8000 fix: filter download_kwargs in BlobReader (#1411) · googleapis/python-storage@0780ea5 · GitHub
[go: up one dir, main page]

8000
Skip to content

Commit 0780ea5

Browse files
committed
fix: filter download_kwargs in BlobReader (#1411)
1 parent 309bad1 commit 0780ea5

File tree

2 files changed

+69
-3
lines changed

2 files changed

+69
-3
lines changed

google/cloud/storage/fileio.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ class BlobReader(io.BufferedIOBase):
9292
configuration changes for Retry objects such as delays and deadlines
9393
are respected.
9494
95+
:type download_kwargs: dict
9596
:param download_kwargs:
9697
Keyword arguments to pass to the underlying API calls.
9798
The following arguments are supported:
@@ -101,9 +102,10 @@ class BlobReader(io.BufferedIOBase):
101102
- ``if_metageneration_match``
102103
- ``if_metageneration_not_match``
103104
- ``timeout``
105+
- ``raw_download``
104106
105-
Note that download_kwargs are also applied to blob.reload(), if a reload
106-
is needed during seek().
107+
Note that download_kwargs (excluding ``raw_download``) are also applied to blob.reload(),
108+
if a reload is needed during seek().
107109
"""
108110

109111
def __init__(self, blob, chunk_size=None, retry=DEFAULT_RETRY, **download_kwargs):
@@ -178,7 +180,10 @@ def seek(self, pos, whence=0):
178180
self._checkClosed() # Raises ValueError if closed.
179181

180182
if self._blob.size is None:
181-
self._blob.reload(**self._download_kwargs)
183+
reload_kwargs = {
184+
k: v for k, v in self._download_kwargs.items() if k != "raw_download"
185+
}
186+
self._blob.reload(**reload_kwargs)
182187

183188
initial_offset = self._pos + self._buffer.tell()
184189

@@ -281,6 +286,7 @@ class BlobWriter(io.BufferedIOBase):
281286
configuration changes for Retry objects such as delays and deadlines
282287
are respected.
283288
289+
:type upload_kwargs: dict
284290
:param upload_kwargs:
285291
Keyword arguments to pass to the underlying API
286292
calls. The following arguments are supported:

tests/system/test_fileio.py

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,3 +76,63 @@ def test_blobwriter_and_blobreader_text_mode(
7676
assert text_data[:100] == reader.read(100)
7777
assert 0 == reader.seek(0)
7878
assert reader.read() == text_data
79+
80+
81+
def test_blobwriter_exit(
82+
shared_bucket,
83+
blobs_to_delete,
84+
service_account,
85+
):
86+
blob = shared_bucket.blob("NeverUploaded")
87+
88+
# no-op when nothing was uploaded yet
89+
with pytest.raises(ValueError, match="SIGTERM received"):
90+
with blob.open("wb") as writer:
91+
writer.write(b"first chunk") # not yet uploaded
92+
raise ValueError("SIGTERM received") # no upload to cancel in __exit__
93+
# blob should not exist
94+
assert not blob.exists()
95+
96+
# unhandled exceptions should cancel the upload
97+
with pytest.raises(ValueError, match="SIGTERM received"):
98+
with blob.open("wb", chunk_size=CHUNK_SIZE_MULTIPLE) as writer:
99+
writer.write(b"first chunk") # not yet uploaded
100+
writer.write(bytes(CHUNK_SIZE_MULTIPLE)) # uploaded
101+
raise ValueError("SIGTERM received") # upload is cancelled in __exit__
102+
# blob should not exist
103+
assert not blob.exists()
104+
105+
# handled exceptions should not cancel the upload
106+
with blob.open("wb", chunk_size=CHUNK_SIZE_MULTIPLE) as writer:
107+
writer.write(b"first chunk") # not yet uploaded
108+
writer.write(bytes(CHUNK_SIZE_MULTIPLE)) # uploaded
109+
try:
110+
raise ValueError("This is fine")
111+
except ValueError:
112+
pass # no exception context passed to __exit__
113+
blobs_to_delete.append(blob)
114+
# blob should have been uploaded
115+
assert blob.exists()
116+
117+
118+
def test_blobreader_w_raw_download(
119+
shared_bucket,
120+
blobs_to_delete,
121+
file_data,
122+
):
123+
blob = shared_bucket.blob("LargeFile")
124+
info = file_data["big"]
125+
with open(info["path"], "rb") as file_obj:
126+
with blob.open("wb", chunk_size=256 * 1024, if_generation_match=0) as writer:
127+
writer.write(file_obj.read())
128+
blobs_to_delete.append(blob)
129+
130+
# Test BlobReader read and seek handles raw downloads.
131+
with open(info["path"], "rb") as file_obj:
132+
with blob.open("rb", chunk_size=256 * 1024, raw_download=True) as reader:
133+
reader.seek(0)
134+
file_obj.seek(0)
135+
assert file_obj.read() == reader.read()
136+
# End of file reached; further reads should be blank but not
137+
# raise an error.
138+
assert reader.read() == b""

0 commit comments

Comments
 (0)
0