From 0780ea5c2c073eedb681af5d65083c8be05b8823 Mon Sep 17 00:00:00 2001 From: cojenco Date: Wed, 22 Jan 2025 14:53:41 -0800 Subject: [PATCH 1/2] fix: filter download_kwargs in BlobReader (#1411) --- google/cloud/storage/fileio.py | 12 +++++-- tests/system/test_fileio.py | 60 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/google/cloud/storage/fileio.py b/google/cloud/storage/fileio.py index 97d234983..7354fa526 100644 --- a/google/cloud/storage/fileio.py +++ b/google/cloud/storage/fileio.py @@ -92,6 +92,7 @@ class BlobReader(io.BufferedIOBase): configuration changes for Retry objects such as delays and deadlines are respected. + :type download_kwargs: dict :param download_kwargs: Keyword arguments to pass to the underlying API calls. The following arguments are supported: @@ -101,9 +102,10 @@ class BlobReader(io.BufferedIOBase): - ``if_metageneration_match`` - ``if_metageneration_not_match`` - ``timeout`` + - ``raw_download`` - Note that download_kwargs are also applied to blob.reload(), if a reload - is needed during seek(). + Note that download_kwargs (excluding ``raw_download``) are also applied to blob.reload(), + if a reload is needed during seek(). """ def __init__(self, blob, chunk_size=None, retry=DEFAULT_RETRY, **download_kwargs): @@ -178,7 +180,10 @@ def seek(self, pos, whence=0): self._checkClosed() # Raises ValueError if closed. if self._blob.size is None: - self._blob.reload(**self._download_kwargs) + reload_kwargs = { + k: v for k, v in self._download_kwargs.items() if k != "raw_download" + } + self._blob.reload(**reload_kwargs) initial_offset = self._pos + self._buffer.tell() @@ -281,6 +286,7 @@ class BlobWriter(io.BufferedIOBase): configuration changes for Retry objects such as delays and deadlines are respected. + :type upload_kwargs: dict :param upload_kwargs: Keyword arguments to pass to the underlying API calls. The following arguments are supported: diff --git a/tests/system/test_fileio.py b/tests/system/test_fileio.py index 21c197eee..80f6b9666 100644 --- a/tests/system/test_fileio.py +++ b/tests/system/test_fileio.py @@ -76,3 +76,63 @@ def test_blobwriter_and_blobreader_text_mode( assert text_data[:100] == reader.read(100) assert 0 == reader.seek(0) assert reader.read() == text_data + + +def test_blobwriter_exit( + shared_bucket, + blobs_to_delete, + service_account, +): + blob = shared_bucket.blob("NeverUploaded") + + # no-op when nothing was uploaded yet + with pytest.raises(ValueError, match="SIGTERM received"): + with blob.open("wb") as writer: + writer.write(b"first chunk") # not yet uploaded + raise ValueError("SIGTERM received") # no upload to cancel in __exit__ + # blob should not exist + assert not blob.exists() + + # unhandled exceptions should cancel the upload + with pytest.raises(ValueError, match="SIGTERM received"): + with blob.open("wb", chunk_size=CHUNK_SIZE_MULTIPLE) as writer: + writer.write(b"first chunk") # not yet uploaded + writer.write(bytes(CHUNK_SIZE_MULTIPLE)) # uploaded + raise ValueError("SIGTERM received") # upload is cancelled in __exit__ + # blob should not exist + assert not blob.exists() + + # handled exceptions should not cancel the upload + with blob.open("wb", chunk_size=CHUNK_SIZE_MULTIPLE) as writer: + writer.write(b"first chunk") # not yet uploaded + writer.write(bytes(CHUNK_SIZE_MULTIPLE)) # uploaded + try: + raise ValueError("This is fine") + except ValueError: + pass # no exception context passed to __exit__ + blobs_to_delete.append(blob) + # blob should have been uploaded + assert blob.exists() + + +def test_blobreader_w_raw_download( + shared_bucket, + blobs_to_delete, + file_data, +): + blob = shared_bucket.blob("LargeFile") + info = file_data["big"] + with open(info["path"], "rb") as file_obj: + with blob.open("wb", chunk_size=256 * 1024, if_generation_match=0) as writer: + writer.write(file_obj.read()) + blobs_to_delete.append(blob) + + # Test BlobReader read and seek handles raw downloads. + with open(info["path"], "rb") as file_obj: + with blob.open("rb", chunk_size=256 * 1024, raw_download=True) as reader: + reader.seek(0) + file_obj.seek(0) + assert file_obj.read() == reader.read() + # End of file reached; further reads should be blank but not + # raise an error. + assert reader.read() == b"" From f3853ed14a2883e3bab4857deafd3e8f08ae8012 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Tue, 4 Feb 2025 22:26:45 +0000 Subject: [PATCH 2/2] chore: release 2.19.1 Release-As: 2.19.1 --- CHANGELOG.md | 7 +++++++ google/cloud/storage/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f3883ec3..88618d235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-cloud-storage/#history +## [2.19.1](https://github.com/googleapis/python-storage/compare/v2.19.0...v2.19.1) (2025-02-04) + + +### Bug Fixes + +* Filter download_kwargs in BlobReader ([#1411](https://github.com/googleapis/python-storage/issues/1411)) ([0780ea5](https://github.com/googleapis/python-storage/commit/0780ea5c2c073eedb681af5d65083c8be05b8823)) + ## [2.19.0](https://github.com/googleapis/python-storage/compare/v2.18.2...v2.19.0) (2024-11-21) diff --git a/google/cloud/storage/version.py b/google/cloud/storage/version.py index 2605c08a3..25b6f2f82 100644 --- a/google/cloud/storage/version.py +++ b/google/cloud/storage/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.19.0" +__version__ = "2.19.1"