From 41a180d30acd734d217380f9b3f61c81bcb75dd2 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Tue, 9 May 2023 13:34:03 +0300 Subject: [PATCH 1/4] add special char integration test --- tests/integration/s3/test_s3.py | 14 +++++ tests/integration/s3/test_s3.snapshot.json | 62 ++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/tests/integration/s3/test_s3.py b/tests/integration/s3/test_s3.py index 481e33510dae1..15c4a43bc1e2c 100644 --- a/tests/integration/s3/test_s3.py +++ b/tests/integration/s3/test_s3.py @@ -626,6 +626,20 @@ def test_upload_file_multipart(self, s3_bucket, tmpdir, snapshot, aws_client): assert obj["Body"].read() == data, f"body did not contain expected data {obj}" snapshot.match("get_object", obj) + @pytest.mark.aws_validated + @pytest.mark.skip_snapshot_verify(paths=["$..ServerSideEncryption"]) + def test_put_get_object_special_character(self, s3_bucket, aws_client, snapshot): + snapshot.add_transformer(snapshot.transform.s3_api()) + key = "test@key/" + resp = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body=b"test") + snapshot.match("put-object-special-char", resp) + resp = aws_client.s3.list_objects_v2(Bucket=s3_bucket) + snapshot.match("list-object-special-char", resp) + resp = aws_client.s3.get_object(Bucket=s3_bucket, Key=key) + snapshot.match("get-object-special-char", resp) + aws_client.s3.delete_object(Bucket=s3_bucket, Key=key) + snapshot.match("del-object-special-char", resp) + @pytest.mark.aws_validated @pytest.mark.parametrize("delimiter", ["/", "%2F"]) def test_list_objects_with_prefix(self, s3_create_bucket, delimiter, snapshot, aws_client): diff --git a/tests/integration/s3/test_s3.snapshot.json b/tests/integration/s3/test_s3.snapshot.json index 320922c540304..daeae495bffa6 100644 --- a/tests/integration/s3/test_s3.snapshot.json +++ b/tests/integration/s3/test_s3.snapshot.json @@ -7414,5 +7414,67 @@ } } } + }, + "tests/integration/s3/test_s3.py::TestS3::test_put_get_object_special_character": { + "recorded-date": "09-05-2023, 13:27:18", + "recorded-content": { + "put-object-special-char": { + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "list-object-special-char": { + "Contents": [ + { + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "Key": "test@key/", + "LastModified": "datetime", + "Size": 4, + "StorageClass": "STANDARD" + } + ], + "EncodingType": "url", + "IsTruncated": false, + "KeyCount": 1, + "MaxKeys": 1000, + "Name": "", + "Prefix": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-object-special-char": { + "AcceptRanges": "bytes", + "Body": "test", + "ContentLength": 4, + "ContentType": "binary/octet-stream", + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "LastModified": "datetime", + "Metadata": {}, + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "del-object-special-char": { + "AcceptRanges": "bytes", + "Body": "", + "ContentLength": 4, + "ContentType": "binary/octet-stream", + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "LastModified": "datetime", + "Metadata": {}, + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } } } From 2180c0bc20d02c67f53ec71da6fe9686db08999f Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Wed, 24 May 2023 19:17:46 +0200 Subject: [PATCH 2/4] add delete snapshot --- tests/integration/s3/test_s3.py | 2 +- tests/integration/s3/test_s3.snapshot.json | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/integration/s3/test_s3.py b/tests/integration/s3/test_s3.py index 15c4a43bc1e2c..29b0ff27ebc0c 100644 --- a/tests/integration/s3/test_s3.py +++ b/tests/integration/s3/test_s3.py @@ -637,7 +637,7 @@ def test_put_get_object_special_character(self, s3_bucket, aws_client, snapshot) snapshot.match("list-object-special-char", resp) resp = aws_client.s3.get_object(Bucket=s3_bucket, Key=key) snapshot.match("get-object-special-char", resp) - aws_client.s3.delete_object(Bucket=s3_bucket, Key=key) + resp = aws_client.s3.delete_object(Bucket=s3_bucket, Key=key) snapshot.match("del-object-special-char", resp) @pytest.mark.aws_validated diff --git a/tests/integration/s3/test_s3.snapshot.json b/tests/integration/s3/test_s3.snapshot.json index daeae495bffa6..70b5ee591d58c 100644 --- a/tests/integration/s3/test_s3.snapshot.json +++ b/tests/integration/s3/test_s3.snapshot.json @@ -7416,7 +7416,7 @@ } }, "tests/integration/s3/test_s3.py::TestS3::test_put_get_object_special_character": { - "recorded-date": "09-05-2023, 13:27:18", + "recorded-date": "24-05-2023, 19:17:18", "recorded-content": { "put-object-special-char": { "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", @@ -7462,17 +7462,9 @@ } }, "del-object-special-char": { - "AcceptRanges": "bytes", - "Body": "", - "ContentLength": 4, - "ContentType": "binary/octet-stream", - "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", - "LastModified": "datetime", - "Metadata": {}, - "ServerSideEncryption": "AES256", "ResponseMetadata": { "HTTPHeaders": {}, - "HTTPStatusCode": 200 + "HTTPStatusCode": 204 } } } From 94280444fabc1e6851b004cc64a5e559b62a2fcf Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Fri, 9 Jun 2023 17:32:51 +0300 Subject: [PATCH 3/4] add test and clean up key name to be consistent with moto --- localstack/services/s3/utils.py | 6 ++- tests/integration/s3/test_s3.py | 8 ++-- tests/integration/s3/test_s3.snapshot.json | 56 ++++++++++++++++++++++ 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/localstack/services/s3/utils.py b/localstack/services/s3/utils.py index 3e511b91ab715..fbb486395c8d3 100644 --- a/localstack/services/s3/utils.py +++ b/localstack/services/s3/utils.py @@ -8,6 +8,7 @@ from botocore.utils import InvalidArnException from moto.s3.exceptions import MissingBucket from moto.s3.models import FakeBucket, FakeDeleteMarker, FakeKey +from moto.s3.utils import clean_key_name from localstack.aws.api import CommonServiceException, ServiceException from localstack.aws.api.s3 import ( @@ -162,10 +163,11 @@ def get_key_from_moto_bucket( ) -> FakeKey | FakeDeleteMarker: # TODO: rework the delete marker handling # we basically need to re-implement moto `get_object` to account for FakeDeleteMarker + clean_key = clean_key_name(key) if version_id is None: - fake_key = moto_bucket.keys.get(key) + fake_key = moto_bucket.keys.get(clean_key) else: - for key_version in moto_bucket.keys.getlist(key, default=[]): + for key_version in moto_bucket.keys.getlist(clean_key, default=[]): if str(key_version.version_id) == str(version_id): fake_key = key_version break diff --git a/tests/integration/s3/test_s3.py b/tests/integration/s3/test_s3.py index 29b0ff27ebc0c..c47acfbeefaad 100644 --- a/tests/integration/s3/test_s3.py +++ b/tests/integration/s3/test_s3.py @@ -628,13 +628,15 @@ def test_upload_file_multipart(self, s3_bucket, tmpdir, snapshot, aws_client): @pytest.mark.aws_validated @pytest.mark.skip_snapshot_verify(paths=["$..ServerSideEncryption"]) - def test_put_get_object_special_character(self, s3_bucket, aws_client, snapshot): + @pytest.mark.parametrize("key", ["file%2Fname", "test@key/"]) + def test_put_get_object_special_character(self, s3_bucket, aws_client, snapshot, key): snapshot.add_transformer(snapshot.transform.s3_api()) - key = "test@key/" resp = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body=b"test") snapshot.match("put-object-special-char", resp) resp = aws_client.s3.list_objects_v2(Bucket=s3_bucket) - snapshot.match("list-object-special-char", resp) + # FIXME: Moto will by default clean up key name, but they will return the cleaned up key name in ListObject... + if "%" not in key or is_aws_cloud(): + snapshot.match("list-object-special-char", resp) resp = aws_client.s3.get_object(Bucket=s3_bucket, Key=key) snapshot.match("get-object-special-char", resp) resp = aws_client.s3.delete_object(Bucket=s3_bucket, Key=key) diff --git a/tests/integration/s3/test_s3.snapshot.json b/tests/integration/s3/test_s3.snapshot.json index 70b5ee591d58c..a9ddfb1c313a7 100644 --- a/tests/integration/s3/test_s3.snapshot.json +++ b/tests/integration/s3/test_s3.snapshot.json @@ -7417,6 +7417,62 @@ }, "tests/integration/s3/test_s3.py::TestS3::test_put_get_object_special_character": { "recorded-date": "24-05-2023, 19:17:18", + "tests/integration/s3/test_s3.py::TestS3::test_put_get_object_special_character[file%2Fname]": { + "recorded-date": "09-06-2023, 17:15:35", + "recorded-content": { + "put-object-special-char": { + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "list-object-special-char": { + "Contents": [ + { + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "Key": "file%2Fname", + "LastModified": "datetime", + "Size": 4, + "StorageClass": "STANDARD" + } + ], + "EncodingType": "url", + "IsTruncated": false, + "KeyCount": 1, + "MaxKeys": 1000, + "Name": "", + "Prefix": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "get-object-special-char": { + "AcceptRanges": "bytes", + "Body": "test", + "ContentLength": 4, + "ContentType": "binary/octet-stream", + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "LastModified": "datetime", + "Metadata": {}, + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "del-object-special-char": { + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 204 + } + } + } + }, + "tests/integration/s3/test_s3.py::TestS3::test_put_get_object_special_character[test@key/]": { + "recorded-date": "09-06-2023, 17:15:39", "recorded-content": { "put-object-special-char": { "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", From f890979d35ed0fd5fbe82f8ff29ab54e273acbae Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Mon, 3 Jul 2023 14:14:59 +0200 Subject: [PATCH 4/4] fix merge --- tests/integration/s3/test_s3.snapshot.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/s3/test_s3.snapshot.json b/tests/integration/s3/test_s3.snapshot.json index a9ddfb1c313a7..8ca9a5cc4b00b 100644 --- a/tests/integration/s3/test_s3.snapshot.json +++ b/tests/integration/s3/test_s3.snapshot.json @@ -7415,8 +7415,6 @@ } } }, - "tests/integration/s3/test_s3.py::TestS3::test_put_get_object_special_character": { - "recorded-date": "24-05-2023, 19:17:18", "tests/integration/s3/test_s3.py::TestS3::test_put_get_object_special_character[file%2Fname]": { "recorded-date": "09-06-2023, 17:15:35", "recorded-content": {