8000 add special char s3 test + fix moto lookup with cleaned up key name by bentsku · Pull Request #8470 · localstack/localstack · GitHub
[go: up one dir, main page]
Skip to content

add special char s3 test + fix moto lookup with cleaned up key name #8470

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jul 3, 2023
Merged

add special char s3 test + fix moto lookup with cleaned up key name #8470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
41a180d
add special char integration test
bentsku May 9, 2023
2180c0b
add delete snapshot
bentsku May 24, 2023
9428044
add test and clean up key name to be consistent with moto
bentsku Jun 9, 2023
f890979
fix merge
bentsku Jul 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions localstack/services/s3/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from botocore.utils import InvalidArnException
from moto.s3.exceptions import MissingBucket
from moto.s3.models import FakeBucket, FakeDeleteMarker, FakeKey
from moto.s3.utils import clean_key_name

from localstack.aws.api import CommonServiceException, ServiceException
from localstack.aws.api.s3 import (
Expand Down Expand Up @@ -162,10 +163,11 @@ def get_key_from_moto_bucket(
) -> FakeKey | FakeDeleteMarker:
# TODO: rework the delete marker handling
# we basically need to re-implement moto `get_object` to account for FakeDeleteMarker
clean_key = clean_key_name(key)
if version_id is None:
fake_key = moto_bucket.keys.get(key)
fake_key = moto_bucket.keys.get(clean_key)
else:
for key_version in moto_bucket.keys.getlist(key, default=[]):
for key_version in moto_bucket.keys.getlist(clean_key, default=[]):
if str(key_version.version_id) == str(version_id):
fake_key = key_version
break
Expand Down
16 changes: 16 additions & 0 deletions tests/integration/s3/test_s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -626,6 +626,22 @@ def test_upload_file_multipart(self, s3_bucket, tmpdir, snapshot, aws_client):
assert obj["Body"].read() == data, f"body did not contain expected data {obj}"
snapshot.match("get_object", obj)

@pytest.mark.aws_validated
@pytest.mark.skip_snapshot_verify(paths=["$..ServerSideEncryption"])
@pytest.mark.parametrize("key", ["file%2Fname", "test@key/"])
def test_put_get_object_special_character(self, s3_bucket, aws_client, snapshot, key):
snapshot.add_transformer(snapshot.transform.s3_api())
resp = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body=b"test")
snapshot.match("put-object-special-char", resp)
resp = aws_client.s3.list_objects_v2(Bucket=s3_bucket)
# FIXME: Moto will by default clean up key name, but they will return the cleaned up key name in ListObject...
if "%" not in key or is_aws_cloud():
snapshot.match("list-object-special-char", resp)
resp = aws_client.s3.get_object(Bucket=s3_bucket, Key=key)
snapshot.match("get-object-special-char", resp)
resp = aws_client.s3.delete_object(Bucket=s3_bucket, Key=key)
snapshot.match("del-object-special-char", resp)

@pytest.mark.aws_validated
@pytest.mark.parametrize("delimiter", ["/", "%2F"])
def test_list_objects_with_prefix(self, s3_create_bucket, delimiter, snapshot, aws_client):
Expand Down
108 changes: 108 additions & 0 deletions tests/integration/s3/test_s3.snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7414,5 +7414,113 @@
}
}
}
},
"tests/integration/s3/test_s3.py::TestS3::test_put_get_object_special_character[file%2Fname]": {
"recorded-date": "09-06-2023, 17:15:35",
"recorded-content": {
"put-object-special-char": {
"ETag": "\"098f6bcd4621d373cade4e832627b4f6\"",
"ServerSideEncryption": "AES256",
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
},
"list-object-special-char": {
"Contents": [
{
"ETag": "\"098f6bcd4621d373cade4e832627b4f6\"",
"Key": "file%2Fname",
"LastModified": "datetime",
"Size": 4,
"StorageClass": "STANDARD"
}
],
"EncodingType": "url",
"IsTruncated": false,
"KeyCount": 1,
F20C "MaxKeys": 1000,
"Name": "<bucket-name:1>",
"Prefix": "",
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
},
"get-object-special-char": {
"AcceptRanges": "bytes",
"Body": "test",
"ContentLength": 4,
"ContentType": "binary/octet-stream",
"ETag": "\"098f6bcd4621d373cade4e832627b4f6\"",
"LastModified": "datetime",
"Metadata": {},
"ServerSideEncryption": "AES256",
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
},
"del-object-special-char": {
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 204
}
}
}
},
"tests/integration/s3/test_s3.py::TestS3::test_put_get_object_special_character[test@key/]": {
"recorded-date": "09-06-2023, 17:15:39",
"recorded-content": {
"put-object-special-char": {
"ETag": "\"098f6bcd4621d373cade4e832627b4f6\"",
"ServerSideEncryption": "AES256",
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
},
"list-object-special-char": {
"Contents": [
{
"ETag": "\"098f6bcd4621d373cade4e832627b4f6\"",
"Key": "test@key/",
"LastModified": "datetime",
"Size": 4,
"StorageClass": "STANDARD"
}
],
"EncodingType": "url",
"IsTruncated": false,
"KeyCount": 1,
"MaxKeys": 1000,
"Name": "<bucket-name:1>",
"Prefix": "",
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
},
"get-object-special-char": {
"AcceptRanges": "bytes",
"Body": "test",
"ContentLength": 4,
"ContentType": "binary/octet-stream",
"ETag": "\"098f6bcd4621d373cade4e832627b4f6\"",
"LastModified": "datetime",
"Metadata": {},
"ServerSideEncryption": "AES256",
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
},
"del-object-special-char": {
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 204
}
}
}
}
}
0