From a8794379eaa30c94fcc68c2a3bf6b2c8b9e02187 Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Wed, 27 Nov 2024 01:23:22 +0100 Subject: [PATCH 1/2] implement S3 IfMatch --- .../localstack/services/s3/provider.py | 62 ++- tests/aws/services/s3/test_s3_api.py | 253 ++++++++++ .../aws/services/s3/test_s3_api.snapshot.json | 440 ++++++++++++++++++ .../services/s3/test_s3_api.validation.json | 21 + 4 files changed, 772 insertions(+), 4 deletions(-) diff --git a/localstack-core/localstack/services/s3/provider.py b/localstack-core/localstack/services/s3/provider.py index 46007b83fa871..5e24508bace46 100644 --- a/localstack-core/localstack/services/s3/provider.py +++ b/localstack-core/localstack/services/s3/provider.py @@ -660,11 +660,20 @@ def put_object( validate_object_key(key) - if (if_none_match := request.get("IfNoneMatch")) and if_none_match != "*": + if_match = request.get("IfMatch") + if (if_none_match := request.get("IfNoneMatch")) and if_match: raise NotImplementedException( "A header you provided implies functionality that is not implemented", - Header="If-None-Match", - additionalMessage="We don't accept the provided value of If-None-Match header for this API", + Header="If-Match,If-None-Match", + additionalMessage="Multiple conditional request headers present in the request", + ) + + elif (if_none_match and if_none_match != "*") or (if_match and if_match == "*"): + header_name = "If-None-Match" if if_none_match else "If-Match" + raise NotImplementedException( + "A header you provided implies functionality that is not implemented", + Header=header_name, + additionalMessage=f"We don't accept the provided value of {header_name} header for this API", ) system_metadata = get_system_metadata_from_request(request) @@ -758,6 +767,9 @@ def put_object( Condition="If-None-Match", ) + elif if_match: + verify_object_equality_precondition_write(s3_bucket, key, if_match) + s3_stored_object.write(body) if ( @@ -2377,7 +2389,14 @@ def complete_multipart_upload( UploadId=upload_id, ) - if if_none_match: + if if_none_match and if_match: + raise NotImplementedException( + "A header you provided implies functionality that is not implemented", + Header="If-Match,If-None-Match", + additionalMessage="Multiple conditional request headers present in the request", + ) + + elif if_none_match: if if_none_match != "*": raise NotImplementedException( "A header you provided implies functionality that is not implemented", @@ -2396,6 +2415,17 @@ def complete_multipart_upload( Key=key, ) + elif if_match: + if if_match == "*": + raise NotImplementedException( + "A header you provided implies functionality that is not implemented", + Header="If-None-Match", + additionalMessage="We don't accept the provided value of If-None-Match header for this API", + ) + verify_object_equality_precondition_write( + s3_bucket, key, if_match, initiated=s3_multipart.initiated + ) + parts = multipart_upload.get("Parts", []) if not parts: raise InvalidRequest("You must specify at least one part") @@ -4395,3 +4425,27 @@ def get_access_control_policy_for_new_resource_request( def object_exists_for_precondition_write(s3_bucket: S3Bucket, key: ObjectKey) -> bool: return (existing := s3_bucket.objects.get(key)) and not isinstance(existing, S3DeleteMarker) + + +def verify_object_equality_precondition_write( + s3_bucket: S3Bucket, + key: ObjectKey, + etag: str, + initiated: datetime.datetime | None = None, +) -> None: + existing = s3_bucket.objects.get(key) + if not existing or isinstance(existing, S3DeleteMarker): + raise NoSuchKey("The specified key does not exist.", Key=key) + + if not existing.etag == etag.strip('"'): + raise PreconditionFailed( + "At least one of the pre-conditions you specified did not hold", + Condition="If-Match", + ) + + if initiated and initiated < existing.last_modified: + raise ConditionalRequestConflict( + "The conditional request cannot succeed due to a conflicting operation against this resource.", + Condition="If-Match", + Key=key, + ) diff --git a/tests/aws/services/s3/test_s3_api.py b/tests/aws/services/s3/test_s3_api.py index 54e12ede32aad..82a367ba84b67 100644 --- a/tests/aws/services/s3/test_s3_api.py +++ b/tests/aws/services/s3/test_s3_api.py @@ -1719,6 +1719,10 @@ def test_bucket_acceleration_configuration_exc( class TestS3ObjectWritePrecondition: + """ + https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-writes.html + """ + @pytest.fixture(autouse=True) def add_snapshot_transformers(self, snapshot): snapshot.add_transformers_list( @@ -1869,3 +1873,252 @@ def test_put_object_if_none_match_versioned_bucket(self, s3_bucket, aws_client, list_object_versions = aws_client.s3.list_object_versions(Bucket=s3_bucket) snapshot.match("list-object-versions", list_object_versions) + + @markers.aws.validated + def test_put_object_if_match(self, s3_bucket, aws_client, snapshot): + key = "test-precondition" + put_obj = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test") + snapshot.match("put-obj", put_obj) + etag = put_obj["ETag"] + + with pytest.raises(ClientError) as e: + # empty object is provided + aws_client.s3.put_object( + Bucket=s3_bucket, Key=key, IfMatch="d41d8cd98f00b204e9800998ecf8427e" + ) + snapshot.match("put-obj-if-match-wrong-etag", e.value.response) + + put_obj_overwrite = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfMatch=etag) + snapshot.match("put-obj-overwrite", put_obj_overwrite) + + del_obj = aws_client.s3.delete_object(Bucket=s3_bucket, Key=key) + snapshot.match("del-obj", del_obj) + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfMatch=etag) + snapshot.match("put-obj-if-match-key-not-exists", e.value.response) + + put_obj_after_del = aws_client.s3.put_object(Bucket=s3_bucket, Key=key) + snapshot.match("put-obj-after-del", put_obj_after_del) + + @markers.aws.validated + def test_put_object_if_match_validation(self, s3_bucket, aws_client, snapshot): + key = "test-precondition-validation" + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfMatch="*") + snapshot.match("put-obj-if-match-star-value", e.value.response) + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfMatch="abcdef") + snapshot.match("put-obj-if-match-bad-value", e.value.response) + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfMatch="bad-char_/") + snapshot.match("put-obj-if-match-bad-value-2", e.value.response) + + @markers.aws.validated + def test_multipart_if_match_with_put(self, s3_bucket, aws_client, snapshot): + key = "test-precondition" + put_obj = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test") + snapshot.match("put-obj", put_obj) + put_obj_etag_1 = put_obj["ETag"] + + create_multipart = aws_client.s3.create_multipart_upload(Bucket=s3_bucket, Key=key) + snapshot.match("create-multipart", create_multipart) + upload_id = create_multipart["UploadId"] + + upload_part = aws_client.s3.upload_part( + Bucket=s3_bucket, Key=key, UploadId=upload_id, Body="test", PartNumber=1 + ) + parts = [{"ETag": upload_part["ETag"], "PartNumber": 1}] + + put_obj_2 = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test2") + snapshot.match("put-obj-during", put_obj_2) + put_obj_etag_2 = put_obj_2["ETag"] + + with pytest.raises(ClientError) as e: + aws_client.s3.complete_multipart_upload( + Bucket=s3_bucket, + Key=key, + MultipartUpload={"Parts": parts}, + UploadId=upload_id, + IfMatch=put_obj_etag_1, + ) + snapshot.match("complete-multipart-if-match-put-before", e.value.response) + + # the previous PutObject request was done between the CreateMultipartUpload and completion, so it takes + # precedence + # you need to restart the whole multipart for it to work + with pytest.raises(ClientError) as e: + aws_client.s3.complete_multipart_upload( + Bucket=s3_bucket, + Key=key, + MultipartUpload={"Parts": parts}, + UploadId=upload_id, + IfMatch=put_obj_etag_2, + ) + snapshot.match("complete-multipart-if-match-put-during", e.value.response) + + create_multipart = aws_client.s3.create_multipart_upload(Bucket=s3_bucket, Key=key) + snapshot.match("create-multipart-again", create_multipart) + upload_id = create_multipart["UploadId"] + + upload_part = aws_client.s3.upload_part( + Bucket=s3_bucket, Key=key, UploadId=upload_id, Body="test", PartNumber=1 + ) + parts = [{"ETag": upload_part["ETag"], "PartNumber": 1}] + + complete_multipart = aws_client.s3.complete_multipart_upload( + Bucket=s3_bucket, + Key=key, + MultipartUpload={"Parts": parts}, + UploadId=upload_id, + IfMatch=put_obj_etag_2, + ) + snapshot.match("complete-multipart-if-match-put-before-restart", complete_multipart) + + @markers.aws.validated + def test_multipart_if_match_with_put_identical(self, s3_bucket, aws_client, snapshot): + key = "test-precondition" + put_obj = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test") + snapshot.match("put-obj", put_obj) + put_obj_etag_1 = put_obj["ETag"] + + create_multipart = aws_client.s3.create_multipart_upload(Bucket=s3_bucket, Key=key) + snapshot.match("create-multipart", create_multipart) + upload_id = create_multipart["UploadId"] + + upload_part = aws_client.s3.upload_part( + Bucket=s3_bucket, Key=key, UploadId=upload_id, Body="test", PartNumber=1 + ) + parts = [{"ETag": upload_part["ETag"], "PartNumber": 1}] + + put_obj_2 = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test") + snapshot.match("put-obj-during", put_obj_2) + # same ETag as first put + put_obj_etag_2 = put_obj_2["ETag"] + assert put_obj_etag_1 == put_obj_etag_2 + + # it seems that even if we overwrite the object with the same content, S3 will still reject the request if a + # write operation was done between creation and completion of the multipart upload, like the `Delete` + # counterpart of `IfNoneMatch` + + with pytest.raises(ClientError) as e: + aws_client.s3.complete_multipart_upload( + Bucket=s3_bucket, + Key=key, + MultipartUpload={"Parts": parts}, + UploadId=upload_id, + IfMatch=put_obj_etag_2, + ) + snapshot.match("complete-multipart-if-match-put-during", e.value.response) + # the previous PutObject request was done between the CreateMultipartUpload and completion, so it takes + # precedence + # you need to restart the whole multipart for it to work + + create_multipart = aws_client.s3.create_multipart_upload(Bucket=s3_bucket, Key=key) + snapshot.match("create-multipart-again", create_multipart) + upload_id = create_multipart["UploadId"] + + upload_part = aws_client.s3.upload_part( + Bucket=s3_bucket, Key=key, UploadId=upload_id, Body="test", PartNumber=1 + ) + parts = [{"ETag": upload_part["ETag"], "PartNumber": 1}] + + complete_multipart = aws_client.s3.complete_multipart_upload( + Bucket=s3_bucket, + Key=key, + MultipartUpload={"Parts": parts}, + UploadId=upload_id, + IfMatch=put_obj_etag_2, + ) + snapshot.match("complete-multipart-if-match-put-before-restart", complete_multipart) + + @markers.aws.validated + def test_multipart_if_match_with_delete(self, s3_bucket, aws_client, snapshot): + key = "test-precondition" + put_obj = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test") + snapshot.match("put-obj", put_obj) + obj_etag = put_obj["ETag"] + + create_multipart = aws_client.s3.create_multipart_upload(Bucket=s3_bucket, Key=key) + snapshot.match("create-multipart", create_multipart) + upload_id = create_multipart["UploadId"] + + upload_part = aws_client.s3.upload_part( + Bucket=s3_bucket, Key=key, UploadId=upload_id, Body="test", PartNumber=1 + ) + parts = [{"ETag": upload_part["ETag"], "PartNumber": 1}] + + del_obj = aws_client.s3.delete_object(Bucket=s3_bucket, Key=key) + snapshot.match("del-obj", del_obj) + + with pytest.raises(ClientError) as e: + aws_client.s3.complete_multipart_upload( + Bucket=s3_bucket, + Key=key, + MultipartUpload={"Parts": parts}, + UploadId=upload_id, + IfMatch=obj_etag, + ) + snapshot.match("complete-multipart-after-del", e.value.response) + + put_obj_2 = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test") + snapshot.match("put-obj-2", put_obj_2) + obj_etag_2 = put_obj_2["ETag"] + + with pytest.raises(ClientError) as e: + # even if we recreated the object, it still fails as it was done after the start of the upload + aws_client.s3.complete_multipart_upload( + Bucket=s3_bucket, + Key=key, + MultipartUpload={"Parts": parts}, + UploadId=upload_id, + IfMatch=obj_etag_2, + ) + snapshot.match("complete-multipart-if-match-after-put", e.value.response) + + @markers.aws.validated + def test_put_object_if_match_versioned_bucket(self, s3_bucket, aws_client, snapshot): + aws_client.s3.put_bucket_versioning( + Bucket=s3_bucket, VersioningConfiguration={"Status": "Enabled"} + ) + key = "test-precondition" + put_obj = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test") + snapshot.match("put-obj", put_obj) + put_obj_etag_1 = put_obj["ETag"] + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfMatch="abcdef") + snapshot.match("put-obj-if-none-match-bad-value", e.value.response) + + del_obj = aws_client.s3.delete_object(Bucket=s3_bucket, Key=key) + snapshot.match("del-obj", del_obj) + + # if the last object is a delete marker, then we can't use IfMatch + with pytest.raises(ClientError) as e: + aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfMatch=put_obj_etag_1) + snapshot.match("put-obj-after-del-exc", e.value.response) + + put_obj_2 = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test-after-del") + snapshot.match("put-obj-after-del", put_obj_2) + put_obj_etag_2 = put_obj_2["ETag"] + + put_obj_3 = aws_client.s3.put_object( + Bucket=s3_bucket, Key=key, Body="test-if-match", IfMatch=put_obj_etag_2 + ) + snapshot.match("put-obj-if-match", put_obj_3) + + list_object_versions = aws_client.s3.list_object_versions(Bucket=s3_bucket) + snapshot.match("list-object-versions", list_object_versions) + + @markers.aws.validated + def test_put_object_if_match_and_if_none_match_validation( + self, s3_bucket, aws_client, snapshot + ): + key = "test-precondition-validation" + + with pytest.raises(ClientError) as e: + aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfNoneMatch="*", IfMatch="abcdef") + snapshot.match("put-obj-both-precondition", e.value.response) diff --git a/tests/aws/services/s3/test_s3_api.snapshot.json b/tests/aws/services/s3/test_s3_api.snapshot.json index f46efc33e6944..599d74f17e149 100644 --- a/tests/aws/services/s3/test_s3_api.snapshot.json +++ b/tests/aws/services/s3/test_s3_api.snapshot.json @@ -3653,5 +3653,445 @@ } } } + }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_put_object_if_match": { + "recorded-date": "26-11-2024, 16:57:18", + "recorded-content": { + "put-obj": { + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "put-obj-if-match-wrong-etag": { + "Error": { + "Code": "PreconditionFailed", + "Condition": "If-Match", + "Message": "At least one of the pre-conditions you specified did not hold" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 412 + } + }, + "put-obj-overwrite": { + "ETag": "\"d41d8cd98f00b204e9800998ecf8427e\"", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "del-obj": { + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 204 + } + }, + "put-obj-if-match-key-not-exists": { + "Error": { + "Code": "NoSuchKey", + "Key": "test-precondition", + "Message": "The specified key does not exist." + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 404 + } + }, + "put-obj-after-del": { + "ETag": "\"d41d8cd98f00b204e9800998ecf8427e\"", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } + }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_put_object_if_match_validation": { + "recorded-date": "26-11-2024, 20:38:49", + "recorded-content": { + "put-obj-if-match-star-value": { + "Error": { + "Code": "NotImplemented", + "Header": "If-Match", + "Message": "A header you provided implies functionality that is not implemented", + "additionalMessage": "We don't accept the provided value of If-Match header for this API" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 501 + } + }, + "put-obj-if-match-bad-value": { + "Error": { + "Code": "NoSuchKey", + "Key": "test-precondition-validation", + "Message": "The specified key does not exist." + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 404 + } + }, + "put-obj-if-match-bad-value-2": { + "Error": { + "Code": "NoSuchKey", + "Key": "test-precondition-validation", + "Message": "The specified key does not exist." + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 404 + } + } + } + }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_multipart_if_match_with_put": { + "recorded-date": "26-11-2024, 20:44:46", + "recorded-content": { + "put-obj": { + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "create-multipart": { + "Bucket": "", + "Key": "test-precondition", + "ServerSideEncryption": "AES256", + "UploadId": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "put-obj-during": { + "ETag": "\"ad0234829205b9033196ba818f7a872b\"", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "complete-multipart-if-match-put-before": { + "Error": { + "Code": "PreconditionFailed", + "Condition": "If-Match", + "Message": "At least one of the pre-conditions you specified did not hold" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 412 + } + }, + "complete-multipart-if-match-put-during": { + "Error": { + "Code": "ConditionalRequestConflict", + "Condition": "If-Match", + "Key": "test-precondition", + "Message": "The conditional request cannot succeed due to a conflicting operation against this resource." + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 409 + } + }, + "create-multipart-again": { + "Bucket": "", + "Key": "test-precondition", + "ServerSideEncryption": "AES256", + "UploadId": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "complete-multipart-if-match-put-before-restart": { + "Bucket": "", + "ETag": "\"60cd54a928cbbcbb6e7b5595bab46a9e-1\"", + "Key": "test-precondition", + "Location": "", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } + }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_multipart_if_match_with_put_identical": { + "recorded-date": "26-11-2024, 23:46:03", + "recorded-content": { + "put-obj": { + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "create-multipart": { + "Bucket": "", + "Key": "test-precondition", + "ServerSideEncryption": "AES256", + "UploadId": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "put-obj-during": { + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "complete-multipart-if-match-put-during": { + "Error": { + "Code": "ConditionalRequestConflict", + "Condition": "If-Match", + "Key": "test-precondition", + "Message": "The conditional request cannot succeed due to a conflicting operation against this resource." + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 409 + } + }, + "create-multipart-again": { + "Bucket": "", + "Key": "test-precondition", + "ServerSideEncryption": "AES256", + "UploadId": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "complete-multipart-if-match-put-before-restart": { + "Bucket": "", + "ETag": "\"60cd54a928cbbcbb6e7b5595bab46a9e-1\"", + "Key": "test-precondition", + "Location": "", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } + }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_multipart_if_match_with_delete": { + "recorded-date": "26-11-2024, 23:47:59", + "recorded-content": { + "put-obj": { + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "create-multipart": { + "Bucket": "", + "Key": "test-precondition", + "ServerSideEncryption": "AES256", + "UploadId": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "del-obj": { + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 204 + } + }, + "complete-multipart-after-del": { + "Error": { + "Code": "NoSuchKey", + "Key": "test-precondition", + "Message": "The specified key does not exist." + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 404 + } + }, + "put-obj-2": { + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "complete-multipart-if-match-after-put": { + "Error": { + "Code": "ConditionalRequestConflict", + "Condition": "If-Match", + "Key": "test-precondition", + "Message": "The conditional request cannot succeed due to a conflicting operation against this resource." + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 409 + } + } + } + }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_put_object_if_match_versioned_bucket": { + "recorded-date": "26-11-2024, 23:49:59", + "recorded-content": { + "put-obj": { + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "ServerSideEncryption": "AES256", + "VersionId": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "put-obj-if-none-match-bad-value": { + "Error": { + "Code": "PreconditionFailed", + "Condition": "If-Match", + "Message": "At least one of the pre-conditions you specified did not hold" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 412 + } + }, + "del-obj": { + "DeleteMarker": true, + "VersionId": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 204 + } + }, + "put-obj-after-del-exc": { + "Error": { + "Code": "NoSuchKey", + "Key": "test-precondition", + "Message": "The specified key does not exist." + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 404 + } + }, + "put-obj-after-del": { + "ETag": "\"b022e6afbcd118faed117e3c2b6e7b19\"", + "ServerSideEncryption": "AES256", + "VersionId": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "put-obj-if-match": { + "ETag": "\"98e41c14fd4ec56bafc444346ecb74b7\"", + "ServerSideEncryption": "AES256", + "VersionId": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "list-object-versions": { + "DeleteMarkers": [ + { + "IsLatest": false, + "Key": "test-precondition", + "LastModified": "datetime", + "Owner": { + "DisplayName": "", + "ID": "" + }, + "VersionId": "" + } + ], + "EncodingType": "url", + "IsTruncated": false, + "KeyMarker": "", + "MaxKeys": 1000, + "Name": "", + "Prefix": "", + "VersionIdMarker": "", + "Versions": [ + { + "ETag": "\"98e41c14fd4ec56bafc444346ecb74b7\"", + "IsLatest": true, + "Key": "test-precondition", + "LastModified": "datetime", + "Owner": { + "DisplayName": "", + "ID": "" + }, + "Size": 13, + "StorageClass": "STANDARD", + "VersionId": "" + }, + { + "ETag": "\"b022e6afbcd118faed117e3c2b6e7b19\"", + "IsLatest": false, + "Key": "test-precondition", + "LastModified": "datetime", + "Owner": { + "DisplayName": "", + "ID": "" + }, + "Size": 14, + "StorageClass": "STANDARD", + "VersionId": "" + }, + { + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "IsLatest": false, + "Key": "test-precondition", + "LastModified": "datetime", + "Owner": { + "DisplayName": "", + "ID": "" + }, + "Size": 4, + "StorageClass": "STANDARD", + "VersionId": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } + }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_put_object_if_match_and_if_none_match_validation": { + "recorded-date": "26-11-2024, 23:54:00", + "recorded-content": { + "put-obj-both-precondition": { + "Error": { + "Code": "NotImplemented", + "Header": "If-Match,If-None-Match", + "Message": "A header you provided implies functionality that is not implemented", + "additionalMessage": "Multiple conditional request headers present in the request" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 501 + } + } + } } } diff --git a/tests/aws/services/s3/test_s3_api.validation.json b/tests/aws/services/s3/test_s3_api.validation.json index 065b5a046f397..75a2c83e9d8d9 100644 --- a/tests/aws/services/s3/test_s3_api.validation.json +++ b/tests/aws/services/s3/test_s3_api.validation.json @@ -119,12 +119,33 @@ "tests/aws/services/s3/test_s3_api.py::TestS3ObjectLock::test_put_object_lock_configuration_on_existing_bucket": { "last_validated_date": "2024-01-15T03:13:25+00:00" }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_multipart_if_match_with_delete": { + "last_validated_date": "2024-11-26T23:47:58+00:00" + }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_multipart_if_match_with_put": { + "last_validated_date": "2024-11-26T20:44:45+00:00" + }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_multipart_if_match_with_put_identical": { + "last_validated_date": "2024-11-26T23:46:02+00:00" + }, "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_multipart_if_none_match_with_delete": { "last_validated_date": "2024-08-21T22:26:26+00:00" }, "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_multipart_if_none_match_with_put": { "last_validated_date": "2024-08-21T22:26:28+00:00" }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_put_object_if_match": { + "last_validated_date": "2024-11-26T16:57:17+00:00" + }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_put_object_if_match_and_if_none_match_validation": { + "last_validated_date": "2024-11-26T23:54:00+00:00" + }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_put_object_if_match_validation": { + "last_validated_date": "2024-11-26T20:38:49+00:00" + }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_put_object_if_match_versioned_bucket": { + "last_validated_date": "2024-11-26T23:49:57+00:00" + }, "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_put_object_if_none_match": { "last_validated_date": "2024-08-21T22:26:21+00:00" }, From 0ed6453ca0a9ba3bcf5fffb5481882a84b68b57f Mon Sep 17 00:00:00 2001 From: Benjamin Simon Date: Wed, 27 Nov 2024 11:35:51 +0100 Subject: [PATCH 2/2] add multipart etag test --- tests/aws/services/s3/test_s3_api.py | 58 ++++++++++++++++ .../aws/services/s3/test_s3_api.snapshot.json | 66 +++++++++++++++++++ .../services/s3/test_s3_api.validation.json | 3 + 3 files changed, 127 insertions(+) diff --git a/tests/aws/services/s3/test_s3_api.py b/tests/aws/services/s3/test_s3_api.py index 82a367ba84b67..d30aec4d8ba6e 100644 --- a/tests/aws/services/s3/test_s3_api.py +++ b/tests/aws/services/s3/test_s3_api.py @@ -2122,3 +2122,61 @@ def test_put_object_if_match_and_if_none_match_validation( with pytest.raises(ClientError) as e: aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfNoneMatch="*", IfMatch="abcdef") snapshot.match("put-obj-both-precondition", e.value.response) + + @markers.aws.validated + def test_multipart_if_match_etag(self, s3_bucket, aws_client, snapshot): + key = "test-precondition" + put_obj = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test") + snapshot.match("put-obj", put_obj) + put_obj_etag_1 = put_obj["ETag"] + + create_multipart = aws_client.s3.create_multipart_upload(Bucket=s3_bucket, Key=key) + snapshot.match("create-multipart", create_multipart) + upload_id = create_multipart["UploadId"] + + upload_part = aws_client.s3.upload_part( + Bucket=s3_bucket, Key=key, UploadId=upload_id, Body="test", PartNumber=1 + ) + parts = [{"ETag": upload_part["ETag"], "PartNumber": 1}] + + complete_multipart_1 = aws_client.s3.complete_multipart_upload( + Bucket=s3_bucket, + Key=key, + MultipartUpload={"Parts": parts}, + UploadId=upload_id, + IfMatch=put_obj_etag_1, + ) + snapshot.match("complete-multipart-if-match", complete_multipart_1) + + multipart_etag = complete_multipart_1["ETag"] + # those are different, because multipart etag contains the amount of parts and is the hash of the hashes of the + # part + assert put_obj_etag_1 != multipart_etag + + create_multipart = aws_client.s3.create_multipart_upload(Bucket=s3_bucket, Key=key) + snapshot.match("create-multipart-overwrite", create_multipart) + upload_id = create_multipart["UploadId"] + + upload_part = aws_client.s3.upload_part( + Bucket=s3_bucket, Key=key, UploadId=upload_id, Body="test", PartNumber=1 + ) + parts = [{"ETag": upload_part["ETag"], "PartNumber": 1}] + + with pytest.raises(ClientError) as e: + aws_client.s3.complete_multipart_upload( + Bucket=s3_bucket, + Key=key, + MultipartUpload={"Parts": parts}, + UploadId=upload_id, + IfMatch=put_obj_etag_1, + ) + snapshot.match("complete-multipart-if-match-true-etag", e.value.response) + + complete_multipart_1 = aws_client.s3.complete_multipart_upload( + Bucket=s3_bucket, + Key=key, + MultipartUpload={"Parts": parts}, + UploadId=upload_id, + IfMatch=multipart_etag, + ) + snapshot.match("complete-multipart-if-match-overwrite-multipart", complete_multipart_1) diff --git a/tests/aws/services/s3/test_s3_api.snapshot.json b/tests/aws/services/s3/test_s3_api.snapshot.json index 599d74f17e149..9f854c113731c 100644 --- a/tests/aws/services/s3/test_s3_api.snapshot.json +++ b/tests/aws/services/s3/test_s3_api.snapshot.json @@ -4093,5 +4093,71 @@ } } } + }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_multipart_if_match_etag": { + "recorded-date": "27-11-2024, 10:35:20", + "recorded-content": { + "put-obj": { + "ETag": "\"098f6bcd4621d373cade4e832627b4f6\"", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "create-multipart": { + "Bucket": "", + "Key": "test-precondition", + "ServerSideEncryption": "AES256", + "UploadId": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "complete-multipart-if-match": { + "Bucket": "", + "ETag": "\"60cd54a928cbbcbb6e7b5595bab46a9e-1\"", + "Key": "test-precondition", + "Location": "", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "create-multipart-overwrite": { + "Bucket": "", + "Key": "test-precondition", + "ServerSideEncryption": "AES256", + "UploadId": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "complete-multipart-if-match-true-etag": { + "Error": { + "Code": "PreconditionFailed", + "Condition": "If-Match", + "Message": "At least one of the pre-conditions you specified did not hold" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 412 + } + }, + "complete-multipart-if-match-overwrite-multipart": { + "Bucket": "", + "ETag": "\"60cd54a928cbbcbb6e7b5595bab46a9e-1\"", + "Key": "test-precondition", + "Location": "", + "ServerSideEncryption": "AES256", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } } } diff --git a/tests/aws/services/s3/test_s3_api.validation.json b/tests/aws/services/s3/test_s3_api.validation.json index 75a2c83e9d8d9..bd31ae704dba0 100644 --- a/tests/aws/services/s3/test_s3_api.validation.json +++ b/tests/aws/services/s3/test_s3_api.validation.json @@ -119,6 +119,9 @@ "tests/aws/services/s3/test_s3_api.py::TestS3ObjectLock::test_put_object_lock_configuration_on_existing_bucket": { "last_validated_date": "2024-01-15T03:13:25+00:00" }, + "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_multipart_if_match_etag": { + "last_validated_date": "2024-11-27T10:35:19+00:00" + }, "tests/aws/services/s3/test_s3_api.py::TestS3ObjectWritePrecondition::test_multipart_if_match_with_delete": { "last_validated_date": "2024-11-26T23:47:58+00:00" },