8000 S3: fix Checksum handling in UploadPartCopy (#12753) · localstack/localstack@df9ebe9 · GitHub
[go: up one dir, main page]
Skip to content

Commit df9ebe9

Browse files
bentskubentsku
authored
S3: fix Checksum handling in UploadPartCopy (#12753)
1 parent bdc489f commit df9ebe9

File tree

7 files changed

+257
-20
lines changed

7 files changed

+257
-20
lines changed

‎localstack-core/localstack/services/s3/provider.py

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2397,11 +2397,19 @@ def upload_part_copy(
23972397
request: UploadPartCopyRequest,
23982398
) -> UploadPartCopyOutput:
23992399
# TODO: handle following parameters:
2400-
# copy_source_if_match: CopySourceIfMatch = None,
2401-
# copy_source_if_modified_since: CopySourceIfModifiedSince = None,
2402-
# copy_source_if_none_match: CopySourceIfNoneMatch = None,
2403-
# copy_source_if_unmodified_since: CopySourceIfUnmodifiedSince = None,
2404-
# request_payer: RequestPayer = None,
2400+
# CopySourceIfMatch: Optional[CopySourceIfMatch]
2401+
# CopySourceIfModifiedSince: Optional[CopySourceIfModifiedSince]
2402+
# CopySourceIfNoneMatch: Optional[CopySourceIfNoneMatch]
2403+
# CopySourceIfUnmodifiedSince: Optional[CopySourceIfUnmodifiedSince]
2404+
# SSECustomerAlgorithm: Optional[SSECustomerAlgorithm]
2405+
# SSECustomerKey: Optional[SSECustomerKey]
2406+
# SSECustomerKeyMD5: Optional[SSECustomerKeyMD5]
2407+
# CopySourceSSECustomerAlgorithm: Optional[CopySourceSSECustomerAlgorithm]
2408+
# CopySourceSSECustomerKey: Optional[CopySourceSSECustomerKey]
2409+
# CopySourceSSECustomerKeyMD5: Optional[CopySourceSSECustomerKeyMD5]
2410+
# RequestPayer: Optional[RequestPayer]
2411+
# ExpectedBucketOwner: Optional[AccountId]
2412+
# ExpectedSourceBucketOwner: Optional[AccountId]
24052413
dest_bucket = request["Bucket"]
24062414
dest_key = request["Key"]
24072415
store = self.get_store(context.account_id, context.region)
@@ -2449,24 +2457,22 @@ def upload_part_copy(
24492457
)
24502458

24512459
source_range = request.get("CopySourceRange")
2452-
# TODO implement copy source IF (done in ASF provider)
2460+
# TODO implement copy source IF
24532461

24542462
range_data: Optional[ObjectRange] = None
24552463
if source_range:
24562464
range_data = parse_copy_source_range_header(source_range, src_s3_object.size)
24572465

24582466
s3_part = S3Part(part_number=part_number)
2467+
if s3_multipart.checksum_algorithm:
2468+
s3_part.checksum_algorithm = s3_multipart.checksum_algorithm
24592469

24602470
stored_multipart = self._storage_backend.get_multipart(dest_bucket, s3_multipart)
24612471
stored_multipart.copy_from_object(s3_part, src_bucket, src_s3_object, range_data)
24622472

24632473
s3_multipart.parts[part_number] = s3_part
24642474

2465-
# TODO: return those fields (checksum not handled currently in moto for parts)
2466-
# ChecksumCRC32: Optional[ChecksumCRC32]
2467-
# ChecksumCRC32C: Optional[ChecksumCRC32C]
2468-
# ChecksumSHA1: Optional[ChecksumSHA1]
2469-
# ChecksumSHA256: Optional[ChecksumSHA256]
2475+
# TODO: return those fields
24702476
# RequestCharged: Optional[RequestCharged]
24712477

24722478
result = CopyPartResult(
@@ -2481,6 +2487,9 @@ def upload_part_copy(
24812487
if src_s3_bucket.versioning_status and src_s3_object.version_id:
24822488
response["CopySourceVersionId"] = src_s3_object.version_id
24832489

2490+
if s3_part.checksum_algorithm:
2491+
result[f"Checksum{s3_part.checksum_algorithm.upper()}"] = s3_part.checksum_value
2492+
24842493
add_encryption_to_response(response, s3_object=s3_multipart.object)
24852494

24862495
return response
@@ -2750,7 +2759,7 @@ def list_parts(
27502759
PartNumber=part_number,
27512760
Size=part.size,
27522761
)
2753-
if s3_multipart.checksum_algorithm:
2762+
if s3_multipart.checksum_algorithm and part.checksum_algorithm:
27542763
part_item[f"Checksum{part.checksum_algorithm.upper()}"] = part.checksum_value
27552764

27562765
parts.append(part_item)

‎localstack-core/localstack/services/s3/storage/ephemeral.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -340,10 +340,12 @@ def copy_from_object(
340340
):
341341
if not range_data:
342342
stored_part.write(src_stored_object)
343-
return
343+
else:
344+
object_slice = LimitedStream(src_stored_object, range_data=range_data)
345+
stored_part.write(object_slice)
344346

345-
object_slice = LimitedStream(src_stored_object, range_data=range_data)
346-
stored_part.write(object_slice)
347+
if s3_part.checksum_algorithm:
348+
s3_part.checksum_value = stored_part.checksum
347349

348350

349351
class BucketTemporaryFileSystem(TypedDict):

‎tests/aws/services/s3/test_s3.py

Lines changed: 78 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -483,7 +483,6 @@ def test_metadata_header_character_decoding(self, s3_bucket, snapshot, aws_clien
483483
assert metadata_saved["Metadata"] == {"test_meta_1": "foo", "__meta_2": "bar"}
484484

485485
@markers.aws.validated
486-
@markers.snapshot.skip_snapshot_verify(paths=["$..ChecksumType"])
487486
def test_upload_file_multipart(self, s3_bucket, tmpdir, snapshot, aws_client):
488487
snapshot.add_transformer(snapshot.transform.s3_api())
489488
key = "my-key"
@@ -13023,6 +13022,84 @@ def test_multipart_size_validation(self, aws_client, s3_bucket, snapshot):
1302313022
)
1302413023
snapshot.match("get-object-attrs", object_attrs)
1302513024

13025+
@markers.aws.validated
13026+
def test_multipart_upload_part_copy_checksum(self, s3_bucket, snapshot, aws_client):
13027+
snapshot.add_transformer(
13028+
[
13029+
snapshot.transform.key_value("Bucket", reference_replacement=False),
13030+
snapshot.transform.key_value("Location"),
13031+
snapshot.transform.key_value("UploadId"),
13032+
snapshot.transform.key_value("DisplayName", reference_replacement=False),
13033+
snapshot.transform.key_value("ID", reference_replacement=False),
13034+
]
13035+
)
13036+
13037+
part_key = "test-part-checksum"
13038+
put_object = aws_client.s3.put_object(
13039+
Bucket=s3_bucket,
13040+
Key=part_key,
13041+
Body="this is a part",
13042+
)
13043+
snapshot.match("put-object", put_object)
13044+
13045+
key_name = "test-multipart-checksum"
13046+
response = aws_client.s3.create_multipart_upload(
13047+
Bucket=s3_bucket, Key=key_name, ChecksumAlgorithm="SHA256"
13048+
)
13049+
snapshot.match("create-mpu-checksum-sha256", response)
13050+
upload_id = response["UploadId"]
13051+
13052+
copy_source_key = f"{s3_bucket}/{part_key}"
13053+
upload_part_copy = aws_client.s3.upload_part_copy(
13054+
Bucket=s3_bucket,
13055+
UploadId=upload_id,
13056+
Key=key_name,
13057+
PartNumber=1,
13058+
CopySource=copy_source_key,
13059+
)
13060+
snapshot.match("upload-part-copy", upload_part_copy)
13061+
13062+
list_parts = aws_client.s3.list_parts(
13063+
Bucket=s3_bucket,
13064+
UploadId=upload_id,
13065+
Key=key_name,
13066+
)
13067+
snapshot.match("list-parts", list_parts)
13068+
13069+
# complete with no checksum type specified, just all default values
13070+
response = aws_client.s3.complete_multipart_upload(
13071+
Bucket=s3_bucket,
13072+
Key=key_name,
13073+
MultipartUpload={
13074+
"Parts": [
13075+
{
13076+
"ETag": upload_part_copy["CopyPartResult"]["ETag"],
13077+
"PartNumber": 1,
13078+
"ChecksumSHA256": upload_part_copy["CopyPartResult"]["ChecksumSHA256"],
13079+
}
13080+
]
13081+
},
13082+
UploadId=upload_id,
13083+
)
13084+
snapshot.match("complete-multipart-checksum", response)
13085+
13086+
get_object_with_checksum = aws_client.s3.get_object(
13087+
Bucket=s3_bucket, Key=key_name, ChecksumMode="ENABLED"
13088+
)
13089+
snapshot.match("get-object-with-checksum", get_object_with_checksum)
13090+
13091+
head_object_with_checksum = aws_client.s3.head_object(
13092+
Bucket=s3_bucket, Key=key_name, ChecksumMode="ENABLED"
13093+
)
13094+
snapshot.match("head-object-with-checksum", head_object_with_checksum)
13095+
13096+
object_attrs = aws_client.s3.get_object_attributes(
13097+
Bucket=s3_bucket,
13098+
Key=key_name,
13099+
ObjectAttributes=["Checksum", "ETag"],
13100+
)
13101+
snapshot.match("get-object-attrs", object_attrs)
13102+
1302613103

1302713104
def _s3_client_pre_signed_client(conf: Config, endpoint_url: str = None):
1302813105
if is_aws_cloud():

‎tests/aws/services/s3/test_s3.snapshot.json

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17411,5 +17411,133 @@
1741117411
}
1741217412
}
1741317413
}
17414+
},
17415+
"tests/aws/services/s3/test_s3.py::TestS3MultipartUploadChecksum::test_multipart_upload_part_copy_checksum": {
17416+
"recorded-date": "13-06-2025, 12:45:49",
17417+
"recorded-content": {
17418+
"put-object": {
17419+
"ChecksumCRC32": "nG7pIA==",
17420+
"ChecksumType": "FULL_OBJECT",
17421+
"ETag": "\"11df95d595559285eb2b042124e74f09\"",
17422+
"ServerSideEncryption": "AES256",
17423+
"ResponseMetadata": {
17424+
"HTTPHeaders": {},
17425+
"HTTPStatusCode": 200
17426+
}
17427+
},
17428+
"create-mpu-checksum-sha256": {
17429+
"Bucket": "bucket",
17430+
"ChecksumAlgorithm": "SHA256",
17431+
"ChecksumType": "COMPOSITE",
17432+
"Key": "test-multipart-checksum",
17433+
"ServerSideEncryption": "AES256",
17434+
"UploadId": "<upload-id:1>",
17435+
"ResponseMetadata": {
17436+
"HTTPHeaders": {},
17437+
"HTTPStatusCode": 200
17438+
}
17439+
},
17440+
"upload-part-copy": {
17441+
"CopyPartResult": {
17442+
"ChecksumSHA256": "+j3Oc5P9QdoIdPJ4lFSyNlAAX0G7Am+wZsxu4FYN+wo=",
17443+
"ETag": "\"11df95d595559285eb2b042124e74f09\"",
17444+
"LastModified": "datetime"
17445+
},
17446+
"ServerSideEncryption" 1241 ;: "AES256",
17447+
"ResponseMetadata": {
17448+
"HTTPHeaders": {},
17449+
"HTTPStatusCode": 200
17450+
}
17451+
},
17452+
"list-parts": {
17453+
"Bucket": "bucket",
17454+
"ChecksumAlgorithm": "SHA256",
17455+
"ChecksumType": "COMPOSITE",
17456+
"Initiator": {
17457+
"DisplayName": "display-name",
17458+
"ID": "i-d"
17459+
},
17460+
"IsTruncated": false,
17461+
"Key": "test-multipart-checksum",
17462+
"MaxParts": 1000,
17463+
"NextPartNumberMarker": 1,
17464+
"Owner": {
17465+
"DisplayName": "display-name",
17466+
"ID": "i-d"
17467+
},
17468+
"PartNumberMarker": 0,
17469+
"Parts": [
17470+
{
17471+
"ChecksumSHA256": "+j3Oc5P9QdoIdPJ4lFSyNlAAX0G7Am+wZsxu4FYN+wo=",
17472+
"ETag": "\"11df95d595559285eb2b042124e74f09\"",
17473+
"LastModified": "datetime",
17474+
"PartNumber": 1,
17475+
"Size": 14
17476+
}
17477+
],
17478+
"StorageClass": "STANDARD",
17479+
"UploadId": "<upload-id:1>",
17480+
"ResponseMetadata": {
17481+
"HTTPHeaders": {},
17482+
"HTTPStatusCode": 200
17483+
}
17484+
},
17485+
"complete-multipart-checksum": {
17486+
"Bucket": "bucket",
17487+
"ChecksumSHA256": "/4+xERoRlzE2Ryan+GX/sqNSrf6Qe30L2IM7APXadSE=-1",
17488+
"ChecksumType": "COMPOSITE",
17489+
"ETag": "\"395d97c07920de036bfa21e7568a2e9f-1\"",
17490+
"Key": "test-multipart-checksum",
17491+
"Location": "<location:1>",
17492+
"ServerSideEncryption": "AES256",
17493+
"ResponseMetadata": {
17494+
"HTTPHeaders": {},
17495+
"HTTPStatusCode": 200
17496+
}
17497+
},
17498+
"get-object-with-checksum": {
17499+
"AcceptRanges": "bytes",
17500+
"Body": "this is a part",
17501+
"ChecksumSHA256": "/4+xERoRlzE2Ryan+GX/sqNSrf6Qe30L2IM7APXadSE=-1",
17502+
"ChecksumType": "COMPOSITE",
17503+
"ContentLength": 14,
17504+
"ContentType": "binary/octet-stream",
17505+
"ETag": "\"395d97c07920de036bfa21e7568a2e9f-1\"",
17506+
"LastModified": "datetime",
17507+
"Metadata": {},
17508+
"ServerSideEncryption": "AES256",
17509+
"ResponseMetadata": {
17510+
"HTTPHeaders": {},
17511+
"HTTPStatusCode": 200
17512+
}
17513+
},
17514+
"head-object-with-checksum": {
17515+
"AcceptRanges": "bytes",
17516+
"ChecksumSHA256": "/4+xERoRlzE2Ryan+GX/sqNSrf6Qe30L2IM7APXadSE=-1",
17517+
"ChecksumType": "COMPOSITE",
17518+
"ContentLength": 14,
17519+
"ContentType": "binary/octet-stream",
17520+
"ETag": "\"395d97c07920de036bfa21e7568a2e9f-1\"",
17521+
"LastModified": "datetime",
17522+
"Metadata": {},
17523+
"ServerSideEncryption": "AES256",
17524+
"ResponseMetadata": {
17525+
"HTTPHeaders": {},
17526+
"HTTPStatusCode": 200
17527+
}
17528+
},
17529+
"get-object-attrs": {
17530+
"Checksum": {
17531+
"ChecksumSHA256": "/4+xERoRlzE2Ryan+GX/sqNSrf6Qe30L2IM7APXadSE=",
17532+
"ChecksumType": "COMPOSITE"
17533+
},
17534+
"ETag": "395d97c07920de036bfa21e7568a2e9f-1",
17535+
"LastModified": "datetime",
17536+
"ResponseMetadata": {
17537+
"HTTPHeaders": {},
17538+
"HTTPStatusCode": 200
17539+
}
17540+
}
17541+
}
1741417542
}
1741517543
}

‎tests/aws/services/s3/test_s3.validation.json

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -671,6 +671,15 @@
671671
"tests/aws/services/s3/test_s3.py::TestS3MultipartUploadChecksum::test_multipart_upload_part_checksum_exception[SHA256]": {
672672
"last_validated_date": "2025-03-17T18:21:07+00:00"
673673
},
674+
"tests/aws/services/s3/test_s3.py::TestS3MultipartUploadChecksum::test_multipart_upload_part_copy_checksum": {
675+
"last_validated_date": "2025-06-13T12:45:50+00:00",
676+
"durations_in_seconds": {
677+
"setup": 0.92,
678+
"call": 1.39,
679+
"teardown": 1.01,
680+
"total": 3.32
681+
}
682+
},
674683
"tests/aws/services/s3/test_s3.py::TestS3ObjectLockLegalHold::test_delete_locked_object": {
675684
"last_validated_date": "2025-01-21T18:17:15+00:00"
676685
},

‎tests/aws/services/s3/test_s3_api.snapshot.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3237,7 +3237,7 @@
32373237
}
32383238
},
32393239
"tests/aws/services/s3/test_s3_api.py::TestS3Multipart::test_upload_part_copy_range": {
3240-
"recorded-date": "21-01-2025, 18:10:14",
3240+
"recorded-date": "13-06-2025, 12:42:54",
32413241
"recorded-content": {
32423242
"put-src-object": {
32433243
"ChecksumCRC32": "poTHxg==",
@@ -3517,7 +3517,7 @@
35173517
}
35183518
},
35193519
"tests/aws/services/s3/test_s3_api.py::TestS3Multipart::test_upload_part_copy_no_copy_source_range": {
3520-
"recorded-date": "21-01-2025, 18:10:16",
3520+
"recorded-date": "13-06-2025, 12:42:57",
35213521
"recorded-content": {
35223522
"put-src-object": {
35233523
"ChecksumCRC32": "poTHxg==",

‎tests/aws/services/s3/test_s3_api.validation.json

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,22 @@
6969
"last_validated_date": "2025-01-21T18:10:31+00:00"
7070
},
7171
"tests/aws/services/s3/test_s3_api.py::TestS3Multipart::test_upload_part_copy_no_copy_source_range": {
72-
"last_validated_date": "2025-01-21T18:10:16+00:00"
72+
"last_validated_date": "2025-06-13T12:42:58+00:00",
73+
"durations_in_seconds": {
74+
"setup": 0.55,
75+
"call": 0.66,
76+
"teardown": 1.07,
77+
"total": 2.28
78+
}
7379
},
7480
"tests/aws/services/s3/test_s3_api.py::TestS3Multipart::test_upload_part_copy_range": {
75-
"last_validated_date": "2025-01-21T18:10:14+00:00"
81+
"last_validated_date": "2025-06-13T12:42:55+00:00",
82+
"durations_in_seconds": {
83+
"setup": 1.02,
84+
"call": 5.28,
85+
"teardown": 1.54,
86+
"total": 7.84
87+
}
7688
},
7789
"tests/aws/services/s3/test_s3_api.py::TestS3ObjectCRUD::test_delete_object": {
7890
"last_validated_date": "2025-01-21T18:09:31+00:00"

0 commit comments

Comments
 (0)
0