8000 implement S3 IfMatch · localstack/localstack@bbd50e1 · GitHub
[go: up one dir, main page]
Skip to content

Commit bbd50e1

Browse files
bentskubentsku
committed
implement S3 IfMatch
1 parent 8ca9935 commit bbd50e1

File tree

4 files changed

+772
-4
lines changed

4 files changed

+772
-4
lines changed

‎localstack-core/localstack/services/s3/provider.py

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -660,11 +660,20 @@ def put_object(
660660

661661
validate_object_key(key)
662662

663-
if (if_none_match := request.get("IfNoneMatch")) and if_none_match != "*":
663+
if_match = request.get("IfMatch")
664+
if (if_none_match := request.get("IfNoneMatch")) and if_match:
664665
raise NotImplementedException(
665666
"A header you provided implies functionality that is not implemented",
666-
Header="If-None-Match",
667-
additionalMessage="We don't accept the provided value of If-None-Match header for this API",
667+
Header="If-Match,If-None-Match",
668+
additionalMessage="Multiple conditional request headers present in the request",
669+
)
670+
671+
elif (if_none_match and if_none_match != "*") or (if_match and if_match == "*"):
672+
header_name = "If-None-Match" if if_none_match else "If-Match"
673+
raise NotImplementedException(
674+
"A header you provided implies functionality that is not implemented",
675+
Header=header_name,
676+
additionalMessage=f"We don't accept the provided value of {header_name} header for this API",
668677
)
669678

670679
system_metadata = get_system_metadata_from_request(request)
@@ -758,6 +767,9 @@ def put_object(
758767
Condition="If-None-Match",
759768
)
760769

770+
elif if_match:
771+
verify_object_equality_precondition_write(s3_bucket, key, if_match)
772+
761773
s3_stored_object.write(body)
762774

763775
if (
@@ -2377,7 +2389,14 @@ def complete_multipart_upload(
23772389
UploadId=upload_id,
23782390
)
23792391

2380-
if if_none_match:
2392+
if if_none_match and if_match:
2393+
raise NotImplementedException(
2394+
"A header you provided implies functionality that is not implemented",
2395+
Header="If-Match,If-None-Match",
2396+
additionalMessage="Multiple conditional request headers present in the request",
2397+
)
2398+
2399+
elif if_none_match:
23812400
if if_none_match != "*":
23822401
raise NotImplementedException(
341A 23832402
"A header you provided implies functionality that is not implemented",
@@ -2396,6 +2415,17 @@ def complete_multipart_upload(
23962415
Key=key,
23972416
)
23982417

2418+
elif if_match:
2419+
if if_match == "*":
2420+
raise NotImplementedException(
2421+
"A header you provided implies functionality that is not implemented",
2422+
Header="If-None-Match",
2423+
additionalMessage="We don't accept the provided value of If-None-Match header for this API",
2424+
)
2425+
verify_object_equality_precondition_write(
2426+
s3_bucket, key, if_match, initiated=s3_multipart.initiated
2427+
)
2428+
23992429
parts = multipart_upload.get("Parts", [])
24002430
if not parts:
24012431
raise InvalidRequest("You must specify at least one part")
@@ -4395,3 +4425,27 @@ def get_access_control_policy_for_new_resource_request(
43954425

43964426
def object_exists_for_precondition_write(s3_bucket: S3Bucket, key: ObjectKey) -> bool:
43974427
return (existing := s3_bucket.objects.get(key)) and not isinstance(existing, S3DeleteMarker)
4428+
4429+
4430+
def verify_object_equality_precondition_write(
4431+
s3_bucket: S3Bucket,
4432+
key: ObjectKey,
4433+
etag: str,
4434+
initiated: datetime.datetime | None = None,
4435+
) -> None:
4436+
existing = s3_bucket.objects.get(key)
4437+
if not existing or isinstance(existing, S3DeleteMarker):
4438+
raise NoSuchKey("The specified key does not exist.", Key=key)
4439+
4440+
if not existing.etag == etag.strip('"'):
4441+
raise PreconditionFailed(
4442+
"At least one of the pre-conditions you specified did not hold",
4443+
Condition="If-Match",
4444+
)
4445+
4446+
if initiated and initiated < existing.last_modified:
4447+
raise ConditionalRequestConflict(
4448+
"The conditional request cannot succeed due to a conflicting operation against this resource.",
4449+
Condition="If-Match",
4450+
Key=key,
4451+
)

‎tests/aws/services/s3/test_s3_api.py

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1719,6 +1719,10 @@ def test_bucket_acceleration_configuration_exc(
17191719

17201720

17211721
class TestS3ObjectWritePrecondition:
1722+
"""
1723+
https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-writes.html
1724+
"""
1725+
17221726
@pytest.fixture(autouse=True)
17231727
def add_snapshot_transformers(self, snapshot):
17241728
snapshot.add_transformers_list(
@@ -1869,3 +1873,252 @@ def test_put_object_if_none_match_versioned_bucket(self, s3_bucket, aws_client,
18691873

18701874
list_object_versions = aws_client.s3.list_object_versions(Bucket=s3_bucket)
18711875
snapshot.match("list-object-versions", list_object_versions)
1876+
1877+
@markers.aws.validated
1878+
def test_put_object_if_match(self, s3_bucket, aws_client, snapshot):
1879+
key = "test-precondition"
1880+
put_obj = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test")
1881+
snapshot.match("put-obj", put_obj)
1882+
etag = put_obj["ETag"]
1883+
1884+
with pytest.raises(ClientError) as e:
1885+
# empty object is provided
1886+
aws_client.s3.put_object(
1887+
Bucket=s3_bucket, Key=key, IfMatch="d41d8cd98f00b204e9800998ecf8427e"
1888+
)
1889+
snapshot.match("put-obj-if-match-wrong-etag", e.value.response)
1890+
1891+
put_obj_overwrite = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfMatch=etag)
1892+
snapshot.match("put-obj-overwrite", put_obj_overwrite)
1893+
1894+
del_obj = aws_client.s3.delete_object(Bucket=s3_bucket, Key=key)
1895+
snapshot.match("del-obj", del_obj)
1896+
1897+
with pytest.raises(ClientError) as e:
1898+
aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfMatch=etag)
1899+
snapshot.match("put-obj-if-match-key-not-exists", e.value.response)
1900+
1901+
put_obj_after_del = aws_client.s3.put_object(Bucket=s3_bucket, Key=key)
1902+
snapshot.match("put-obj-after-del", put_obj_after_del)
1903+
1904+
@markers.aws.validated
1905+
def test_put_object_if_match_validation(self, s3_bucket, aws_client, snapshot):
1906+
key = "test-precondition-validation"
1907+
1908+
with pytest.raises(ClientError) as e:
1909+
aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfMatch="*")
1910+
snapshot.match("put-obj-if-match-star-value", e.value.response)
1911+
1912+
with pytest.raises(ClientError) as e:
1913+
aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfMatch="abcdef")
1914+
snapshot.match("put-obj-if-match-bad-value", e.value.response)
1915+
1916+
with pytest.raises(ClientError) as e:
1917+
aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfMatch="bad-char_/")
1918+
snapshot.match("put-obj-if-match-bad-value-2", e.value.response)
1919+
1920+
@markers.aws.validated
1921+
def test_multipart_if_match_with_put(self, s3_bucket, aws_client, snapshot):
1922+
key = "test-precondition"
1923+
put_obj = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test")
1924+
snapshot.match("put-obj", put_obj)
1925+
put_obj_etag_1 = put_obj["ETag"]
1926+
1927+
create_multipart = aws_client.s3.create_multipart_upload(Bucket=s3_bucket, Key=key)
1928+
snapshot.match("create-multipart", create_multipart)
1929+
upload_id = create_multipart["UploadId"]
1930+
1931+
upload_part = aws_client.s3.upload_part(
1932+
Bucket=s3_bucket, Key=key, UploadId=upload_id, Body="test", PartNumber=1
1933+
)
1934+
parts = [{"ETag": upload_part["ETag"], "PartNumber": 1}]
1935+
1936+
put_obj_2 = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test2")
1937+
snapshot.match("put-obj-during", put_obj_2)
1938+
put_obj_etag_2 = put_obj_2["ETag"]
1939+
1940+
with pytest.raises(ClientError) as e:
1941+
aws_client.s3.complete_multipart_upload(
1942+
Bucket=s3_bucket,
1943+
Key=key,
1944+
MultipartUpload={"Parts": parts},
1945+
UploadId=upload_id,
1946+
IfMatch=put_obj_etag_1,
1947+
)
1948+
snapshot.match("complete-multipart-if-match-put-before", e.value.response)
1949+
1950+
# the previous PutObject request was done between the CreateMultipartUpload and completion, so it takes
1951+
# precedence
1952+
# you need to restart the whole multipart for it to work
1953+
with pytest.raises(ClientError) as e:
1954+
aws_client.s3.complete_multipart_upload(
1955+
Bucket=s3_bucket,
1956+
Key=key,
1957+
MultipartUpload={"Parts": parts},
1958+
UploadId=upload_id,
1959+
IfMatch=put_obj_etag_2,
1960+
)
1961+
snapshot.match("complete-multipart-if-match-put-during", e.value.response)
1962+
1963+
create_multipart = aws_client.s3.create_multipart_upload(Bucket=s3_bucket, Key=key)
1964+
snapshot.match("create-multipart-again", create_multipart)
1965+
upload_id = create_multipart["UploadId"]
1966+
1967+
upload_part = aws_client.s3.upload_part(
1968+
Bucket=s3_bucket, Key=key, UploadId=upload_id, Body="test", PartNumber=1
1969+
)
1970+
parts = [{"ETag": upload_part["ETag"], "PartNumber": 1}]
1971+
1972+
complete_multipart = aws_client.s3.complete_multipart_upload(
1973+
Bucket=s3_bucket,
1974+
Key=key,
1975+
MultipartUpload={"Parts": parts},
1976+
UploadId=upload_id,
1977+
IfMatch=put_obj_etag_2,
1978+
)
1979+
snapshot.match("complete-multipart-if-match-put-before-restart", complete_multipart)
1980+
1981+
@markers.aws.validated
1982+
def test_multipart_if_match_with_put_identical(self, s3_bucket, aws_client, snapshot):
1983+
key = "test-precondition"
1984+
put_obj = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test")
1985+
snapshot.match("put-obj", put_obj)
1986+
put_obj_etag_1 = put_obj["ETag"]
1987+
1988+
create_multipart = aws_client.s3.create_multipart_upload(Bucket=s3_bucket, Key=key)
1989+
snapshot.match("create-multipart", create_multipart)
1990+
upload_id = create_multipart["UploadId"]
1991+
1992+
upload_part = aws_client.s3.upload_part(
1993+
Bucket=s3_bucket, Key=key, UploadId=upload_id, Body="test", PartNumber=1
1994+
)
1995+
parts = [{"ETag": upload_part["ETag"], "PartNumber": 1}]
1996+
1997+
put_obj_2 = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test")
1998+
snapshot.match("put-obj-during", put_obj_2)
1999+
# same ETag as first put
2000+
put_obj_etag_2 = put_obj_2["ETag"]
2001+
assert put_obj_etag_1 == put_obj_etag_2
2002+
2003+
# it seems that even if we overwrite the object with the same content, S3 will still reject the request if a
2004+
# write operation was done between creation and completion of the multipart upload, like the `Delete`
2005+
# counterpart of `IfNoneMatch`
2006+
2007+
with pytest.raises(ClientError) as e:
2008+
aws_client.s3.complete_multipart_upload(
2009+
Bucket=s3_bucket,
2010+
Key=key,
2011+
MultipartUpload={"Parts": parts},
2012+
UploadId=upload_id,
2013+
IfMatch=put_obj_etag_2,
2014+
)
2015+
snapshot.match("complete-multipart-if-match-put-during", e.value.response)
2016+
# the previous PutObject request was done between the CreateMultipartUpload and completion, so it takes
2017+
# precedence
2018+
# you need to restart the whole multipart for it to work
2019+
2020+
create_multipart = aws_client.s3.create_multipart_upload(Bucket=s3_bucket, Key=key)
2021+
snapshot.match("create-multipart-again", create_multipart)
2022+
upload_id = create_multipart["UploadId"]
2023+
2024+
upload_part = aws_client.s3.upload_part(
2025+
Bucket=s3_bucket, Key=key, UploadId=upload_id, Body="test", PartNumber=1
2026+
)
2027+
parts = [{"ETag": upload_part["ETag"], "PartNumber": 1}]
2028+
2029+
complete_multipart = aws_client.s3.complete_multipart_upload(
2030+
Bucket=s3_bucket,
2031+
Key=key,
2032+
MultipartUpload={"Parts": parts},
2033+
UploadId=upload_id,
2034+
IfMatch=put_obj_etag_2,
2035+
)
2036+
snapshot.match("complete-multipart-if-match-put-before-restart", complete_multipart)
2037+
2038+
@markers.aws.validated
2039+
def test_multipart_if_match_with_delete(self, s3_bucket, aws_client, snapshot):
2040+
key = "test-precondition"
2041+
put_obj = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test")
2042+
snapshot.match("put-obj", put_obj)
2043+
obj_etag = put_obj["ETag"]
2044+
2045+
create_multipart = aws_client.s3.create_multipart_upload(Bucket=s3_bucket, Key=key)
2046+
snapshot.match("create-multipart", create_multipart)
2047+
upload_id = create_multipart["UploadId"]
2048+
2049+
upload_part = aws_client.s3.upload_part(
2050+
Bucket=s3_bucket, Key=key, UploadId=upload_id, Body="test", PartNumber=1
2051+
)
2052+
parts = [{"ETag": upload_part["ETag"], "PartNumber": 1}]
2053+
2054+
del_obj = aws_client.s3.delete_object(Bucket=s3_bucket, Key=key)
2055+
snapshot.match("del-obj", del_obj)
2056+
2057+
with pytest.raises(ClientError) as e:
2058+
aws_client.s3.complete_multipart_upload(
2059+
Bucket=s3_bucket,
2060+
Key=key,
2061+
MultipartUpload={"Parts": parts},
2062+
UploadId=upload_id,
2063+
IfMatch=obj_etag,
2064+
)
2065+
snapshot.match("complete-multipart-after-del", e.value.response)
2066+
2067+
put_obj_2 = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test")
2068+
snapshot.match("put-obj-2", put_obj_2)
2069+
obj_etag_2 = put_obj_2["ETag"]
2070+
2071+
with pytest.raises(ClientError) as e:
2072+
# even if we recreated the object, it still fails as it was done after the start of the upload
2073+
aws_client.s3.complete_multipart_upload(
2074+
Bucket=s3_bucket,
2075+
Key=key,
2076+
MultipartUpload={"Parts": parts},
2077+
UploadId=upload_id,
2078+
IfMatch=obj_etag_2,
2079+
)
2080+
snapshot.match("complete-multipart-if-match-after-put", e.value.response)
2081+
2082+
@markers.aws.validated
2083+
def test_put_object_if_match_versioned_bucket(self, s3_bucket, aws_client, snapshot):
2084+
aws_client.s3.put_bucket_versioning(
2085+
Bucket=s3_bucket, VersioningConfiguration={"Status": "Enabled"}
2086+
)
2087+
key = "test-precondition"
2088+
put_obj = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test")
2089+
snapshot.match("put-obj", put_obj)
2090+
put_obj_etag_1 = put_obj["ETag"]
2091+
2092+
with pytest.raises(ClientError) as e:
2093+
aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfMatch="abcdef")
2094+
snapshot.match("put-obj-if-none-match-bad-value", e.value.response)
2095+
2096+
del_obj = aws_client.s3.delete_object(Bucket=s3_bucket, Key=key)
2097+
snapshot.match("del-obj", del_obj)
2098+
2099+
# if the last object is a delete marker, then we can't use IfMatch
2100+
with pytest.raises(ClientError) as e:
2101+
aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfMatch=put_obj_etag_1)
2102+
snapshot.match("put-obj-after-del-exc", e.value.response)
2103+
2104+
put_obj_2 = aws_client.s3.put_object(Bucket=s3_bucket, Key=key, Body="test-after-del")
2105+
snapshot.match("put-obj-after-del", put_obj_2)
2106+
put_obj_etag_2 = put_obj_2["ETag"]
2107+
2108+
put_obj_3 = aws_client.s3.put_object(
2109+
Bucket=s3_bucket, Key=key, Body="test-if-match", IfMatch=put_obj_etag_2
2110+
)
2111+
snapshot.match("put-obj-if-match", put_obj_3)
2112+
2113+
list_object_versions = aws_client.s3.list_object_versions(Bucket=s3_bucket)
2114+
snapshot.match("list-object-versions", list_object_versions)
2115+
2116+
@markers.aws.validated
2117+
def test_put_object_if_match_and_if_none_match_validation(
2118+
self, s3_bucket, aws_client, snapshot
2119+
):
2120+
key = "test-precondition-validation"
2121+
2122+
with pytest.raises(ClientError) as e:
2123+
aws_client.s3.put_object(Bucket=s3_bucket, Key=key, IfNoneMatch="*", IfMatch="abcdef")
2124+
snapshot.match("put-obj-both-precondition", e.value.response)

0 commit comments

Comments
 (0)
0