8000 improved support for object retention (#8647) · codeperl/localstack@3a6a330 · GitHub
[go: up one dir, main page]
Skip to content

Commit 3a6a330

Browse files
macnev2013macnev2013
authored
improved support for object retention (localstack#8647)
1 parent 0908315 commit 3a6a330

File tree

3 files changed

+217
-0
lines changed

3 files changed

+217
-0
lines changed

‎localstack/services/s3/provider.py

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@
6666
GetObjectAttributesRequest,
6767
GetObjectOutput,
6868
GetObjectRequest,
69+
GetObjectRetentionOutput,
6970
GetObjectTaggingOutput,
7071
GetObjectTaggingRequest,
7172
HeadObjectOutput,
@@ -102,6 +103,7 @@
102103
NotificationConfiguration,
103104
ObjectIdentifier,
104105
ObjectKey,
106+
ObjectLockRetention,
105107
ObjectLockToken,
106108
ObjectVersionId,
107109
OutputSerialization,
@@ -116,6 +118,7 @@
116118
PutObjectAclRequest,
117119
PutObjectOutput,
118120
PutObjectRequest,
121+
PutObjectRetentionOutput,
119122
PutObjectTaggingOutput,
120123
PutObjectTaggingRequest,
121124
ReplicationConfiguration,
@@ -176,6 +179,7 @@
176179
from localstack.utils.collections import get_safe
177180
from localstack.utils.patch import patch
178181
from localstack.utils.strings import short_uid
182+
from localstack.utils.time import parse_timestamp
179183
from localstack.utils.urls import localstack_host
180184

181185
LOG = logging.getLogger(__name__)
@@ -1104,6 +1108,62 @@ def get_bucket_acl(
11041108

11051109
return response
11061110

1111+
def get_object_retention(
1112+
self,
1113+
context: RequestContext,
1114+
bucket: BucketName,
1115+
key: ObjectKey,
1116+
version_id: ObjectVersionId = None,
1117+
request_payer: RequestPayer = None,
1118+
expected_bucket_owner: AccountId = None,
1119+
) -> GetObjectRetentionOutput:
1120+
moto_backend = get_moto_s3_backend(context)
1121+
key = get_key_from_moto_bucket(
1122+
get_bucket_from_moto(moto_backend, bucket=bucket), key=key, version_id=version_id
1123+
)
1124+
if not key.lock_mode and not key.lock_until:
1125+
raise InvalidRequest("Bucket is missing Object Lock Configuration")
1126+
return GetObjectRetentionOutput(
1127+
Retention=ObjectLockRetention(
1128+
Mode=key.lock_mode,
1129+
RetainUntilDate=parse_timestamp(key.lock_until),
1130+
)
1131+
)
1132+
1133+
@handler("PutObjectRetention")
1134+
def put_object_retention(
1135+
self,
1136+
context: RequestContext,
1137+
bucket: BucketName,
1138+
key: ObjectKey,
1139+
retention: ObjectLockRetention = None,
1140+
request_payer: RequestPayer = None,
1141+
version_id: ObjectVersionId = None,
1142+
bypass_governance_retention: BypassGovernanceRetention = None,
1143+
content_md5: ContentMD5 = None,
1144+
checksum_algorithm: ChecksumAlgorithm = None,
1145+
expected_bucket_owner: AccountId = None,
1146+
) -> PutObjectRetentionOutput:
1147+
moto_backend = get_moto_s3_backend(context)
1148+
moto_bucket = get_bucket_from_moto(moto_backend, bucket=bucket)
1149+
1150+
try:
1151+
moto_key = get_key_from_moto_bucket(moto_bucket, key=key, version_id=version_id)
1152+
except NoSuchKey:
1153+
moto_key = None
1154+
1155+
if not moto_key and version_id:
1156+
raise InvalidArgument("Invalid version id specified")
1157+
if not moto_bucket.object_lock_enabled:
1158+
raise InvalidRequest("Bucket is missing Object Lock Configuration")
1159+
if not moto_key and not version_id:
1160+
raise NoSuchKey("The specified key does not exist.", Key=key)
1161+
1162+
moto_key.lock_mode = retention.get("Mode")
1163+
retention_date = retention.get("RetainUntilDate")
1164+
retention_date = retention_date.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
1165+
moto_key.lock_until = retention_date
1166+
11071167
@handler("PutBucketAcl", expand=False)
11081168
def put_bucket_acl(
11091169
self,
@@ -2177,6 +2237,23 @@ def bucket_get_permission(fn, self, *args, **kwargs):
21772237

21782238
return fn(self, *args, **kwargs)
21792239

2240+
def key_is_locked(self):
2241+
"""
2242+
Apply a patch to check if a key is locked
2243+
"""
2244+
if self.lock_legal_status == "ON":
2245+
return True
2246+
2247+
if self.lock_mode in ["GOVERNANCE", "COMPLIANCE"]:
2248+
now = datetime.datetime.utcnow()
2249+
until = parse_timestamp(self.lock_until)
2250+
if until > now:
2251+
return True
2252+
2253+
return False
2254+
2255+
setattr(moto_s3_models.FakeKey, "is_locked", property(key_is_locked))
2256+
21802257

21812258
def register_custom_handlers():
21822259
serve_custom_service_request_handlers.append(s3_cors_request_handler)

‎tests/integration/s3/test_s3.py

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4864,6 +4864,107 @@ def test_s3_get_object_headers(self, aws_client, s3_create_bucket, snapshot):
48644864
aws_client.s3.get_object(Bucket=bucket, Key=key, IfMatch="etag")
48654865
snapshot.match("if_match_err_1", e.value.response["Error"])
48664866

4867+
@pytest.mark.aws_validated
4868+
def test_s3_object_hold(self, aws_client, s3_create_bucket, snapshot):
4869+
bucket_name_with_lock = f"bucket-with-lock-{short_uid()}"
4870+
bucket_name_without_lock = f"bucket-without-lock-{short_uid()}"
4871+
key_1 = "test1.txt"
4872+
key_2 = "test2.txt"
4873+
4874+
s3_create_bucket(Bucket=bucket_name_with_lock, ObjectLockEnabledForBucket=True)
4875+
aws_client.s3.put_object(Bucket=bucket_name_with_lock, Key=key_1, Body="test")
4876+
key_1_version_id = aws_client.s3.get_object(Bucket=bucket_name_with_lock, Key=key_1)[
4877+
"VersionId"
4878+
]
4879+
key_2_version_id = aws_client.s3.put_object(Bucket=bucket_name_with_lock, Key=key_2)[
4880+
"VersionId"
4881+
]
4882+
4883+
s3_create_bucket(Bucket=bucket_name_without_lock, ObjectLockEnabledForBucket=False)
4884+
aws_client.s3.put_object(Bucket=bucket_name_without_lock, Key=key_1, Body="test")
4885+
4886+
# non-existing bucket
4887+
with pytest.raises(ClientError) as exc:
4888+
aws_client.s3.put_object_retention(
4889+
Bucket="non-existing-bucket",
4890+
Key=key_1,
4891+
Retention={"Mode": "GOVERNANCE", "RetainUntilDate": datetime.datetime(2030, 1, 1)},
4892+
)
4893+
snapshot.match("put_object_retention_1", exc.value.response["Error"])
4894+
4895+
# non-existing key
4896+
with pytest.raises(ClientError) as exc:
4897+
aws_client.s3.put_object_retention(
4898+
Bucket=bucket_name_with_lock,
4899+
Key="non-existing-key",
4900+
Retention={"Mode": "GOVERNANCE", "RetainUntilDate": datetime.datetime(2030, 1, 1)},
4901+
)
4902+
snapshot.match("put_object_retention_2", exc.value.response["Error"])
4903+
4904+
# no lock on bucket
4905+
with pytest.raises(ClientError) as exc:
4906+
aws_client.s3.get_object_retention(Bucket=bucket_name_without_lock, Key=key_1)
4907+
snapshot.match("get_object_retention_1", exc.value.response["Error"])
4908+
4909+
response = aws_client.s3.put_object_retention(
4910+
Bucket=bucket_name_with_lock,
4911+
Key=key_1,
4912+
Retention={"Mode": "GOVERNANCE", "RetainUntilDate": datetime.datetime(2030, 1, 1)},
4913+
)
4914+
snapshot.match("put_object_retention_4", response["ResponseMetadata"]["HTTPStatusCode"])
4915+
4916+
response = aws_client.s3.get_object_retention(Bucket=bucket_name_with_lock, Key=key_1)
4917+
snapshot.match("get_object_retention_2", response)
4918+
4919+
# delete object with lock without bypass
4920+
with pytest.raises(ClientError) as exc:
4921+
aws_client.s3.delete_object(
4922+
Bucket=bucket_name_with_lock, Key=key_1, VersionId=key_1_version_id
4923+
)
4924+
snapshot.match("delete_object_1", exc.value.response["Error"])
4925+
4926+
# delete object with lock with bypass
4927+
response = aws_client.s3.delete_object(
4928+
Bucket=bucket_name_with_lock,
4929+
Key=key_1,
4930+
VersionId=key_1_version_id,
4931+
BypassGovernanceRetention=True,
4932+
)
4933+
snapshot.match("delete_object_2", response["ResponseMetadata"]["HTTPStatusCode"])
4934+
4935+
# add object retention to key_2 with 5 seconds retention
4936+
aws_client.s3.put_object_retention(
4937+
Bucket=bucket_name_with_lock,
4938+
Key=key_2,
4939+
Retention={
4940+
"Mode": "GOVERNANCE",
4941+
"RetainUntilDate": datetime.datetime.utcnow() + datetime.timedelta(seconds=5),
4942+
},
4943+
)
4944+
4945+
# delete object with lock without bypass before 5 seconds
4946+
with pytest.raises(ClientError):
4947+
aws_client.s3.delete_object(
4948+
Bucket=bucket_name_with_lock, Key=key_2, VersionId=key_2_version_id
4949+
)
4950+
4951+
# delete object with lock without bypass after 5 seconds
4952+
time.sleep(6)
4953+
aws_client.s3.delete_object(
4954+
Bucket=bucket_name_with_lock,
4955+
Key=key_2,
4956+
VersionId=key_2_version_id,
4957+
)
4958+
4959+
# put object retention on bucket without lock configured
4960+
with pytest.raises(ClientError) as exc:
4961+
aws_client.s3.put_object_retention(
4962+
Bucket=bucket_name_without_lock,
4963+
Key=key_1,
4964+
Retention={"Mode": "GOVERNANCE", "RetainUntilDate": datetime.datetime(2030, 1, 1)},
4965+
)
4966+
snapshot.match("put_object_retention_5", exc.value.response["Error"])
4967+
48674968
@pytest.mark.aws_validated
48684969
def test_put_bucket_logging(self, aws_client, s3_create_bucket, snapshot):
48694970
snapshot.add_transformer(

‎tests/integration/s3/test_s3.snapshot.json

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8595,5 +8595,44 @@
85958595
}
85968596
}
85978597
}
8598+
},
8599+
"tests/integration/s3/test_s3.py::TestS3::test_s3_object_hold": {
8600+
"recorded-date": "08-07-2023, 00:52:18",
8601+
"recorded-content": {
8602+
"put_object_retention_1": {
8603+
"BucketName": "non-existing-bucket",
8604+
"Code": "NoSuchBucket",
8605+
"Message": "The specified bucket does not exist"
8606+
},
8607+
"put_object_retention_2": {
8608+
"Code": "NoSuchKey",
8609+
"Key": "non-existing-key",
8610+
"Message": "The specified key does not exist."
8611+
},
8612+
"get_object_retention_1": {
8613+
"Code": "InvalidRequest",
8614+
"Message": "Bucket is missing Object Lock Configuration"
8615+
},
8616+
"put_object_retention_4": 200,
8617+
"get_object_retention_2": {
8618+
"Retention": {
8619+
"Mode": "GOVERNANCE",
8620+
"RetainUntilDate": "datetime"
8621+
},
8622+
"ResponseMetadata": {
8623+
"HTTPHeaders": {},
8624+
"HTTPStatusCode": 200
8625+
}
8626+
},
8627+
"delete_object_1": {
8628+
"Code": "AccessDenied",
8629+
"Message": "Access Denied"
8630+
},
8631+
"delete_object_2": 204,
8632+
"put_object_retention_5": {
8633+
"Code": "InvalidRequest",
8634+
"Message": "Bucket is missing Object Lock Configuration"
8635+
}
8636+
}
85988637
}
85998638
}

0 commit comments

Comments
 (0)
0