10BC0 SQS: Improve update support for CloudFormation handlers. (#13477) · localstack/localstack@ca6b22b · GitHub
[go: up one dir, main page]
Skip to content

Commit ca6b22b

Browse files
peter-smith-phdpeter-smith-phd
authored
SQS: Improve update support for CloudFormation handlers. (#13477)
1 parent 2b704c1 commit ca6b22b

34 files changed

+1417
-414
lines changed

‎localstack-core/localstack/services/cloudformation/provider_utils.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,26 @@ def recursive_convert(obj):
275275
return recursive_convert(input_dict)
276276

277277

278+
def resource_tags_to_remove_or_update(
279+
prev_tags: list[dict], new_tags: list[dict]
280+
) -> tuple[list[str], dict[str, str]]:
281+
"""
282+
When updating resources that have tags, we need to determine which tags to remove and which to add/update,
283+
as these are typically done in separate API calls. The format of prev_tags and new_tags is expected to
284+
be [{ "Key": tagName, "Value": tagValue }, ...]. The return value will be a tuple of (tags_to_remove, tags_to_update),
285+
where:
286+
- tags_to_remove is a list of tag keys that are present in prev_tags but not in new_tags.
287+
- tags_to_update is a dict of tags to add or update, with the format: { tagName: tagValue, ... }.
288+
"""
289+
prev_tag_keys = [tag["Key"] for tag in prev_tags]
290+
new_tag_keys = [tag["Key"] for tag in new_tags]
291+
tags_to_remove = list(set(prev_tag_keys) - set(new_tag_keys))
292+
293+
# convert from list of dicts, to a single dict because that's what tag_queue APIs expect.
294+
tags_to_update = {tag["Key"]: tag["Value"] for tag in new_tags}
295+
return (tags_to_remove, tags_to_update)
296+
297+
278298
# LocalStack specific utilities
279299
def get_schema_path(file_path: Path) -> dict:
280300
file_name_base = file_path.name.removesuffix(".py").removesuffix(".py.enc")

‎localstack-core/localstack/services/sqs/constants.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@
3131
QueueAttributeName.QueueArn,
3232
]
3333

34+
#
35+
# If these attributes are set to their default values, they are effectively
36+
# deleted from the queue attributes and not returned in future calls to get_queue_attributes()
37+
#
38+
DELETE_IF_DEFAULT = {"KmsMasterKeyId": "", "KmsDataKeyReusePeriodSeconds": "300"}
39+
3440
INVALID_STANDARD_QUEUE_ATTRIBUTES = [
3541
QueueAttributeName.FifoQueue,
3642
QueueAttributeName.ContentBasedDeduplication,

‎localstack-core/localstack/services/sqs/provider.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1273,7 +1273,11 @@ def set_queue_attributes(
12731273
for k, v in attributes.items():
12741274
if k in sqs_constants.INTERNAL_QUEUE_ATTRIBUTES:
12751275
raise InvalidAttributeName(f"Unknown Attribute {k}.")
1276-
queue.attributes[k] = v
1276+
if k in sqs_constants.DELETE_IF_DEFAULT and v == sqs_constants.DELETE_IF_DEFAULT[k]:
1277+
if k in queue.attributes:
1278+
del queue.attributes[k]
1279+
else:
1280+
queue.attributes[k] = v
12771281

12781282
# Special cases
12791283
if queue.attributes.get(QueueAttributeName.Policy) == "":

‎localstack-core/localstack/services/sqs/resource_providers/aws_sqs_queue.py‎

Lines changed: 61 additions & 46 deletions
1AF0
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,35 @@ class SQSQueueProvider(ResourceProvider[SQSQueueProperties]):
6464
TYPE = "AWS::SQS::Queue" # Autogenerated. Don't change
6565
SCHEMA = util.get_schema_path(Path(__file__)) # Autogenerated. Don't change
6666

67+
# Values used when a property is removed from a template and needs to be set to its default.
68+
# If AWS changes their defaults in the future, our parity tests should break.
69+
DEFAULT_ATTRIBUTE_VALUES = {
70+
"ReceiveMessageWaitTimeSeconds": "0",
71+
"DelaySeconds": "0",
72+
"KmsMasterKeyId": "",
73+
"RedrivePolicy": "",
74+
"MessageRetentionPeriod": "345600",
75+
"MaximumMessageSize": "262144", # Note: CloudFormation sets this to 256KB on update, but 1MB on create
76+
"VisibilityTimeout": "30",
77+
"KmsDataKeyReusePeriodSeconds": "300",
78+
}
79+
80+
# Private method for creating a unique queue name, if none is specified.
81+
def _autogenerated_queue_name(self, request: ResourceRequest[SQSQueueProperties]) -> str:
82+
queue_name = util.generate_default_name(request.stack_name, request.logical_resource_id)
83+
isFifoQueue = request.desired_state.get("FifoQueue")
84+
85+
# Note that it's an SQS FIFO queue only if the FifoQueue property is set to boolean True, or the string "true"
86+
# (case insensitive). If it's None (property was omitted) or False, or any type of string (e.g. a typo
87+
# such as "Fasle"), then it's not a FIFO queue. This extra check is needed because the CloudFormation engine
88+
# doesn't fully validate the FifoQueue property before passing it to the resource provider.
89+
if (
90+
isFifoQueue == True # noqa: E712
91+
or (isinstance(isFifoQueue, str) and isFifoQueue.lower() == "true")
92+
):
93+
queue_name = f"{queue_name[:-5]}.fifo"
94+
return queue_name
95+
6796
def create(
6897
self,
6998
request: ResourceRequest[SQSQueueProperties],
@@ -74,8 +103,6 @@ def create(
74103
Primary identifier fields:
75104
- /properties/QueueUrl
76105
77-
78-
79106
Create-only properties:
80107
- /properties/FifoQueue
81108
- /properties/QueueName
@@ -92,26 +119,13 @@ def create(
92119
- sqs:TagQueue
93120
94121
"""
95-
# TODO: validations
122+
# TODO: validations - what validations are needed?
96123
model = request.desired_state
97124
sqs = request.aws_client_factory.sqs
98125

99-
if model.get("FifoQueue", False):
100-
model["FifoQueue"] = model["FifoQueue"]
101-
102-
queue_name = model.get("QueueName")
103-
if not queue_name:
104-
# TODO: verify patterns here
105-
if model.get("FifoQueue"):
106-
queue_name = util.generate_default_name(
107-
request.stack_name, request.logical_resource_id
108-
)[:-5]
109-
queue_name = f"{queue_name}.fifo"
110-
else:
111-
queue_name = util.generate_default_name(
112-
request.stack_name, request.logical_resource_id
113-
)
114-
model["QueueName"] = queue_name
126+
# if no QueueName is specified, automatically generate one
127+
if not model.get("QueueName"):
128+
model["QueueName"] = self._autogenerated_queue_name(request)
115129

116130
attributes = self._compile_sqs_queue_attributes(model)
117131
result = request.aws_client_factory.sqs.create_queue(
@@ -184,38 +198,30 @@ def update(
184198
"""
185199
sqs = request.aws_client_factory.sqs
186200
model = request.desired_state
201+
prev_model = request.previous_state
187202

188203
assert request.previous_state is not None
189204

190-
should_replace = (
191-
request.desired_state.get("QueueName", request.previous_state["QueueName"])
192-
!= request.previous_state["QueueName"]
193-
) or (
194-
request.desired_state.get("FifoQueue", request.previous_state.get("FifoQueue"))
195-
!= request.previous_state.get("FifoQueue")
205+
queue_url = prev_model["QueueUrl"]
206+
self._populate_missing_attributes_with_defaults(model)
207+
sqs.set_queue_attributes(
208+
QueueUrl=queue_url, Attributes=self._compile_sqs_queue_attributes(model)
196209
)
197210

198-
if not should_replace:
199-
return ProgressEvent(OperationStatus.SUCCESS, resource_model=request.previous_state)
200-
201-
# TODO: copied from the create handler, extract?
202-
if model.get("FifoQueue"):
203-
queue_name = util.generate_default_name(
204-
request.stack_name, request.logical_resource_id
205-
)[:-5]
206-
queue_name = f"{queue_name}.fifo"
207-
else:
208-
queue_name = util.generate_default_name(request.stack_name, request.logical_resource_id)
209-
210-
# replacement (TODO: find out if we should handle this in the provider or outside of it)
211-
# delete old queue
212-
sqs.delete_queue(QueueUrl=request.previous_state["QueueUrl"])
213-
# create new queue (TODO: re-use create logic to make this more robust, e.g. for
214-
# auto-generated queue names)
215-
model["QueueUrl"] = sqs.create_queue(QueueName=queue_name)["QueueUrl"]
216-
model["Arn"] = sqs.get_queue_attributes(
217-
QueueUrl=model["QueueUrl"], AttributeNames=["QueueArn"]
218-
)["Attributes"]["QueueArn"]
211+
(tags_to_remove, tags_to_add_or_update) = util.resource_tags_to_remove_or_update(
212+
prev_model.get("Tags", []), model.get("Tags", [])
213+
)
214+
sqs.untag_queue(QueueUrl=queue_url, TagKeys=tags_to_remove)
215+
sqs.tag_queue(QueueUrl=queue_url, Tags=tags_to_add_or_update)
216+
217+
model["QueueUrl"] = queue_url
218+
model["Arn"] = request.previous_state["Arn"]
219+
220+
# For QueueName and FifoQueue, always use the value from the previous model. These fields
221+
# are create-only, so they cannot be changed via an update (even though they might be omitted)
222+
model["QueueName"] = prev_model.get("QueueName")
223+
model["FifoQueue"] = prev_model.get("FifoQueue", False)
224+
< F0D0 /code>
219225
return ProgressEvent(OperationStatus.SUCCESS, resource_model=model)
220226

221227
def _compile_sqs_queue_attributes(self, properties: SQSQueueProperties) -> dict[str, str]:
@@ -250,6 +256,15 @@ def _compile_sqs_queue_attributes(self, properties: SQSQueueProperties) -> dict[
250256

251257
return result
252258

259+
def _populate_missing_attributes_with_defaults(self, properties: SQSQueueProperties) -> None:
260+
"""
261+
For any attribute that is missing from the desired state, populate it with the default value.
262+
This is the only way to remove an attribute from an existing SQS queue's configuration.
263+
:param properties: the properties passed from cloudformation
264+
"""
265+
for k, v in self.DEFAULT_ATTRIBUTE_VALUES.items():
266+
properties.setdefault(k, v)
267+
253268
def list(
254269
self,
255270
request: ResourceRequest[SQSQueueProperties],

‎tests/aws/services/cloudformation/api/test_stacks.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,7 @@ def test_update_stack_with_same_template_withoutchange_transformation(
340340
)
341341

342342
@markers.aws.validated
343+
@skip_if_legacy_engine()
343344
def test_update_stack_actual_update(self, deploy_cfn_template, aws_client):
344345
template = load_file(
345346
os.path.join(os.path.dirname(__file__), "../../../templates/sqs_queue_update.yml")

‎tests/aws/services/cloudformation/resources/test_sqs.py‎

Lines changed: 0 additions & 143 deletions
This file was deleted.

0 commit comments

Comments
 (0)
0