8000 CloudFormation v2 Engine: V1 Test Porting and Annotations and Batch of Parity Improvements by MEPalma · Pull Request #12660 · localstack/localstack · GitHub
[go: up one dir, main page]

Skip to content

CloudFormation v2 Engine: V1 Test Porting and Annotations and Batch of Parity Improvements #12660

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Jun 2, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -523,7 +523,6 @@ def _resolve_intrinsic_function_fn_get_att(self, arguments: ChangeSetEntity) ->
def _resolve_intrinsic_function_ref(self, arguments: ChangeSetEntity) -> ChangeType:
if arguments.change_type != ChangeType.UNCHANGED:
return arguments.change_type
# TODO: add support for nested functions, here we assume the argument is a logicalID.
if not isinstance(arguments, TerminalValue):
return arguments.change_type

Expand Down Expand Up @@ -1170,7 +1169,7 @@ def _retrieve_parameter_if_exists(self, parameter_name: str) -> Optional[NodePar
parameters_scope, parameter_name, before_parameters, after_parameters
)
node_parameter = self._visit_parameter(
parameters_scope,
parameter_scope,
parameter_name,
before_parameter=before_parameter,
after_parameter=after_parameter,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import localstack.aws.api.cloudformation as cfn_api
from localstack.services.cloudformation.engine.v2.change_set_model import (
NodeIntrinsicFunction,
NodeProperty,
NodeResource,
PropertiesKey,
)
Expand Down Expand Up @@ -45,26 +46,36 @@ def visit_node_intrinsic_function_fn_get_att(
# artificially limit the precision of our output to match AWS's?

arguments_delta = self.visit(node_intrinsic_function.arguments)
before_argument_list = arguments_delta.before
after_argument_list = arguments_delta.after
before_argument: Optional[list[str]] = arguments_delta.before
if isinstance(before_argument, str):
before_argument = before_argument.split(".")
after_argument: Optional[list[str]] = arguments_delta.after
if isinstance(after_argument, str):
after_argument = after_argument.split(".")

before = None
if before_argument_list:
before_logical_name_of_resource = before_argument_list[0]
before_attribute_name = before_argument_list[1]
if before_argument:
before_logical_name_of_resource = before_argument[0]
before_attribute_name = before_argument[1]
before_node_resource = self._get_node_resource_for(
resource_name=before_logical_name_of_resource, node_template=self._node_template
)
before_node_property = self._get_node_property_for(
before_node_property: Optional[NodeProperty] = self._get_node_property_for(
property_name=before_attribute_name, node_resource=before_node_resource
)
before_property_delta = self.visit(before_node_property)
before = before_property_delta.before
if before_node_property is not None:
before_property_delta = self.visit(before_node_property)
before = before_property_delta.before
else:
before = self._before_deployed_property_value_of(
resource_logical_id=before_logical_name_of_resource,
property_name=before_attribute_name,
)

after = None
if after_argument_list:
after_logical_name_of_resource = after_argument_list[0]
after_attribute_name = after_argument_list[1]
if after_argument:
after_logical_name_of_resource = after_argument[0]
after_attribute_name = after_argument[1]
after_node_resource = self._get_node_resource_for(
resource_name=after_logical_name_of_resource, node_template=self._node_template
)
Expand All @@ -74,12 +85,18 @@ def visit_node_intrinsic_function_fn_get_att(
)
if after_node_property is not None:
after_property_delta = self.visit(after_node_property)
if after_property_delta.before == after_property_delta.after:
after = after_property_delta.after
else:
after = CHANGESET_KNOWN_AFTER_APPLY
else:
after_property_delta = PreprocEntityDelta(after=CHANGESET_KNOWN_AFTER_APPLY)
if after_property_delta.before == after_property_delta.after:
after = after_property_delta.after
else:
after = CHANGESET_KNOWN_AFTER_APPLY
try:
after = self._after_deployed_property_value_of(
resource_logical_id=after_logical_name_of_resource,
property_name=after_attribute_name,
)
except RuntimeError:
after = CHANGESET_KNOWN_AFTER_APPLY

return PreprocEntityDelta(before=before, after=after)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,40 +103,44 @@ def visit_node_resource(
`after` delta with the physical resource ID, if side effects resulted in an update.
"""
delta = super().visit_node_resource(node_resource=node_resource)
self._execute_on_resource_change(
name=node_resource.name, before=delta.before, after=delta.after
)
after_resource = delta.after
if after_resource is not None and delta.before != delta.after:
after_logical_id = after_resource.logical_id
after_physical_id: Optional[str] = self._after_resource_physical_id(
before = delta.before
after = delta.after

if before != after:
# There are changes for this resource.
self._execute_resource_change(name=node_resource.name, before=before, after=after)
else:
# There are no updates for this resource; iff the resource was previously
# deployed, then the resolved details are copied in the current state for
# references or other downstream operations.
if before is not None:
before_logical_id = delta.before.logical_id
before_resource = self._before_resolved_resources.get(before_logical_id, dict())
self.resources[before_logical_id] = before_resource

# Update the latest version of this resource for downstream references.
if after is not None:
after_logical_id = after.logical_id
after_physical_id: str = self._after_resource_physical_id(
resource_logical_id=after_logical_id
)
if after_physical_id is None:
raise RuntimeError(
f"No PhysicalResourceId was found for resource '{after_physical_id}' post-update."
)
after_resource.physical_resource_id = after_physical_id
after.physical_resource_id = after_physical_id
return delta

def visit_node_output(
self, node_output: NodeOutput
) -> PreprocEntityDelta[PreprocOutput, PreprocOutput]:
delta = super().visit_node_output(node_output=node_output)
if delta.after is None:
# handling deletion so the output does not really matter
# TODO: are there other situations?
after = delta.after
if after is None or (isinstance(after, PreprocOutput) and after.condition is False):
return delta

self.outputs[delta.after.name] = delta.after.value
return delta

def _execute_on_resource_change(
def _execute_resource_change(
self, name: str, before: Optional[PreprocResource], after: Optional[PreprocResource]
) -> None:
if before == after:
# unchanged: nothing to do.
return
# Changes are to be made about this resource.
# TODO: this logic is a POC and should be revised.
if before is not None and after is not None:
# Case: change on same type.
Expand Down Expand Up @@ -257,11 +261,34 @@ def _execute_resource_action(
case OperationStatus.SUCCESS:
# merge the resources state with the external state
# TODO: this is likely a duplicate of updating from extra_resource_properties

# TODO: add typing
# TODO: avoid the use of string literals for sampling from the object, use typed classes instead
# TODO: avoid sampling from resources and use tmp var reference
# TODO: add utils functions to abstract this logic away (resource.update(..))
# TODO: avoid the use of setdefault (debuggability/readability)
# TODO: review the use of merge

self.resources[logical_resource_id]["Properties"].update(event.resource_model)
self.resources[logical_resource_id].update(extra_resource_properties)
# XXX for legacy delete_stack compatibility
self.resources[logical_resource_id]["LogicalResourceId"] = logical_resource_id
self.resources[logical_resource_id]["Type"] = resource_type

# TODO: review why the physical id is returned as None during updates
# TODO: abstract this in member function of resource classes instead
physical_resource_id = None
try:
physical_resource_id = self._after_resource_physical_id(logical_resource_id)
except RuntimeError:
# The physical id is missing or is set to None, which is invalid.
pass
if physical_resource_id is None:
# The physical resource id is None after an update that didn't rewrite the resource, the previous
# resource id is therefore the current physical id of this resource.
physical_resource_id = self._before_resource_physical_id(logical_resource_id)
self.resources[logical_resource_id]["PhysicalResourceId"] = physical_resource_id

case OperationStatus.FAILED:
reason = event.message
LOG.warning(
Expand Down
< B41A tr data-hunk="722aba30cd17396a4c73460ca97e4e074d0337d3f00b4ab430a6c1ca1f03e0c7" class="show-top-border">
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ def _get_node_resource_for(
# TODO: this could be improved with hashmap lookups if the Node contained bindings and not lists.
for node_resource in node_template.resources.resources:
if node_resource.name == resource_name:
self.visit(node_resource)
return node_resource
raise RuntimeError(f"No resource '{resource_name}' was found")

Expand All @@ -177,6 +178,7 @@ def _get_node_property_for(
# TODO: this could be improved with hashmap lookups if the Node contained bindings and not lists.
for node_property in node_resource.properties.properties:
if node_property.name == property_name:
self.visit(node_property)
return node_property
return None

Expand All @@ -189,10 +191,9 @@ def _deployed_property_value_of(
# process the resource if this wasn't processed already. Ideally, values should only
# be accessible through delta objects, to ensure computation is always complete at
# every level.
node_resource = self._get_node_resource_for(
_ = self._get_node_resource_for(
resource_name=resource_logical_id, node_template=self._node_template
)
self.visit(node_resource)

resolved_resource = resolved_resources.get(resource_logical_id)
if resolved_resource is None:
Expand Down Expand Up @@ -228,6 +229,7 @@ def _get_node_mapping(self, map_name: str) -> NodeMapping:
# TODO: another scenarios suggesting property lookups might be preferable.
for mapping in mappings:
if mapping.name == map_name:
self.visit(mapping)
return mapping
# TODO
raise RuntimeError()
Expand All @@ -237,6 +239,7 @@ def _get_node_parameter_if_exists(self, parameter_name: str) -> Optional[NodePar
# TODO: another scenarios suggesting property lookups might be preferable.
for parameter in parameters:
if parameter.name == parameter_name:
self.visit(parameter)
return parameter
return None

Expand All @@ -245,6 +248,7 @@ def _get_node_condition_if_exists(self, condition_name: str) -> Optional[NodeCon
# TODO: another scenarios suggesting property lookups might be preferable.
for condition in conditions:
if condition.name == condition_name:
self.visit(condition)
return condition
return None

Expand Down Expand Up @@ -372,15 +376,19 @@ def visit_node_object(self, node_object: NodeObject) -> PreprocEntityDelta:
def visit_node_intrinsic_function_fn_get_att(
self, node_intrinsic_function: NodeIntrinsicFunction
) -> PreprocEntityDelta:
arguments_delta = self.visit(node_intrinsic_function.arguments)
# TODO: validate the return value according to the spec.
before_argument_list = arguments_delta.before
after_argument_list = arguments_delta.after
arguments_delta = self.visit(node_intrinsic_function.arguments)
before_argument: Optional[list[str]] = arguments_delta.before
if isinstance(before_argument, str):
before_argument = before_argument.split(".")
after_argument: Optional[list[str]] = arguments_delta.after
if isinstance(after_argument, str):
after_argument = after_argument.split(".")

before = None
if before_argument_list:
before_logical_name_of_resource = before_argument_list[0]
before_attribute_name = before_argument_list[1]
if before_argument:
before_logical_name_of_resource = before_argument[0]
before_attribute_name = before_argument[1]

before_node_resource = self._get_node_resource_for(
resource_name=before_logical_name_of_resource, node_template=self._node_template
Expand All @@ -401,9 +409,9 @@ def visit_node_intrinsic_function_fn_get_att(
)

after = None
if after_argument_list:
after_logical_name_of_resource = after_argument_list[0]
after_attribute_name = after_argument_list[1]
if after_argument:
after_logical_name_of_resource = after_argument[0]
after_attribute_name = after_argument[1]
after_node_resource = self._get_node_resource_for(
resource_name=after_logical_name_of_resource, node_template=self._node_template
)
Expand Down Expand Up @@ -452,10 +460,14 @@ def _compute_delta_for_if_statement(args: list[Any]) -> PreprocEntityDelta:
)

# TODO: add support for this being created or removed.
before_outcome_delta = _compute_delta_for_if_statement(arguments_delta.before)
before = before_outcome_delta.before
after_outcome_delta = _compute_delta_for_if_statement(arguments_delta.after)
after = after_outcome_delta.after
before = None
if arguments_delta.before:
before_outcome_delta = _compute_delta_for_if_statement(arguments_delta.before)
before = before_outcome_delta.before
after = None
if arguments_delta.after:
after_outcome_delta = _compute_delta_for_if_statement(arguments_delta.after)
after = after_outcome_delta.after
return PreprocEntityDelta(before=before, after=after)

def visit_node_intrinsic_function_fn_not(
Expand Down Expand Up @@ -558,7 +570,7 @@ def _compute_join(args: list[Any]) -> str:
delimiter: str = str(args[0])
values: list[Any] = args[1]
if not isinstance(values, list):
raise RuntimeError("Invalid arguments list definition for Fn::Join")
raise RuntimeError(f"Invalid arguments list definition for Fn::Join: '{args}'")
join_result = delimiter.join(map(str, values))
return join_result

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,12 @@ def visit_children(self, change_set_entity: ChangeSetEntity):
self.visit(child)

def visit_node_template(self, node_template: NodeTemplate):
self.visit_children(node_template)
# Visit the resources, which will lazily evaluate all the referenced (direct and indirect)
# entities (parameters, mappings, conditions, etc.). Then compute the output fields; computing
# only the output fields would only result in the deployment logic of the referenced outputs
# being evaluated, hence enforce the visiting of all the resources first.
self.visit(node_template.resources)
self.visit(node_template.outputs)

def visit_node_outputs(self, node_outputs: NodeOutputs):
self.visit_children(node_outputs)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import pytest

from localstack.aws.api.cloudformation import StackEvent
from localstack.aws.api.cloudformation import DescribeChangeSetOutput, StackEvent
from localstack.aws.connect import ServiceLevelClientFactory
from localstack.utils.functions import call_safe
from localstack.utils.strings import short_uid
Expand All @@ -29,6 +29,12 @@ def capture(stack_name: str) -> PerResourceStackEvents:
return capture


def _normalise_describe_change_set_output(value: DescribeChangeSetOutput) -> None:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel the order of changes in the output gives us clues into the implementation on the AWS side. If we have a difference then perhaps our visiting order needs adjusting instead?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From what I’ve observed, AWS returns changes in the same order as they are declared in the template.

Our implementation currently mirrors this behavior (it is also true that the json library preserving key order after 3.7, but this is not the only reason for our compliance here), except in cases where the user doesn’t explicitly sort dependencies in the template. In those situations, our system places the dependee’s resource change first (if a change exists there), though it otherwise respects the original order of the template for all other changes.

I’ve decided to omit these specifics for now, but it’s unfortunate that the snapshot is being influenced by the normalization process.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we either add a skipped test that exercises this behaviour or at least a TODO comment in the describe_change_set provider method to come back to in the future? I agree it's not critical for parity at this stage, but it is still a discrepancy.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes absolutely, I'm adding this TODO in the PR for Fn::Split, after I merge this train of PRs today

value.get("Changes", list()).sort(
key=lambda change: change.get("ResourceChange", dict()).get("LogicalResourceId", str())
)


@pytest.fixture
def capture_update_process(aws_client_no_retry, cleanups, capture_per_resource_events):
"""
Expand Down Expand Up @@ -84,12 +90,15 @@ def inner(
ChangeSetName=change_set_id, IncludePropertyValues=True
)
)
_normalise_describe_change_set_output(describe_change_set_with_prop_values)
snapshot.match("describe-change-set-1-prop-values", describe_change_set_with_prop_values)

describe_change_set_without_prop_values = (
aws_client_no_retry.cloudformation.describe_change_set(
ChangeSetName=change_set_id, IncludePropertyValues=False
)
)
_normalise_describe_change_set_output(describe_change_set_without_prop_values)
snapshot.match("describe-change-set-1", describe_change_set_without_prop_values)

execute_results = aws_client_no_retry.cloudformation.execute_change_set(
Expand Down Expand Up @@ -132,12 +141,15 @@ def inner(
ChangeSetName=change_set_id, IncludePropertyValues=True
)
)
_normalise_describe_change_set_output(describe_change_set_with_prop_values)
snapshot.match("describe-change-set-2-prop-values", describe_change_set_with_prop_values)

describe_change_set_without_prop_values = (
aws_client_no_retry.cloudformation.describe_change_set(
ChangeSetName=change_set_id, IncludePropertyValues=False
)
)
_normalise_describe_change_set_output(describe_change_set_without_prop_values)
snapshot.match("describe-change-set-2", describe_change_set_without_prop_values)

execute_results = aws_client_no_retry.cloudformation.execute_change_set(
Expand Down
Empty file.
Loading
Loading
0