8000 Improve CFn provider framework / scaffolding (#8594) · codeperl/localstack@384bdb0 · GitHub
[go: up one dir, main page]
Skip to content

Commit 384bdb0

Browse files
dominikschubertdominikschubert
authored
Improve CFn provider framework / scaffolding (localstack#8594)
1 parent f0d049f commit 384bdb0

40 files changed

+965
-464
lines changed

‎localstack/services/cloudformation/deployment_utils.py

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ def replace(params, logical_resource_id, *args, **kwargs):
3333
return replace
3434

3535

36+
# TODO: remove
3637
def param_defaults(param_func, defaults):
3738
def replace(properties: dict, logical_resource_id: str, *args, **kwargs):
3839
result = param_func(properties, logical_resource_id, *args, **kwargs)
@@ -102,6 +103,7 @@ def do_replace(params, logical_resource_id, *args, **kwargs):
102103
return do_replace
103104

104105

106+
# TODO: remove
105107
def params_select_attributes(*attrs):
106108
def do_select(params, logical_resource_id, *args, **kwargs):
107109
result = {}
@@ -151,18 +153,6 @@ def generate_default_name_without_stack(logical_resource_id: str):
151153
return f"{resource_id_part}-{random_id_part}"
152154

153155

154-
def pre_create_default_name(key: str) -> Callable[[str, dict, str, dict, str], None]:
155-
def _pre_create_default_name(
156-
resource_id: str, resources: dict, resource_type: str, func: dict, stack_name: str
157-
):
158-
resource = resources[resource_id]
159-
props = resource["Properties"]
160-
if not props.get(key):
161-
props[key] = generate_default_name(stack_name, resource_id)
162-
163-
return _pre_create_default_name
164-
165-
166156
# Utils for parameter conversion
167157

168158
# TODO: handling of multiple valid types
@@ -201,7 +191,6 @@ def fix_boto_parameters_based_on_report(original_params: dict, report: str) -> d
201191
cast_class = getattr(builtins, valid_class)
202192
old_value = get_nested(params, param_name)
203193

204-
new_value = None
205194
if cast_class == bool and str(old_value).lower() in ["true", "false"]:
206195
new_value = str(old_value).lower() == "true"
207196
else:

‎localstack/services/cloudformation/engine/quirks.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
"""
2+
We can't always automatically determine which value serves as the physical resource ID.
3+
=> This needs to be determined manually by testing against AWS (!)
4+
5+
There's also a reason that the mapping is located here instead of closer to the resource providers themselves.
6+
If the resources were compliant with the generic AWS resource provider framework that AWS provides for your own resource types, we wouldn't need this.
7+
For legacy resources (and even some of the ones where they are open-sourced), AWS still has a layer of "secret sauce" that defines what the actual physical resource ID is.
8+
An extension schema only defines the primary identifiers but not directly the physical resource ID that is generated based on those.
9+
Since this is therefore rather part of the cloudformation layer and *not* the resource providers responsibility, we've put the mapping closer to the cloudformation engine.
10+
"""
11+
12+
# note: format here is subject to change (e.g. it might not be a pure str -> str mapping, it could also involve more sophisticated handlers
13+
PHYSICAL_RESOURCE_ID_SPECIAL_CASES = {
14+
# Example
15+
# "AWS::ApiGateway::Resource": "/properties/ResourceId",
16+
}
17+
18+
# You can usually find the available GetAtt targets in the official resource documentation:
19+
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-template-resource-type-ref.html
20+
# Use the scaffolded exploration test to verify against AWS which attributes you can access.
21+
# This mapping is not in use yet (!)
22+
VALID_GETATT_PROPERTIES = {
23+
# Other Examples
24+
# "AWS::ApiGateway::Resource": ["ResourceId"],
25+
"AWS::IAM::User": ["Arn"], # TODO: not validated yet
26+
"AWS::SSM::Parameter": ["Type", "Value"], # TODO: not validated yet
27+
"AWS::OpenSearchService::Domain": [
28+
"AdvancedSecurityOptions.AnonymousAuthDisableDate",
29+
"Arn",
30+
"DomainArn",
31+
"DomainEndpoint",
32+
"DomainEndpoints",
33+
"Id",
34+
"ServiceSoftwareOptions",
35+
"ServiceSoftwareOptions.AutomatedUpdateDate",
36+
"ServiceSoftwareOptions.Cancellable",
37+
"ServiceSoftwareOptions.CurrentVersion",
38+
"ServiceSoftwareOptions.Description",
39+
"ServiceSoftwareOptions.NewVersion",
40+
"ServiceSoftwareOptions.OptionalDeployment",
41+
"ServiceSoftwareOptions.UpdateAvailable",
42+
"ServiceSoftwareOptions.UpdateStatus",
43+
], # TODO: not validated yet
44+
}

‎localstack/services/cloudformation/engine/schema.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import json
2+
import os
3+
import zipfile
4+
5+
6+
# TODO: unify with scaffolding
7+
class SchemaProvider:
8+
def __init__(self, zipfile_path: str | os.PathLike[str]):
9+
self.schemas = {}
10+
with zipfile.ZipFile(zipfile_path) as infile:
11+
for filename in infile.namelist():
12+
with infile.open(filename) as schema_file:
13+
schema = json.load(schema_file)
14+
typename = schema["typeName"]
15+
self.schemas[typename] = schema

‎localstack/services/cloudformation/provider_utils.py

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
"""
2+
A set of utils for use in resource providers.
3+
4+
Avoid any imports to localstack here and keep external imports to a minimum!
5+
This is because we want to be able to package a resource provider without including localstack code.
6+
"""
7+
import builtins
8+
import json
9+
import re
10+
import uuid
11+
from copy import deepcopy
12+
from pathlib import Path
13+
14+
15+
def generate_default_name(stack_name: str, logical_resource_id: str):
16+
random_id_part = str(uuid.uuid4())[0:8]
17+
resource_id_part = logical_resource_id[:24]
18+
stack_name_part = stack_name[: 63 - 2 - (len(random_id_part) + len(resource_id_part))]
19+
return f"{stack_name_part}-{resource_id_part}-{random_id_part}"
20+
21+
22+
def generate_default_name_without_stack(logical_resource_id: str):
23+
random_id_part = str(uuid.uuid4())[0:8]
24+
resource_id_part = logical_resource_id[: 63 - 1 - len(random_id_part)]
25+
return f"{resource_id_part}-{random_id_part}"
26+
27+
28+
# ========= Helpers for boto calls ==========
29+
# (equivalent to the old ones in deployment_utils.py)
30+
31+
32+
def deselect_attributes(model: dict, params: list[str]) -> dict:
33+
return {k: v for k, v in model.items() if k not in params}
34+
35+
36+
def select_attributes(model: dict, params: list[str]) -> dict:
37+
return {k: v for k, v in model.items() if k in params}
38+
39+
40+
def keys_lower(model: dict) -> dict:
41+
return {k.lower(): v for k, v in model.items()}
42+
43+
44+
# FIXME: this shouldn't be necessary in the future
45+
param_validation = re.compile(
46+
r"Invalid type for parameter (?P<param>\w+), value: (?P<value>\w+), type: <class '(?P<wrong_class>\w+)'>, valid types: <class '(?P<valid_class>\w+)'>"
47+
)
48+
49+
50+
def get_nested(obj: dict, path: str):
51+
parts = path.split(".")
52+
result = obj
53+
for p in parts[:-1]:
54+
result = result.get(p, {})
55+
return result.get(parts[-1])
56+
57+
58+
def set_nested(obj: dict, path: str, value):
59+
parts = path.split(".")
60+
result = obj
61+
for p in parts[:-1]:
62+
result = result.get(p, {})
63+
result[parts[-1]] = value
64+
65+
66+
def fix_boto_parameters_based_on_report(original_params: dict, report: str) -> dict:
67+
"""
68+
Fix invalid type parameter validation errors in boto request parameters
69+
70+
:param original_params: original boto request parameters that lead to the parameter validation error
71+
:param report: error report from botocore ParamValidator
72+
:return: a copy of original_params with all values replaced by their correctly cast ones
73+
"""
74+
params = deepcopy(original_params)
75+
for found in param_validation.findall(report):
76+
param_name, value, wrong_class, valid_class = found
77+
cast_class = getattr(builtins, valid_class)
78+
old_value = get_nested(params, param_name)
79+
80+
if cast_class == bool and str(old_value).lower() in ["true", "false"]:
81+
new_value = str(old_value).lower() == "true"
82+
else:
83+
new_value = cast_class(old_value)
84+
set_nested(params, param_name, new_value)
85+
return params
86+
87+
88+
# LocalStack specific utilities
89+
def get_schema_path(file_path: Path) -> Path:
90+
file_name_base = file_path.name.removesuffix(".py")
91+
with Path(file_path).parent.joinpath(f"{file_name_base}.schema.json").open() as fd:
92+
return json.load(fd)

‎localstack/services/cloudformation/resource_provider.py

Lines changed: 68 additions & 34 deletions
614
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
fix_boto_parameters_based_on_report,
2424
remove_none_values,
2525
)
26+
from localstack.services.cloudformation.engine.quirks import PHYSICAL_RESOURCE_ID_SPECIAL_CASES
2627
from localstack.services.cloudformation.service_models import KEY_RESOURCE_STATE, GenericBaseModel
2728
from localstack.utils.aws import aws_stack
2829

@@ -537,6 +538,21 @@ class NoResourceProvider(Exception):
537538
pass
538539

539540

541+
def resolve_json_pointer(resource_props: Properties, primary_id_path: str) -> str:
542+
primary_id_path = primary_id_path.replace("/properties", "")
543+
parts = [p for p in primary_id_path.split("/") if p]
544+
545+
resolved_part = resource_props.copy()
546+
for i in range(len(parts)):
547+
part = parts[i]
548+
resolved_part = resolved_part.get(part)
549+
if i == len(parts) - 1:
550+
# last part
551+
return resolved_part
552+
553+
raise Exception(f"Resource properties is missing field: {part}")
554+
555+
540556
class ResourceProviderExecutor:
541557
"""
542558
Point of abstraction between our integration with generic base models, and the new providers.
@@ -561,19 +577,32 @@ def deploy_loop(
561577
) -> ProgressEvent[Properties]:
562578
payload = copy.deepcopy(raw_payload)
563579

564-
for _ in range(max_iterations):
565-
event = self.execute_action(payload)
580+
for current_iteration in range(max_iterations):
581+
resource_type = get_resource_type(
582+
{"Type": raw_payload["resourceType"]}
583+
) # TODO: simplify signature of get_resource_type to just take the type
584+
resource_provider = self.load_resource_provider(resource_type)
585+
event = self.execute_action(resource_provider, payload)
566586

567587
if event.status == OperationStatus.SUCCESS:
568-
# TODO: validate physical_resource_id is not None
569588
logical_resource_id = raw_payload["requestData"]["logicalResourceId"]
570589
resource = self.resources[logical_resource_id]
571590
if "PhysicalResourceId" not in resource:
572591
# branch for non-legacy providers
573592
# TODO: move out of if? (physical res id can be set earlier possibly)
574-
resource_type_schema = self.load_resource_schema(raw_payload["resourceType"])
593+
if isinstance(resource_provider, LegacyResourceProvider):
594+
raise Exception(
595+
"A GenericBaseModel should always have a PhysicalResourceId set after deployment"
596+
)
597+
598+
if not hasattr(resource_provider, "SCHEMA"):
599+
raise Exception(
600+
"A ResourceProvider should always have a SCHEMA property defined."
601+
)
602+
603+
resource_type_schema = resource_provider.SCHEMA
575604
physical_resource_id = self.extract_physical_resource_id_from_model_with_schema(
576-
event.resource_model, resource_type_schema
605+
event.resource_model, raw_payload["resourceType"], resource_type_schema
577606
)
578607

579608
resource["PhysicalResourceId"] = physical_resource_id
@@ -585,34 +614,30 @@ def deploy_loop(
585
payload["callbackContext"] = context
586615
payload["requestData"]["resourceProperties"] = event.resource_model
587616

588-
time.sleep(sleep_time)
617+
if current_iteration == 0:
618+
time.sleep(0)
619+
else:
620+
time.sleep(sleep_time)
589621
else:
590622
raise TimeoutError("Could not perform deploy loop action")
591623

592-
def execute_action(self, raw_payload: ResourceProviderPayload) -> ProgressEvent[Properties]:
593-
resource_type = get_resource_type(
594-
{"Type": raw_payload["resourceType"]}
595-
) # TODO: simplify signature of get_resource_type to just take the type
596-
resource_provider = self.load_resource_provider(resource_type)
597-
if resource_provider:
598-
change_type = raw_payload["action"]
599-
request = convert_payload(
600-
stack_name=self.stack_name, stack_id=self.stack_id, payload=raw_payload
601-
)
602-
603-
match change_type:
604-
case "Add":
605-
return resource_provider.create(request)
606-
case "Dynamic" | "Modify":
607-
return resource_provider.update(request)
608-
case "Remove":
609-
return resource_provider.delete(request)
610-
case _:
611-
raise NotImplementedError(change_type)
624+
def execute_action(
625+
self, resource_provider: ResourceProvider, raw_payload: ResourceProviderPayload
626+
) -> ProgressEvent[Properties]:
627+
change_type = raw_payload["action"]
628+
request = convert_payload(
629+
stack_name=self.stack_name, stack_id=self.stack_id, payload=raw_payload
630+
)
612631

613-
else:
614-
# custom provider
615-
raise NoResourceProvider
632+
match change_type:
633+
case "Add":
634+
return resource_provider.create(request)
635+
case "Dynamic" | "Modify":
636+
return resource_provider.update(request)
637+
case "Remove":
638+
return resource_provider.delete(request)
639+
case _:
640+
raise NotImplementedError(change_type)
616641

617642
def load_resource_provider(self, resource_type: str) -> Optional[ResourceProvider]:
618643
# TODO: unify behavior here in regards to raising NoResourceProvider
@@ -643,13 +668,22 @@ def _load_legacy_resource_provider(self, resource_type: str) -> LegacyResourcePr
643668
raise NoResourceProvider
644669

645670
def extract_physical_resource_id_from_model_with_schema(
646-
self, resource_model: Properties, resource_type_schema: dict
671+
self, resource_model: Properties, resource_type: str, resource_type_schema: dict
647672
) -> str:
648-
# id_path = resource_type_schema['primaryIdentifier'][0]
649-
return resource_model["Id"]
673+
if resource_type in PHYSICAL_RESOURCE_ID_SPECIAL_CASES:
674+
primary_id_path = PHYSICAL_RESOURCE_ID_SPECIAL_CASES[resource_type]
675+
physical_resource_id = resolve_json_pointer(resource_model, primary_id_path)
676+
else:
677+
primary_id_paths = resource_type_schema["primaryIdentifier"]
678+
if len(primary_id_paths) > 1:
679+
# TODO: auto-merge. Verify logic here with AWS
680+
physical_resource_id = "-".join(
681+
[resolve_json_pointer(resource_model, pip) for pip in primary_id_paths]
682+
)
683+
else:
684+
physical_resource_id = resolve_json_pointer(resource_model, primary_id_paths[0])
650685

651-
def load_resource_schema(self, resource_type: str) -> dict:
652-
return {}
686+
return physical_resource_id
653687

654688

655689
plugin_manager = PluginManager(CloudFormationResourceProviderPlugin.namespace)

0 commit comments

Comments
 (0)
0