8000 Merge branch 'master' into MEP-sfn-sync-and-sfn · localstack/localstack@e4985ee · GitHub
[go: up one dir, main page]
Skip to content

Commit e4985ee

Browse files
MEPalmaMEPalma
committed
Merge branch 'master' into MEP-sfn-sync-and-sfn
2 parents 562c236 + 751c1ec commit e4985ee

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

82 files changed

+923
-787
lines changed

‎localstack/aws/connect.py

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
INTERNAL_AWS_ACCESS_KEY_ID,
2222
INTERNAL_AWS_SECRET_ACCESS_KEY,
2323
MAX_POOL_CONNECTIONS,
24-
TEST_AWS_ACCESS_KEY_ID,
2524
TEST_AWS_SECRET_ACCESS_KEY,
2625
)
2726
from localstack.utils.aws.aws_stack import get_local_service_url, get_s3_hostname
@@ -447,9 +446,9 @@ def get_client(
447446
:param region_name: Name of the AWS region to be associated with the client
448447
If set to None, loads from botocore session.
449448
:param aws_access_key_id: Access key to use for the client.
450-
Defaults to dummy value ("test")
449+
If set to None, loads from botocore session.
451450
:param aws_secret_access_key: Secret key to use for the client.
452-
Defaults to "dummy value ("test")
451+
If set to None, uses a placeholder value
453452
:param aws_session_token: Session token to use for the client.
454453
Not being used if not set.
455454
:param endpoint_url: Full endpoint URL to be used by the client.
@@ -466,14 +465,19 @@ def get_client(
466465
if re.match(r"https?://localhost(:[0-9]+)?", endpoint_url):
467466
endpoint_url = endpoint_url.replace("://localhost", f"://{get_s3_hostname()}")
468467

468+
# Prevent `PartialCredentialsError` when only access key ID is provided
469+
# The value of secret access key is insignificant and can be set to anything
470+
if aws_access_key_id:
471+
aws_secret_access_key = aws_secret_access_key or TEST_AWS_SECRET_ACCESS_KEY
472+
469473
return self._get_client(
470474
service_name=service_name,
471475
region_name=region_name or config.region_name or self._get_region(),
472476
use_ssl=self._use_ssl,
473477
verify=self._verify,
474478
endpoint_url=endpoint_url,
475-
aws_access_key_id=aws_access_key_id or TEST_AWS_ACCESS_KEY_ID,
476-
aws_secret_access_key=aws_secret_access_key or TEST_AWS_SECRET_ACCESS_KEY,
479+
aws_access_key_id=aws_access_key_id,
480+
aws_secret_access_key=aws_secret_access_key,
477481
aws_session_token=aws_session_token,
478482
config=config,
479483
)

‎localstack/aws/forwarder.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@
1818
ServiceResponse,
1919
)
2020
from localstack.aws.client import parse_response, raise_service_exception
21+
from localstack.aws.connect import connect_to
2122
from localstack.aws.skeleton import DispatchTable, create_dispatch_table
2223
from localstack.aws.spec import load_service
2324
from localstack.http import Response
2425
from localstack.http.proxy import forward
25-
from localstack.utils.aws import aws_stack
2626
from localstack.utils.strings import to_str
2727

2828

@@ -160,7 +160,7 @@ def create_aws_request_context(
160160
# we re-use botocore internals here to serialize the HTTP request,
161161
# but deactivate validation (validation errors should be handled by the backend)
162162
# and don't send it yet
163-
client = aws_stack.connect_to_service(
163+
client = connect_to.get_client(
164164
service_name,
165165
endpoint_url=endpoint_url,
166166
region_name=region,

‎localstack/constants.py

Lines changed: 16 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -41,17 +41,6 @@
4141
# host to bind to when starting the services
4242
BIND_HOST = "0.0.0.0"
4343

44-
# Fallback Account ID if not available in the client request
45-
DEFAULT_AWS_ACCOUNT_ID = "000000000000"
46-
47-
# AWS user account ID used for tests - TODO move to config.py
48-
if "TEST_AWS_ACCOUNT_ID" not in os.environ:
49-
os.environ["TEST_AWS_ACCOUNT_ID"] = DEFAULT_AWS_ACCOUNT_ID
50-
51-
# Values used by tests
52-
TEST_AWS_ACCOUNT_ID = os.environ["TEST_AWS_ACCOUNT_ID"]
53-
TEST_AWS_REGION_NAME = "us-east-1"
54-
5544
# root code folder
5645
MODULE_MAIN_PATH = os.path.dirname(os.path.realpath(__file__))
5746
# TODO rename to "ROOT_FOLDER"!
@@ -158,21 +147,24 @@
158147
except Exception:
159148
MAX_POOL_CONNECTIONS = 150
160149

161-
# credentials used in the test suite
162-
TEST_AWS_ACCESS_KEY_ID = "test"
163-
TEST_AWS_SECRET_ACCESS_KEY = (
164-
"test" # NOTE: In the near future, this will be set to a structured Access Key ID
165-
)
166-
167-
# additional credentials used in the test suite (mainly for cross-account access)
168-
SECONDARY_TEST_AWS_ACCOUNT_ID = "000000000002"
169-
SECONDARY_TEST_AWS_ACCESS_KEY_ID = (
170-
"000000000002" # NOTE: In the near future, this will be set to a structured Access Key ID
171-
)
172-
SECONDARY_TEST_AWS_SECRET_ACCESS_KEY = "test2"
150+
# Fallback Account ID if not available in the client request
151+
DEFAULT_AWS_ACCOUNT_ID = "000000000000"
152+
153+
# Credentials used in the test suite
154+
# These can be overridden if the tests are being run against AWS
155+
# If a structured access key ID is used, it must correspond to the account ID
156+
TEST_AWS_ACCOUNT_ID = os.getenv("TEST_AWS_ACCOUNT_ID") or DEFAULT_AWS_ACCOUNT_ID
157+
TEST_AWS_ACCESS_KEY_ID = os.getenv("TEST_AWS_ACCESS_KEY_ID") or "test"
158+
TEST_AWS_SECRET_ACCESS_KEY = os.getenv("TEST_AWS_SECRET_ACCESS_KEY") or "test"
159+
TEST_AWS_REGION_NAME = "us-east-1"
160+
161+
# Additional credentials used in the test suite (when running cross-account tests)
162+
SECONDARY_TEST_AWS_ACCOUNT_ID = os.getenv("SECONDARY_TEST_AWS_ACCOUNT_ID") or "000000000002"
163+
SECONDARY_TEST_AWS_ACCESS_KEY_ID = os.getenv("SECONDARY_TEST_AWS_ACCESS_KEY_ID") or "000000000002"
164+
SECONDARY_TEST_AWS_SECRET_ACCESS_KEY = os.getenv("SECONDARY_TEST_AWS_SECRET_ACCESS_KEY") or "test2"
173165
SECONDARY_TEST_AWS_REGION_NAME = "ap-southeast-1"
174166

175-
# credentials being used for internal calls
167+
# Credentials used for internal calls
176168
INTERNAL_AWS_ACCESS_KEY_ID = "__internal_call__"
177169
INTERNAL_AWS_SECRET_ACCESS_KEY = "__internal_call__"
178170

‎localstack/services/apigateway/helpers.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
Model,
2929
RequestValidator,
3030
)
31+
from localstack.aws.connect import connect_to
3132
from localstack.constants import (
3233
APPLICATION_JSON,
3334
HEADER_LOCALSTACK_EDGE_URL,
@@ -528,7 +529,7 @@ def get_stage_variables(context: ApiInvocationContext) -> Optional[Dict[str, str
528529
return {}
529530

530531
_, region_name = get_api_account_id_and_region(context.api_id)
531-
api_gateway_client = aws_stack.connect_to_service("apigateway", region_name=region_name)
532+
api_gateway_client = connect_to(region_name=region_name).apigateway
532533
try:
533534
response = api_gateway_client.get_stage(restApiId=context.api_id, stageName=context.stage)
534535
return response.get("variables")
@@ -617,7 +618,7 @@ def get_cors_response(headers):
617618

618619

619620
def get_rest_api_paths(rest_api_id, region_name=None):
620-
apigateway = aws_stack.connect_to_service(service_name="apigateway", region_name=region_name)
621+
apigateway = connect_to(region_name=region_name).apigateway
621622
resources = apigateway.get_resources(restApiId=rest_api_id, limit=100)
622623
resource_map = {}
623624
for resource in resources["items"]:
@@ -1467,7 +1468,7 @@ def _swagger_export(self, api_id: str, stage: str, export_format: str) -> str:
14671468
"""
14681469
https://github.com/OAI/OpenAPI-Specification/blob/main/versions/2.0.md
14691470
"""
1470-
apigateway_client = aws_stack.connect_to_service("apigateway")
1471+
apigateway_client = connect_to().apigateway
14711472

14721473
rest_api = apigateway_client.get_rest_api(restApiId=api_id)
14731474
resources = apigateway_client.get_resources(restApiId=api_id)
@@ -1488,7 +1489,7 @@ def _oas30_export(self, api_id: str, stage: str, export_format: str) -> str:
14881489
"""
14891490
https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.1.0.md
14901491
"""
1491-
apigateway_client = aws_stack.connect_to_service("apigateway")
1492+
apigateway_client = connect_to().apigateway
14921493

14931494
rest_api = apigateway_client.get_rest_api(restApiId=api_id)
14941495
resources = apigateway_client.get_resources(restApiId=api_id)

‎localstack/services/apigateway/integration.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -573,7 +573,7 @@ def invoke(self, invocation_context: ApiInvocationContext):
573573
relative_path, query_string_params = extract_query_string_params(path=invocation_path)
574574
uri = integration.get("uri") or integration.get("integrationUri") or ""
575575

576-
s3 = aws_stack.connect_to_service("s3")
576+
s3 = connect_to().s3
577577
uri = apply_request_parameters(
578578
uri,
579579
integration=integration,
@@ -619,7 +619,7 @@ def invoke(self, invocation_context: ApiInvocationContext):
619619

620620
if ":servicediscovery:" in uri:
621621
# check if this is a servicediscovery integration URI
622-
client = aws_stack.connect_to_service("servicediscovery")
622+
client = connect_to().servicediscovery
623623
service_id = uri.split("/")[-1]
624624
instances = client.list_instances(ServiceId=service_id)["Instances"]
625625
instance = (instances or [None])[0]
@@ -752,7 +752,7 @@ def invoke(self, invocation_context: ApiInvocationContext):
752752
else:
753753
payload = json.loads(invocation_context.data)
754754

755-
client = aws_stack.connect_to_service("stepfunctions")
755+
client = connect_to().stepfunctions
756756
if isinstance(payload.get("input"), dict):
757757
payload["input"] = json.dumps(payload["input"])
758758

‎localstack/services/apigateway/invocations.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ def run_authorizer(invocation_context: ApiInvocationContext, authorizer: Dict):
149149

150150
def authorize_invocation(invocation_context: ApiInvocationContext):
151151
region_name = invocation_context.region_name or aws_stack.get_region()
152-
client = aws_stack.connect_to_service("apigateway", region_name=region_name)
152+
client = connect_to(region_name=region_name).apigateway
153153
authorizers = client.get_authorizers(restApiId=invocation_context.api_id, limit=100).get(
154154
"items", []
155155
)

‎localstack/services/awslambda/lambda_api.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
from localstack import config, constants
2727
from localstack.aws.accounts import get_aws_account_id
28+
from localstack.aws.connect import connect_to
2829
from localstack.constants import APPLICATION_JSON
2930
from localstack.http import Request
3031
from localstack.http import Response as HttpResponse
@@ -432,7 +433,7 @@ def run_lambda(
432433
# (e.g., persistence) have been executed when the run_lambda(..) function gets called (e.g., from API GW).
433434
LOG.debug("Running lambda %s", func_arn)
434435
if not hasattr(run_lambda, "_provider_initialized"):
435-
aws_stack.connect_to_service("lambda").list_functions()
436+
connect_to().awslambda.list_functions()
436437
run_lambda._provider_initialized = True
437438

438439
store = get_awslambda_store_for_arn(func_arn)
@@ -909,7 +910,7 @@ def forward_to_fallback_url(func_arn, data):
909910
lambda_name = arns.lambda_function_name(func_arn)
910911
if config.LAMBDA_FALLBACK_URL.startswith("dynamodb://"):
911912
table_name = urlparse(config.LAMBDA_FALLBACK_URL.replace("dynamodb://", "http://")).netloc
912-
dynamodb = aws_stack.connect_to_service("dynamodb")
913+
dynamodb = connect_to().dynamodb
913914
item = {
914915
"id": {"S": short_uid()},
915916
"timestamp": {"N": str(now_utc())},
@@ -938,7 +939,7 @@ def forward_to_fallback_url(func_arn, data):
938939

939940

940941
def get_lambda_policy(function, qualifier=None):
941-
iam_client = aws_stack.connect_to_service("iam")
942+
iam_client = connect_to().iam
942943
policies = iam_client.list_policies(Scope="Local", MaxItems=500)["Policies"]
943944
docs = []
944945
for p in policies:
@@ -1631,7 +1632,7 @@ def add_permission_policy_statement(
16311632
):
16321633
store = get_awslambda_store_for_arn(resource_arn)
16331634
data = json.loads(to_str(request.data))
1634-
iam_client = aws_stack.connect_to_service("iam")
1635+
iam_client = connect_to().iam
16351636
sid = data.get("StatementId")
16361637
action = data.get("Action")
16371638
principal = data.get("Principal")
@@ -1690,7 +1691,7 @@ def add_permission_policy_statement(
16901691
@app.route("%s/functions/<function>/policy/<statement>" % API_PATH_ROOT, methods=["DELETE"])
16911692
def remove_permission(function, statement):
16921693
qualifier = request.args.get("Qualifier")
1693-
iam_client = aws_stack.connect_to_service("iam")
1694+
iam_client = connect_to().iam
16941695
policy = get_lambda_policy(function, qualifier=qualifier)
16951696
if not policy:
16961697
return not_found_error('Unable to find policy for Lambda function "%s"' % function)
@@ -1842,7 +1843,7 @@ def _create_response(invocation_result, status_code=200, headers=None):
18421843
),
18431844
mode="rb",
18441845
)
1845-
lambda_client = aws_stack.connect_to_service("lambda")
1846+
lambda_client = connect_to().awslambda
18461847
lambda_client.create_function(
18471848
FunctionName="localstack-internal-awssdk",
18481849
Runtime=LAMBDA_RUNTIME_NODEJS14X,

‎localstack/services/awslambda/lambda_executors.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
2020

2121
from localstack import config
22+
from localstack.aws.connect import connect_to
2223
from localstack.constants import DEFAULT_LAMBDA_CONTAINER_REGISTRY
2324
from localstack.runtime.hooks import hook_spec
2425
from localstack.services.awslambda.lambda_utils import (
@@ -878,7 +879,7 @@ def invoke_lambda(
878879
) -> InvocationResult:
879880
full_url = self._get_lambda_stay_open_url(lambda_docker_ip)
880881

881-
client = aws_stack.connect_to_service("lambda", endpoint_url=full_url)
882+
client = connect_to(endpoint_url=full_url).awslambda
882883
event = inv_context.event or "{}"
883884

884885
LOG.debug(f"Calling {full_url} to run invocation in docker-reuse Lambda container")

‎localstack/services/awslambda/lambda_starter.py

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,13 @@
33
from moto.awslambda import models as moto_awslambda_models
44

55
from localstack import config
6+
from localstack.aws.connect import connect_to
67
from localstack.services.awslambda.lambda_api import handle_lambda_url_invocation
78
from localstack.services.awslambda.lambda_utils import get_default_executor_mode
89
from localstack.services.edge import ROUTER
910
from localstack.services.plugins import ServiceLifecycleHook
1011
from localstack.utils.analytics import log
11-
from localstack.utils.aws import arns, aws_stack
12+
from localstack.utils.aws import arns
1213
from localstack.utils.aws.request_context import AWS_REGION_REGEX
1314
from localstack.utils.patch import patch
1415
from localstack.utils.platform import is_linux
@@ -77,7 +78,6 @@ def check_lambda(expect_shutdown=False, print_error=False):
7778
out = None
7879
try:
7980
from localstack.services.infra import PROXY_LISTENERS
80-
from localstack.utils.aws import aws_stack
8181
from localstack.utils.common import wait_for_port_open
8282

8383
# wait for port to be opened
@@ -86,9 +86,7 @@ def check_lambda(expect_shutdown=False, print_error=False):
8686
wait_for_port_open(port, sleep_time=0.5, retries=20)
8787

8888
endpoint_url = f"http://127.0.0.1:{port}"
89-
out = aws_stack.connect_to_service(
90-
service_name="lambda", endpoint_url=endpoint_url
91-
).list_functions()
89+
out = connect_to(endpoint_url=endpoint_url).awslambda.list_functions()
9290
except Exception:
9391
if print_error:
9492
LOG.exception("Lambda health check failed")
@@ -104,7 +102,7 @@ def get_function(fn, self, *args, **kwargs):
104102
if result:
105103
return result
106104

107-
client = aws_stack.connect_to_service("lambda")
105+
client = connect_to().awslambda
108106
lambda_name = arns.lambda_function_name(args[0])
109107
response = client.get_function(FunctionName=lambda_name)
110108

@@ -119,5 +117,5 @@ def get_function(fn, self, *args, **kwargs):
119117
@patch(moto_awslambda_models.LambdaFunction.invoke)
120118
def invoke(fn, self, *args, **kwargs):
121119
payload = to_bytes(args[0])
122-
client = aws_stack.connect_to_service("lambda")
120+
client = connect_to().awslambda
123121
return client.invoke(FunctionName=self.function_name, Payload=payload)

‎localstack/services/cloudformation/api_utils.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55
from requests.structures import CaseInsensitiveDict
66

77
from localstack import config, constants
8+
from localstack.aws.connect import connect_to
89
from localstack.services.s3 import s3_listener, s3_utils
9-
from localstack.utils.aws import aws_stack
1010
from localstack.utils.functions import run_safe
1111
from localstack.utils.http import safe_requests
1212
from localstack.utils.strings import to_str
@@ -45,7 +45,7 @@ def get_template_body(req_data: dict) -> str:
4545
if is_local_service_url(url):
4646
parsed_path = urlparse(url).path.lstrip("/")
4747
parts = parsed_path.partition("/")
48-
client = aws_stack.connect_to_service("s3")
48+
client = connect_to().s3
4949
LOG.debug(
5050
"Download CloudFormation template content from local S3: %s - %s",
5151
parts[0],

‎localstack/services/cloudformation/engine/entities.py

Lines changed: 3 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
from localstack.services.cloudformation.engine.parameters import (
66
StackParameter,
77
convert_stack_parameters_to_list,
8-
map_to_legacy_structure,
98
strip_parameter_type,
109
)
1110
from localstack.utils.aws import arns
@@ -244,26 +243,10 @@ def stack_name(self):
244243
def stack_id(self):
245244
return self.metadata["StackId"]
246245

247-
# TODO: potential performance issues due to many stack_parameters calls (cache or limit actual invocations)
248246
@property
249-
def resources(self): # TODO: not actually resources, split apart
250-
"""Return dict of resources, parameters, conditions, and other stack metadata."""
251-
result = dict(self.template_resources)
252-
253-
result.update(
254-
{k: map_to_legacy_structure(k, v) for k, v in self.resolved_parameters.items()}
255-
)
256-
257-
# TODO: conditions don't really belong here and should be handled separately
258-
for name, value in self.conditions.items():
259-
if name not in result:
260-
result[name] = {
261-
"Type": "Parameter",
262-
"LogicalResourceId": name,
263-
"Properties": {"Value": value},
264-
}
265-
266-
return result
247+
def resources(self):
248+
"""Return dict of resources"""
249+
return dict(self.template_resources)
267250

268251
@property
269252
def template_resources(self):

0 commit comments

Comments
 (0)
0