From 84e01f85452dde16e335d4200ccd28e17aa569eb Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 18 Mar 2023 22:51:39 +0100 Subject: [PATCH 1/8] add support for VPC endpoints for private API Gateway REST APIs --- localstack/services/apigateway/helpers.py | 2 +- localstack/services/apigateway/invocations.py | 3 +- localstack/services/apigateway/router_asf.py | 17 +- localstack/services/ec2/provider.py | 89 +++++++- localstack/testing/pytest/fixtures.py | 35 ++++ tests/integration/apigateway/conftest.py | 1 + .../test_apigateway_integrations.py | 197 +++++++++++++++++- ...test_apigateway_integrations.snapshot.json | 75 +++++++ 8 files changed, 406 insertions(+), 13 deletions(-) diff --git a/localstack/services/apigateway/helpers.py b/localstack/services/apigateway/helpers.py index 0ebeeb8ef64ea..c10311ec808ed 100644 --- a/localstack/services/apigateway/helpers.py +++ b/localstack/services/apigateway/helpers.py @@ -44,7 +44,7 @@ r"^/restapis/([A-Za-z0-9_\\-]+)(?:/([A-Za-z0-9\_($|%%24)\\-]+))?/%s/(.*)$" % PATH_USER_REQUEST ) # URL pattern for invocations -HOST_REGEX_EXECUTE_API = r"(?:.*://)?([a-zA-Z0-9-]+)\.execute-api\.(localhost.localstack.cloud|([^\.]+)\.amazonaws\.com)(.*)" +HOST_REGEX_EXECUTE_API = r"(?:.*://)?([a-zA-Z0-9]+)(?:(-vpce-[^.]+))?\.execute-api\.(localhost.localstack.cloud|([^\.]+)\.amazonaws\.com)(.*)" # regex path patterns PATH_REGEX_MAIN = r"^/restapis/([A-Za-z0-9_\-]+)/[a-z]+(\?.*)?" diff --git a/localstack/services/apigateway/invocations.py b/localstack/services/apigateway/invocations.py index 55df5419215aa..7b0b558f9a428 100644 --- a/localstack/services/apigateway/invocations.py +++ b/localstack/services/apigateway/invocations.py @@ -123,7 +123,8 @@ def run_authorizer(invocation_context: ApiInvocationContext, authorizer: Dict): def authorize_invocation(invocation_context: ApiInvocationContext): - client = aws_stack.connect_to_service("apigateway") + region_name = invocation_context.region_name or aws_stack.get_region() + client = aws_stack.connect_to_service("apigateway", region_name=region_name) authorizers = client.get_authorizers(restApiId=invocation_context.api_id, limit=100).get( "items", [] ) diff --git a/localstack/services/apigateway/router_asf.py b/localstack/services/apigateway/router_asf.py index 52833e7da5ecf..6e7ccf4f81d34 100644 --- a/localstack/services/apigateway/router_asf.py +++ b/localstack/services/apigateway/router_asf.py @@ -99,27 +99,28 @@ def __init__(self, router: Router[Handler]): def register_routes(self) -> None: """Registers parameterized routes for API Gateway user invocations.""" if self.registered: - LOG.debug("Skipped API gateway route registration (routes already registered).") + LOG.debug("Skipped API Gateway route registration (routes already registered).") return self.registered = True - LOG.debug("Registering parameterized API gateway routes.") + LOG.debug("Registering parameterized API Gateway routes.") + host_pattern = ".execute-api." self.router.add( "/", - host=".execute-api.", + host=host_pattern, endpoint=self.invoke_rest_api, defaults={"path": "", "stage": None}, strict_slashes=True, ) self.router.add( "//", - host=".execute-api.", + host=host_pattern, endpoint=self.invoke_rest_api, defaults={"path": ""}, strict_slashes=False, ) self.router.add( "//", - host=".execute-api.", + host=host_pattern, endpoint=self.invoke_rest_api, strict_slashes=True, ) @@ -136,10 +137,12 @@ def register_routes(self) -> None: strict_slashes=True, ) - def invoke_rest_api(self, request: Request, **url_params: Dict[str, Any]) -> Response: - if not get_api_account_id_and_region(url_params["api_id"])[1]: + def invoke_rest_api(self, request: Request, **url_params: Dict[str, str]) -> Response: + _, region_name = get_api_account_id_and_region(url_params["api_id"]) + if not region_name: return Response(status=404) invocation_context = to_invocation_context(request, url_params) + invocation_context.region_name = region_name result = invoke_rest_api_from_request(invocation_context) if result is not None: return convert_response(result) diff --git a/localstack/services/ec2/provider.py b/localstack/services/ec2/provider.py index 66db3fa31b681..f892ef39d5a91 100644 --- a/localstack/services/ec2/provider.py +++ b/localstack/services/ec2/provider.py @@ -1,3 +1,4 @@ +import json import re from abc import ABC from datetime import datetime, timezone @@ -5,9 +6,16 @@ from botocore.parsers import ResponseParserError from moto.core.utils import camelcase_to_underscores, underscores_to_camelcase from moto.ec2.exceptions import InvalidVpcEndPointIdError -from moto.ec2.models import SubnetBackend, TransitGatewayAttachmentBackend +from moto.ec2.models import ( + EC2Backend, + SubnetBackend, + TransitGatewayAttachmentBackend, + VPCBackend, + ec2_backends, +) from moto.ec2.models.launch_templates import LaunchTemplate as MotoLaunchTemplate from moto.ec2.models.subnets import Subnet +from moto.ec2.models.vpcs import VPCEndPoint from localstack.aws.api import RequestContext, handler from localstack.aws.api.ec2 import ( @@ -30,7 +38,13 @@ DescribeSubnetsResult, DescribeTransitGatewaysRequest, DescribeTransitGatewaysResult, + DescribeVpcEndpointServicesRequest, + DescribeVpcEndpointServicesResult, + DescribeVpcEndpointsRequest, + DescribeVpcEndpointsResult, + DnsOptions, DnsOptionsSpecification, + DnsRecordIpType, Ec2Api, InstanceType, IpAddressType, @@ -68,7 +82,7 @@ from localstack.services.moto import call_moto from localstack.utils.aws import aws_stack from localstack.utils.patch import patch -from localstack.utils.strings import first_char_to_upper, long_uid +from localstack.utils.strings import first_char_to_upper, long_uid, short_uid # additional subnet attributes not yet supported upstream ADDITIONAL_SUBNET_ATTRS = ("private_dns_name_options_on_launch", "enable_dns64") @@ -381,6 +395,67 @@ def modify_launch_template( return result + @handler("DescribeVpcEndpointServices", expand=False) + def describe_vpc_endpoint_services( + self, + context: RequestContext, + request: DescribeVpcEndpointServicesRequest, + ) -> DescribeVpcEndpointServicesResult: + ep_services = VPCBackend._collect_default_endpoint_services( + account_id=context.account_id, region=context.region + ) + + moto_backend = get_moto_backend(context) + service_names = [s["ServiceName"] for s in ep_services] + execute_api_name = f"com.amazonaws.{context.region}.execute-api" + + if execute_api_name not in service_names: + # ensure that the service entry for execute-api exists + service_names.append(execute_api_name) + zones = moto_backend.describe_availability_zones() + zones = [zone.name for zone in zones] + private_dns_name = f"*.execute-api.{context.region}.amazonaws.com" + service = { + "ServiceName": execute_api_name, + "ServiceId": f"vpce-svc-{short_uid()}", + "ServiceType": [{"ServiceType": "Interface"}], + "AvailabilityZones": zones, + "Owner": "amazon", + "BaseEndpointDnsNames": [f"execute-api.{context.region}.vpce.amazonaws.com"], + "PrivateDnsName": private_dns_name, + "PrivateDnsNames": [{"PrivateDnsName": private_dns_name}], + "VpcEndpointPolicySupported": True, + "AcceptanceRequired": False, + "ManagesVpcEndpoints": False, + "PrivateDnsNameVerificationState": "verified", + "SupportedIpAddressTypes": ["ipv4"], + } + ep_services.append(service) + + return call_moto(context) + + @handler("DescribeVpcEndpoints", expand=False) + def describe_vpc_endpoints( + self, + context: RequestContext, + request: DescribeVpcEndpointsRequest, + ) -> DescribeVpcEndpointsResult: + result: DescribeVpcEndpointsResult = call_moto(context) + + for endpoint in result.get("VpcEndpoints"): + endpoint.setdefault("DnsOptions", DnsOptions(DnsRecordIpType=DnsRecordIpType.ipv4)) + endpoint.setdefault("IpAddressType", IpAddressType.ipv4) + endpoint.setdefault("RequesterManaged", False) + endpoint.setdefault("RouteTableIds", []) + # AWS parity: Version should not be contained in the policy response + policy = endpoint.get("PolicyDocument") + if policy and '"Version":' in policy: + policy = json.loads(policy) + policy.pop("Version", None) + endpoint["PolicyDocument"] = json.dumps(policy) + + return result + @patch(SubnetBackend.modify_subnet_attribute) def modify_subnet_attribute(fn, self, subnet_id, attr_name, attr_value): @@ -399,6 +474,11 @@ def modify_subnet_attribute(fn, self, subnet_id, attr_name, attr_value): return fn(self, subnet_id, attr_name, attr_value) +def get_moto_backend(context: RequestContext) -> EC2Backend: + """Get the moto EC2 backend for the given request context""" + return ec2_backends[context.account_id][context.region] + + @patch(Subnet.get_filter_value) def get_filter_value(fn, self, filter_name): if filter_name in ( @@ -414,3 +494,8 @@ def delete_transit_gateway_vpc_attachment(fn, self, transit_gateway_attachment_i transit_gateway_attachment = self.transit_gateway_attachments.get(transit_gateway_attachment_id) transit_gateway_attachment.state = "deleted" return transit_gateway_attachment + + +# fix a bug in upstream moto where a space is encoded in the "Statement" key - TODO remove once fixed upstream +if "Statement " in VPCEndPoint.DEFAULT_POLICY: + VPCEndPoint.DEFAULT_POLICY["Statement"] = VPCEndPoint.DEFAULT_POLICY.pop("Statement ") diff --git a/localstack/testing/pytest/fixtures.py b/localstack/testing/pytest/fixtures.py index 79ce964857c62..03dc1f530ce78 100644 --- a/localstack/testing/pytest/fixtures.py +++ b/localstack/testing/pytest/fixtures.py @@ -1879,6 +1879,41 @@ def factory(email_address: str) -> None: ses_client.delete_identity(Identity=identity) +@pytest.fixture +def ec2_create_security_group(ec2_client): + ec2_sgs = [] + + def factory(ports=None, **kwargs): + if "GroupName" not in kwargs: + kwargs["GroupName"] = f"test-sg-{short_uid()}" + security_group = ec2_client.create_security_group(**kwargs) + + permissions = [ + { + "FromPort": port, + "IpProtocol": "tcp", + "IpRanges": [{"CidrIp": "0.0.0.0/0"}], + "ToPort": port, + } + for port in ports or [] + ] + ec2_client.authorize_security_group_ingress( + GroupName=kwargs["GroupName"], + IpPermissions=permissions, + ) + + ec2_sgs.append(security_group["GroupId"]) + return security_group + + yield factory + + for sg_group_id in ec2_sgs: + try: + ec2_client.delete_security_group(GroupId=sg_group_id) + except Exception as e: + LOG.debug(f"Error cleaning up ec2 security group: {sg_group_id}, {e}") + + @pytest.fixture def cleanups(ec2_client): cleanup_fns = [] diff --git a/tests/integration/apigateway/conftest.py b/tests/integration/apigateway/conftest.py index 4db7168e6a74d..627e61a2dedda 100644 --- a/tests/integration/apigateway/conftest.py +++ b/tests/integration/apigateway/conftest.py @@ -85,6 +85,7 @@ def _factory( resourceId=resource_id, httpMethod="POST", authorizationType="NONE", + apiKeyRequired=False, ) # set AWS policy to give API GW access to backend resources diff --git a/tests/integration/apigateway/test_apigateway_integrations.py b/tests/integration/apigateway/test_apigateway_integrations.py index b35cf7e906a52..1f15bd5f54239 100644 --- a/tests/integration/apigateway/test_apigateway_integrations.py +++ b/tests/integration/apigateway/test_apigateway_integrations.py @@ -1,22 +1,32 @@ import json +import textwrap +from urllib.parse import urlparse import pytest import requests from botocore.exceptions import ClientError +from localstack import config from localstack.aws.accounts import get_aws_account_id +from localstack.constants import APPLICATION_JSON from localstack.services.apigateway.helpers import path_based_url -from localstack.services.awslambda.lambda_utils import LAMBDA_RUNTIME_PYTHON39 +from localstack.services.awslambda.lambda_utils import ( + LAMBDA_RUNTIME_PYTHON39, + get_main_endpoint_from_container, +) +from localstack.testing.aws.util import is_aws_cloud from localstack.utils.aws import arns, aws_stack -from localstack.utils.strings import short_uid +from localstack.utils.strings import short_uid, to_bytes, to_str from localstack.utils.sync import retry from localstack.utils.testutil import create_lambda_function from tests.integration.apigateway.apigateway_fixtures import ( api_invoke_url, + create_rest_api_deployment, create_rest_api_integration, create_rest_resource, create_rest_resource_method, ) +from tests.integration.apigateway.conftest import DEFAULT_STAGE_NAME from tests.integration.awslambda.test_lambda import TEST_LAMBDA_AWS_PROXY, TEST_LAMBDA_HELLO_WORLD @@ -652,6 +662,189 @@ def test_put_integration_validation(apigateway_client): ) +@pytest.fixture +def default_vpc(ec2_client): + vpcs = ec2_client.describe_vpcs() + for vpc in vpcs["Vpcs"]: + if vpc.get("IsDefault"): + return vpc + raise Exception("Default VPC not found") + + +@pytest.fixture +def create_vpc_endpoint(ec2_client, default_vpc): + endpoints = [] + + def _create(**kwargs): + kwargs.setdefault("VpcId", default_vpc["VpcId"]) + result = ec2_client.create_vpc_endpoint(**kwargs) + endpoints.append(result["VpcEndpoint"]["VpcEndpointId"]) + return result["VpcEndpoint"] + + yield _create + + for endpoint in endpoints: + with contextlib.suppress(Exception): + ec2_client.delete_vpc_endpoints(VpcEndpointIds=[endpoint]) + + +@pytest.mark.skip_snapshot_verify( + paths=["$..endpointConfiguration.types", "$..policy.Statement..Resource"] +) +def test_create_execute_api_vpc_endpoint( + create_rest_api_with_integration, + dynamodb_create_table, + create_vpc_endpoint, + default_vpc, + create_lambda_function, + ec2_create_security_group, + ec2_client, + apigateway_client, + dynamodb_resource, + lambda_client, + snapshot, +): + poll_sleep = 5 if is_aws_cloud() else 1 + # TODO: create a re-usable ec2_api() transformer + snapshot.add_transformer(snapshot.transform.key_value("DnsName")) + snapshot.add_transformer(snapshot.transform.key_value("GroupId")) + snapshot.add_transformer(snapshot.transform.key_value("GroupName")) + snapshot.add_transformer(snapshot.transform.key_value("SubnetIds")) + snapshot.add_transformer(snapshot.transform.key_value("VpcId")) + snapshot.add_transformer(snapshot.transform.key_value("VpcEndpointId")) + snapshot.add_transformer(snapshot.transform.key_value("HostedZoneId")) + snapshot.add_transformer(snapshot.transform.key_value("id")) + snapshot.add_transformer(snapshot.transform.key_value("name")) + + # create table + table = dynamodb_create_table()["TableDescription"] + table_name = table["TableName"] + + # insert items + dynamodb_table = dynamodb_resource.Table(table_name) + item_ids = ("test", "test2", "test 3") + for item_id in item_ids: + dynamodb_table.put_item(Item={"id": item_id}) + + # construct request mapping template + request_templates = {APPLICATION_JSON: json.dumps({"TableName": table_name})} + + # deploy REST API with integration + region_name = apigateway_client.meta.region_name + integration_uri = f"arn:aws:apigateway:{region_name}:dynamodb:action/Scan" + api_id = create_rest_api_with_integration( + integration_uri=integration_uri, + req_templates=request_templates, + integration_type="AWS", + ) + + # get service names + service_name = f"com.amazonaws.{region_name}.execute-api" + service_names = ec2_client.describe_vpc_endpoint_services()["ServiceNames"] + assert service_name in service_names + + # create security group + vpc_id = default_vpc["VpcId"] + security_group = ec2_create_security_group( + VpcId=vpc_id, Description="Test SG for API GW", ports=[443] + ) + security_group = security_group["GroupId"] + subnets = ec2_client.describe_subnets(Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]) + subnets = [sub["SubnetId"] for sub in subnets["Subnets"]] + + # get or create execute-api VPC endpoint + endpoints = ec2_client.describe_vpc_endpoints(MaxResults=1000)["VpcEndpoints"] + matching = [ep for ep in endpoints if ep["ServiceName"] == service_name] + if matching: + endpoint_id = matching[0]["VpcEndpointId"] + else: + result = create_vpc_endpoint( + ServiceName=service_name, + VpcEndpointType="Interface", + SubnetIds=subnets, + SecurityGroupIds=[security_group], + ) + endpoint_id = result["VpcEndpointId"] + + # wait until VPC endpoint is in state "available" + def _check_available(): + result = ec2_client.describe_vpc_endpoints(VpcEndpointIds=[endpoint_id]) + endpoint_details = result["VpcEndpoints"][0] + # may have multiple entries in AWS + endpoint_details["DnsEntries"] = endpoint_details["DnsEntries"][:1] + endpoint_details.pop("SubnetIds", None) + endpoint_details.pop("NetworkInterfaceIds", None) + assert endpoint_details["State"] == "available" + snapshot.match("endpoint-details", endpoint_details) + + retry(_check_available, retries=30, sleep=poll_sleep) + + # update API with VPC endpoint + patches = [ + {"op": "replace", "path": "/endpointConfiguration/types/EDGE", "value": "PRIVATE"}, + {"op": "add", "path": "/endpointConfiguration/vpcEndpointIds", "value": endpoint_id}, + ] + apigateway_client.update_rest_api(restApiId=api_id, patchOperations=patches) + + # create Lambda that invokes API via VPC endpoint (required as the endpoint is only accessible within the VPC) + subdomain = f"{api_id}-{endpoint_id}" + endpoint = api_invoke_url(subdomain, stage=DEFAULT_STAGE_NAME, path="/test") + host_header = urlparse(endpoint).netloc + + # create Lambda function that invokes the API GW (private VPC endpoint not accessible from outside of AWS) + if not is_aws_cloud(): + endpoint = endpoint.replace( + host_header, f"{get_main_endpoint_from_container()}:{config.get_edge_port_http()}" + ) + print("!endpoint", endpoint, host_header) + lambda_code = textwrap.dedent( + f""" + def handler(event, context): + import requests + headers = {{"content-type": "application/json", "host": "{host_header}"}} + result = requests.post("{endpoint}", headers=headers) + return {{"content": result.content.decode("utf-8"), "code": result.status_code}} + """ + ) + func_name = f"test-{short_uid()}" + vpc_config = { + "SubnetIds": subnets, + "SecurityGroupIds": [security_group], + } + create_lambda_function( + func_name=func_name, handler_file=lambda_code, timeout=10, VpcConfig=vpc_config + ) + + # create resource policy + statement = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": "*", + "Action": "execute-api:Invoke", + "Resource": ["execute-api:/*"], + } + ], + } + patches = [{"op": "replace", "path": "/policy", "value": json.dumps(statement)}] + result = apigateway_client.update_rest_api(restApiId=api_id, patchOperations=patches) + result["policy"] = json.loads(to_bytes(result["policy"]).decode("unicode_escape")) + snapshot.match("api-details", result) + + # re-deploy API + create_rest_api_deployment(apigateway_client, restApiId=api_id, stageName=DEFAULT_STAGE_NAME) + + def _invoke_api(): + result = lambda_client.invoke(FunctionName=func_name, Payload="{}") + result = json.loads(to_str(result["Payload"].read())) + items = json.loads(result["content"])["Items"] + assert len(items) == len(item_ids) + + # invoke Lambda and assert result + retry(_invoke_api, retries=15, sleep=poll_sleep) + + # TODO - remove the code below? # # def test_aws_integration_dynamodb(apigateway_client): diff --git a/tests/integration/apigateway/test_apigateway_integrations.snapshot.json b/tests/integration/apigateway/test_apigateway_integrations.snapshot.json index cdd61f637e032..37d7691c8de05 100644 --- a/tests/integration/apigateway/test_apigateway_integrations.snapshot.json +++ b/tests/integration/apigateway/test_apigateway_integrations.snapshot.json @@ -772,5 +772,80 @@ "stageVariables": null } } + }, + "tests/integration/apigateway/test_apigateway_integrations.py::test_create_execute_api_vpc_endpoint": { + "recorded-date": "18-03-2023, 22:01:10", + "recorded-content": { + "endpoint-details": { + "CreationTimestamp": "timestamp", + "DnsEntries": [ + { + "DnsName": "", + "HostedZoneId": "" + } + ], + "DnsOptions": { + "DnsRecordIpType": "ipv4" + }, + "Groups": [ + { + "GroupId": "", + "GroupName": "" + } + ], + "IpAddressType": "ipv4", + "OwnerId": "111111111111", + "PolicyDocument": { + "Statement": [ + { + "Action": "*", + "Effect": "Allow", + "Principal": "*", + "Resource": "*" + } + ] + }, + "PrivateDnsEnabled": true, + "RequesterManaged": false, + "RouteTableIds": [], + "ServiceName": "com.amazonaws..execute-api", + "State": "available", + "Tags": [], + "VpcEndpointId": "", + "VpcEndpointType": "Interface", + "VpcId": "" + }, + "api-details": { + "apiKeySource": "HEADER", + "createdDate": "datetime", + "disableExecuteApiEndpoint": false, + "endpointConfiguration": { + "types": [ + "PRIVATE" + ], + "vpcEndpointIds": [ + "" + ] + }, + "id": "", + "name": "", + "policy": { + "Statement": [ + { + "Action": "execute-api:Invoke", + "Effect": "Allow", + "Principal": "*", + "Resource": "arn:aws:execute-api::111111111111:/*" + } + ], + "Version": "2012-10-17" + }, + "tags": {}, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } } } From 1848f83f29fd6ee868fcff14b92825fe916bcbd8 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 18 Mar 2023 23:35:29 +0100 Subject: [PATCH 2/8] fix linter --- tests/integration/apigateway/test_apigateway_integrations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/apigateway/test_apigateway_integrations.py b/tests/integration/apigateway/test_apigateway_integrations.py index 1f15bd5f54239..bafbe12298fba 100644 --- a/tests/integration/apigateway/test_apigateway_integrations.py +++ b/tests/integration/apigateway/test_apigateway_integrations.py @@ -1,3 +1,4 @@ +import contextlib import json import textwrap from urllib.parse import urlparse From a834f0c41ea9b1df94d1df0ebd403781304c2fdc Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sun, 19 Mar 2023 02:09:14 +0100 Subject: [PATCH 3/8] minor cleanup --- localstack/services/awslambda/lambda_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/localstack/services/awslambda/lambda_utils.py b/localstack/services/awslambda/lambda_utils.py index 800c6dbb2ef7a..e030bfaf7ccf1 100644 --- a/localstack/services/awslambda/lambda_utils.py +++ b/localstack/services/awslambda/lambda_utils.py @@ -185,7 +185,6 @@ def store_lambda_logs( def get_main_endpoint_from_container() -> str: - global DOCKER_MAIN_CONTAINER_IP if config.HOSTNAME_FROM_LAMBDA: return config.HOSTNAME_FROM_LAMBDA return get_endpoint_for_network(network=get_container_network_for_lambda()) From aea14056dde07db60208778037da4fbc5b09ffaa Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sun, 19 Mar 2023 13:36:11 +0100 Subject: [PATCH 4/8] add tmp CI debug logs --- localstack/utils/container_utils/docker_sdk_client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/localstack/utils/container_utils/docker_sdk_client.py b/localstack/utils/container_utils/docker_sdk_client.py index 6be847146ae91..1af3931deb9bc 100644 --- a/localstack/utils/container_utils/docker_sdk_client.py +++ b/localstack/utils/container_utils/docker_sdk_client.py @@ -40,9 +40,15 @@ class SdkDockerClient(ContainerClient): def __init__(self): try: + print("!get client") # TODO: CI debugging + LOG.debug("Get Docker client from environment") self.docker_client = docker.from_env() logging.getLogger("urllib3").setLevel(logging.INFO) - except DockerException: + print("!get client", self.docker_client) # TODO: CI debugging + LOG.debug("Created Docker client: %s", self.docker_client) + except DockerException as e: + print("!ERR", e, type(e)) # TODO: CI debugging + LOG.exception("Unable to created Docker client") self.docker_client = None def client(self): From 0c4c85e8538abed69b3e813c6bf8438bcc59475f Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sun, 19 Mar 2023 15:58:18 +0100 Subject: [PATCH 5/8] adjust endpoint for local Lambda executor mode --- localstack/services/awslambda/lambda_utils.py | 4 ++++ localstack/testing/pytest/fixtures.py | 2 +- localstack/utils/container_utils/docker_sdk_client.py | 8 +------- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/localstack/services/awslambda/lambda_utils.py b/localstack/services/awslambda/lambda_utils.py index e030bfaf7ccf1..9f42c38bf6e14 100644 --- a/localstack/services/awslambda/lambda_utils.py +++ b/localstack/services/awslambda/lambda_utils.py @@ -14,6 +14,7 @@ from localstack import config from localstack.aws.accounts import get_aws_account_id from localstack.aws.api.lambda_ import FilterCriteria, Runtime +from localstack.constants import LOCALHOST from localstack.services.awslambda.lambda_models import AwsLambdaStore, awslambda_stores from localstack.utils.aws import aws_stack from localstack.utils.aws.arns import extract_account_id_from_arn, extract_region_from_arn @@ -187,6 +188,9 @@ def store_lambda_logs( def get_main_endpoint_from_container() -> str: if config.HOSTNAME_FROM_LAMBDA: return config.HOSTNAME_FROM_LAMBDA + if config.LAMBDA_EXECUTOR == "local": + # special case: return localhost for local Lambda executor (TODO remove after full switch to v2 provider) + return LOCALHOST return get_endpoint_for_network(network=get_container_network_for_lambda()) diff --git a/localstack/testing/pytest/fixtures.py b/localstack/testing/pytest/fixtures.py index 03dc1f530ce78..ed6406158aec6 100644 --- a/localstack/testing/pytest/fixtures.py +++ b/localstack/testing/pytest/fixtures.py @@ -1911,7 +1911,7 @@ def factory(ports=None, **kwargs): try: ec2_client.delete_security_group(GroupId=sg_group_id) except Exception as e: - LOG.debug(f"Error cleaning up ec2 security group: {sg_group_id}, {e}") + LOG.debug("Error cleaning up EC2 security group: %s, %s", sg_group_id, e) @pytest.fixture diff --git a/localstack/utils/container_utils/docker_sdk_client.py b/localstack/utils/container_utils/docker_sdk_client.py index 1af3931deb9bc..6be847146ae91 100644 --- a/localstack/utils/container_utils/docker_sdk_client.py +++ b/localstack/utils/container_utils/docker_sdk_client.py @@ -40,15 +40,9 @@ class SdkDockerClient(ContainerClient): def __init__(self): try: - print("!get client") # TODO: CI debugging - LOG.debug("Get Docker client from environment") self.docker_client = docker.from_env() logging.getLogger("urllib3").setLevel(logging.INFO) - print("!get client", self.docker_client) # TODO: CI debugging - LOG.debug("Created Docker client: %s", self.docker_client) - except DockerException as e: - print("!ERR", e, type(e)) # TODO: CI debugging - LOG.exception("Unable to created Docker client") + except DockerException: self.docker_client = None def client(self): From e6b196fd8b0253541d93f681fc59eb39f34f99d0 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sun, 19 Mar 2023 17:24:39 +0100 Subject: [PATCH 6/8] cleanup tmp debugging --- tests/integration/apigateway/test_apigateway_integrations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/apigateway/test_apigateway_integrations.py b/tests/integration/apigateway/test_apigateway_integrations.py index bafbe12298fba..8b383facfe06e 100644 --- a/tests/integration/apigateway/test_apigateway_integrations.py +++ b/tests/integration/apigateway/test_apigateway_integrations.py @@ -797,7 +797,6 @@ def _check_available(): endpoint = endpoint.replace( host_header, f"{get_main_endpoint_from_container()}:{config.get_edge_port_http()}" ) - print("!endpoint", endpoint, host_header) lambda_code = textwrap.dedent( f""" def handler(event, context): From 93d1815e4b009914e2c1b63ca5443e9e718ee7b5 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Mon, 20 Mar 2023 12:27:01 +0100 Subject: [PATCH 7/8] move local executor check into tests --- localstack/services/awslambda/lambda_utils.py | 4 ---- .../apigateway/test_apigateway_integrations.py | 12 ++++++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/localstack/services/awslambda/lambda_utils.py b/localstack/services/awslambda/lambda_utils.py index 9f42c38bf6e14..e030bfaf7ccf1 100644 --- a/localstack/services/awslambda/lambda_utils.py +++ b/localstack/services/awslambda/lambda_utils.py @@ -14,7 +14,6 @@ from localstack import config from localstack.aws.accounts import get_aws_account_id from localstack.aws.api.lambda_ import FilterCriteria, Runtime -from localstack.constants import LOCALHOST from localstack.services.awslambda.lambda_models import AwsLambdaStore, awslambda_stores from localstack.utils.aws import aws_stack from localstack.utils.aws.arns import extract_account_id_from_arn, extract_region_from_arn @@ -188,9 +187,6 @@ def store_lambda_logs( def get_main_endpoint_from_container() -> str: if config.HOSTNAME_FROM_LAMBDA: return config.HOSTNAME_FROM_LAMBDA - if config.LAMBDA_EXECUTOR == "local": - # special case: return localhost for local Lambda executor (TODO remove after full switch to v2 provider) - return LOCALHOST return get_endpoint_for_network(network=get_container_network_for_lambda()) diff --git a/tests/integration/apigateway/test_apigateway_integrations.py b/tests/integration/apigateway/test_apigateway_integrations.py index 8b383facfe06e..fc79e4c725ca2 100644 --- a/tests/integration/apigateway/test_apigateway_integrations.py +++ b/tests/integration/apigateway/test_apigateway_integrations.py @@ -9,12 +9,13 @@ from localstack import config from localstack.aws.accounts import get_aws_account_id -from localstack.constants import APPLICATION_JSON +from localstack.constants import APPLICATION_JSON, LOCALHOST from localstack.services.apigateway.helpers import path_based_url from localstack.services.awslambda.lambda_utils import ( LAMBDA_RUNTIME_PYTHON39, get_main_endpoint_from_container, ) +from localstack.testing.aws.lambda_utils import is_old_provider from localstack.testing.aws.util import is_aws_cloud from localstack.utils.aws import arns, aws_stack from localstack.utils.strings import short_uid, to_bytes, to_str @@ -794,9 +795,12 @@ def _check_available(): # create Lambda function that invokes the API GW (private VPC endpoint not accessible from outside of AWS) if not is_aws_cloud(): - endpoint = endpoint.replace( - host_header, f"{get_main_endpoint_from_container()}:{config.get_edge_port_http()}" - ) + if config.LAMBDA_EXECUTOR == "local" and is_old_provider(): + # special case: return localhost for local Lambda executor (TODO remove after full switch to v2 provider) + api_host = LOCALHOST + else: + api_host = get_main_endpoint_from_container() + endpoint = endpoint.replace(host_header, f"{api_host}:{config.get_edge_port_http()}") lambda_code = textwrap.dedent( f""" def handler(event, context): From dbd6a3bd2e04322fbe19fe45634e033c2b4d2ae7 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Tue, 21 Mar 2023 20:38:26 +0100 Subject: [PATCH 8/8] remove unnecessary statement --- localstack/services/ec2/provider.py | 1 - 1 file changed, 1 deletion(-) diff --git a/localstack/services/ec2/provider.py b/localstack/services/ec2/provider.py index f892ef39d5a91..372b12d7ca2e6 100644 --- a/localstack/services/ec2/provider.py +++ b/localstack/services/ec2/provider.py @@ -411,7 +411,6 @@ def describe_vpc_endpoint_services( if execute_api_name not in service_names: # ensure that the service entry for execute-api exists - service_names.append(execute_api_name) zones = moto_backend.describe_availability_zones() zones = [zone.name for zone in zones] private_dns_name = f"*.execute-api.{context.region}.amazonaws.com"