diff --git a/localstack/services/apigateway/helpers.py b/localstack/services/apigateway/helpers.py index 0ebeeb8ef64ea..c10311ec808ed 100644 --- a/localstack/services/apigateway/helpers.py +++ b/localstack/services/apigateway/helpers.py @@ -44,7 +44,7 @@ r"^/restapis/([A-Za-z0-9_\\-]+)(?:/([A-Za-z0-9\_($|%%24)\\-]+))?/%s/(.*)$" % PATH_USER_REQUEST ) # URL pattern for invocations -HOST_REGEX_EXECUTE_API = r"(?:.*://)?([a-zA-Z0-9-]+)\.execute-api\.(localhost.localstack.cloud|([^\.]+)\.amazonaws\.com)(.*)" +HOST_REGEX_EXECUTE_API = r"(?:.*://)?([a-zA-Z0-9]+)(?:(-vpce-[^.]+))?\.execute-api\.(localhost.localstack.cloud|([^\.]+)\.amazonaws\.com)(.*)" # regex path patterns PATH_REGEX_MAIN = r"^/restapis/([A-Za-z0-9_\-]+)/[a-z]+(\?.*)?" diff --git a/localstack/services/apigateway/invocations.py b/localstack/services/apigateway/invocations.py index 55df5419215aa..7b0b558f9a428 100644 --- a/localstack/services/apigateway/invocations.py +++ b/localstack/services/apigateway/invocations.py @@ -123,7 +123,8 @@ def run_authorizer(invocation_context: ApiInvocationContext, authorizer: Dict): def authorize_invocation(invocation_context: ApiInvocationContext): - client = aws_stack.connect_to_service("apigateway") + region_name = invocation_context.region_name or aws_stack.get_region() + client = aws_stack.connect_to_service("apigateway", region_name=region_name) authorizers = client.get_authorizers(restApiId=invocation_context.api_id, limit=100).get( "items", [] ) diff --git a/localstack/services/apigateway/router_asf.py b/localstack/services/apigateway/router_asf.py index 52833e7da5ecf..6e7ccf4f81d34 100644 --- a/localstack/services/apigateway/router_asf.py +++ b/localstack/services/apigateway/router_asf.py @@ -99,27 +99,28 @@ def __init__(self, router: Router[Handler]): def register_routes(self) -> None: """Registers parameterized routes for API Gateway user invocations.""" if self.registered: - LOG.debug("Skipped API gateway route registration (routes already registered).") + LOG.debug("Skipped API Gateway route registration (routes already registered).") return self.registered = True - LOG.debug("Registering parameterized API gateway routes.") + LOG.debug("Registering parameterized API Gateway routes.") + host_pattern = ".execute-api." self.router.add( "/", - host=".execute-api.", + host=host_pattern, endpoint=self.invoke_rest_api, defaults={"path": "", "stage": None}, strict_slashes=True, ) self.router.add( "//", - host=".execute-api.", + host=host_pattern, endpoint=self.invoke_rest_api, defaults={"path": ""}, strict_slashes=False, ) self.router.add( "//", - host=".execute-api.", + host=host_pattern, endpoint=self.invoke_rest_api, strict_slashes=True, ) @@ -136,10 +137,12 @@ def register_routes(self) -> None: strict_slashes=True, ) - def invoke_rest_api(self, request: Request, **url_params: Dict[str, Any]) -> Response: - if not get_api_account_id_and_region(url_params["api_id"])[1]: + def invoke_rest_api(self, request: Request, **url_params: Dict[str, str]) -> Response: + _, region_name = get_api_account_id_and_region(url_params["api_id"]) + if not region_name: return Response(status=404) invocation_context = to_invocation_context(request, url_params) + invocation_context.region_name = region_name result = invoke_rest_api_from_request(invocation_context) if result is not None: return convert_response(result) diff --git a/localstack/services/awslambda/lambda_utils.py b/localstack/services/awslambda/lambda_utils.py index 800c6dbb2ef7a..e030bfaf7ccf1 100644 --- a/localstack/services/awslambda/lambda_utils.py +++ b/localstack/services/awslambda/lambda_utils.py @@ -185,7 +185,6 @@ def store_lambda_logs( def get_main_endpoint_from_container() -> str: - global DOCKER_MAIN_CONTAINER_IP if config.HOSTNAME_FROM_LAMBDA: return config.HOSTNAME_FROM_LAMBDA return get_endpoint_for_network(network=get_container_network_for_lambda()) diff --git a/localstack/services/ec2/provider.py b/localstack/services/ec2/provider.py index 66db3fa31b681..372b12d7ca2e6 100644 --- a/localstack/services/ec2/provider.py +++ b/localstack/services/ec2/provider.py @@ -1,3 +1,4 @@ +import json import re from abc import ABC from datetime import datetime, timezone @@ -5,9 +6,16 @@ from botocore.parsers import ResponseParserError from moto.core.utils import camelcase_to_underscores, underscores_to_camelcase from moto.ec2.exceptions import InvalidVpcEndPointIdError -from moto.ec2.models import SubnetBackend, TransitGatewayAttachmentBackend +from moto.ec2.models import ( + EC2Backend, + SubnetBackend, + TransitGatewayAttachmentBackend, + VPCBackend, + ec2_backends, +) from moto.ec2.models.launch_templates import LaunchTemplate as MotoLaunchTemplate from moto.ec2.models.subnets import Subnet +from moto.ec2.models.vpcs import VPCEndPoint from localstack.aws.api import RequestContext, handler from localstack.aws.api.ec2 import ( @@ -30,7 +38,13 @@ DescribeSubnetsResult, DescribeTransitGatewaysRequest, DescribeTransitGatewaysResult, + DescribeVpcEndpointServicesRequest, + DescribeVpcEndpointServicesResult, + DescribeVpcEndpointsRequest, + DescribeVpcEndpointsResult, + DnsOptions, DnsOptionsSpecification, + DnsRecordIpType, Ec2Api, InstanceType, IpAddressType, @@ -68,7 +82,7 @@ from localstack.services.moto import call_moto from localstack.utils.aws import aws_stack from localstack.utils.patch import patch -from localstack.utils.strings import first_char_to_upper, long_uid +from localstack.utils.strings import first_char_to_upper, long_uid, short_uid # additional subnet attributes not yet supported upstream ADDITIONAL_SUBNET_ATTRS = ("private_dns_name_options_on_launch", "enable_dns64") @@ -381,6 +395,66 @@ def modify_launch_template( return result + @handler("DescribeVpcEndpointServices", expand=False) + def describe_vpc_endpoint_services( + self, + context: RequestContext, + request: DescribeVpcEndpointServicesRequest, + ) -> DescribeVpcEndpointServicesResult: + ep_services = VPCBackend._collect_default_endpoint_services( + account_id=context.account_id, region=context.region + ) + + moto_backend = get_moto_backend(context) + service_names = [s["ServiceName"] for s in ep_services] + execute_api_name = f"com.amazonaws.{context.region}.execute-api" + + if execute_api_name not in service_names: + # ensure that the service entry for execute-api exists + zones = moto_backend.describe_availability_zones() + zones = [zone.name for zone in zones] + private_dns_name = f"*.execute-api.{context.region}.amazonaws.com" + service = { + "ServiceName": execute_api_name, + "ServiceId": f"vpce-svc-{short_uid()}", + "ServiceType": [{"ServiceType": "Interface"}], + "AvailabilityZones": zones, + "Owner": "amazon", + "BaseEndpointDnsNames": [f"execute-api.{context.region}.vpce.amazonaws.com"], + "PrivateDnsName": private_dns_name, + "PrivateDnsNames": [{"PrivateDnsName": private_dns_name}], + "VpcEndpointPolicySupported": True, + "AcceptanceRequired": False, + "ManagesVpcEndpoints": False, + "PrivateDnsNameVerificationState": "verified", + "SupportedIpAddressTypes": ["ipv4"], + } + ep_services.append(service) + + return call_moto(context) + + @handler("DescribeVpcEndpoints", expand=False) + def describe_vpc_endpoints( + self, + context: RequestContext, + request: DescribeVpcEndpointsRequest, + ) -> DescribeVpcEndpointsResult: + result: DescribeVpcEndpointsResult = call_moto(context) + + for endpoint in result.get("VpcEndpoints"): + endpoint.setdefault("DnsOptions", DnsOptions(DnsRecordIpType=DnsRecordIpType.ipv4)) + endpoint.setdefault("IpAddressType", IpAddressType.ipv4) + endpoint.setdefault("RequesterManaged", False) + endpoint.setdefault("RouteTableIds", []) + # AWS parity: Version should not be contained in the policy response + policy = endpoint.get("PolicyDocument") + if policy and '"Version":' in policy: + policy = json.loads(policy) + policy.pop("Version", None) + endpoint["PolicyDocument"] = json.dumps(policy) + + return result + @patch(SubnetBackend.modify_subnet_attribute) def modify_subnet_attribute(fn, self, subnet_id, attr_name, attr_value): @@ -399,6 +473,11 @@ def modify_subnet_attribute(fn, self, subnet_id, attr_name, attr_value): return fn(self, subnet_id, attr_name, attr_value) +def get_moto_backend(context: RequestContext) -> EC2Backend: + """Get the moto EC2 backend for the given request context""" + return ec2_backends[context.account_id][context.region] + + @patch(Subnet.get_filter_value) def get_filter_value(fn, self, filter_name): if filter_name in ( @@ -414,3 +493,8 @@ def delete_transit_gateway_vpc_attachment(fn, self, transit_gateway_attachment_i transit_gateway_attachment = self.transit_gateway_attachments.get(transit_gateway_attachment_id) transit_gateway_attachment.state = "deleted" return transit_gateway_attachment + + +# fix a bug in upstream moto where a space is encoded in the "Statement" key - TODO remove once fixed upstream +if "Statement " in VPCEndPoint.DEFAULT_POLICY: + VPCEndPoint.DEFAULT_POLICY["Statement"] = VPCEndPoint.DEFAULT_POLICY.pop("Statement ") diff --git a/localstack/testing/pytest/fixtures.py b/localstack/testing/pytest/fixtures.py index 79ce964857c62..ed6406158aec6 100644 --- a/localstack/testing/pytest/fixtures.py +++ b/localstack/testing/pytest/fixtures.py @@ -1879,6 +1879,41 @@ def factory(email_address: str) -> None: ses_client.delete_identity(Identity=identity) +@pytest.fixture +def ec2_create_security_group(ec2_client): + ec2_sgs = [] + + def factory(ports=None, **kwargs): + if "GroupName" not in kwargs: + kwargs["GroupName"] = f"test-sg-{short_uid()}" + security_group = ec2_client.create_security_group(**kwargs) + + permissions = [ + { + "FromPort": port, + "IpProtocol": "tcp", + "IpRanges": [{"CidrIp": "0.0.0.0/0"}], + "ToPort": port, + } + for port in ports or [] + ] + ec2_client.authorize_security_group_ingress( + GroupName=kwargs["GroupName"], + IpPermissions=permissions, + ) + + ec2_sgs.append(security_group["GroupId"]) + return security_group + + yield factory + + for sg_group_id in ec2_sgs: + try: + ec2_client.delete_security_group(GroupId=sg_group_id) + except Exception as e: + LOG.debug("Error cleaning up EC2 security group: %s, %s", sg_group_id, e) + + @pytest.fixture def cleanups(ec2_client): cleanup_fns = [] diff --git a/tests/integration/apigateway/conftest.py b/tests/integration/apigateway/conftest.py index 4db7168e6a74d..627e61a2dedda 100644 --- a/tests/integration/apigateway/conftest.py +++ b/tests/integration/apigateway/conftest.py @@ -85,6 +85,7 @@ def _factory( resourceId=resource_id, httpMethod="POST", authorizationType="NONE", + apiKeyRequired=False, ) # set AWS policy to give API GW access to backend resources diff --git a/tests/integration/apigateway/test_apigateway_integrations.py b/tests/integration/apigateway/test_apigateway_integrations.py index b35cf7e906a52..fc79e4c725ca2 100644 --- a/tests/integration/apigateway/test_apigateway_integrations.py +++ b/tests/integration/apigateway/test_apigateway_integrations.py @@ -1,22 +1,34 @@ +import contextlib import json +import textwrap +from urllib.parse import urlparse import pytest import requests from botocore.exceptions import ClientError +from localstack import config from localstack.aws.accounts import get_aws_account_id +from localstack.constants import APPLICATION_JSON, LOCALHOST from localstack.services.apigateway.helpers import path_based_url -from localstack.services.awslambda.lambda_utils import LAMBDA_RUNTIME_PYTHON39 +from localstack.services.awslambda.lambda_utils import ( + LAMBDA_RUNTIME_PYTHON39, + get_main_endpoint_from_container, +) +from localstack.testing.aws.lambda_utils import is_old_provider +from localstack.testing.aws.util import is_aws_cloud from localstack.utils.aws import arns, aws_stack -from localstack.utils.strings import short_uid +from localstack.utils.strings import short_uid, to_bytes, to_str from localstack.utils.sync import retry from localstack.utils.testutil import create_lambda_function from tests.integration.apigateway.apigateway_fixtures import ( api_invoke_url, + create_rest_api_deployment, create_rest_api_integration, create_rest_resource, create_rest_resource_method, ) +from tests.integration.apigateway.conftest import DEFAULT_STAGE_NAME from tests.integration.awslambda.test_lambda import TEST_LAMBDA_AWS_PROXY, TEST_LAMBDA_HELLO_WORLD @@ -652,6 +664,191 @@ def test_put_integration_validation(apigateway_client): ) +@pytest.fixture +def default_vpc(ec2_client): + vpcs = ec2_client.describe_vpcs() + for vpc in vpcs["Vpcs"]: + if vpc.get("IsDefault"): + return vpc + raise Exception("Default VPC not found") + + +@pytest.fixture +def create_vpc_endpoint(ec2_client, default_vpc): + endpoints = [] + + def _create(**kwargs): + kwargs.setdefault("VpcId", default_vpc["VpcId"]) + result = ec2_client.create_vpc_endpoint(**kwargs) + endpoints.append(result["VpcEndpoint"]["VpcEndpointId"]) + return result["VpcEndpoint"] + + yield _create + + for endpoint in endpoints: + with contextlib.suppress(Exception): + ec2_client.delete_vpc_endpoints(VpcEndpointIds=[endpoint]) + + +@pytest.mark.skip_snapshot_verify( + paths=["$..endpointConfiguration.types", "$..policy.Statement..Resource"] +) +def test_create_execute_api_vpc_endpoint( + create_rest_api_with_integration, + dynamodb_create_table, + create_vpc_endpoint, + default_vpc, + create_lambda_function, + ec2_create_security_group, + ec2_client, + apigateway_client, + dynamodb_resource, + lambda_client, + snapshot, +): + poll_sleep = 5 if is_aws_cloud() else 1 + # TODO: create a re-usable ec2_api() transformer + snapshot.add_transformer(snapshot.transform.key_value("DnsName")) + snapshot.add_transformer(snapshot.transform.key_value("GroupId")) + snapshot.add_transformer(snapshot.transform.key_value("GroupName")) + snapshot.add_transformer(snapshot.transform.key_value("SubnetIds")) + snapshot.add_transformer(snapshot.transform.key_value("VpcId")) + snapshot.add_transformer(snapshot.transform.key_value("VpcEndpointId")) + snapshot.add_transformer(snapshot.transform.key_value("HostedZoneId")) + snapshot.add_transformer(snapshot.transform.key_value("id")) + snapshot.add_transformer(snapshot.transform.key_value("name")) + + # create table + table = dynamodb_create_table()["TableDescription"] + table_name = table["TableName"] + + # insert items + dynamodb_table = dynamodb_resource.Table(table_name) + item_ids = ("test", "test2", "test 3") + for item_id in item_ids: + dynamodb_table.put_item(Item={"id": item_id}) + + # construct request mapping template + request_templates = {APPLICATION_JSON: json.dumps({"TableName": table_name})} + + # deploy REST API with integration + region_name = apigateway_client.meta.region_name + integration_uri = f"arn:aws:apigateway:{region_name}:dynamodb:action/Scan" + api_id = create_rest_api_with_integration( + integration_uri=integration_uri, + req_templates=request_templates, + integration_type="AWS", + ) + + # get service names + service_name = f"com.amazonaws.{region_name}.execute-api" + service_names = ec2_client.describe_vpc_endpoint_services()["ServiceNames"] + assert service_name in service_names + + # create security group + vpc_id = default_vpc["VpcId"] + security_group = ec2_create_security_group( + VpcId=vpc_id, Description="Test SG for API GW", ports=[443] + ) + security_group = security_group["GroupId"] + subnets = ec2_client.describe_subnets(Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]) + subnets = [sub["SubnetId"] for sub in subnets["Subnets"]] + + # get or create execute-api VPC endpoint + endpoints = ec2_client.describe_vpc_endpoints(MaxResults=1000)["VpcEndpoints"] + matching = [ep for ep in endpoints if ep["ServiceName"] == service_name] + if matching: + endpoint_id = matching[0]["VpcEndpointId"] + else: + result = create_vpc_endpoint( + ServiceName=service_name, + VpcEndpointType="Interface", + SubnetIds=subnets, + SecurityGroupIds=[security_group], + ) + endpoint_id = result["VpcEndpointId"] + + # wait until VPC endpoint is in state "available" + def _check_available(): + result = ec2_client.describe_vpc_endpoints(VpcEndpointIds=[endpoint_id]) + endpoint_details = result["VpcEndpoints"][0] + # may have multiple entries in AWS + endpoint_details["DnsEntries"] = endpoint_details["DnsEntries"][:1] + endpoint_details.pop("SubnetIds", None) + endpoint_details.pop("NetworkInterfaceIds", None) + assert endpoint_details["State"] == "available" + snapshot.match("endpoint-details", endpoint_details) + + retry(_check_available, retries=30, sleep=poll_sleep) + + # update API with VPC endpoint + patches = [ + {"op": "replace", "path": "/endpointConfiguration/types/EDGE", "value": "PRIVATE"}, + {"op": "add", "path": "/endpointConfiguration/vpcEndpointIds", "value": endpoint_id}, + ] + apigateway_client.update_rest_api(restApiId=api_id, patchOperations=patches) + + # create Lambda that invokes API via VPC endpoint (required as the endpoint is only accessible within the VPC) + subdomain = f"{api_id}-{endpoint_id}" + endpoint = api_invoke_url(subdomain, stage=DEFAULT_STAGE_NAME, path="/test") + host_header = urlparse(endpoint).netloc + + # create Lambda function that invokes the API GW (private VPC endpoint not accessible from outside of AWS) + if not is_aws_cloud(): + if config.LAMBDA_EXECUTOR == "local" and is_old_provider(): + # special case: return localhost for local Lambda executor (TODO remove after full switch to v2 provider) + api_host = LOCALHOST + else: + api_host = get_main_endpoint_from_container() + endpoint = endpoint.replace(host_header, f"{api_host}:{config.get_edge_port_http()}") + lambda_code = textwrap.dedent( + f""" + def handler(event, context): + import requests + headers = {{"content-type": "application/json", "host": "{host_header}"}} + result = requests.post("{endpoint}", headers=headers) + return {{"content": result.content.decode("utf-8"), "code": result.status_code}} + """ + ) + func_name = f"test-{short_uid()}" + vpc_config = { + "SubnetIds": subnets, + "SecurityGroupIds": [security_group], + } + create_lambda_function( + func_name=func_name, handler_file=lambda_code, timeout=10, VpcConfig=vpc_config + ) + + # create resource policy + statement = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": "*", + "Action": "execute-api:Invoke", + "Resource": ["execute-api:/*"], + } + ], + } + patches = [{"op": "replace", "path": "/policy", "value": json.dumps(statement)}] + result = apigateway_client.update_rest_api(restApiId=api_id, patchOperations=patches) + result["policy"] = json.loads(to_bytes(result["policy"]).decode("unicode_escape")) + snapshot.match("api-details", result) + + # re-deploy API + create_rest_api_deployment(apigateway_client, restApiId=api_id, stageName=DEFAULT_STAGE_NAME) + + def _invoke_api(): + result = lambda_client.invoke(FunctionName=func_name, Payload="{}") + result = json.loads(to_str(result["Payload"].read())) + items = json.loads(result["content"])["Items"] + assert len(items) == len(item_ids) + + # invoke Lambda and assert result + retry(_invoke_api, retries=15, sleep=poll_sleep) + + # TODO - remove the code below? # # def test_aws_integration_dynamodb(apigateway_client): diff --git a/tests/integration/apigateway/test_apigateway_integrations.snapshot.json b/tests/integration/apigateway/test_apigateway_integrations.snapshot.json index cdd61f637e032..37d7691c8de05 100644 --- a/tests/integration/apigateway/test_apigateway_integrations.snapshot.json +++ b/tests/integration/apigateway/test_apigateway_integrations.snapshot.json @@ -772,5 +772,80 @@ "stageVariables": null } } + }, + "tests/integration/apigateway/test_apigateway_integrations.py::test_create_execute_api_vpc_endpoint": { + "recorded-date": "18-03-2023, 22:01:10", + "recorded-content": { + "endpoint-details": { + "CreationTimestamp": "timestamp", + "DnsEntries": [ + { + "DnsName": "", + "HostedZoneId": "" + } + ], + "DnsOptions": { + "DnsRecordIpType": "ipv4" + }, + "Groups": [ + { + "GroupId": "", + "GroupName": "" + } + ], + "IpAddressType": "ipv4", + "OwnerId": "111111111111", + "PolicyDocument": { + "Statement": [ + { + "Action": "*", + "Effect": "Allow", + "Principal": "*", + "Resource": "*" + } + ] + }, + "PrivateDnsEnabled": true, + "RequesterManaged": false, + "RouteTableIds": [], + "ServiceName": "com.amazonaws..execute-api", + "State": "available", + "Tags": [], + "VpcEndpointId": "", + "VpcEndpointType": "Interface", + "VpcId": "" + }, + "api-details": { + "apiKeySource": "HEADER", + "createdDate": "datetime", + "disableExecuteApiEndpoint": false, + "endpointConfiguration": { + "types": [ + "PRIVATE" + ], + "vpcEndpointIds": [ + "" + ] + }, + "id": "", + "name": "", + "policy": { + "Statement": [ + { + "Action": "execute-api:Invoke", + "Effect": "Allow", + "Principal": "*", + "Resource": "arn:aws:execute-api::111111111111:/*" + } + ], + "Version": "2012-10-17" + }, + "tags": {}, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } } }