localstack · simonrw · Mar 20, 2023 · Feb 23, 2023 · Feb 27, 2023 · Feb 27, 2023
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -147,7 +147,7 @@ jobs:
           name: Test ASF Lambda provider
           environment:
             PROVIDER_OVERRIDE_LAMBDA: "asf"
-            TEST_PATH: "tests/integration/awslambda/test_lambda.py tests/integration/awslambda/test_lambda_api.py tests/integration/awslambda/test_lambda_common.py tests/integration/awslambda/test_lambda_integration_sqs.py tests/integration/cloudformation/resources/test_lambda.py tests/integration/awslambda/test_lambda_integration_dynamodbstreams.py tests/integration/awslambda/test_lambda_integration_kinesis.py tests/integration/awslambda/test_lambda_developer_tools.py"
+            TEST_PATH: "tests/integration/awslambda/test_lambda.py tests/integration/awslambda/test_lambda_api.py tests/integration/awslambda/test_lambda_common.py tests/integration/awslambda/test_lambda_integration_sqs.py tests/integration/cloudformation/resources/test_lambda.py tests/integration/awslambda/test_lambda_integration_dynamodbstreams.py tests/integration/awslambda/test_lambda_integration_kinesis.py tests/integration/awslambda/test_lambda_developer_tools.py tests/integration/test_network_configuration.py::TestLambda"
             PYTEST_ARGS: "--reruns 3 --junitxml=target/reports/lambda_asf.xml -o junit_suite_name='lambda_asf'"
             COVERAGE_ARGS: "-p"
           command: make test-coverage
@@ -172,7 +172,7 @@ jobs:
           name: Test ASF S3 provider
           environment:
             PROVIDER_OVERRIDE_S3: "asf"
-            TEST_PATH: "tests/integration/s3/"
+            TEST_PATH: "tests/integration/s3/ tests/integration/test_network_configuration.py::TestS3"
             PYTEST_ARGS: "--reruns 3 --junitxml=target/reports/s3_asf.xml -o junit_suite_name='s3_asf'"
             COVERAGE_ARGS: "-p"
           command: make test-coverage

@@ -42,7 +42,7 @@ concurrency:
 jobs:
   run-integration-tests:
     runs-on: ubuntu-latest
-    timeout-minutes: 110
+    timeout-minutes: 120
     defaults:
       run:
         working-directory: localstack-ext

diff --git a/localstack/services/awslambda/lambda_api.py b/localstack/services/awslambda/lambda_api.py
@@ -25,7 +25,7 @@
 
 from localstack import config, constants
 from localstack.aws.accounts import get_aws_account_id
-from localstack.constants import APPLICATION_JSON, LOCALHOST_HOSTNAME
+from localstack.constants import APPLICATION_JSON
 from localstack.http import Request
 from localstack.http import Response as HttpResponse
 from localstack.services.awslambda import lambda_executors
@@ -81,6 +81,7 @@
     now_utc,
     timestamp,
 )
+from localstack.utils.urls import localstack_host
 
 LOG = logging.getLogger(__name__)
 
@@ -1511,7 +1512,10 @@ def create_url_config(function):
 
     custom_id = md5(str(random()))
     region_name = aws_stack.get_region()
-    url = f"http://{custom_id}.lambda-url.{region_name}.{LOCALHOST_HOSTNAME}:{config.EDGE_PORT_HTTP or config.EDGE_PORT}/"
+    host_definition = localstack_host(
+        use_localhost_cloud=True, custom_port=config.EDGE_PORT_HTTP or config.EDGE_PORT
+    )
+    url = f"http://{custom_id}.lambda-url.{region_name}.{host_definition.host_and_port()}/"
     # TODO: HTTPS support
 
     data = json.loads(to_str(request.data))

diff --git a/localstack/services/awslambda/provider.py b/localstack/services/awslambda/provider.py
@@ -134,7 +134,6 @@
 
10000
    UpdateFunctionUrlConfigResponse,
     Version,
 )
-from localstack.constants import LOCALHOST_HOSTNAME
 from localstack.services.awslambda import api_utils
 from localstack.services.awslambda import hooks as lambda_hooks
 from localstack.services.awslambda.api_utils import STATEMENT_ID_REGEX
@@ -193,6 +192,7 @@
 from localstack.utils.files import load_file
 from localstack.utils.strings import get_random_hex, long_uid, short_uid, to_bytes, to_str
 from localstack.utils.sync import poll_condition
+from localstack.utils.urls import localstack_host
 
 LOG = logging.getLogger(__name__)
 
@@ -1632,12 +1632,16 @@ def create_function_url_config(
 
         # create function URL config
         url_id = api_utils.generate_random_url_id()
+
+        host_definition = localstack_host(
+            use_localhost_cloud=True, custom_port=config.EDGE_PORT_HTTP or config.EDGE_PORT
+        )
         fn.function_url_configs[normalized_qualifier] = FunctionUrlConfig(
             function_arn=function_arn,
             function_name=function_name,
             cors=cors,
             url_id=url_id,
-            url=f"http://{url_id}.lambda-url.{context.region}.{LOCALHOST_HOSTNAME}:{config.EDGE_PORT_HTTP or config.EDGE_PORT}/",  # TODO: https support
+            url=f"http://{url_id}.lambda-url.{context.region}.{host_definition.host_and_port()}/",  # TODO: https support
             auth_type=auth_type,
             creation_time=api_utils.generate_lambda_date(),
             last_modified_time=api_utils.generate_lambda_date(),

diff --git a/localstack/services/opensearch/cluster_manager.py b/localstack/services/opensearch/cluster_manager.py
@@ -8,7 +8,7 @@
 from localstack import config
 from localstack.aws.api.opensearch import DomainEndpointOptions, EngineType
 from localstack.config import EDGE_BIND_HOST
-from localstack.constants import LOCALHOST, LOCALHOST_HOSTNAME
+from localstack.constants import LOCALHOST
 from localstack.services.opensearch import versions
 from localstack.services.opensearch.cluster import (
     CustomEndpoint,
@@ -28,6 +28,7 @@
     start_thread,
 )
 from localstack.utils.serving import Server
+from localstack.utils.urls import localstack_host
 
 LOG = logging.getLogger(__name__)
 
@@ -115,11 +116,16 @@ def build_cluster_endpoint(
                 assigned_port = external_service_ports.reserve_port()
         else:
             assigned_port = external_service_ports.reserve_port()
-        return f"{config.LOCALSTACK_HOSTNAME}:{assigned_port}"
+
+        host_definition = localstack_host(use_localstack_hostname=True, custom_port=assigned_port)
+        return host_definition.host_and_port()
     if config.OPENSEARCH_ENDPOINT_STRATEGY == "path":
-        return f"{config.LOCALSTACK_HOSTNAME}:{config.EDGE_PORT}/{engine_domain}/{domain_key.region}/{domain_key.domain_name}"
+        host_definition = localstack_host(use_localstack_hostname=True)
+        return f"{host_definition.host_and_port()}/{engine_domain}/{domain_key.region}/{domain_key.domain_name}"
+
     # or through a subdomain (domain-name.region.opensearch.localhost.localstack.cloud)
-    return f"{domain_key.domain_name}.{domain_key.region}.{engine_domain}.{LOCALHOST_HOSTNAME}:{config.EDGE_PORT}"
+    host_definition = localstack_host(use_localhost_cloud=True)
+    return f"{domain_key.domain_name}.{domain_key.region}.{engine_domain}.{host_definition.host_and_port()}"
 
 
 def determine_custom_endpoint(

diff --git a/localstack/services/s3/provider.py b/localstack/services/s3/provider.py
@@ -94,7 +94,6 @@
     preprocess_request,
     serve_custom_service_request_handlers,
 )
-from localstack.constants import LOCALHOST_HOSTNAME
 from localstack.services.edge import ROUTER
 from localstack.services.moto import call_moto
 from localstack.services.plugins import ServiceLifecycleHook
@@ -130,6 +129,7 @@
 from localstack.utils.collections import get_safe
 from localstack.utils.patch import patch
 from localstack.utils.strings import short_uid
+from localstack.utils.urls import localstack_host
 
 LOG = logging.getLogger(__name__)
 
@@ -166,8 +166,13 @@ def __init__(self, message=None):
 
 def get_full_default_bucket_location(bucket_name):
     if config.HOSTNAME_EXTERNAL != config.LOCALHOST:
-        return f"{config.get_protocol()}://{config.HOSTNAME_EXTERNAL}:{config.get_edge_port_http()}/{bucket_name}/"
-    return f"{config.get_protocol()}://{bucket_name}.s3.{LOCALHOST_HOSTNAME}:{config.get_edge_port_http()}/"
+        host_definition = localstack_host(
+            use_hostname_external=True, custom_port=config.get_edge_port_http()
+        )
+        return f"{config.get_protocol()}://{host_definition.host_and_port()}/{bucket_name}/"
+    else:
+        host_definition = localstack_host(use_localhost_cloud=True)
+        return f"{config.get_protocol()}://{bucket_name}.s3.{host_definition.host_and_port()}/"
 
 
 class S3Provider(S3Api, ServiceLifecycleHook):

diff --git a/localstack/services/s3/s3_listener.py b/localstack/services/s3/s3_listener.py
@@ -64,6 +64,7 @@
     to_str,
 )
 from localstack.utils.time import timestamp_millis
+from localstack.utils.urls import localstack_host
 from localstack.utils.xml import strip_xmlns
 
 # backend port (configured in s3_starter.py on startup)
@@ -1346,6 +1347,7 @@ def is_multipart_upload(query):
 
     @staticmethod
     def get_201_response(key, bucket_name):
+        host_definition = localstack_host(use_hostname_external=True)
         return """
                 <PostResponse>
                     <Location>{protocol}://{host}/{encoded_key}</Location>
@@ -1355,7 +1357,7 @@ def get_201_response(key, bucket_name):
                 </PostResponse>
                 """.format(
             protocol=get_service_protocol(),
-            host=config.HOSTNAME_EXTERNAL,
+            host=host_definition.host,
             encoded_key=quote(key, safe=""),
             key=key,
             bucket=bucket_name,
@@ -1366,12 +1368,13 @@ def get_201_response(key, bucket_name):
     def _update_location(content, bucket_name):
         bucket_name = normalize_bucket_name(bucket_name)
 
-        host = config.HOSTNAME_EXTERNAL
-        if ":" not in host:
-            host = f"{host}:{config.service_port('s3')}"
+        host_definition = localstack_host(
+            use_hostname_external=True, custom_port=config.get_edge_port_http()
+        )
         return re.sub(
             r"<Location>\s*([a-zA-Z0-9\-]+)://[^/]+/([^<]+)\s*</Location>",
-            r"<Location>%s://%s/%s/\2</Location>" % (get_service_protocol(), host, bucket_name),
+            r"<Location>%s://%s/%s/\2</Location>"
+            % (get_service_protocol(), host_definition.host_and_port(), bucket_name),
             content,
             flags=re.MULTILINE,
         )

diff --git a/localstack/services/sqs/models.py b/localstack/services/sqs/models.py
@@ -9,7 +9,7 @@
 from queue import PriorityQueue
 from typing import Dict, NamedTuple, Optional, Set
 
-from localstack import config, constants
+from localstack import config
 from localstack.aws.api import RequestContext
 from localstack.aws.api.sqs import (
     InvalidAttributeName,
@@ -21,7 +21,7 @@
     ReceiptHandleIsInvalid,
     TagMap,
 )
-from localstack.config import external_service_url
+from localstack.config import get_protocol
 from localstack.services.sqs import constants as sqs_constants
 from localstack.services.sqs.exceptions import (
     InvalidAttributeValue,
@@ -35,6 +35,7 @@
 )
 from localstack.services.stores import AccountRegionBundle, BaseStore, LocalAttribute
 from localstack.utils.time import now
+from localstack.utils.urls import localstack_host
 
 LOG = logging.getLogger(__name__)
 
@@ -249,13 +250,18 @@ def url(self, context: RequestContext) -> str:
             # or us-east-2.queue.localhost.localstack.cloud:4566/000000000000/my-queue
             region = "" if self.region == "us-east-1" else self.region + "."
             scheme = context.request.scheme
-            host_url = f"{scheme}://{region}queue.{constants.LOCALHOST_HOSTNAME}:{config.EDGE_PORT}"
+
+            host_definition = localstack_host(use_localhost_cloud=True)
+            host_url = f"{scheme}://{region}queue.{host_definition.host_and_port()}"
         elif config.SQS_ENDPOINT_STRATEGY == "path":
             # https?://localhost:4566/queue/us-east-1/00000000000/my-queue (us-east-1)
             host_url = f"{context.request.host_url}queue/{self.region}"
         else:
             if config.SQS_PORT_EXTERNAL:
-                host_url = external_service_url("sqs")
+                host_definition = localstack_host(
+                    use_hostname_external=True, custom_port=config.SQS_PORT_EXTERNAL
+                )
+                host_url = f"{get_protocol()}://{host_definition.host_and_port()}"
 
         return "{host}/{account_id}/{name}".format(
             host=host_url.rstrip("/"),

diff --git a/localstack/testing/pytest/fixtures.py b/localstack/testing/pytest/fixtures.py
@@ -4,6 +4,7 @@
 import logging
 import os
 import re
+import socket
 import time
 from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple
 
@@ -21,7 +22,7 @@
 from pytes
F438
t_httpserver import HTTPServer
 from werkzeug import Request, Response
 
-from localstack import config
+from localstack import config, constants
 from localstack.aws.accounts import get_aws_account_id
 from localstack.constants import TEST_AWS_ACCESS_KEY_ID, TEST_AWS_SECRET_ACCESS_KEY
 from localstack.services.stores import (
@@ -2006,6 +2007,69 @@ def factory(**kwargs):
             LOG.debug(f"Error cleaning up AppSync API: {api}, {e}")
 
 
+@pytest.fixture
+def assert_host_customisation(monkeypatch):
+    hostname_external = f"external-host-{short_uid()}"
+    # `LOCALSTACK_HOSTNAME` is really an internal variable that has been
+    # exposed to the user at some point in the past. It is used by some
+    # services that start resources (e.g. OpenSearch) to determine if the
+    # service has been started correctly (i.e. a health check). This means that
+    # the value must be resolvable by LocalStack or else the service resources
+    # won't start properly.
+    #
+    # One hostname that's always resolvable is the hostname of the process
+    # running LocalStack, so use that here.
+    #
+    # Note: We cannot use `localhost` since we explicitly check that the URL
+    # passed in does not contain `localhost`, unless it is requried to.
+    localstack_hostname = socket.gethostname()
+    monkeypatch.setattr(config, "HOSTNAME_EXTERNAL", hostname_external)
+    monkeypatch.setattr(config, "LOCALSTACK_HOSTNAME", localstack_hostname)
+
+    def asserter(
+        url: str,
+        *,
+        use_hostname_external: bool = False,
+        use_localstack_hostname: bool = False,
+        use_localstack_cloud: bool = False,
+        use_localhost: bool = False,
+        custom_host: Optional[str] = None,
+    ):
+        if use_hostname_external:
+            assert hostname_external in url
+
+            assert localstack_hostname not in url
+            assert constants.LOCALHOST_HOSTNAME not in url
+            assert constants.LOCALHOST not in url
+        elif use_localstack_hostname:
+            assert localstack_hostname in url
+
+            assert hostname_external not in url
+            assert constants.LOCALHOST_HOSTNAME not in url
+            assert constants.LOCALHOST not in url
+        elif use_localstack_cloud:
+            assert constants.LOCALHOST_HOSTNAME in url
+
+            assert hostname_external not in url
+            assert localstack_hostname not in url
+        elif use_localhost:
+            assert constants.LOCALHOST in url
+
+            assert constants.LOCALHOST_HOSTNAME not in url
+            assert hostname_external not in url
+            assert localstack_hostname not in url
+        elif custom_host is not None:
+            assert custom_host in url
+
+            assert constants.LOCALHOST_HOSTNAME not in url
+            assert hostname_external not in url
+            assert localstack_hostname not in url
+        else:
+            raise ValueError("no assertions made")
+
+    yield asserter
+
+
 @pytest.fixture
 def echo_http_server(httpserver: HTTPServer):
     """Spins up a local HTTP echo server and returns the endpoint URL"""

diff --git a/localstack/utils/urls.py b/localstack/utils/urls.py
@@ -1,6 +1,47 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from localstack import config, constants
+
+
 def path_from_url(url: str) -> str:
     return f'/{url.partition("://")[2].partition("/")[2]}' if "://" in url else url
 
 
 def hostname_from_url(url: str) -> str:
     return url.split("://")[-1].split("/")[0].split(":")[0]
+
+
+@dataclass
+class HostDefinition:
+    host: str
+    port: int
+
+    def host_and_port(self):
+        return f"{self.host}:{self.port}"
+
+
+def localstack_host(
+    use_hostname_external: bool = False,
+    use_localstack_hostname: bool = False,
+    use_localhost_cloud: bool = False,
+    custom_port: Optional[int] = None,
+) -> HostDefinition:
+    """
+    Determine the host and port to return to the user based on:
+    - the user's configuration (e.g environment variable overrides)
+    - the defaults of the system
+    """
+    port = config.EDGE_PORT
+    if custom_port is not None:
+        port = custom_port
+
+    host = config.LOCALHOST
+    if use_hostname_external:
+        host = config.HOSTNAME_EXTERNAL
+    elif use_localstack_hostname:
+        host = config.LOCALSTACK_HOSTNAME
+    elif use_localhost_cloud:
+        host = constants.LOCALHOST_HOSTNAME
+
+    return HostDefinition(host=host, port=port)