localstack · simonrw · Mar 20, 2023 · Feb 23, 2023 · Feb 27, 2023 · Feb 27, 2023
diff --git a/localstack/services/awslambda/lambda_api.py b/localstack/services/awslambda/lambda_api.py
@@ -25,7 +25,7 @@
 
 from localstack import config, constants
 from localstack.aws.accounts import get_aws_account_id
-from localstack.constants import APPLICATION_JSON, LOCALHOST_HOSTNAME
 from localstack.constants import APPLICATION_JSON
 from localstack.http import Request
 from localstack.http import Response as HttpResponse
 from localstack.services.awslambda import lambda_executors
@@ -81,6 +81,7 @@
     now_utc,
     timestamp,
 )
+from localstack.utils.urls import localstack_host
 
 LOG = logging.getLogger(__name__)
 
@@ -1511,7 +1512,10 @@ def create_url_config(function):
 
     custom_id = md5(str(random()))
     region_name = aws_stack.get_region()
-    url = f"http://{custom_id}.lambda-url.{region_name}.{LOCALHOST_HOSTNAME}:{config.EDGE_PORT_HTTP or config.EDGE_PORT}/"
+    host_definition = localstack_host(
+        use_localhost_cloud=True, custom_port=config.EDGE_PORT_HTTP or config.EDGE_PORT
+    )
+    url = f"http://{custom_id}.lambda-url.{region_name}.{host_definition.host_and_port()}/"
     # TODO: HTTPS support
 
     data = json.loads(to_str(request.data))

diff --git a/localstack/services/awslambda/provider.py b/localstack/services/awslambda/provider.py
@@ -134,7 +134,6 @@
     UpdateFunctionUrlConfigResponse,
     Version,
 )
-from localstack.constants import LOCALHOST_HOSTNAME
 from localstack.services.awslambda import api_utils
 from localstack.services.awslambda import hooks as lambda_hooks
 from localstack.services.awslambda.api_utils import STATEMENT_ID_REGEX
@@ -193,6 +192,7 @@
 from localstack.utils.files import load_file
 from localstack.utils.strings import get_random_hex, long_uid, short_uid, to_bytes, to_str
 from localstack.utils.sync import poll_condition
+from localstack.utils.urls import localstack_host
 
 LOG = logging.getLogger(__name__)
 
@@ -1632,12 +1632,16 @@ def create_function_url_config(
 
         # create function URL config
         url_id = api_utils.generate_random_url_id()
+
+        host_definition = localstack_host(
+            use_localhost_cloud=True, custom_port=config.EDGE_PORT_HTTP or config.EDGE_PORT
+        )
         fn.function_url_configs[normalized_qualifier] = FunctionUrlConfig(
             function_arn=function_arn,
             function_name=function_name,
             cors=cors,
             url_id=url_id,
-            url=f"http://{url_id}.lambda-url.{context.region}.{LOCALHOST_HOSTNAME}:{config.EDGE_PORT_HTTP or config.EDGE_PORT}/",  # TODO: https support
+            url=f"http://{url_id}.lambda-url.{context.region}.{host_definition.host_and_port()}/",  # TODO: https support
             auth_type=auth_type,
             creation_time=api_utils.generate_lambda_date(),
             last_modified_time=api_utils.generate_lambda_date(),

@@ -8,7 +8,7 @@
 from localstack import config
 from localstack.aws.api.opensearch import DomainEndpointOptions, EngineType
 from localstack.config import EDGE_BIND_HOST
-from localstack.constants import LOCALHOST, LOCALHOST_HOSTNAME
+from localstack.constants import LOCALHOST
 from localstack.services.opensearch import versions
 from localstack.services.opensearch.cluster import (
     CustomEndpoint,
@@ -28,6 +28,7 @@
     start_thread,
 )
 from localstack.utils.serving import Server
+from localstack.utils.urls import localstack_host
 
 LOG = logging.getLogger(__name__)
 
@@ -115,11 +116,16 @@ def build_cluster_endpoint(
                 assigned_port = external_service_ports.reserve_port()
         else:
             assigned_port = external_service_ports.reserve_port()
-        return f"{config.LOCALSTACK_HOSTNAME}:{assigned_port}"
+
+        host_definition = localstack_host(use_localstack_hostname=True, custom_port=assigned_port)
+        return host_definition.host_and_port()
     if config.OPENSEARCH_ENDPOINT_STRATEGY == "path":
-        return f"{config.LOCALSTACK_HOSTNAME}:{config.EDGE_PORT}/{engine_domain}/{domain_key.region}/{domain_key.domain_name}"
+        host_definition = localstack_host(use_localstack_hostname=True)
+        return f"{host_definition.host_and_port()}/{engine_domain}/{domain_key.region}/{domain_key.domain_name}"
+
     # or through a subdomain (domain-name.region.opensearch.localhost.localstack.cloud)
-    return f"{domain_key.domain_name}.{domain_key.region}.{engine_domain}.{LOCALHOST_HOSTNAME}:{config.EDGE_PORT}"
+    host_definition = localstack_host(use_localhost_cloud=True)
+    return f"{domain_key.domain_name}.{domain_key.region}.{engine_domain}.{host_definition.host_and_port()}"
 
 
 def determine_custom_endpoint(

@@ -94,7 +94,6 @@
     preprocess_request,
     serve_custom_service_request_handlers,
 )
-from localstack.constants import LOCALHOST_HOSTNAME
 from localstack.services.edge import ROUTER
 from localstack.services.moto import call_moto
 from localstack.services.plugins import ServiceLifecycleHook
@@ -130,6 +129,7 @@
 from localstack.utils.collections import get_safe
 from localstack.utils.patch import patch
 from localstack.utils.strings import short_uid
+from localstack.utils.urls import localstack_host
 
 LOG = logging.getLogger(__name__)
 
@@ -166,8 +166,11 @@ def __init__(self, message=None):
 
 def get_full_default_bucket_location(bucket_name):
     if config.HOSTNAME_EXTERNAL != config.LOCALHOST:
-        return f"{config.get_protocol()}://{config.HOSTNAME_EXTERNAL}:{config.get_edge_port_http()}/{bucket_name}/"
-    return f"{config.get_protocol()}://{bucket_name}.s3.{LOCALHOST_HOSTNAME}:{config.get_edge_port_http()}/"
+        host_definition = localstack_host(use_hostname_external=True)
+        return f"{config.get_protocol()}://{host_definition.host_and_port()}/{bucket_name}/"
+    else:
+        host_definition = localstack_host(use_localhost_cloud=True)
+        return f"{config.get_protocol()}://{bucket_name}.s3.{host_definition.host_and_port()}/"
 
 
 class S3Provider(S3Api, ServiceLifecycleHook):

@@ -64,6 +64,7 @@
     to_str,
 )
 from localstack.utils.time import timestamp_millis
+from localstack.utils.urls import localstack_host
 from localstack.utils.xml import strip_xmlns
 
 # backend port (configured in s3_starter.py on startup)
@@ -1346,6 +1347,7 @@ def is_multipart_upload(query):
 
     @staticmethod
     def get_201_response(key, bucket_name):
+        host_definition = localstack_host(use_hostname_external=True)
         return """
                 <PostResponse>
                     <Location>{protocol}://{host}/{encoded_key}</Location>
@@ -1355,7 +1357,7 @@ def get_201_response(key, bucket_name):
                 </PostResponse>
                 """.format(
             protocol=get_service_protocol(),
-            host=config.HOSTNAME_EXTERNAL,
+            host=host_definition.host,
             encoded_key=quote(key, safe=""),
             key=key,
             bucket=bucket_name,

@@ -3,23 +3,24 @@
 from urllib.parse import urlsplit, urlunsplit
 
 from localstack.config import LEGACY_S3_PROVIDER
-from localstack.constants import LOCALHOST_HOSTNAME
+from localstack.constants import LOCALHOST, LOCALHOST_HOSTNAME
 from localstack.http import Request, Response
 from localstack.http.proxy import Proxy
 from localstack.runtime import hooks
 from localstack.services.edge import ROUTER
 from localstack.services.s3.utils import S3_VIRTUAL_HOST_FORWARDED_HEADER
 from localstack.utils.aws.request_context import AWS_REGION_REGEX
+from localstack.utils.urls import localstack_host
 
 LOG = logging.getLogger(__name__)
 
 # virtual-host style: https://{bucket-name}.s3.{region}.localhost.localstack.cloud.com/{key-name}
-VHOST_REGEX_PATTERN = f"<regex('.*'):bucket>.s3.<regex('({AWS_REGION_REGEX}\\.)?'):region>{LOCALHOST_HOSTNAME}<regex('(?::\\d+)?'):port>"
+VHOST_REGEX_PATTERN = "<regex('.*'):bucket>.s3.<regex('({aws_region_regex}\\.)?'):region>{hostname}<regex('(?::\\d+)?'):port>"
 
 # path addressed request with the region in the hostname
 # https://s3.{region}.localhost.localstack.cloud.com/{bucket-name}/{key-name}
 PATH_WITH_REGION_PATTERN = (
-    f"s3.<regex('({AWS_REGION_REGEX}\\.)'):region>{LOCALHOST_HOSTNAME}<regex('(?::\\d+)?'):port>"
+    "s3.<regex('({aws_region_regex}\\.)'):region>{hostname}<regex('(?::\\d+)?'):port>"
 )
 
 
@@ -32,7 +33,6 @@ class S3VirtualHostProxyHandler:
     def __call__(self, request: Request, **kwargs) -> Response:
         # TODO region pattern currently not working -> removing it from url
         rewritten_url = self._rewrite_url(request.url, kwargs.get("bucket"), kwargs.get("region"))
-
         LOG.debug(f"Rewritten original host url: {request.url} to path-style url: {rewritten_url}")
 
         forward_to_url = urlsplit(rewritten_url)
@@ -62,6 +62,10 @@ def _rewrite_url(url: str, bucket: str, region: str) -> str:
 
         If the region is contained in the host-name we remove it (for now) as moto cannot handle the region correctly
 
+        If the url contains a customised hostname, for example if the user sets `HOSTNAME_EXTERNAL` then re-write the
+        host to localhost.localstack.cloud since the request is coming from inside LocalStack itself, and `HOSTNAME_EXTERNAL`
+        may not be resolvable.
+
         :param url: the original url
         :param bucket: the bucket name
         :param region: the region name
@@ -79,6 +83,13 @@ def _rewrite_url(url: str, bucket: str, region: str) -> str:
         if region:
             netloc = netloc.replace(f"{region}", "")
 
+        # if the user specifies a custom hostname for LocalStack, this name may not be resolvable by
+        # LocalStack. We are proxying the request to ourself, so replace their custom hostname with
+        # `localhost.localstack.cloud` which also matches the PATH matchers.
+        host_definition = localstack_host(use_hostname_external=True)
+        if host_definition.host != LOCALHOST:
+            netloc = netloc.replace(host_definition.host, LOCALHOST_HOSTNAME)
+
         return urlunsplit((splitted.scheme, netloc, path, splitted.query, splitted.fragment))
 
 
@@ -89,28 +100,83 @@ def register_virtual_host_routes():
 
     """
     s3_proxy_handler = S3VirtualHostProxyHandler()
+    host_definition = localstack_host(use_hostname_external=True)
+    # Add additional routes if the user specifies a custom HOSTNAME_EXTERNAL
+    # as we should match on these routes, and also match on localhost.localstack.cloud
+    # to maintain backwards compatibility.
+    if host_definition.host != LOCALHOST:
+        ROUTER.add(
+            path="/",
+            host=VHOST_REGEX_PATTERN.format(
+                aws_region_regex=AWS_REGION_REGEX,
+                hostname=host_definition.host,
+            ),
+            endpoint=s3_proxy_handler,
+            defaults={"path": "/"},
+        )
+
+        ROUTER.add(
+            path="/<path:path>",
+            host=VHOST_REGEX_PATTERN.format(
+                aws_region_regex=AWS_REGION_REGEX,
+                hostname=host_definition.host,
+            ),
+            endpoint=s3_proxy_handler,
+        )
+
+        ROUTER.add(
+            path="/<regex('.+'):bucket>",
+            host=PATH_WITH_REGION_PATTERN.format(
+                aws_region_regex=AWS_REGION_REGEX,
+                hostname=host_definition.host,
+            ),
+            endpoint=s3_proxy_handler,
+            defaults={"path": "/"},
+        )
+
+        ROUTER.add(
+            path="/<regex('.+'):bucket>/<path:path>",
+            host=PATH_WITH_REGION_PATTERN.format(
+                aws_region_regex=AWS_REGION_REGEX,
+                hostname=host_definition.host,
+            ),
+            endpoint=s3_proxy_handler,
+        )
+
     ROUTER.add(
         path="/",
-        host=VHOST_REGEX_PATTERN,
+        host=VHOST_REGEX_PATTERN.format(
+            aws_region_regex=AWS_REGION_REGEX,
+            hostname=LOCALHOST_HOSTNAME,
+        ),
         endpoint=s3_proxy_handler,
         defaults={"path": "/"},
     )
 
     ROUTER.add(
         path="/<path:path>",
-        host=VHOST_REGEX_PATTERN,
+        host=VHOST_REGEX_PATTERN.format(
+            aws_region_regex=AWS_REGION_REGEX,
+            hostname=LOCALHOST_HOSTNAME,
+        ),
         endpoint=s3_proxy_handler,
     )
 
     ROUTER.add(
         path="/<regex('.+'):bucket>",
-        host=PATH_WITH_REGION_PATTERN,
+        host=PATH_WITH_REGION_PATTERN.format(
+            aws_region_regex=AWS_REGION_REGEX,
+            hostname=LOCALHOST_HOSTNAME,
+        ),
         endpoint=s3_proxy_handler,
         defaults={"path": "/"},
     )
 
     ROUTER.add(
         path="/<regex('.+'):bucket>/<path:path>",
-        host=PATH_WITH_REGION_PATTERN,
+        host=PATH_WITH_REGION_PATTERN.format(
+            aws_region_regex=AWS_REGION_REGEX,
+            hostname=LOCALHOST_HOSTNAME,
+        ),
         endpoint=s3_proxy_handler,
     )
@@ -9,7 +9,7 @@
 from queue import PriorityQueue
 from typing import Dict, NamedTuple, Optional, Set
 
-from localstack import config, constants
+from localstack import config
 from localstack.aws.api import RequestContext
 from localstack.aws.api.sqs import (
     InvalidAttributeName,
@@ -21,7 +21,7 @@
     ReceiptHandleIsInvalid,
     TagMap,
 )
-from localstack.config import external_service_url
+from localstack.config import get_protocol
 from localstack.services.sqs import constants as sqs_constants
 from localstack.services.sqs.exceptions import (
     InvalidAttributeValue,
@@ -35,6 +35,7 @@
 )
 from localstack.services.stores import AccountRegionBundle, BaseStore, LocalAttribute
 from localstack.utils.time import now
+from localstack.utils.urls import localstack_host
 
 LOG = logging.getLogger(__name__)
 
@@ -249,13 +250,18 @@ def url(self, context: RequestContext) -> str:
             # or us-east-2.queue.localhost.localstack.cloud:4566/000000000000/my-queue
             region = "" if self.region == "us-east-1" else self.region + "."
             scheme = context.request.scheme
-            host_url = f"{scheme}://{region}queue.{constants.LOCALHOST_HOSTNAME}:{config.EDGE_PORT}"
+
+            host_definition = localstack_host(use_localhost_cloud=True)
+            host_url = f"{scheme}://{region}queue.{host_definition.host_and_port()}"
         elif config.SQS_ENDPOINT_STRATEGY == "path":
             # https?://localhost:4566/queue/us-east-1/00000000000/my-queue (us-east-1)
             host_url = f"{context.request.host_url}/queue/{self.region}"
         else:
             if config.SQS_PORT_EXTERNAL:
-                host_url = external_service_url("sqs")
+                host_definition = localstack_host(
+                    use_hostname_external=True, custom_port=config.SQS_PORT_EXTERNAL
+                )
+                host_url = f"{get_protocol()}://{host_definition.host_and_port()}"
 
         return "{host}/{account_id}/{name}".format(
             host=host_url.rstrip("/"),