8000 Unify hostnames in returned URLs (#7774) · localstack/localstack@3b41e13 · GitHub
[go: up one dir, main page]
Skip to content

Commit 3b41e13

Browse files
simonrwsimonrw
authored
Unify hostnames in returned URLs (#7774)
This patch adds a series of tests that capture what we currently do regarding returning URLs to the user.
1 parent 38d1403 commit 3b41e13

File tree

11 files changed

+390
-24
lines changed

11 files changed

+390
-24
lines changed

‎.circleci/config.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ jobs:
147147
name: Test ASF Lambda provider
148148
environment:
149149
PROVIDER_OVERRIDE_LAMBDA: "asf"
150-
TEST_PATH: "tests/integration/awslambda/test_lambda.py tests/integration/awslambda/test_lambda_api.py tests/integration/awslambda/test_lambda_common.py tests/integration/awslambda/test_lambda_integration_sqs.py tests/integration/cloudformation/resources/test_lambda.py tests/integration/awslambda/test_lambda_integration_dynamodbstreams.py tests/integration/awslambda/test_lambda_integration_kinesis.py tests/integration/awslambda/test_lambda_developer_tools.py"
150+
TEST_PATH: "tests/integration/awslambda/test_lambda.py tests/integration/awslambda/test_lambda_api.py tests/integration/awslambda/test_lambda_common.py tests/integration/awslambda/test_lambda_integration_sqs.py tests/integration/cloudformation/resources/test_lambda.py tests/integration/awslambda/test_lambda_integration_dynamodbstreams.py tests/integration/awslambda/test_lambda_integration_kinesis.py tests/integration/awslambda/test_lambda_developer_tools.py tests/integration/test_network_configuration.py::TestLambda"
151151
PYTEST_ARGS: "--reruns 3 --junitxml=target/reports/lambda_asf.xml -o junit_suite_name='lambda_asf'"
152152
COVERAGE_ARGS: "-p"
153153
command: make test-coverage
@@ -172,7 +172,7 @@ jobs:
172172
name: Test ASF S3 provider
173173
environment:
174174
PROVIDER_OVERRIDE_S3: "asf"
175-
TEST_PATH: "tests/integration/s3/"
175+
TEST_PATH: "tests/integration/s3/ tests/integration/test_network_configuration.py::TestS3"
176176
PYTEST_ARGS: "--reruns 3 --junitxml=target/reports/s3_asf.xml -o junit_suite_name='s3_asf'"
177177
COVERAGE_ARGS: "-p"
178178
command: make test-coverage

‎.github/workflows/pro-integration.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ concurrency:
4242
jobs:
4343
run-integration-tests:
4444
runs-on: ubuntu-latest
45-
timeout-minutes: 110
45+
timeout-minutes: 120
4646
defaults:
4747
run:
4848
working-directory: localstack-ext

‎localstack/services/awslambda/lambda_api.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525

2626
from localstack import config, constants
2727
from localstack.aws.accounts import get_aws_account_id
28-
from localstack.constants import APPLICATION_JSON, LOCALHOST_HOSTNAME
28+
from localstack.constants import APPLICATION_JSON
2929
from localstack.http import Request
3030
from localstack.http import Response as HttpResponse
3131
from localstack.services.awslambda import lambda_executors
@@ -81,6 +81,7 @@
8181
now_utc,
8282
timestamp,
8383
)
84+
from localstack.utils.urls import localstack_host
8485

8586
LOG = logging.getLogger(__name__)
8687

@@ -1511,7 +1512,10 @@ def create_url_config(function):
15111512

15121513
custom_id = md5(str(random()))
15131514
region_name = aws_stack.get_region()
1514-
url = f"http://{custom_id}.lambda-url.{region_name}.{LOCALHOST_HOSTNAME}:{config.EDGE_PORT_HTTP or config.EDGE_PORT}/"
1515+
host_definition = localstack_host(
1516+
use_localhost_cloud=True, custom_port=config.EDGE_PORT_HTTP or config.EDGE_PORT
1517+
)
1518+
url = f"http://{custom_id}.lambda-url.{region_name}.{host_definition.host_and_port()}/"
15151519
# TODO: HTTPS support
15161520

15171521
data = json.loads(to_str(request.data))

‎localstack/services/awslambda/provider.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,6 @@
134134
UpdateFunctionUrlConfigResponse,
135135
Version,
136136
)
137-
from localstack.constants import LOCALHOST_HOSTNAME
138137
from localstack.services.awslambda import api_utils
139138
from localstack.services.awslambda import hooks as lambda_hooks
140139
from localstack.services.awslambda.api_utils import STATEMENT_ID_REGEX
@@ -193,6 +192,7 @@
193192
from localstack.utils.files import load_file
194193
from localstack.utils.strings import get_random_hex, long_uid, short_uid, to_bytes, to_str
195194
from localstack.utils.sync import poll_condition
195+
from localstack.utils.urls import localstack_host
196196

197197
LOG = logging.getLogger(__name__)
198198

@@ -1632,12 +1632,16 @@ def create_function_url_config(
16321632

16331633
# create function URL config
16341634
url_id = api_utils.generate_random_url_id()
1635+
1636+
host_definition = localstack_host(
1637+
use_localhost_cloud=True, custom_port=config.EDGE_PORT_HTTP or config.EDGE_PORT
1638+
)
16351639
fn.function_url_configs[normalized_qualifier] = FunctionUrlConfig(
16361640
function_arn=function_arn,
16371641
function_name=function_name,
16381642
cors=cors,
16391643
url_id=url_id,
1640-
url=f"http://{url_id}.lambda-url.{context.region}.{LOCALHOST_HOSTNAME}:{config.EDGE_PORT_HTTP or config.EDGE_PORT}/", # TODO: https support
1644+
url=f"http://{url_id}.lambda-url.{context.region}.{host_definition.host_and_port()}/", # TODO: https support
16411645
auth_type=auth_type,
16421646
creation_time=api_utils.generate_lambda_date(),
16431647
last_modified_time=api_utils.generate_lambda_date(),

‎localstack/services/opensearch/cluster_manager.py

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from localstack import config
99
from localstack.aws.api.opensearch import DomainEndpointOptions, EngineType
1010
from localstack.config import EDGE_BIND_HOST
11-
from localstack.constants import LOCALHOST, LOCALHOST_HOSTNAME
11+
from localstack.constants import LOCALHOST
1212
from localstack.services.opensearch import versions
1313
from localstack.services.opensearch.cluster import (
1414
CustomEndpoint,
@@ -28,6 +28,7 @@
2828
start_thread,
2929
)
3030
from localstack.utils.serving import Server
31+
from localstack.utils.urls import localstack_host
3132

3233
LOG = logging.getLogger(__name__)
3334

@@ -115,11 +116,16 @@ def build_cluster_endpoint(
115116
assigned_port = external_service_ports.reserve_port()
116117
else:
117118
assigned_port = external_service_ports.reserve_port()
118-
return f"{config.LOCALSTACK_HOSTNAME}:{assigned_port}"
119+
120+
host_definition = localstack_host(use_localstack_hostname=True, custom_port=assigned_port)
121+
return host_definition.host_and_port()
119122
if config.OPENSEARCH_ENDPOINT_STRATEGY == "path":
120-
return f"{config.LOCALSTACK_HOSTNAME}:{config.EDGE_PORT}/{engine_domain}/{domain_key.region}/{domain_key.domain_name}"
123+
host_definition = localstack_host(use_localstack_hostname=True)
124+
return f"{host_definition.host_and_port()}/{engine_domain}/{domain_key.region}/{domain_key.domain_name}"
125+
121126
# or through a subdomain (domain-name.region.opensearch.localhost.localstack.cloud)
122-
return f"{domain_key.domain_name}.{domain_key.region}.{engine_domain}.{LOCALHOST_HOSTNAME}:{config.EDGE_PORT}"
127+
host_definition = localstack_host(use_localhost_cloud=True)
128+
return f"{domain_key.domain_name}.{domain_key.region}.{engine_domain}.{host_definition.host_and_port()}"
123129

124130

125131
def determine_custom_endpoint(

‎localstack/services/s3/provider.py

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,6 @@
9494
preprocess_request,
9595
serve_custom_service_request_handlers,
9696
)
97-
from localstack.constants import LOCALHOST_HOSTNAME
9897
from localstack.services.edge import ROUTER
9998
from localstack.services.moto import call_moto
10099
from localstack.services.plugins import ServiceLifecycleHook
@@ -130,6 +129,7 @@
130129
from localstack.utils.collections import get_safe
131130
from localstack.utils.patch import patch
132131
from localstack.utils.strings import short_uid
132+
from localstack.utils.urls import localstack_host
133133

134134
LOG = logging.getLogger(__name__)
135135

@@ -166,8 +166,13 @@ def __init__(self, message=None):
166166

167167
def get_full_default_bucket_location(bucket_name):
168168
if config.HOSTNAME_EXTERNAL != config.LOCALHOST:
169-
return f"{config.get_protocol()}://{config.HOSTNAME_EXTERNAL}:{config.get_edge_port_http()}/{bucket_name}/"
170-
return f"{config.get_protocol()}://{bucket_name}.s3.{LOCALHOST_HOSTNAME}:{config.get_edge_port_http()}/"
169+
host_definition = localstack_host(
170+
use_hostname_external=True, custom_port=config.get_edge_port_http()
171+
)
172+
return f"{config.get_protocol()}://{host_definition.host_and_port()}/{bucket_name}/"
173+
else:
174+
host_definition = localstack_host(use_localhost_cloud=True)
175+
return f"{config.get_protocol()}://{bucket_name}.s3.{host_definition.host_and_port()}/"
171176

172177

173178
class S3Provider(S3Api, ServiceLifecycleHook):

‎localstack/services/s3/s3_listener.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
to_str,
6565
)
6666
from localstack.utils.time import timestamp_millis
67+
from localstack.utils.urls import localstack_host
6768
from localstack.utils.xml import strip_xmlns
6869

6970
# backend port (configured in s3_starter.py on startup)
@@ -1346,6 +1347,7 @@ def is_multipart_upload(query):
13461347

13471348
@staticmethod
13481349
def get_201_response(key, bucket_name):
1350+
host_definition = localstack_host(use_hostname_external=True)
13491351
return """
13501352
<PostResponse>
13511353
<Location>{protocol}://{host}/{encoded_key}</Location>
@@ -1355,7 +1357,7 @@ def get_201_response(key, bucket_name):
13551357
</PostResponse>
13561358
""".format(
13571359
protocol=get_service_protocol(),
1358-
host=config.HOSTNAME_EXTERNAL,
1360+
host=host_definition.host,
13591361
encoded_key=quote(key, safe=""),
13601362
key=key,
13611363
bucket=bucket_name,
@@ -1366,12 +1368,13 @@ def get_201_response(key, bucket_name):
13661368
def _update_location(content, bucket_name):
13671369
bucket_name = normalize_bucket_name(bucket_name)
13681370

1369-
host = config.HOSTNAME_EXTERNAL
1370-
if ":" not in host:
1371-
host = f"{host}:{config.service_port('s3')}"
1371+
host_definition = localstack_host(
1372+
use_hostname_external=True, custom_port=config.get_edge_port_http()
1373+
)
13721374
return re.sub(
13731375
r"<Location>\s*([a-zA-Z0-9\-]+)://[^/]+/([^<]+)\s*</Location>",
1374-
r"<Location>%s://%s/%s/\2</Location>" % (get_service_protocol(), host, bucket_name),
1376+
r"<Location>%s://%s/%s/\2</Location>"
1377+
% (get_service_protocol(), host_definition.host_and_port(), bucket_name),
13751378
content,
13761379
flags=re.MULTILINE,
13771380
)

‎localstack/services/sqs/models.py

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from queue import PriorityQueue
1010
from typing import Dict, NamedTuple, Optional, Set
1111

12-
from localstack import config, constants
12+
from localstack import config
1313
from localstack.aws.api import RequestContext
1414
from localstack.aws.api.sqs import (
1515
InvalidAttributeName,
@@ -21,7 +21,7 @@
2121
ReceiptHandleIsInvalid,
2222
TagMap,
2323
)
24-
from localstack.config import external_service_url
24+
from localstack.config import get_protocol
2525
from localstack.services.sqs import constants as sqs_constants
2626
from localstack.services.sqs.exceptions import (
2727
InvalidAttributeValue,
@@ -35,6 +35,7 @@
3535
)
3636
from localstack.services.stores import AccountRegionBundle, BaseStore, LocalAttribute
3737
from localstack.utils.time import now
38+
from localstack.utils.urls import localstack_host
3839

3940
LOG = logging.getLogger(__name__)
4041

@@ -249,13 +250,18 @@ def url(self, context: RequestContext) -> str:
249250
# or us-east-2.queue.localhost.localstack.cloud:4566/000000000000/my-queue
250251
region = "" if self.region == "us-east-1" else self.region + "."
251252
scheme = context.request.scheme
252-
host_url = f"{scheme}://{region}queue.{constants.LOCALHOST_HOSTNAME}:{config.EDGE_PORT}"
253+
254+
host_definition = localstack_host(use_localhost_cloud=True)
255+
host_url = f"{scheme}://{region}queue.{host_definition.host_and_port()}"
253256
elif config.SQS_ENDPOINT_STRATEGY == "path":
254257
# https?://localhost:4566/queue/us-east-1/00000000000/my-queue (us-east-1)
255258
host_url = f"{context.request.host_url}queue/{self.region}"
256259
else:
257260
if config.SQS_PORT_EXTERNAL:
258-
host_url = external_service_url("sqs")
261+
host_definition = localstack_host(
262+
use_hostname_external=True, custom_port=config.SQS_PORT_EXTERNAL
263+
)
264+
host_url = f"{get_protocol()}://{host_definition.host_and_port()}"
259265

260266
return "{host}/{account_id}/{name}".format(
261267
host=host_url.rstrip("/"),

‎localstack/testing/pytest/fixtures.py

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import logging
55
import os
66
import re
7+
import socket
78
import time
89
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple
910

@@ -21,7 +22,7 @@
2122
from pytest_httpserver import HTTPServer
2223
from werkzeug import Request, Response
2324

24-
from localstack import config
25+
from localstack import config, constants
2526
from localstack.aws.accounts import get_aws_account_id
2627
from localstack.constants import TEST_AWS_ACCESS_KEY_ID, TEST_AWS_SECRET_ACCESS_KEY
2728
from localstack.services.stores import (
@@ -2006,6 +2007,69 @@ def factory(**kwargs):
20062007
LOG.debug(f"Error cleaning up AppSync API: {api}, {e}")
20072008

20082009

2010+
@pytest.fixture
2011+
def assert_host_customisation(monkeypatch):
2012+
hostname_external = f"external-host-{short_uid()}"
2013+
# `LOCALSTACK_HOSTNAME` is really an internal variable that has been
2014+
# exposed to the user at some point in the past. It is used by some
2015+
# services that start resources (e.g. OpenSearch) to determine if the
2016+
# service has been started correctly (i.e. a health check). This means that
2017+
# the value must be resolvable by LocalStack or else the service resources
2018+
# won't start properly.
2019+
#
2020+
# One hostname that's always resolvable is the hostname of the process
2021+
# running LocalStack, so use that here.
2022+
#
2023+
# Note: We cannot use `localhost` since we explicitly check that the URL
2024+
# passed in does not contain `localhost`, unless it is requried to.
2025+
localstack_hostname = socket.gethostname()
2026+
monkeypatch.setattr(config, "HOSTNAME_EXTERNAL", hostname_external)
2027+
monkeypatch.setattr(config, "LOCALSTACK_HOSTNAME", localstack_hostname)
2028+
2029+
def asserter(
2030+
url: str,
2031+
*,
2032+
use_hostname_external: bool = False,
2033+
use_localstack_hostname: bool = False,
2034+
use_localstack_cloud: bool = False,
2035+
use_localhost: bool = False,
2036+
custom_host: Optional[str] = None,
2037+
):
2038+
if use_hostname_external:
2039+
assert hostname_external in url
2040+
2041+
assert localstack_hostname not in url
2042+
assert constants.LOCALHOST_HOSTNAME not in url
2043+
assert constants.LOCALHOST not in url
2044+
elif use_localstack_hostname:
2045+
assert localstack_hostname in url
2046+
2047+
assert hostname_external not in url
2048+
assert constants.LOCALHOST_HOSTNAME not in url
2049+
assert constants.LOCALHOST not in url
2050+
elif use_localstack_cloud:
2051+
assert constants.LOCALHOST_HOSTNAME in url
2052+
2053+
assert hostname_external not in url
2054+
assert localstack_hostname not in url
2055+
elif use_localhost:
2056+
assert constants.LOCALHOST in url
2057+
2058+
assert constants.LOCALHOST_HOSTNAME not in url
2059+
assert hostname_external not in url
2060+
assert localstack_hostname not in url
2061+
elif custom_host is not None:
2062+
assert custom_host in url
2063+
2064+
assert constants.LOCALHOST_HOSTNAME not in url
2065+
assert hostname_external not in url
2066+
assert localstack_hostname not in url
2067+
else:
2068+
raise ValueError("no asse 10000 rtions made")
2069+
2070+
yield asserter
2071+
2072+
20092073
@pytest.fixture
20102074
def echo_http_server(httpserver: HTTPServer):
20112075
"""Spins up a local HTTP echo server and returns the endpoint URL"""

‎localstack/utils/urls.py

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,47 @@
1+
from dataclasses import dataclass
2+
from typing import Optional
3+
4+
from localstack import config, constants
5+
6+
17
def path_from_url(url: str) -> str:
28
return f'/{url.partition("://")[2].partition("/")[2]}' if "://" in url else url
39

410

511
def hostname_from_url(url: str) -> str:
612
return url.split("://")[-1].split("/")[0].split(":")[0]
13+
14+
15+
@dataclass
16+
class HostDefinition:
17+
host: str
18+
port: int
19+
20+
def host_and_port(self):
21+
return f"{self.host}:{self.port}"
22+
23+
24+
def localstack_host(
25+
use_hostname_external: bool = False,
26+
use_localstack_hostname: bool = False,
27+
use_localhost_cloud: bool = False,
28+
custom_port: Optional[int] = None,
29+
) -> HostDefinition:
30+
"""
31+
Determine the host and port to return to the user based on:
32+
- the user's configuration (e.g environment variable overrides)
33+
- the defaults of the system
34+
"""
35+
port = config.EDGE_PORT
36+
if custom_port is not None:
37+
port = custom_port
38+
39+
host = config.LOCALHOST
40+
if use_hostname_external:
41+
host = config.HOSTNAME_EXTERNAL
42+
elif use_localstack_hostname:
43+
host = config.LOCALSTACK_HOSTNAME
44+
elif use_localhost_cloud:
45+
host = constants.LOCALHOST_HOSTNAME
46+
47+
return HostDefinition(host=host, port=port)

0 commit comments

Comments
 (0)
0