From 56377128eacc8cf56594f596c502ce956a4393c7 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Thu, 23 Feb 2023 17:42:29 +0000 Subject: [PATCH 01/24] Start tests for url returning Get opensearch and sqs handled --- .../integration/test_network_configuration.py | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 tests/integration/test_network_configuration.py diff --git a/tests/integration/test_network_configuration.py b/tests/integration/test_network_configuration.py new file mode 100644 index 0000000000000..738554d0059bf --- /dev/null +++ b/tests/integration/test_network_configuration.py @@ -0,0 +1,132 @@ +""" +This test file captures the _current_ state of returning URLs before making +sweeping changes. This is to ensure that the refactoring does not cause +external breaking behaviour. In the future we can update this test suite to +correspond to the behaviour we want, and we get a todo list of things to +change 😂 +""" +import pytest + +from localstack import config, constants +from localstack.utils.strings import short_uid + +# TODO: how do we test `localstack_hostname` - this variable configures the +# host that services make requests to when starting up (e.g. opensearch) and +# they won't start if we override the variable. + + +pytestmark = [pytest.mark.only_localstack] + + +@pytest.fixture +def patch_hostnames(monkeypatch): + """ + Update both HOSTNAME_EXTERNAL and LOCALSTACK_HOSTNAME to custom values to configure the running localstack instance. + """ + hostname_external = f"external-host-{short_uid()}" + localstack_hostname = f"localstack-hostname={short_uid()}" + monkeypatch.setattr(config, "HOSTNAME_EXTERNAL", hostname_external) + # monkeypatch.setattr(config, "LOCALSTACK_HOSTNAME", localstack_hostname) + yield hostname_external, localstack_hostname + + +class TestSQS: + def test_off_strategy(self, monkeypatch, sqs_create_queue, patch_hostnames): + external_port = "12345" + + monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "off") + monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) + + external_hostname, localstack_hostname = patch_hostnames + + queue_url = sqs_create_queue() + + assert external_hostname in queue_url + + assert localstack_hostname not in queue_url + + def test_domain_strategy(self, monkeypatch, sqs_create_queue, patch_hostnames): + external_port = "12345" + + monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "domain") + monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) + + external_hostname, localstack_hostname = patch_hostnames + queue_url = sqs_create_queue() + + assert constants.LOCALHOST_HOSTNAME in queue_url + + assert external_hostname not in queue_url + assert localstack_hostname not in queue_url + + def test_path_strategy(self, monkeypatch, sqs_create_queue, patch_hostnames): + external_port = "12345" + + monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "path") + monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) + + external_hostname, localstack_hostname = patch_hostnames + queue_url = sqs_create_queue() + + assert "localhost" in queue_url + + assert constants.LOCALHOST_HOSTNAME not in queue_url + assert external_hostname not in queue_url + assert localstack_hostname not in queue_url + + +class TestOpenSearch: + """ + OpenSearch does not respect any customisations and just returns a domain with localhost.localstack.cloud in. + """ + + def test_default_strategy( + self, opensearch_client, opensearch_wait_for_cluster, patch_hostnames + ): + domain_name = f"domain-{short_uid()}" + res = opensearch_client.create_domain(DomainName=domain_name) + opensearch_wait_for_cluster(domain_name) + endpoint = res["DomainStatus"]["Endpoint"] + + hostname_external, localstack_hostname = patch_hostnames + + assert constants.LOCALHOST_HOSTNAME in endpoint + + assert hostname_external not in endpoint + assert localstack_hostname not in endpoint + + def test_port_strategy( + self, monkeypatch, opensearch_client, opensearch_wait_for_cluster, patch_hostnames + ): + monkeypatch.setattr(config, "OPENSEARCH_ENDPOINT_STRATEGY", "port") + + domain_name = f"domain-{short_uid()}" + res = opensearch_client.create_domain(DomainName=domain_name) + opensearch_wait_for_cluster(domain_name) + endpoint = res["DomainStatus"]["Endpoint"] + + hostname_external, localstack_hostname = patch_hostnames + + assert "127.0.0.1" in endpoint + + assert hostname_external not in endpoint + assert localstack_hostname not in endpoint + assert constants.LOCALHOST_HOSTNAME not in endpoint + + def test_path_strategy( + self, monkeypatch, opensearch_client, opensearch_wait_for_cluster, patch_hostnames + ): + monkeypatch.setattr(config, "OPENSEARCH_ENDPOINT_STRATEGY", "path") + + domain_name = f"domain-{short_uid()}" + res = opensearch_client.create_domain(DomainName=domain_name) + opensearch_wait_for_cluster(domain_name) + endpoint = res["DomainStatus"]["Endpoint"] + + hostname_external, localstack_hostname = patch_hostnames + + assert "localhost" in endpoint + + assert hostname_external not in endpoint + assert localstack_hostname not in endpoint + assert constants.LOCALHOST_HOSTNAME not in endpoint From 0325017fc8e4c1191b86889f33b166f31d192614 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Mon, 27 Feb 2023 11:53:41 +0000 Subject: [PATCH 02/24] Add patch_hostnames fixture This is used to configure LocalStack after startup --- localstack/testing/pytest/fixtures.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/localstack/testing/pytest/fixtures.py b/localstack/testing/pytest/fixtures.py index a406e7cd83957..00391d31510a5 100644 --- a/localstack/testing/pytest/fixtures.py +++ b/localstack/testing/pytest/fixtures.py @@ -1978,3 +1978,16 @@ def factory(**kwargs): appsync_client.delete_graphql_api(apiId=api) except Exception as e: LOG.debug(f"Error cleaning up AppSync API: {api}, {e}") + + +@pytest.fixture +def patch_hostnames(monkeypatch): + """ + Update both HOSTNAME_EXTERNAL and LOCALSTACK_HOSTNAME to custom values to + configure the running localstack instance. + """ + hostname_external = f"external-host-{short_uid()}" + localstack_hostname = f"localstack-hostname={short_uid()}" + monkeypatch.setattr(config, "HOSTNAME_EXTERNAL", hostname_external) + # monkeypatch.setattr(config, "LOCALSTACK_HOSTNAME", localstack_hostname) + yield hostname_external, localstack_hostname From 480bb2f6aa93153050edde9618bb5d2677357819 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Mon, 27 Feb 2023 11:54:48 +0000 Subject: [PATCH 03/24] Add test for s3 location --- .../integration/test_network_configuration.py | 182 ++++++++++++------ 1 file changed, 125 insertions(+), 57 deletions(-) diff --git a/tests/integration/test_network_configuration.py b/tests/integration/test_network_configuration.py index 738554d0059bf..4983d4e8765b2 100644 --- a/tests/integration/test_network_configuration.py +++ b/tests/integration/test_network_configuration.py @@ -18,63 +18,6 @@ pytestmark = [pytest.mark.only_localstack] -@pytest.fixture -def patch_hostnames(monkeypatch): - """ - Update both HOSTNAME_EXTERNAL and LOCALSTACK_HOSTNAME to custom values to configure the running localstack instance. - """ - hostname_external = f"external-host-{short_uid()}" - localstack_hostname = f"localstack-hostname={short_uid()}" - monkeypatch.setattr(config, "HOSTNAME_EXTERNAL", hostname_external) - # monkeypatch.setattr(config, "LOCALSTACK_HOSTNAME", localstack_hostname) - yield hostname_external, localstack_hostname - - -class TestSQS: - def test_off_strategy(self, monkeypatch, sqs_create_queue, patch_hostnames): - external_port = "12345" - - monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "off") - monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) - - external_hostname, localstack_hostname = patch_hostnames - - queue_url = sqs_create_queue() - - assert external_hostname in queue_url - - assert localstack_hostname not in queue_url - - def test_domain_strategy(self, monkeypatch, sqs_create_queue, patch_hostnames): - external_port = "12345" - - monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "domain") - monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) - - external_hostname, localstack_hostname = patch_hostnames - queue_url = sqs_create_queue() - - assert constants.LOCALHOST_HOSTNAME in queue_url - - assert external_hostname not in queue_url - assert localstack_hostname not in queue_url - - def test_path_strategy(self, monkeypatch, sqs_create_queue, patch_hostnames): - external_port = "12345" - - monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "path") - monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) - - external_hostname, localstack_hostname = patch_hostnames - queue_url = sqs_create_queue() - - assert "localhost" in queue_url - - assert constants.LOCALHOST_HOSTNAME not in queue_url - assert external_hostname not in queue_url - assert localstack_hostname not in queue_url - - class TestOpenSearch: """ OpenSearch does not respect any customisations and just returns a domain with localhost.localstack.cloud in. @@ -130,3 +73,128 @@ def test_path_strategy( assert hostname_external not in endpoint assert localstack_hostname not in endpoint assert constants.LOCALHOST_HOSTNAME not in endpoint + + +class TestS3: + @pytest.mark.skipif( + condition=config.LEGACY_S3_PROVIDER, reason="Not implemented for legacy provider" + ) + def test_non_us_east_1_location( + self, monkeypatch, patch_hostnames, s3_resource, s3_client, cleanups + ): + monkeypatch.setattr(config, "LEGACY_S3_PROVIDER", False) + monkeypatch.setenv("PROVIDER_OVERRIDE_S3", "asf") + + bucket_name = f"bucket-{short_uid()}" + res = s3_client.create_bucket( + Bucket=bucket_name, + CreateBucketConfiguration={ + "LocationConstraint": "eu-west-1", + }, + ) + + def cleanup(): + bucket = s3_resource.Bucket(bucket_name) + bucket.objects.all().delete() + bucket.object_versions.all().delete() + bucket.delete() + + cleanups.append(cleanup) + + hostname_external, localstack_hostname = patch_hostnames + + url = res["Location"] + + assert hostname_external in url + + assert localstack_hostname not in url + assert constants.LOCALHOST_HOSTNAME not in url + + def test_multipart_upload(self, patch_hostnames, s3_bucket, s3_client): + key_name = f"key-{short_uid()}" + upload_id = s3_client.create_multipart_upload(Bucket=s3_bucket, Key=key_name)["UploadId"] + part_etag = s3_client.upload_part( + Bucket=s3_bucket, Key=key_name, Body=b"bytes", PartNumber=1, UploadId=upload_id + )["ETag"] + res = s3_client.complete_multipart_upload( + Bucket=s3_bucket, + Key=key_name, + MultipartUpload={"Parts": [{"ETag": part_etag, "PartNumber": 1}]}, + UploadId=upload_id, + ) + location = res["Location"] + + hostname_external, localstack_hostname = patch_hostnames + + assert hostname_external in location + + assert localstack_hostname not in location + assert constants.LOCALHOST_HOSTNAME not in location + + @pytest.mark.parametrize("method", ["put_object", "get_object", "head_object"]) + def test_presigned_urls(self, method, patch_hostnames, s3_bucket, s3_client): + key_name = f"key-{short_uid()}" + url = s3_client.generate_presigned_url( + ClientMethod=method, Params=dict(Bucket=s3_bucket, Key=key_name) + ) + + hostname_external, localstack_hostname = patch_hostnames + + assert constants.LOCALHOST_HOSTNAME in url + assert hostname_external not in url + assert localstack_hostname not in url + + def test_presigned_post(self, patch_hostnames, s3_bucket, s3_client): + key_name = f"key-{short_uid()}" + url = s3_client.generate_presigned_post(Bucket=s3_bucket, Key=key_name)["url"] + + hostname_external, localstack_hostname = patch_hostnames + + assert constants.LOCALHOST_HOSTNAME in url + assert hostname_external not in url + assert localstack_hostname not in url + + +class TestSQS: + def test_off_strategy(self, monkeypatch, sqs_create_queue, patch_hostnames): + external_port = "12345" + + monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "off") + monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) + + external_hostname, localstack_hostname = patch_hostnames + + queue_url = sqs_create_queue() + + assert external_hostname in queue_url + + assert localstack_hostname not in queue_url + + def test_domain_strategy(self, monkeypatch, sqs_create_queue, patch_hostnames): + external_port = "12345" + + monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "domain") + monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) + + external_hostname, localstack_hostname = patch_hostnames + queue_url = sqs_create_queue() + + assert constants.LOCALHOST_HOSTNAME in queue_url + + assert external_hostname not in queue_url + assert localstack_hostname not in queue_url + + def test_path_strategy(self, monkeypatch, sqs_create_queue, patch_hostnames): + external_port = "12345" + + monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "path") + monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) + + external_hostname, localstack_hostname = patch_hostnames + queue_url = sqs_create_queue() + + assert "localhost" in queue_url + + assert constants.LOCALHOST_HOSTNAME not in queue_url + assert external_hostname not in queue_url + assert localstack_hostname not in queue_url From 61c4994606108b2bb9ae0a9a4264ff2a5b90996b Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Wed, 1 Mar 2023 15:31:23 +0000 Subject: [PATCH 04/24] Add helpers for configuring the hostname --- localstack/utils/urls.py | 41 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/localstack/utils/urls.py b/localstack/utils/urls.py index 2d7b58eb3c0cd..0dbe2aa22066b 100644 --- a/localstack/utils/urls.py +++ b/localstack/utils/urls.py @@ -1,6 +1,47 @@ +from dataclasses import dataclass +from typing import Optional + +from localstack import config, constants + + def path_from_url(url: str) -> str: return f'/{url.partition("://")[2].partition("/")[2]}' if "://" in url else url def hostname_from_url(url: str) -> str: return url.split("://")[-1].split("/")[0].split(":")[0] + + +@dataclass +class HostDefinition: + host: str + port: int + + def __str__(self): + return f"{self.host}:{self.port}" + + +def localstack_host( + use_hostname_external: bool = False, + use_localstack_hostname: bool = False, + use_localhost_cloud: bool = False, + custom_port: Optional[int] = None, +) -> HostDefinition: + """ + Determine the host and port to return to the user based on: + - the user's configuration (e.g environment variable overrides) + - the defaults of the system + """ + port = config.EDGE_PORT + if custom_port is not None: + port = custom_port + + host = config.LOCALHOST + if use_hostname_external: + host = config.HOSTNAME_EXTERNAL + elif use_localstack_hostname: + host = config.LOCALSTACK_HOSTNAME + elif use_localhost_cloud: + host = constants.LOCALHOST_HOSTNAME + + return HostDefinition(host=host, port=port) From 3ede981a8f9eda0f175f505acb54a16f7e05cf82 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Wed, 1 Mar 2023 15:39:37 +0000 Subject: [PATCH 05/24] Use host utils in opensearch --- localstack/services/opensearch/cluster_manager.py | 14 ++++++++++---- localstack/utils/urls.py | 2 +- tests/integration/test_network_configuration.py | 5 ++++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/localstack/services/opensearch/cluster_manager.py b/localstack/services/opensearch/cluster_manager.py index 19630d3b78cfe..f018f6c918106 100644 --- a/localstack/services/opensearch/cluster_manager.py +++ b/localstack/services/opensearch/cluster_manager.py @@ -8,7 +8,7 @@ from localstack import config from localstack.aws.api.opensearch import DomainEndpointOptions, EngineType from localstack.config import EDGE_BIND_HOST -from localstack.constants import LOCALHOST, LOCALHOST_HOSTNAME +from localstack.constants import LOCALHOST from localstack.services.opensearch import versions from localstack.services.opensearch.cluster import ( CustomEndpoint, @@ -28,6 +28,7 @@ start_thread, ) from localstack.utils.serving import Server +from localstack.utils.urls import localstack_host LOG = logging.getLogger(__name__) @@ -115,11 +116,16 @@ def build_cluster_endpoint( assigned_port = external_service_ports.reserve_port() else: assigned_port = external_service_ports.reserve_port() - return f"{config.LOCALSTACK_HOSTNAME}:{assigned_port}" + + host_definition = localstack_host(use_localstack_hostname=True, custom_port=assigned_port) + return host_definition.host_and_port() if config.OPENSEARCH_ENDPOINT_STRATEGY == "path": - return f"{config.LOCALSTACK_HOSTNAME}:{config.EDGE_PORT}/{engine_domain}/{domain_key.region}/{domain_key.domain_name}" + host_definition = localstack_host(use_localstack_hostname=True) + return f"{host_definition.host_and_port()}/{engine_domain}/{domain_key.region}/{domain_key.domain_name}" + # or through a subdomain (domain-name.region.opensearch.localhost.localstack.cloud) - return f"{domain_key.domain_name}.{domain_key.region}.{engine_domain}.{LOCALHOST_HOSTNAME}:{config.EDGE_PORT}" + host_definition = localstack_host(use_localhost_cloud=True) + return f"{domain_key.domain_name}.{domain_key.region}.{engine_domain}.{host_definition.host_and_port()}" def determine_custom_endpoint( diff --git a/localstack/utils/urls.py b/localstack/utils/urls.py index 0dbe2aa22066b..04f605a553fd4 100644 --- a/localstack/utils/urls.py +++ b/localstack/utils/urls.py @@ -17,7 +17,7 @@ class HostDefinition: host: str port: int - def __str__(self): + def host_and_port(self): return f"{self.host}:{self.port}" diff --git a/tests/integration/test_network_configuration.py b/tests/integration/test_network_configuration.py index 4983d4e8765b2..49c3f514e6412 100644 --- a/tests/integration/test_network_configuration.py +++ b/tests/integration/test_network_configuration.py @@ -50,7 +50,10 @@ def test_port_strategy( hostname_external, localstack_hostname = patch_hostnames - assert "127.0.0.1" in endpoint + if config.is_in_docker: + assert constants.LOCALHOST in endpoint + else: + assert "127.0.0.1" in endpoint assert hostname_external not in endpoint assert localstack_hostname not in endpoint From dd812b56f762b4574de934d8212f5de927c9ecf4 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Thu, 2 Mar 2023 17:33:07 +0000 Subject: [PATCH 06/24] Update S3 to use new host config --- localstack/services/s3/provider.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/localstack/services/s3/provider.py b/localstack/services/s3/provider.py index 2d86149824b6b..e6351ccbf8887 100644 --- a/localstack/services/s3/provider.py +++ b/localstack/services/s3/provider.py @@ -94,7 +94,6 @@ preprocess_request, serve_custom_service_request_handlers, ) -from localstack.constants import LOCALHOST_HOSTNAME from localstack.services.edge import ROUTER from localstack.services.moto import call_moto from localstack.services.plugins import ServiceLifecycleHook @@ -130,6 +129,7 @@ from localstack.utils.collections import get_safe from localstack.utils.patch import patch from localstack.utils.strings import short_uid +from localstack.utils.urls import localstack_host LOG = logging.getLogger(__name__) @@ -166,8 +166,11 @@ def __init__(self, message=None): def get_full_default_bucket_location(bucket_name): if config.HOSTNAME_EXTERNAL != config.LOCALHOST: - return f"{config.get_protocol()}://{config.HOSTNAME_EXTERNAL}:{config.get_edge_port_http()}/{bucket_name}/" - return f"{config.get_protocol()}://{bucket_name}.s3.{LOCALHOST_HOSTNAME}:{config.get_edge_port_http()}/" + host_definition = localstack_host(use_hostname_external=True) + return f"{config.get_protocol()}://{host_definition.host_and_port()}/{bucket_name}/" + else: + host_definition = localstack_host(use_localhost_cloud=True) + return f"{config.get_protocol()}://{bucket_name}.s3.{host_definition.host_and_port()}/" class S3Provider(S3Api, ServiceLifecycleHook): From fe4d980d094c29cd4928cafe3727e292003e75f1 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 3 Mar 2023 14:37:34 +0000 Subject: [PATCH 07/24] Cover more cases with SQS tests These combine the existing hostname changes with external port changes. --- .../integration/test_network_configuration.py | 63 +++++++++++++------ 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/tests/integration/test_network_configuration.py b/tests/integration/test_network_configuration.py index 49c3f514e6412..907c44dddffb1 100644 --- a/tests/integration/test_network_configuration.py +++ b/tests/integration/test_network_configuration.py @@ -159,45 +159,72 @@ def test_presigned_post(self, patch_hostnames, s3_bucket, s3_client): class TestSQS: - def test_off_strategy(self, monkeypatch, sqs_create_queue, patch_hostnames): - external_port = "12345" + """ + Test all combinations of: - monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "off") - monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) + * endpoint_strategy + * sqs_port_external + * hostname_external + """ - external_hostname, localstack_hostname = patch_hostnames + def test_off_strategy_without_external_port( + self, monkeypatch, sqs_create_queue, patch_hostnames + ): + monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "off") - queue_url = sqs_create_queue() + queue_name = f"queue-{short_uid()}" + queue_url = sqs_create_queue(QueueName=queue_name) - assert external_hostname in queue_url + hostname_external, localstack_hostname = patch_hostnames + assert constants.LOCALHOST in queue_url + assert queue_name in queue_url + assert hostname_external not in queue_url + assert constants.LOCALHOST_HOSTNAME not in queue_url assert localstack_hostname not in queue_url - def test_domain_strategy(self, monkeypatch, sqs_create_queue, patch_hostnames): - external_port = "12345" + def test_off_strategy_with_external_port(self, monkeypatch, sqs_create_queue, patch_hostnames): + external_port = 12345 + monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "off") + monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) + + queue_name = f"queue-{short_uid()}" + queue_url = sqs_create_queue(QueueName=queue_name) + hostname_external, localstack_hostname = patch_hostnames + assert hostname_external in queue_url + assert str(external_port) in queue_url + assert queue_name in queue_url + + assert localstack_hostname not in queue_url + + @pytest.mark.parametrize("external_port", [0, 12345]) + def test_domain_strategy(self, external_port, monkeypatch, sqs_create_queue, patch_hostnames): monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "domain") monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) - external_hostname, localstack_hostname = patch_hostnames - queue_url = sqs_create_queue() + queue_name = f"queue-{short_uid()}" + queue_url = sqs_create_queue(QueueName=queue_name) + hostname_external, localstack_hostname = patch_hostnames assert constants.LOCALHOST_HOSTNAME in queue_url + assert queue_name in queue_url - assert external_hostname not in queue_url + assert hostname_external not in queue_url assert localstack_hostname not in queue_url - def test_path_strategy(self, monkeypatch, sqs_create_queue, patch_hostnames): - external_port = "12345" - + @pytest.mark.parametrize("external_port", [0, 12345]) + def test_path_strategy(self, external_port, monkeypatch, sqs_create_queue, patch_hostnames): monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "path") monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) - external_hostname, localstack_hostname = patch_hostnames - queue_url = sqs_create_queue() + queue_name = f"queue-{short_uid()}" + queue_url = sqs_create_queue(QueueName=queue_name) + hostname_external, localstack_hostname = patch_hostnames assert "localhost" in queue_url + assert queue_name in queue_url assert constants.LOCALHOST_HOSTNAME not in queue_url - assert external_hostname not in queue_url + assert hostname_external not in queue_url assert localstack_hostname not in queue_url From 74abbfb42800e69ea36bb70dbbb5ad63b4e4bc0d Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 3 Mar 2023 11:08:34 +0000 Subject: [PATCH 08/24] Partially update SQS to use the new config The logic seems a little fragmented - the default is to use the `Host` request header, but then some circumstances can partially override this with new behaviour. --- localstack/services/sqs/models.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/localstack/services/sqs/models.py b/localstack/services/sqs/models.py index da33bbeba7b6a..1c4c222be5491 100644 --- a/localstack/services/sqs/models.py +++ b/localstack/services/sqs/models.py @@ -9,7 +9,7 @@ from queue import PriorityQueue from typing import Dict, NamedTuple, Optional, Set -from localstack import config, constants +from localstack import config from localstack.aws.api import RequestContext from localstack.aws.api.sqs import ( InvalidAttributeName, @@ -21,7 +21,7 @@ ReceiptHandleIsInvalid, TagMap, ) -from localstack.config import external_service_url +from localstack.config import get_protocol from localstack.services.sqs import constants as sqs_constants from localstack.services.sqs.exceptions import ( InvalidAttributeValue, @@ -35,6 +35,7 @@ ) from localstack.services.stores import AccountRegionBundle, BaseStore, LocalAttribute from localstack.utils.time import now +from localstack.utils.urls import localstack_host LOG = logging.getLogger(__name__) @@ -249,13 +250,18 @@ def url(self, context: RequestContext) -> str: # or us-east-2.queue.localhost.localstack.cloud:4566/000000000000/my-queue region = "" if self.region == "us-east-1" else self.region + "." scheme = context.request.scheme - host_url = f"{scheme}://{region}queue.{constants.LOCALHOST_HOSTNAME}:{config.EDGE_PORT}" + + host_definition = localstack_host(use_localhost_cloud=True) + host_url = f"{scheme}://{region}queue.{host_definition.host_and_port()}" elif config.SQS_ENDPOINT_STRATEGY == "path": # https?://localhost:4566/queue/us-east-1/00000000000/my-queue (us-east-1) host_url = f"{context.request.host_url}/queue/{self.region}" else: if config.SQS_PORT_EXTERNAL: - host_url = external_service_url("sqs") + host_definition = localstack_host( + use_hostname_external=True, custom_port=config.SQS_PORT_EXTERNAL + ) + host_url = f"{get_protocol()}://{host_definition.host_and_port()}" return "{host}/{account_id}/{name}".format( host=host_url.rstrip("/"), From 1a1df4ee1af717570e01c94f1cf024c52af1b614 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 3 Mar 2023 16:21:40 +0000 Subject: [PATCH 09/24] Add `assert_host_customisation` fixture This replaces the `patch_hostnames` fixture, and performs the correct assertions. --- localstack/testing/pytest/fixtures.py | 52 +++++++-- .../integration/test_network_configuration.py | 110 +++++------------- 2 files changed, 75 insertions(+), 87 deletions(-) diff --git a/localstack/testing/pytest/fixtures.py b/localstack/testing/pytest/fixtures.py index 00391d31510a5..12b97c0af40a1 100644 --- a/localstack/testing/pytest/fixtures.py +++ b/localstack/testing/pytest/fixtures.py @@ -20,7 +20,7 @@ from moto.core import BackendDict, BaseBackend from pytest_httpserver import HTTPServer -from localstack import config +from localstack import config, constants from localstack.aws.accounts import get_aws_account_id from localstack.constants import TEST_AWS_ACCESS_KEY_ID, TEST_AWS_SECRET_ACCESS_KEY from localstack.services.stores import ( @@ -1981,13 +1981,51 @@ def factory(**kwargs): @pytest.fixture -def patch_hostnames(monkeypatch): - """ - Update both HOSTNAME_EXTERNAL and LOCALSTACK_HOSTNAME to custom values to - configure the running localstack instance. - """ +def assert_host_customisation(monkeypatch): hostname_external = f"external-host-{short_uid()}" localstack_hostname = f"localstack-hostname={short_uid()}" monkeypatch.setattr(config, "HOSTNAME_EXTERNAL", hostname_external) # monkeypatch.setattr(config, "LOCALSTACK_HOSTNAME", localstack_hostname) - yield hostname_external, localstack_hostname + + def asserter( + url: str, + *, + use_hostname_external: bool = False, + use_localstack_hostname: bool = False, + use_localstack_cloud: bool = False, + use_localhost: bool = False, + custom_host: Optional[str] = None, + ): + if use_hostname_external: + assert hostname_external in url + + assert localstack_hostname not in url + assert constants.LOCALHOST_HOSTNAME not in url + assert constants.LOCALHOST not in url + elif use_localstack_hostname: + assert localstack_hostname in url + + assert hostname_external not in url + assert constants.LOCALHOST_HOSTNAME not in url + assert constants.LOCALHOST not in url + elif use_localstack_cloud: + assert constants.LOCALHOST_HOSTNAME in url + + assert hostname_external not in url + assert localstack_hostname not in url + elif use_localhost: + assert constants.LOCALHOST in url + + assert constants.LOCALHOST_HOSTNAME not in url + assert hostname_external not in url + assert localstack_hostname not in url + elif custom_host is not None: + assert custom_host in url + + assert constants.LOCALHOST_HOSTNAME not in url + assert hostname_external not in url + assert localstack_hostname not in url + else: + raise ValueError("no assertions made") + + yield asserter diff --git a/tests/integration/test_network_configuration.py b/tests/integration/test_network_configuration.py index 907c44dddffb1..329a872a69ce6 100644 --- a/tests/integration/test_network_configuration.py +++ b/tests/integration/test_network_configuration.py @@ -7,7 +7,7 @@ """ import pytest -from localstack import config, constants +from localstack import config from localstack.utils.strings import short_uid # TODO: how do we test `localstack_hostname` - this variable configures the @@ -24,22 +24,17 @@ class TestOpenSearch: """ def test_default_strategy( - self, opensearch_client, opensearch_wait_for_cluster, patch_hostnames + self, opensearch_client, opensearch_wait_for_cluster, assert_host_customisation ): domain_name = f"domain-{short_uid()}" res = opensearch_client.create_domain(DomainName=domain_name) opensearch_wait_for_cluster(domain_name) endpoint = res["DomainStatus"]["Endpoint"] - hostname_external, localstack_hostname = patch_hostnames - - assert constants.LOCALHOST_HOSTNAME in endpoint - - assert hostname_external not in endpoint - assert localstack_hostname not in endpoint + assert_host_customisation(endpoint, use_localstack_cloud=True) def test_port_strategy( - self, monkeypatch, opensearch_client, opensearch_wait_for_cluster, patch_hostnames + self, monkeypatch, opensearch_client, opensearch_wait_for_cluster, assert_host_customisation ): monkeypatch.setattr(config, "OPENSEARCH_ENDPOINT_STRATEGY", "port") @@ -48,19 +43,13 @@ def test_port_strategy( opensearch_wait_for_cluster(domain_name) endpoint = res["DomainStatus"]["Endpoint"] - hostname_external, localstack_hostname = patch_hostnames - if config.is_in_docker: - assert constants.LOCALHOST in endpoint + assert_host_customisation(endpoint, use_localhost=True) else: - assert "127.0.0.1" in endpoint - - assert hostname_external not in endpoint - assert localstack_hostname not in endpoint - assert constants.LOCALHOST_HOSTNAME not in endpoint + assert_host_customisation(endpoint, custom_host="127.0.0.1") def test_path_strategy( - self, monkeypatch, opensearch_client, opensearch_wait_for_cluster, patch_hostnames + self, monkeypatch, opensearch_client, opensearch_wait_for_cluster, assert_host_customisation ): monkeypatch.setattr(config, "OPENSEARCH_ENDPOINT_STRATEGY", "path") @@ -69,13 +58,7 @@ def test_path_strategy( opensearch_wait_for_cluster(domain_name) endpoint = res["DomainStatus"]["Endpoint"] - hostname_external, localstack_hostname = patch_hostnames - - assert "localhost" in endpoint - - assert hostname_external not in endpoint - assert localstack_hostname not in endpoint - assert constants.LOCALHOST_HOSTNAME not in endpoint + assert_host_customisation(endpoint, use_localhost=True) class TestS3: @@ -83,7 +66,7 @@ class TestS3: condition=config.LEGACY_S3_PROVIDER, reason="Not implemented for legacy provider" ) def test_non_us_east_1_location( - self, monkeypatch, patch_hostnames, s3_resource, s3_client, cleanups + self, monkeypatch, s3_resource, s3_client, cleanups, assert_host_customisation ): monkeypatch.setattr(config, "LEGACY_S3_PROVIDER", False) monkeypatch.setenv("PROVIDER_OVERRIDE_S3", "asf") @@ -104,16 +87,9 @@ def cleanup(): cleanups.append(cleanup) - hostname_external, localstack_hostname = patch_hostnames + assert_host_customisation(res["Location"], use_hostname_external=True) - url = res["Location"] - - assert hostname_external in url - - assert localstack_hostname not in url - assert constants.LOCALHOST_HOSTNAME not in url - - def test_multipart_upload(self, patch_hostnames, s3_bucket, s3_client): + def test_multipart_upload(self, s3_bucket, s3_client, assert_host_customisation): key_name = f"key-{short_uid()}" upload_id = s3_client.create_multipart_upload(Bucket=s3_bucket, Key=key_name)["UploadId"] part_etag = s3_client.upload_part( @@ -125,37 +101,23 @@ def test_multipart_upload(self, patch_hostnames, s3_bucket, s3_client): MultipartUpload={"Parts": [{"ETag": part_etag, "PartNumber": 1}]}, UploadId=upload_id, ) - location = res["Location"] - - hostname_external, localstack_hostname = patch_hostnames - - assert hostname_external in location - assert localstack_hostname not in location - assert constants.LOCALHOST_HOSTNAME not in location + assert_host_customisation(res["Location"], use_hostname_external=True) @pytest.mark.parametrize("method", ["put_object", "get_object", "head_object"]) - def test_presigned_urls(self, method, patch_hostnames, s3_bucket, s3_client): + def test_presigned_urls(self, method, s3_bucket, s3_client, assert_host_customisation): key_name = f"key-{short_uid()}" url = s3_client.generate_presigned_url( ClientMethod=method, Params=dict(Bucket=s3_bucket, Key=key_name) ) - hostname_external, localstack_hostname = patch_hostnames + assert_host_customisation(url, use_localstack_cloud=True) - assert constants.LOCALHOST_HOSTNAME in url - assert hostname_external not in url - assert localstack_hostname not in url - - def test_presigned_post(self, patch_hostnames, s3_bucket, s3_client): + def test_presigned_post(self, s3_bucket, s3_client, assert_host_customisation): key_name = f"key-{short_uid()}" url = s3_client.generate_presigned_post(Bucket=s3_bucket, Key=key_name)["url"] - hostname_external, localstack_hostname = patch_hostnames - - assert constants.LOCALHOST_HOSTNAME in url - assert hostname_external not in url - assert localstack_hostname not in url + assert_host_customisation(url, use_localstack_cloud=True) class TestSQS: @@ -168,22 +130,19 @@ class TestSQS: """ def test_off_strategy_without_external_port( - self, monkeypatch, sqs_create_queue, patch_hostnames + self, monkeypatch, sqs_create_queue, assert_host_customisation ): monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "off") queue_name = f"queue-{short_uid()}" queue_url = sqs_create_queue(QueueName=queue_name) - hostname_external, localstack_hostname = patch_hostnames - assert constants.LOCALHOST in queue_url + assert_host_customisation(queue_url, use_localhost=True) assert queue_name in queue_url - assert hostname_external not in queue_url - assert constants.LOCALHOST_HOSTNAME not in queue_url - assert localstack_hostname not in queue_url - - def test_off_strategy_with_external_port(self, monkeypatch, sqs_create_queue, patch_hostnames): + def test_off_strategy_with_external_port( + self, monkeypatch, sqs_create_queue, assert_host_customisation + ): external_port = 12345 monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "off") monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) @@ -191,40 +150,31 @@ def test_off_strategy_with_external_port(self, monkeypatch, sqs_create_queue, pa queue_name = f"queue-{short_uid()}" queue_url = sqs_create_queue(QueueName=queue_name) - hostname_external, localstack_hostname = patch_hostnames - assert hostname_external in queue_url - assert str(external_port) in queue_url + assert_host_customisation(queue_url, use_hostname_external=True) assert queue_name in queue_url - assert localstack_hostname not in queue_url - @pytest.mark.parametrize("external_port", [0, 12345]) - def test_domain_strategy(self, external_port, monkeypatch, sqs_create_queue, patch_hostnames): + def test_domain_strategy( + self, external_port, monkeypatch, sqs_create_queue, assert_host_customisation + ): monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "domain") monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) queue_name = f"queue-{short_uid()}" queue_url = sqs_create_queue(QueueName=queue_name) - hostname_external, localstack_hostname = patch_hostnames - assert constants.LOCALHOST_HOSTNAME in queue_url + assert_host_customisation(queue_url, use_localstack_cloud=True) assert queue_name in queue_url - assert hostname_external not in queue_url - assert localstack_hostname not in queue_url - @pytest.mark.parametrize("external_port", [0, 12345]) - def test_path_strategy(self, external_port, monkeypatch, sqs_create_queue, patch_hostnames): + def test_path_strategy( + self, external_port, monkeypatch, sqs_create_queue, assert_host_customisation + ): monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "path") monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) queue_name = f"queue-{short_uid()}" queue_url = sqs_create_queue(QueueName=queue_name) - hostname_external, localstack_hostname = patch_hostnames - assert "localhost" in queue_url + assert_host_customisation(queue_url, use_localhost=True) assert queue_name in queue_url - - assert constants.LOCALHOST_HOSTNAME not in queue_url - assert hostname_external not in queue_url - assert localstack_hostname not in queue_url From 91e483f3965e1e60557f8d07709e30e141d184c5 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Tue, 7 Mar 2023 11:52:50 +0000 Subject: [PATCH 10/24] Update lambda function urls to use new localstack_host function New and old providers --- localstack/services/awslambda/lambda_api.py | 8 ++- localstack/services/awslambda/provider.py | 8 ++- .../integration/test_network_configuration.py | 60 +++++++++++++++++++ 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/localstack/services/awslambda/lambda_api.py b/localstack/services/awslambda/lambda_api.py index 5a9498792670f..a1091e4e0e16f 100644 --- a/localstack/services/awslambda/lambda_api.py +++ b/localstack/services/awslambda/lambda_api.py @@ -25,7 +25,7 @@ from localstack import config, constants from localstack.aws.accounts import get_aws_account_id -from localstack.constants import APPLICATION_JSON, LOCALHOST_HOSTNAME +from localstack.constants import APPLICATION_JSON from localstack.http import Request from localstack.http import Response as HttpResponse from localstack.services.awslambda import lambda_executors @@ -81,6 +81,7 @@ now_utc, timestamp, ) +from localstack.utils.urls import localstack_host LOG = logging.getLogger(__name__) @@ -1511,7 +1512,10 @@ def create_url_config(function): custom_id = md5(str(random())) region_name = aws_stack.get_region() - url = f"http://{custom_id}.lambda-url.{region_name}.{LOCALHOST_HOSTNAME}:{config.EDGE_PORT_HTTP or config.EDGE_PORT}/" + host_definition = localstack_host( + use_localhost_cloud=True, custom_port=config.EDGE_PORT_HTTP or config.EDGE_PORT + ) + url = f"http://{custom_id}.lambda-url.{region_name}.{host_definition.host_and_port()}/" # TODO: HTTPS support data = json.loads(to_str(request.data)) diff --git a/localstack/services/awslambda/provider.py b/localstack/services/awslambda/provider.py index 2eb85b1ecfaba..c3eded17c0734 100644 --- a/localstack/services/awslambda/provider.py +++ b/localstack/services/awslambda/provider.py @@ -134,7 +134,6 @@ UpdateFunctionUrlConfigResponse, Version, ) -from localstack.constants import LOCALHOST_HOSTNAME from localstack.services.awslambda import api_utils from localstack.services.awslambda import hooks as lambda_hooks from localstack.services.awslambda.api_utils import STATEMENT_ID_REGEX @@ -193,6 +192,7 @@ from localstack.utils.files import load_file from localstack.utils.strings import get_random_hex, long_uid, short_uid, to_bytes, to_str from localstack.utils.sync import poll_condition +from localstack.utils.urls import localstack_host LOG = logging.getLogger(__name__) @@ -1632,12 +1632,16 @@ def create_function_url_config( # create function URL config url_id = api_utils.generate_random_url_id() + + host_definition = localstack_host( + use_localhost_cloud=True, custom_port=config.EDGE_PORT_HTTP or config.EDGE_PORT + ) fn.function_url_configs[normalized_qualifier] = FunctionUrlConfig( function_arn=function_arn, function_name=function_name, cors=cors, url_id=url_id, - url=f"http://{url_id}.lambda-url.{context.region}.{LOCALHOST_HOSTNAME}:{config.EDGE_PORT_HTTP or config.EDGE_PORT}/", # TODO: https support + url=f"http://{url_id}.lambda-url.{context.region}.{host_definition.host_and_port()}/", # TODO: https support auth_type=auth_type, creation_time=api_utils.generate_lambda_date(), last_modified_time=api_utils.generate_lambda_date(), diff --git a/tests/integration/test_network_configuration.py b/tests/integration/test_network_configuration.py index 329a872a69ce6..d56cad6687c35 100644 --- a/tests/integration/test_network_configuration.py +++ b/tests/integration/test_network_configuration.py @@ -5,9 +5,15 @@ correspond to the behaviour we want, and we get a todo list of things to change 😂 """ +import json + import pytest +from botocore.auth import SigV4Auth from localstack import config +from localstack.aws.api.lambda_ import Runtime +from localstack.testing.aws.lambda_utils import is_new_provider, is_old_provider +from localstack.utils.files import new_tmp_file, save_file from localstack.utils.strings import short_uid # TODO: how do we test `localstack_hostname` - this variable configures the @@ -178,3 +184,57 @@ def test_path_strategy( assert_host_customisation(queue_url, use_localhost=True) assert queue_name in queue_url + + +class TestLambda: + @pytest.mark.skipif(condition=is_old_provider(), reason="Not implemented for legacy provider") + def test_function_url(self, assert_host_customisation, lambda_client, create_lambda_function): + function_name = f"function-{short_uid()}" + handler_code = "" + handler_file = new_tmp_file() + save_file(handler_file, handler_code) + + create_lambda_function( + func_name=function_name, + handler_file=handler_file, + runtime=Runtime.python3_9, + ) + + function_url = lambda_client.create_function_url_config( + FunctionName=function_name, + AuthType="NONE", + )["FunctionUrl"] + + assert_host_customisation(function_url, use_localstack_cloud=True) + + @pytest.mark.skipif(condition=is_new_provider(), reason="Not implemented for new provider") + def test_http_api_for_function_url( + self, assert_host_customisation, create_lambda_function, aws_http_client_factory + ): + function_name = f"function-{short_uid()}" + handler_code = "" + handler_file = new_tmp_file() + save_file(handler_file, handler_code) + + create_lambda_function( + func_name=function_name, + handler_file=handler_file, + runtime=Runtime.python3_9, + ) + + client = aws_http_client_factory("lambda", signer_factory=SigV4Auth) + url = f"/2021-10-31/functions/{function_name}/url" + r = client.post( + url, + data=json.dumps( + { + "AuthType": "NONE", + } + ), + params={"Qualifier": "$LATEST"}, + ) + r.raise_for_status() + + function_url = r.json()["FunctionUrl"] + + assert_host_customisation(function_url, use_localstack_cloud=True) From 2b5d625da6d963a75f25e24d52f718623b2c5fc0 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Wed, 8 Mar 2023 09:53:29 +0000 Subject: [PATCH 11/24] Correct typo with localstack-hostname value --- localstack/testing/pytest/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localstack/testing/pytest/fixtures.py b/localstack/testing/pytest/fixtures.py index 12b97c0af40a1..80d9aa3337ae9 100644 --- a/localstack/testing/pytest/fixtures.py +++ b/localstack/testing/pytest/fixtures.py @@ -1983,7 +1983,7 @@ def factory(**kwargs): @pytest.fixture def assert_host_customisation(monkeypatch): hostname_external = f"external-host-{short_uid()}" - localstack_hostname = f"localstack-hostname={short_uid()}" + localstack_hostname = f"localstack-hostname-{short_uid()}" monkeypatch.setattr(config, "HOSTNAME_EXTERNAL", hostname_external) # monkeypatch.setattr(config, "LOCALSTACK_HOSTNAME", localstack_hostname) From c94c3f176b8771812ae00bbb0b083e1159b2bd87 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Wed, 8 Mar 2023 13:44:40 +0000 Subject: [PATCH 12/24] Set localstack_hostname to the hostname of the host This enables tests to access the host by `config.LOCALSTACK_HOSTNAME`, but ensures that it is not `localhost`. --- localstack/testing/pytest/fixtures.py | 5 +++-- tests/integration/test_network_configuration.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/localstack/testing/pytest/fixtures.py b/localstack/testing/pytest/fixtures.py index 80d9aa3337ae9..16dcdef739b0d 100644 --- a/localstack/testing/pytest/fixtures.py +++ b/localstack/testing/pytest/fixtures.py @@ -4,6 +4,7 @@ import logging import os import re +import socket import time from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple @@ -1983,9 +1984,9 @@ def factory(**kwargs): @pytest.fixture def assert_host_customisation(monkeypatch): hostname_external = f"external-host-{short_uid()}" - localstack_hostname = f"localstack-hostname-{short_uid()}" + localstack_hostname = socket.gethostname() monkeypatch.setattr(config, "HOSTNAME_EXTERNAL", hostname_external) - # monkeypatch.setattr(config, "LOCALSTACK_HOSTNAME", localstack_hostname) + monkeypatch.setattr(config, "LOCALSTACK_HOSTNAME", localstack_hostname) def asserter( url: str, diff --git a/tests/integration/test_network_configuration.py b/tests/integration/test_network_configuration.py index d56cad6687c35..94c6cd32e1504 100644 --- a/tests/integration/test_network_configuration.py +++ b/tests/integration/test_network_configuration.py @@ -64,7 +64,7 @@ def test_path_strategy( opensearch_wait_for_cluster(domain_name) endpoint = res["DomainStatus"]["Endpoint"] - assert_host_customisation(endpoint, use_localhost=True) + assert_host_customisation(endpoint, use_localstack_hostname=True) class TestS3: From edad51b0f8baa2fd1791feefeceefc7b060a91e9 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Wed, 8 Mar 2023 15:17:08 +0000 Subject: [PATCH 13/24] Handle S3 201 response hostname returning --- localstack/services/s3/s3_listener.py | 4 ++- .../integration/test_network_configuration.py | 30 ++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/localstack/services/s3/s3_listener.py b/localstack/services/s3/s3_listener.py index 17f2cda76e220..d697ce6671b18 100644 --- a/localstack/services/s3/s3_listener.py +++ b/localstack/services/s3/s3_listener.py @@ -64,6 +64,7 @@ to_str, ) from localstack.utils.time import timestamp_millis +from localstack.utils.urls import localstack_host from localstack.utils.xml import strip_xmlns # backend port (configured in s3_starter.py on startup) @@ -1346,6 +1347,7 @@ def is_multipart_upload(query): @staticmethod def get_201_response(key, bucket_name): + host_definition = localstack_host(use_hostname_external=True) return """ {protocol}://{host}/{encoded_key} @@ -1355,7 +1357,7 @@ def get_201_response(key, bucket_name): """.format( protocol=get_service_protocol(), - host=config.HOSTNAME_EXTERNAL, + host=host_definition.host, encoded_key=quote(key, safe=""), key=key, bucket=bucket_name, diff --git a/tests/integration/test_network_configuration.py b/tests/integration/test_network_configuration.py index 94c6cd32e1504..95b653b9a15a4 100644 --- a/tests/integration/test_network_configuration.py +++ b/tests/integration/test_network_configuration.py @@ -8,6 +8,8 @@ import json import pytest +import requests +import xmltodict from botocore.auth import SigV4Auth from localstack import config @@ -110,20 +112,26 @@ def test_multipart_upload(self, s3_bucket, s3_client, assert_host_customisation) assert_host_customisation(res["Location"], use_hostname_external=True) - @pytest.mark.parametrize("method", ["put_object", "get_object", "head_object"]) - def test_presigned_urls(self, method, s3_bucket, s3_client, assert_host_customisation): + def test_201_response(self, s3_bucket, s3_client, assert_host_customisation): key_name = f"key-{short_uid()}" - url = s3_client.generate_presigned_url( - ClientMethod=method, Params=dict(Bucket=s3_bucket, Key=key_name) + body = "body" + presigned_request = s3_client.generate_presigned_post( + Bucket=s3_bucket, + Key=key_name, + Fields={"success_action_status": "201"}, + Conditions=[{"bucket": s3_bucket}, ["eq", "$success_action_status", "201"]], ) + files = {"file": ("my-file", body)} + res = requests.post( + presigned_request["url"], + data=presigned_request["fields"], + files=files, + verify=False, + ) + res.raise_for_status() + json_response = xmltodict.parse(res.content)["PostResponse"] - assert_host_customisation(url, use_localstack_cloud=True) - - def test_presigned_post(self, s3_bucket, s3_client, assert_host_customisation): - key_name = f"key-{short_uid()}" - url = s3_client.generate_presigned_post(Bucket=s3_bucket, Key=key_name)["url"] - - assert_host_customisation(url, use_localstack_cloud=True) + assert_host_customisation(json_response["Location"], use_hostname_external=True) class TestSQS: From a912dc9748a46ee481b460328e5ae9c5f2cfec75 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Thu, 9 Mar 2023 10:15:05 +0000 Subject: [PATCH 14/24] S3 vpath: handle HOSTNAME_EXTERNAL if supplied --- localstack/services/s3/virtual_host.py | 81 +++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 9 deletions(-) diff --git a/localstack/services/s3/virtual_host.py b/localstack/services/s3/virtual_host.py index 6afcfb35c5427..530e98a74a65c 100644 --- a/localstack/services/s3/virtual_host.py +++ b/localstack/services/s3/virtual_host.py @@ -2,8 +2,8 @@ import logging from urllib.parse import urlsplit, urlunsplit -from localstack.config import LEGACY_S3_PROVIDER -from localstack.constants import LOCALHOST_HOSTNAME +from localstack.config import HOSTNAME_EXTERNAL, LEGACY_S3_PROVIDER +from localstack.constants import LOCALHOST, LOCALHOST_HOSTNAME from localstack.http import Request, Response from localstack.http.proxy import Proxy from localstack.runtime import hooks @@ -14,12 +14,12 @@ LOG = logging.getLogger(__name__) # virtual-host style: https://{bucket-name}.s3.{region}.localhost.localstack.cloud.com/{key-name} -VHOST_REGEX_PATTERN = f".s3.{LOCALHOST_HOSTNAME}" +VHOST_REGEX_PATTERN = ".s3.{hostname}" # path addressed request with the region in the hostname # https://s3.{region}.localhost.localstack.cloud.com/{bucket-name}/{key-name} PATH_WITH_REGION_PATTERN = ( - f"s3.{LOCALHOST_HOSTNAME}" + "s3.{hostname}" ) @@ -32,7 +32,6 @@ class S3VirtualHostProxyHandler: def __call__(self, request: Request, **kwargs) -> Response: # TODO region pattern currently not working -> removing it from url rewritten_url = self._rewrite_url(request.url, kwargs.get("bucket"), kwargs.get("region")) - LOG.debug(f"Rewritten original host url: {request.url} to path-style url: {rewritten_url}") forward_to_url = urlsplit(rewritten_url) @@ -62,6 +61,10 @@ def _rewrite_url(url: str, bucket: str, region: str) -> str: If the region is contained in the host-name we remove it (for now) as moto cannot handle the region correctly + If the url contains a customised hostname, for example if the user sets `HOSTNAME_EXTERNAL` then re-write the + host to localhost.localstack.cloud since the request is coming from inside LocalStack itself, and `HOSTNAME_EXTERNAL` + may not be resolvable. + :param url: the original url :param bucket: the bucket name :param region: the region name @@ -79,6 +82,12 @@ def _rewrite_url(url: str, bucket: str, region: str) -> str: if region: netloc = netloc.replace(f"{region}", "") + # if the user specifies a custom hostname for LocalStack, this name may not be resolvable by + # LocalStack. We are proxying the request to ourself, so replace their custom hostname with + # `localhost.localstack.cloud` which also matches the PATH matchers. + if HOSTNAME_EXTERNAL != LOCALHOST: + netloc = netloc.replace(HOSTNAME_EXTERNAL, LOCALHOST_HOSTNAME) + return urlunsplit((splitted.scheme, netloc, path, splitted.query, splitted.fragment)) @@ -89,28 +98,82 @@ def register_virtual_host_routes(): """ s3_proxy_handler = S3VirtualHostProxyHandler() + # Add additional routes if the user specifies a custom HOSTNAME_EXTERNAL + # as we should match on these routes, and also match on localhost.localstack.cloud + # to maintain backwards compatibility. + if HOSTNAME_EXTERNAL != LOCALHOST: + ROUTER.add( + path="/", + host=VHOST_REGEX_PATTERN.format( + aws_region_regex=AWS_REGION_REGEX, + hostname=HOSTNAME_EXTERNAL, + ), + endpoint=s3_proxy_handler, + defaults={"path": "/"}, + ) + + ROUTER.add( + path="/", + host=VHOST_REGEX_PATTERN.format( + aws_region_regex=AWS_REGION_REGEX, + hostname=HOSTNAME_EXTERNAL, + ), + endpoint=s3_proxy_handler, + ) + + ROUTER.add( + path="/", + host=PATH_WITH_REGION_PATTERN.format( + aws_region_regex=AWS_REGION_REGEX, + hostname=HOSTNAME_EXTERNAL, + ), + endpoint=s3_proxy_handler, + defaults={"path": "/"}, + ) + + ROUTER.add( + path="//", + host=PATH_WITH_REGION_PATTERN.format( + aws_region_regex=AWS_REGION_REGEX, + hostname=HOSTNAME_EXTERNAL, + ), + endpoint=s3_proxy_handler, + ) + ROUTER.add( path="/", - host=VHOST_REGEX_PATTERN, + host=VHOST_REGEX_PATTERN.format( + aws_region_regex=AWS_REGION_REGEX, + hostname=LOCALHOST_HOSTNAME, + ), endpoint=s3_proxy_handler, defaults={"path": "/"}, ) ROUTER.add( path="/", - host=VHOST_REGEX_PATTERN, + host=VHOST_REGEX_PATTERN.format( + aws_region_regex=AWS_REGION_REGEX, + hostname=LOCALHOST_HOSTNAME, + ), endpoint=s3_proxy_handler, ) ROUTER.add( path="/", - host=PATH_WITH_REGION_PATTERN, + host=PATH_WITH_REGION_PATTERN.format( + aws_region_regex=AWS_REGION_REGEX, + hostname=LOCALHOST_HOSTNAME, + ), endpoint=s3_proxy_handler, defaults={"path": "/"}, ) ROUTER.add( path="//", - host=PATH_WITH_REGION_PATTERN, + host=PATH_WITH_REGION_PATTERN.format( + aws_region_regex=AWS_REGION_REGEX, + hostname=LOCALHOST_HOSTNAME, + ), endpoint=s3_proxy_handler, ) From bae2c5936e7764d8161d3467cb5b60446aef51dc Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Thu, 9 Mar 2023 11:31:52 +0000 Subject: [PATCH 15/24] Update s3 vhosts to use new localstack_host config --- localstack/services/s3/virtual_host.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/localstack/services/s3/virtual_host.py b/localstack/services/s3/virtual_host.py index 530e98a74a65c..c39b9b7f7830c 100644 --- a/localstack/services/s3/virtual_host.py +++ b/localstack/services/s3/virtual_host.py @@ -2,7 +2,7 @@ import logging from urllib.parse import urlsplit, urlunsplit -from localstack.config import HOSTNAME_EXTERNAL, LEGACY_S3_PROVIDER +from localstack.config import LEGACY_S3_PROVIDER from localstack.constants import LOCALHOST, LOCALHOST_HOSTNAME from localstack.http import Request, Response from localstack.http.proxy import Proxy @@ -10,6 +10,7 @@ from localstack.services.edge import ROUTER from localstack.services.s3.utils import S3_VIRTUAL_HOST_FORWARDED_HEADER from localstack.utils.aws.request_context import AWS_REGION_REGEX +from localstack.utils.urls import localstack_host LOG = logging.getLogger(__name__) @@ -85,8 +86,9 @@ def _rewrite_url(url: str, bucket: str, region: str) -> str: # if the user specifies a custom hostname for LocalStack, this name may not be resolvable by # LocalStack. We are proxying the request to ourself, so replace their custom hostname with # `localhost.localstack.cloud` which also matches the PATH matchers. - if HOSTNAME_EXTERNAL != LOCALHOST: - netloc = netloc.replace(HOSTNAME_EXTERNAL, LOCALHOST_HOSTNAME) + host_definition = localstack_host(use_hostname_external=True) + if host_definition.host != LOCALHOST: + netloc = netloc.replace(host_definition.host, LOCALHOST_HOSTNAME) return urlunsplit((splitted.scheme, netloc, path, splitted.query, splitted.fragment)) @@ -98,15 +100,16 @@ def register_virtual_host_routes(): """ s3_proxy_handler = S3VirtualHostProxyHandler() + host_definition = localstack_host(use_hostname_external=True) # Add additional routes if the user specifies a custom HOSTNAME_EXTERNAL # as we should match on these routes, and also match on localhost.localstack.cloud # to maintain backwards compatibility. - if HOSTNAME_EXTERNAL != LOCALHOST: + if host_definition.host != LOCALHOST: ROUTER.add( path="/", host=VHOST_REGEX_PATTERN.format( aws_region_regex=AWS_REGION_REGEX, - hostname=HOSTNAME_EXTERNAL, + hostname=host_definition.host, ), endpoint=s3_proxy_handler, defaults={"path": "/"}, @@ -116,7 +119,7 @@ def register_virtual_host_routes(): path="/", host=VHOST_REGEX_PATTERN.format( aws_region_regex=AWS_REGION_REGEX, - hostname=HOSTNAME_EXTERNAL, + hostname=host_definition.host, ), endpoint=s3_proxy_handler, ) @@ -125,7 +128,7 @@ def register_virtual_host_routes(): path="/", host=PATH_WITH_REGION_PATTERN.format( aws_region_regex=AWS_REGION_REGEX, - hostname=HOSTNAME_EXTERNAL, + hostname=host_definition.host, ), endpoint=s3_proxy_handler, defaults={"path": "/"}, @@ -135,7 +138,7 @@ def register_virtual_host_routes(): path="//", host=PATH_WITH_REGION_PATTERN.format( aws_region_regex=AWS_REGION_REGEX, - hostname=HOSTNAME_EXTERNAL, + hostname=host_definition.host, ), endpoint=s3_proxy_handler, ) From 888059a5d8905dd2c3fa0b103169c552124a8626 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 10 Mar 2023 11:09:01 +0000 Subject: [PATCH 16/24] Run ASF tests correctly --- .circleci/config.yml | 2 +- tests/integration/test_network_configuration.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3edc86341ea3f..4fb8cedbcf122 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,7 +172,7 @@ jobs: name: Test ASF S3 provider environment: PROVIDER_OVERRIDE_S3: "asf" - TEST_PATH: "tests/integration/s3/" + TEST_PATH: "tests/integration/s3/ tests/integration/test_network_configuration.py::TestS3" PYTEST_ARGS: "--reruns 3 --junitxml=target/reports/s3_asf.xml -o junit_suite_name='s3_asf'" COVERAGE_ARGS: "-p" command: make test-coverage diff --git a/tests/integration/test_network_configuration.py b/tests/integration/test_network_configuration.py index 95b653b9a15a4..af28be4dbdd9d 100644 --- a/tests/integration/test_network_configuration.py +++ b/tests/integration/test_network_configuration.py @@ -74,11 +74,8 @@ class TestS3: condition=config.LEGACY_S3_PROVIDER, reason="Not implemented for legacy provider" ) def test_non_us_east_1_location( - self, monkeypatch, s3_resource, s3_client, cleanups, assert_host_customisation + self, s3_resource, s3_client, cleanups, assert_host_customisation ): - monkeypatch.setattr(config, "LEGACY_S3_PROVIDER", False) - monkeypatch.setenv("PROVIDER_OVERRIDE_S3", "asf") - bucket_name = f"bucket-{short_uid()}" res = s3_client.create_bucket( Bucket=bucket_name, From fef6b5c1bb7160027ed348ac5caf6467f487b0fc Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 10 Mar 2023 11:11:36 +0000 Subject: [PATCH 17/24] Run ASF lambda tests correctly --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4fb8cedbcf122..159f7dc61000e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -147,7 +147,7 @@ jobs: name: Test ASF Lambda provider environment: PROVIDER_OVERRIDE_LAMBDA: "asf" - TEST_PATH: "tests/integration/awslambda/test_lambda.py tests/integration/awslambda/test_lambda_api.py tests/integration/awslambda/test_lambda_common.py tests/integration/awslambda/test_lambda_integration_sqs.py tests/integration/cloudformation/resources/test_lambda.py tests/integration/awslambda/test_lambda_integration_dynamodbstreams.py tests/integration/awslambda/test_lambda_integration_kinesis.py tests/integration/awslambda/test_lambda_developer_tools.py" + TEST_PATH: "tests/integration/awslambda/test_lambda.py tests/integration/awslambda/test_lambda_api.py tests/integration/awslambda/test_lambda_common.py tests/integration/awslambda/test_lambda_integration_sqs.py tests/integration/cloudformation/resources/test_lambda.py tests/integration/awslambda/test_lambda_integration_dynamodbstreams.py tests/integration/awslambda/test_lambda_integration_kinesis.py tests/integration/awslambda/test_lambda_developer_tools.py tests/integration/test_network_configuration.py::TestLambda" PYTEST_ARGS: "--reruns 3 --junitxml=target/reports/lambda_asf.xml -o junit_suite_name='lambda_asf'" COVERAGE_ARGS: "-p" command: make test-coverage From 53dab3ac64b4a0102e7154d3477fa176276a3701 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 10 Mar 2023 14:25:45 +0000 Subject: [PATCH 18/24] Add explanation comment for socket.gethostname --- localstack/testing/pytest/fixtures.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/localstack/testing/pytest/fixtures.py b/localstack/testing/pytest/fixtures.py index 16dcdef739b0d..5e5faf31b0a8d 100644 --- a/localstack/testing/pytest/fixtures.py +++ b/localstack/testing/pytest/fixtures.py @@ -1984,6 +1984,18 @@ def factory(**kwargs): @pytest.fixture def assert_host_customisation(monkeypatch): hostname_external = f"external-host-{short_uid()}" + # `LOCALSTACK_HOSTNAME` is really an internal variable that has been + # exposed to the user at some point in the past. It is used by some + # services that start resources (e.g. OpenSearch) to determine if the + # service has been started correctly (i.e. a health check). This means that + # the value must be resolvable by LocalStack or else the service resources + # won't start properly. + # + # One hostname that's always resolvable is the hostname of the process + # running LocalStack, so use that here. + # + # Note: We cannot use `localhost` since we explicitly check that the URL + # passed in does not contain `localhost`, unless it is requried to. localstack_hostname = socket.gethostname() monkeypatch.setattr(config, "HOSTNAME_EXTERNAL", hostname_external) monkeypatch.setattr(config, "LOCALSTACK_HOSTNAME", localstack_hostname) From 9845857a0e55485992f1b0e68491d0c96504eb3a Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 10 Mar 2023 14:26:23 +0000 Subject: [PATCH 19/24] Update `s3_listener._update_location` This removes a usage of HOSTNAME_EXTERNAL --- localstack/services/s3/s3_listener.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/localstack/services/s3/s3_listener.py b/localstack/services/s3/s3_listener.py index d697ce6671b18..1ebe660e06999 100644 --- a/localstack/services/s3/s3_listener.py +++ b/localstack/services/s3/s3_listener.py @@ -1368,12 +1368,11 @@ def get_201_response(key, bucket_name): def _update_location(content, bucket_name): bucket_name = normalize_bucket_name(bucket_name) - host = config.HOSTNAME_EXTERNAL - if ":" not in host: - host = f"{host}:{config.service_port('s3')}" + host_definition = localstack_host(use_hostname_external=True) return re.sub( r"\s*([a-zA-Z0-9\-]+)://[^/]+/([^<]+)\s*", - r"%s://%s/%s/\2" % (get_service_protocol(), host, bucket_name), + r"%s://%s/%s/\2" + % (get_service_protocol(), host_definition.host_and_port(), bucket_name), content, flags=re.MULTILINE, ) From 600d22f09ec2fbdf47e9f6dc2386e2a3beea7f8d Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 10 Mar 2023 16:19:19 +0000 Subject: [PATCH 20/24] Make suggested changes to sqs tests --- tests/integration/test_network_configuration.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_network_configuration.py b/tests/integration/test_network_configuration.py index af28be4dbdd9d..1d2d8b14da14c 100644 --- a/tests/integration/test_network_configuration.py +++ b/tests/integration/test_network_configuration.py @@ -163,13 +163,10 @@ def test_off_strategy_with_external_port( assert_host_customisation(queue_url, use_hostname_external=True) assert queue_name in queue_url + assert f":{external_port}" in queue_url - @pytest.mark.parametrize("external_port", [0, 12345]) - def test_domain_strategy( - self, external_port, monkeypatch, sqs_create_queue, assert_host_customisation - ): + def test_domain_strategy(self, monkeypatch, sqs_create_queue, assert_host_customisation): monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "domain") - monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) queue_name = f"queue-{short_uid()}" queue_url = sqs_create_queue(QueueName=queue_name) @@ -177,12 +174,8 @@ def test_domain_strategy( assert_host_customisation(queue_url, use_localstack_cloud=True) assert queue_name in queue_url - @pytest.mark.parametrize("external_port", [0, 12345]) - def test_path_strategy( - self, external_port, monkeypatch, sqs_create_queue, assert_host_customisation - ): + def test_path_strategy(self, monkeypatch, sqs_create_queue, assert_host_customisation): monkeypatch.setattr(config, "SQS_ENDPOINT_STRATEGY", "path") - monkeypatch.setattr(config, "SQS_PORT_EXTERNAL", external_port) queue_name = f"queue-{short_uid()}" queue_url = sqs_create_queue(QueueName=queue_name) From 75176fe614920d07f50d2a4eeeb2aa74cafec9bb Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Mon, 13 Mar 2023 10:53:11 +0000 Subject: [PATCH 21/24] Remove outdated comment --- tests/integration/test_network_configuration.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/integration/test_network_configuration.py b/tests/integration/test_network_configuration.py index 1d2d8b14da14c..62a291630727b 100644 --- a/tests/integration/test_network_configuration.py +++ b/tests/integration/test_network_configuration.py @@ -18,11 +18,6 @@ from localstack.utils.files import new_tmp_file, save_file from localstack.utils.strings import short_uid -# TODO: how do we test `localstack_hostname` - this variable configures the -# host that services make requests to when starting up (e.g. opensearch) and -# they won't start if we override the variable. - - pytestmark = [pytest.mark.only_localstack] From 2c5899f1e5f8c86529c4c9fa36f00ca6148482fc Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 17 Mar 2023 10:15:19 +0000 Subject: [PATCH 22/24] Fix pro integration test with s3 We need to listen on the HTTP port rather than the HTTPS port --- localstack/services/s3/provider.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/localstack/services/s3/provider.py b/localstack/services/s3/provider.py index e6351ccbf8887..845436ad796f0 100644 --- a/localstack/services/s3/provider.py +++ b/localstack/services/s3/provider.py @@ -166,7 +166,9 @@ def __init__(self, message=None): def get_full_default_bucket_location(bucket_name): if config.HOSTNAME_EXTERNAL != config.LOCALHOST: - host_definition = localstack_host(use_hostname_external=True) + host_definition = localstack_host( + use_hostname_external=True, custom_port=config.get_edge_port_http() + ) return f"{config.get_protocol()}://{host_definition.host_and_port()}/{bucket_name}/" else: host_definition = localstack_host(use_localhost_cloud=True) From 46ea0e97a08ddc429f7c3f93b12cb6ef92d97c80 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 17 Mar 2023 11:58:18 +0000 Subject: [PATCH 23/24] Fix issue with legacy s3 provider --- localstack/services/s3/s3_listener.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/localstack/services/s3/s3_listener.py b/localstack/services/s3/s3_listener.py index 1ebe660e06999..17deada128585 100644 --- a/localstack/services/s3/s3_listener.py +++ b/localstack/services/s3/s3_listener.py @@ -1368,7 +1368,9 @@ def get_201_response(key, bucket_name): def _update_location(content, bucket_name): bucket_name = normalize_bucket_name(bucket_name) - host_definition = localstack_host(use_hostname_external=True) + host_definition = localstack_host( + use_hostname_external=True, custom_port=config.get_edge_port_http() + ) return re.sub( r"\s*([a-zA-Z0-9\-]+)://[^/]+/([^<]+)\s*", r"%s://%s/%s/\2" From ba95b63adc6dfde3aeebda2a45700cbaa1c5c296 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 17 Mar 2023 16:05:34 +0000 Subject: [PATCH 24/24] Increase timeout of tests --- .github/workflows/pro-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pro-integration.yml b/.github/workflows/pro-integration.yml index e44bc7ee5c33a..7406eea1f9ce3 100644 --- a/.github/workflows/pro-integration.yml +++ b/.github/workflows/pro-integration.yml @@ -42,7 +42,7 @@ concurrency: jobs: run-integration-tests: runs-on: ubuntu-latest - timeout-minutes: 110 + timeout-minutes: 120 defaults: run: working-directory: localstack-ext