8000 Add lambda config to ignore architecture (#7890) · codeperl/localstack@10e77c0 · GitHub
[go: up one dir, main page]
Skip to content

Commit 10e77c0

Browse files
joe4devjoe4dev
authored
Add lambda config to ignore architecture (localstack#7890)
1 parent 3b41e13 commit 10e77c0

File tree

5 files changed

+134
-15
lines changed

5 files changed

+134
-15
lines changed

‎localstack/config.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -645,6 +645,10 @@ def in_docker():
645645
# additional flags passed to Lambda Docker run/create commands
646646
LAMBDA_DOCKER_FLAGS = os.environ.get("LAMBDA_DOCKER_FLAGS", "").strip()
647647

648+
# Enable this flag to run cross-platform compatible lambda functions natively (i.e., Docker selects architecture) and
649+
# ignore the AWS architectures (i.e., x86_64, arm64) configured for the lambda function.
650+
LAMBDA_IGNORE_ARCHITECTURE = is_env_true("LAMBDA_IGNORE_ARCHITECTURE")
651+
648652
# prebuild images before execution? Increased cold start time on the tradeoff of increased time until lambda is ACTIVE
649653
LAMBDA_PREBUILD_IMAGES = is_env_true("LAMBDA_PREBUILD_IMAGES")
650654

‎localstack/services/awslambda/invocation/docker_runtime_executor.py

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
COPY code/ /var/task
5151
"""
5252

53-
PULLED_IMAGES: set[str] = set()
53+
PULLED_IMAGES: set[(str, DockerPlatform)] = set()
5454

5555
HOT_RELOADING_ENV_VARIABLE = "LOCALSTACK_HOT_RELOADING_PATHS"
5656

@@ -64,15 +64,17 @@
6464
)
6565

6666

67-
def docker_platform(lambda_architecture: Architecture) -> DockerPlatform:
67+
def docker_platform(lambda_architecture: Architecture) -> DockerPlatform | None:
6868
"""
6969
Convert an AWS Lambda architecture into a Docker platform flag. Examples:
7070
* docker_platform("x86_64") == "linux/amd64"
7171
* docker_platform("arm64") == "linux/arm64"
7272
7373
:param lambda_architecture: the instruction set that the function supports
74-
:return: Docker platform in the format ``os[/arch[/variant]]``
74+
:return: Docker platform in the format ``os[/arch[/variant]]`` or None if configured to ignore the architecture
7575
"""
76+
if config.LAMBDA_IGNORE_ARCHITECTURE:
77+
return None
7678
return ARCHITECTURE_PLATFORM_MAPPING[lambda_architecture]
7779

7880

@@ -378,9 +380,11 @@ def prepare_version(cls, function_version: FunctionVersion) -> None:
378380
if function_version.config.code:
379381
function_version.config.code.prepare_for_execution()
380382
image_name = resolver.get_image_for_runtime(function_version.config.runtime)
381-
if image_name not in PULLED_IMAGES:
382-
CONTAINER_CLIENT.pull_image(image_name)
383-
PULLED_IMAGES.add(image_name)
383+
platform = docker_platform(function_version.config.architectures[0])
384+
# Pull image for a given platform upon function creation such that invocations do not time out.
385+
if (image_name, platform) not in PULLED_IMAGES:
386+
CONTAINER_CLIENT.pull_image(image_name, platform)
387+
PULLED_IMAGES.add((image_name, platform))
384388
if config.LAMBDA_PREBUILD_IMAGES:
385389
target_path = function_version.config.code.get_unzipped_code_location()
386390
prepare_image(target_path, function_version)

‎localstack/testing/aws/lambda_utils.py

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import json
22
import os
3-
import platform
43
from typing import Literal
54

5+
from localstack.services.awslambda.lambda_api import use_docker
66
from localstack.utils.common import to_str
77
from localstack.utils.sync import ShortCircuitWaitException, retry
88
from localstack.utils.testutil import get_lambda_log_events
@@ -117,6 +117,14 @@ def get_events():
117117
return retry(get_events, retries=retries, sleep_before=2)
118118

119119

120+
def is_old_local_executor() -> bool:
121+
"""Returns True if running in local executor mode and False otherwise.
122+
The new provider ignores the LAMBDA_EXECUTOR flag and `not use_docker()` covers the fallback case if
123+
the Docker socket is not available.
124+
"""
125+
return is_old_provider() and not use_docker()
126+
127+
120128
def is_old_provider():
121129
return os.environ.get("TEST_TARGET") != "AWS_CLOUD" and os.environ.get(
122130
"PROVIDER_OVERRIDE_LAMBDA"
@@ -127,7 +135,3 @@ def is_new_provider():
127135
return os.environ.get("TEST_TARGET") != "AWS_CLOUD" and os.environ.get(
128136
"PROVIDER_OVERRIDE_LAMBDA"
129137
) in ["asf", "v2"]
130-
131-
132-
def is_arm_compatible():
133-
return platform.machine() == "arm64"

‎localstack/utils/platform.py

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,21 @@ def is_redhat() -> bool:
2828
return "rhel" in load_file("/etc/os-release", "")
2929

3030

31+
class Arch(str):
32+
"""LocalStack standardised machine architecture names"""
33+
34+
amd64 = "amd64"
35+
arm64 = "arm64"
36+
37+
3138
def standardized_arch(arch: str):
3239
"""
3340
Returns LocalStack standardised machine architecture name.
3441
"""
3542
if arch == "x86_64":
36-
return "amd64"
43+
return Arch.amd64
3744
if arch == "aarch64":
38-
return "arm64"
45+
return Arch.arm64
3946
return arch
4047

4148

@@ -47,6 +54,11 @@ def get_arch() -> str:
4754
return standardized_arch(arch)
4855

4956

57+
def is_arm_compatible() -> bool:
58+
"""Returns true if the current machine is compatible with ARM instructions and false otherwise."""
59+
return get_arch() == Arch.arm64
60+
61+
5062
def get_os() -> str:
5163
if is_mac_os():
5264
return "osx"

‎tests/integration/awslambda/test_lambda.py

Lines changed: 97 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,22 +10,24 @@
1010
import pytest
1111
from botocore.response import StreamingBody
1212

13+
from localstack import config
1314
from localstack.aws.api.lambda_ import Architecture, Runtime
1415
from localstack.services.awslambda.lambda_api import use_docker
1516
from localstack.testing.aws.lambda_utils import (
1617
concurrency_update_done,
1718
get_invoke_init_type,
18-
is_arm_compatible,
19+
is_old_local_executor,
1920
is_old_provider,
2021
update_done,
2122
)
2223
from localstack.testing.aws.util import create_client_with_keys
2324
from localstack.testing.pytest.snapshot import is_aws
2425
from localstack.testing.snapshots.transformer import KeyValueBasedTransformer
2526
from localstack.testing.snapshots.transformer_utility import PATTERN_UUID
26-
from localstack.utils import files, testutil
27+
from localstack.utils import files, platform, testutil
2728
from localstack.utils.files import load_file
2829
from localstack.utils.http import safe_requests
30+
from localstack.utils.platform import is_arm_compatible, standardized_arch
2931
from localstack.utils.strings import short_uid, to_bytes, to_str
3032
from localstack.utils.sync import retry, wait_until
3133
from localstack.utils.testutil import create_lambda_archive
@@ -377,6 +379,99 @@ def test_runtime_introspection_arm(self, lambda_client, create_lambda_function,
377379
invoke_result = lambda_client.invoke(FunctionName=func_name)
378380
snapshot.match("invoke_runtime_arm_introspection", invoke_result)
379381

382+
@pytest.mark.skipif(is_old_provider(), reason="unsupported in old provider")
383+
@pytest.mark.skipif(
384+
is_old_local_executor(),
385+
reason="Monkey-patching of Docker flags is not applicable because no new container is spawned",
386+
)
387+
@pytest.mark.only_localstack
388+
def test_ignore_architecture(
389+
self, lambda_client, create_lambda_function, snapshot, monkeypatch
390+
):
391+
"""Test configuration to ignore lambda architecture by creating a lambda with non-native architecture."""
392+
monkeypatch.setattr(config, "LAMBDA_IGNORE_ARCHITECTURE", True)
393+
394+
# Assumes that LocalStack runs on native Docker host architecture
395+
# This assumption could be violated when using remote Lambda executors
396+
native_arch = platform.get_arch()
397+
non_native_architecture = (
398+
Architecture.x86_64 if native_arch == "arm64" else Architecture.arm64
399+
)
400+
func_name = f"test_lambda_arch_{short_uid()}"
401+
create_lambda_function(
402+
func_name=func_name,
403+
handler_file=TEST_LAMBDA_INTROSPECT_PYTHON,
404+
runtime=Runtime.python3_9,
405+
Architectures=[non_native_architecture],
406+
)
407+
408+
invoke_result = lambda_client.invoke(FunctionName=func_name)
409+
payload = json.loads(to_str(invoke_result["Payload"].read()))
410+
lambda_arch = standardized_arch(payload.get("platform_machine"))
411+
assert lambda_arch == native_arch
412+
413+
@pytest.mark.skipif(is_old_provider(), reason="unsupported in old provider")
414+
@pytest.mark.skipif(
415+
not is_arm_compatible() and not is_aws(),
416+
reason="ARM architecture not supported on this host",
417+
)
418+
@pytest.mark.aws_validated
419+
def test_mixed_architecture(self, lambda_client, create_lambda_function):
420+
"""Test emulation and interaction of lambda functions with different architectures.
421+
Limitation: only works on ARM hosts that support x86 emulation.
422+
"""
423+
func_name = f"test_lambda_x86_{short_uid()}"
424+
create_lambda_function(
425+
func_name=func_name,
426+
handler_file=TEST_LAMBDA_INTROSPECT_PYTHON,
427+
runtime=Runtime.python3_9,
428+
Architectures=[Architecture.x86_64],
429+
)
430+
431+
invoke_result = lambda_client.invoke(FunctionName=func_name)
432+
assert "FunctionError" not in invoke_result
433+
payload = json.loads(invoke_result["Payload"].read())
434+
assert payload.get("platform_machine") == "x86_64"
435+
436+
func_name_arm = f"test_lambda_arm_{short_uid()}"
437+
create_lambda_function(
438+
func_name=func_name_arm,
439+
handler_file=TEST_LAMBDA_INTROSPECT_PYTHON,
440+
runtime=Runtime.python3_9,
441+
Architectures=[Architecture.arm64],
442+
)
443+
444+
invoke_result_arm = lambda_client.invoke(FunctionName=func_name_arm)
445+
assert "FunctionError" not in invoke_result_arm
446+
payload_arm = json.loads(invoke_result_arm["Payload"].read())
447+
assert payload_arm.get("platform_machine") == "aarch64"
448+
449+
v1_result = lambda_client.publish_version(FunctionName=func_name)
450+
v1 = v1_result["Version"]
451+
452+
# assert version is available(!)
453+
lambda_client.get_waiter(waiter_name="function_active_v2").wait(
454+
FunctionName=func_name, Qualifier=v1
455+
)
456+
457+
arm_v1_result = lambda_client.publish_version(FunctionName=func_name_arm)
458+
arm_v1 = arm_v1_result["Version"]
459+
460+
# assert version is available(!)
461+
lambda_client.get_waiter(waiter_name="function_active_v2").wait(
462+
FunctionName=func_name_arm, Qualifier=arm_v1
463+
)
464+
465+
invoke_result_2 = lambda_client.invoke(FunctionName=func_name, Qualifier=v1)
466+
assert "FunctionError" not in invoke_result_2
467+
payload_2 = json.loads(invoke_result_2["Payload"].read())
468+
assert payload_2.get("platform_machine") == "x86_64"
469+
470+
invoke_result_arm_2 = lambda_client.invoke(FunctionName=func_name_arm, Qualifier=arm_v1)
471+
assert "FunctionError" not in invoke_result_arm_2
472+
payload_arm_2 = json.loads(invoke_result_arm_2["Payload"].read())
473+
assert payload_arm_2.get("platform_machine") == "aarch64"
474+
380475
@pytest.mark.skip_snapshot_verify(
381476
condition=is_old_provider, paths=["$..Payload", "$..LogResult"]
382477
)

0 commit comments

Comments
 (0)
0