8000 Fix docker runtime executor flags (#7675) · localstack/localstack@27b4e32 · GitHub
[go: up one dir, main page]
Skip to content

Commit 27b4e32

Browse files
joe4devjoe4dev
authored
Fix docker runtime executor flags (#7675)
1 parent 7159b90 commit 27b4e32

File tree

14 files changed

+309
-30
lines changed

14 files changed

+309
-30
lines changed

‎localstack/services/awslambda/invocation/docker_runtime_executor.py

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from typing import Callable, Dict, Literal, Optional
88

99
from localstack import config
10-
from localstack.aws.api.lambda_ import PackageType, Runtime
10+
from localstack.aws.api.lambda_ import Architecture, PackageType, Runtime
1111
from localstack.services.awslambda import hooks as lambda_hooks
1212
from localstack.services.awslambda.invocation.executor_endpoint import (
1313
INVOCATION_PORT,
@@ -27,6 +27,7 @@
2727
from localstack.utils.container_networking import get_main_container_name
2828
from localstack.utils.container_utils.container_client import (
2929
ContainerConfiguration,
30+
DockerPlatform,
3031
PortMappings,
3132
VolumeBind,
3233
VolumeMappings,
@@ -54,6 +55,27 @@
5455
HOT_RELOADING_ENV_VARIABLE = "LOCALSTACK_HOT_RELOADING_PATHS"
5556

5657

58+
"""Map AWS Lambda architecture to Docker platform flags. Example: arm64 => linux/arm64"""
59+
ARCHITECTURE_PLATFORM_MAPPING: dict[Architecture, DockerPlatform] = dict(
60+
{
61+
Architecture.x86_64: DockerPlatform.linux_amd64,
62+
Architecture.arm64: DockerPlatform.linux_arm64,
63+
}
64+
)
65+
66+
67+
def docker_platform(lambda_architecture: Architecture) -> DockerPlatform:
68+
"""
69+
Convert an AWS Lambda architecture into a Docker platform flag. Examples:
70+
* docker_platform("x86_64") == "linux/amd64"
71+
* docker_platform("arm64") == "linux/arm64"
72+
73+
:param lambda_architecture: the instruction set that the function supports
74+
:return: Docker platform in the format ``os[/arch[/variant]]``
75+
"""
76+
return ARCHITECTURE_PLATFORM_MAPPING[lambda_architecture]
77+
78+
5779
def get_image_name_for_function(function_version: FunctionVersion) -> str:
5880
return f"localstack/lambda-{function_version.id.qualified_arn().replace(':', '_').replace('$', '_').lower()}"
5981

@@ -245,6 +267,7 @@ def start(self, env_vars: dict[str, str]) -> None:
245267
env_vars=env_vars,
246268
network=network,
247269
entrypoint=RAPID_ENTRYPOINT,
270+
platform=docker_platform(self.function_version.config.architectures[0]),
248271
additional_flags=config.LAMBDA_DOCKER_FLAGS,
249272
)
250273
if self.function_version.config.package_type == PackageType.Zip:

‎localstack/services/awslambda/provider.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -551,8 +551,18 @@ def create_function(
551551
)
552552

553553
architectures = request.get("Architectures")
554-
if architectures and Architecture.arm64 in architectures:
555-
raise ServiceException("ARM64 is currently not supported by this provider")
554+
if architectures:
555+
if len(architectures) != 1:
556+
raise ValidationException(
557+
f"1 validation error detected: Value '[{', '.join(architectures)}]' at 'architectures' failed to "
558+
f"satisfy constraint: Member must have length less than or equal to 1",
559+
)
560+
if architectures[0] not in [Architecture.x86_64, Architecture.arm64]:
561+
raise ValidationException(
562+
f"1 validation error detected: Value '[{', '.join(architectures)}]' at 'architectures' failed to "
563+ f"satisfy constraint: Member must satisfy constraint: [Member must satisfy enum value set: "
564+
f"[x86_64, arm64], Member must not be null]",
565+
)
556566

557567
if env_vars := request.get("Environment", {}).get("Variables"):
558568
self._verify_env_variables(env_vars)
@@ -568,7 +578,7 @@ def create_function(
568578
package_type = request.get("PackageType", PackageType.Zip)
569579
if package_type == PackageType.Zip and request.get("Runtime") not in IMAGE_MAPPING:
570580
raise InvalidParameterValueException(
571-
f"Value {request.get('Runtime')} at 'runtime' failed to satisfy constraint: Member must satisfy enum value set: [nodejs12.x, provided, nodejs16.x, nodejs14.x, ruby2.7, java11, dotnet6, go1.x, provided.al2, java8, java8.al2, dotnetcore3.1, python3.7, python3.8, python3.9] or be a valid ARN",
581+
f"Value {request.get('Runtime')} at 'runtime' failed to satisfy constraint: Member must satisfy enum value set: [nodejs12.x, provided, nodejs16.x, nodejs14.x, ruby2.7, java11, dotnet6, go1.x, nodejs18.x, provided.al2, java8, java8.al2, dotnetcore3.1, python3.7, python3.8, python3.9] or be a valid ARN",
572582
Type="User",
573583
)
574584
state = lambda_stores[context.account_id][context.region]
@@ -637,7 +647,7 @@ def create_function(
637647
package_type=package_type,
638648
reserved_concurrent_executions=0,
639649
environment=env_vars,
640-
architectures=request.get("Architectures") or ["x86_64"], # TODO
650+
architectures=request.get("Architectures") or [Architecture.x86_64],
641651
tracing_config_mode=TracingMode.PassThrough, # TODO
642652
image=image,
643653
image_config=image_config,

‎localstack/testing/aws/lambda_utils.py

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import json
22
import os
3+
import platform
34
from typing import Literal
45

56
from localstack.utils.common import to_str
@@ -117,14 +118,16 @@ def get_events():
117118

118119

119120
def is_old_provider():
120-
return (
121-
os.environ.get("TEST_TARGET") != "AWS_CLOUD"
122-
and os.environ.get("PROVIDER_OVERRIDE_LAMBDA") != "asf"
123-
)
121+
return os.environ.get("TEST_TARGET") != "AWS_CLOUD" and os.environ.get(
122+
"PROVIDER_OVERRIDE_LAMBDA"
123+
) not in ["asf", "v2"]
124124

125125

126126
def is_new_provider():
127-
return (
128-
os.environ.get("TEST_TARGET") != "AWS_CLOUD"
129-
and os.environ.get("PROVIDER_OVERRIDE_LAMBDA") == "asf"
130-
)
127+
return os.environ.get("TEST_TARGET") != "AWS_CLOUD" and os.environ.get(
128+
"PROVIDER_OVERRIDE_LAMBDA"
129+
) in ["asf", "v2"]
130+
131+
132+
def is_arm_compatible():
133+
return platform.machine() == "arm64"

‎localstack/utils/container_utils/container_client.py

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,13 @@ def close(self):
9696
raise NotImplementedError
9797

9898

99+
class DockerPlatform(str):
100+
"""Platform in the format ``os[/arch[/variant]]``"""
101+
102+
linux_amd64 = "linux/amd64"
103+
linux_arm64 = "linux/arm64"
104+
105+
99106
# defines the type for port mappings (source->target port range)
100107
PortRange = Union[List, HashableList]
101108

@@ -366,6 +373,7 @@ class ContainerConfiguration:
366373
network: Optional[str] = None
367374
dns: Optional[str] = None
368375
workdir: Optional[str] = None
376+
platform: Optional[str] = None
369377

370378

371379
@dataclasses.dataclass
@@ -378,6 +386,8 @@ class DockerRunFlags:
378386
extra_hosts: Optional[Dict[str, str]]
379387
network: Optional[str]
380388
labels: Optional[Dict[str, str]]
389+
user: Optional[str]
390+
platform: Optional[DockerPlatform]
381391

382392

383393
class ContainerClient(metaclass=ABCMeta):
@@ -496,7 +506,7 @@ def copy_from_container(
496506
pass
497507

498508
@abstractmethod
499-
def pull_image(self, docker_image: str) -> None:
509+
def pull_image(self, docker_image: str, platform: Optional[DockerPlatform] = None) -> None:
500510
"""Pulls an image with a given name from a Docker registry"""
501511
pass
502512

@@ -687,6 +697,7 @@ def create_container_from_config(self, container_config: ContainerConfiguration)
687697
additional_flags=container_config.additional_flags,
688< 9E7A code>698
workdir=container_config.workdir,
689699
privileged=container_config.privileged,
700+
platform=container_config.platform,
690701
)
691702

692703
@abstractmethod
@@ -714,6 +725,7 @@ def create_container(
714725
workdir: Optional[str] = None,
715726
privileged: Optional[bool] = None,
716727
labels: Optional[Dict[str, str]] = None,
728+
platform: Optional[DockerPlatform] = None,
717729
) -> str:
718730
"""Creates a container with the given image
719731
@@ -888,13 +900,17 @@ def parse_additional_flags(
888900
ports: PortMappings = None,
889901
mounts: List[SimpleVolumeBind] = None,
890902
network: Optional[str] = None,
903+
user: Optional[str] = None,
904+
platform: Optional[DockerPlatform] = None,
891905
) -> DockerRunFlags:
892906
"""Parses environment, volume and port flags passed as string
893907
:param additional_flags: String which contains the flag definitions
894908
:param env_vars: Dict with env vars. Will be modified in place.
895909
:param ports: PortMapping object. Will be modified in place.
896910
:param mounts: List of mount tuples (host_path, container_path). Will be modified in place.
897911
:param network: Existing network name (optional). Warning will be printed if network is overwritten in flags.
912+
:param user: User to run firs F438 t process. Warning will be printed if user is overwritten in flags.
913+
:param platform: Platform to execute container. Warning will be printed if platform is overwritten in flags.
898914
:return: A DockerRunFlags object containing the env_vars, ports, mount, extra_hosts, network, and labels.
899915
The result will return new objects if respective parameters were None and additional flags contained
900916
a flag for that object, the same which are passed otherwise.
@@ -917,6 +933,10 @@ def parse_additional_flags(
917933
cur_state = "set-network"
918934
elif flag == "--label":
919935
cur_state = "add-label"
936+
elif flag in ["-u", "--user"]:
937+
cur_state = "user"
938+
elif flag == "--platform":
939+
cur_state = "platform"
920940
else:
921941
raise NotImplementedError(
922942
f"Flag {flag} is currently not supported by this Docker client."
@@ -982,6 +1002,18 @@ def parse_additional_flags(
9821002
labels[key] = value
9831003
else:
9841004
LOG.warning("Invalid --label specified, unable to parse: '%s'", flag)
1005+
elif cur_state == "user":
1006+
if user:
1007+
LOG.warning(
1008+
f"Overwriting Docker container user {user} with new value {flag}"
1009+
)
1010+
user = flag
1011+
elif cur_state == "platform":
1012+
if platform:
1013+
LOG.warning(
1014+
f"Overwriting Docker container platform {platform} with new value {flag}"
1015+
)
1016+
platform = flag
9851017

9861018
cur_state = None
9871019

@@ -992,6 +1024,8 @@ def parse_additional_flags(
9921024
extra_hosts=extra_hosts,
9931025
network=network,
9941026
labels=labels,
1027+
user=user,
1028+
platform=platform,
9951029
)
9961030

9971031
@staticmethod

‎localstack/utils/container_utils/docker_cmd_client.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
ContainerClient,
1515
ContainerException,
1616
DockerContainerStatus,
17+
DockerPlatform,
1718
NoSuchContainer,
1819
NoSuchImage,
1920
NoSuchNetwork,
@@ -263,9 +264,11 @@ def copy_from_container(
263264
"Docker process returned with errorcode %s" % e.returncode, e.stdout, e.stderr
264265
) from e
265266

266-
def pull_image(self, docker_image: str) -> None:
267+
def pull_image(self, docker_image: str, platform: Optional[DockerPlatform] = None) -> None:
267268
cmd = self._docker_cmd()
268269
cmd += ["pull", docker_image]
270+
if platform:
271+
cmd += ["--platform", platform]
269272
LOG.debug("Pulling image with cmd: %s", cmd)
270273
try:
271274
run(cmd)
@@ -616,6 +619,7 @@ def _build_run_create_cmd(
616619
workdir: Optional[str] = None,
617620
privileged: Optional[bool] = None,
618621
labels: Optional[Dict[str, str]] = None,
622+
platform: Optional[DockerPlatform] = None,
619623
) -> Tuple[List[str], str]:
620624
env_file = None
621625
cmd = self._docker_cmd() + [action]
@@ -663,6 +667,8 @@ def _build_run_create_cmd(
663667
if labels:
664668
for key, value in labels.items():
665669
cmd += ["--label", f"{key}={value}"]
670+
if platform:
671+
cmd += ["--platform", platform]
666672

667673
if additional_flags:
668674
cmd += shlex.split(additional_flags)

‎localstack/utils/container_utils/docker_sdk_client.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
ContainerClient,
1818
ContainerException,
1919
DockerContainerStatus,
20+
DockerPlatform,
2021
NoSuchContainer,
2122
NoSuchImage,
2223
NoSuchNetwork,
@@ -213,11 +214,11 @@ def copy_from_container(
213214
except APIError as e:
214215
raise ContainerException() from e
215216

216-
def pull_image(self, docker_image: str) -> None:
217+
def pull_image(self, docker_image: str, platform: Optional[DockerPlatform] = None) -> None:
217218
LOG.debug("Pulling Docker image: %s", docker_image)
218219
# some path in the docker image string indicates a custom repository
219220
try:
220-
self.client().images.pull(docker_image)
221+
self.client().images.pull(docker_image, platform=platform)
221222
except ImageNotFound:
222223
raise NoSuchImage(docker_image)
223224
except APIError as e:
@@ -507,19 +508,22 @@ def create_container(
507508
workdir: Optional[str] = None,
508509
privileged: Optional[bool] = None,
509510
labels: Optional[Dict[str, str]] = None,
511+
platform: Optional[DockerPlatform] = None,
510512
) -> str:
511513
LOG.debug("Creating container with attributes: %s", locals())
512514
extra_hosts = None
513515
if additional_flags:
514516
parsed_flags = Util.parse_additional_flags(
515-
additional_flags, env_vars, ports, mount_volumes, network
517+
additional_flags, env_vars, ports, mount_volumes, network, user, platform
516518
)
517519
env_vars = parsed_flags.env_vars
518520
ports = parsed_flags.ports
519521
mount_volumes = parsed_flags.mounts
520522
extra_hosts = parsed_flags.extra_hosts
521523
network = parsed_flags.network
522524
labels = parsed_flags.labels
525+
user = parsed_flags.user
526+
platform = parsed_flags.platform
523527

524528
try:
525529
kwargs = {}
@@ -558,13 +562,14 @@ def create_container():
558562
network=network,
559563
volumes=mounts,
560564
extra_hosts=extra_hosts,
565+
platform=platform,
561566
**kwargs,
562567
)
563568

564569
try:
565570
container = create_container()
566571
except ImageNotFound:
567-
self.pull_image(image_name)
572+
self.pull_image(image_name, platform)
568573
container = create_container()
569574
return container.id
570575
except ImageNotFound:

‎setup.cfg

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ runtime =
7575
cryptography
7676
dnslib>=0.9.10
7777
dnspython>=1.16.0
78-
docker==5.0.0
78+
docker==6.0.1
7979
flask==2.1.3
8080
flask-cors>=3.0.3,<3.1.0
8181
flask_swagger==0.2.12

‎tests/bootstrap/test_cli.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,8 +180,11 @@ def test_container_starts_non_root(self, runner, monkeypatch, container_client):
180180
output = container_client.exec_in_container(config.MAIN_CONTAINER_NAME, ["ps", "-u", user])
181181
assert "supervisord" in to_str(output[0])
182182

183+
@pytest.mark.skip(reason="skip temporarily until compatible localstack image is on DockerHub")
183184
def test_start_cli_within_container(self, runner, container_client):
184185
output = container_client.run_container(
186+
# CAVEAT: Updates to the Docker image are not immediately reflected when using the latest image from
187+
# DockerHub in the CI. Re-build the Docker image locally through `make docker-build` for local testing.
185188
"localstack/localstack",
186189
remove=True,
187190
entrypoint="",

‎tests/integration/awslambda/functions/lambda_introspect.py

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1+
import getpass
12
import os
3+
import platform
4+
import stat
5+
import subprocess
26
import time
37

48

@@ -8,6 +12,14 @@ def handler(event, context):
812
time.sleep(event["wait"])
913

1014
return {
11-
"env": {k: v for k, v in os.environ.items()},
15+
# Tested in tests/integration/awslambda/test_lambda_common.py
16+
# "environment": dict(os.environ),
1217
"event": event,
18+
# user behavior: https://stackoverflow.com/a/25574419
19+
"user_login_name": getpass.getuser(),
20+
"user_whoami": subprocess.getoutput("whoami"),
21+
"platform_system": platform.system(),
22+
"platform_machine": platform.machine(),
23+
"pwd_filemode": stat.filemode(os.stat(".").st_mode),
24+
"opt_filemode": stat.filemode(os.stat("/opt").st_mode),
1325
}

0 commit comments

Comments
 (0)
0