10000 Add Lambda runtime parity configuration for ulimit (#7871) · codeperl/localstack@77bd2c2 · GitHub
[go: up one dir, main page]
Skip to content

Commit 77bd2c2

Browse files
joe4devjoe4dev
authored
Add Lambda runtime parity configuration for ulimit (localstack#7871)
1 parent 10e77c0 commit 77bd2c2

File tree

8 files changed

+500
-235
lines changed

8 files changed

+500
-235
lines changed

‎localstack/utils/container_utils/container_client.py

Lines changed: 203 additions & 126 deletions
Large diffs are not rendered by default.

‎localstack/utils/container_utils/docker_cmd_client.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
PortMappings,
2323
RegistryConnectionError,
2424
SimpleVolumeBind,
25+
Ulimit,
2526
Util,
2627
VolumeBind,
2728
)
@@ -629,6 +630,7 @@ def _build_run_create_cmd(
629630
privileged: Optional[bool] = None,
630631
labels: Optional[Dict[str, str]] = None,
631632
platform: Optional[DockerPlatform] = None,
633+
ulimits: Optional[List[Ulimit]] = None,
632634
) -> Tuple[List[str], str]:
633635
env_file = None
634636
cmd = self._docker_cmd() + [action]
@@ -678,6 +680,10 @@ def _build_run_create_cmd(
678680
cmd += ["--label", f"{key}={value}"]
679681
if platform:
680682
cmd += ["--platform", platform]
683+
if ulimits:
684+
cmd += list(
685+
itertools.chain.from_iterable(["--ulimits", str(ulimit)] for ulimit in ulimits)
686+
)
681687

682688
if additional_flags:
683689
cmd += shlex.split(additional_flags)

‎localstack/utils/container_utils/docker_sdk_client.py

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
PortMappings,
2525
RegistryConnectionError,
2626
SimpleVolumeBind,
27+
Ulimit,
2728
Util,
2829
)
2930
from localstack.utils.strings import to_bytes, to_str
@@ -516,29 +517,32 @@ def create_container(
516517
privileged: Optional[bool] = None,
517518
labels: Optional[Dict[str, str]] = None,
518519
platform: Optional[DockerPlatform] = None,
520+
ulimits: Optional[List[Ulimit]] = None,
519521
) -> str:
520522
LOG.debug("Creating container with attributes: %s", locals())
521523
extra_hosts = None
522524
if additional_flags:
523525
parsed_flags = Util.parse_additional_flags(
524526
additional_flags,
525-
env_vars,
526-
ports,
527-
mount_volumes,
528-
network,
529-
user,
530-
platform,
531-
privileged,
527+
env_vars=env_vars,
528+
mounts=mount_volumes,
529+
network=network,
530+
platform=platform,
531+
privileged=privileged,
532+
ports=ports,
533+
ulimits=ulimits,
534+
user=user,
532535
)
533536
env_vars = parsed_flags.env_vars
534-
ports = parsed_flags.ports
535-
mount_volumes = parsed_flags.mounts
536537
extra_hosts = parsed_flags.extra_hosts
537-
network = parsed_flags.network
538+
mount_volumes = parsed_flags.mounts
538539
labels = parsed_flags.labels
539-
user = parsed_flags.user
540+
network = parsed_flags.network
540541
platform = parsed_flags.platform
541542
privileged = parsed_flags.privileged
543+
ports = parsed_flags.ports
544+
ulimits = parsed_flags.ulimits
545+
user = parsed_flags.user
542546

543547
try:
544548
kwargs = {}
@@ -558,6 +562,13 @@ def create_container(
558562
kwargs["privileged"] = True
559563
if labels:
560564
kwargs["labels"] = labels
565+
if ulimits:
566+
kwargs["ulimits"] = [
567+
docker.types.Ulimit(
568+
name=ulimit.name, soft=ulimit.soft_limit, hard=ulimit.hard_limit
569+
)
570+
for ulimit in ulimits
571+
]
561572
mounts = None
562573
if mount_volumes:
563574
mounts = Util.convert_mount_list_to_dict(mount_volumes)
@@ -615,11 +626,21 @@ def run_container(
615626
dns: Optional[str] = None,
616627
additional_flags: Optional[str] = None,
617628
workdir: Optional[str] = None,
629+
platform: Optional[DockerPlatform] = None,
618630
privileged: Optional[bool] = None,
631+
ulimits: Optional[List[Ulimit]] = None,
619632
) -> Tuple[bytes, bytes]:
620633
LOG.debug("Running container with image: %s", image_name)
621634
container = None
622635
try:
636+
kwargs = {}
637+
if ulimits:
638+
kwargs["ulimits"] = [
639+
docker.types.Ulimit(
640+
name=ulimit.name, soft=ulimit.soft_limit, hard=ulimit.hard_limit
641+
)
642+
for ulimit in ulimits
643+
]
623644
container = self.create_container(
624645
image_name,
625646
name=name,
@@ -641,6 +662,8 @@ def run_container(
641662
additional_flags=additional_flags,
642663
workdir=workdir,
643664
privileged=privileged,
665+
platform=platform,
666+
**kwargs,
644667
)
645668
result = self.start_container(
646669
container_name_or_id=container,

‎localstack/utils/no_exit_argument_parser.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import argparse
2+
import logging
3+
from typing import NoReturn, Optional
4+
5+
LOG = logging.getLogger(__name__)
6+
7+
8+
class NoExitArgumentParser(argparse.ArgumentParser):
9+
"""Implements the `exit_on_error=False` behavior introduced in Python 3.9 to support older Python versions
10+
and prevents further SystemExit for other error categories.
11+
* Limitations of error categories: https://stackoverflow.com/a/67891066/6875981
12+
* ArgumentParser subclassing example: https://stackoverflow.com/a/59072378/6875981
13+
"""
14+
15+
def exit(self, status: int = ..., message: Optional[str] = ...) -> NoReturn:
16+
LOG.warning(f"Error in argument parser but preventing exit: {message}")
17+
18+
def error(self, message: str) -> NoReturn:
19+
raise NotImplementedError(f"Unsupported flag by this Docker client: {message}")

‎tests/integration/awslambda/functions/lambda_ulimits.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import resource
2+
3+
4+
def handler(event, context):
5+
# https://docs.python.org/3/library/resource.html
6+
ulimit_names = {
7+
"RLIMIT_AS": resource.RLIMIT_AS,
8+
"RLIMIT_CORE": resource.RLIMIT_CORE,
9+
"RLIMIT_CPU": resource.RLIMIT_CPU,
10+
"RLIMIT_DATA": resource.RLIMIT_DATA,
11+
"RLIMIT_FSIZE": resource.RLIMIT_FSIZE,
12+
"RLIMIT_MEMLOCK": resource.RLIMIT_M 10000 EMLOCK,
13+
"RLIMIT_NOFILE": resource.RLIMIT_NOFILE,
14+
"RLIMIT_NPROC": resource.RLIMIT_NPROC,
15+
"RLIMIT_RSS": resource.RLIMIT_RSS,
16+
"RLIMIT_STACK": resource.RLIMIT_STACK,
17+
}
18+
return {label: resource.getrlimit(res) for label, res in ulimit_names.items()}

‎tests/integration/awslambda/test_lambda.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
TEST_LAMBDA_TIMEOUT_ENV_PYTHON = os.path.join(THIS_FOLDER, "functions/lambda_timeout_env.py")
8484
TEST_LAMBDA_SLEEP_ENVIRONMENT = os.path.join(THIS_FOLDER, "functions/lambda_sleep_environment.py")
8585
TEST_LAMBDA_INTROSPECT_PYTHON = os.path.join(THIS_FOLDER, "functions/lambda_introspect.py")
86+
TEST_LAMBDA_ULIMITS = os.path.join(THIS_FOLDER, "functions/lambda_ulimits.py")
8687
TEST_LAMBDA_VERSION = os.path.join(THIS_FOLDER, "functions/lambda_version.py")
8788

8889
TEST_GOLANG_LAMBDA_URL_TEMPLATE = "https://github.com/localstack/awslamba-go-runtime/releases/download/v{version}/example-handler-{os}-{arch}.tar.gz"
@@ -379,6 +380,31 @@ def test_runtime_introspection_arm(self, lambda_client, create_lambda_function,
379380
invoke_result = lambda_client.invoke(FunctionName=func_name)
380381
snapshot.match("invoke_runtime_arm_introspection", invoke_result)
381382

383+
@pytest.mark.skipif(
384+
is_old_local_executor(),
385+
reason="Monkey-patching of Docker flags is not applicable because no new container is spawned",
386+
)
387+
@pytest.mark.skip_snapshot_verify(condition=is_old_provider, paths=["$..LogResult"])
388+
@pytest.mark.aws_validated
389+
def test_runtime_ulimits(self, lambda_client, create_lambda_function, snapshot, monkeypatch):
390+
"""We consider ulimits parity as opt-in because development environments could hit these limits unlike in
391+
optimized production deployments."""
392+
monkeypatch.setattr(
393+
config,
394+
"LAMBDA_DOCKER_FLAGS",
395+
"--ulimit nofile=1024:1024 --ulimit nproc=735:735 --ulimit core=-1:-1 --ulimit stack=8388608:-1",
396+
)
397+
398+
func_name = f"test_lambda_ulimits_{short_uid()}"
399+
create_lambda_function(
400+
func_name=func_name,
401+
handler_file=TEST_LAMBDA_ULIMITS,
402+
runtime=Runtime.python3_9,
403+
)
404+
405+
invoke_result = lambda_client.invoke(FunctionName=func_name)
406+
snapshot.match("invoke_runtime_ulimits", invoke_result)
407+
382408
@pytest.mark.skipif(is_old_provider(), reason="unsupported in old provider")
383409
@pytest.mark.skipif(
384410
is_old_local_executor(),

‎tests/integration/awslambda/test_lambda.snapshot.json

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2658,5 +2658,60 @@
26582658
}
26592659
}
26602660
}
2661+
},
2662+
"tests/integration/awslambda/test_lambda.py::TestLambdaBehavior::test_runtime_ulimits": {
2663+
"recorded-date": "15-03-2023, 00:11:15",
2664+
"recorded-content": {
2665+
"invoke_runtime_ulimits": {
2666+
"ExecutedVersion": "$LATEST",
2667+
"Payload": {
2668+
"RLIMIT_AS": [
2669+
-1,
2670+
-1
2671+
],
2672+
"RLIMIT_CORE": [
2673+
-1,
2674+
-1
2675+
],
2676+
"RLIMIT_CPU": [
2677+
-1,
2678+
-1
2679+
],
2680+
"RLIMIT_DATA": [
2681+
-1,
2682+
-1
2683+
],
2684+
"RLIMIT_FSIZE": [
2685+
-1,
2686+
-1
2687+
],
2688+
"RLIMIT_MEMLOCK": [
2689+
65536,
2690+
65536
2691+
],
2692+
"RLIMIT_NOFILE": [
2693+
1024,
2694+
1024
2695+
],
2696+
"RLIMIT_NPROC": [
2697+
735,
2698+
735
2699+
],
2700+
"RLIMIT_RSS": [
2701+
-1,
2702+
-1
2703+
],
2704+
"RLIMIT_STACK": [
2705+
8388608,
2706+
-1
2707+
]
2708+
},
2709+
"StatusCode": 200,
2710+
"ResponseMetadata": {
2711+
"HTTPHeaders": {},
2712+
"HTTPStatusCode": 200
2713+
}
2714+
}
2715+
}
26612716
}
26622717
}

0 commit comments

Comments
 (0)
0