8000 reduce docker desktop lambda creation times (#6595) · localstack/localstack@e269ebf · GitHub
[go: up one dir, main page]
Skip to content

Commit e269ebf

Browse files
dfangldfangl
authored
reduce docker desktop lambda creation times (#6595)
1 parent 3f7a7fe commit e269ebf

File tree

5 files changed

+51
-56
lines changed

5 files changed

+51
-56
lines changed

‎localstack/services/awslambda/lambda_api.py

Lines changed: 21 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
event_source_arn_matches,
3939
get_executor_mode,
4040
get_handler_file_from_name,
41+
get_lambda_extraction_dir,
4142
get_lambda_runtime,
4243
get_zip_bytes,
4344
multi_value_dict_for_list,
@@ -709,13 +710,16 @@ def set_archive_code(code: Dict, lambda_name: str, zip_file_content: bytes = Non
709710
latest_version = lambda_details.get_version(VERSION_LATEST)
710711
latest_version["CodeSize"] = len(zip_file_content)
711712
latest_version["CodeSha256"] = code_sha_256.decode("utf-8")
712-
tmp_dir = "%s/zipfile.%s" % (config.dirs.tmp, short_uid())
713-
mkdir(tmp_dir)
714-
tmp_file = "%s/%s" % (tmp_dir, LAMBDA_ZIP_FILE_NAME)
713+
zip_dir_name = f"function.zipfile.{short_uid()}"
714+
zip_dir = f"{config.dirs.tmp}/{zip_dir_name}"
715+
mkdir(zip_dir)
716+
tmp_file = f"{zip_dir}/{LAMBDA_ZIP_FILE_NAME}"
715717
save_file(tmp_file, zip_file_content)
716-
TMP_FILES.append(tmp_dir)
717-
lambda_details.cwd = tmp_dir
718-
return tmp_dir
718+
TMP_FILES.append(zip_dir)
719+
lambda_details.zip_dir = zip_dir
720+
lambda_details.cwd = f"{get_lambda_extraction_dir()}/{zip_dir_name}"
721+
mkdir(lambda_details.cwd)
722+
return zip_dir
719723

720724

721725
def set_function_code(lambda_function: LambdaFunction):
@@ -749,22 +753,22 @@ def store_and_get_lambda_code_archive(
749753
in case this is a Lambda with the special bucket marker __local__, used for code mounting."""
750754
code_passed = lambda_function.code
751755
is_local_mount = code_passed.get("S3Bucket") == config.BUCKET_MARKER_LOCAL
752-
lambda_cwd = lambda_function.cwd
756+
lambda_zip_dir = lambda_function.zip_dir
753757

754758
if code_passed:
755-
lambda_cwd = lambda_cwd or set_archive_code(code_passed, lambda_function.name())
759+
lambda_zip_dir = lambda_zip_dir or set_archive_code(code_passed, lambda_function.name())
756760
if not zip_file_content and not is_local_mount:
757761
# Save the zip file to a temporary file that the lambda executors can reference
758762
zip_file_content = get_zip_bytes(code_passed)
759763
else:
760764
lambda_details = LambdaRegion.get().lambdas[lambda_function.arn()]
761-
lambda_cwd = lambda_cwd or lambda_details.cwd
765+
lambda_zip_dir = lambda_zip_dir or lambda_details.zip_dir
762766

763-
if not lambda_cwd:
767+
if not lambda_zip_dir:
764768
return
765769

766770
# construct archive name
767-
archive_file = os.path.join(lambda_cwd, LAMBDA_ZIP_FILE_NAME)
771+
archive_file = os.path.join(lambda_zip_dir, LAMBDA_ZIP_FILE_NAME)
768772

769773
if not zip_file_content:
770774
zip_file_content = load_file(archive_file, mode="rb")
@@ -773,7 +777,7 @@ def store_and_get_lambda_code_archive(
773777
save_file(archive_file, zip_file_content)
774778
# remove content from code attribute, if present
775779
lambda_function.code.pop("ZipFile", None)
776-
return lambda_cwd, archive_file, zip_file_content
780+
return lambda_zip_dir, archive_file, zip_file_content
777781

778782

779783
def do_set_function_code(lambda_function: LambdaFunction):
@@ -804,7 +808,8 @@ def generic_handler(*_):
804808
_result = store_and_get_lambda_code_archive(lambda_function)
805809
if not _result:
806810
return
807-
lambda_cwd, archive_file, zip_file_content = _result
811+
lambda_zip_dir, archive_file, zip_file_content = _result
812+
lambda_cwd = lambda_function.cwd
808813

809814
# Set the appropriate Lambda handler.
810815
lambda_handler = generic_handler
@@ -841,19 +846,15 @@ def generic_handler(*_):
841846
# Obtain handler details for any non-Java Lambda function
842847
if not is_java:
843848
handler_file = get_handler_file_from_name(handler_name, runtime=runtime)
844-
main_file = "%s/%s" % (lambda_cwd, handler_file)
849+
main_file = f"{lambda_cwd}/{handler_file}"
845850

846851
if CHECK_HANDLER_ON_CREATION and not os.path.exists(main_file):
847852
# Raise an error if (1) this is not a local mount lambda, or (2) we're
848853
# running Lambdas locally (not in Docker), or (3) we're using remote Docker.
849854
# -> We do *not* want to raise an error if we're using local mount in non-remote Docker
850855
if not is_local_mount or not use_docker() or config.LAMBDA_REMOTE_DOCKER:
851-
file_list = run('cd "%s"; du -d 3 .' % lambda_cwd)
852-
config_debug = 'Config for local mount, docker, remote: "%s", "%s", "%s"' % (
853-
is_local_mount,
854-
use_docker(),
855-
config.LAMBDA_REMOTE_DOCKER,
856-
)
856+
file_list = run(f'cd "{lambda_cwd}"; du -d 3 .')
857+
config_debug = f'Config for local mount, docker, remote: "{is_local_mount}", "{use_docker()}", "{config.LAMBDA_REMOTE_DOCKER}"'
857858
LOG.debug("Lambda archive content:\n%s", file_list)
858859
raise ClientError(
859860
error_response(
@@ -1277,27 +1278,6 @@ def update_function_code(function):
12771278
return jsonify(result or {})
12781279

12791280

1280-
@app.route("%s/functions/<function>/code" % API_PATH_ROOT, methods=["GET"])
1281-
def get_function_code(function):
1282-
"""Get the code of an existing function
1283-
---
1284-
operationId: 'getFunctionCode'
1285-
parameters:
1286-
"""
1287-
region = LambdaRegion.get()
1288-
arn = func_arn(function)
1289-
lambda_function = region.lambdas.get(arn)
1290-
if not lambda_function:
1291-
return not_found_error(arn)
1292-
lambda_cwd = lambda_function.cwd
1293-
tmp_file = "%s/%s" % (lambda_cwd, LAMBDA_ZIP_FILE_NAME)
1294-
return Response(
1295-
load_file(tmp_file, mode="rb"),
1296-
mimetype="application/zip",
1297-
headers={"Content-Disposition": "attachment; filename=lambda_archive.zip"},
1298-
)
1299-
1300-
13011281
@app.route("%s/functions/<function>/configuration" % API_PATH_ROOT, methods=["GET"])
13021282
def get_function_configuration(function):
13031283
"""Get the configuration of an existing function

‎localstack/services/awslambda/lambda_executors.py

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1449,7 +1449,7 @@ def execute_java_lambda(
14491449

14501450
classpath = "%s:%s:%s" % (
14511451
main_file,
1452-
Util.get_java_classpath(main_file),
1452+
Util.get_java_classpath(lambda_function.cwd),
14531453
LAMBDA_EXECUTOR_JAR,
14541454
)
14551455
cmd = "java %s -cp %s %s %s" % (
@@ -1628,27 +1628,25 @@ def docker_image_for_lambda(cls, lambda_function: LambdaFunction):
16281628
return "%s:%s" % (docker_image, docker_tag)
16291629

16301630
@classmethod
1631-
def get_java_classpath(cls, archive):
1631+
def get_java_classpath(cls, lambda_cwd):
16321632
"""
1633-
Return the Java classpath, using the parent folder of the
1634-
given archive as the base folder.
1633+
Return the Java classpath, using the given working directory as the base folder.
16351634
1636-
The result contains any *.jar files in the base folder, as
1635+
The result contains any *.jar files in the workdir folder, as
16371636
well as any JAR files in the "lib/*" subfolder living
16381637
alongside the supplied java archive (.jar or .zip).
16391638
1640-
:param archive: an absolute path to a .jar or .zip Java archive
1641-
:return: the Java classpath, relative to the base dir of "archive"
1639+
:param lambda_cwd: an absolute path to a working directory folder of a java lambda
1640+
:return: the Java classpath, relative to the base dir of the working directory
16421641
"""
16431642
entries = ["."]
1644-
base_dir = os.path.dirname(archive)
16451643
for pattern in ["%s/*.jar", "%s/lib/*.jar", "%s/java/lib/*.jar", "%s/*.zip"]:
1646-
for entry in glob.glob(pattern % base_dir):
1647-
if os.path.realpath(archive) != os.path.realpath(entry):
1648-
entries.append(os.path.relpath(entry, base_dir))
1644+
for entry in glob.glob(pattern % lambda_cwd):
1645+
if os.path.realpath(lambda_cwd) != os.path.realpath(entry):
1646+
entries.append(os.path.relpath(entry, lambda_cwd))
16491647
# make sure to append the localstack-utils.jar at the end of the classpath
16501648
# https://github.com/localstack/localstack/issues/1160
1651-
entries.append(os.path.relpath(archive, base_dir))
1649+
entries.append(os.path.relpath(lambda_cwd, lambda_cwd))
16521650
entries.append("*.jar")
16531651
entries.append("java/lib/*.jar")
16541652
result = ":".join(entries)

‎localstack/services/awslambda/lambda_utils.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import logging
33
import os
44
import re
5+
import tempfile
56
import time
67
from collections import defaultdict
78
from functools import lru_cache
@@ -223,6 +224,21 @@ def get_record_from_event(event: Dict, key: str) -> Any:
223224
return None
224225

225226

227+
def get_lambda_extraction_dir() -> str:
228+
"""
229+
Get the directory a lambda is supposed to use as working directory (= the directory to extract the contents to).
230+
This method is needed due to performance problems for IO on bind volumes when running inside Docker Desktop, due to
231+
the file sharing with the host being slow when using gRPC-FUSE.
232+
By extracting to a not-mounted directory, we can improve performance significantly.
233+
The lambda zip file itself, however, should still be located on the mount.
234+
235+
:return: directory path
236+
"""
237+
if config.LAMBDA_REMOTE_DOCKER:
238+
return tempfile.gettempdir()
239+
return config.dirs.tmp
240+
241+
226242
def get_zip_bytes(function_code):
227243
"""Returns the ZIP file contents from a FunctionCode dict.
228244

‎localstack/utils/aws/aws_models.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ def __init__(self, arn):
187187
self.runtime = None
188188
self.handler = None
189189
self.cwd = None
190+
self.zip_dir = None
190191
self.timeout = None
191192
self.last_modified = None
192193
self.vpc_config = None

‎tests/unit/test_lambda.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -971,7 +971,7 @@ def prepare_java_opts(self, java_opts):
971971
def test_get_java_lib_folder_classpath(self):
972972
jar_file = os.path.join(new_tmp_dir(), "foo.jar")
973973
save_file(jar_file, "")
974-
classpath = lambda_executors.Util.get_java_classpath(jar_file)
974+
classpath = lambda_executors.Util.get_java_classpath(os.path.dirname(jar_file))
975975
self.assertIn(".:foo.jar", classpath)
976976
self.assertIn("*.jar", classpath)
977977

@@ -982,13 +982,13 @@ def test_get_java_lib_folder_classpath_no_directories(self):
982982
lib_file = os.path.join(base_dir, "lib", "lib.jar")
983983
mkdir(os.path.dirname(lib_file))
984984
save_file(lib_file, "")
985-
classpath = lambda_executors.Util.get_java_classpath(jar_file)
985+
classpath = lambda_executors.Util.get_java_classpath(os.path.dirname(jar_file))
986986
self.assertIn(":foo.jar", classpath)
987987
self.assertIn("lib/lib.jar:", classpath)
988988
self.assertIn(":*.jar", classpath)
989989

990990
def test_get_java_lib_folder_classpath_archive_is_None(self):
991-
self.assertRaises(TypeError, lambda_executors.Util.get_java_classpath, None)
991+
self.assertRaises(ValueError, lambda_executors.Util.get_java_classpath, None)
992992

993993
@mock.patch("localstack.utils.cloudwatch.cloudwatch_util.store_cloudwatch_logs")
994994
def test_executor_store_logs_can_handle_milliseconds(self, mock_store_cloudwatch_logs):

0 commit comments

Comments
 (0)
0