8000 some preliminary work · localstack/localstack@15d4011 · GitHub
[go: up one dir, main page]
Skip to content

Commit 15d4011

Browse files
dfangldfangl
committed
some preliminary work
1 parent 7e3045d commit 15d4011

File tree

7 files changed

+317
-46
lines changed

7 files changed

+317
-46
lines changed

‎Dockerfile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ RUN apt-get update && \
5050
git make openssl tar pixz zip unzip groff-base iputils-ping nss-passwords \
5151
# Postgres
5252
postgresql postgresql-client postgresql-plpython3 \
53+
# OpenSSH client (for kubernetes proxy) \
54+
openssh-client \
5355
# NodeJS
5456
nodejs && \
5557
apt-get clean && rm -rf /var/lib/apt/lists/*

‎localstack/config.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -690,6 +690,12 @@ def in_docker():
690690
"ES_MULTI_CLUSTER"
691691
)
692692

693+
# Kubernetes hackathon config variables
694+
LAMBDA_KUBERNETES_IMAGE_PREFIX = (
695+
os.environ.get("LAMBDA_KUBERNETES_IMAGE_PREFIX") or "dfangl/localstack-lambda-images"
696+
)
697+
LAMBDA_KUBERNETES_ENABLED = is_env_not_false("LAMBDA_KUBERNETES_ENABLED")
698+
693699
# list of environment variable names used for configuration.
694700
# Make sure to keep this in sync with the above!
695701
# Note: do *not* include DATA_DIR in this list, as it is treated separately

‎localstack/services/awslambda/invocation/executor_endpoint.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
from localstack.utils.serving import Server
1616

1717
LOG = logging.getLogger(__name__)
18-
INVOCATION_PORT = 9563
1918

2019

2120
class InvokeSendError(Exception):
@@ -34,10 +33,12 @@ def __init__(
3433
service_endpoint: ServiceEndpoint,
3534
host: str = "0.0.0.0",
3635
container_address: Optional[str] = None,
36+
invocation_port: int = 9563,
3737
) -> None:
3838
super().__init__(port, host)
3939
self.service_endpoint = service_endpoint
4040
self.container_address = container_address
41+
self.invocation_port = invocation_port
4142

4243
def _create_endpoint(self) -> Flask:
4344
executor_endpoint = Flask(f"executor_endpoint_{self.port}")
@@ -96,7 +97,7 @@ def do_shutdown(self) -> None:
9697
def invoke(self, payload: Dict[str, str]) -> None:
9798
if not self.container_address:
9899
raise ValueError("Container address not set, but got an invoke.")
99-
invocation_url = f"http://{self.container_address}:{INVOCATION_PORT}/invoke"
100+
invocation_url = f"http://{self.container_address}:{self.invocation_port}/invoke"
100101
response = requests.post(url=invocation_url, json=payload)
101102
if not response.ok:
102103
raise InvokeSendError(

‎localstack/services/awslambda/invocation/runtime_environment.py

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
from datetime import date, datetime
55
from enum import Enum, auto
66
from threading import RLock, Timer
7-
from typing import TYPE_CHECKING, Dict, Literal, Optional
7+
from typing import TYPE_CHECKING, Dict, Literal, Optional, Type
88

99
from localstack import config
1010
from localstack.services.awslambda.invocation.executor_endpoint import ServiceEndpoint
@@ -42,6 +42,17 @@ def generate_runtime_id() -> str:
4242
return "".join(random.choices(string.hexdigits[:16], k=32)).lower()
4343

4444

45+
def get_runtime_executor() -> Type[RuntimeExecutor]:
46+
if config.LAMBDA_KUBERNETES_ENABLED:
47+
from localstack.services.awslambda.invocation.runtime_executor_kubernetes import (
48+
KubernetesRuntimeExecutor,
49+
)
50+
51+
return KubernetesRuntimeExecutor
52+
else:
53+
return RuntimeExecutor
54+
55+
4556
class RuntimeEnvironment:
4657
runtime_executor: RuntimeExecutor
4758
status_lock: RLock
@@ -61,7 +72,7 @@ def __init__(
6172
self.status_lock = RLock()
6273
self.function_version = function_version
6374
self.initialization_type = initialization_type
64-
self.runtime_executor = RuntimeExecutor(
75+
self.runtime_executor = get_runtime_executor()(
6576
self.id, function_version, service_endpoint=service_endpoint
6677
)
6778
self.last_returned = datetime.min
@@ -81,7 +92,7 @@ def get_environment_variables(self) -> Dict[str, str]:
8192
env_vars = {
8293
# Runtime API specifics
8394
"LOCALSTACK_RUNTIME_ID": self.id,
84-
"LOCALSTACK_RUNTIME_ENDPOINT": f"http://{self.runtime_executor.get_endpoint_from_executor()}:{self.runtime_executor.executor_endpoint.port}",
95+
"LOCALSTACK_RUNTIME_ENDPOINT": f"http://{self.runtime_executor.get_executor_endpoint_from_executor()}",
8596
# General Lambda Environment Variables
8697
"AWS_LAMBDA_LOG_GROUP_NAME": self.get_log_group_name(),
8798
"AWS_LAMBDA_LOG_STREAM_NAME": self.get_log_stream_name(),
@@ -101,9 +112,9 @@ def get_environment_variables(self) -> Dict[str, str]:
101112
"AWS_SESSION_TOKEN": "test",
102113
# TODO xray
103114
# LocalStack endpoint specifics
104-
"LOCALSTACK_HOSTNAME": self.runtime_executor.get_endpoint_from_executor(),
115+
"LOCALSTACK_HOSTNAME": self.runtime_executor.get_localstack_endpoint_from_executor(),
105116
"EDGE_PORT": str(config.EDGE_PORT),
106-
"AWS_ENDPOINT_URL": f"http://{self.runtime_executor.get_endpoint_from_executor()}:{config.EDGE_PORT}",
117+
"AWS_ENDPOINT_URL": f"http://{self.runtime_executor.get_localstack_endpoint_from_executor()}:{config.EDGE_PORT}",
107118
}
108119
if self.function_version.config.handler:
109120
env_vars["_HANDLER"] = self.function_version.config.handler

‎localstack/services/awslambda/invocation/runtime_executor.py

Lines changed: 36 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -112,38 +112,6 @@ def prepare_image(target_path: Path, function_version: FunctionVersion) -> None:
112112
)
113113

114114

115-
def prepare_version(function_version: FunctionVersion) -> None:
116-
if not function_version.code.zip_file:
117-
raise NotImplementedError("Images without zipfile are currently not supported")
118-
time_before = time.perf_counter()
119-
target_path = get_path_for_function(function_version)
120-
target_path.mkdir(parents=True, exist_ok=True)
121-
# write code to disk
122-
target_code = get_code_path_for_function(function_version)
123-
with NamedTemporaryFile() as file:
124-
file.write(function_version.code.zip_file)
125-
file.flush()
126-
unzip(file.name, str(target_code))
127-
if config.LAMBDA_PREBUILD_IMAGES:
128-
prepare_image(target_path, function_version)
129-
LOG.debug("Version preparation took %0.2fms", (time.perf_counter() - time_before) * 1000)
130-
131-
132-
def cleanup_version(function_version: FunctionVersion) -> None:
133-
function_path = get_path_for_function(function_version)
134-
try:
135-
shutil.rmtree(function_path)
136-
except OSError as e:
137-
LOG.debug(
138-
"Could not cleanup function %s due to error %s while deleting file %s",
139-
function_version.qualified_arn,
140-
e.strerror,
141-
e.filename,
142-
)
143-
if config.LAMBDA_PREBUILD_IMAGES:
144-
CONTAINER_CLIENT.remove_image(get_image_name_for_function(function_version))
145-
146-
147115
class LambdaRuntimeException(Exception):
148116
def __init__(self, message: str):
149117
super().__init__(message)
@@ -163,6 +131,38 @@ def __init__(
163131
self.ip = None
164132
self.executor_endpoint = self._build_executor_endpoint(service_endpoint)
165133

134+
@staticmethod
135+
def prepare_version(function_version: FunctionVersion) -> None:
136+
if not function_version.code.zip_file:
137+
raise NotImplementedError("Images without zipfile are currently not supported")
138+
time_before = time.perf_counter()
139+
target_path = get_path_for_function(function_version)
140+
target_path.mkdir(parents=True, exist_ok=True)
141+
# write code to disk
142+
target_code = get_code_path_for_function(function_version)
143+
with NamedTemporaryFile() as file:
144+
file.write(function_version.code.zip_file)
145+
file.flush()
146+
unzip(file.name, str(target_code))
147+
if config.LAMBDA_PREBUILD_IMAGES:
148+
prepare_image(target_path, function_version)
149+
LOG.debug("Version preparation took %0.2fms", (time.perf_counter() - time_before) * 1000)
150+
151+
@staticmethod
152+
def cleanup_version(function_version: FunctionVersion) -> None:
153+
function_path = get_path_for_function(function_version)
154+
try:
155+
shutil.rmtree(function_path)
156+
except OSError as e:
157+
LOG.debug(
158+
"Could not cleanup function %s due to error %s while deleting file %s",
159+
function_version.qualified_arn,
160+
e.strerror,
161+
e.filename,
162+
)
163+
if config.LAMBDA_PREBUILD_IMAGES:
164+
CONTAINER_CLIENT.remove_image(get_image_name_for_function(function_version))
165+
166166
def get_image(self) -> str:
167167
if not self.function_version.config.runtime:
168168
raise NotImplementedError("Custom images are currently not supported")
@@ -229,7 +229,10 @@ def get_address(self) -> str:
229229
raise LambdaRuntimeException(f"IP address of executor '{self.id}' unknown")
230230
return self.ip
231231

232-
def get_endpoint_from_executor(self) -> str:
232+
def get_executor_endpoint_from_executor(self) -> str:
233+
return f"{get_main_endpoint_from_container()}:{self.executor_endpoint.port}"
234+
235+
def get_localstack_endpoint_from_executor(self) -> str:
233236
return get_main_endpoint_from_container()
234237

235238
def get_network_for_executor(self) -> str:

0 commit comments

Comments
 (0)
0