8000 Merge branch 'master' into region-override · localstack/localstack@a898079 · GitHub
[go: up one dir, main page]
Skip to content

Commit a898079

Browse files
viren-nadkarniviren-nadkarni
committed
Merge branch 'master' into region-override
2 parents e4b1475 + 601aecb commit a898079

File tree

27 files changed

+1061
-392
lines changed

27 files changed

+1061
-392
lines changed

‎Dockerfile

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,12 @@ RUN --mount=type=cache,target=/root/.cache \
168168
RUN echo /var/lib/localstack/lib/extensions/python_venv/lib/python3.10/site-packages > localstack-extensions-venv.pth && \
169169
mv localstack-extensions-venv.pth .venv/lib/python*/site-packages/
170170

171+
# link the python package installer virtual environments into the localstack venv
172+
RUN echo /var/lib/localstack/lib/python-packages/lib/python3.10/site-packages > localstack-var-python-packages-venv.pth && \
173+
mv localstack-var-python-packages-venv.pth .venv/lib/python*/site-packages/
174+
RUN echo /usr/lib/localstack/python-packages/lib/python3.10/site-packages > localstack-static-python-packages-venv.pth && \
175+
mv localstack-static-python-packages-venv.pth .venv/lib/python*/site-packages/
176+
171177
# Install the latest version of the LocalStack Persistence Plugin
172178
RUN --mount=type=cache,target=/root/.cache \
173179
(. .venv/bin/activate && pip3 install --upgrade localstack-plugin-persistence)

‎localstack/packages/api.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ def __init__(self, name: str, version: str, install_lock: Optional[RLock] = None
6868
self.name = name
6969
self.version = version
7070
self.install_lock = install_lock or RLock()
71+
self._setup_for_target: dict[InstallTarget, bool] = defaultdict(lambda: False)
7172

7273
def install(self, target: Optional[InstallTarget] = None) -> None:
7374
"""
@@ -92,6 +93,9 @@ def install(self, target: Optional[InstallTarget] = None) -> None:
9293
LOG.debug("Installation of %s finished.", self.name)
9394
else:
9495
LOG.debug("Installation of %s skipped (already installed).", self.name)
96+
if not self._setup_for_target[target]:
97+
LOG.debug("Performing runtime setup for already installed package.")
98+
self._setup_existing_installation(target)
9599
except PackageException as e:
96100
raise e
97101
except Exception as e:
@@ -135,6 +139,17 @@ def _get_install_marker_path(self, install_dir: str) -> str:
135139
"""
136140
raise NotImplementedError()
137141

142+
def _setup_existing_installation(self, target: InstallTarget) -> None:
143+
"""
144+
Internal function to perform the setup for an existing installation, f.e. adding a path to an environment.
145+
This is only necessary for certain installers (like the PythonPackageInstaller).
146+
This function will _always_ be executed _exactly_ once within a Python session for a specific installer
147+
instance and target, if #install is called for the respective target.
148+
:param target: of the installation
149+
:return: None
150+
"""
151+
pass
152+
138153
def _prepare_installation(self, target: InstallTarget) -> None:
139154
"""
140155
Internal function to prepare an installation, f.e. by downloading some data or installing an OS package repo.

‎localstack/packages/core.py

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,20 @@
11
import logging
22
import os
3+
import re
34
from abc import ABC
45
from functools import lru_cache
6+
from sys import version_info
57
from typing import Optional
68

79
import requests
810

9-
from localstack import config
11+
from localstack import config, constants
1012

1113
from ..utils.archives import download_and_extract
1214
from ..utils.files import chmod_r, chown_r, mkdir, rm_rf
1315
from ..utils.http import download
1416
from ..utils.run import is_root, run
17+
from ..utils.venv import VirtualEnvironment
1518
from .api import InstallTarget, PackageException, PackageInstaller
1619

1720
LOG = logging.getLogger(__name__)
@@ -229,3 +232,65 @@ def _install(self, target: InstallTarget) -> None:
229232
# if the package was installed as root, set the ownership manually
230233
LOG.debug("Setting ownership root:root on %s", target_dir)
231234
chown_r(target_dir, "root")
235+
236+
237+
LOCALSTACK_VENV = VirtualEnvironment(os.path.join(constants.LOCALSTACK_ROOT_FOLDER, ".venv"))
238+
239+
240+
class PythonPackageInstaller(PackageInstaller):
241+
"""
242+
Package installer which allows the runtime-installation of additional python packages used by certain services.
243+
f.e. vosk as offline speech recognition toolkit (which is ~7MB in size compressed and ~26MB uncompressed).
244+
"""
245+
246+
normalized_name: str
247+
"""Normalized package name according to PEP440."""
248+
249+
def __init__(self, name: str, version: str, *args, **kwargs):
250+
super().__init__(name, version, *args, **kwargs)
251+
self.normalized_name = self._normalize_package_name(name)
252+
253+
def _normalize_package_name(self, name: str):
254+
"""
255+
Normalized the Python package name according to PEP440.
256+
https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization
257+
"""
258+
return re.sub(r"[-_.]+", "-", name).lower()
259+
260+
def _get_install_dir(self, target: InstallTarget) -> str:
261+
# all python installers share a venv
262+
return os.path.join(target.value, "python-packages")
263+
264+
def _get_install_marker_path(self, install_dir: str) -> str:
265+
python_subdir = f"python{version_info[0]}.{version_info[1]}"
266+
dist_info_dir = f"{self.normalized_name}-{self.version}.dist-info"
267+
# the METADATA file is mandatory, use it as install marker
268+
return os.path.join(
269+
install_dir, "lib", python_subdir, "site-packages", dist_info_dir, "METADATA"
270+
)
271+
272+
def _get_venv(self, target: InstallTarget) -> VirtualEnvironment:
273+
venv_dir = self._get_install_dir(target)
274+
return VirtualEnvironment(venv_dir)
275+
276+
def _prepare_installation(self, target: InstallTarget) -> None:
277+
# make sure the venv is properly set up before installing the package
278+
venv = self._get_venv(target)
279+
if not venv.exists:
280+
LOG.info("creating virtual environment at %s", venv.venv_dir)
281+
venv.create()
282+
LOG.info("adding localstack venv path %s", venv.venv_dir)
283+
venv.add_pth("localstack-venv", LOCALSTACK_VENV)
284+
LOG.debug("injecting venv into path %s", venv.venv_dir)
285+
venv.inject_to_sys_path()
286+
287+
def _install(self, target: InstallTarget) -> None:
288+
venv = self._get_venv(target)
289+
python_bin = os.path.join(venv.venv_dir, "bin/python")
290+
291+
# run pip via the python binary of the venv
292+
run([python_bin, "-m", "pip", "install", f"{self.name}=={self.version}"], print_error=False)
293+
294+
def _setup_existing_installation(self, target: InstallTarget) -> None:
295+
"""If the venv is already present, it just needs to be initialized once."""
296+
self._prepare_installation(target)

‎localstack/services/cloudformation/models/dynamodb.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,7 @@ def _handle_result(
144144
default=None,
145145
)
146146
),
147+
"Tags": "Tags",
147148
},
148149
"result_handler": _handle_result,
149150
},

‎localstack/services/cloudformation/resource_provider.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@
4343
# by default we use the GenericBaseModel (the legacy model), unless the resource is listed below
4444
# add your new provider here when you want it to be the default
4545
PROVIDER_DEFAULTS = {
46-
"AWS::SQS::Queue": "ResourceProvider"
46+
"AWS::SQS::Queue": "ResourceProvider",
47+
"AWS::SQS::QueuePolicy": "ResourceProvider",
4748
# "AWS::IAM::User": "ResourceProvider",
4849
# "AWS::SSM::Parameter": "GenericBaseModel",
4950
# "AWS::OpenSearchService::Domain": "GenericBaseModel",

‎localstack/services/cloudwatch/provider.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from moto.cloudwatch import cloudwatch_backends
66
from moto.cloudwatch.models import CloudWatchBackend, FakeAlarm, MetricDatum
77

8-
from localstack.aws.accounts import get_aws_account_id
8+
from localstack.aws.accounts import get_account_id_from_access_key_id, get_aws_account_id
99
from localstack.aws.api import CommonServiceException, RequestContext, handler
1010
from localstack.aws.api.cloudwatch import (
1111
AlarmNames,
@@ -284,7 +284,10 @@ def delete_alarms(self, context: RequestContext, alarm_names: AlarmNames) -> Non
284284
def get_raw_metrics(self, request: Request):
285285
region = aws_stack.extract_region_from_auth_header(request.headers)
286286
account_id = (
287-
extract_access_key_id_from_auth_header(request.headers) or DEFAULT_AWS_ACCOUNT_ID
287+
get_account_id_from_access_key_id(
288+
extract_access_key_id_from_auth_header(request.headers)
289+
)
290+
or DEFAULT_AWS_ACCOUNT_ID
288291
)
289292
backend = cloudwatch_backends[account_id][region]
290293
if backend:

‎localstack/services/lambda_/invocation/version_manager.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ def run_log_loop(self, *args, **kwargs) -> None:
106106
return
107107
try:
108108
store_cloudwatch_logs(
109-
log_item.log_group, log_item.log_stream, log_item.logs, logs_client=logs_client
109+
logs_client, log_item.log_group, log_item.log_stream, log_item.logs
110110
)
111111
except Exception as e:
112112
LOG.warning(

‎localstack/services/lambda_/lambda_utils.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,15 @@ def store_lambda_logs(
184184
invocation_time_secs = int(invocation_time / 1000)
185185
time_str = time.strftime("%Y/%m/%d", time.gmtime(invocation_time_secs))
186186
log_stream_name = "%s/[LATEST]%s" % (time_str, container_id)
187-
return store_cloudwatch_logs(log_group_name, log_stream_name, log_output, invocation_time)
187+
188+
arn = lambda_function.arn()
189+
account_id = extract_account_id_from_arn(arn)
190+
region_name = extract_region_from_arn(arn)
191+
logs_client = connect_to(aws_access_key_id=account_id, region_name=region_name).logs
192+
193+
return store_cloudwatch_logs(
194+
logs_client, log_group_name, log_stream_name, log_output, invocation_time
195+
)
188196

189197

190198
def get_main_endpoint_from_container() -> str:

‎localstack/services/ses/provider.py

Lines changed: 17 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -340,43 +340,33 @@ def send_email(
340340
tags: MessageTagList = None,
341341
configuration_set_name: ConfigurationSetName = None,
342342
) -> SendEmailResponse:
343-
response = call_moto(context)
344-
345-
backend = get_ses_backend(context)
346-
emitter = SNSEmitter(context)
347-
recipients = recipients_from_destination(destination)
348-
349343
if tags:
350344
for tag in tags:
351345
tag_name = tag.get("Name", "")
352346
tag_value = tag.get("Value", "")
353347
if tag_name == "":
354-
raise CommonServiceException(
355-
"InvalidParameterValue", "The tag name must be specified."
356-
)
348+
raise InvalidParameterValue("The tag name must be specified.")
357349
if tag_value == "":
358-
raise CommonServiceException(
359-
"InvalidParameterValue", "The tag value must be specified."
360-
)
350+
raise InvalidParameterValue("The tag value must be specified.")
361351
if len(tag_name) > 255:
362-
raise CommonServiceException(
363-
"InvalidParameterValue", "Tag name cannot exceed 255 characters."
364-
)
352+
raise InvalidParameterValue("Tag name cannot exceed 255 characters.")
365353
if not re.match(ALLOWED_TAG_CHARS, tag_name):
366-
raise CommonServiceException(
367-
"InvalidParameterValue",
354+
raise InvalidParameterValue(
368355
f"Invalid tag name <{tag_name}>: only alphanumeric ASCII characters, '_', and '-' are allowed.",
369356
)
370357
if len(tag_value) > 255:
371-
raise CommonServiceException(
372-
"InvalidParameterValue", "Tag value cannot exceed 255 characters."
373-
)
358+
raise InvalidParameterValue("Tag value cannot exceed 255 characters.")
374359
if not re.match(ALLOWED_TAG_CHARS, tag_value):
375-
raise CommonServiceException(
376-
"InvalidParameterValue",
360+
raise InvalidParameterValue(
377361
f"Invalid tag value <{tag_value}>: only alphanumeric ASCII characters, '_', and '-' are allowed.",
378362
)
379363

364+
response = call_moto(context)
365+
366+
backend = get_ses_backend(context)
367+
emitter = SNSEmitter(context)
368+
recipients = recipients_from_destination(destination)
369+
380370
for event_destination in backend.config_set_event_destination.values():
381371
if not event_destination["Enabled"]:
382372
continue
@@ -651,3 +641,8 @@ def _client_for_topic(topic_arn: str) -> "SNSClient":
651641
aws_access_key_id=access_key_id,
652642
aws_secret_access_key=TEST_AWS_SECRET_ACCESS_KEY,
653643
).sns
644+
645+
646+
class InvalidParameterValue(CommonServiceException):
647+
def __init__(self, message=None):
648+
super().__init__("InvalidParameterValue", status_code=400, message=message)

‎localstack/services/sns/publisher.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
SnsSubscription,
2727
)
2828
from localstack.utils.aws.arns import (
29+
extract_account_id_from_arn,
2930
extract_region_from_arn,
3031
extract_resource_from_arn,
3132
parse_arn,
@@ -873,7 +874,13 @@ def store_delivery_log(
873874

874875
log_output = json.dumps(delivery_log)
875876

876-
return store_cloudwatch_logs(log_group_name, log_stream_name, log_output, invocation_time)
877+
account_id = extract_account_id_from_arn(subscriber["TopicArn"])
878+
region_name = extract_region_from_arn(subscriber["TopicArn"])
879+
logs_client = connect_to(aws_access_key_id=account_id, region_name=region_name).logs
880+
881+
return store_cloudwatch_logs(
882+
logs_client, log_group_name, log_stream_name, log_output, invocation_time
883+
)
877884

878885

879886
def create_subscribe_url(external_url, topic_arn, subscription_token):

0 commit comments

Comments
 (0)
0