8000 Merge branch 'master' into efritz/async-lambda-events · localstack/localstack@5a50769 · GitHub
[go: up one dir, main page]
Skip to content

Commit 5a50769

Browse files
whummerwhummer
committed
Merge branch 'master' into efritz/async-lambda-events
2 parents 8020012 + 6a380e5 commit 5a50769

27 files changed

+649
-115
lines changed

‎Makefile

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,13 +105,14 @@ web: ## Start web application (dashboard)
105105

106106
test: ## Run automated tests
107107
make lint && \
108-
($(VENV_RUN); DEBUG=$(DEBUG) PYTHONPATH=`pwd` nosetests --with-timer --with-coverage --logging-level=WARNING --nocapture --no-skip --exe --cover-erase --cover-tests --cover-inclusive --cover-package=localstack --with-xunit --exclude='$(VENV_DIR).*' --ignore-files='lambda_python3.py' $(TEST_PATH))
108+
($(VENV_RUN); DEBUG=$(DEBUG) PYTHONPATH=`pwd` nosetests $(NOSE_ARGS) --with-timer --with-coverage --logging-level=WARNING --nocapture --no-skip --exe --cover-erase --cover-tests --cover-inclusive --cover-package=localstack --with-xunit --exclude='$(VENV_DIR).*' --ignore-files='lambda_python3.py' $(TEST_PATH))
109109

110110
test-docker: ## Run automated tests in Docker
111111
ENTRYPOINT="--entrypoint=" CMD="make test" make docker-run
112112

113113
test-docker-mount:
114-
ENTRYPOINT="--entrypoint= -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/tests:/opt/code/localstack/tests -e TEST_PATH=$(TEST_PATH) -e LAMBDA_JAVA_OPTS=$(LAMBDA_JAVA_OPTS)" CMD="make test" make docker-run
114+
MOTO_DIR=$$(echo $$(pwd)/.venv/lib/python*/site-packages/moto | awk '{print $$NF}'); \
115+
ENTRYPOINT="--entrypoint= -v `pwd`/localstack/config.py:/opt/code/localstack/localstack/config.py -v `pwd`/localstack/constants.py:/opt/code/localstack/localstack/constants.py -v `pwd`/localstack/utils:/opt/code/localstack/localstack/utils -v `pwd`/localstack/services:/opt/code/localstack/localstack/services -v `pwd`/tests:/opt/code/localstack/tests -v `pwd`/Makefile:/opt/code/localstack/Makefile -v $$MOTO_DIR:/opt/code/localstack/.venv/lib/python3.8/site-packages/moto/ -e TEST_PATH=$(TEST_PATH) -e NOSE_ARGS=-v -e LAMBDA_JAVA_OPTS=$(LAMBDA_JAVA_OPTS)" CMD="make test" make docker-run
115116

116117
reinstall-p2: ## Re-initialize the virtualenv with Python 2.x
117118
rm -rf $(VENV_DIR)

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ You can pass the following environment variables to LocalStack:
213213
* `LAMBDA_CONTAINER_REGISTRY` Use an alternative docker registry to pull lambda execution containers (default: `lambci/lambda`).
214214
* `LAMBDA_REMOVE_CONTAINERS`: Whether to remove containers after Lambdas finished executing (default: `true`).
215215
* `DATA_DIR`: Local directory for saving persistent data (currently only supported for these services:
216-
Kinesis, DynamoDB, Elasticsearch, S3, Secretsmanager, SSM, SQS). Set it to `/tmp/localstack/data` to enable persistence
216+
Kinesis, DynamoDB, Elasticsearch, S3, Secretsmanager, SSM, SQS, SNS). Set it to `/tmp/localstack/data` to enable persistence
217217
(`/tmp/localstack` is mounted into the Docker container), leave blank to disable
218218
persistence (default).
219219
* `PORT_WEB_UI`: Port for the Web user interface / dashboard (default: `8080`).

‎localstack/config.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,9 @@ def is_env_not_false(env_var_name):
9797
# temporary folder of the host (required when running in Docker). Fall back to local tmp folder if not set
9898
HOST_TMP_FOLDER = os.environ.get('HOST_TMP_FOLDER', TMP_FOLDER)
9999

100+
# whether to enable verbose debug logging
101+
DEBUG = os.environ.get('DEBUG', '').lower() in TRUE_STRINGS
102+
100103
# whether to use SSL encryption for the services
101104
USE_SSL = is_env_true('USE_SSL')
102105

@@ -143,6 +146,9 @@ def is_env_not_false(env_var_name):
143146
# path prefix for windows volume mounting
144147
WINDOWS_DOCKER_MOUNT_PREFIX = os.environ.get('WINDOWS_DOCKER_MOUNT_PREFIX', '/host_mnt')
145148

149+
# whether to use a proxy server with HTTP/2 support. TODO: remove in the future
150+
USE_HTTP2_SERVER = os.environ.get('USE_HTTP2_SERVER', '').strip() not in FALSE_STRINGS
151+
146152

147153
def has_docker():
148154
try:
@@ -184,7 +190,7 @@ def is_linux():
184190
'START_WEB', 'DOCKER_BRIDGE_IP', 'DEFAULT_REGION', 'LAMBDA_JAVA_OPTS', 'LOCALSTACK_API_KEY',
185191
'LAMBDA_CONTAINER_REGISTRY', 'TEST_AWS_ACCOUNT_ID', 'DISABLE_EVENTS', 'EDGE_PORT',
186192
'EDGE_PORT_HTTP', 'SKIP_INFRA_DOWNLOADS', 'STEPFUNCTIONS_LAMBDA_ENDPOINT',
187-
'WINDOWS_DOCKER_MOUNT_PREFIX',
193+
'WINDOWS_DOCKER_MOU F438 NT_PREFIX', 'USE_HTTP2_SERVER',
188194
'SYNCHRONOUS_API_GATEWAY_EVENTS', 'SYNCHRONOUS_KINESIS_EVENTS',
189195
'SYNCHRONOUS_SNS_EVENTS', 'SYNCHRONOUS_SQS_EVENTS', 'SYNCHRONOUS_DYNAMODB_EVENTS']
190196

@@ -332,7 +338,11 @@ def external_service_url(service_key, host=None):
332338
populate_configs()
333339

334340
# set log level
341+
<<<<<<< HEAD
335342
if is_env_true('DEBUG'):
343+
=======
344+
if DEBUG:
345+
>>>>>>> master
336346
logging.getLogger('').setLevel(logging.DEBUG)
337347
logging.getLogger('localstack').setLevel(logging.DEBUG)
338348

‎localstack/services/apigateway/apigateway_listener.py

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,42 @@ def authorize_invocation(api_id, headers):
152152
run_authorizer(api_id, headers, authorizer)
153153

154154

155+
def validate_api_key(api_key, stage):
156+
157+
key = None
158+
usage_plan_id = None
159+
160+
client = aws_stack.connect_to_service('apigateway')
161+
usage_plans = client.get_usage_plans()
162+
for item in usage_plans.get('items', []):
163+
api_stages = item.get('apiStages', [])
164+
for api_stage in api_stages:
165+
if api_stage.get('stage') == stage:
166+
usage_plan_id = item.get('id')
167+
if not usage_plan_id:
168+
return False
169+
170+
usage_plan_keys = client.get_usage_plan_keys(usagePlanId=usage_plan_id)
171+
for item in usage_plan_keys.get('items', []):
172+
key = item.get('value')
173+
174+
if key != api_key:
175+
return False
176+
177+
return True
178+
179+
180+
def is_api_key_valid(is_api_key_required, headers, stage):
181+
if not is_api_key_required:
182+
return True
183+
184+
api_key = headers.get('X-API-Key')
185+
if not api_key:
186+
return False
187+
188+
return validate_api_key(api_key, stage)
189+
190+
155191
def invoke_rest_api(api_id, stage, method, invocation_path, data, headers, path=None):
156192
path = path or invocation_path
157193
relative_path, query_string_params = extract_query_string_params(path=invocation_path)
@@ -165,6 +201,10 @@ def invoke_rest_api(api_id, stage, method, invocation_path, data, headers, path=
165201
except Exception:
166202
return make_error_response('Unable to find path %s' % path, 404)
167203

204+
if not is_api_key_valid(path_map.get(relative_path, {}).
205+
get('resourceMethods', {}).get(method, {}).get('apiKeyRequired'), headers, stage):
206+
return make_error_response('Acess Denied Exception.', 403)
207+
168208
integrations = resource.get('resourceMethods', {})
169209
integration = integrations.get(method, {})
170210
if not integration:

‎localstack/services/awslambda/lambda_api.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,12 @@
3838
LAMBDA_RUNTIME_RUBY,
3939
LAMBDA_RUNTIME_RUBY25,
4040
LAMBDA_RUNTIME_PROVIDED)
41+
from localstack.services.awslambda.multivalue_transformer import multi_value_dict_for_list
4142
from localstack.utils.common import (to_str, load_file, save_file, TMP_FILES, ensure_readable,
4243
mkdir, unzip, is_zip_file, zip_contains_jar_entries, run, short_uid,
43-
timestamp_millis, parse_chunked_data, now_utc, safe_requests, FuncThread,
44-
isoformat_milliseconds)
44+
timestamp_millis, now_utc, safe_requests, FuncThread, isoformat_milliseconds)
4545
from localstack.utils.analytics import event_publisher
46+
from localstack.utils.http_utils import parse_chunked_data
4647
from localstack.utils.aws.aws_models import LambdaFunction
4748

4849
APP_NAME = 'lambda_api'
@@ -246,12 +247,14 @@ def process_apigateway_invocation(func_arn, path, payload, headers={},
246247
event = {
247248
'path': path,
248249
'headers': dict(headers),
250+
'multiValueHeaders': multi_value_dict_for_list(headers),
249251
'pathParameters': dict(path_params),
250252
'body': payload,
251253
'isBase64Encoded': False,
252254
'resource': resource_path,
253255
'httpMethod': method,
254256
'queryStringParameters': query_string_params,
257+
'multiValueQueryStringParameters': multi_value_dict_for_list(query_string_params),
255258
'requestContext': request_context,
256259
'stageVariables': {} # TODO
257260
}

‎localstack/services/awslambda/multivalue_transformer.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
from collections import defaultdict
2+
from localstack.utils.common import to_str
3+
4+
5+
def multi_value_dict_for_list(elements):
6+
temp_mv_dict = defaultdict(list)
7+
for key in elements:
8+
if isinstance(key, (list, tuple)):
9+
key, value = key
10+
else:
11+
value = elements[key]
12+
key = to_str(key)
13+
temp_mv_dict[key].append(value)
14+
15+
return dict((k, tuple(v)) for k, v in temp_mv_dict.items())

‎localstack/services/edge.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from localstack.constants import HEADER_LOCALSTACK_TARGET, HEADER_LOCALSTACK_EDGE_URL, LOCALSTACK_ROOT_FOLDER
99
from localstack.utils.common import run, is_root, TMP_THREADS, to_bytes, truncate
1010
from localstack.utils.common import safe_requests as requests
11-
from localstack.services.generic_proxy import ProxyListener, GenericProxy
11+
from localstack.services.generic_proxy import ProxyListener, start_proxy_server
1212

1313
LOG = logging.getLogger(__name__)
1414

@@ -169,8 +169,7 @@ def do_start_edge(port, use_ssl, asynchronous=False):
169169
# get port and start Edge
170170
print('Starting edge router (http%s port %s)...' % ('s' if use_ssl else '', port))
171171
# use use=True here because our proxy allows both, HTTP and HTTPS traffic
172-
proxy = GenericProxy(port, ssl=True, update_listener=ProxyListenerEdge())
173-
proxy.start()
172+
proxy = start_proxy_server(port, use_ssl=True, update_listener=ProxyListenerEdge())
174173
if not asynchronous:
175174
proxy.join()
176175
return proxy

‎localstack/services/events/events_starter.py

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from localstack.constants import (
88
APPLICATION_AMZ_JSON_1_1, TEST_AWS_ACCOUNT_ID)
99
from localstack.utils.aws import aws_stack
10-
from localstack.utils.common import short_uid
10+
from localstack.utils.common import short_uid, to_bytes
1111
from localstack.services.infra import start_moto_server
1212
from localstack.services.events.scheduler import JobScheduler
1313
from localstack.services.awslambda.lambda_api import run_lambda
@@ -35,6 +35,14 @@ def send_event_to_lambda(event, arn):
3535
run_lambda(event=json.loads(event['Detail']), context={}, func_arn=arn, asynchronous=True)
3636

3737

38+
def send_event_to_firehose(event, arn):
39+
delivery_stream_name = aws_stack.firehose_name(arn)
40+
firehose_client = aws_stack.connect_to_service('firehose')
41+
firehose_client.put_record(
42+
DeliveryStreamName=delivery_stream_name,
43+
Record={'Data': to_bytes(event['Detail'])})
44+
45+
3846
def process_events(event, targets):
3947
for target in targets:
4048
arn = target['Arn']
@@ -46,6 +54,9 @@ def process_events(event, targets):
4654
elif service == 'lambda':
4755
send_event_to_lambda(event, arn)
4856

57+
elif service == 'firehose':
58+
send_event_to_firehose(event, arn)
59+
4960
else:
5061
LOG.warning('Unsupported Events target service type "%s"' % service)
5162

0 commit comments

Comments
 (0)
0