8000 Fix lambda destination default retries by dominikschubert · Pull Request #7933 · localstack/localstack · GitHub
[go: up one dir, main page]
Skip to content

Fix lambda destination default retries #7933

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 22, 2023
Merged

Fix lambda destination default retries #7933

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
25ad977
speed up sqs destination test and add default max retry
dominikschubert Mar 22, 2023
976f2de
add additional snapshot for default rendering
dominikschubert Mar 22, 2023
4ddc083
add test for default retries
dominikschubert Mar 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions localstack/services/awslambda/invocation/version_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,8 @@ def process_event_destinations(
)

max_retry_attempts = event_invoke_config.maximum_retry_attempts
if max_retry_attempts is None:
max_retry_attempts = 2 # default
previous_retry_attempts = queued_invocation.retries

if self.function.reserved_concurrent_executions == 0:
Expand Down
5 changes: 5 additions & 0 deletions tests/integration/awslambda/test_lambda_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1764,6 +1764,11 @@ def test_lambda_eventinvokeconfig_lifecycle(
)
snapshot.match("put_published_invokeconfig", put_published_invokeconfig)

get_published_invokeconfig = lambda_client.get_function_event_invoke_config(
FunctionName=function_name, Qualifier=publish_version_result["Version"]
)
snapshot.match("get_published_invokeconfig", get_published_invokeconfig)

# list paging
list_paging_single = lambda_client.list_function_event_invoke_configs(
FunctionName=function_name, MaxItems=1
Expand Down
15 changes: 14 additions & 1 deletion tests/integration/awslambda/test_lambda_api.snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9906,7 +9906,7 @@
}
},
"tests/integration/awslambda/test_lambda_api.py::TestLambdaEventInvokeConfig::test_lambda_eventinvokeconfig_lifecycle": {
"recorded-date": "17-02-2023, 11:39:34",
"recorded-date": "22-03-2023, 18:49:13",
"recorded-content": {
"put_invokeconfig_retries_0": {
"DestinationConfig": {
Expand Down Expand Up @@ -10073,6 +10073,19 @@
"HTTPStatusCode": 200
}
},
"get_published_invokeconfig": {
"DestinationConfig": {
"OnFailure": {},
"OnSuccess": {}
},
"FunctionArn": "arn:aws:lambda:<region>:111111111111:function:<function-name:1>:1",
"LastModified": "datetime",
"MaximumEventAgeInSeconds": 120,
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
},
"list_paging_nolimit_postdelete": {
"FunctionEventInvokeConfigs": [
{
Expand Down
72 changes: 63 additions & 9 deletions tests/integration/awslambda/test_lambda_destinations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

from localstack import config
from localstack.aws.api.lambda_ import Runtime
from localstack.testing.aws.lambda_utils import is_new_provider, is_old_provider
from localstack.testing.aws.lambda_utils import is_old_provider
from localstack.testing.aws.util import is_aws_cloud
from localstack.utils.strings import short_uid, to_bytes, to_str
from localstack.utils.sync import retry, wait_until
Expand Down Expand Up @@ -104,12 +104,6 @@ class TestLambdaDestinationSqs:
"$..stackTrace",
],
)
@pytest.mark.skip_snapshot_verify(
condition=is_new_provider,
paths=[
"$..approximateInvokeCount", # TODO: retry support
],
)
@pytest.mark.parametrize(
"payload",
[
Expand Down Expand Up @@ -142,12 +136,12 @@ def test_assess_lambda_destination_invocation(
create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON,
func_name=lambda_name,
libs=TEST_LAMBDA_LIBS,
role=lambda_su_role,
)

put_event_invoke_config_response = lambda_client.put_function_event_invoke_config(
FunctionName=lambda_name,
MaximumRetryAttempts=0,
DestinationConfig={
"OnSuccess": {"Destination": queue_arn},
"OnFailure": {"Destination": queue_arn},
Expand All @@ -162,10 +156,70 @@ def test_assess_lambda_destination_invocation(
)

def receive_message():
rs = sqs_client.receive_message(QueueUrl=queue_url, MessageAttributeNames=["All"])
rs = sqs_client.receive_message(
QueueUrl=queue_url, WaitTimeSeconds=2, MessageAttributeNames=["All"]
)
assert len(rs["Messages"]) > 0
return rs

receive_message_result = retry(receive_message, retries=120, sleep=1)
snapshot.match("receive_message_result", receive_message_result)

@pytest.mark.skipif(
condition=is_old_provider(), reason="config variable only supported in new provider"
)
def test_lambda_destination_default_retries(
self,
lambda_client,
sqs_client,
create_lambda_function,
sqs_create_queue,
sqs_queue_arn,
lambda_su_role,
snapshot,
monkeypatch,
):
snapshot.add_transformer(snapshot.transform.lambda_api())
snapshot.add_transformer(snapshot.transform.sqs_api())
snapshot.add_transformer(snapshot.transform.key_value("MD5OfBody"))

if not is_aws_cloud():
monkeypatch.setattr(config, "LAMBDA_RETRY_BASE_DELAY_SECONDS", 5)

# create DLQ and Lambda function
queue_name = f"test-{short_uid()}"
lambda_name = f"test-{short_uid()}"
queue_url = sqs_create_queue(QueueName=queue_name)
queue_arn = sqs_queue_arn(queue_url)
create_lambda_function(
handler_file=TEST_LAMBDA_PYTHON,
func_name=lambda_name,
role=lambda_su_role,
)

put_event_invoke_config_response = lambda_client.put_function_event_invoke_config(
FunctionName=lambda_name,
DestinationConfig={
"OnSuccess": {"Destination": queue_arn},
"OnFailure": {"Destination": queue_arn},
},
)
snapshot.match("put_function_event_invoke_config", put_event_invoke_config_response)

lambda_client.invoke(
FunctionName=lambda_name,
Payload=json.dumps({lambda_integration.MSG_BODY_RAISE_ERROR_FLAG: 1}),
InvocationType="Event",
)

def receive_message():
rs = sqs_client.receive_message(
QueueUrl=queue_url, WaitTimeSeconds=2, MessageAttributeNames=["All"]
)
assert len(rs["Messages"]) > 0
return rs

# this will take at least 3 minutes on AWS
receive_message_result = retry(receive_message, retries=120, sleep=3)
snapshot.match("receive_message_result", receive_message_result)

Expand Down
67 changes: 64 additions & 3 deletions tests/integration/awslambda/test_lambda_destinations.snapshot.json
View file
Open in desktop
6D4E
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@
}
},
"tests/integration/awslambda/test_lambda_destinations.py::TestLambdaDestinationSqs::test_assess_lambda_destination_invocation[payload0]": {
"recorded-date": "27-02-2023, 16:02:33",
"recorded-date": "22-03-2023, 18:42:11",
"recorded-content": {
"put_function_event_invoke_config": {
"DestinationConfig": {
Expand All @@ -134,6 +134,7 @@
},
"FunctionArn": "arn:aws:lambda:<region>:111111111111:function:<resource:2>:$LATEST",
"LastModified": "datetime",
"MaximumRetryAttempts": 0,
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
Expand Down Expand Up @@ -183,7 +184,7 @@
}
},
"tests/integration/awslambda/test_lambda_destinations.py::TestLambdaDestinationSqs::test_assess_lambda_destination_invocation[payload1]": {
"recorded-date": "27-02-2023, 16:08:45",
"recorded-date": "22-03-2023, 18:42:16",
"recorded-content": {
"put_function_event_invoke_config": {
"DestinationConfig": {
Expand All @@ -196,6 +197,7 @@
},
"FunctionArn": "arn:aws:lambda:<region>:111111111111:function:<resource:2>:$LATEST",
"LastModified": "datetime",
"MaximumRetryAttempts": 0,
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
Expand All @@ -211,7 +213,7 @@
"requestId": "<uuid:1>",
"functionArn": "arn:aws:lambda:<region>:111111111111:function:<resource:2>:$LATEST",
"condition": "RetriesExhausted",
"approximateInvokeCount": 3
"approximateInvokeCount": 1
},
"requestPayload": {
"raise_error": 1
Expand Down Expand Up @@ -525,5 +527,64 @@
"ReceiptHandle": "<receipt-handle:2>"
}
}
},
"tests/integration/awslambda/test_lambda_destinations.py::TestLambdaDestinationSqs::test_lambda_destination_default_retries": {
"recorded-date": "22-03-2023, 19:02:29",
"recorded-content": {
"put_function_event_invoke_config": {
"DestinationConfig": {
"OnFailure": {
"Destination": "arn:aws:sqs:<region>:111111111111:<resource:1>"
},
"OnSuccess": {
"Destination": "arn:aws:sqs:<region>:111111111111:<resource:1>"
}
},
"FunctionArn": "arn:aws:lambda:<region>:111111111111:function:<resource:2>:$LATEST",
"LastModified": "datetime",
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
},
"receive_message_result": {
"Messages": [
{
"Body": {
"version": "1.0",
"timestamp": "date",
"requestContext": {
"requestId": "<uuid:1>",
"functionArn": "arn:aws:lambda:<region>:111111111111:function:<resource:2>:$LATEST",
"condition": "RetriesExhausted",
"approximateInvokeCount": 3
},
"requestPayload": {
"raise_error": 1
},
"responseContext": {
"statusCode": 200,
"executedVersion": "$LATEST",
"functionError": "Unhandled"
},
"responsePayload": {
"errorMessage": "Test exception (this is intentional)",
"errorType": "Exception",
"stackTrace": [
" File \"/var/task/handler.py\", line 54, in handler\n raise Exception(\"Test exception (this is intentional)\")\n"
]
}
},
"MD5OfBody": "<m-d5-of-body:1>",
"MessageId& 5132 quot;: "<uuid:2>",
"ReceiptHandle": "<receipt-handle:1>"
}
],
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
}
}
}
}
0