8000 [SFN] Add support for Task Timeouts by MEPalma · Pull Request #8376 · localstack/localstack · GitHub
[go: up one dir, main page]
Skip to content

[SFN] Add support for Task Timeouts #8376

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 15 commits into from
Jun 25, 2023
Merged

[SFN] Add support for Task Timeouts #8376

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
3b0c2e9
base
MEPalma May 8, 2023
2f392e7
fixes
MEPalma May 10, 2023
4d63d30
update snap test
MEPalma May 10, 2023
21f10ee
reworked parameters, improved double dep of lambda state tasks, tests
MEPalma May 11, 2023
0b750d2
minor conflicts
MEPalma May 16, 2023
0cd04f9
Merge branch 'MEP-sfn-callbacks' into MEP-sfn-lambda-fix-params
MEPalma May 16, 2023
d2f539b
Merge branch 'master' into MEP-sfn-lambda-fix-params
MEPalma May 24, 2023
a24ee43
timeouts support
MEPalma May 26, 2023
406f3c0
Merge branch 'master' into MEP-sfn-timeout
MEPalma Jun 21, 2023
ae5ee1e
minor
MEPalma Jun 21, 2023
d0a65b1
pr items
MEPalma Jun 22, 2023
2682ffa
minor cleanup
MEPalma Jun 22, 2023
95ee8e0
Merge branch 'master' into MEP-sfn-timeout
MEPalma Jun 22, 2023
de9a320
fix cycle on send success
MEPalma Jun 22, 2023
387016f
Merge branch 'master' into MEP-sfn-timeout
MEPalma Jun 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
pr items
  • Loading branch information
@MEPalma
MEPalma committed Jun 22, 2023
commit d0a65b1cd51a7deae4fd9d2a18350a8d7b15ba1f
27 changes: 27 additions & 0 deletions localstack/testing/snapshots/transformer_utility.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,6 +526,33 @@ def sfn_sm_exec_arn(start_exec: StartExecutionOutput, index: int):
arn_part: str = "".join(start_exec["executionArn"].rpartition(":")[-1])
return RegexTransformer(arn_part, arn_part_repl)

@staticmethod
def stepfunctions_api():
return [
JsonpathTransformer(
"$..SdkHttpMetadata.AllHttpHeaders.Date",
"date",
replace_reference=False,
),
JsonpathTransformer(
"$..SdkHttpMetadata.AllHttpHeaders.X-Amzn-Trace-Id",
"X-Amzn-Trace-Id",
replace_reference=False,
),
JsonpathTransformer(
"$..SdkHttpMetadata.HttpHeaders.Date",
"date",
replace_reference=False,
),
JsonpathTransformer(
"$..SdkHttpMetadata.HttpHeaders.X-Amzn-Trace-Id",
"X-Amzn-Trace-Id",
replace_reference=False,
),
SortingTransformer("VersionStages"),
SortingTransformer("Versions", lambda e: e.get("CreatedDate")),
]

# TODO add example
# @staticmethod
# def custom(fn: Callable[[dict], dict]) -> Transformer:
Expand Down
6 changes: 6 additions & 0 deletions tests/integration/stepfunctions/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,12 @@
LOG = logging.getLogger(__name__)


@pytest.fixture
def sfn_snapshot(snapshot):
snapshot.add_transformers_list(snapshot.transform.stepfunctions_api())
return snapshot


@pytest.fixture
def create_iam_role_for_sfn(aws_client, cleanups, create_state_machine):
iam_client = aws_client.iam
Expand Down
28 changes: 14 additions & 14 deletions tests/integration/stepfunctions/v2/callback/test_callback.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@ def test_sqs_wait_for_task_tok(
create_state_machine,
sqs_create_queue,
sqs_send_task_success_state_machine,
snapshot,
sfn_snapshot,
):
snapshot.add_transformer(snapshot.transform.sqs_api())
snapshot.add_transformer(
sfn_snapshot.add_transformer(sfn_snapshot.transform.sqs_api())
sfn_snapshot.add_transformer(
JsonpathTransformer(
jsonpath="$..TaskToken",
replacement="<task_token>",
Expand All @@ -45,8 +45,8 @@ def test_sqs_wait_for_task_tok(

queue_name = f"queue-{short_uid()}"
queue_url = sqs_create_queue(QueueName=queue_name)
snapshot.add_transformer(RegexTransformer(queue_url, "<sqs_queue_url>"))
snapshot.add_transformer(RegexTransformer(queue_name, "<sqs_queue_name>"))
sfn_snapshot.add_transformer(RegexTransformer(queue_url, "<sqs_queue_url>"))
sfn_snapshot.add_transformer(RegexTransformer(queue_name, "<sqs_queue_name>"))

sqs_send_task_success_state_machine(queue_url)

Expand All @@ -59,23 +59,23 @@ def test_sqs_wait_for_task_tok(
aws_client.stepfunctions,
create_iam_role_for_sfn,
create_state_machine,
snapshot,
sfn_snapshot,
definition,
exec_input,
)

@pytest.mark.skip_snapshot_verify(paths=["$..MD5OfMessageBody", "$..QueueUrl"])
def test_sqs_wait_for_task_tok_timeout(
@pytest.mark.skip_snapshot_verify(paths=["$..MD5OfMessageBody"])
def test_sqs_wait_for_task_token_timeout(
self,
aws_client,
create_iam_role_for_sfn,
create_state_machine,
sqs_create_queue,
sqs_send_task_success_state_machine,
snapshot,
sfn_snapshot,
):
snapshot.add_transformer(snapshot.transform.sqs_api())
snapshot.add_transformer(
sfn_snapshot.add_transformer(sfn_snapshot.transform.sqs_api())
sfn_snapshot.add_transformer(
JsonpathTransformer(
jsonpath="$..TaskToken",
replacement="<task_token>",
Expand All @@ -85,8 +85,8 @@ def test_sqs_wait_for_task_tok_timeout(

queue_name = f"queue-{short_uid()}"
queue_url = sqs_create_queue(QueueName=queue_name)
snapshot.add_transformer(RegexTransformer(queue_name, "<sqs_queue_name>"))
snapshot.add_transformer(RegexTransformer(queue_url, "<sqs_queue_url>"))
sfn_snapshot.add_transformer(RegexTransformer(queue_url, "<sqs_queue_url>"))
sfn_snapshot.add_transformer(RegexTransformer(queue_name, "<sqs_queue_name>"))

template = CT.load_sfn_template(CT.SQS_WAIT_FOR_TASK_TOKEN_WITH_TIMEOUT)
definition = json.dumps(template)
Expand All @@ -97,7 +97,7 @@ def test_sqs_wait_for_task_tok_timeout(
aws_client.stepfunctions,
create_iam_role_for_sfn,
create_state_machine,
snapshot,
sfn_snapshot,
definition,
exec_input,
)
145 changes: 139 additions & 6 deletions tests/integration/stepfunctions/v2/callback/test_callback.snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"tests/integration/stepfunctions/v2/callback/test_callback.py::TestCallback::test_sqs_wait_for_task_tok": {
"recorded-date": "10-05-2023, 15:43:08",
"recorded-date": "22-06-2023, 13:05:16",
"recorded-content": {
"get_execution_history": {
"events": [
Expand Down Expand Up @@ -69,7 +69,7 @@
"previousEventId": 4,
"taskSubmittedEventDetails": {
"output": {
"MD5OfMessageBody": "95adacacec73703a3538430d3decf18f",
"MD5OfMessageBody": "476a20e5904cb4822a4cd978ced5e607",
"MessageId": "<uuid:1>",
"SdkHttpMetadata": {
"AllHttpHeaders": {
Expand All @@ -79,17 +79,15 @@
"Content-Length": [
"378"
],
"Date": [
"Wed, 10 May 2023 13:43:06 GMT"
],
"Date": "date",
"Content-Type": [
"text/xml"
]
},
"HttpHeaders": {
"Content-Length": "378",
"Content-Type": "text/xml",
"Date": "Wed, 10 May 2023 13:43:06 GMT",
"Date": "date",
"x-amzn-RequestId": "<uuid:2>"
},
"HttpStatusCode": 200
Expand Down Expand Up @@ -290,5 +288,140 @@
}
}
}
},
"tests/integration/stepfunctions/v2/callback/test_callback.py::TestCallback::test_sqs_wait_for_task_token_timeout": {
"recorded-date": "22-06-2023, 14:42:04",
"recorded-content": {
"get_execution_history": {
"events": [
{
"executionStartedEventDetails": {
"input": {
"QueueUrl": "<sqs_queue_url>",
"Message": "test_message_txt"
},
"inputDetails": {
"truncated": false
},
"roleArn": "arn:aws:iam::111111111111:role/<resource:1>"
},
"id": 1,
"previousEventId": 0,
"timestamp": "timestamp",
"type": "ExecutionStarted"
},
{
"id": 2,
"previousEventId": 0,
"stateEnteredEventDetails": {
"input": {
"QueueUrl": "<sqs_queue_url>",
"Message": "test_message_txt"
},
"inputDetails": {
"truncated": false
},
"name": "SendMessageWithWaitAndTimeout"
},
"timestamp": "timestamp",
"type": "TaskStateEntered"
},
{
"id": 3,
"previousEventId": 2,
"taskScheduledEventDetails": {
"parameters": {
"MessageBody": {
"Message": "test_message_txt",
"TaskToken": "<<task_token>:1>"
},
"QueueUrl": "<sqs_queue_url>"
},
"region": "<region>",
"resource": "sendMessage.waitForTaskToken",
"resourceType": "sqs",
"timeoutInSeconds": 1
},
"timestamp": "timestamp",
"type": "TaskScheduled"
},
{
"id": 4,
"previousEventId": 3,
"taskStartedEventDetails": {
"resource": "sendMessage.waitForTaskToken",
"resourceType": "sqs"
},
"timestamp": "timestamp",
"type": "TaskStarted"
},
{
"id": 5,
"previousEventId": 4,
"taskSubmittedEventDetails": {
"output": {
"MD5OfMessageBody": "823533eb403070c450a065ce6b7bedf6",
"MessageId": "<uuid:1>",
"SdkHttpMetadata": {
"AllHttpHeaders": {
"x-amzn-RequestId": [
"<uuid:2>"
],
"Content-Length": [
"378"
],
"Date": "date",
"Content-Type": [
"text/xml"
]
},
"HttpHeaders": {
"Content-Length": "378",
"Content-Type": "text/xml",
"Date": "date",
"x-amzn-RequestId": "<uuid:2>"
},
"HttpStatusCode": 200
},
"SdkResponseMetadata": {
"RequestId": "<uuid:2>"
}
},
"outputDetails": {
"truncated": false
},
"resource": "sendMessage.waitForTaskToken",
"resourceType": "sqs"
},
"timestamp": "timestamp",
"type": "TaskSubmitted"
},
{
"id": 6,
"previousEventId": 5,
"taskTimedOutEventDetails": {
"error": "States.Timeout",
"resource": "sendMessage.waitForTaskToken",
"resourceType": "sqs"
},
"timestamp": "timestamp",
"type": "TaskTimedOut"
},
{
"executionFailedEventDetails": {
"error": "States.Timeout"
},
"id": 7,
"previousEventId": 6,
"timestamp": "timestamp",
"type": "ExecutionFailed"
}
],
"ResponseMetadata": {
"HTTPHeaders": {},
"HTTPStatusCode": 200
}
}
}
}
}
8 changes: 4 additions & 4 deletions tests/integration/stepfunctions/v2/choice_operators/test_boolean_equals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,25 +19,25 @@
)
class TestBooleanEquals:
def test_boolean_equals(
self, create_iam_role_for_sfn, create_state_machine, snapshot, aws_client
self, create_iam_role_for_sfn, create_state_machine, sfn_snapshot, aws_client
):
create_and_test_comparison_function(
aws_client.stepfunctions,
create_iam_role_for_sfn,
create_state_machine,
snapshot,
sfn_snapshot,
"BooleanEquals",
comparisons=TYPE_COMPARISONS,
)

def test_boolean_equals_path(
self, create_iam_role_for_sfn, create_state_machine, snapshot, aws_client
self, create_iam_role_for_sfn, create_state_machine, sfn_snapshot, aws_client
):
create_and_test_comparison_function(
aws_client.stepfunctions,
create_iam_role_for_sfn,
create_state_machine,
snapshot,
sfn_snapshot,
"BooleanEqualsPath",
comparisons=TYPE_COMPARISONS,
add_literal_value=False,
Expand Down
4 changes: 2 additions & 2 deletions tests/integration/stepfunctions/v2/choice_operators/test_boolean_equals.snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"tests/integration/stepfunctions/v2/choice_operators/test_boolean_equals.py::TestBooleanEquals::test_boolean_equals": {
"recorded-date": "24-04-2023, 09:05:03",
"recorded-date": "22-06-2023, 13:06:32",
"recorded-content": {
"cases": [
{
Expand Down Expand Up @@ -339,7 +339,7 @@
}
},
"tests/integration/stepfunctions/v2/choice_operators/test_boolean_equals.py::TestBooleanEquals::test_boolean_equals_path": {
"recorded-date": "24-04-2023, 09:05:36",
"recorded-date": "22-06-2023, 13:07:20",
"recorded-content": {
"cases": [
{
Expand Down
Loading
0