8000 Set up acceptance test marker & CI job (#9045) · codeperl/localstack@5557e15 · GitHub
[go: up one dir, main page]
Skip to content

Commit 5557e15

Browse files
dominikschubertdominikschubert
authored
Set up acceptance test marker & CI job (localstack#9045)
1 parent 7974a9c commit 5557e15

File tree

4 files changed

+63
-15
lines changed

4 files changed

+63
-15
lines changed

‎.circleci/config.yml

Lines changed: 58 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -88,12 +88,11 @@ jobs:
8888
TEST_PATH: "tests/unit"
8989
COVERAGE_ARGS: "-p"
9090
command: |
91-
PYTEST_ARGS="${TINYBIRD_PYTEST_ARGS}--junitxml=target/reports/unit-tests.xml -o junit_suite_name=unit-tests" make test-coverage
91+
COVERAGE_FILE="target/coverage/.coverage.unit.${CIRCLE_NODE_INDEX}" \
92+
PYTEST_ARGS="${TINYBIRD_PYTEST_ARGS}--junitxml=target/reports/unit-tests.xml -o junit_suite_name=unit-tests" \
93+
make test-coverage
9294
- store_test_results:
9395
path: target/reports/
94-
- run:
95-
name: Store coverage results
96-
command: mv .coverage.* target/coverage/
9796
- persist_to_workspace:
9897
root:
9998
/tmp/workspace
@@ -115,10 +114,9 @@ jobs:
115114
TEST_PATH: "tests/aws/services/lambda_/ tests/aws/test_integration.py tests/aws/services/apigateway/test_apigateway_basic.py tests/aws/services/cloudformation/resources/test_lambda.py"
116115
COVERAGE_ARGS: "-p"
117116
command: |
118-
PYTEST_ARGS="${TINYBIRD_PYTEST_ARGS}--reruns 2 --junitxml=target/reports/lambda-docker.xml -o junit_suite_name='legacy-lambda-local'" make test-coverage
119-
- run:
120-
name: Store coverage results
121-
command: mv .coverage.* target/coverage/
117+
COVERAGE_FILE="target/coverage/.coverage.lambdav1.${CIRCLE_NODE_INDEX}" \
118+
PYTEST_ARGS="${TINYBIRD_PYTEST_ARGS}--reruns 2 --junitxml=target/reports/lambda-docker.xml -o junit_suite_name='legacy-lambda-local'" \
119+
make test-coverage
122120
- persist_to_workspace:
123121
root:
124122
/tmp/workspace
@@ -141,14 +139,40 @@ jobs:
141139
TEST_PATH: "tests/aws/services/stepfunctions/v2/"
142140
COVERAGE_ARGS: "-p"
143141
command: |
144-
PYTEST_ARGS="${TINYBIRD_PYTEST_ARGS}--reruns 3 --junitxml=target/reports/sfn_v2.xml -o junit_suite_name='sfn_v2'" make test-coverage
142+
COVERAGE_FILE="target/coverage/.coverage.sfnv2.${CIRCLE_NODE_INDEX}" \
143+
PYTEST_ARGS="${TINYBIRD_PYTEST_ARGS}--reruns 3 --junitxml=target/reports/sfn_v2.xml -o junit_suite_name='sfn_v2'" \
144+
make test-coverage
145+
- persist_to_workspace:
146+
root:
147+
/tmp/workspace
148+
paths:
149+
- repo/target/coverage/
150+
- store_test_results:
151+
path: target/reports/
152+
153+
acceptance-tests:
154+
executor: ubuntu-machine-amd64
155+
working_directory: /tmp/workspace/repo
156+
steps:
157+
- attach_workspace:
158+
at: /tmp/workspace
159+
- prepare-pytest-tinybird
145160
- run:
146-
name: Store coverage results
147-
command: mv .coverage.* target/coverage/
161+
name: Acceptance tests (EXPERIMENTAL)
162+
environment:
163+
TEST_PATH: "tests/aws/"
164+
COVERAGE_ARGS: "-p"
165+
LOCALSTACK_INTERNAL_TEST_COLLECT_METRIC: 1
166+
command: |
167+
COVERAGE_FILE="target/coverage/.coverage.acceptance.${CIRCLE_NODE_INDEX}" \
168+
PYTEST_ARGS="${TINYBIRD_PYTEST_ARGS}--reruns 3 -m acceptance_test --junitxml=target/reports/acceptance_test.xml -o junit_suite_name='acceptance_test'" \
169+
make test-coverage
148170
- persist_to_workspace:
149171
root:
150172
/tmp/workspace
151173
paths:
174+
- repo/target/reports/
175+
- repo/target/metric_reports/
152176
- repo/target/coverage/
153177
- store_test_results:
154178
path: target/reports/
@@ -167,10 +191,9 @@ jobs:
167191
TEST_PATH: "tests/aws/services/s3/"
168192
COVERAGE_ARGS: "-p"
169193
command: |
170-
PYTEST_ARGS="${TINYBIRD_PYTEST_ARGS}--reruns 3 --junitxml=target/reports/s3_stream.xml -o junit_suite_name='s3_stream'" make test-coverage
171-
- run:
172-
name: Store coverage results
173-
command: mv .coverage.* target/coverage/
194+
COVERAGE_FILE="target/coverage/.coverage.s3stream.${CIRCLE_NODE_INDEX}" \
195+
PYTEST_ARGS="${TINYBIRD_PYTEST_ARGS}--reruns 3 --junitxml=target/reports/s3_stream.xml -o junit_suite_name='s3_stream'" \
196+
make test-coverage
174197
- persist_to_workspace:
175198
root:
176199
/tmp/workspace
@@ -346,6 +369,13 @@ jobs:
346369
steps:
347370
- attach_workspace:
348371
at: /tmp/workspace
372+
# store (uncombined) coverage from acceptance tests
373+
- run:
374+
name: fetch isolated acceptance coverage
375+
command: |
376+
cp target/coverage/.coverage.acceptance.* .coverage.acceptance
377+
- store_artifacts:
378+
path: .coverage.acceptance
349379
- run:
350380
name: Collect coverage
351381
command: |
@@ -372,6 +402,13 @@ jobs:
372402
python -m scripts.tinybird.upload_raw_test_metrics_and_coverage
373403
- store_artifacts:
374404
path: parity_metrics/
405+
- run:
406+
name: store acceptance parity metrics
407+
command: |
408+
mkdir acceptance_parity_metrics
409+
mv target/metric_reports/metric-report*acceptance* acceptance_parity_metrics/
410+
- store_artifacts:
411+
path: acceptance_parity_metrics/
375412
- store_artifacts:
376413
path: scripts/implementation_coverage_aggregated.csv
377414
destination: community/implementation_coverage_aggregated.csv
@@ -381,6 +418,7 @@ jobs:
381418
- store_artifacts:
382419
path: .coverage
383420

421+
384422
push:
385423
executor: ubuntu-machine-amd64
386424
working_directory: /tmp/workspace/repo
@@ -438,6 +476,9 @@ workflows:
438476
- preflight:
439477
requires:
440478
- install
479+
- acceptance-tests:
480+
requires:
481+
- preflight
441482
- itest-lambda-legacy-local:
442483
requires:
443484
- preflight
@@ -491,6 +532,7 @@ workflows:
491532
requires:
492533
- itest-lambda-legacy-local
493534
- itest-sfn-v2-provider
535+
- acceptance-tests
494536
- docker-test-amd64
495537
- docker-test-arm64
496538
- collect-not-implemented
@@ -502,6 +544,7 @@ workflows:
502544
requires:
503545
- itest-lambda-legacy-local
504546
- itest-sfn-v2-provider
547+
- acceptance-tests
505548
- docker-test-amd64
506549
- docker-test-arm64
507550
- unit-tests

‎localstack/testing/pytest/marking.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,9 @@ class Markers:
5555
multiruntime: MultiRuntimeMarker = pytest.mark.multiruntime
5656

5757
# test selection
58+
acceptance_test_beta = (
59+
pytest.mark.acceptance_test
60+
) # for now with a _beta suffix to make clear they are not really used as acceptance tests yet
5861
skip_offline = pytest.mark.skip_offline
5962
only_on_amd64 = pytest.mark.only_on_amd64
6063
resource_heavy = pytest.mark.resource_heavy

‎tests/aws/scenario/test_apigateway_scenario.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ def handler(event, context):
1818
"""
1919

2020

21+
@markers.acceptance_test_beta
2122
class TestApigatewayLambdaIntegrationScenario:
2223
@pytest.fixture(scope="class", autouse=True)
2324
def infrastructure(self, aws_client):

‎tests/aws/scenario/test_bookstore.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ def setup_lambda(s3_client: "S3Client", bucket_name: str, key_name: str, code_pa
9393
os.remove(tmp_zip_path)
9494

9595

96+
@markers.acceptance_test_beta
9697
class TestBookstoreApplication:
9798
@pytest.fixture(scope="class")
9899
def patch_opensearch_strategy(self):

0 commit comments

Comments
 (0)
0