8000 v2 pipeline changes (#7927) · codeperl/localstack@b4e4c5d · GitHub
[go: up one dir, main page]
Skip to content

Commit b4e4c5d

Browse files
alexrashedalexrashed
authored
v2 pipeline changes (localstack#7927)
1 parent 85a00cb commit b4e4c5d

File tree

16 files changed

+377
-472
lines changed

16 files changed

+377
-472
lines changed

‎.circleci/config.yml

Lines changed: 33 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -228,35 +228,17 @@ jobs:
228228
type: string
229229
machine:
230230
image: << parameters.machine_image >>
231-
# TODO re-enable docker layer caching after file system problems are fixed
232-
# docker_layer_caching: true
233231
resource_class: << parameters.resource_class >>
234232
working_directory: /tmp/workspace/repo
235233
steps:
236234
- attach_workspace:
237235
at: /tmp/workspace
238-
- run:
239-
name: Build full docker image
240-
command: make docker-build-full
241236
- run:
242237
name: Build community docker image
243238
command: make docker-build
244239
- run:
245-
name: Build light docker image
246-
command: make docker-build-light
247-
- run:
248-
name: Build pro docker image
249-
command: make docker-build-pro
250-
- run:
251-
name: Save docker images
252-
command: PLATFORM="<< parameters.platform >>" make docker-save-images
253-
- when:
254-
condition:
255-
equal: [ master, << pipeline.git.branch >> ]
256-
steps:
257-
- run:
258-
name: Run pre-release smoke tests
259-
command: echo "make ci-pro-smoke-tests"
240+
name: Save docker image
241+
command: PLATFORM="<< parameters.platform >>" make docker-save-image
260242
- persist_to_workspace:
261243
root:
262244
/tmp/workspace
@@ -286,8 +268,8 @@ jobs:
286268
- attach_workspace:
287269
at: /tmp/workspace
288270
- run:
289-
name: Load docker localstack-full image
290-
command: docker load -i target/localstack-docker-images-<< parameters.platform >>.tar
271+
name: Load docker image
272+
command: docker load -i target/localstack-docker-image-<< parameters.platform >>.tar
291273
- run:
292274
name: Run integration tests
293275
# circleci split returns newline separated list, so `tr` is necessary to prevent problems in the Makefile
@@ -308,31 +290,25 @@ jobs:
308290
- repo/target/metric_reports
309291

310292
capture-not-implemented:
311-
parameters:
312-
pro:
313-
description: "Run tests against Pro?"
314-
default: false
315-
type: boolean
316293
executor: ubuntu-machine-amd64
317294
working_directory: /tmp/workspace/repo
318295
steps:
319296
- attach_workspace:
320297
at: /tmp/workspace
321298
- run:
322-
name: Load docker localstack-full image
323-
command: docker load -i target/localstack-docker-images-amd64.tar
299+
name: Load docker image
300+
command: docker load -i target/localstack-docker-image-amd64.tar
324301
- run:
325302
name: Run localstack
326303
command: |
327-
<<#parameters.pro>>LOCALSTACK_API_KEY=$TEST_LOCALSTACK_API_KEY<</parameters.pro>> DEBUG=1 DISABLE_EVENTS="1" IMAGE_NAME="localstack/localstack-full:latest" bin/localstack start -d
304+
DEBUG=1 DISABLE_EVENTS="1" IMAGE_NAME="localstack/localstack:latest" bin/localstack start -d
328305
bin/localstack wait -t 120 || (bin/localstack logs && false)
329306
- run:
330307
name: Run capture-not-implemented
331308
command: |
332309
source .venv/bin/activate
333310
cd scripts
334-
<<#parameters.pro>>mkdir -p pro<</parameters.pro>>
335-
python -m capture_notimplemented_responses <<#parameters.pro>>./pro<</parameters.pro>>
311+
python -m capture_notimplemented_responses
336312
- run:
337313
name: Print the logs
338314
command: |
@@ -347,8 +323,8 @@ jobs:
347323
root:
348324
/tmp/workspace
349325
paths:
350-
- repo/scripts/<<#parameters.pro>>pro/<</parameters.pro>>implementation_coverage_aggregated.csv
351-
- repo/scripts/<<#parameters.pro>>pro/<</parameters.pro>>implementation_coverage_full.csv
326+
- repo/scripts/implementation_coverage_aggregated.csv
327+
- repo/scripts/implementation_coverage_full.csv
352328

353329
report:
354330
executor: ubuntu-machine-amd64
@@ -383,7 +359,6 @@ jobs:
383359
METRIC_REPORT_FILE=$(find parity_metrics -type f -iname "metric-report-raw-data-all-*.csv")
384360
METRIC_REPORT_PATH=$METRIC_REPORT_FILE \
385361
COMMUNITY_IMPL_COV_PATH=scripts/implementation_coverage_full.csv \
386-
PRO_IMPL_COV_PATH=scripts/pro/implementation_coverage_full.csv \
387362
python -m scripts.tinybird.upload_raw_test_metrics_and_coverage
388363
- store_artifacts:
389364
path: parity_metrics/
@@ -393,62 +368,56 @@ jobs:
393368
- store_artifacts:
394369
path: scripts/implementation_coverage_full.csv
395370
destination: community/implementation_coverage_full.csv
396-
- store_artifacts:
397-
path: scripts/pro/implementation_coverage_aggregated.csv
398-
destination: pro/implementation_coverage_aggregated.csv
399-
- store_artifacts:
400-
path: scripts/pro/implementation_coverage_full.csv
401-
destination: pro/implementation_coverage_full.csv
402371

403-
docker-push:
372+
push:
404373
executor: ubuntu-machine-amd64
405374
working_directory: /tmp/workspace/repo
406375
steps:
407376
- attach_workspace:
408377
at: /tmp/workspace
409378
- run:
410-
name: Load docker images - amd64
379+
name: Load docker image - amd64
411380
command: |
412-
# Load all images for AMD64
413-
docker load -i target/localstack-docker-images-amd64.tar
381+
# Load all image for AMD64
382+
docker load -i target/localstack-docker-image-amd64.tar
414383
- run:
415384
name: Log in to ECR registry
416385
command: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
417386
- run:
418-
name: Push docker images - amd64
387+
name: Push docker image - amd64
419388
command: |
420389
# Push to Docker Hub
421-
PLATFORM="amd64" make docker-push-master-all
390+
PLATFORM="amd64" make docker-push-master
422391
# Push to Amazon Public ECR
423392
PLATFORM="amd64" SOURCE_IMAGE_NAME="localstack/localstack" TARGET_IMAGE_NAME="public.ecr.aws/localstack/localstack" make docker-push-master
424-
PLATFORM="amd64" SOURCE_IMAGE_NAME="localstack/localstack-pro" TARGET_IMAGE_NAME="public.ecr.aws/localstack/localstack-pro" make docker-push-master
425393
# Load and push per architecture (load overwrites the previous ones)
426394
- run:
427395
name: Load docker image - arm64
428396
command: |
429-
# Load all images for AMD64
430-
docker load -i target/localstack-docker-images-arm64.tar
397+
# Load all image for AMD64
398+
docker load -i target/localstack-docker-image-arm64.tar
431399
- run:
432-
name: Push docker images - arm64
400+
name: Push docker image - arm64
433401
command: |
434402
# Push to Docker Hub
435-
PLATFORM="arm64" make docker-push-master-all
403+
PLATFORM="arm64" make docker-push-master
436404
# Push to Amazon Public ECR
437405
PLATFORM="arm64" SOURCE_IMAGE_NAME="localstack/localstack" TARGET_IMAGE_NAME="public.ecr.aws/localstack/localstack" make docker-push-master
438-
PLATFORM="arm64" SOURCE_IMAGE_NAME="localstack/localstack-pro" TARGET_IMAGE_NAME="public.ecr.aws/localstack/localstack-pro" make docker-push-master
439406
- run:
440-
name: Create multi-platform manifests - full
407+
name: Create multi-platform manifests
441408
command: |
442409
# Push to Docker Hub
443-
MANIFEST_IMAGE_NAME="localstack/localstack-full" make docker-create-push-manifests
444-
- run:
445-
name: Create multi-platform manifests - light
446-
command: |
447-
# Push to Docker Hub
448-
make docker-create-push-manifests-light
410+
make docker-create-push-manifests
449411
# Push to Amazon Public ECR
450412
MANIFEST_IMAGE_NAME="public.ecr.aws/localstack/localstack" make docker-create-push-manifests
451-
MANIFEST_IMAGE_NAME="public.ecr.aws/localstack/localstack-pro" make docker-create-push-manifests
413+
- run:
414+
name: Publish a dev release
415+
command: |
416+
source .venv/bin/activate
417+
bin/release-helper.sh set-ver $(bin/release-helper.sh next-dev-ver)
418+
make publish
419+
420+
452421
453422
workflows:
454423
main:
@@ -503,12 +472,7 @@ workflows:
503472
requires:
504473
- docker-build-amd64
505474
- capture-not-implemented:
506-
name: collect-not-implemented-community
507-
requires:
508-
- docker-build-amd64
509-
- capture-not-implemented:
510-
name: collect-not-implemented-pro
511-
pro: true
475+
name: collect-not-implemented
512476
requires:
513477
- docker-build-amd64
514478
- report:
@@ -519,10 +483,9 @@ workflows:
519483
- itest-sfn-v2-provider
520484
- docker-test-amd64
521485
- docker-test-arm64
522-
- collect-not-implemented-community
523-
- collect-not-implemented-pro
486+
- collect-not-implemented
524487
- unit-tests
525-
- docker-push:
488+
- push:
526489
filters:
527490
branches:
528491
only: master

‎.github/workflows/asf-updates.yml

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ jobs:
1010
runs-on: ubuntu-latest
1111
steps:
1212
- name: Checkout Open Source
13-
uses: actions/checkout@v2
13+
uses: actions/checkout@v3
1414
with:
1515
fetch-depth: 0
1616

@@ -21,12 +21,12 @@ jobs:
2121
2222
- name: Set up Python 3.8
2323
id: setup-python
24-
uses: actions/setup-python@v2
24+
uses: actions/setup-python@v4
2525
with:
26-
python-version: 3.8
26+
python-version: '3.10'
2727

2828
- name: Cache LocalStack community dependencies (venv)
29-
uses: actions/cache@v2
29+
uses: actions/cache@v3
3030
with:
3131
path: .venv
3232
key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('setup.cfg') }}
@@ -52,10 +52,12 @@ jobs:
5252
# Store the result in target/diff-check.log and store the diff count in the GitHub Action output "diff-count"
5353
mkdir -p target
5454
(git diff --name-only origin/asf-auto-updates localstack/aws/api/ 2>/dev/null || git diff --name-only origin/master localstack/aws/api/ 2>/dev/null) | tee target/diff-check.log
55-
echo "::set-output name=diff-count::$(cat target/diff-check.log | wc -l)"
55+
echo "diff-count=$(cat target/diff-check.log | wc -l)" >> $GITHUB_OUTPUT
5656
5757
# Store a (multiline-sanitized) list of changed services (compared to the master) in the GitHub Action output "changed-services"
58-
echo "::set-output name=changed-services::$(git diff --name-only origin/master localstack/aws/api/ | sed 's#localstack/aws/api/#- #g' | sed 's#/__init__.py##g' | sed 's/_/-/g' | sed -z 's/\n/%0A/g' | sed -z 's/\r/%0D/g')"
58+
echo "changed-services<<EOF" >> $GITHUB_OUTPUT
59+
echo "$(git diff --name-only origin/master localstack/aws/api/ | sed 's#localstack/aws/api/#- #g' | sed 's#/__init__.py##g' | sed 's/_/-/g' | sed -z 's/\n/%0A/g' | sed -z 's/\r/%0D/g')" >> $GITHUB_OUTPUT
60+
echo "EOF" >> $GITHUB_OUTPUT
5961
6062
- name: Read PR markdown template
6163
if: ${{ success() && steps.check-for-changes.outputs.diff-count != '0' && steps.check-for-changes.outputs.diff-count != '' }}
@@ -67,14 +69,15 @@ jobs:
6769
- name: Add changed services to template
6870
if: ${{ success() && steps.check-for-changes.outputs.diff-count != '0' && steps.check-for-changes.outputs.diff-count != '' }}
6971
id: markdown
72+
# TODO upgrade after https://github.com/mad9000/actions-find-and-replace-string/pull/11 has been released
7073
uses: mad9000/actions-find-and-replace-string@2
7174
with:
7275
source: ${{ steps.template.outputs.content }}
7376
find: '{{ SERVICES }}'
7477
replace: ${{ steps.check-for-changes.outputs.changed-services }}
7578

7679
- name: Create PR
77-
uses: peter-evans/create-pull-request@v3
80+
uses: peter-evans/create-pull-request@v4
7881
if: ${{ success() && steps.check-for-changes.outputs.diff-count != '0' && steps.check-for-changes.outputs.diff-count != '' }}
7982
with:
8083
title: "Update ASF APIs"

‎.github/workflows/cli-tests.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: cli-tests
1+
name: CLI Tests
22
on:
33
workflow_dispatch:
44
pull_request:
@@ -44,10 +44,10 @@ jobs:
4444
timeout-minutes: 10
4545
steps:
4646
- name: Checkout
47-
uses: actions/checkout@v2
47+
uses: actions/checkout@v3
4848
- name: Set up Python
4949
id: setup-python
50-
uses: actions/setup-python@v2
50+
uses: actions/setup-python@v4
5151
with:
5252
python-version: ${{ matrix.python-version }}
5353
- name: Install CLI test dependencies

‎.github/workflows/pro-integration.yml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: integration-tests-against-pro
1+
name: Community Integration Tests against Pro
22
on:
33
workflow_dispatch:
44
inputs:
@@ -108,19 +108,19 @@ jobs:
108108
console.log("Ext repo does not have a companion ref. Using default: ", DEFAULT_REF)
109109
return DEFAULT_REF
110110
- name: Checkout Pro
111-
uses: actions/checkout@v2
111+
uses: actions/checkout@v3
112112
with:
113113
repository: localstack/localstack-ext
114114
ref: ${{steps.determine-companion-ref.outputs.result}}
115115
token: ${{ secrets.PRO_ACCESS_TOKEN }}
116116
path: localstack-ext
117117
- name: Checkout Open Source
118-
uses: actions/checkout@v2
118+
uses: actions/checkout@v3
119119
with:
120120
path: localstack
121121
- name: Set up Python 3.10
122122
id: setup-python
123-
uses: actions/setup-python@v2
123+
uses: actions/setup-python@v4
124124
with:
125125
python-version: '3.10'
126126
- name: Set up Node 14.x
@@ -140,15 +140,15 @@ jobs:
140140
sudo apt-get update
141141
sudo apt-get install -y --allow-downgrades libsasl2-dev jq postgresql-14=14.7-0ubuntu0.22.04.1 postgresql-client postgresql-plpython3
142142
- name: Cache LocalStack-ext dependencies (venv)
143-
uses: actions/cache@v2
143+
uses: actions/cache@v3
144144
id: ext-cache
145145
with:
146146
path: localstack-ext/.venv
147147
key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-venv-${{ hashFiles('localstack-ext/setup.cfg', 'localstack-ext/pyproject.toml', 'localstack/localstack/services/install.py', 'localstack/setup.cfg', 'localstack/localstack/constants.py') }}
148148
- name: Install Python Dependencies for LocalStack Pro
149149
run: make install
150150
- name: Cache LocalStack community dependencies (venv, static libs)
151-
uses: actions/cache@v2
151+
uses: actions/cache@v3
152152
id: os-cache
153153
with:
154154
path: |

‎.github/workflows/rebase-release-prs.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Rebase release PRs
1+
name: Rebase Release PRs
22
on:
33
push:
44
branches:

‎.github/workflows/rebase-release-targeting-prs.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Rebase PRs targeting release branches
1+
name: Rebase PRs targeting Release Branches
22
on:
33
push:
44
branches:

‎.github/workflows/rh-support.yml

Lines changed: 0 additions & 34 deletions
This file was deleted.

‎.github/workflows/stale-bot.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: "Triage Stale issues"
1+
name: Triage Stale Issues
22

33
on:
44
schedule:

0 commit comments

Comments
 (0)
0