From 2664f48e31f5ce4a31e0c1b88135bf4be4deab6a Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 3 Jun 2023 09:36:39 +0200 Subject: [PATCH] Optimize CI runs, by avoiding unnecessary rebuilds --- .github/workflows/push.yml | 208 ++++++++----------------------------- Makefile | 67 +++++------- 2 files changed, 72 insertions(+), 203 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index bf38f43fbb..58890b5efb 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -53,8 +53,8 @@ jobs: parallel: true flag-name: run-${{ matrix.os }}-${{ matrix.python-version }} - ubuntu_build_apk: - name: Unit test apk [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] + ubuntu_build: + name: Build test APP [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] needs: [flake8] runs-on: ${{ matrix.runs_on }} continue-on-error: true @@ -68,36 +68,41 @@ jobs: target: testapps-with-scipy - name: webview target: testapps-webview + - name: service_library + target: testapps-service_library-aar steps: - name: Checkout python-for-android uses: actions/checkout@v3 - # helps with GitHub runner getting out of space - - name: Free disk space - run: | - df -h - sudo swapoff -a - sudo rm -f /swapfile - sudo apt -y clean - docker rmi $(docker image ls -aq) - df -h - - name: Pull docker image - run: | - make docker/pull - - name: Build multi-arch apk Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) - run: | - mkdir -p apks - make docker/run/make/with-artifact/apk/${{ matrix.bootstrap.target }} - - name: Rename apk artifact to include the build platform name - run: | - mv apks/${{ env.APK_ARTIFACT_FILENAME }} apks/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - - name: Upload apk artifact - uses: actions/upload-artifact@v1 + - name: Build python-for-android docker image + run: | + docker build --tag=kivy/python-for-android . + - name: Build multi-arch ${{ matrix.bootstrap.target }} artifact with docker + run: | + docker run --name p4a-latest kivy/python-for-android make ${{ matrix.bootstrap.target }} + - name: Copy produced artifacts from docker container (*.apk, *.aab) + if: matrix.bootstrap.name != 'service_library' + run: | + mkdir -p dist + docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.APK_ARTIFACT_FILENAME }} dist/ + docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.AAB_ARTIFACT_FILENAME }} dist/ + - name: Copy produced artifacts from docker container (*.aar) + if: matrix.bootstrap.name == 'service_library' + run: | + mkdir -p dist + docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.AAR_ARTIFACT_FILENAME }} dist/ + - name: Rename artifacts to include the build platform name (*.apk, *.aab, *.aar) + run: | + if [ -f dist/${{ env.APK_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.APK_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}; fi + if [ -f dist/${{ env.AAB_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAB_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}; fi + if [ -f dist/${{ env.AAR_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAR_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAR_ARTIFACT_FILENAME }}; fi + - name: Upload artifacts + uses: actions/upload-artifact@v3 with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - path: apks/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} + name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-artifacts + path: dist - macos_build_apk: - name: Unit test apk [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] + macos_build: + name: Build test APP [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] needs: [flake8] runs-on: ${{ matrix.runs_on }} continue-on-error: true @@ -137,146 +142,21 @@ jobs: source ci/osx_ci.sh arm64_set_path_and_python_version 3.9.7 make ${{ matrix.bootstrap.target }} - - name: Rename apk artifact to include the build platform name - run: | - mv testapps/on_device_unit_tests/${{ env.APK_ARTIFACT_FILENAME }} ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - - name: Upload apk artifact - uses: actions/upload-artifact@v1 - with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - path: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - - ubuntu_build_aab: - name: Unit test aab [ ${{ matrix.runs_on }} ] - needs: [flake8] - runs-on: ${{ matrix.runs_on }} - continue-on-error: true - strategy: - matrix: - runs_on: [ubuntu-latest] - bootstrap: - - name: sdl2 - target: testapps-with-numpy-aab - - name: webview - target: testapps-webview-aab - steps: - - name: Checkout python-for-android - uses: actions/checkout@v3 - # helps with GitHub runner getting out of space - - name: Free disk space - run: | - df -h - sudo swapoff -a - sudo rm -f /swapfile - sudo apt -y clean - docker rmi $(docker image ls -aq) - df -h - - name: Pull docker image - run: | - make docker/pull - - name: Build Android App Bundle Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) - run: | - mkdir -p aabs - make docker/run/make/with-artifact/aab/${{ matrix.bootstrap.target }} - - name: Rename artifact to include the build platform name - run: | - mv aabs/${{ env.AAB_ARTIFACT_FILENAME }} aabs/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }} - - name: Upload apk artifact - uses: actions/upload-artifact@v1 - with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }} - path: aabs/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }} - - - ubuntu_build_aar: - name: Unit test aar [ ${{ matrix.runs_on }} ] - needs: [flake8] - runs-on: ${{ matrix.runs_on }} - continue-on-error: true - strategy: - matrix: - runs_on: [ubuntu-latest] - bootstrap: - - name: service_library - target: testapps-service_library-aar - steps: - - name: Checkout python-for-android - uses: actions/checkout@v3 - # helps with GitHub runner getting out of space - - name: Free disk space - run: | - df -h - sudo swapoff -a - sudo rm -f /swapfile - sudo apt -y clean - docker rmi $(docker image ls -aq) - df -h - - name: Pull docker image - run: | - make docker/pull - - name: Build Android AAR Python 3 (arm64-v8a) - run: | - mkdir -p aars - make docker/run/make/with-artifact/aar/${{ matrix.bootstrap.target }} - - name: Rename artifact to include the build platform name - run: | - mv aars/${{ env.AAR_ARTIFACT_FILENAME }} aars/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }} - - name: Upload aar artifact - uses: actions/upload-artifact@v1 - with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }} - path: aars/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }} - - - macos_build_aab: - name: Unit test aab [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] - needs: [flake8] - runs-on: ${{ matrix.runs_on }} - continue-on-error: true - strategy: - matrix: - runs_on: [macos-latest, apple-silicon-m1] - bootstrap: - - name: sdl2 - target: testapps-with-numpy-aab - - name: webview - target: testapps-webview-aab - env: - ANDROID_HOME: ${HOME}/.android - ANDROID_SDK_ROOT: ${HOME}/.android/android-sdk - ANDROID_SDK_HOME: ${HOME}/.android/android-sdk - ANDROID_NDK_HOME: ${HOME}/.android/android-ndk - steps: - - name: Checkout python-for-android - uses: actions/checkout@v3 - - name: Install python-for-android - run: | - source ci/osx_ci.sh - arm64_set_path_and_python_version 3.9.7 - python3 -m pip install -e . - - name: Install prerequisites via pythonforandroid/prerequisites.py (Experimental) - run: | - source ci/osx_ci.sh - arm64_set_path_and_python_version 3.9.7 - python3 pythonforandroid/prerequisites.py - - name: Install dependencies (Legacy) + - name: Copy produced artifacts into dist/ (*.apk, *.aab) run: | - source ci/osx_ci.sh - arm64_set_path_and_python_version 3.9.7 - make --file ci/makefiles/osx.mk - - name: Build multi-arch sdl2 aab Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) - run: | - source ci/osx_ci.sh - arm64_set_path_and_python_version 3.9.7 - make ${{ matrix.bootstrap.target }} - - name: Rename sdl2 artifact to include the build platform name + mkdir -p dist + cp testapps/on_device_unit_tests/*.apk dist/ + cp testapps/on_device_unit_tests/*.aab dist/ + ls -l dist/ + - name: Rename artifacts to include the build platform name (*.apk, *.aab) run: | - mv testapps/on_device_unit_tests/${{ env.AAB_ARTIFACT_FILENAME }} ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }} - - name: Upload sdl2 apk artifact - uses: actions/upload-artifact@v1 + if [ -f dist/${{ env.APK_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.APK_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}; fi + if [ -f dist/${{ env.AAB_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAB_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}; fi + - name: Upload artifacts + uses: actions/upload-artifact@v3 with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }} - path: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }} + name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-artifacts + path: dist ubuntu_rebuild_updated_recipes: name: Test updated recipes for arch ${{ matrix.android_arch }} [ ubuntu-latest ] diff --git a/Makefile b/Makefile index ea5adadcd8..5b7166b277 100644 --- a/Makefile +++ b/Makefile @@ -36,47 +36,51 @@ rebuild_updated_recipes: virtualenv ANDROID_SDK_HOME=$(ANDROID_SDK_HOME) ANDROID_NDK_HOME=$(ANDROID_NDK_HOME) \ $(PYTHON) ci/rebuild_updated_recipes.py $(REBUILD_UPDATED_RECIPES_EXTRA_ARGS) -testapps-with-numpy: virtualenv +testapps-with-numpy: testapps-with-numpy/debug/apk testapps-with-numpy/release/aab + +# testapps-with-numpy/MODE/ARTIFACT +testapps-with-numpy/%: virtualenv + $(eval MODE := $(word 2, $(subst /, ,$@))) + $(eval ARTIFACT := $(word 3, $(subst /, ,$@))) + @echo Building testapps-with-numpy for $(MODE) mode and $(ARTIFACT) artifact . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \ --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 \ --permission "(name=android.permission.WRITE_EXTERNAL_STORAGE;maxSdkVersion=18)" --permission "(name=android.permission.INTERNET)" -testapps-with-scipy: virtualenv +testapps-with-scipy: testapps-with-scipy/debug/apk testapps-with-scipy/release/aab + +# testapps-with-scipy/MODE/ARTIFACT +testapps-with-scipy/%: virtualenv + $(eval MODE := $(word 2, $(subst /, ,$@))) + $(eval ARTIFACT := $(word 3, $(subst /, ,$@))) + @echo Building testapps-with-scipy for $(MODE) mode and $(ARTIFACT) artifact . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ export LEGACY_NDK=$(ANDROID_NDK_HOME_LEGACY) && \ - python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ --requirements python3,scipy,kivy \ --arch=armeabi-v7a --arch=arm64-v8a -testapps-with-numpy-aab: virtualenv - . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py aab --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ - --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \ - --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 --release \ - --permission "(name=android.permission.WRITE_EXTERNAL_STORAGE;maxSdkVersion=18)" --permission "(name=android.permission.INTERNET)" +testapps-webview: testapps-webview/debug/apk testapps-webview/release/aab -testapps-service_library-aar: virtualenv +# testapps-webview/MODE/ARTIFACT +testapps-webview/%: virtualenv + $(eval MODE := $(word 2, $(subst /, ,$@))) + $(eval ARTIFACT := $(word 3, $(subst /, ,$@))) + @echo Building testapps-webview for $(MODE) mode and $(ARTIFACT) artifact . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py aar --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ - --bootstrap service_library \ - --requirements python3 \ - --arch=arm64-v8a --arch=x86 --release - -testapps-webview: virtualenv - . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ --bootstrap webview \ --requirements sqlite3,libffi,openssl,pyjnius,flask,python3,genericndkbuild \ --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 -testapps-webview-aab: virtualenv +testapps-service_library-aar: virtualenv . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py aab --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ - --bootstrap webview \ - --requirements sqlite3,libffi,openssl,pyjnius,flask,python3,genericndkbuild \ - --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 --release + python setup.py aar --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + --bootstrap service_library \ + --requirements python3 \ + --arch=arm64-v8a --arch=x86 --release testapps/%: virtualenv $(eval $@_APP_ARCH := $(shell basename $*)) @@ -106,21 +110,6 @@ docker/run/test: docker/build docker/run/command: docker/build docker run --rm --env-file=.env $(DOCKER_IMAGE) /bin/sh -c "$(COMMAND)" -docker/run/make/with-artifact/apk/%: docker/build - docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $* - docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-debug-1.1.apk ./apks - docker rm -fv p4a-latest - -docker/run/make/with-artifact/aar/%: docker/build - docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $* - docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-release-1.1.aar ./aars - docker rm -fv p4a-latest - -docker/run/make/with-artifact/aab/%: docker/build - docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $* - docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-release-1.1.aab ./aabs - docker rm -fv p4a-latest - docker/run/make/rebuild_updated_recipes: docker/build docker run --name p4a-latest -e REBUILD_UPDATED_RECIPES_EXTRA_ARGS --env-file=.env $(DOCKER_IMAGE) make rebuild_updated_recipes