From b2594586b34a7d8f1d8de9c2c6db2d7aed6fa073 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Sun, 13 Aug 2023 22:21:49 +0200 Subject: [PATCH 1/4] CI: move some jobs in `build_test.yml` to Meson --- .github/actions/action.yml | 1 - .github/workflows/build_test.yml | 129 ++++++++++++++---------------- .github/workflows/linux_meson.yml | 5 +- build_requirements.txt | 1 + 4 files changed, 60 insertions(+), 76 deletions(-) diff --git a/.github/actions/action.yml b/.github/actions/action.yml index 20a239026e4f..2c271c0c5726 100644 --- a/.github/actions/action.yml +++ b/.github/actions/action.yml @@ -11,7 +11,6 @@ runs: echo DOWNLOAD_OPENBLAS $DOWNLOAD_OPENBLAS echo USE_DEBUG $USE_DEBUG echo NPY_USE_BLAS_ILP64 $NPY_USE_BLAS_ILP64 - echo NUMPY_EXPERIMENTAL_ARRAY_FUNCTION $NUMPY_EXPERIMENTAL_ARRAY_FUNCTION echo USE_ASV $USE_ASV echo PATH $PATH echo python `which python` diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 66f4acd45adc..2d56e3bfe683 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -16,7 +16,6 @@ defaults: env: DOWNLOAD_OPENBLAS: 1 - PYTHON_VERSION: 3.9 concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -37,7 +36,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.9' - name: Install linter requirements run: python -m pip install -r linter_requirements.txt @@ -57,7 +56,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.9' - uses: ./.github/meson_actions basic: @@ -66,7 +65,7 @@ jobs: if: github.event_name != 'push' strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "pypy3.9-v7.3.12"] + python-version: ["3.9", "pypy3.9-v7.3.12"] env: EXPECT_CPU_FEATURES: "SSE SSE2 SSE3 SSSE3 SSE41 POPCNT SSE42 AVX F16C FMA3 AVX2 AVX512F AVX512CD AVX512_KNL AVX512_KNM AVX512_SKX AVX512_CLX AVX512_CNL AVX512_ICL" steps: @@ -81,7 +80,7 @@ jobs: old_gcc: needs: [smoke_test] - # provides GCC 7, 8 + # provides GCC 8 runs-on: ubuntu-20.04 if: github.event_name != 'push' steps: @@ -89,7 +88,6 @@ jobs: with: submodules: recursive fetch-depth: 0 - # comes with python3.6 - name: Install Python3.9 run: | sudo apt update @@ -100,21 +98,14 @@ jobs: sudo ln -s /usr/bin/python3.9 /usr/bin/pythonx pythonx -m pip install --upgrade pip setuptools wheel pythonx -m pip install -r test_requirements.txt - - name: Install Compilers - run: sudo apt install g++-7 g++-8 -y - - name: Build gcc-7 - run: | - export CC=/usr/bin/gcc-7 - export CXX=/usr/bin/g++-7 - rm -rf build && pythonx setup.py install --user - - name: Runtests gcc-7 - run: pythonx runtests.py -n + - name: Install GCC 8 + run: sudo apt install g++-8 -y - name: Build gcc-8 run: | export CC=/usr/bin/gcc-8 export CXX=/usr/bin/g++-8 rm -rf build && pythonx setup.py install --user - - name: Runtests gcc-8 + - name: Run test suite run: pythonx runtests.py -n without_optimizations: @@ -146,7 +137,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.10' - uses: ./.github/meson_actions without_avx512: @@ -162,7 +153,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.10' - uses: ./.github/meson_actions without_avx512_avx2_fma3: @@ -178,7 +169,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.10' - uses: ./.github/meson_actions debug: @@ -194,7 +185,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.11' - uses: ./.github/actions @@ -211,17 +202,13 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.11' - uses: ./.github/actions full: + # Build a wheel, install it, then run the full test suite with code coverage needs: [smoke_test] runs-on: ubuntu-22.04 - env: - USE_WHEEL: 1 - RUN_FULL_TESTS: 1 - RUN_COVERAGE: 1 - INSTALL_PICKLE5: 1 steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 with: @@ -229,8 +216,27 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/actions + python-version: '3.9' + - name: Install build and test dependencies from PyPI + run: | + pip install -r build_requirements.txt + pip install -r test_requirements.txt + - name: Install gfortran and OpenBLAS (MacPython build) + run: | + set -xe + sudo apt install gfortran libgfortran5 + target=$(python tools/openblas_support.py) + sudo cp -r $target/lib/* /usr/lib + sudo cp $target/include/* /usr/include + - name: Build a wheel + run: | + python -m build --wheel --no-isolation --skip-dependency-checks + - name: Run full test suite + run: | + cd doc + pytest --pyargs numpy --cov-report=html:build/coverage + # TODO: gcov + benchmark: needs: [smoke_test] @@ -251,7 +257,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.9' - uses: ./.github/actions relaxed_strides_debug: @@ -269,50 +275,13 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/actions - - use_wheel: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - USE_WHEEL: 1 - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/actions - - no_openblas: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - BLAS: None - LAPACK: None - ATLAS: None - DOWNLOAD_OPENBLAS: '' - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.11' - uses: ./.github/actions sdist: needs: [smoke_test] runs-on: ubuntu-latest if: github.event_name != 'push' - env: - USE_SDIST: 1 steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 with: @@ -320,8 +289,26 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/actions + python-version: '3.12' + - name: Install gfortran and OpenBLAS (MacPython build) + run: | + set -xe + sudo apt install gfortran libgfortran5 + target=$(python tools/openblas_support.py) + sudo cp -r $target/lib/* /usr/lib + sudo cp $target/include/* /usr/include + - name: Build a wheel via an sdist + run: | + python -m build + pip install dist/numpy*.whl + - name: Install test dependencies + run: | + pip install -r test_requirements.txt + - name: Run test suite + run: | + cd doc + pytest --pyargs numpy -m "not slow" + armv7_simd_test: needs: [smoke_test] @@ -391,7 +378,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.9' - name: Install Intel SDE run: | curl -o /tmp/sde.tar.xz https://downloadmirror.intel.com/732268/sde-external-9.7.0-2022-05-09-lin.tar.xz @@ -420,7 +407,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.11' - name: Install Intel SDE run: | curl -o /tmp/sde.tar.xz https://downloadmirror.intel.com/784319/sde-external-9.24.0-2023-07-13-lin.tar.xz diff --git a/.github/workflows/linux_meson.yml b/.github/workflows/linux_meson.yml index 8ef0e5752119..7ded28ac61f3 100644 --- a/.github/workflows/linux_meson.yml +++ b/.github/workflows/linux_meson.yml @@ -10,9 +10,6 @@ defaults: run: shell: bash -env: - PYTHON_VERSION: 3.11 - concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -37,7 +34,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.11' - name: Install dependencies run: | pip install -r build_requirements.txt diff --git a/build_requirements.txt b/build_requirements.txt index e12ac1cf4e2c..7f598f623252 100644 --- a/build_requirements.txt +++ b/build_requirements.txt @@ -3,3 +3,4 @@ Cython>=3.0 wheel==0.38.1 ninja spin==0.5 +build From ed1aac08730da9814887339f31b24aa8a953e590 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Tue, 22 Aug 2023 10:01:15 +0200 Subject: [PATCH 2/4] CI: merge the two GitHub Actions files for Windows builds --- .../{windows_meson.yml => windows.yml} | 28 +++++-- .github/workflows/windows_clangcl.yml | 82 ------------------- 2 files changed, 20 insertions(+), 90 deletions(-) rename .github/workflows/{windows_meson.yml => windows.yml} (82%) delete mode 100644 .github/workflows/windows_clangcl.yml diff --git a/.github/workflows/windows_meson.yml b/.github/workflows/windows.yml similarity index 82% rename from .github/workflows/windows_meson.yml rename to .github/workflows/windows.yml index f6f5c932ee10..52bea0e9e00c 100644 --- a/.github/workflows/windows_meson.yml +++ b/.github/workflows/windows.yml @@ -1,4 +1,4 @@ -name: Test Meson build (Windows) +name: Windows tests on: pull_request: @@ -6,9 +6,6 @@ on: - main - maintenance/** -env: - PYTHON_VERSION: 3.11 - concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -17,10 +14,13 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: - msvc_64bit_python_openblas: - name: MSVC, x86-64, LP64 OpenBLAS + python64bit_openblas: + name: x86-64, LP64 OpenBLAS runs-on: windows-2019 if: "github.repository == 'numpy/numpy'" + strategy: + matrix: + compiler: ["MSVC", "Clang-cl"] steps: - name: Checkout uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 @@ -31,7 +31,7 @@ jobs: - name: Setup Python uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.11' - name: Install build dependencies from PyPI run: | @@ -48,10 +48,22 @@ jobs: unzip -d c:\opt openblas-v0.3.21-win_amd64-gcc_10_3_0.zip echo "PKG_CONFIG_PATH=c:\opt\64\lib\pkgconfig;" >> $env:GITHUB_ENV - - name: Install NumPy + - name: Install Clang-cl + if: matrix.compiler == 'Clang-cl' + run: | + choco install llvm -y --version=16.0.6 + + - name: Install NumPy (MSVC) + if: matrix.compiler == 'MSVC' run: | spin build -j2 -- --vsenv + - name: Install NumPy (Clang-cl) + if: matrix.compiler == 'Clang-cl' + run: | + "[binaries]","c = 'clang-cl'","cpp = 'clang-cl'","ar = 'llvm-lib'","c_ld = 'lld-link'","cpp_ld = 'lld-link'" | Out-File $PWD/clang-cl-build.ini -Encoding ascii + spin build -j2 -- --vsenv --native-file=$PWD/clang-cl-build.ini + - name: Copy OpenBLAS DLL, write _distributor_init.py run: | # Getting the OpenBLAS DLL to the right place so it loads diff --git a/.github/workflows/windows_clangcl.yml b/.github/workflows/windows_clangcl.yml deleted file mode 100644 index 223d4809da89..000000000000 --- a/.github/workflows/windows_clangcl.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: Test Clang-CL Build (Windows) - -on: - pull_request: - branches: - - main - - maintenance/** - -env: - PYTHON_VERSION: 3.11 - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read # to fetch code (actions/checkout) - -jobs: - meson: - name: Meson windows build/test - runs-on: windows-2019 - if: "github.repository == 'numpy/numpy'" - steps: - - name: Checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - submodules: recursive - fetch-depth: 0 - - name: Setup Python - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install build dependencies from PyPI - run: | - pip install spin Cython - - - name: Install OpenBLAS and Clang-cl - run: | - # Download and install pre-built OpenBLAS library with 32-bit - # interfaces Unpack it in the pkg-config hardcoded path - choco install unzip -y - choco install wget -y - # Install llvm, which contains clang-cl - choco install llvm -y --version=16.0.6 - choco install -y --checksum 6004DF17818F5A6DBF19CB335CC92702 pkgconfiglite - wget https://anaconda.org/multibuild-wheels-staging/openblas-libs/v0.3.21/download/openblas-v0.3.21-win_amd64-gcc_10_3_0.zip - unzip -d c:\opt openblas-v0.3.21-win_amd64-gcc_10_3_0.zip - echo "PKG_CONFIG_PATH=c:\opt\64\lib\pkgconfig;" >> $env:GITHUB_ENV - - - name: Write native file for Clang-cl binaries - run: | - # TODO: this job is identical to the one in `windows_meson.yml` aside - # from installing Clang-cl and usage of this .ini file. So merge the - # two and use a matrix'ed CI job run. - "[binaries]","c = 'clang-cl'","cpp = 'clang-cl'","ar = 'llvm-lib'","c_ld = 'lld-link'","cpp_ld = 'lld-link'" | Out-File $PWD/clang-cl-build.ini -Encoding ascii - - - name: Install NumPy - run: | - spin build -j2 -- --vsenv --native-file=$PWD/clang-cl-build.ini - - - name: Copy OpenBLAS DLL, write _distributor_init.py - run: | - # Getting the OpenBLAS DLL to the right place so it loads - $installed_path = "$PWD\build-install\usr\Lib\site-packages" - $numpy_path = "${installed_path}\numpy" - $libs_path = "${numpy_path}\.libs" - mkdir ${libs_path} - $ob_path = "C:/opt/64/bin/" - cp $ob_path/*.dll $libs_path - # Write _distributor_init.py to load .libs DLLs. - python -c "from tools import openblas_support; openblas_support.make_init(r'${numpy_path}')" - - - name: Install test dependencies - run: | - python -m pip install -r test_requirements.txt - python -m pip install threadpoolctl - - - name: Run test suite - run: | - spin test From 1a81895ae6881d39a2cc756fd749b167081895ad Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Tue, 22 Aug 2023 10:58:33 +0200 Subject: [PATCH 3/4] CI: split `build_test.yml` into three GHA jobs files Also documents better what is being run. See gh-24410 for the overall restructuring plan for GitHub Actions CI. --- .github/workflows/linux.yml | 208 +++++++++++++++++ .github/workflows/linux_blas.yml | 36 +++ .../{build_test.yml => linux_simd.yml} | 214 +++--------------- 3 files changed, 277 insertions(+), 181 deletions(-) create mode 100644 .github/workflows/linux.yml create mode 100644 .github/workflows/linux_blas.yml rename .github/workflows/{build_test.yml => linux_simd.yml} (64%) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml new file mode 100644 index 000000000000..d73de33c7640 --- /dev/null +++ b/.github/workflows/linux.yml @@ -0,0 +1,208 @@ +name: Linux tests + +# This file is meant for testing across supported Python versions, build types +# and interpreters (PyPy, python-dbg, a pre-release Python in summer time), +# build-via-sdist, run benchmarks, measure code coverage, and other build +# options like relaxed-strides. + +on: + push: + branches: + # coverage comparison in the "full" step needs to run on main after merges + - main + pull_request: + branches: + - main + - maintenance/** + +defaults: + run: + shell: bash + +env: + DOWNLOAD_OPENBLAS: 1 + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + lint: + if: github.repository == 'numpy/numpy' && github.event_name != 'push' + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.9' + - name: Install linter requirements + run: + python -m pip install -r linter_requirements.txt + - name: Run linter on PR diff + run: + python tools/linter.py --branch origin/${{ github.base_ref }} + + smoke_test: + if: "github.repository == 'numpy/numpy'" + runs-on: ubuntu-latest + env: + MESON_ARGS: "-Dallow-noblas=true -Dcpu-baseline=none -Dcpu-dispatch=none" + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.9' + - uses: ./.github/meson_actions + + basic: + needs: [smoke_test] + runs-on: ubuntu-latest + if: github.event_name != 'push' + strategy: + matrix: + python-version: ["3.9", "pypy3.9-v7.3.12"] + env: + EXPECT_CPU_FEATURES: "SSE SSE2 SSE3 SSSE3 SSE41 POPCNT SSE42 AVX F16C FMA3 AVX2 AVX512F AVX512CD AVX512_KNL AVX512_KNM AVX512_SKX AVX512_CLX AVX512_CNL AVX512_ICL" + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: ${{ matrix.python-version }} + - uses: ./.github/actions + + debug: + needs: [smoke_test] + runs-on: ubuntu-latest + if: github.event_name != 'push' + env: + USE_DEBUG: 1 + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.11' + + - uses: ./.github/actions + + full: + # Build a wheel, install it, then run the full test suite with code coverage + needs: [smoke_test] + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.9' + - name: Install build and test dependencies from PyPI + run: | + pip install -r build_requirements.txt + pip install -r test_requirements.txt + - name: Install gfortran and OpenBLAS (MacPython build) + run: | + set -xe + sudo apt install gfortran libgfortran5 + target=$(python tools/openblas_support.py) + sudo cp -r $target/lib/* /usr/lib + sudo cp $target/include/* /usr/include + - name: Build a wheel + run: | + python -m build --wheel --no-isolation --skip-dependency-check + pip install dist/numpy*.whl + - name: Run full test suite + run: | + cd tools + pytest --pyargs numpy --cov-report=html:build/coverage + # TODO: gcov + + benchmark: + needs: [smoke_test] + runs-on: ubuntu-latest + if: github.event_name != 'push' + env: + PYTHONOPTIMIZE: 2 + BLAS: None + LAPACK: None + ATLAS: None + NPY_BLAS_ORDER: mkl,blis,openblas,atlas,blas + NPY_LAPACK_ORDER: MKL,OPENBLAS,ATLAS,LAPACK + USE_ASV: 1 + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.9' + - uses: ./.github/actions + + relaxed_strides_debug: + needs: [smoke_test] + runs-on: ubuntu-latest + if: github.event_name != 'push' + env: + CHECK_BLAS: 1 + NPY_USE_BLAS_ILP64: 1 + NPY_RELAXED_STRIDES_DEBUG: 1 + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.11' + - uses: ./.github/actions + + sdist: + needs: [smoke_test] + runs-on: ubuntu-latest + if: github.event_name != 'push' + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.11' + - name: Install gfortran and OpenBLAS (MacPython build) + run: | + set -xe + sudo apt install gfortran libgfortran5 + target=$(python tools/openblas_support.py) + sudo cp -r $target/lib/* /usr/lib + sudo cp $target/include/* /usr/include + - name: Build a wheel via an sdist + run: | + pip install build + python -m build + pip install dist/numpy*.whl + - name: Install test dependencies + run: | + pip install -r test_requirements.txt + pip install ninja + - name: Run test suite + run: | + cd tools + pytest --pyargs numpy -m "not slow" + diff --git a/.github/workflows/linux_blas.yml b/.github/workflows/linux_blas.yml new file mode 100644 index 000000000000..cf91ef5f1f7a --- /dev/null +++ b/.github/workflows/linux_blas.yml @@ -0,0 +1,36 @@ +name: BLAS tests (Linux) + +on: + pull_request: + branches: + - main + - maintenance/** + +defaults: + run: + shell: bash + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + openblas64: + runs-on: ubuntu-latest + if: github.event_name != 'push' + env: + DOWNLOAD_OPENBLAS: 1 + NPY_USE_BLAS_ILP64: 1 + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.11' + - uses: ./.github/actions + diff --git a/.github/workflows/build_test.yml b/.github/workflows/linux_simd.yml similarity index 64% rename from .github/workflows/build_test.yml rename to .github/workflows/linux_simd.yml index 2d56e3bfe683..33a2b4620a4e 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/linux_simd.yml @@ -1,10 +1,38 @@ -name: Build_Test +name: SIMD tests (Linux) + +# This file is meant for testing different SIMD-related build options and +# optimization levels. See `meson_options.txt` for the available build options. +# +# Jobs and their purpose: +# +# - smoke_test: +# Meant to complete as quickly as possible, and acts as a filter for +# the other, more expensive jobs (those only start once `smoke_test` +# passes). +# - old_gcc: +# Tests the oldest supported GCC version with the default build +# settings. +# - without_optimizations: +# Completely disables both all SIMD optimizations and other compiler +# optimizations like loop unrolling. +# - with_baseline_only: +# Only uses the baseline SIMD settings, but no runtime dispatch based +# on compiler features detected at runtime. +# - without_avx512_avx2_fma3: +# Uses runtime SIMD dispatching, with AVX2, FMA3 and AVX512 disabled. +# - without_avx512: +# Uses runtime SIMD dispatching, with AVX512 disabled. +# - armv7_simd_test: +# Cross-compiles from x86-64 to armv7, and then runs only the +# SIMD-specific tests under QEMU. +# - sde_simd_avx512_test: +# Runs only the SIMD tests for several AVX512-xxx instruction sets +# under the Intel Software Development Emulator (SDE). +# - intel_spr_sde_test: +# Similar to the SDE test above, but for AVX512-SPR which requires some +# special-casing. on: - push: - branches: - # coverage comparison in the "full" step needs to run on main after merges - - main pull_request: branches: - main @@ -25,25 +53,6 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: - lint: - if: github.repository == 'numpy/numpy' && github.event_name != 'push' - runs-on: ubuntu-latest - continue-on-error: true - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: '3.9' - - name: Install linter requirements - run: - python -m pip install -r linter_requirements.txt - - name: Run linter on PR diff - run: - python tools/linter.py --branch origin/${{ github.base_ref }} - smoke_test: if: "github.repository == 'numpy/numpy'" runs-on: ubuntu-latest @@ -59,25 +68,6 @@ jobs: python-version: '3.9' - uses: ./.github/meson_actions - basic: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - strategy: - matrix: - python-version: ["3.9", "pypy3.9-v7.3.12"] - env: - EXPECT_CPU_FEATURES: "SSE SSE2 SSE3 SSSE3 SSE41 POPCNT SSE42 AVX F16C FMA3 AVX2 AVX512F AVX512CD AVX512_KNL AVX512_KNM AVX512_SKX AVX512_CLX AVX512_CNL AVX512_ICL" - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ matrix.python-version }} - - uses: ./.github/actions - old_gcc: needs: [smoke_test] # provides GCC 8 @@ -172,144 +162,6 @@ jobs: python-version: '3.10' - uses: ./.github/meson_actions - debug: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - USE_DEBUG: 1 - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: '3.11' - - - uses: ./.github/actions - - blas64: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - NPY_USE_BLAS_ILP64: 1 - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: '3.11' - - uses: ./.github/actions - - full: - # Build a wheel, install it, then run the full test suite with code coverage - needs: [smoke_test] - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: '3.9' - - name: Install build and test dependencies from PyPI - run: | - pip install -r build_requirements.txt - pip install -r test_requirements.txt - - name: Install gfortran and OpenBLAS (MacPython build) - run: | - set -xe - sudo apt install gfortran libgfortran5 - target=$(python tools/openblas_support.py) - sudo cp -r $target/lib/* /usr/lib - sudo cp $target/include/* /usr/include - - name: Build a wheel - run: | - python -m build --wheel --no-isolation --skip-dependency-checks - - name: Run full test suite - run: | - cd doc - pytest --pyargs numpy --cov-report=html:build/coverage - # TODO: gcov - - - benchmark: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - PYTHONOPTIMIZE: 2 - BLAS: None - LAPACK: None - ATLAS: None - NPY_BLAS_ORDER: mkl,blis,openblas,atlas,blas - NPY_LAPACK_ORDER: MKL,OPENBLAS,ATLAS,LAPACK - USE_ASV: 1 - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: '3.9' - - uses: ./.github/actions - - relaxed_strides_debug: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - CHECK_BLAS: 1 - NPY_USE_BLAS_ILP64: 1 - NPY_RELAXED_STRIDES_DEBUG: 1 - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: '3.11' - - uses: ./.github/actions - - sdist: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: '3.12' - - name: Install gfortran and OpenBLAS (MacPython build) - run: | - set -xe - sudo apt install gfortran libgfortran5 - target=$(python tools/openblas_support.py) - sudo cp -r $target/lib/* /usr/lib - sudo cp $target/include/* /usr/include - - name: Build a wheel via an sdist - run: | - python -m build - pip install dist/numpy*.whl - - name: Install test dependencies - run: | - pip install -r test_requirements.txt - - name: Run test suite - run: | - cd doc - pytest --pyargs numpy -m "not slow" - - armv7_simd_test: needs: [smoke_test] # make sure this matches the base docker image below From 6d3438067a4c6b644cff4d58d968716ef7558222 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Tue, 22 Aug 2023 11:07:43 +0200 Subject: [PATCH 4/4] CI: merge `linux_meson.yml` into `linux_blas.yml` [skip azp] [skip cirrus] [skip circle] [skip travis] --- .github/workflows/linux_blas.yml | 77 ++++++++++++++++++++++++++++++- .github/workflows/linux_meson.yml | 69 --------------------------- 2 files changed, 75 insertions(+), 71 deletions(-) delete mode 100644 .github/workflows/linux_meson.yml diff --git a/.github/workflows/linux_blas.yml b/.github/workflows/linux_blas.yml index cf91ef5f1f7a..699381f6584f 100644 --- a/.github/workflows/linux_blas.yml +++ b/.github/workflows/linux_blas.yml @@ -1,5 +1,23 @@ name: BLAS tests (Linux) +# This file is meant for testing different BLAS/LAPACK flavors and build +# options on Linux. All other yml files for Linux will only test without BLAS +# (mostly because that's easier and faster to build) or with the same 64-bit +# OpenBLAS build that is used in the wheel jobs. +# +# Jobs and their purpose: +# +# - openblas64_setuppy: +# This job uses the default 64-bit build of OpenBLAS with the +# `numpy.distutils`-based build. It can be removed once we remove +# support for those builds. +# - openblas32_stable_nightly: +# Uses the 32-bit OpenBLAS builds, both the latest stable release and a +# nightly build. +# +# TODO: coverage here is limited, we should add non-OpenBLAS libraries and +# exercise the BLAS-related build options (see `meson_options.txt`). + on: pull_request: branches: @@ -18,9 +36,9 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: - openblas64: + openblas64_setuppy: runs-on: ubuntu-latest - if: github.event_name != 'push' + if: "github.repository == 'numpy/numpy'" env: DOWNLOAD_OPENBLAS: 1 NPY_USE_BLAS_ILP64: 1 @@ -34,3 +52,58 @@ jobs: python-version: '3.11' - uses: ./.github/actions + openblas32_stable_nightly: + if: "github.repository == 'numpy/numpy'" + runs-on: ubuntu-latest + strategy: + matrix: + USE_NIGHTLY_OPENBLAS: [false, true] + env: + USE_NIGHTLY_OPENBLAS: ${{ matrix.USE_NIGHTLY_OPENBLAS }} + name: "Test Linux (${{ matrix.USE_NIGHTLY_OPENBLAS && 'nightly' || 'stable' }} OpenBLAS)" + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install -r build_requirements.txt + # Install OpenBLAS + set -xe + if [[ $USE_NIGHTLY_OPENBLAS == "true" ]]; then + target=$(python tools/openblas_support.py --nightly) + else + target=$(python tools/openblas_support.py) + fi + sudo cp -r $target/lib/* /usr/lib + sudo cp $target/include/* /usr/include + + - name: Build + shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"' + env: + TERM: xterm-256color + run: + spin build -- --werror + + - name: Check build-internal dependencies + run: + ninja -C build -t missingdeps + + - name: Check installed test and stub files + run: + python tools/check_installed_files.py $(find ./build-install -path '*/site-packages/numpy') + + - name: Test + shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"' + env: + TERM: xterm-256color + LD_LIBRARY_PATH: "/usr/local/lib/" # to find libopenblas.so.0 + + run: | + pip install pytest pytest-xdist hypothesis typing_extensions + spin test -j auto diff --git a/.github/workflows/linux_meson.yml b/.github/workflows/linux_meson.yml deleted file mode 100644 index 7ded28ac61f3..000000000000 --- a/.github/workflows/linux_meson.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Test Meson build (Linux) - -on: - pull_request: - branches: - - main - - maintenance/** - -defaults: - run: - shell: bash - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read # to fetch code (actions/checkout) - -jobs: - meson_spin: - if: "github.repository == 'numpy/numpy'" - runs-on: ubuntu-latest - strategy: - matrix: - USE_NIGHTLY_OPENBLAS: [false, true] - env: - USE_NIGHTLY_OPENBLAS: ${{ matrix.USE_NIGHTLY_OPENBLAS }} - name: "Test Linux (${{ matrix.USE_NIGHTLY_OPENBLAS && 'nightly' || 'stable' }} OpenBLAS)" - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: '3.11' - - name: Install dependencies - run: | - pip install -r build_requirements.txt - # Install OpenBLAS - set -xe - if [[ $USE_NIGHTLY_OPENBLAS == "true" ]]; then - target=$(python tools/openblas_support.py --nightly) - else - target=$(python tools/openblas_support.py) - fi - sudo cp -r $target/lib/* /usr/lib - sudo cp $target/include/* /usr/include - - name: Build - shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"' - env: - TERM: xterm-256color - run: - spin build -- --werror - - name: Check build-internal dependencies - run: - ninja -C build -t missingdeps - - name: Check installed test and stub files - run: - python tools/check_installed_files.py $(find ./build-install -path '*/site-packages/numpy') - - name: Test - shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"' - env: - TERM: xterm-256color - LD_LIBRARY_PATH: "/usr/local/lib/" # to find libopenblas.so.0 - run: | - pip install pytest pytest-xdist hypothesis typing_extensions - spin test -j auto