8000 CI: GitHub Actions CI job restructuring by rgommers · Pull Request #24493 · numpy/numpy · GitHub
[go: up one dir, main page]
Skip to content

CI: GitHub Actions CI job restructuring #24493

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 22, 2023
Merged

CI: GitHub Actions CI job restructuring #24493

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b259458
CI: move some jobs in `build_test.yml` to Meson
rgommers Aug 13, 2023
ed1aac0
CI: merge the two GitHub Actions files for Windows builds
rgommers Aug 22, 2023
1a81895
CI: split `build_test.yml` into three GHA jobs files
rgommers Aug 22, 2023
6d34380
CI: merge `linux_meson.yml` into `linux_blas.yml`
rgommers Aug 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .github/actions/action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ runs:
echo DOWNLOAD_OPENBLAS $DOWNLOAD_OPENBLAS
echo USE_DEBUG $USE_DEBUG
echo NPY_USE_BLAS_ILP64 $NPY_USE_BLAS_ILP64
echo NUMPY_EXPERIMENTAL_ARRAY_FUNCTION $NUMPY_EXPERIMENTAL_ARRAY_FUNCTION
echo USE_ASV $USE_ASV
echo PATH $PATH
echo python `which python`
Expand Down
208 changes: 208 additions & 0 deletions .github/workflows/linux.yml
View file
Open in desktop
< 8000 /tr>
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
name: Linux tests

# This file is meant for testing across supported Python versions, build types
# and interpreters (PyPy, python-dbg, a pre-release Python in summer time),
# build-via-sdist, run benchmarks, measure code coverage, and other build
# options like relaxed-strides.

on:
push:
branches:
# coverage comparison in the "full" step needs to run on main after merges
- main
pull_request:
branches:
- main
- maintenance/**

defaults:
run:
shell: bash

env:
DOWNLOAD_OPENBLAS: 1

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

permissions:
contents: read # to fetch code (actions/checkout)

jobs:
lint:
if: github.repository == 'numpy/numpy' && github.event_name != 'push'
runs-on: ubuntu-latest
continue-on-error: true
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: '3.9'
- name: Install linter requirements
run:
python -m pip install -r linter_requirements.txt
- name: Run linter on PR diff
run:
python tools/linter.py --branch origin/${{ github.base_ref }}

smoke_test:
if: "github.repository == 'numpy/numpy'"
runs-on: ubuntu-latest
env:
MESON_ARGS: "-Dallow-noblas=true -Dcpu-baseline=none -Dcpu-dispatch=none"
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: '3.9'
- uses: ./.github/meson_actions

basic:
needs: [smoke_test]
runs-on: ubuntu-latest
if: github.event_name != 'push'
strategy:
matrix:
python-version: ["3.9", "pypy3.9-v7.3.12"]
env:
EXPECT_CPU_FEATURES: "SSE SSE2 SSE3 SSSE3 SSE41 POPCNT SSE42 AVX F16C FMA3 AVX2 AVX512F AVX512CD AVX512_KNL AVX512_KNM AVX512_SKX AVX512_CLX AVX512_CNL AVX512_ICL"
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: ${{ matrix.python-version }}
- uses: ./.github/actions

debug:
needs: [smoke_test]
runs-on: ubuntu-latest
if: github.event_name != 'push'
env:
USE_DEBUG: 1
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: '3.11'

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could get rid of the blank line at some point.

- uses: ./.github/actions

full:
# Build a wheel, install it, then run the full test suite with code coverage
needs: [smoke_test]
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: '3.9'
- name: Install build and test dependencies from PyPI
run: |
pip install -r build_requirements.txt
pip install -r test_requirements.txt
- name: Install gfortran and OpenBLAS (MacPython build)
run: |
set -xe
sudo apt install gfortran libgfortran5
target=$(python tools/openblas_support.py)
sudo cp -r $target/lib/* /usr/lib
sudo cp $target/include/* /usr/include
- name: Build a wheel
run: |
python -m build --wheel --no-isolation --skip-dependency-check
pip install dist/numpy*.whl
- name: Run full test suite
run: |
cd tools
pytest --pyargs numpy --cov-report=html:build/coverage
# TODO: gcov

benchmark:
needs: [smoke_test]
runs-on: ubuntu-latest
if: github.event_name != 'push'
env:
PYTHONOPTIMIZE: 2
BLAS: None
LAPACK: None
ATLAS: None
NPY_BLAS_ORDER: mkl,blis,openblas,atlas,blas
NPY_LAPACK_ORDER: MKL,OPENBLAS,ATLAS,LAPACK
USE_ASV: 1
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: '3.9'
- uses: ./.github/actions

relaxed_strides_debug:
needs: [smoke_test]
runs-on: ubuntu-latest
if: github.event_name != 'push'
env:
CHECK_BLAS: 1
NPY_USE_BLAS_ILP64: 1
NPY_RELAXED_STRIDES_DEBUG: 1
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: '3.11'
- uses: ./.github/actions

sdist:
needs: [smoke_test]
runs-on: ubuntu-latest
if: github.event_name != 'push'
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: '3.11'
- name: Install gfortran and OpenBLAS (MacPython build)
run: |
set -xe
sudo apt install gfortran libgfortran5
target=$(python tools/openblas_support.py)
sudo cp -r $target/lib/* /usr/lib
sudo cp $target/include/* /usr/include
- name: Build a wheel via an sdist
run: |
pip install build
< 8000 /td> python -m build
pip install dist/numpy*.whl
- name: Install test dependencies
run: |
pip install -r test_requirements.txt
pip install ninja
- name: Run test suite
run: |
cd tools
pytest --pyargs numpy -m "not slow"

49 changes: 43 additions & 6 deletions .github/workflows/linux_meson.yml → .github/workflows/linux_blas.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,22 @@
name: Test Meson build (Linux)
name: BLAS tests (Linux)

# This file is meant for testing different BLAS/LAPACK flavors and build
# options on Linux. All other yml files for Linux will only test without BLAS
# (mostly because that's easier and faster to build) or with the same 64-bit
# OpenBLAS build that is used in the wheel jobs.
#
# Jobs and their purpose:
#
# - openblas64_setuppy:
# This job uses the default 64-bit build of OpenBLAS with the
# `numpy.distutils`-based build. It can be removed once we remove
# support for those builds.
# - openblas32_stable_nightly:
# Uses the 32-bit OpenBLAS builds, both the latest stable release and a
# nightly build.
#
# TODO: coverage here is limited, we should add non-OpenBLAS libraries and
# exercise the BLAS-related build options (see `meson_options.txt`).

on:
pull_request:
Expand All @@ -10,9 +28,6 @@ defaults:
run:
shell: bash

env:
PYTHON_VERSION: 3.11

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
Expand All @@ -21,7 +36,23 @@ permissions:
contents: read # to fetch code (actions/checkout)

jobs:
meson_spin:
openblas64_setuppy:
runs-on: ubuntu-latest
if: "github.repository == 'numpy/numpy'"
env:
DOWNLOAD_OPENBLAS: 1
NPY_USE_BLAS_ILP64: 1
steps:
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
with:
submodules: recursive
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: '3.11'
- uses: ./.github/actions

openblas32_stable_nightly:
if: "github.repository == 'numpy/numpy'"
runs-on: ubuntu-latest
strategy:
Expand All @@ -37,7 +68,8 @@ jobs:
fetch-depth: 0
- uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
with:
python-version: ${{ env.PYTHON_VERSION }}
python-version: '3.11'

- name: Install dependencies
run: |
pip install -r build_requirements.txt
Expand All @@ -50,23 +82,28 @@ jobs:
fi
sudo cp -r $target/lib/* /usr/lib
sudo cp $target/include/* /usr/include

- name: Build
shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"'
env:
TERM: xterm-256color
run:
spin build -- --werror

- name: Check build-internal dependencies
run:
ninja -C build -t missingdeps

- name: Check installed test and stub files
run:
python tools/check_installed_files.py $(find ./build-install -path '*/site-packages/numpy')

- name: Test
shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"'
env:
TERM: xterm-256color
LD_LIBRARY_PATH: "/usr/local/lib/" # to find libopenblas.so.0

run: |
pip install pytest pytest-xdist hypothesis typing_extensions
spin test -j auto
Loading
0