From 68ec29452ac9b537c6943c74f9eaa5c1b723aa9c Mon Sep 17 00:00:00 2001
From: Ralf Gommers <ralf.gommers@gmail.com>
Date: Thu, 24 Aug 2023 14:23:58 +0200
Subject: [PATCH 1/5] CI: remove TravisCI jobs

There are timing out a lot, and ppc64le was recently disabled.
We now have QEMU-based jobs in GitHub Action for these platforms.

[skip azp] [skip cirrus] [skip circle] [skip actions]
---
 .travis.yml | 59 -----------------------------------------------------
 1 file changed, 59 deletions(-)
 delete mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 72845eefac09..000000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,59 +0,0 @@
-# After changing this file, check it on:
-#   http://lint.travis-ci.org/
-language: python
-group: travis_latest
-os: linux
-dist: focal
-
-# Travis allows these packages, additions can be requested
-#   https://github.com/travis-ci/apt-package-safelist
-addons:
-  apt:
-    packages: &common_packages
-      - gfortran
-      - libgfortran5
-      - libatlas-base-dev
-      # Speedup builds, particularly when USE_CHROOT=1
-      - eatmydata
-
-# Disable clone depth
-git:
-  depth: false
-
-cache:
-  directories:
-    - $HOME/.cache/pip
-
-jobs:
-  include:
-# The ppc64le for these tests is usually missing, resulting in
-# test failure most of the time. Let's not do that.
-#    - python: "3.9"
-#      os: linux
-#      arch: ppc64le
-#      env:
-#       # use OpenBLAS build, not system ATLAS
-#       - DOWNLOAD_OPENBLAS=1
-#       # - NPY_USE_BLAS_ILP64=1   # the openblas build fails
-#       - ATLAS=None
-#       # VSX4 still not supported by ubuntu/gcc-11
-#       - EXPECT_CPU_FEATURES="VSX VSX2 VSX3"
-
-    - python: "3.9"
-      os: linux
-      arch: s390x
-      # fixes VX assembler ambiguous errors
-      # due to compiler incompatibility
-      install: sudo apt update && sudo apt -y --only-upgrade install binutils
-      env:
-       # use OpenBLAS build, not system ATLAS
-       - DOWNLOAD_OPENBLAS=1
-       - NPY_USE_BLAS_ILP64=1
-       - ATLAS=None
-       - EXPECT_CPU_FEATURES="VX VXE VXE2"
-
-before_install:
-  - ./tools/travis-before-install.sh
-
-script:
-  - ./tools/travis-test.sh

From 5db5d413ec7c8f84a5d33fc6a739f981f573a4cd Mon Sep 17 00:00:00 2001
From: Ralf Gommers <ralf.gommers@gmail.com>
Date: Thu, 24 Aug 2023 14:28:46 +0200
Subject: [PATCH 2/5] CI: remove GHA job that built with setup.py and ILP64
 OpenBLAS

We still have several other jobs that use ILP64, and this setup.py
based job does nothing special. Hence, let's remove it.

[skip cirrus] [skip circle] [skip azp]
---
 .github/workflows/linux_blas.yml | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/.github/workflows/linux_blas.yml b/.github/workflows/linux_blas.yml
index 9b65d38ba91a..f4996ca02218 100644
--- a/.github/workflows/linux_blas.yml
+++ b/.github/workflows/linux_blas.yml
@@ -36,22 +36,6 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 
 jobs:
-  openblas64_setuppy:
-    runs-on: ubuntu-latest
-    if: "github.repository == 'numpy/numpy'"
-    env:
-      DOWNLOAD_OPENBLAS: 1
-      NPY_USE_BLAS_ILP64: 1
-    steps:
-    - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
-      with:
-        submodules: recursive
-        fetch-depth: 0
-    - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
-      with:
-        python-version: '3.11'
-    - uses: ./.github/actions
-
   openblas32_stable_nightly:
     if: "github.repository == 'numpy/numpy'"
     runs-on: ubuntu-latest

From 5585785c53b952262dcefbb4a954814713bb0e3d Mon Sep 17 00:00:00 2001
From: Ralf Gommers <ralf.gommers@gmail.com>
Date: Thu, 24 Aug 2023 14:53:09 +0200
Subject: [PATCH 3/5] CI: move conda and macOS Azure job to Meson

The use of conda and testing on macOS is nice to combine;
no need for the separate jobs. Same for the almost 100%
duplication of the two macOS Azure jobs - we can use only
one here.

Code coverage is removed, because it's done in another job on
Linux already, and uploading to Azure isn't all that useful.
Some more cleanups to comments and code structure, so that the
job is reasonably readable now.
---
 .github/workflows/macos.yml | 100 ++++++++++++++++++++++++++++++++++++
 azure-pipelines.yml         |  96 ++++++++++------------------------
 environment.yml             |   1 +
 3 files changed, 128 insertions(+), 69 deletions(-)
 create mode 100644 .github/workflows/macos.yml

diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
new file mode 100644
index 000000000000..0abb9d0479c8
--- /dev/null
+++ b/.github/workflows/macos.yml
@@ -0,0 +1,100 @@
+name: macOS tests (meson)
+
+on:
+  pull_request:
+    branches:
+      - main
+      - maintenance/**
+
+permissions:
+   contents: read  # to fetch code (actions/checkout)
+
+env:
+  CCACHE_DIR: "${{ github.workspace }}/.ccache"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  x86_conda:
+    name: macOS x86-64 conda
+    if: "github.repository == 'numpy/numpy'"
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        python-version: ["3.11"]
+
+    steps:
+    - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      with:
+        submodules: recursive
+        fetch-depth: 0
+
+    - name:  Prepare cache dirs and timestamps
+      id:    prep-ccache
+      shell: bash -l {0}
+      run: |
+        mkdir -p "${CCACHE_DIR}"
+        echo "dir=$CCACHE_DIR" >> $GITHUB_OUTPUT
+        NOW=$(date -u +"%F-%T")
+        echo "timestamp=${NOW}" >> $GITHUB_OUTPUT
+        echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
+
+    - name: Setup compiler cache
+      uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
+      id:    cache-ccache
+      with:
+        path: ${{ steps.prep-ccache.outputs.dir }}
+        key:  ${{ github.workflow }}-${{ matrix.python-version }}-ccache-macos-${{ steps.prep-ccache.outputs.timestamp }}
+        restore-keys: |
+          ${{ github.workflow }}-${{ matrix.python-version }}-ccache-macos-
+
+    - name: Setup Mambaforge
+      uses: conda-incubator/setup-miniconda@3b0f2504dd76ef23b6d31f291f4913fb60ab5ff3 # v2.2.0
+      with:
+        python-version: ${{ matrix.python-version }}
+        channels: conda-forge
+        channel-priority: true
+        activate-environment: numpy-dev
+        use-only-tar-bz2: false
+        miniforge-variant: Mambaforge
+        miniforge-version: latest
+        use-mamba: true
+
+    # Updates if `environment.yml` or the date changes. The latter is needed to
+    # ensure we re-solve once a day (since we don't lock versions). Could be
+    # replaced by a conda-lock based approach in the future.
+    - name: Cache conda environment
+      uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
+      env:
+        # Increase this value to reset cache if environment.yml has not changed
+        CACHE_NUMBER: 1
+      with:
+        path: ${{ env.CONDA }}/envs/numpy-dev
+        key:
+          ${{ runner.os }}--${{ steps.prep-ccache.outputs.today }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }}
+      id: envcache
+
+    - name: Update Conda Environment
+      run: mamba env update -n numpy-dev -f environment.yml
+      if: steps.envcache.outputs.cache-hit != 'true'
+
+    - name: Build and Install NumPy
+      shell: bash -l {0}
+      run: |
+        conda activate numpy-dev
+        CC="ccache $CC" spin build -j2
+
+    - name: Run test suite (full)
+      shell: bash -l {0}
+      run: |
+        conda activate numpy-dev
+        export OMP_NUM_THREADS=2
+        spin test -j2 -m full
+
+    - name: Ccache statistics
+      shell: bash -l {0}
+      run: |
+        conda activate numpy-dev
+        ccache -s
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index d14e73b27edc..0c2109fdf387 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -89,15 +89,13 @@ stages:
     pool:
       vmImage: 'macOS-11'
     strategy:
-      maxParallel: 3
+      maxParallel: 2
       matrix:
           Python39:
             PYTHON_VERSION: '3.9'
-            USE_OPENBLAS: '1'
           Python39-ILP64:
             PYTHON_VERSION: '3.9'
             NPY_USE_BLAS_ILP64: '1'
-            USE_OPENBLAS: '1'
     steps:
     - script: |
         git submodule update --init
@@ -123,9 +121,8 @@ stages:
         source tools/wheels/gfortran_utils.sh
         install_gfortran
       displayName: 'install gfortran'
-    # use the pre-built openblas binary that most closely
-    # matches our MacOS wheel builds -- currently based
-    # primarily on file size / name details
+    # use the pre-built openblas binary that most closely matches our MacOS
+    # wheel builds -- currently based primarily on file size / name details
     - script: |
         set -xe
         target=$(python tools/openblas_support.py)
@@ -135,8 +132,7 @@ stages:
         cp $target/include/* /usr/local/include/
         otool -L /usr/local/lib/libopenblas*
       displayName: 'install pre-built openblas'
-      condition: eq(variables['USE_OPENBLAS'], '1')
-    - script: python -m pip install --upgrade pip 'setuptools<49.2.0' wheel
+    - script: python -m pip install --upgrade pip
       displayName: 'Install tools'
     - script: |
         python -m pip install -r test_requirements.txt
@@ -149,61 +145,29 @@ stages:
     - script: git submodule update --init
       displayName: 'Fetch submodules'
 
-    # prefer usage of clang over gcc proper
-    # to match likely scenario on many user mac machines
-    - script: python setup.py build -j 4 build_src --verbose-cfg install
+    # TODO: pick up the correct OpenBLAS libraries once we can install those through wheels
+    - script: python -m pip install . -Ccompile-args="-j4" -Csetup-args="-Dallow-noblas=true"
       displayName: 'Build NumPy'
-      env:
-        BLAS: None
-        LAPACK: None
-        ATLAS: None
-        CC: /usr/bin/clang
 
-    # wait until after dev build of NumPy to pip
-    # install matplotlib to avoid pip install of older numpy
+    # only install matplotlib here, to avoid pulling in an older numpy
     - script: python -m pip install matplotlib
       displayName: 'Install matplotlib before refguide run'
 
-    - script: python runtests.py -g --refguide-check
+    - script: |
+        set -xe
+        cd tools
+        python refguide_check.py --doctests
       displayName: 'Run Refguide Check'
-      condition: eq(variables['USE_OPENBLAS'], '1')
 
     - script: |
+        cd tools
         echo LIBRARY_PATH ${LIBRARY_PATH}
-        python runtests.py -n --mode=full -- -rsx --junitxml=junit/test-results.xml
+        pytest --pyargs numpy
       displayName: 'Run Full NumPy Test Suite'
-      condition: eq(variables['USE_OPENBLAS'], '1')
       env:
         # gfortran installed above adds -lSystem, so this is needed to find it (gh-22043)
         LIBRARY_PATH: /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib
 
-    - bash: |
-        python -m pip install threadpoolctl
-        python tools/openblas_support.py --check_version
-      displayName: 'Verify OpenBLAS version'
-      condition: eq(variables['USE_OPENBLAS'], '1')
-
-    # import doesn't work when in numpy src directory , so do a pip dev install of build lib to test
-    - script: |
-        #!/bin/bash -v
-        set +e
-        python -c "import numpy as np" > test_output.log 2>&1
-        check_output_code=$?
-        cat test_output.log
-        grep "buggy Accelerate backend"  test_output.log
-        check_message=$?
-        if [ $check_output_code == 1 ] && [ $check_message == 0 ]; then exit 0; else exit 1;fi
-      displayName: "Check if numpy import fails with accelerate"
-      condition: eq(variables['USE_OPENBLAS'], '0')
-
-    - task: PublishTestResults@2
-      condition: succeededOrFailed()
-      inputs:
-        testResultsFiles: '**/test-*.xml'
-        failTaskOnFailedTests: true
-        testRunTitle: 'Publish test results for Python 3.9 64-bit full Mac OS'
-
-
   - job: Windows
     pool:
       vmImage: 'windows-2019'
@@ -242,25 +206,19 @@ stages:
         git submodule update --init
       displayName: 'Fetch submodules'
     - script: |
-            # create and activate conda environment
-            conda env create -f environment.yml
+        conda env create -f environment.yml
       displayName: 'Create conda environment.'
     - script: |
-            # >>> conda initialize >>>
-            # !! Contents within this block are 'conda init' !!
-            # see https://github.com/conda/conda/issues/7980
-            __conda_setup="$('conda' 'shell.bash' 'hook' 2> /dev/null)"
-            eval "$__conda_setup"
-            unset __conda_setup
-            # <<< conda initialize <<<
-            conda activate numpy-dev
-            # Run native baseline Build / Tests
-            python runtests.py --show-build-log --cpu-baseline=native --cpu-dispatch=none \
-            --debug-info --mode=full -- -rsx --junitxml=junit/test-results.xml
-      displayName: 'Run native baseline Build / Tests in conda.'
-    - task: PublishTestResults@2
-      condition: succeededOrFailed()
-      inputs:
-        testResultsFiles: '**/test-*.xml'
-        failTaskOnFailedTests: true
-        testRunTitle: 'Publish test results for conda installation'
+        # >>> conda initialize >>>
+        # !! Contents within this block are 'conda init' !!
+        # see https://github.com/conda/conda/issues/7980
+        __conda_setup="$('conda' 'shell.bash' 'hook' 2> /dev/null)"
+        eval "$__conda_setup"
+        unset __conda_setup
+        # <<< conda initialize <<<
+        conda activate numpy-dev
+        # Note: conda env activation doesn't carry over between steps, so
+        # build/test are both in this step
+        spin build -- -Dcpu-baseline=native -Dcpu-dispatch=none
+        spin test -m full
+      displayName: 'Build with native baseline, run full test suite'
diff --git a/environment.yml b/environment.yml
index ea8e1ba4f6a2..4fde472e8fac 100644
--- a/environment.yml
+++ b/environment.yml
@@ -18,6 +18,7 @@ dependencies:
   - meson-python
   - pip
   - spin
+  - ccache
   # For testing
   - pytest
   - pytest-cov

From b7d76b19d79fcce1c6109e0c9d54ded1e6379508 Mon Sep 17 00:00:00 2001
From: Ralf Gommers <ralf.gommers@gmail.com>
Date: Fri, 25 Aug 2023 11:41:56 +0200
Subject: [PATCH 4/5] CI: factor out custom code checks (ninja, vulture,
 `test_*` installed)

[skip cirrus] [skip circle]
---
 .github/meson_actions/action.yml |  8 --------
 .github/workflows/linux.yml      | 31 +++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/.github/meson_actions/action.yml b/.github/meson_actions/action.yml
index aff70da169bc..4b0c24577f41 100644
--- a/.github/meson_actions/action.yml
+++ b/.github/meson_actions/action.yml
@@ -12,14 +12,6 @@ runs:
       TERM: xterm-256color
     run:
       spin build -- ${MESON_ARGS[@]}
-  - name: Check build-internal dependencies
-    shell: bash
-    run:
-      ninja -C build -t missingdeps
-  - name: Check installed test and stub files
-    shell: bash
-    run:
-      python tools/check_installed_files.py $(find ./build-install -path '*/site-packages/numpy')
   - name: Test
     shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"'
     env:
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index 963232b3f0c8..8b944899304e 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -206,3 +206,34 @@ jobs:
         cd tools
         pytest --pyargs numpy -m "not slow"
 
+  custom_checks:
+    needs: [smoke_test]
+    runs-on: ubuntu-latest
+    if: github.event_name != 'push'
+    steps:
+    - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      with:
+        submodules: recursive
+        fetch-depth: 0
+    - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+      with:
+        python-version: '3.11'
+    - name: Install build and test dependencies from PyPI
+      run: |
+        pip install -r build_requirements.txt
+        pip install -r test_requirements.txt
+        pip install vulture
+    - name: Build and install NumPy
+      run: |
+        # Install using the fastests way to build (no BLAS, no SIMD)
+        spin build -j2 -- -Dallow-noblas=true -Dcpu-baseline=none -Dcpu-dispatch=none
+    - name: Check build-internal dependencies
+      run: |
+        ninja -C build -t missingdeps
+    - name: Check installed test and stub files
+      run: |
+        python tools/check_installed_files.py $(find ./build-install -path '*/site-packages/numpy')
+    - name: Check for unreachable code paths in Python modules
+      run: |
+        # Need the explicit `bash -c` here because `grep` returns exit code 1 for no matches
+        bash -c "! vulture . --min-confidence 100 --exclude doc/,numpy/distutils/,vendored-meson/ | grep 'unreachable'"

From a6234ad55e69dd1ef390e5b0356a9de73f00ffd4 Mon Sep 17 00:00:00 2001
From: Ralf Gommers <ralf.gommers@gmail.com>
Date: Fri, 25 Aug 2023 12:39:06 +0200
Subject: [PATCH 5/5] CI: remove macOS and conda jobs from Azure

[skip cirrus] [skip circle]
---
 azure-pipelines.yml | 110 --------------------------------------------
 1 file changed, 110 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 0c2109fdf387..ce78d07777fa 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -84,90 +84,6 @@ stages:
             python3 -m pytest --pyargs numpy"
       displayName: 'Run 32-bit manylinux2014 Docker Build / Tests'
 
-
-  - job: macOS
-    pool:
-      vmImage: 'macOS-11'
-    strategy:
-      maxParallel: 2
-      matrix:
-          Python39:
-            PYTHON_VERSION: '3.9'
-          Python39-ILP64:
-            PYTHON_VERSION: '3.9'
-            NPY_USE_BLAS_ILP64: '1'
-    steps:
-    - script: |
-        git submodule update --init
-      displayName: 'Fetch submodules'
-    # the @0 refers to the (major) version of the *task* on Microsoft's
-    # end, not the order in the build matrix nor anything to do
-    # with version of Python selected
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: $(PYTHON_VERSION)
-        addToPath: true
-        architecture: 'x64'
-    - script: |
-        set -xe
-        [ -n "$USE_XCODE_10" ] && /bin/bash -c "sudo xcode-select -s /Applications/Xcode_10.app/Contents/Developer"
-        clang --version
-      displayName: 'report clang version'
-
-    - script: |
-        if [[ $PLATFORM == "macosx-arm64" ]]; then
-            PLAT="arm64"
-        fi
-        source tools/wheels/gfortran_utils.sh
-        install_gfortran
-      displayName: 'install gfortran'
-    # use the pre-built openblas binary that most closely matches our MacOS
-    # wheel builds -- currently based primarily on file size / name details
-    - script: |
-        set -xe
-        target=$(python tools/openblas_support.py)
-        ls -lR $target
-        # manually link to appropriate system paths
-        cp $target/lib/lib* /usr/local/lib/
-        cp $target/include/* /usr/local/include/
-        otool -L /usr/local/lib/libopenblas*
-      displayName: 'install pre-built openblas'
-    - script: python -m pip install --upgrade pip
-      displayName: 'Install tools'
-    - script: |
-        python -m pip install -r test_requirements.txt
-        # Don't use doc_requirements.txt since that messes up tests
-        python -m pip install vulture sphinx==4.3.0 numpydoc==1.4.0 ninja
-      displayName: 'Install dependencies; some are optional to avoid test skips'
-    - script: /bin/bash -c "! vulture . --min-confidence 100 --exclude doc/,numpy/distutils/ | grep 'unreachable'"
-      displayName: 'Check for unreachable code paths in Python modules'
-
-    - script: git submodule update --init
-      displayName: 'Fetch submodules'
-
-    # TODO: pick up the correct OpenBLAS libraries once we can install those through wheels
-    - script: python -m pip install . -Ccompile-args="-j4" -Csetup-args="-Dallow-noblas=true"
-      displayName: 'Build NumPy'
-
-    # only install matplotlib here, to avoid pulling in an older numpy
-    - script: python -m pip install matplotlib
-      displayName: 'Install matplotlib before refguide run'
-
-    - script: |
-        set -xe
-        cd tools
-        python refguide_check.py --doctests
-      displayName: 'Run Refguide Check'
-
-    - script: |
-        cd tools
-        echo LIBRARY_PATH ${LIBRARY_PATH}
-        pytest --pyargs numpy
-      displayName: 'Run Full NumPy Test Suite'
-      env:
-        # gfortran installed above adds -lSystem, so this is needed to find it (gh-22043)
-        LIBRARY_PATH: /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib
-
   - job: Windows
     pool:
       vmImage: 'windows-2019'
@@ -196,29 +112,3 @@ stages:
 
     steps:
     - template: azure-steps-windows.yml
-
-
-  - job: Linux_conda
-    pool:
-      vmImage: 'ubuntu-20.04'
-    steps:
-    - script: |
-        git submodule update --init
-      displayName: 'Fetch submodules'
-    - script: |
-        conda env create -f environment.yml
-      displayName: 'Create conda environment.'
-    - script: |
-        # >>> conda initialize >>>
-        # !! Contents within this block are 'conda init' !!
-        # see https://github.com/conda/conda/issues/7980
-        __conda_setup="$('conda' 'shell.bash' 'hook' 2> /dev/null)"
-        eval "$__conda_setup"
-        unset __conda_setup
-        # <<< conda initialize <<<
-        conda activate numpy-dev
-        # Note: conda env activation doesn't carry over between steps, so
-        # build/test are both in this step
-        spin build -- -Dcpu-baseline=native -Dcpu-dispatch=none
-        spin test -m full
-      displayName: 'Build with native baseline, run full test suite'