diff --git a/.cirrus.star b/.cirrus.star index c503f25720a7..e1188b461332 100644 --- a/.cirrus.star +++ b/.cirrus.star @@ -51,4 +51,4 @@ def main(ctx): if int(pr_number) < 0: return [] - return fs.read("tools/ci/cirrus_arm.yml") + # DISABLED: return fs.read("tools/ci/cirrus_arm.yml") diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 7fad1948fb7f..e3ec479a6ec9 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -73,6 +73,8 @@ jobs: source .venv-pyodide/bin/activate pip install dist/*.whl python -c "import sys; print(sys.platform)" + # TODO: when re-enabled this workflow, install test deps differently + # since ninja isn't installable with pyodide pip install -r test_requirements.txt - name: Test run: | diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 7e3f7a52b4da..707526896802 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -202,7 +202,6 @@ jobs: - name: Install test dependencies run: | pip install -r test_requirements.txt - pip install ninja - name: Run test suite run: | cd tools diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 63d3e0decd6c..1437ebe63ddd 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -196,7 +196,6 @@ jobs: # TODO: Don't run test suite, and instead build wheels from sdist # Depends on pypa/cibuildwheel#1020 python -m pip install dist/*.gz -Csetup-args=-Dallow-noblas=true - pip install ninja pip install -r test_requirements.txt cd .. # Can't import numpy within numpy src directory python -c "import numpy, sys; print(numpy.__version__); sys.exit(numpy.test() is False)" diff --git a/numpy/f2py/tests/util.py b/numpy/f2py/tests/util.py index 75b257cdb825..d8c15135fe79 100644 --- a/numpy/f2py/tests/util.py +++ b/numpy/f2py/tests/util.py @@ -21,7 +21,8 @@ from pathlib import Path from numpy._utils import asunicode -from numpy.testing import temppath, IS_WASM +from numpy.exceptions import VisibleDeprecationWarning +from numpy.testing import temppath, IS_WASM, suppress_warnings from importlib import import_module # @@ -409,13 +410,16 @@ def setup_method(self): ) if self.sources is not None: - self.module = build_module( - self.sources, - options=self.options, - skip=self.skip, - only=self.only, - module_name=self.module_name, - ) + with suppress_warnings() as sup: + sup.filter(VisibleDeprecationWarning, + "distutils has been deprecated since") + self.module = build_module( + self.sources, + options=self.options, + skip=self.skip, + only=self.only, + module_name=self.module_name, + ) # diff --git a/pyproject.toml b/pyproject.toml index 03cdfe34aaa8..dee74a5d5eee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -140,34 +140,28 @@ tracker = "https://github.com/numpy/numpy/issues" # Note: the below skip command doesn't do much currently, the platforms to # build wheels for in CI are controlled in `.github/workflows/wheels.yml` and # `tools/ci/cirrus_wheels.yml`. +build-frontend = "build" skip = "cp36-* cp37-* cp-38* pp37-* *-manylinux_i686 *_ppc64le *_s390x" build-verbosity = "3" before-build = "bash {project}/tools/wheels/cibw_before_build.sh {project}" -config-settings = "setup-args=-Duse-ilp64=true setup-args=-Dblas=openblas setup-args=-Dlapack=openblas setup-args=-Dblas-symbol-suffix=64_ setup-args=-Dallow-noblas=false" -# meson has a hard dependency on ninja, and we need meson to build -# c-extensions in tests. There is a ninja PyPI package used in -# build_requirements.txt for macOS, windows, linux but it cannot be in -# test_requirements.txt since pyodide, which uses test_requirements.txt, does -# not have it. -before-test = "pip install ninja && pip install -r {project}/test_requirements.txt" +# The build will use openblas64 everywhere, except on arm64 macOS >=14.0 (uses Accelerate) +config-settings = "setup-args=-Duse-ilp64=true setup-args=-Dallow-noblas=false build-dir=build" +before-test = "pip install -r {project}/test_requirements.txt" test-command = "bash {project}/tools/wheels/cibw_test_command.sh {project}" [tool.cibuildwheel.linux] manylinux-x86_64-image = "manylinux2014" manylinux-aarch64-image = "manylinux2014" musllinux-x86_64-image = "musllinux_1_1" -environment = {CFLAGS="-fno-strict-aliasing", LDFLAGS="-Wl,--strip-debug", RUNNER_OS="Linux"} +# RUNNER_OS is a GitHub Actions specific env var; define it here so it works on Cirrus CI too +environment = {RUNNER_OS="Linux"} [tool.cibuildwheel.macos] -# For universal2 wheels, we will need to fuse them manually -# instead of going through cibuildwheel -# This is because cibuildwheel tries to make a fat wheel -# https://github.com/multi-build/multibuild/blame/devel/README.rst#L541-L565 -# for more info +# universal2 wheels are not supported (see gh-21233), use `delocate-fuse` if you need them archs = "x86_64 arm64" test-skip = "*_universal2:arm64" # MACOS linker doesn't support stripping symbols. -environment = {CFLAGS="-fno-strict-aliasing", RUNNER_OS="macOS"} +environment = {RUNNER_OS="macOS"} [tool.cibuildwheel.windows] environment = {PKG_CONFIG_PATH="C:/opt/64/lib/pkgconfig"} diff --git a/test_requirements.txt b/test_requirements.txt index ff1ed284e37d..ecf4bd5c924b 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -7,6 +7,7 @@ pytest==7.4.0 pytz==2023.3 pytest-cov==4.1.0 meson +ninja pytest-xdist # for numpy.random.test.test_extending cffi; python_version < '3.10' diff --git a/tools/ci/cirrus_arm.yml b/tools/ci/cirrus_arm.yml index 7e394f55442b..cf85ca607c38 100644 --- a/tools/ci/cirrus_arm.yml +++ b/tools/ci/cirrus_arm.yml @@ -95,7 +95,6 @@ macos_arm64_test_task: python --version RUNNER_OS="macOS" - SDKROOT=/Applications/Xcode-14.0.0.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.3.sdk # NOTE: OpenBLAS is not used in this job; if that's done in the future, ensure # PKG_CONFIG_PATH points to the directory containing the openblas.pc file diff --git a/tools/ci/cirrus_wheels.yml b/tools/ci/cirrus_wheels.yml index 2fc7b204cacb..c7e5dbc8cd79 100644 --- a/tools/ci/cirrus_wheels.yml +++ b/tools/ci/cirrus_wheels.yml @@ -1,52 +1,13 @@ build_and_store_wheels: &BUILD_AND_STORE_WHEELS install_cibuildwheel_script: - - python -m pip install cibuildwheel + - python3 -m pip install cibuildwheel cibuildwheel_script: - cibuildwheel + always: + show_meson_log_script: cat build/meson-logs/meson-log.txt wheels_artifacts: path: "wheelhouse/*" -###################################################################### -# Build linux_aarch64 natively -###################################################################### - -linux_aarch64_task: - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' - compute_engine_instance: - image_project: cirrus-images - image: family/docker-builder-arm64 - architecture: arm64 - platform: linux - cpu: 1 - memory: 8G - matrix: - # build in a matrix because building and testing all four wheels in a - # single task takes longer than 60 mins (the default time limit for a - # cirrus-ci task). - - env: - CIRRUS_CLONE_SUBMODULES: true - CIBW_BUILD: cp39-* - EXPECT_CPU_FEATURES: NEON NEON_FP16 NEON_VFPV4 ASIMD ASIMDHP ASIMDDP ASIMDFHM - - env: - CIRRUS_CLONE_SUBMODULES: true - CIBW_BUILD: cp310-* - - env: - CIRRUS_CLONE_SUBMODULES: true - CIBW_BUILD: cp311-* - - env: - CIRRUS_CLONE_SUBMODULES: true - CIBW_PRERELEASE_PYTHONS: True - CIBW_BUILD: cp312-* - - build_script: | - apt update - apt install -y python3-venv python-is-python3 gfortran libatlas-base-dev libgfortran5 eatmydata - git fetch origin - bash ./tools/wheels/cibw_before_build.sh ${PWD} - which python - echo $CIRRUS_CHANGE_MESSAGE - <<: *BUILD_AND_STORE_WHEELS - ###################################################################### # Build macosx_arm64 natively @@ -55,112 +16,32 @@ linux_aarch64_task: macosx_arm64_task: use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' macos_instance: - image: ghcr.io/cirruslabs/macos-monterey-xcode:14 + matrix: + - image: ghcr.io/cirruslabs/macos-monterey-xcode:14 + #- image: ghcr.io/cirruslabs/macos-sonoma-xcode:latest matrix: - env: CIRRUS_CLONE_SUBMODULES: true - CIBW_BUILD: cp39-* - - env: - CIRRUS_CLONE_SUBMODULES: true - CIBW_BUILD: cp310-* cp311-* - - env: - CIRRUS_CLONE_SUBMODULES: true - CIBW_PRERELEASE_PYTHONS: True - CIBW_BUILD: cp312-* + CIBW_BUILD: cp310-* + # Specifying CIBW_ENVIRONMENT_MACOS overrides pyproject.toml, so include + # all the settings from there, otherwise they're lost. env: PATH: /opt/homebrew/opt/python@3.10/bin:/usr/local/lib:/usr/local/include:$PATH CIBW_ARCHS: arm64 - # Specifying CIBW_ENVIRONMENT_MACOS overrides pyproject.toml, so include - # all the settings from there, otherwise they're lost. - # SDKROOT needs to be set for repackaged conda-forge gfortran compilers - # supplied by isuruf. - # Find out SDKROOT via `xcrun --sdk macosx --show-sdk-path` CIBW_ENVIRONMENT_MACOS: > RUNNER_OS=macOS - SDKROOT=/Applications/Xcode-14.0.0.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.3.sdk + MACOSX_DEPLOYMENT_TARGET=11.0 # 14.0 LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH build_script: - brew install python@3.10 - ln -s python3 /opt/homebrew/opt/python@3.10/bin/python - - which python + - which python3 # needed for submodules - git submodule update --init # need to obtain all the tags so setup.py can determine FULLVERSION - git fetch origin - uname -m - - python -c "import platform;print(platform.python_version());print(platform.system());print(platform.machine())" + - python3 -c "import platform; print(platform.python_version()); print(platform.system()); print(platform.machine())" - clang --version <<: *BUILD_AND_STORE_WHEELS - - -###################################################################### -# Upload all wheels -###################################################################### - -wheels_upload_task: - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' - # Artifacts don't seem to be persistent from task to task. - # Rather than upload wheels at the end of each cibuildwheel run we do a - # final upload here. This is because a run may be on different OS for - # which bash, etc, may not be present. - depends_on: - - linux_aarch64 - - macosx_arm64 - compute_engine_instance: - image_project: cirrus-images - image: family/docker-builder - platform: linux - cpu: 1 - - env: - NUMPY_STAGING_UPLOAD_TOKEN: ENCRYPTED[!5a69522ae0c2af9edb2bc1cdfeaca6292fb3666d9ecd82dca0615921834a6ce3b702352835d8bde4ea2a9ed5ef8424ac!] - NUMPY_NIGHTLY_UPLOAD_TOKEN: ENCRYPTED[ef04347663cfcb58d121385707e55951dc8e03b009edeed988aa4a33ba8205c54ca9980ac4da88e1adfdebff8b9d7ed4] - - upload_script: | - apt-get update - apt-get install -y curl wget - export IS_SCHEDULE_DISPATCH="false" - export IS_PUSH="false" - - # cron job - if [[ "$CIRRUS_CRON" == "nightly" ]]; then - export IS_SCHEDULE_DISPATCH="true" - fi - - # a manual build was started - if [[ "$CIRRUS_BUILD_SOURCE" == "api" && "$CIRRUS_COMMIT_MESSAGE" == "API build for null" ]]; then - export IS_SCHEDULE_DISPATCH="true" - fi - - # only upload wheels to staging if it's a tag beginning with 'v' and you're - # on a maintenance branch - if [[ "$CIRRUS_TAG" == v* ]] && [[ $CIRRUS_TAG != *"dev0"* ]]; then - export IS_PUSH="true" - fi - - if [[ $IS_PUSH == "true" ]] || [[ $IS_SCHEDULE_DISPATCH == "true" ]]; then - # install miniconda in the home directory. For some reason HOME isn't set by Cirrus - export HOME=$PWD - - # install miniconda for uploading to anaconda - wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda3 - $HOME/miniconda3/bin/conda init bash - source $HOME/miniconda3/bin/activate - conda install -y anaconda-client - - # The name of the zip file is derived from the `wheels_artifact` line. - # If you change the artifact line to `myfile_artifact` then it would be - # called myfile.zip - - curl https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/wheels.zip --output wheels.zip - unzip wheels.zip - - source ./tools/wheels/upload_wheels.sh - # IS_PUSH takes precedence over IS_SCHEDULE_DISPATCH - set_upload_vars - - # Will be skipped if not a push/tag/scheduled build - upload_wheels - fi diff --git a/tools/wheels/cibw_before_build.sh b/tools/wheels/cibw_before_build.sh index 7ce1edc68654..85f6554ad9a5 100644 --- a/tools/wheels/cibw_before_build.sh +++ b/tools/wheels/cibw_before_build.sh @@ -16,8 +16,19 @@ elif [[ $RUNNER_OS == "Windows" ]]; then cat $PROJECT_DIR/tools/wheels/LICENSE_win32.txt >> $PROJECT_DIR/LICENSE.txt fi +install_openblas=true +if [[ $RUNNER_OS == "macOS" ]]; then + if [[ $MACOSX_DEPLOYMENT_VERSION == "14.0" ]]; then + # This is the wheel build with Accelerate + install_openblas=false + else + # Done in gfortran_utils.sh + echo "deployment target determined from Python interpreter" + fi +fi + # Install Openblas -if [[ $RUNNER_OS == "Linux" || $RUNNER_OS == "macOS" ]] ; then +if [[ $RUNNER_OS == "Linux" || ($RUNNER_OS == "macOS" && $install_openblas) ]] ; then basedir=$(python tools/openblas_support.py --use-ilp64) if [[ $RUNNER_OS == "macOS" && $PLATFORM == "macosx-arm64" ]]; then # /usr/local/lib doesn't exist on cirrus-ci runners @@ -52,6 +63,7 @@ if [[ $RUNNER_OS == "macOS" ]]; then if [[ $PLATFORM == "macosx-arm64" ]]; then PLAT="arm64" fi + source $PROJECT_DIR/tools/wheels/gfortran_utils.sh install_gfortran pip install "delocate==0.10.4" diff --git a/tools/wheels/gfortran_utils.sh b/tools/wheels/gfortran_utils.sh index 9102ba1277ac..66f9b5736417 100644 --- a/tools/wheels/gfortran_utils.sh +++ b/tools/wheels/gfortran_utils.sh @@ -109,12 +109,15 @@ function get_gf_lib_for_suf { } if [ "$(uname)" == "Darwin" ]; then + # Set deployment target to match the one Python was built for, if the env + # var wasn't explicitly set already. mac_target=${MACOSX_DEPLOYMENT_TARGET:-$(get_macosx_target)} export MACOSX_DEPLOYMENT_TARGET=$mac_target # Keep this for now as some builds might depend on this being # available before install_gfortran is called export GFORTRAN_SHA=c469a420d2d003112749dcdcbe3c684eef42127e - # Set SDKROOT env variable if not set + # SDKROOT needs to be set for repackaged conda-forge gfortran compilers + # supplied by isuruf. So set it if it's not already set. export SDKROOT=${SDKROOT:-$(xcrun --show-sdk-path)} function download_and_unpack_gfortran { @@ -144,6 +147,7 @@ if [ "$(uname)" == "Darwin" ]; then pushd /opt sudo tar -xvf gfortran-darwin-${arch}-${type}.tar.gz sudo rm gfortran-darwin-${arch}-${type}.tar.gz + find gfortran-darwin-${arch}-${type} -name "libgfortran.spec" -exec sed -i.bak "s@\-lm@-lm -isysroot ${SDKROOT}@g" {} + popd if [[ "${type}" == "native" ]]; then # Link these into /usr/local so that there's no need to add rpath or -L