From e19398cb26f8616c941934f88691c019c672ed2e Mon Sep 17 00:00:00 2001 From: mattip Date: Mon, 17 Feb 2020 12:02:26 +0200 Subject: [PATCH 1/5] update submodules to latest versions and fixes for manylinux2010 --- .travis.yml | 7 +++++-- config.sh | 26 ++++++++++---------------- env_vars_32.sh | 4 ++++ gfortran-install | 2 +- multibuild | 2 +- numpy | 2 +- 6 files changed, 22 insertions(+), 21 deletions(-) create mode 100644 env_vars_32.sh diff --git a/.travis.yml b/.travis.yml index 9ef3739..00c1160 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,11 +19,11 @@ env: language: python # Default Python version is usually 2.7 python: 3.6 -sudo: required dist: bionic services: docker +os: linux -matrix: +jobs: exclude: # Exclude the default Python 3.6 build - python: 3.6 @@ -37,6 +37,7 @@ matrix: - MB_PYTHON_VERSION=3.6 - MB_ML_VER=2010 - PLAT=i686 + - ENV_VARS_PATH=env_vars_32.sh - os: linux env: - MB_PYTHON_VERSION=3.7 @@ -46,6 +47,7 @@ matrix: - MB_PYTHON_VERSION=3.7 - MB_ML_VER=2010 - PLAT=i686 + - ENV_VARS_PATH=env_vars_32.sh - os: linux env: - MB_PYTHON_VERSION=3.8 @@ -55,6 +57,7 @@ matrix: - MB_PYTHON_VERSION=3.8 - MB_ML_VER=2010 - PLAT=i686 + - ENV_VARS_PATH=env_vars_32.sh - os: osx language: generic osx_image: xcode10.1 diff --git a/config.sh b/config.sh index 0329262..70cc33f 100644 --- a/config.sh +++ b/config.sh @@ -7,6 +7,9 @@ function build_wheel { local lib_plat=$PLAT if [ -n "$IS_OSX" ]; then install_gfortran + else + # For manylinux2010 builds with manylinux1 openblas builds + $use_sudo yum install -y libgfortran-4.4.7 fi build_libs $lib_plat # Fix version error for development wheels by using bdist_wheel @@ -14,37 +17,28 @@ function build_wheel { } function build_libs { - local plat=${1:-$PLAT} - # Force 64-bit OpenBLAS library for macOS intel (dual arch) - # builds. For these builds, we pretend to be dual arch, but in - # fact we're only using the 64-bit build of OpenBLAS - if [ -n $IS_OSX ] && [ $plat == intel ]; then - plat=x86_64 - fi - local tar_fname=$(get_gf_lib "openblas-${OPENBLAS_VERSION}" "$plat") - local tar_path=$(abspath $tar_fname) - # Sudo needed for macOS - local use_sudo="" - [ -n "$IS_OSX" ] && use_sudo="sudo" - (cd / && $use_sudo tar zxf $tar_path) + # Use the same incantation as numpy/tools/travis-before-install.sh + target=$(python numpy/tools/openblas_support.py) + $use_sudo cp -r $target/lib/* /usr/local/lib + $use_sudo cp $target/include/* /usr/local/include } function get_test_cmd { local extra_argv=${1:-$EXTRA_ARGV} echo "import sys; import numpy; \ sys.exit(not numpy.test('full', \ - extra_argv=[${extra_argv}]))" + extra_argv=['-vv', ${extra_argv}]))" } function run_tests { # Runs tests on installed distribution from an empty directory - # We only run the 64 bit tests as of NumPy 1.16. if [ -z "$IS_OSX" ]; then apt-get -y update && apt-get install -y gfortran fi python -c "$(get_test_cmd)" # Check bundled license file python ../check_license.py - # Show BLAS / LAPACK used + # Show BLAS / LAPACK used. Since this uses a wheel we cannot use + # tools/openblas_config.py; tools is not part of what is shipped python -c 'import numpy; numpy.show_config()' } diff --git a/env_vars_32.sh b/env_vars_32.sh new file mode 100644 index 0000000..ac3a1b8 --- /dev/null +++ b/env_vars_32.sh @@ -0,0 +1,4 @@ +# Environment variables for build +OPENBLAS_VERSION="v0.3.7" +MACOSX_DEPLOYMENT_TARGET=10.9 +CFLAGS="-msse2 -std=c99 -fno-strict-aliasing" diff --git a/gfortran-install b/gfortran-install index e9764d0..d430fe6 160000 --- a/gfortran-install +++ b/gfortran-install @@ -1 +1 @@ -Subproject commit e9764d0a8f4e3adcd385c319e7742177193484c7 +Subproject commit d430fe6e38b6c5149c53f775a4437964e2f7b883 diff --git a/multibuild b/multibuild index 0c4a992..b943f33 160000 --- a/multibuild +++ b/multibuild @@ -1 +1 @@ -Subproject commit 0c4a9920fa4635419fcea61f10d6dc0cd50e3639 +Subproject commit b943f33a92772fd52c2fa38c03d08aac974c53bd diff --git a/numpy b/numpy index 9e419a0..f1a247f 160000 --- a/numpy +++ b/numpy @@ -1 +1 @@ -Subproject commit 9e419a0359cb5dbb9ae0b2711ab7a59b121a98cf +Subproject commit f1a247f9952085db72c3f8248f8336a0961d825c From 130bc10a8ae240f488ae81172734c14226daaeda Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 26 Feb 2020 11:58:19 +0200 Subject: [PATCH 2/5] add urllib3 needed for openblas_support --- .appveyor.yml | 2 +- config.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index e0b598c..a216858 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -113,7 +113,7 @@ install: ls $destination # Upgrade to the latest pip, setuptools, and wheel. - - python -m pip install -U pip setuptools wheel + - python -m pip install -U pip setuptools wheel urllib3 # Install build requirements. - pip install "%CYTHON_BUILD_DEP%" --install-option="--no-cython-compile" diff --git a/config.sh b/config.sh index 70cc33f..f1f3ecc 100644 --- a/config.sh +++ b/config.sh @@ -18,6 +18,7 @@ function build_wheel { function build_libs { # Use the same incantation as numpy/tools/travis-before-install.sh + python -mpip install urllib3 target=$(python numpy/tools/openblas_support.py) $use_sudo cp -r $target/lib/* /usr/local/lib $use_sudo cp $target/include/* /usr/local/include From c9ab82ebb0a230a343c73a2004f8f02431c7ec81 Mon Sep 17 00:00:00 2001 From: mattip Date: Fri, 6 Mar 2020 15:43:44 +0200 Subject: [PATCH 3/5] Update multibuild and use xenial test images on 32-bit --- .travis.yml | 9 ++++----- multibuild | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 00c1160..d4898c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,16 +17,12 @@ env: - EXTRA_ARGV="'--disable-pytest-warnings'" language: python -# Default Python version is usually 2.7 -python: 3.6 +python: 3.7 dist: bionic services: docker os: linux jobs: - exclude: - # Exclude the default Python 3.6 build - - python: 3.6 include: - os: linux env: @@ -38,6 +34,7 @@ jobs: - MB_ML_VER=2010 - PLAT=i686 - ENV_VARS_PATH=env_vars_32.sh + - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} - os: linux env: - MB_PYTHON_VERSION=3.7 @@ -48,6 +45,7 @@ jobs: - MB_ML_VER=2010 - PLAT=i686 - ENV_VARS_PATH=env_vars_32.sh + - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} - os: linux env: - MB_PYTHON_VERSION=3.8 @@ -58,6 +56,7 @@ jobs: - MB_ML_VER=2010 - PLAT=i686 - ENV_VARS_PATH=env_vars_32.sh + - DOCKER_TEST_IMAGE=multibuild/xenial_{PLAT} - os: osx language: generic osx_image: xcode10.1 diff --git a/multibuild b/multibuild index b943f33..3bd75ee 160000 --- a/multibuild +++ b/multibuild @@ -1 +1 @@ -Subproject commit b943f33a92772fd52c2fa38c03d08aac974c53bd +Subproject commit 3bd75eee8fa98ee70e29d59f4ec5a15291362cc2 From 52f4b91ed6b9b0dd54165dd97913e0a5e1ce5c13 Mon Sep 17 00:00:00 2001 From: mattip Date: Sat, 7 Mar 2020 19:26:30 +0200 Subject: [PATCH 4/5] fixes from review, cleanups, improve README --- .travis.yml | 6 +++--- README.rst | 12 +++++++----- config.sh | 11 +++++++---- env_vars_32.sh | 4 +++- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index d4898c2..fcf26e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,6 @@ env: - BUILD_COMMIT=e94cec800304a6a467cf90ce4e7d3e207770b4b4 - BUILD_DEPENDS=cython - TEST_DEPENDS="pytest hypothesis" - - PLAT=x86_64 - - UNICODE_WIDTH=32 - WHEELHOUSE_UPLOADER_USERNAME=travis-worker # Following generated with # travis encrypt -r MacPython/numpy-wheels WHEELHOUSE_UPLOADER_SECRET= @@ -17,7 +15,6 @@ env: - EXTRA_ARGV="'--disable-pytest-warnings'" language: python -python: 3.7 dist: bionic services: docker os: linux @@ -28,6 +25,7 @@ jobs: env: - MB_PYTHON_VERSION=3.6 - MB_ML_VER=2010 + - PLAT=x86_64 - os: linux env: - MB_PYTHON_VERSION=3.6 @@ -39,6 +37,7 @@ jobs: env: - MB_PYTHON_VERSION=3.7 - MB_ML_VER=2010 + - PLAT=x86_64 - os: linux env: - MB_PYTHON_VERSION=3.7 @@ -50,6 +49,7 @@ jobs: env: - MB_PYTHON_VERSION=3.8 - MB_ML_VER=2010 + - PLAT=x86_64 - os: linux env: - MB_PYTHON_VERSION=3.8 diff --git a/README.rst b/README.rst index 6c71b8e..d788105 100644 --- a/README.rst +++ b/README.rst @@ -19,10 +19,12 @@ How it works The wheel-building repository: -* does a fresh build of any required C / C++ libraries; -* builds a numpy wheel, linking against these fresh builds; +* checks out either a known version of NumPy or master's HEAD +* downloads OpenBLAS using numpy/tools/openblas_support.py +* builds a numpy wheel, linking against the downloaded library and using + the appropriate additional variables from `env_vars.sh` or `env_vars32.sh` * processes the wheel using delocate_ (OSX) or auditwheel_ ``repair`` - (Manylinux1_). ``delocate`` and ``auditwheel`` copy the required dynamic + (manylinux_). ``delocate`` and ``auditwheel`` copy the required dynamic libraries into the wheel and relinks the extension modules against the copied libraries; * uploads the built wheels to http://wheels.scipy.org (a Rackspace container @@ -30,7 +32,7 @@ The wheel-building repository: The resulting wheels are therefore self-contained and do not need any external dynamic libraries apart from those provided as standard by OSX / Linux as -defined by the manylinux1 standard. +defined by the manylinux standard. The ``.travis.yml`` file in this repository has a line containing the API key for the Rackspace container encrypted with an RSA key that is unique to the @@ -132,7 +134,7 @@ upload them to PyPI. Of course, you will need permissions to upload to PyPI, for this to work. -.. _manylinux1: https://www.python.org/dev/peps/pep-0513 +.. _manylinux: https://www.python.org/dev/peps/pep-0513 .. _twine: https://pypi.python.org/pypi/twine .. _bs4: https://pypi.python.org/pypi/beautifulsoup4 .. _delocate: https://pypi.python.org/pypi/delocate diff --git a/config.sh b/config.sh index f1f3ecc..e515500 100644 --- a/config.sh +++ b/config.sh @@ -17,11 +17,14 @@ function build_wheel { } function build_libs { - # Use the same incantation as numpy/tools/travis-before-install.sh + # Use the same incantation as numpy/tools/travis-before-install.sh to + # download and un-tar the openblas libraries. The python call returns + # the un-tar root directory, then the files are copied into /usr/local. + # Could utilize a site.cfg instead to prevent the copy. python -mpip install urllib3 - target=$(python numpy/tools/openblas_support.py) - $use_sudo cp -r $target/lib/* /usr/local/lib - $use_sudo cp $target/include/* /usr/local/include + basedir=$(python numpy/tools/openblas_support.py) + $use_sudo cp -r $basedir/lib/* /usr/local/lib + $use_sudo cp $basedir/include/* /usr/local/include } function get_test_cmd { diff --git a/env_vars_32.sh b/env_vars_32.sh index ac3a1b8..46b1aaa 100644 --- a/env_vars_32.sh +++ b/env_vars_32.sh @@ -1,4 +1,6 @@ -# Environment variables for build +# Environment variables for 32-bit build. +# The important difference from the 64-bit build is `-msse2` to +# compile sse loops for ufuncs. OPENBLAS_VERSION="v0.3.7" MACOSX_DEPLOYMENT_TARGET=10.9 CFLAGS="-msse2 -std=c99 -fno-strict-aliasing" From 1e6dae670cc03320f8c1ce06c141b7df0f9fc373 Mon Sep 17 00:00:00 2001 From: mattip Date: Sat, 7 Mar 2020 20:11:28 +0200 Subject: [PATCH 5/5] update .appveyor to get openblas from numpy/tools/openblas_support --- .appveyor.yml | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index a216858..b5bb25f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -12,8 +12,6 @@ environment: global: MINGW_32: C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin MINGW_64: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin - OPENBLAS_32: "https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.7-win32-gcc_7_1_0.zip" - OPENBLAS_64: "https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.7-win_amd64-gcc_7_1_0.zip" CYTHON_BUILD_DEP: Cython TEST_MODE: fast @@ -88,33 +86,18 @@ install: python -c "import sys,platform,struct; print(sys.platform, platform.machine(), struct.calcsize('P') * 8, )" + # Upgrade to the latest pip, setuptools, and wheel. + - python -m pip install -U pip setuptools wheel urllib3 + # Download and install static "openblas.a" to PYTHON\lib - ps: | - $PYTHON_ARCH = $env:PYTHON_ARCH $PYTHON = $env:PYTHON - If ($PYTHON_ARCH -eq 32) { - $OPENBLAS = $env:OPENBLAS_32 - } Else { - $OPENBLAS = $env:OPENBLAS_64 - } - $clnt = new-object System.Net.WebClient - $file = "$(New-TemporaryFile).zip" - $tmpdir = New-TemporaryFile | %{ rm $_; mkdir $_ } + $lib = python numpy/tools/openblas_support.py $destination = "$PYTHON\lib\openblas.a" - echo $file - echo $tmpdir - echo $OPENBLAS - $clnt.DownloadFile($OPENBLAS,$file) - Expand-Archive $file $tmpdir - rm $tmpdir\$PYTHON_ARCH\lib\*.dll.a - $lib = ls $tmpdir\$PYTHON_ARCH\lib\*.a | ForEach { ls $_ } | Select-Object -first 1 echo $lib cp $lib $destination ls $destination - # Upgrade to the latest pip, setuptools, and wheel. - - python -m pip install -U pip setuptools wheel urllib3 - # Install build requirements. - pip install "%CYTHON_BUILD_DEP%" --install-option="--no-cython-compile"