scikit-learn · ogrisel · Dec 22, 2020 · Dec 4, 2020 · Dec 4, 2020 · Dec 4, 2020
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -22,6 +22,7 @@ jobs:
   check_build_trigger:
     name: Check build trigger
     runs-on: ubuntu-latest
+    if: github.repository == 'scikit-learn/scikit-learn'
     outputs:
       build: ${{ steps.check_build_trigger.outputs.build }}
 
@@ -86,7 +87,7 @@ jobs:
                             SKLEARN_SKIP_NETWORK_TESTS=1
                             SKLEARN_BUILD_PARALLEL=3
           CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
-          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }} 
+          CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }}
           CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }}
           CIBW_TEST_REQUIRES: pytest pandas threadpoolctl
           CIBW_TEST_COMMAND: bash {project}/build_tools/github/test_wheels.sh

diff --git a/.travis.yml b/.travis.yml
@@ -1,58 +1,90 @@
-# make it explicit that we favor the new container-based travis workers
+# Make it explicit that we favor the
+# new container-based Travis workers
 language: python
+dist: xenial
 
 cache:
   apt: true
   directories:
-  - $HOME/.cache/pip
-  - $HOME/.ccache
-
-dist: xenial
+    - $HOME/.cache/pip
+    - $HOME/.ccache
 
 env:
   global:
-    # Directory where tests are run from
-    - TEST_DIR=/tmp/sklearn
+    - CPU_COUNT=3
+    - TEST_DIR=/tmp/sklearn  # Test directory for continuous integration jobs
+    - PYTEST_VERSION=latest
     - OMP_NUM_THREADS=2
     - OPENBLAS_NUM_THREADS=2
-    - PYTEST_VERSION=latest
+    - SKLEARN_BUILD_PARALLEL=3
+    - SKLEARN_SKIP_NETWORK_TESTS=1
+    # Custom environment variables for the ARM wheel builder
+    - CIBW_BUILD_VERBOSITY=1
+    - CIBW_TEST_REQUIRES="pytest pytest-xdist threadpoolctl"
+    - CIBW_TEST_COMMAND="bash {project}/build_tools/travis/test_wheels.sh"
+    - CIBW_ENVIRONMENT="CPU_COUNT=8
+                        OMP_NUM_THREADS=2
+                        OPENBLAS_NUM_THREADS=2
+                        SKLEARN_BUILD_PARALLEL=8
+                        SKLEARN_SKIP_NETWORK_TESTS=1"
 
-matrix:
+jobs:
   include:
-    # Linux environment to test scikit-learn against numpy and scipy master
-    # installed from their CI wheels in a virtualenv with the Python
-    # interpreter provided by travis.
     - python: 3.7
       env:
-        - CHECK_WARNINGS="true"
-        - CI_CPU_COUNT="3"
-      if: type = cron OR commit_message =~ /\[scipy-dev\]/
+        - CHECK_WARNING=true
+        - BUILD_WITH_ICC=true
+      if: type = cron OR commit_message =~ /\[icc-build\]/
 
-    # As above but build scikit-learn with Intel C compiler (ICC).
     - python: 3.7
+      os: linux
+      arch: arm64
+      if: type = cron OR commit_message =~ /\[arm64\]/
       env:
-        - CHECK_WARNING="true"
-        - BUILD_WITH_ICC="true"
-        - CI_CPU_COUNT="3"
-      if: type = cron OR commit_message =~ /\[icc-build\]/
+        - CPU_COUNT=8
+
+    # Linux environments to build the scikit-learn wheels
+    # for the ARM64 arquitecture and Python 3.6 and newer
+    - python: 3.6
+      os: linux
+      arch: arm64
+      if: type = cron or commit_message =~ /\[cd build\]/
+      env:
+        - BUILD_WHEEL=true
+        - CIBW_BUILD=cp36-manylinux_aarch64
 
     - python: 3.7
+      os: linux
+      arch: arm64
+      if: type = cron or commit_message =~ /\[cd build\]/
       env:
-        - CI_CPU_COUNT="8"
+        - BUILD_WHEEL=true
+        - CIBW_BUILD=cp37-manylinux_aarch64
+
+    - python: 3.8
       os: linux
       arch: arm64
-      if: type = cron OR commit_message =~ /\[arm64\]/
+      if: type = cron or commit_message =~ /\[cd build\]/
+      env:
+        - BUILD_WHEEL=true
+        - CIBW_BUILD=cp38-manylinux_aarch64
+
+    - python: 3.9
+      os: linux
+      arch: arm64
+      if: type = cron or commit_message =~ /\[cd build\]/
+      env:
+        - BUILD_WHEEL=true
+        - CIBW_BUILD=cp39-manylinux_aarch64
+
+install: source build_tools/travis/install.sh || travis_terminate 1
+script: source build_tools/travis/script.sh || travis_terminate 1
+after_success: source build_tools/travis/after_success.sh || travis_terminate 1
 
-install: source build_tools/travis/install.sh
-script:
-  - bash build_tools/travis/test_script.sh || travis_terminate 1
-  - bash build_tools/travis/test_docs.sh || travis_terminate 1
-  - bash build_tools/travis/test_pytest_soft_dependency.sh || travis_terminate 1
-after_success: source build_tools/travis/after_success.sh
 notifications:
   webhooks:
     urls:
       - https://webhooks.gitter.im/e/4ffabb4df010b70cd624
-    on_success: change  # options: [always|never|change] default: always
-    on_failure: always  # options: [always|never|change] default: always
-    on_start: never     # options: [always|never|change] default: always
+    on_success: change
+    on_failure: always
+    on_start: never
diff --git a/build_tools/circle/build_test_pypy.sh b/build_tools/circle/build_test_pypy.sh
@@ -35,5 +35,3 @@ export SKLEARN_BUILD_PARALLEL=3
 pip install --no-build-isolation -e .
 
 python -m pytest sklearn
-python -m pytest doc/sphinxext/
-python -m pytest $(find doc -name '*.rst' | sort)
diff --git a/build_tools/travis/after_success.sh b/build_tools/travis/after_success.sh
@@ -1,19 +1,26 @@
 #!/bin/bash
-# This script is meant to be called by the "after_success" step defined in
-# .travis.yml. See https://docs.travis-ci.com/ for more details.
 
-# License: 3-clause BSD
+# This script is meant to be called by the "after_success" step
+# defined in ".travis.yml". In particular, we upload the wheels
+# of the ARM64 architecture for the continuous deployment jobs.
 
 set -e
 
-if [[ "$COVERAGE" == "true" ]]; then
-    # Need to run codecov from a git checkout, so we copy .coverage
-    # from TEST_DIR where pytest has been run
-    cp $TEST_DIR/.coverage $TRAVIS_BUILD_DIR
+# The wheels cannot be uploaded on PRs
+if [[ $BUILD_WHEEL == true && $TRAVIS_EVENT_TYPE != pull_request ]]; then
+    # Nightly upload token and staging upload token are set in
+    # Travis settings (originally generated at Anaconda cloud)
+    if [[ $TRAVIS_EVENT_TYPE == cron ]]; then
+        ANACONDA_ORG="scipy-wheels-nightly"
+        ANACONDA_TOKEN="$SCIKIT_LEARN_NIGHTLY_UPLOAD_TOKEN"
+    else
+        ANACONDA_ORG="scikit-learn-wheels-staging"
+        ANACONDA_TOKEN="$SCIKIT_LEARN_STAGING_UPLOAD_TOKEN"
+    fi
 
-    # Ignore codecov failures as the codecov server is not
-    # very reliable but we don't want travis to report a failure
-    # in the github UI just because the coverage report failed to
-    # be published.
-    codecov --root $TRAVIS_BUILD_DIR || echo "codecov upload failed"
+    pip install git+https://github.com/Anaconda-Server/anaconda-client
+
+    # Force a replacement if the remote file already exists
+    anaconda -t $ANACONDA_TOKEN upload --force -u $ANACONDA_ORG wheelhouse/*.whl
+    echo "Index: https://pypi.anaconda.org/$ANACONDA_ORG/simple"
 fi
diff --git a/build_tools/travis/install.sh b/build_tools/travis/install.sh
@@ -1,111 +1,13 @@
 #!/bin/bash
-# This script is meant to be called by the "install" step defined in
-# .travis.yml. See https://docs.travis-ci.com/ for more details.
-# The behavior of the script is controlled by environment variabled defined
-# in the .travis.yml in the top level folder of the project.
 
-# License: 3-clause BSD
-
-# Travis clone scikit-learn/scikit-learn repository in to a local repository.
-# We use a cached directory with three scikit-learn repositories (one for each
-# matrix entry) from which we pull from local Travis repository. This allows
-# us to keep build artefact for gcc + cython, and gain time
+# This script is meant to be called by the "install" step
+# defined in the ".travis.yml" file. In particular, it is
+# important that we call to the right installation script.
 
 set -e
 
-# Fail fast
-echo "CPU Arch: ${TRAVIS_CPU_ARCH}"
-
-# jq is used in travis_fastfail.sh, it's already pre-installed in non arm64
-# environments
-sudo apt-get install jq
-
-build_tools/travis/travis_fastfail.sh
-
-# Imports get_dep
-source build_tools/shared.sh
-
-echo "List files from cached directories"
-echo "pip:"
-ls $HOME/.cache/pip
-
-export CC=/usr/lib/ccache/gcc
-export CXX=/usr/lib/ccache/g++
-# Useful for debugging how ccache is used
-# export CCACHE_LOGFILE=/tmp/ccache.log
-# ~60M is used by .ccache when compiling from scratch at the time of writing
-ccache --max-size 100M --show-stats
-
-# Deactivate the travis-provided virtual environment and setup a
-# conda-based environment instead
-# If Travvis has language=generic, deactivate does not exist. `|| :` will pass.
-deactivate || :
-
-
-# Install miniconda
-if [[ "$TRAVIS_CPU_ARCH" == "arm64" ]]; then
-    wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh -O miniconda.sh
+if [[ $BUILD_WHEEL == true ]]; then
+    source build_tools/travis/install_wheels.sh
 else
-    fname=Miniconda3-latest-Linux-x86_64.sh
-    wget https://repo.continuum.io/miniconda/$fname -O miniconda.sh
+    source build_tools/travis/install_master.sh
 fi
-MINICONDA_PATH=$HOME/miniconda
-chmod +x miniconda.sh && ./miniconda.sh -b -p $MINICONDA_PATH
-export PATH=$MINICONDA_PATH/bin:$PATH
-conda update --yes conda
-
-# Create environment and install dependencies
-conda create -n testenv --yes python=3.7
-
-source activate testenv
-
-if [[ "$TRAVIS_CPU_ARCH" == "amd64" ]]; then
-    pip install --upgrade pip setuptools
-    echo "Installing numpy and scipy master wheels"
-    dev_anaconda_url=https://pypi.anaconda.org/scipy-wheels-nightly/simple
-    pip install --pre --upgrade --timeout=60 --extra-index $dev_anaconda_url numpy scipy pandas
-    pip install --pre cython
-    echo "Installing joblib master"
-    pip install https://github.com/joblib/joblib/archive/master.zip
-    echo "Installing pillow master"
-    pip install https://github.com/python-pillow/Pillow/archive/master.zip
-else
-    conda install -y scipy numpy pandas cython
-    pip install joblib threadpoolctl
-fi
-
-pip install $(get_dep pytest $PYTEST_VERSION) pytest-cov pytest-xdist
-
-# Build scikit-learn in the install.sh script to collapse the verbose
-# build output in the travis output when it succeeds.
-python --version
-python -c "import numpy; print('numpy %s' % numpy.__version__)"
-python -c "import scipy; print('scipy %s' % scipy.__version__)"
-
-if [[ "$BUILD_WITH_ICC" == "true" ]]; then
-    wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-    sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-    rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-    sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
-    sudo apt-get update
-    sudo apt-get install intel-oneapi-icc
-    source /opt/intel/oneapi/setvars.sh
-
-    # The build_clib command is implicitly used to build libsvm-skl. To compile
-    # with a different compiler we also need to specify the compiler for this
-    # command.
-    python setup.py build_ext --compiler=intelem -i -j "${CI_CPU_COUNT}" build_clib --compiler=intelem
-else
-    # Use setup.py instead of `pip install -e .` to be able to pass the -j flag
-    # to speed-up the building multicore CI machines.
-    python setup.py build_ext --inplace -j "${CI_CPU_COUNT}"
-fi
-
-python setup.py develop
-
-ccache --show-stats
-# Useful for debugging how ccache is used
-# cat $CCACHE_LOGFILE
-
-# fast fail
-build_tools/travis/travis_fastfail.sh