scikit-learn · jeremiedbb · Jan 19, 2021 · Dec 22, 2020 · Dec 22, 2020 · Dec 22, 2020
diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml
@@ -40,10 +40,10 @@ jobs:
         user: __token__
         password: ${{ secrets.TEST_PYPI_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
-      if: ${{ github.event.inputs.pypi_repo }} == 'testpypi'
+      if: ${{ github.event.inputs.pypi_repo == 'testpypi' }}
     - name: Publish package to PyPI
       uses: pypa/gh-action-pypi-publish@v1.4.1
       with:
         user: __token__
         password: ${{ secrets.PYPI_TOKEN }}
-      if: ${{ github.event.inputs.pypi_repo }} == 'pypi'
+      if: ${{ github.event.inputs.pypi_repo == 'pypi' }}
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -86,6 +86,7 @@ jobs:
                             OPENBLAS_NUM_THREADS=2
                             SKLEARN_SKIP_NETWORK_TESTS=1
                             SKLEARN_BUILD_PARALLEL=3
+                            MACOSX_DEPLOYMENT_TARGET=10.13
           CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
           CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: bash build_tools/github/repair_windows_wheels.sh {wheel} {dest_dir} ${{ matrix.bitness }}
           CIBW_BEFORE_TEST_WINDOWS: bash build_tools/github/build_minimal_windows_image.sh ${{ matrix.python }} ${{ matrix.bitness }}

diff --git a/.travis.yml b/.travis.yml
@@ -36,15 +36,21 @@ jobs:
         - BUILD_WITH_ICC=true
       if: type = cron OR commit_message =~ /\[icc-build\]/
 
-    - python: 3.7
+    # Manual trigger of linux/arm64 tests in PR without triggering the full
+    # wheel building process for all the Python versions. 
+    - python: 3.9
       os: linux
       arch: arm64
-      if: type = cron OR commit_message =~ /\[arm64\]/
+      if: commit_message =~ /\[arm64\]/
       env:
         - CPU_COUNT=8
 
-    # Linux environments to build the scikit-learn wheels
-    # for the ARM64 arquitecture and Python 3.6 and newer
+        # Linux environments to build the scikit-learn wheels for the ARM64
+    # architecture and Python 3.6 and newer. This is used both at release time
+    # with the manual trigger in the commit message in the release branch and as
+    # a scheduled task to build the weekly dev build on the master branch. The
+    # weekly frequency is meant to avoid depleting the Travis CI credits too
+    # fast.
     - python: 3.6
       os: linux
       arch: arm64

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -8,66 +8,65 @@ schedules:
   always: true
 
 jobs:
-- job: linting
-  displayName: Linting
+- job: git_commit
+  displayName: Get Git Commit
   pool:
     vmImage: ubuntu-18.04
   steps:
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.9'
-    - bash: |
-        pip install flake8 mypy==0.782
-      displayName: Install linters
     - bash: |
         set -ex
         if [[ $BUILD_REASON == "PullRequest" ]]; then
           # By default pull requests use refs/pull/PULL_ID/merge as the source branch
           # which has a "Merge ID into ID" as a commit message. The latest commit
           # message is the second to last commit
           COMMIT_ID=$(echo $BUILD_SOURCEVERSIONMESSAGE | awk '{print $2}')
-          COMMIT_MESSAGE=$(git log $COMMIT_ID -1 --pretty=%B)
+          message=$(git log $COMMIT_ID -1 --pretty=%B)
         else
-          COMMIT_MESSAGE=$BUILD_SOURCEVERSIONMESSAGE
+          message=$BUILD_SOURCEVERSIONMESSAGE
         fi
-        echo "##vso[task.setvariable variable=COMMIT_MESSAGE]$COMMIT_MESSAGE"
+        echo "##vso[task.setvariable variable=message;isOutput=true]$message"
+      name: commit
       displayName: Get source version message
+
+- job: linting
+  dependsOn: [git_commit]
+  condition: |
+    and(
+      succeeded(),
+      not(contains(dependencies['git_commit']['outputs']['commit.message'], '[lint skip]')),
+      not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]'))
+    )
+  displayName: Linting
+  pool:
+    vmImage: ubuntu-18.04
+  steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '3.9'
     - bash: |
-        set -ex
-        if [[ "$COMMIT_MESSAGE" =~ "[lint skip]" ]]; then
-          # skip linting
-          echo "Skipping flake8 linting"
-          exit 0
-        else
-          ./build_tools/circle/linting.sh
-        fi
+        pip install flake8 mypy==0.782
+      displayName: Install linters
+    - bash: |
+        ./build_tools/circle/linting.sh
       displayName: Run linting
     - bash: |
-        set -ex
-        if [[ "$COMMIT_MESSAGE" =~ "[lint skip]" ]]; then
-          # skip linting
-          echo "Skipping mypy linting"
-          exit 0
-        else
-          mypy sklearn/
-        fi
+        mypy sklearn/
       displayName: Run mypy
-    - bash: |
-        if [[ "$COMMIT_MESSAGE" =~ "[scipy-dev]" ]] || [[ $BUILD_REASON == "Schedule" ]]; then
-          echo "Running scipy-dev"
-          echo "##vso[task.setvariable variable=runScipyDev;isOutput=true]true"
-        else
-          echo "##vso[task.setvariable variable=runScipyDev;isOutput=true]false"
-        fi
-      name: gitCommitMessage
-      displayName: Determine to run scipy-dev
 
 - template: build_tools/azure/posix.yml
   parameters:
     name: Linux_Nightly
     vmImage: ubuntu-18.04
-    dependsOn: [linting]
-    condition: eq(dependencies['linting']['outputs']['gitCommitMessage.runScipyDev'], 'true')
+    dependsOn: [git_commit, linting]
+    condition: |
+      and(
+        succeeded(),
+        not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')),
+        or(eq(variables['Build.Reason'], 'Schedule'),
+           contains(dependencies['git_commit']['outputs']['commit.message'], '[scipy-dev]'
+          )
+        )
+      )
     matrix:
       pylatest_pip_scipy_dev:
         DISTRIB: 'conda-pip-scipy-dev'
@@ -84,6 +83,12 @@ jobs:
   parameters:
     name: Linux_Runs
     vmImage: ubuntu-18.04
+    dependsOn: [git_commit]
+    condition: |
+      and(
+        succeeded(),
+        not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]'))
+      )
     matrix:
       pylatest_conda_mkl:
         DISTRIB: 'conda'
@@ -95,8 +100,13 @@ jobs:
   parameters:
     name: Linux
     vmImage: ubuntu-18.04
-    dependsOn: [linting]
-    condition: and(ne(variables['Build.Reason'], 'Schedule'), succeeded('linting'))
+    dependsOn: [linting, git_commit]
+    condition: |
+      and(
+        succeeded(),
+        not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')),
+        ne(variables['Build.Reason'], 'Schedule')
+      )
     matrix:
       # Linux environment to test that scikit-learn can be built against
       # versions of numpy, scipy with ATLAS that comes with Ubuntu Bionic 18.04
@@ -139,8 +149,13 @@ jobs:
   parameters:
     name: Linux32
     vmImage: ubuntu-18.04
-    dependsOn: [linting]
-    condition: and(ne(variables['Build.Reason'], 'Schedule'), succeeded('linting'))
+    dependsOn: [linting, git_commit]
+    condition: |
+      and(
+        succeeded(),
+        not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')),
+        ne(variables['Build.Reason'], 'Schedule')
+      )
     matrix:
       py36_ubuntu_atlas_32bit:
         DISTRIB: 'ubuntu-32'
@@ -157,8 +172,13 @@ jobs:
   parameters:
     name: macOS
     vmImage: macOS-10.14
-    dependsOn: [linting]
-    condition: and(ne(variables['Build.Reason'], 'Schedule'), succeeded('linting'))
+    dependsOn: [linting, git_commit]
+    condition: |
+      and(
+        succeeded(),
+        not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')),
+        ne(variables['Build.Reason'], 'Schedule')
+      )
     matrix:
       pylatest_conda_forge_mkl:
         DISTRIB: 'conda'
@@ -174,8 +194,13 @@ jobs:
   parameters:
     name: Windows
     vmImage: vs2017-win2016
-    dependsOn: [linting]
-    condition: and(ne(variables['Build.Reason'], 'Schedule'), succeeded('linting'))
+    dependsOn: [linting, git_commit]
+    condition: |
+      and(
+        succeeded(),
+        not(contains(dependencies['git_commit']['outputs']['commit.message'], '[ci skip]')),
+        ne(variables['Build.Reason'], 'Schedule')
+      )
     matrix:
       py37_conda_mkl:
         PYTHON_VERSION: '3.7'

diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh
@@ -116,8 +116,8 @@ fi
 
 if [[ "$CIRCLE_BRANCH" =~ ^master$|^[0-9]+\.[0-9]+\.X$ && -z "$CI_PULL_REQUEST" ]]
 then
-    # PDF linked into HTML
-    make_args="dist LATEXMKOPTS=-halt-on-error"
+    # ZIP linked into HTML
+    make_args=dist
 elif [[ "$build_type" =~ ^QUICK ]]
 then
     make_args=html-noplot
@@ -133,13 +133,10 @@ fi
 make_args="SPHINXOPTS=-T $make_args"  # show full traceback on exception
 
 # Installing required system packages to support the rendering of math
-# notation in the HTML documentation
+# notation in the HTML documentation and to optimize the image files
 sudo -E apt-get -yq update
-sudo -E apt-get -yq remove texlive-binaries --purge
 sudo -E apt-get -yq --no-install-suggests --no-install-recommends \
-    install dvipng texlive-latex-base texlive-latex-extra \
-    texlive-latex-recommended texlive-fonts-recommended \
-    latexmk gsfonts ccache
+    install dvipng gsfonts ccache zip optipng
 
 # deactivate circleci virtualenv and setup a miniconda env instead
 if [[ `type -t deactivate` ]]; then

diff --git a/build_tools/circle/list_versions.py b/build_tools/circle/list_versions.py
@@ -8,6 +8,7 @@
 from distutils.version import LooseVersion
 from urllib.request import urlopen
 
+
 def json_urlread(url):
     try:
         return json.loads(urlopen(url).read().decode('utf8'))
@@ -32,10 +33,23 @@ def human_readable_data_quantity(quantity, multiple=1024):
             quantity /= multiple
 
 
-def get_pdf_size(version):
+def get_file_extension(version):
+    if 'dev' in version:
+        # The 'dev' branch should be explictly handled
+        return 'zip'
+
+    current_version = LooseVersion(version)
+    min_zip_version = LooseVersion('1.0.0')
+
+    return 'zip' if current_version >= min_zip_version else 'pdf'
+
+
+def get_file_size(version):
     api_url = ROOT_URL + '%s/_downloads' % version
     for path_details in json_urlread(api_url):
-        if path_details['name'] == 'scikit-learn-docs.pdf':
+        file_extension = get_file_extension(version)
+        file_path = f'scikit-learn-docs.{file_extension}'
+        if path_details['name'] == file_path:
             return human_readable_data_quantity(path_details['size'], 1000)
 
 
@@ -64,8 +78,8 @@ def get_pdf_size(version):
     if path_details['type'] == 'dir':
         html = urlopen(RAW_FMT % name).read().decode('utf8')
         version_num = VERSION_RE.search(html).group(1)
-        pdf_size = get_pdf_size(name)
-        dirs[name] = (version_num, pdf_size)
+        file_size = get_file_size(name)
+        dirs[name] = (version_num, file_size)
 
     if path_details['type'] == 'symlink':
         symlinks[name] = json_urlread(path_details['_links']['self'])['target']
@@ -81,7 +95,7 @@ def get_pdf_size(version):
 for name in (NAMED_DIRS +
              sorted((k for k in dirs if k[:1].isdigit()),
                     key=LooseVersion, reverse=True)):
-    version_num, pdf_size = dirs[name]
+    version_num, file_size = dirs[name]
     if version_num in seen:
         # symlink came first
         continue
@@ -91,7 +105,8 @@ def get_pdf_size(version):
     path = 'https://scikit-learn.org/%s/' % name
     out = ('* `Scikit-learn %s%s documentation <%s>`_'
            % (version_num, name_display, path))
-    if pdf_size is not None:
-        out += (' (`PDF %s <%s/_downloads/scikit-learn-docs.pdf>`_)'
-                % (pdf_size, path))
+    if file_size is not None:
+        file_extension = get_file_extension(version_num)
+        out += (f' (`{file_extension.upper()} {file_size} <{path}/'
+                f'_downloads/scikit-learn-docs.{file_extension}>`_)')
     print(out)
diff --git a/build_tools/github/build_wheels.sh b/build_tools/github/build_wheels.sh
@@ -5,7 +5,15 @@ set -x
 
 # OpenMP is not present on macOS by default
 if [[ "$RUNNER_OS" == "macOS" ]]; then
-    brew install libomp
+    # Make sure to use a libomp version binary compatible with the oldest
+    # supported version of the macos SDK as libomp will be vendored into the
+    # scikit-learn wheels for macos. The list of bottles can be found at:
+    # https://formulae.brew.sh/api/formula/libomp.json. Currently, the oldest
+    # supported macos version is: High Sierra / 10.13. When upgrading this, be
+    # sure to update the MACOSX_DEPLOYMENT_TARGET environment variable in
+    # wheels.yml accordingly.
+    wget https://homebrew.bintray.com/bottles/libomp-11.0.0.high_sierra.bottle.tar.gz
+    brew install libomp-11.0.0.high_sierra.bottle.tar.gz
     export CC=/usr/bin/clang
     export CXX=/usr/bin/clang++
     export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp"

diff --git a/build_tools/github/check_wheels.py b/build_tools/github/check_wheels.py
@@ -20,6 +20,16 @@
 # plus one more for the sdist
 n_wheels += 1
 
+# aarch64 builds from travis
+travis_config_path = Path.cwd() / ".travis.yml"
+with travis_config_path.open('r') as f:
+    travis_config = yaml.safe_load(f)
+
+jobs = travis_config['jobs']['include']
+travis_builds = [j for j in jobs
+                 if any("CIBW_BUILD" in env for env in j["env"])]
+n_wheels += len(travis_builds)
+
 dist_files = list(Path("dist").glob('**/*'))
 n_dist_files = len(dist_files)
 

diff --git a/build_tools/travis/install_master.sh b/build_tools/travis/install_master.sh
@@ -50,23 +50,8 @@ conda update --yes conda
 conda create -n testenv --yes python=3.7
 
 source activate testenv
-
-if [[ $TRAVIS_CPU_ARCH == amd64 ]]; then
-    echo "Upgrading pip and setuptools."
-    pip install --upgrade pip setuptools
-    echo "Installing numpy, scipy and pandas master wheels."
-    dev_anaconda_url=https://pypi.anaconda.org/scipy-wheels-nightly/simple
-    pip install --pre --upgrade --timeout=60 --extra-index $dev_anaconda_url numpy scipy pandas
-    echo "Installing cython pre-release wheels."
-    pip install --pre cython
-    echo "Installing joblib master."
-    pip install https://github.com/joblib/joblib/archive/master.zip
-    echo "Installing pillow master."
-    pip install https://github.com/python-pillow/Pillow/archive/master.zip
-else
-    conda install -y scipy numpy pandas cython
-    pip install joblib threadpoolctl
-fi
+conda install -y scipy numpy pandas cython
+pip install joblib threadpoolctl
 
 pip install $(get_dep pytest $PYTEST_VERSION) pytest-xdist