diff --git a/.circleci/config.yml b/.circleci/config.yml index c1330a6d239e2..eb634d02aeefa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,38 +1,78 @@ version: 2.1 -# Parameters required to trigger the execution -# of the "doc-min-dependencies" and "doc" jobs -parameters: - GITHUB_RUN_URL: - type: string - default: "none" - jobs: doc-min-dependencies: docker: - image: cimg/python:3.8.12 environment: - - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_URL >>/doc-min-dependencies.zip + - MKL_NUM_THREADS: 2 + - OPENBLAS_NUM_THREADS: 2 + - CONDA_ENV_NAME: testenv + - LOCK_FILE: build_tools/circle/doc_min_dependencies_linux-64_conda.lock + # Sphinx race condition in doc-min-dependencies is causing job to stall + # Here we run the job serially + - SPHINX_NUMJOBS: 1 steps: - checkout - - run: bash build_tools/circle/download_documentation.sh + - run: ./build_tools/circle/checkout_merge_commit.sh + - restore_cache: + key: v1-doc-min-deps-datasets-{{ .Branch }} + - restore_cache: + keys: + - doc-min-deps-ccache-{{ .Branch }} + - doc-min-deps-ccache + - run: ./build_tools/circle/build_doc.sh + - save_cache: + key: doc-min-deps-ccache-{{ .Branch }}-{{ .BuildNum }} + paths: + - ~/.ccache + - ~/.cache/pip + - save_cache: + key: v1-doc-min-deps-datasets-{{ .Branch }} + paths: + - ~/scikit_learn_data - store_artifacts: path: doc/_build/html/stable destination: doc + - store_artifacts: + path: ~/log.txt + destination: log.txt doc: docker: - image: cimg/python:3.8.12 environment: - - GITHUB_ARTIFACT_URL: << pipeline.parameters.GITHUB_RUN_URL >>/doc.zip + - MKL_NUM_THREADS: 2 + - OPENBLAS_NUM_THREADS: 2 + - CONDA_ENV_NAME: testenv + - LOCK_FILE: build_tools/circle/doc_linux-64_conda.lock steps: - checkout - - run: bash build_tools/circle/download_documentation.sh + - run: ./build_tools/circle/checkout_merge_commit.sh + - restore_cache: + key: v1-doc-datasets-{{ .Branch }} + - restore_cache: + keys: + - doc-ccache-{{ .Branch }} + - doc-ccache + - run: ./build_tools/circle/build_doc.sh + - save_cache: + key: doc-ccache-{{ .Branch }}-{{ .BuildNum }} + paths: + - ~/.ccache + - ~/.cache/pip + - save_cache: + key: v1-doc-datasets-{{ .Branch }} + paths: + - ~/scikit_learn_data - store_artifacts: path: doc/_build/html/stable destination: doc - # Persists the generated documentation, so that it - # can be attached and deployed in the "deploy" job + - store_artifacts: + path: ~/log.txt + destination: log.txt + # Persists generated documentation so that it can be attached and deployed + # in the 'deploy' step. - persist_to_workspace: root: doc/_build/html paths: . @@ -54,36 +94,12 @@ jobs: bash build_tools/circle/push_doc.sh doc/_build/html/stable fi - # This noop job is required for the pipeline to exist, so that the - # documentation related jobs can be triggered. - noop: - docker: - - image: cimg/python:3.8.12 - steps: - - run: | - echo "This is no-op job for the pipeline to exist, so that it triggers " - echo "Circle CI jobs pushing the artifacts of the documentation built " - echo "via GitHub actions." - workflows: version: 2 - build-doc-and-deploy: - when: - not: - equal: [ "none", << pipeline.parameters.GITHUB_RUN_URL >> ] - # The jobs should run only when triggered by the workflow jobs: - - doc-min-dependencies - doc + - doc-min-dependencies - deploy: requires: - doc - - noop: - when: - equal: [ "none", << pipeline.parameters.GITHUB_RUN_URL >> ] - # Prevent double execution of this job: on push - # by default and when triggered by the workflow - jobs: - - noop diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml deleted file mode 100644 index a57abe7214504..0000000000000 --- a/.github/workflows/build-docs.yml +++ /dev/null @@ -1,75 +0,0 @@ -# Workflow to build the documentation -name: Documentation builder - -on: - push: - branches: - - main - # Release branches - - "[0-9]+.[0-9]+.X" - pull_request: - branches: - - main - - "[0-9]+.[0-9]+.X" - -jobs: - # Build the documentation against the minimum version of the dependencies - doc-min-dependencies: - # This prevents this workflow from running on a fork. - # To test this workflow on a fork, uncomment the following line. - if: github.repository == 'scikit-learn/scikit-learn' - - runs-on: ubuntu-latest - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v3 - with: - # needed by build_doc.sh to compute the list of changed doc files: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - - name: Build documentation - run: bash build_tools/github/build_doc.sh - env: - OMP_NUM_THREADS: 2 - MKL_NUM_THREADS: 2 - CONDA_ENV_NAME: testenv - # Sphinx race condition in doc-min-dependencies is causing job to stall - # Here we run the job serially - SPHINX_NUMJOBS: 1 - LOCK_FILE: build_tools/github/doc_min_dependencies_linux-64_conda.lock - - - name: Upload documentation - uses: actions/upload-artifact@v3 - with: - name: doc-min-dependencies - path: doc/_build/html/stable - - # Build the documentation against the latest version of the dependencies - doc: - # This prevents this workflow from running on a fork. - # To test this workflow on a fork, uncomment the following line. - if: github.repository == 'scikit-learn/scikit-learn' - - runs-on: ubuntu-latest - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v3 - with: - # needed by build_doc.sh to compute the list of changed doc files: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - - name: Build documentation - run: bash build_tools/github/build_doc.sh - env: - OMP_NUM_THREADS: 2 - MKL_NUM_THREADS: 2 - CONDA_ENV_NAME: testenv - LOCK_FILE: build_tools/github/doc_linux-64_conda.lock - - - name: Upload documentation - uses: actions/upload-artifact@v3 - with: - name: doc - path: doc/_build/html/stable diff --git a/.github/workflows/trigger-hosting.yml b/.github/workflows/trigger-hosting.yml deleted file mode 100644 index 9d0c2c1a56070..0000000000000 --- a/.github/workflows/trigger-hosting.yml +++ /dev/null @@ -1,38 +0,0 @@ -# Workflow to trigger the jobs that will host the documentation -name: Documentation push trigger -on: - workflow_run: - # Run the workflow after the separate "Documentation builder" workflow completes - workflows: [Documentation builder] - types: - - completed - -jobs: - push: - runs-on: ubuntu-latest - # Run the job only if the "Documentation builder" workflow succeeded - # Prevents this workflow from running on a fork. - # To test this workflow on a fork remove the `github.repository == scikit-learn/scikit-learn` condition - if: github.repository == 'scikit-learn/scikit-learn' && github.event.workflow_run.conclusion == 'success' - steps: - - name: Checkout scikit-learn - uses: actions/checkout@v3 - - - name: Trigger hosting jobs - run: bash build_tools/github/trigger_hosting.sh - env: - # Note: the CIRCLE_CI_TOKEN needs to be a user token instead of a - # project-wide token created using: - # - # https://support.circleci.com/hc/en-us/articles/360050351292-How-to-Trigger-a-Workflow-via-CircleCI-API-v2 - # - # At the time of writing, secrets.CIRCLE_CI_TOKEN is valued with a - # token created using the ogrisel circleci account using: - # https://app.circleci.com/settings/user/tokens - CIRCLE_CI_TOKEN: ${{ secrets.CIRCLE_CI_TOKEN }} - EVENT: ${{ github.event.workflow_run.event }} - RUN_ID: ${{ github.event.workflow_run.id }} - HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} - COMMIT_SHA: ${{ github.event.workflow_run.head_sha }} - REPO_NAME: ${{ github.event.workflow_run.head_repository.full_name }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/build_tools/github/build_doc.sh b/build_tools/circle/build_doc.sh similarity index 96% rename from build_tools/github/build_doc.sh rename to build_tools/circle/build_doc.sh index 249dd82e798b6..bc7fb67b33f40 100755 --- a/build_tools/github/build_doc.sh +++ b/build_tools/circle/build_doc.sh @@ -16,9 +16,12 @@ set -e # If the inspection of the current commit fails for any reason, the default # behavior is to quick build the documentation. +# defines the get_dep and show_installed_libraries functions +source build_tools/shared.sh + if [ -n "$GITHUB_ACTION" ] then - # Map the variables for the new documentation builder to the old one + # Map the variables from Github Action to CircleCI CIRCLE_SHA1=$(git log -1 --pretty=format:%H) CIRCLE_JOB=$GITHUB_JOB @@ -169,11 +172,12 @@ ccache -M 512M export CCACHE_COMPRESS=1 # pin conda-lock to latest released version (needs manual update from time to time) -mamba install conda-lock==1.0.5 -y +mamba install "$(get_dep conda-lock min)" -y + conda-lock install --log-level WARNING --name $CONDA_ENV_NAME $LOCK_FILE source activate $CONDA_ENV_NAME -mamba list +show_installed_libraries # Set parallelism to 3 to overlap IO bound tasks with CPU bound tasks on CI # workers with 2 cores when building the compiled extensions of scikit-learn. @@ -185,12 +189,18 @@ ccache -s export OMP_NUM_THREADS=1 +# Avoid CI job getting killed because it uses too much memory +if [[ -z $SPHINX_NUMJOBS ]]; then + export SPHINX_NUMJOBS=2 +fi + if [[ "$CIRCLE_BRANCH" =~ ^main$ && -z "$CI_PULL_REQUEST" ]] then # List available documentation versions if on main python build_tools/circle/list_versions.py > doc/versions.rst fi + # The pipefail is requested to propagate exit code set -o pipefail && cd doc && make $make_args 2>&1 | tee ~/log.txt diff --git a/build_tools/github/doc_environment.yml b/build_tools/circle/doc_environment.yml similarity index 100% rename from build_tools/github/doc_environment.yml rename to build_tools/circle/doc_environment.yml diff --git a/build_tools/github/doc_linux-64_conda.lock b/build_tools/circle/doc_linux-64_conda.lock similarity index 100% rename from build_tools/github/doc_linux-64_conda.lock rename to build_tools/circle/doc_linux-64_conda.lock diff --git a/build_tools/github/doc_min_dependencies_environment.yml b/build_tools/circle/doc_min_dependencies_environment.yml similarity index 100% rename from build_tools/github/doc_min_dependencies_environment.yml rename to build_tools/circle/doc_min_dependencies_environment.yml diff --git a/build_tools/github/doc_min_dependencies_linux-64_conda.lock b/build_tools/circle/doc_min_dependencies_linux-64_conda.lock similarity index 100% rename from build_tools/github/doc_min_dependencies_linux-64_conda.lock rename to build_tools/circle/doc_min_dependencies_linux-64_conda.lock diff --git a/build_tools/github/trigger_hosting.sh b/build_tools/github/trigger_hosting.sh deleted file mode 100755 index 2a8e28ff164ff..0000000000000 --- a/build_tools/github/trigger_hosting.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -e -set -x - -GITHUB_RUN_URL=https://nightly.link/$GITHUB_REPOSITORY/actions/runs/$RUN_ID - -if [ "$EVENT" == pull_request ] -then - PULL_REQUEST_NUMBER=$(curl \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: token $GITHUB_TOKEN" \ - https://api.github.com/repos/$REPO_NAME/commits/$COMMIT_SHA/pulls 2>/dev/null \ - | jq '.[0].number') - - if [[ "$PULL_REQUEST_NUMBER" == "null" ]]; then - # The pull request is on the main (default) branch of the fork. The above API - # call is unable to get the PR number associated with the commit: - # https://docs.github.com/en/rest/commits/commits#list-pull-requests-associated-with-a-commit - # We fallback to the search API here. The search API is not used everytime - # because it has a lower rate limit. - PULL_REQUEST_NUMBER=$(curl \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: token $GITHUB_TOKEN" \ - "https://api.github.com/search/issues?q=$COMMIT_SHA+repo:$GITHUB_REPOSITORY" 2>/dev/null \ - | jq '.items[0].number') - fi - - BRANCH=pull/$PULL_REQUEST_NUMBER/head -else - BRANCH=$HEAD_BRANCH -fi - -curl --request POST \ - --url https://circleci.com/api/v2/project/gh/$GITHUB_REPOSITORY/pipeline \ - --header "Circle-Token: $CIRCLE_CI_TOKEN" \ - --header "content-type: application/json" \ - --header "x-attribution-actor-id: github_actions" \ - --header "x-attribution-login: github_actions" \ - --data \{\"branch\":\"$BRANCH\",\"parameters\":\{\"GITHUB_RUN_URL\":\"$GITHUB_RUN_URL\"\}\} diff --git a/build_tools/update_environments_and_lock_files.py b/build_tools/update_environments_and_lock_files.py index 856ec8d7e6fd4..98fe49a80080a 100644 --- a/build_tools/update_environments_and_lock_files.py +++ b/build_tools/update_environments_and_lock_files.py @@ -241,7 +241,7 @@ def remove_from(alist, to_remove): }, { "build_name": "doc_min_dependencies", - "folder": "build_tools/github", + "folder": "build_tools/circle", "platform": "linux-64", "channel": "conda-forge", "conda_dependencies": common_dependencies_without_coverage @@ -276,7 +276,7 @@ def remove_from(alist, to_remove): }, { "build_name": "doc", - "folder": "build_tools/github", + "folder": "build_tools/circle", "platform": "linux-64", "channel": "conda-forge", "conda_dependencies": common_dependencies_without_coverage diff --git a/doc/templates/index.html b/doc/templates/index.html index af69ae570a785..29b7cb540c80f 100644 --- a/doc/templates/index.html +++ b/doc/templates/index.html @@ -166,6 +166,8 @@

News

  • On-going development: What's new (Changelog)
    • +
  • January 2023. scikit-learn 1.2.1 is available for download (Changelog). +
  • December 2022. scikit-learn 1.2.0 is available for download (Changelog).
  • October 2022. scikit-learn 1.1.3 is available for download (Changelog).