scikit-learn · ogrisel · Apr 20, 2022 · Apr 1, 2022 · Apr 1, 2022 · Apr 5, 2022
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -13,18 +13,7 @@ jobs:
   pool:
     vmImage: ubuntu-20.04
   steps:
-    - bash: |
-        set -ex
-        if [[ $BUILD_REASON == "PullRequest" ]]; then
-          # By default pull requests use refs/pull/PULL_ID/merge as the source branch
-          # which has a "Merge ID into ID" as a commit message. The latest commit
-          # message is the second to last commit
-          COMMIT_ID=$(echo $BUILD_SOURCEVERSIONMESSAGE | awk '{print $2}')
-          message=$(git log $COMMIT_ID -1 --pretty=%B)
-        else
-          message=$BUILD_SOURCEVERSIONMESSAGE
-        fi
-        echo "##vso[task.setvariable variable=message;isOutput=true]$message"
+    - bash: python build_tools/azure/get_commit_message.py
       name: commit
       displayName: Get source version message
 

diff --git a/build_tools/azure/get_commit_message.py b/build_tools/azure/get_commit_message.py
@@ -0,0 +1,29 @@
+import os
+import subprocess
+
+
+def get_commit_message():
+    """Retrieve the commit message."""
+    build_source_version_message = os.environ["BUILD_SOURCEVERSIONMESSAGE"]
+
+    if os.environ["BUILD_REASON"] == "PullRequest":
+        # By default pull requests use refs/pull/PULL_ID/merge as the source branch
+        # which has a "Merge ID into ID" as a commit message. The latest commit
+        # message is the second to last commit
+        commit_id = build_source_version_message.split()[1]
+        git_cmd = ["git", "log", commit_id, "-1", "--pretty=%B"]
+        commit_message = subprocess.run(
+            git_cmd, capture_output=True, text=True
+        ).stdout.strip()
+    else:
+        commit_message = build_source_version_message
+
+    return commit_message
+
+
+if __name__ == "__main__":
+    # set the environment variable to be propagated to other steps
+    commit_message = get_commit_message()
+    print(f"##vso[task.setvariable variable=message;isOutput=true]{commit_message}")
+
+    print(f"commit message: {commit_message}")  # helps debugging
diff --git a/build_tools/azure/get_selected_tests.py b/build_tools/azure/get_selected_tests.py
@@ -0,0 +1,31 @@
+from get_commit_message import get_commit_message
+
+
+def get_selected_tests():
+    """Parse the commit message to check if pytest should run only specific tests.
+
+    If so, selected tests will be run with SKLEARN_TESTS_GLOBAL_RANDOM_SEED="all".
+
+    The commit message must take the form:
+        <title> [all random seeds]
+        <test_name_1>
+        <test_name_2>
+        ...
+    """
+    commit_message = get_commit_message()
+
+    if "[all random seeds]" in commit_message:
+        selected_tests = commit_message.split("[all random seeds]")[1].strip()
+        selected_tests = selected_tests.replace("\n", " or ")
+    else:
+        selected_tests = ""
+
+    return selected_tests
+
+
+if __name__ == "__main__":
+    # set the environment variable to be propagated to other steps
+    selected_tests = get_selected_tests()
+    print(f"##vso[task.setvariable variable=SELECTED_TESTS]'{selected_tests}'")
+
+    print(f"selected tests: {selected_tests}")  # helps debugging
diff --git a/build_tools/azure/posix-docker.yml b/build_tools/azure/posix-docker.yml
@@ -45,6 +45,15 @@ jobs:
       ${{ insert }}: ${{ parameters.matrix }}
 
   steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '3.9'
+        addToPath: false
+      name: pyTools
+      displayName: Select python version to run CI python scripts
+    - bash: $(pyTools.pythonLocation)/bin/python build_tools/azure/get_selected_tests.py
+      displayName: Check selected tests for all random seeds
+      condition: eq(variables['Build.Reason'], 'PullRequest')
     - task: Cache@2
       inputs:
         key: '"ccache-v1" | "$(Agent.JobName)" | "$(Build.BuildNumber)"'
@@ -85,6 +94,7 @@ jobs:
         -e OPENBLAS_NUM_THREADS=$OPENBLAS_NUM_THREADS
         -e SKLEARN_SKIP_NETWORK_TESTS=$SKLEARN_SKIP_NETWORK_TESTS
         -e BLAS=$BLAS
+        -e SELECTED_TESTS="$SELECTED_TESTS"
         -e CPU_COUNT=$CPU_COUNT
         -e CCACHE_DIR=/ccache
         -e CCACHE_COMPRESS=$CCACHE_COMPRESS
@@ -107,12 +117,6 @@ jobs:
         docker container stop skcontainer
       displayName: 'Stop container'
       condition: always()
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.9'
-      displayName: Place Python into path to update issue tracker
-      condition: and(succeededOrFailed(), eq(variables['CREATE_ISSUE_ON_TRACKER'], 'true'),
-                     eq(variables['Build.Reason'], 'Schedule'))
     - bash: |
         set -ex
         if [[ $(BOT_GITHUB_TOKEN) == "" ]]; then
@@ -124,8 +128,8 @@ jobs:
         CI_NAME="$SYSTEM_JOBIDENTIFIER"
         ISSUE_REPO="$BUILD_REPOSITORY_NAME"
 
-        pip install defusedxml PyGithub
-        python maint_tools/update_tracking_issue.py \
+        $(pyTools.pythonLocation)/bin/pip install defusedxml PyGithub
+        $(pyTools.pythonLocation)/bin/python maint_tools/update_tracking_issue.py \
           $(BOT_GITHUB_TOKEN) \
           $CI_NAME \
           $ISSUE_REPO \

diff --git a/build_tools/azure/posix.yml b/build_tools/azure/posix.yml
@@ -42,6 +42,15 @@ jobs:
       ${{ insert }}: ${{ parameters.matrix }}
 
   steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '3.9'
+        addToPath: false
+      name: pyTools
+      displayName: Select python version to run CI python scripts
+    - bash: $(pyTools.pythonLocation)/bin/python build_tools/azure/get_selected_tests.py
+      displayName: Check selected tests for all random seeds
+      condition: eq(variables['Build.Reason'], 'PullRequest')
     - bash: echo "##vso[task.prependpath]$CONDA/bin"
       displayName: Add conda to PATH
       condition: startsWith(variables['DISTRIB'], 'conda')
@@ -65,22 +74,18 @@ jobs:
     - script: |
         build_tools/azure/test_docs.sh
       displayName: 'Test Docs'
+      condition: eq(variables['SELECTED_TESTS'], '')
     - script: |
         build_tools/azure/test_pytest_soft_dependency.sh
       displayName: 'Test Soft Dependency'
-      condition: eq(variables['CHECK_PYTEST_SOFT_DEPENDENCY'], 'true')
+      condition: and(eq(variables['CHECK_PYTEST_SOFT_DEPENDENCY'], 'true'),
+                     eq(variables['SELECTED_TESTS'], ''))
     - task: PublishTestResults@2
       inputs:
         testResultsFiles: '$(TEST_DIR)/$(JUNITXML)'
         testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }}
       displayName: 'Publish Test Results'
       condition: succeededOrFailed()
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.9'
-      displayName: Place Python into path to update issue tracker
-      condition: and(succeededOrFailed(), eq(variables['CREATE_ISSUE_ON_TRACKER'], 'true'),
-                     eq(variables['Build.Reason'], 'Schedule'))
     - bash: |
         set -ex
         if [[ $(BOT_GITHUB_TOKEN) == "" ]]; then
@@ -92,8 +97,8 @@ jobs:
         CI_NAME="$SYSTEM_JOBIDENTIFIER"
         ISSUE_REPO="$BUILD_REPOSITORY_NAME"
 
-        pip install defusedxml PyGithub
-        python maint_tools/update_tracking_issue.py \
+        $(pyTools.pythonLocation)/bin/pip install defusedxml PyGithub
+        $(pyTools.pythonLocation)/bin/python maint_tools/update_tracking_issue.py \
           $(BOT_GITHUB_TOKEN) \
           $CI_NAME \
           $ISSUE_REPO \
@@ -106,7 +111,8 @@ jobs:
                      eq(variables['Build.Reason'], 'Schedule'))
     - script: |
         build_tools/azure/upload_codecov.sh
-      condition: and(succeeded(), eq(variables['COVERAGE'], 'true'))
+      condition: and(succeeded(), eq(variables['COVERAGE'], 'true'),
+                     eq(variables['SELECTED_TESTS'], ''))
       displayName: 'Upload To Codecov'
       env:
         CODECOV_TOKEN: $(CODECOV_TOKEN)
diff --git a/build_tools/azure/test_script.sh b/build_tools/azure/test_script.sh
@@ -65,6 +65,13 @@ if [[ "$SHOW_SHORT_SUMMARY" == "true" ]]; then
     TEST_CMD="$TEST_CMD -ra"
 fi
 
+if [[ "$SELECTED_TESTS" != "" ]]; then
+    TEST_CMD="$TEST_CMD -k $SELECTED_TESTS"
+
    # Override to make selected tests run on all random seeds
+    export SKLEARN_TESTS_GLOBAL_RANDOM_SEED="all"
+fi
+
 set -x
 eval "$TEST_CMD --pyargs sklearn"
 set +x
diff --git a/build_tools/azure/windows.yml b/build_tools/azure/windows.yml
@@ -26,6 +26,9 @@ jobs:
       ${{ insert }}: ${{ parameters.matrix }}
 
   steps:
+    - bash: python build_tools/azure/get_selected_tests.py
+      displayName: Check selected tests for all random seeds
+      condition: eq(variables['Build.Reason'], 'PullRequest')
     - bash: echo "##vso[task.prependpath]$CONDA/Scripts"
       displayName: Add conda to PATH for 64 bit Python
       condition: eq(variables['PYTHON_ARCH'], '64')
@@ -41,7 +44,8 @@ jobs:
     - bash: ./build_tools/azure/test_script.sh
       displayName: 'Test Library'
     - bash: ./build_tools/azure/upload_codecov.sh
-      condition: and(succeeded(), eq(variables['COVERAGE'], 'true'))
+      condition: and(succeeded(), eq(variables['COVERAGE'], 'true'),
+                     eq(variables['SELECTED_TESTS'], ''))
       displayName: 'Upload To Codecov'
       env:
         CODECOV_TOKEN: $(CODECOV_TOKEN)