diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 96b52055328a8..d845ed8526928 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,18 +13,7 @@ jobs: pool: vmImage: ubuntu-20.04 steps: - - bash: | - set -ex - if [[ $BUILD_REASON == "PullRequest" ]]; then - # By default pull requests use refs/pull/PULL_ID/merge as the source branch - # which has a "Merge ID into ID" as a commit message. The latest commit - # message is the second to last commit - COMMIT_ID=$(echo $BUILD_SOURCEVERSIONMESSAGE | awk '{print $2}') - message=$(git log $COMMIT_ID -1 --pretty=%B) - else - message=$BUILD_SOURCEVERSIONMESSAGE - fi - echo "##vso[task.setvariable variable=message;isOutput=true]$message" + - bash: python build_tools/azure/get_commit_message.py name: commit displayName: Get source version message diff --git a/build_tools/azure/get_commit_message.py b/build_tools/azure/get_commit_message.py new file mode 100644 index 0000000000000..d20bf99139071 --- /dev/null +++ b/build_tools/azure/get_commit_message.py @@ -0,0 +1,29 @@ +import os +import subprocess + + +def get_commit_message(): + """Retrieve the commit message.""" + build_source_version_message = os.environ["BUILD_SOURCEVERSIONMESSAGE"] + + if os.environ["BUILD_REASON"] == "PullRequest": + # By default pull requests use refs/pull/PULL_ID/merge as the source branch + # which has a "Merge ID into ID" as a commit message. The latest commit + # message is the second to last commit + commit_id = build_source_version_message.split()[1] + git_cmd = ["git", "log", commit_id, "-1", "--pretty=%B"] + commit_message = subprocess.run( + git_cmd, capture_output=True, text=True + ).stdout.strip() + else: + commit_message = build_source_version_message + + return commit_message + + +if __name__ == "__main__": + # set the environment variable to be propagated to other steps + commit_message = get_commit_message() + print(f"##vso[task.setvariable variable=message;isOutput=true]{commit_message}") + + print(f"commit message: {commit_message}") # helps debugging diff --git a/build_tools/azure/get_selected_tests.py b/build_tools/azure/get_selected_tests.py new file mode 100644 index 0000000000000..09b8a6eef5b28 --- /dev/null +++ b/build_tools/azure/get_selected_tests.py @@ -0,0 +1,31 @@ +from get_commit_message import get_commit_message + + +def get_selected_tests(): + """Parse the commit message to check if pytest should run only specific tests. + + If so, selected tests will be run with SKLEARN_TESTS_GLOBAL_RANDOM_SEED="all". + + The commit message must take the form: + [all random seeds] + <test_name_1> + <test_name_2> + ... + """ + commit_message = get_commit_message() + + if "[all random seeds]" in commit_message: + selected_tests = commit_message.split("[all random seeds]")[1].strip() + selected_tests = selected_tests.replace("\n", " or ") + else: + selected_tests = "" + + return selected_tests + + +if __name__ == "__main__": + # set the environment variable to be propagated to other steps + selected_tests = get_selected_tests() + print(f"##vso[task.setvariable variable=SELECTED_TESTS]'{selected_tests}'") + + print(f"selected tests: {selected_tests}") # helps debugging diff --git a/build_tools/azure/posix-docker.yml b/build_tools/azure/posix-docker.yml index c11fa00e4d70a..1514f5e548666 100644 --- a/build_tools/azure/posix-docker.yml +++ b/build_tools/azure/posix-docker.yml @@ -45,6 +45,15 @@ jobs: ${{ insert }}: ${{ parameters.matrix }} steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.9' + addToPath: false + name: pyTools + displayName: Select python version to run CI python scripts + - bash: $(pyTools.pythonLocation)/bin/python build_tools/azure/get_selected_tests.py + displayName: Check selected tests for all random seeds + condition: eq(variables['Build.Reason'], 'PullRequest') - task: Cache@2 inputs: key: '"ccache-v1" | "$(Agent.JobName)" | "$(Build.BuildNumber)"' @@ -85,6 +94,7 @@ jobs: -e OPENBLAS_NUM_THREADS=$OPENBLAS_NUM_THREADS -e SKLEARN_SKIP_NETWORK_TESTS=$SKLEARN_SKIP_NETWORK_TESTS -e BLAS=$BLAS + -e SELECTED_TESTS="$SELECTED_TESTS" -e CPU_COUNT=$CPU_COUNT -e CCACHE_DIR=/ccache -e CCACHE_COMPRESS=$CCACHE_COMPRESS @@ -107,12 +117,6 @@ jobs: docker container stop skcontainer displayName: 'Stop container' condition: always() - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.9' - displayName: Place Python into path to update issue tracker - condition: and(succeededOrFailed(), eq(variables['CREATE_ISSUE_ON_TRACKER'], 'true'), - eq(variables['Build.Reason'], 'Schedule')) - bash: | set -ex if [[ $(BOT_GITHUB_TOKEN) == "" ]]; then @@ -124,8 +128,8 @@ jobs: CI_NAME="$SYSTEM_JOBIDENTIFIER" ISSUE_REPO="$BUILD_REPOSITORY_NAME" - pip install defusedxml PyGithub - python maint_tools/update_tracking_issue.py \ + $(pyTools.pythonLocation)/bin/pip install defusedxml PyGithub + $(pyTools.pythonLocation)/bin/python maint_tools/update_tracking_issue.py \ $(BOT_GITHUB_TOKEN) \ $CI_NAME \ $ISSUE_REPO \ diff --git a/build_tools/azure/posix.yml b/build_tools/azure/posix.yml index e2c8e0881818f..4bd83e8713f3f 100644 --- a/build_tools/azure/posix.yml +++ b/build_tools/azure/posix.yml @@ -42,6 +42,15 @@ jobs: ${{ insert }}: ${{ parameters.matrix }} steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.9' + addToPath: false + name: pyTools + displayName: Select python version to run CI python scripts + - bash: $(pyTools.pythonLocation)/bin/python build_tools/azure/get_selected_tests.py + displayName: Check selected tests for all random seeds + condition: eq(variables['Build.Reason'], 'PullRequest') - bash: echo "##vso[task.prependpath]$CONDA/bin" displayName: Add conda to PATH condition: startsWith(variables['DISTRIB'], 'conda') @@ -65,22 +74,18 @@ jobs: - script: | build_tools/azure/test_docs.sh displayName: 'Test Docs' + condition: eq(variables['SELECTED_TESTS'], '') - script: | build_tools/azure/test_pytest_soft_dependency.sh displayName: 'Test Soft Dependency' - condition: eq(variables['CHECK_PYTEST_SOFT_DEPENDENCY'], 'true') + condition: and(eq(variables['CHECK_PYTEST_SOFT_DEPENDENCY'], 'true'), + eq(variables['SELECTED_TESTS'], '')) - task: PublishTestResults@2 inputs: testResultsFiles: '$(TEST_DIR)/$(JUNITXML)' testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }} displayName: 'Publish Test Results' condition: succeededOrFailed() - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.9' - displayName: Place Python into path to update issue tracker - condition: and(succeededOrFailed(), eq(variables['CREATE_ISSUE_ON_TRACKER'], 'true'), - eq(variables['Build.Reason'], 'Schedule')) - bash: | set -ex if [[ $(BOT_GITHUB_TOKEN) == "" ]]; then @@ -92,8 +97,8 @@ jobs: CI_NAME="$SYSTEM_JOBIDENTIFIER" ISSUE_REPO="$BUILD_REPOSITORY_NAME" - pip install defusedxml PyGithub - python maint_tools/update_tracking_issue.py \ + $(pyTools.pythonLocation)/bin/pip install defusedxml PyGithub + $(pyTools.pythonLocation)/bin/python maint_tools/update_tracking_issue.py \ $(BOT_GITHUB_TOKEN) \ $CI_NAME \ $ISSUE_REPO \ @@ -106,7 +111,8 @@ jobs: eq(variables['Build.Reason'], 'Schedule')) - script: | build_tools/azure/upload_codecov.sh - condition: and(succeeded(), eq(variables['COVERAGE'], 'true')) + condition: and(succeeded(), eq(variables['COVERAGE'], 'true'), + eq(variables['SELECTED_TESTS'], '')) displayName: 'Upload To Codecov' env: CODECOV_TOKEN: $(CODECOV_TOKEN) diff --git a/build_tools/azure/test_script.sh b/build_tools/azure/test_script.sh index 73e9ba0b618ac..b3088bc62a84d 100755 --- a/build_tools/azure/test_script.sh +++ b/build_tools/azure/test_script.sh @@ -65,6 +65,13 @@ if [[ "$SHOW_SHORT_SUMMARY" == "true" ]]; then TEST_CMD="$TEST_CMD -ra" fi +if [[ "$SELECTED_TESTS" != "" ]]; then + TEST_CMD="$TEST_CMD -k $SELECTED_TESTS" + + # Override to make selected tests run on all random seeds + export SKLEARN_TESTS_GLOBAL_RANDOM_SEED="all" +fi + set -x eval "$TEST_CMD --pyargs sklearn" set +x diff --git a/build_tools/azure/windows.yml b/build_tools/azure/windows.yml index 7402f1739fc3c..3e1d282f3d79a 100644 --- a/build_tools/azure/windows.yml +++ b/build_tools/azure/windows.yml @@ -26,6 +26,9 @@ jobs: ${{ insert }}: ${{ parameters.matrix }} steps: + - bash: python build_tools/azure/get_selected_tests.py + displayName: Check selected tests for all random seeds + condition: eq(variables['Build.Reason'], 'PullRequest') - bash: echo "##vso[task.prependpath]$CONDA/Scripts" displayName: Add conda to PATH for 64 bit Python condition: eq(variables['PYTHON_ARCH'], '64') @@ -41,7 +44,8 @@ jobs: - bash: ./build_tools/azure/test_script.sh displayName: 'Test Library' - bash: ./build_tools/azure/upload_codecov.sh - condition: and(succeeded(), eq(variables['COVERAGE'], 'true')) + condition: and(succeeded(), eq(variables['COVERAGE'], 'true'), + eq(variables['SELECTED_TESTS'], '')) displayName: 'Upload To Codecov' env: CODECOV_TOKEN: $(CODECOV_TOKEN)