8000 CI allow to run only selected tests but on all random seeds (#23026) · jjerphan/scikit-learn@72211d1 · GitHub
[go: up one dir, main page]
Skip to content

Commit 72211d1

Browse files
8000
jeremiedbbjjerphan
jeremiedbb
authored and
jjerphan
committed
CI allow to run only selected tests but on all random seeds (scikit-learn#23026)
1 parent 1d6ebf3 commit 72211d1

File tree

7 files changed

+101
-31
lines changed

7 files changed

+101
-31
lines changed

‎azure-pipelines.yml

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,7 @@ jobs:
1313
pool:
1414
vmImage: ubuntu-20.04
1515
steps:
16-
- bash: |
17-
set -ex
18-
if [[ $BUILD_REASON == "PullRequest" ]]; then
19-
# By default pull requests use refs/pull/PULL_ID/merge as the source branch
20-
# which has a "Merge ID into ID" as a commit message. The latest commit
21-
# message is the second to last commit
22-
COMMIT_ID=$(echo $BUILD_SOURCEVERSIONMESSAGE | awk '{print $2}')
23-
message=$(git log $COMMIT_ID -1 --pretty=%B)
24-
else
25-
message=$BUILD_SOURCEVERSIONMESSAGE
26-
fi
27-
echo "##vso[task.setvariable variable=message;isOutput=t 8000 rue]$message"
16+
- bash: python build_tools/azure/get_commit_message.py
2817
name: commit
2918
displayName: Get source version message
3019

‎build_tools/azure/get_commit_message.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import os
2+
import subprocess
3+
4+
5+
def get_commit_message():
6+
"""Retrieve the commit message."""
7+
build_source_version_message = os.environ["BUILD_SOURCEVERSIONMESSAGE"]
8+
9+
if os.environ["BUILD_REASON"] == "PullRequest":
10+
# By default pull requests use refs/pull/PULL_ID/merge as the source branch
11+
# which has a "Merge ID into ID" as a commit message. The latest commit
12+
# message is the second to last commit
13+
commit_id = build_source_version_message.split()[1]
14+
git_cmd = ["git", "log", commit_id, "-1", "--pretty=%B"]
15+
commit_message = subprocess.run(
16+
git_cmd, capture_output=True, text=True
17+
).stdout.strip()
18+
else:
19+
commit_message = build_source_version_message
20+
21+
return commit_message
22+
23+
24+
if __name__ == "__main__":
25+
# set the environment variable to be propagated to other steps
26+
commit_message = get_commit_message()
27+
print(f"##vso[task.setvariable variable=message;isOutput=true]{commit_message}")
28+
29+
print(f"commit message: {commit_message}") # helps debugging

‎build_tools/azure/get_selected_tests.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
from get_commit_message import get_commit_message
2+
3+
4+
def get_selected_tests():
5+
"""Parse the commit message to check if pytest should run only specific tests.
6+
7+
If so, selected tests will be run with SKLEARN_TESTS_GLOBAL_RANDOM_SEED="all".
8+
9+
The commit message must take the form:
10+
<title> [all random seeds]
11+
<test_name_1>
12+
<test_name_2>
13+
...
14+
"""
15+
commit_message = get_commit_message()
16+
17+
if "[all random seeds]" in commit_message:
18+
selected_tests = commit_message.split("[all random seeds]")[1].strip()
19+
selected_tests = selected_tests.replace("\n", " or ")
20+
else:
21+
selected_tests = ""
22+
23+
return selected_tests
24+
25+
26+
if __name__ == "__main__":
27+
# set the environment variable to be propagated to other steps
28+
selected_tests = get_selected_tests()
29+
print(f"##vso[task.setvariable variable=SELECTED_TESTS]'{selected_tests}'")
30+
31+
print(f"selected tests: {selected_tests}") # helps debugging

‎build_tools/azure/posix-docker.yml

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,15 @@ jobs:
4545
${{ insert }}: ${{ parameters.matrix }}
4646

4747
steps:
48+
- task: UsePythonVersion@0
49+
inputs:
50+
versionSpec: '3.9'
51+
addToPath: false
52+
name: pyTools
53+
displayName: Select python version to run CI python scripts
54+
- bash: $(pyTools.pythonLocation)/bin/python build_tools/azure/get_selected_tests.py
55+
displayName: Check selected tests for all random seeds
56+
condition: eq(variables['Build.Reason'], 'PullRequest')
4857
- task: Cache@2
4958
inputs:
5059
key: '"ccache-v1" | "$(Agent.JobName)" | "$(Build.BuildNumber)"'
@@ -85,6 +94,7 @@ jobs:
8594
-e OPENBLAS_NUM_THREADS=$OPENBLAS_NUM_THREADS
8695
-e SKLEARN_SKIP_NETWORK_TESTS=$SKLEARN_SKIP_NETWORK_TESTS
8796
-e BLAS=$BLAS
97+
-e SELECTED_TESTS="$SELECTED_TESTS"
8898
-e CPU_COUNT=$CPU_COUNT
8999
-e CCACHE_DIR=/ccache
90100
-e CCACHE_COMPRESS=$CCACHE_COMPRESS
@@ -107,12 +117,6 @@ jobs:
107117
docker container stop skcontainer
108118
displayName: 'Stop container'
109119
condition: always()
110-
- task: UsePythonVersion@0
111-
inputs:
112-
versionSpec: '3.9'
113-
displayName: Place Python into path to update issue tracker
114-
condition: and(succeededOrFailed(), eq(variables['CREATE_ISSUE_ON_TRACKER'], 'true'),
115-
eq(variables['Build.Reason'], 'Schedule'))
116120
- bash: |
117121
set -ex
118122
if [[ $(BOT_GITHUB_TOKEN) == "" ]]; then
@@ -124,8 +128,8 @@ jobs:
124128
CI_NAME="$SYSTEM_JOBIDENTIFIER"
125129
ISSUE_REPO="$BUILD_REPOSITORY_NAME"
126130
127-
pip install defusedxml PyGithub
128-
python maint_tools/update_tracking_issue.py \
131+
$(pyTools.pythonLocation)/bin/pip install defusedxml PyGithub
132+
$(pyTools.pythonLocation)/bin/python maint_tools/update_tracking_issue.py \
129133
$(BOT_GITHUB_TOKEN) \
130134
$CI_NAME \
131135
$ISSUE_REPO \

‎build_tools/azure/posix.yml

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,15 @@ jobs:
4242
${{ insert }}: ${{ parameters.matrix }}
4343

4444
steps:
45+
- task: UsePythonVersion@0
46+
inputs:
47+
versionSpec: '3.9'
48+
addToPath: false
49+
name: pyTools
50+
displayName: Select python version to run CI python scripts
51+
- bash: $(pyTools.pythonLocation)/bin/python build_tools/azure/get_selected_tests.py
52+
displayName: Check selected tests for all random seeds
53+
condition: eq(variables['Build.Reason'], 'PullRequest')
4554
- bash: echo "##vso[task.prependpath]$CONDA/bin"
4655
displayName: Add conda to PATH
4756
condition: startsWith(variables['DISTRIB'], 'conda')
@@ -65,22 +74,18 @@ jobs:
6574
- script: |
6675
build_tools/azure/test_docs.sh
6776
displayName: 'Test Docs'
77+
condition: eq(variables['SELECTED_TESTS'], '')
6878
- script: |
6979
build_tools/azure/test_pytest_soft_dependency.sh
7080
displayName: 'Test Soft Dependency'
71-
condition: eq(variables['CHECK_PYTEST_SOFT_DEPENDENCY'], 'true')
81+
condition: and(eq(variables['CHECK_PYTEST_SOFT_DEPENDENCY'], 'true'),
82+
eq(variables['SELECTED_TESTS'], ''))
7283
- task: PublishTestResults@2
7384
inputs:
7485
testResultsFiles: '$(TEST_DIR)/$(JUNITXML)'
7586
testRunTitle: ${{ format('{0}-$(Agent.JobName)', parameters.name) }}
7687
displayName: 'Publish Test Results'
7788
condition: succeededOrFailed()
78-
- task: UsePythonVersion@0
79-
inputs:
80-
versionSpec: '3.9'
81-
displayName: Place Python into path to update issue tracker
82-
condition: and(succeededOrFailed(), eq(variables['CREATE_ISSUE_ON_TRACKER'], 'true'),
83-
eq(variables['Build.Reason'], 'Schedule'))
8489
- bash: |
8590
set -ex
8691
if [[ $(BOT_GITHUB_TOKEN) == "" ]]; then
@@ -92,8 +97,8 @@ jobs:
9297
CI_NAME="$SYSTEM_JOBIDENTIFIER"
9398
ISSUE_REPO="$BUILD_REPOSITORY_NAME"
9499
95-
pip install defusedxml PyGithub
96-
python maint_tools/update_tracking_issue.py \
100+
$(pyTools.pythonLocation)/bin/pip install defusedxml PyGithub
101+
$(pyTools.pythonLocation)/bin/python maint_tools/update_tracking_issue.py \
97102
$(BOT_GITHUB_TOKEN) \
98103
$CI_NAME \
99104
$ISSUE_REPO \
@@ -106,7 +111,8 @@ jobs:
106111
eq(variables['Build.Reason'], 'Schedule'))
107112
- script: |
108113
build_tools/azure/upload_codecov.sh
109-
condition: and(succeeded(), eq(variables['COVERAGE'], 'true'))
114+
condition: and(succeeded(), eq(variables['COVERAGE'], 'true'),
115+
eq(variables['SELECTED_TESTS'], ''))
110116
displayName: 'Upload To Codecov'
111117
env:
112118
CODECOV_TOKEN: $(CODECOV_TOKEN)

‎build_tools/azure/test_script.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,13 @@ if [[ "$SHOW_SHORT_SUMMARY" == "true" ]]; then
7070
TEST_CMD="$TEST_CMD -ra"
7171
fi
7272

73+
if [[ "$SELECTED_TESTS" != "" ]]; then
74+
TEST_CMD="$TEST_CMD -k $SELECTED_TESTS"
75+
76+
# Override to make selected tests run on all random seeds
77+
export SKLEARN_TESTS_GLOBAL_RANDOM_SEED="all"
78+
fi
79+
7380
set -x
7481
eval "$TEST_CMD --pyargs sklearn"
7582
set +x

‎build_tools/azure/windows.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ jobs:
2626
${{ insert }}: ${{ parameters.matrix }}
2727

2828
steps:
29+
- bash: python build_tools/azure/get_selected_tests.py
30+
displayName: Check selected tests for all random seeds
31+
condition: eq(variables['Build.Reason'], 'PullRequest')
2932
- bash: echo "##vso[task.prependpath]$CONDA/Scripts"
3033
displayName: Add conda to PATH for 64 bit Python
3134
condition: eq(variables['PYTHON_ARCH'], '64')
@@ -41,7 +44,8 @@ jobs:
4144
- bash: ./build_tools/azure/test_script.sh
4245
displayName: 'Test Library'
4346
- bash: ./build_tools/azure/upload_codecov.sh
44-
condition: and(succeeded(), eq(variables['COVERAGE'], 'true'))
47+
condition: and(succeeded(), eq(variables['COVERAGE'], 'true'),
48+
eq(variables['SELECTED_TESTS'], ''))
4549
displayName: 'Upload To Codecov'
4650
env:
4751
CODECOV_TOKEN: $(CODECOV_TOKEN)

0 commit comments

Comments
 (0)
0