8000 Pass --fail-rerun option on the main branch (#387) · python/buildmaster-config@0fbab0b · GitHub
[go: up one dir, main page]
Skip to content

Commit 0fbab0b

Browse files
vstinnervstinner
authored
Pass --fail-rerun option on the main branch (#387)
* Treat regrtest exit code 5 (EXITCODE_RERUN_FAIL) as WARNINGS, rather than FAILURE. * Rename ONLY_MASTER_BRANCH to ONLY_MAIN_BRANCH. * Add MAIN_BRANCH_NAME and JUNIT_FILENAME constants. * Always pass the branch to Test() steps. * Test steps use lists for test options, rather than strings. * Remove unused SlowSharedUnixBuild and FreezeBuild factories.
1 parent c76a343 commit 0fbab0b

File tree

6 files changed

+100
-77
lines changed

6 files changed

+100
-77
lines changed

‎master/custom/__init__.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
MAIN_BRANCH_VERSION = "3.13"
2+
CUSTOM_BRANCH_NAME = "custom"
3+
# The Git branch is called "main", but we give it a different name in buildbot.
4+
# See git_branches in master/master.cfg.
5+
MAIN_BRANCH_NAME = "3.x"
6+
JUNIT_FILENAME = "test-results.xml"

‎master/custom/builders.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,7 @@ def get_builders(settings):
244244

245245
# Match builder name (excluding the branch name) of builders that should only
246246
# run on the master and "custom" branches.
247-
ONLY_MASTER_BRANCH = (
247+
ONLY_MAIN_BRANCH = (
248248
"Alpine Linux",
249249
# Cygwin is not supported on 2.7, 3.6, 3.7
250250
"Cygwin",

‎master/custom/factories.py

Lines changed: 61 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99

1010
from buildbot.plugins import util
1111

12+
from . import (MAIN_BRANCH_VERSION, CUSTOM_BRANCH_NAME, MAIN_BRANCH_NAME,
13+
JUNIT_FILENAME)
1214
from .steps import (
1315
Test,
1416
Clean,
@@ -18,8 +20,13 @@
1820
UploadTestResults,
1921
)
2022

21-
main_branch_version = "3.13"
22-
CUSTOM_BRANCH_NAME = "custom"
23+
# Python branches with regrtest --fail-rerun option
24+
# https://github.com/python/cpython/issues/108834
25+
BRANCH_WITH_FAIL_RERUN = (
26+
MAIN_BRANCH_NAME,
27+
#"3.12",
28+
#"3.11",
29+
)
2330

2431
# This (default) timeout is for each individual test file.
2532
# It is a bit more than the default faulthandler timeout in regrtest.py
@@ -40,42 +47,21 @@ def __init__(self, source, *, extra_tags=[], **kwargs):
4047
self.tags = self.factory_tags + extra_tags
4148

4249

43-
class FreezeBuild(TaggedBuildFactory):
44-
buildersuffix = ".freeze" # to get unique directory names on master
45-
test_timeout = None
46-
factory_tags = ["freeze"]
47-
48-
def setup(self, **kwargs):
49-
self.addStep(Configure(command=["./configure", "--prefix", "$(PWD)/target"]))
50-
self.addStep(Compile(command=["make"]))
51-
self.addStep(
52-
ShellCommand(
53-
name="install", description="installing", command=["make", "install"]
54-
)
55-
)
56-
self.addStep(
57-
Test(
58-
command=[
59-
"make",
60-
"-C",
61-
"Tools/freeze/test",
62-
"PYTHON=../../../target/bin/python3",
63-
"OUTDIR=../../../target/freezetest",
64-
]
65-
)
66-
)
67-
68-
6950
##############################################################################
7051
############################### UNIX BUILDS ################################
7152
##############################################################################
7253

7354

55+
def has_option(option, test_options):
56+
# return True for option='-j' and test_options=['-uall', '-j2']
57+
return option in ' '.join(test_options)
58+
59+
7460
class UnixBuild(TaggedBuildFactory):
7561
configureFlags = ["--with-pydebug"]
7662
compile_environ = {}
7763
interpreterFlags = ""
78-
testFlags = "-j2"
64+
testFlags = ["-j2"]
7965
makeTarget = "all"
8066
test_timeout = None
8167
test_environ = {}
@@ -106,22 +92,24 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
10692
Configure(command=configure_cmd, **oot_kwargs)
10793
)
10894
compile = ["make", self.makeTarget]
109-
testopts = self.testFlags
110-
if "-R" not in self.testFlags:
111-
testopts += " --junit-xml test-results.xml"
95+
testopts = list(self.testFlags)
96+
if not has_option("-R", self.testFlags):
97+
testopts.extend(("--junit-xml", JUNIT_FILENAME))
11298
# Timeout for the buildworker process
11399
self.test_timeout = self.test_timeout or TEST_TIMEOUT
114100
# Timeout for faulthandler
115101
faulthandler_timeout = self.test_timeout - 5 * 60
116102
if parallel:
117103
compile = ["make", parallel, self.makeTarget]
118-
testopts = testopts + " " + parallel
119-
if "-j" not in testopts:
120-
testopts = "-j2 " + testopts
104+
testopts.append(parallel)
105+
if not has_option("-j", testopts):
106+
testopts.append("-j2")
107+
if branch in BRANCH_WITH_FAIL_RERUN:
108+
testopts.append("--fail-rerun")
121109
test = [
122110
"make",
123111
"buildbottest",
124-
"TESTOPTS=" + testopts + " ${BUILDBOT_TESTOPTS}",
112+
"TESTOPTS=" + " ".join(testopts) + " ${BUILDBOT_TESTOPTS}",
125113
"TESTPYTHONOPTS=" + self.interpreterFlags,
126114
"TESTTIMEOUT=" + str(faulthandler_timeout),
127115
]
@@ -148,8 +136,8 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
148136
**oot_kwargs
149137
)
150138
)
151-
if branch not in ("3",) and "-R" not in self.testFlags:
152-
filename = "test-results.xml"
139+
if branch not in ("3",) and not has_option("-R", self.testFlags):
140+
filename = JUNIT_FILENAME
153141
if self.build_out_of_tree:
154142
filename = os.path.join(out_of_tree_dir, filename)
155143
self.addStep(UploadTestResults(branch, filename=filename))
@@ -174,7 +162,7 @@ class UnixVintageParserBuild(UnixBuild):
174162

175163
class UnixRefleakBuild(UnixBuild):
176164
buildersuffix = ".refleak"
177-
testFlags = "-R 3:3 -u-cpu"
165+
testFlags = ["-R", "3:3", "-u-cpu"]
178166
# -R 3:3 is supposed to only require timeout x 6, but in practice,
179167
# it's much more slower. Use timeout x 10 to prevent timeout
180168
# caused by --huntrleaks.
@@ -210,8 +198,8 @@ class UnixInstalledBuild(TaggedBuildFactory):
210198
factory_tags = ["installed"]
211199

212200
def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
213-
if branch == "3.x":
214-
branch = main_branch_version
201+
if branch == MAIN_BRANCH_NAME:
202+
branch = MAIN_BRANCH_VERSION
215203
elif branch == "custom":
216204
branch = "3"
217205
installed_python = "./target/bin/python%s" % branch
@@ -224,19 +212,23 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
224212

225213
compile = ["make", self.makeTarget]
226214
install = ["make", self.installTarget]
227-
testopts = self.defaultTestOpts[:]
215+
testopts = list(self.defaultTestOpts)
228216
# Timeout for the buildworker process
229217
self.test_timeout = self.test_timeout or TEST_TIMEOUT
230218
# Timeout for faulthandler
231219
faulthandler_timeout = self.test_timeout - 5 * 60
232-
testopts += [f"--timeout={faulthandler_timeout}"]
220+
testopts.append(f"--timeout={faulthandler_timeout}")
233221
if parallel:
234222
compile = ["make", parallel, self.makeTarget]
235223
install = ["make", parallel, self.installTarget]
236-
testopts = testopts + [parallel]
224+
testopts.append(parallel)
225+
if branch in BRANCH_WITH_FAIL_RERUN:
226+
testopts.append("--fail-rerun")
237227

238-
test = [installed_python] + self.interpreterFlags
239-
test += ["-m", "test.regrtest"] + testopts
228+
test = [installed_python,
229+
*self.interpreterFlags,
230+
"-m", "test",
231+
*testopts]
240232

241233
self.addStep(Compile(command=compile))
242234
self.addStep(Install(command=install))
@@ -362,10 +354,6 @@ class SlowNonDebugUnixBuild(NonDebugUnixBuild):
362354
test_timeout = SLOW_TIMEOUT
363355

364356

365-
class SlowSharedUnixBuild(SharedUnixBuild):
366-
test_timeout = SLOW_TIMEOUT
367-
368-
369357
class SlowUnixInstalledBuild(UnixInstalledBuild):
370358
test_timeout = SLOW_TIMEOUT
371359

@@ -516,7 +504,8 @@ class MacOSArmWithBrewBuild(UnixBuild):
516504
"LDFLAGS=-L/opt/homebrew/lib",
517505
]
518506
# These tests are known to crash on M1 macs (see bpo-45289).
519-
testFlags = UnixBuild.testFlags + " -x test_dbm test_dbm_ndbm test_shelve"
507+
testFlags = [*UnixBuild.testFlags,
508+
"-x", "test_dbm", "test_dbm_ndbm", "test_shelve"]
520509

521510
##############################################################################
522511
############################ WINDOWS BUILDS ################################
@@ -536,12 +525,14 @@ class BaseWindowsBuild(TaggedBuildFactory):
536525

537526
def setup(self, parallel, branch, **kwargs):
538527
build_command = self.build_command + self.buildFlags
539-
test_command = self.test_command + self.testFlags
540-
if "-R" not in self.testFlags:
541-
test_command += [r"--junit-xml", r"test-results.xml"]
528+
test_command = [*self.test_command, *self.testFlags]
529+
if not has_option("-R", self.testFlags):
530+
test_command.extend((r"--junit-xml", JUNIT_FILENAME))
542531
clean_command = self.clean_command + self.cleanFlags
543532
if parallel:
544533
test_command.append(parallel)
534+
if branch in BRANCH_WITH_FAIL_RERUN:
535+
test_command.append("--fail-rerun")
545536
self.addStep(Compile(command=build_command))
546537
self.addStep(
547538
ShellCommand(
@@ -552,9 +543,9 @@ def setup(self, parallel, branch, **kwargs):
552543
)
553544
)
554545
timeout = self.test_timeout if self.test_timeout else TEST_TIMEOUT
555-
test_command += ["--timeout", timeout - (5 * 60)]
546+
test_command.extend(("--timeout", timeout - (5 * 60)))
556547
self.addStep(Test(command=test_command, timeout=timeout))
557-
if branch not in ("3",) and "-R" not in self.testFlags:
548+
if branch not in ("3",) and not has_option("-R", self.testFlags):
558549
self.addStep(UploadTestResults(branch))
559550
self.addStep(Clean(command=clean_command))
560551

@@ -596,7 +587,7 @@ class Windows64BigmemBuild(BaseWindowsBuild):
596587

597588
class Windows64RefleakBuild(Windows64Build):
598589
buildersuffix = ".refleak"
599-
testFlags = ["-p", "x64"] + WindowsRefleakBuild.testFlags
590+
testFlags = ["-p", "x64", *WindowsRefleakBuild.testFlags]
600591
# -R 3:3 is supposed to only require timeout x 6, but in practice,
601592
# it's much more slower. Use timeout x 10 to prevent timeout
602593
# caused by --huntrleaks.
@@ -607,7 +598,7 @@ class Windows64RefleakBuild(Windows64Build):
607598
class Windows64ReleaseBuild(Windows64Build):
608599
buildersuffix = ".nondebug"
609600
buildFlags = Windows64Build.buildFlags + ["-c", "Release"]
610-
testFlags = Windows64Build.testFlags + ["+d"]
601+
testFlags = [*Windows64Build.testFlags, "+d"]
611602
# keep default cleanFlags, both configurations get cleaned
612603
factory_tags = ["win64", "nondebug"]
613604

@@ -622,7 +613,7 @@ class WindowsARM64Build(BaseWindowsBuild):
622613
class WindowsARM64ReleaseBuild(WindowsARM64Build):
623614
buildersuffix = ".nondebug"
624615
buildFlags = WindowsARM64Build.buildFlags + ["-c", "Release"]
625-
testFlags = WindowsARM64Build.testFlags + ["+d"]
616+
testFlags = [*WindowsARM64Build.testFlags, "+d"]
626617
# keep default cleanFlags, both configurations get cleaned
627618
factory_tags = ["win-arm64", "nondebug"]
628619

@@ -714,13 +705,15 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
714705
)
715706
)
716707

717-
testopts = self.testFlags
718-
if "-R" not in self.testFlags:
719-
testopts += " --junit-xml test-results.xml"
708+
testopts = list(self.testFlags)
709+
if not has_option("-R", self.testFlags):
710+
testopts.extend((" --junit-xml", JUNIT_FILENAME))
720711
if parallel:
721-
testopts = testopts + " " + parallel
722-
if "-j" not in testopts:
723-
testopts = "-j2 " + testopts
712+
testopts.append(parallel)
713+
if not has_option("-j", self.testFlags):
714+
testopts.append("-j2")
715+
if branch in BRANCH_WITH_FAIL_RERUN:
716+
testopts.append("--fail-rerun")
724717

725718
# Timeout for the buildworker process
726719
self.test_timeout = self.test_timeout or TEST_TIMEOUT
@@ -730,7 +723,7 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
730723
test = [
731724
"make",
732725
"buildbottest",
733-
"TESTOPTS=" + testopts + " ${BUILDBOT_TESTOPTS}",
726+
"TESTOPTS=" + " ".join(testopts) + " ${BUILDBOT_TESTOPTS}",
734727
"TESTPYTHONOPTS=" + self.interpreterFlags,
735728
"TESTTIMEOUT=" + str(faulthandler_timeout),
736729
]
@@ -767,8 +760,8 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
767760
workdir=oot_host_path,
768761
)
769762
)
770-
if branch not in ("3",) and "-R" not in self.testFlags:
771-
filename = os.path.join(oot_host_path, "test-results.xml")
763+
if branch not in ("3",) and not has_option("-R", self.testFlags):
764+
filename = os.path.join(oot_host_path, JUNIT_FILENAME)
772765
self.addStep(UploadTestResults(branch, filename=filename))
773766
self.addStep(
774767
Clean(

‎master/custom/steps.py

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
import re
22

3-
from buildbot.steps.shell import ShellCommand, Test as BaseTest
43
from buildbot.plugins import steps, util
4+
from buildbot.process.results import SUCCESS, WARNINGS
5+
from buildbot.steps.shell import ShellCommand, Test as BaseTest
56
from buildbot.steps.source.git import Git as _Git
67
from buildbot.steps.source.github import GitHub as _GitHub
78

9+
from . import JUNIT_FILENAME
10+
811

912
class Git(_Git):
1013
# GH-68: If "git clone" fails, mark the whole build as WARNING
@@ -77,6 +80,25 @@ class Test(BaseTest):
7780
# Give SIGTERM 30 seconds to shut things down before SIGKILL.
7881
sigtermTime = 30
7982

83+
# Treat "regrtest --fail-rerun" exit code (5) as WARNINGS
84+
# https://github.com/python/cpython/issues/108834
85+
decodeRC = {
86+
0: SUCCESS,
87+
88+
# Treat --fail-rerun exit code (5) to WARNINGS, when a test failed but
89+
# passed when run again in verbose mode in a fresh process (unstable
90+
# test).
91+
5: WARNINGS, # EXITCODE_RERUN_FAIL
92+
93+
# Any exit code not present in the dictionary is treated as FAILURE.
94+
# So there is no need to map each regrtest exit code to FAILURE.
95+
#
96+
# 2: FAILURE, # EXITCODE_BAD_TEST
97+
# 3: FAILURE, # EXITCODE_ENV_CHANGED
98+
# 4: FAILURE, # EXITCODE_NO_TESTS_RAN
99+
# 130: FAILURE, # EXITCODE_INTERRUPTED
100+
}
101+
80102
def evaluateCommand(self, cmd):
81103
if cmd.didFail():
82104
self.setProperty("test_failed_to_build", True)
@@ -139,7 +161,7 @@ class UploadTestResults(steps.FileUpload):
139161
def _has_the_build_failed(self, build):
140162
return self.getProperty("test_failed_to_build")
141163

142-
def __init__(self, branch, filename="test-results.xml"):
164+
def __init__(self, branch, filename=JUNIT_FILENAME):
143165
super().__init__(
144166
doStepIf=self._has_the_build_failed,
145167
workersrc=filename,

‎master/custom/workers.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66

77
from buildbot.plugins import worker as _worker
88

9+
from custom.factories import MAIN_BRANCH_NAME
10+
911

1012
# By default, the buildmaster sends a simple, non-blocking message to each
1113
# worker every hour. These keepalives ensure that traffic is flowing over the
@@ -232,7 +234,7 @@ def get_workers(settings):
232234
cpw(
233235
name="ware-alpine",
234236
tags=['linux', 'unix', 'alpine', 'docker', 'amd64', 'x86-64'],
235-
branches=['3.x'],
237+
branches=[MAIN_BRANCH_NAME],
236238
),
237239
cpw(
238240
name="ware-freebsd",

0 commit comments

Comments
 (0)
0