8000 `absltest`: do not fail tests on Python 3.12+ when no tests ran AND: · abseil/abseil-py@16bcbeb · GitHub
[go: up one dir, main page]

Skip to content

Commit 16bcbeb

Browse files
yileicopybara-github
authored andcommitted
absltest: do not fail tests on Python 3.12+ when no tests ran AND:
- Either test filtering is used - Sharding is used and the shard index > 0, i.e. only the first shard will fail when no tests ran on Python 3.12+. Context: Python 3.12 unittest will now fail when no tests ran after the change from python/cpython#102051. Since `absltest` is built on top of `unittest`, it will follow this behavior change in Python 3.12. However, when test filtering is used in `absltest`, often used via `bazel test --test_filter=<my_filter>`, the current user expectation is the `bazel test` command should NOT fail is at least one test ran. Since the test runner here has no visibility of the overall `bazel` invocation, we'll make the test not fail when test filtering is used. This is the existing behavior before Python 3.12. Also test absl-py on Python 3.12. PiperOrigin-RevId: 565190992
1 parent f9281cb commit 16bcbeb

File tree

2 files changed

+82
-23
lines changed

2 files changed

+82
-23
lines changed

.github/workflows/test.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
strategy:
2020
fail-fast: false
2121
matrix:
22-
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
22+
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
2323
os: [ubuntu-latest, macOS-latest, windows-latest]
2424

2525
steps:

absl/testing/absltest.py

Lines changed: 81 additions & 22 deletions
+
Args:
Original file line numberDiff line numberDiff line change
@@ -2317,8 +2317,7 @@ def get_default_xml_output_filename():
23172317
os.path.splitext(os.path.basename(sys.argv[0]))[0] + '.xml')
23182318

23192319

2320-
def _setup_filtering(argv):
2321-
# type: (MutableSequence[Text]) -> None
2320+
def _setup_filtering(argv: MutableSequence[str]) -> bool:
23222321
"""Implements the bazel test filtering protocol.
23232322
23242323
The following environment variable is used in this method:
@@ -2333,16 +2332,20 @@ def _setup_filtering(argv):
23332332
23342333
Args:
23352334
argv: the argv to mutate in-place.
2335+
2336+
Returns:
2337+
Whether test filtering is requested.
23362338
"""
23372339
test_filter = os.environ.get('TESTBRIDGE_TEST_ONLY')
23382340
if argv is None or not test_filter:
2339-
return
2341+
return False
23402342

23412343
filters = shlex.split(test_filter)
23422344
if sys.version_info[:2] >= (3, 7):
23432345
filters = ['-k=' + test_filter for test_filter in filters]
23442346

23452347
argv[1:1] = filters
2348+
return True
23462349

23472350

23482351
def _setup_test_runner_fail_fast(argv):
@@ -2369,8 +2372,9 @@ def _setup_test_runner_fail_fast(argv):
23692372
argv[1:1] = ['--failfast']
23702373

23712374

2372-
def _setup_sharding(custom_loader=None):
2373-
# type: (Optional[unittest.TestLoader]) -> unittest.TestLoader
2375+
def _setup_sharding(
2376+
custom_loader: Optional[unittest.TestLoader] = None,
2377+
) -> Tuple[unittest.TestLoader, Optional[int]]:
23742378
"""Implements the bazel sharding protocol.
23752379
23762380
The following environment variables are used in this method:
@@ -2389,8 +2393,10 @@ def _setup_sharding(custom_loader=None):
23892393
custom_loader: A TestLoader to be made sharded.
23902394
23912395
Returns:
2392-
The test loader for shard-filtering or the standard test loader, depending
2393-
on the sharding environment variables.
2396+
A tuple of ``(test_loader, shard_index)``. ``test_loader`` is for
2397+
shard-filtering or the standard test loader depending on the sharding
2398+
environment variables. ``shard_index`` is the shard index, or ``None`` when
2399+
sharding is not used.
23942400
"""
23952401

23962402
# It may be useful to write the shard file even if the other sharding
@@ -2408,7 +2414,7 @@ def _setup_sharding(custom_loader=None):
24082414
base_loader = custom_loader or TestLoader()
24092415
if 'TEST_TOTAL_SHARDS' not in os.environ:
24102416
# Not using sharding, use the expected test loader.
2411-
return base_loader
2417+
return base_loader, None
24122418

24132419
total_shards = int(os.environ['TEST_TOTAL_SHARDS'])
24142420
shard_index = int(os.environ['TEST_SHARD_INDEX'])
@@ -2437,25 +2443,69 @@ def getShardedTestCaseNames(testCaseClass):
24372443
return [x for x in ordered_names if x in filtered_names]
24382444

24392445
base_loader.getTestCaseNames = getShardedTestCaseNames
2440-
return base_loader
2446+
return base_loader, shard_index
2447+
2448+
2449+
def _run_and_get_tests_result(
2450+
argv: MutableSequence[str],
2451+
args: Sequence[Any],
2452+
kwargs: MutableMapping[str, Any],
2453+
xml_test_runner_class: Type[unittest.TextTestRunner],
2454+
) -> Tuple[unittest.TestResult, bool]:
2455+
"""Same as run_tests, but it doesn't exit.
24412456
2457
2458+
argv: sys.argv with the command-line flags removed from the front, i.e. the
2459+
argv with which :func:`app.run()<absl.app.run>` has called
2460+
``__main__.main``. It is passed to
2461+
``unittest.TestProgram.__init__(argv=)``, which does its own flag parsing.
2462+
It is ignored if kwargs contains an argv entry.
2463+
args: Positional arguments passed through to
2464+
``unittest.TestProgram.__init__``.
2465+
kwargs: Keyword arguments passed through to
2466+
``unittest.TestProgram.__init__``.
2467+
xml_test_runner_class: The type of the test runner class.
24422468
2443-
# pylint: disable=line-too-long
2444-
def _run_and_get_tests_result(argv, args, kwargs, xml_test_runner_class):
2445-
# type: (MutableSequence[Text], Sequence[Any], MutableMapping[Text, Any], Type) -> unittest.TestResult
2446-
# pylint: enable=line-too-long
2447-
"""Same as run_tests, except it returns the result instead of exiting."""
2469+
Returns:
2470+
A tuple of ``(test_result, fail_when_no_tests_ran)``.
2471+
``fail_when_no_tests_ran`` indicates whether the test should fail when
2472+
no tests ran.
2473+
"""
24482474

24492475
# The entry from kwargs overrides argv.
24502476
argv = kwargs.pop('argv', argv)
24512477

2478+
if sys.version_info[:2] >= (3, 12):
2479+
# Python 3.12 unittest changed the behavior from PASS to FAIL in
2480+
# https://github.com/python/cpython/pull/102051. absltest follows this.
2481+
fail_when_no_tests_ran = True
2482+
else:
2483+
# Historically, absltest and unittest before Python 3.12 passes if no tests
2484+
# ran.
2485+
fail_when_no_tests_ran = False
2486+
24522487
# Set up test filtering if requested in environment.
2453-
_setup_filtering(argv)
2488+
if _setup_filtering(argv):
2489+
# When test filtering is requested, ideally we also want to fail when no
2490+
# tests ran. However, the test filters are usually done when running bazel.
2491+
# When you run multiple targets, e.g. `bazel test //my_dir/...
2492+
# --test_filter=MyTest`, you don't necessarily want individual tests to fail
2493+
# because no tests match in that particular target.
2494+
# Due to this use case, we don't fail when test filtering is requested.
2495+
fail_when_no_tests_ran = False
2496+
24542497
# Set up --failfast as requested in environment
24552498
_setup_test_runner_fail_fast(argv)
24562499

24572500
# Shard the (default or custom) loader if sharding is turned on.
2458-
kwargs['testLoader'] = _setup_sharding(kwargs.get('testLoader', None))
2501+
kwargs['testLoader'], shard_index = _setup_sharding(
2502+
kwargs.get('testLoader', None)
2503+
)
2504+
if shard_index is not None and shard_index > 0:
2505+
# When sharding is requested, all the shards except the first one shall not
2506+
# fail when no tests ran. This happens when the shard count is greater than
2507+
# the test case count.
2508+
fail_when_no_tests_ran = False
24592509

24602510
# XML file name is based upon (sorted by priority):
24612511
# --xml_output_file flag, XML_OUTPUT_FILE variable,
@@ -2533,9 +2583,13 @@ def _run_and_get_tests_result(argv, args, kwargs, xml_test_runner_class):
25332583
# on argv, which is sys.argv without the command-line flags.
25342584
kwargs['argv'] = argv
25352585

2586+
# Request unittest.TestProgram to not exit. The exit will be handled by
2587+
# `absltest.run_tests`.
2588+
kwargs['exit'] = False
2589+
25362590
try:
25372591
test_program = unittest.TestProgram(*args, **kwargs)
2538-
return test_program.result
2592+
return test_program.result, fail_when_no_tests_ran
25392593
finally:
25402594
if xml_buffer:
25412595
try:
@@ -2545,9 +2599,11 @@ def _run_and_get_tests_result(argv, args, kwargs, xml_test_runner_class):
25452599
xml_buffer.close()
25462600

25472601

2548-
def run_tests(argv, args, kwargs): # pylint: disable=line-too-long
2549-
# type: (MutableSequence[Text], Sequence[Any], MutableMapping[Text, Any]) -> None
2550-
# pylint: enable=line-too-long
2602+
def run_tests(
2603+
argv: MutableSequence[Text],
2604+
args: Sequence[Any],
2605+
kwargs: MutableMapping[Text, Any],
2606+
) -> None:
25512607
"""Executes a set of Python unit tests.
25522608
25532609
Most users should call absltest.main() instead of run_tests.
@@ -2568,8 +2624,11 @@ def run_tests(argv, args, kwargs): # pylint: disable=line-too-long
25682624
kwargs: Keyword arguments passed through to
25692625
``unittest.TestProgram.__init__``.
25702626
"""
2571-
result = _run_and_get_tests_result(
2572-
argv, args, kwargs, xml_reporter.TextAndXMLTestRunner)
2627+
result, fail_when_no_tests_ran = _run_and_get_tests_result(
2628+
argv, args, kwargs, xml_reporter.TextAndXMLTestRunner
2629+
)
2630+
if fail_when_no_tests_ran and result.testsRun == 0:
2631+
sys.exit(5)
25732632
sys.exit(not result.wasSuccessful())
25742633

25752634

0 commit comments

Comments
 (0)
0