From 6144df4c52bd447f5399eb0e0076329cb2f0f259 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Sun, 11 May 2025 12:52:04 +0200 Subject: [PATCH 1/2] ENH: add ARRAY_API_TESTS_XFAIL_MARK to turn xfails into skips --- README.md | 12 ++++++++++++ conftest.py | 18 +++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c97d25e9..5a6d4c9f 100644 --- a/README.md +++ b/README.md @@ -305,6 +305,18 @@ ARRAY_API_TESTS_SKIP_DTYPES=uint16,uint32,uint64 pytest array_api_tests/ Note that skipping certain essential dtypes such as `bool` and the default floating-point dtype is not supported. +#### Turning xfails into skips + +Keeping a large number of ``xfails`` can have drastic effects on the run time. This is due +to the way `hypothesis` works: when it detects a failure, it does a large amount +of work to simplify the failing example. +If the run time of the test suite becomes a problem, you can use the +``ARRAY_API_TESTS_XFAIL_MARK`` environment variable: setting it to ``skip`` skips the +entries from the ``xfail.txt`` file instead of xfailing them. Anecdotally, we saw +speed-ups by a factor of 4-5---which allowed us to use 4-5 larger values of +``--max-examples`` within the same time budget. + + ## Contributing ### Remain in-scope diff --git a/conftest.py b/conftest.py index 2d7dac22..05baebc1 100644 --- a/conftest.py +++ b/conftest.py @@ -144,6 +144,20 @@ def check_id_match(id_, pattern): return False +def get_xfail_mark(): + """Skip or xfail tests from the xfails-file.txt.""" + m = os.environ.get("ARRAY_API_TESTS_XFAIL_MARK", "xfail") + if m == "xfail": + return mark.xfail + elif m == "skip": + return mark.skip + else: + raise ValueError( + f'ARRAY_API_TESTS_XFAIL_MARK value should be one of "skip" or "xfail" ' + f'got {m} instead.' + ) + + def pytest_collection_modifyitems(config, items): # 1. Prepare for iterating over items # ----------------------------------- @@ -187,6 +201,8 @@ def pytest_collection_modifyitems(config, items): # 2. Iterate through items and apply markers accordingly # ------------------------------------------------------ + xfail_mark = get_xfail_mark() + for item in items: markers = list(item.iter_markers()) # skip if specified in skips file @@ -198,7 +214,7 @@ def pytest_collection_modifyitems(config, items): # xfail if specified in xfails file for id_ in xfail_ids: if check_id_match(item.nodeid, id_): - item.add_marker(mark.xfail(reason=f"--xfails-file ({xfails_file})")) + item.add_marker(xfail_mark(reason=f"--xfails-file ({xfails_file})")) xfail_id_matched[id_] = True break # skip if disabled or non-existent extension From 0da70106ec216970dcbbaadbf93ee42388ea3230 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Mon, 12 May 2025 14:51:24 +0200 Subject: [PATCH 2/2] ENH: make MAX_ARRAY_SIZE configurable, lower the default to 1024 Changing the default is via the environment variable ARRAY_API_TESTS_MAX_ARRAY_SIZE. --- README.md | 8 ++++++++ array_api_tests/hypothesis_helpers.py | 3 ++- array_api_tests/test_creation_functions.py | 4 ++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5a6d4c9f..fa17b763 100644 --- a/README.md +++ b/README.md @@ -316,6 +316,14 @@ entries from the ``xfail.txt`` file instead of xfailing them. Anecdotally, we sa speed-ups by a factor of 4-5---which allowed us to use 4-5 larger values of ``--max-examples`` within the same time budget. +#### Limiting the array sizes + +The test suite generates random arrays as inputs to functions it tests. "unvectorized" +tests iterate over elements of arrays, which might be slow. If the run time becomes +a problem, you can limit the maximum number of elements in generated arrays by +setting the environment variable ``ARRAY_API_TESTS_MAX_ARRAY_SIZE`` to the +desired value. By default, it is set to 1024. + ## Contributing diff --git a/array_api_tests/hypothesis_helpers.py b/array_api_tests/hypothesis_helpers.py index 3d1a3ec9..e1df108c 100644 --- a/array_api_tests/hypothesis_helpers.py +++ b/array_api_tests/hypothesis_helpers.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import re from contextlib import contextmanager from functools import wraps @@ -232,7 +233,7 @@ def all_floating_dtypes() -> SearchStrategy[DataType]: lambda i: getattr(xp, i)) # Limit the total size of an array shape -MAX_ARRAY_SIZE = 10000 +MAX_ARRAY_SIZE = int(os.environ.get("ARRAY_API_TESTS_MAX_ARRAY_SIZE", 1024)) # Size to use for 2-dim arrays SQRT_MAX_ARRAY_SIZE = int(math.sqrt(MAX_ARRAY_SIZE)) diff --git a/array_api_tests/test_creation_functions.py b/array_api_tests/test_creation_functions.py index 8c504a2a..1f144c72 100644 --- a/array_api_tests/test_creation_functions.py +++ b/array_api_tests/test_creation_functions.py @@ -499,7 +499,7 @@ def test_meshgrid(dtype, data): shapes = data.draw( st.integers(1, 5).flatmap( lambda n: hh.mutually_broadcastable_shapes( - n, min_dims=1, max_dims=1, max_side=5 + n, min_dims=1, max_dims=1, max_side=4 ) ), label="shapes", @@ -509,7 +509,7 @@ def test_meshgrid(dtype, data): x = data.draw(hh.arrays(dtype=dtype, shape=shape), label=f"x{i}") arrays.append(x) # sanity check - assert math.prod(math.prod(x.shape) for x in arrays) <= hh.MAX_ARRAY_SIZE + # assert math.prod(math.prod(x.shape) for x in arrays) <= hh.MAX_ARRAY_SIZE out = xp.meshgrid(*arrays) for i, x in enumerate(out): ph.assert_dtype("meshgrid", in_dtype=dtype, out_dtype=x.dtype, repr_name=f"out[{i}].dtype")