From 5597ec755d44cb005f01601b3c2193f9f56b604f Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Sun, 11 May 2025 12:56:49 +0200 Subject: [PATCH 1/6] CI: use ARRAY_API_TESTS_XFAIL_MARK on CI --- .github/workflows/array-api-tests-dask.yml | 2 ++ .github/workflows/array-api-tests-numpy-1-22.yml | 2 ++ .github/workflows/array-api-tests-numpy-1-26.yml | 2 ++ .github/workflows/array-api-tests-numpy-dev.yml | 2 ++ .github/workflows/array-api-tests-numpy-latest.yml | 2 ++ .github/workflows/array-api-tests-torch.yml | 1 + 6 files changed, 11 insertions(+) diff --git a/.github/workflows/array-api-tests-dask.yml b/.github/workflows/array-api-tests-dask.yml index afc67975..964fb52d 100644 --- a/.github/workflows/array-api-tests-dask.yml +++ b/.github/workflows/array-api-tests-dask.yml @@ -16,3 +16,5 @@ jobs: # the full test suite with at least 200 examples. pytest-extra-args: --max-examples=5 python-versions: '[''3.10'', ''3.13'']' + extra-env-vars: | + ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-1-22.yml b/.github/workflows/array-api-tests-numpy-1-22.yml index d8f60432..1cf6e26d 100644 --- a/.github/workflows/array-api-tests-numpy-1-22.yml +++ b/.github/workflows/array-api-tests-numpy-1-22.yml @@ -10,3 +10,5 @@ jobs: package-version: '== 1.22.*' xfails-file-extra: '-1-22' python-versions: '[''3.10'']' + extra-env-vars: | + ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-1-26.yml b/.github/workflows/array-api-tests-numpy-1-26.yml index 33780760..a2788d2f 100644 --- a/.github/workflows/array-api-tests-numpy-1-26.yml +++ b/.github/workflows/array-api-tests-numpy-1-26.yml @@ -10,3 +10,5 @@ jobs: package-version: '== 1.26.*' xfails-file-extra: '-1-26' python-versions: '[''3.10'', ''3.12'']' + extra-env-vars: | + ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-dev.yml b/.github/workflows/array-api-tests-numpy-dev.yml index d6de1a53..dce0813f 100644 --- a/.github/workflows/array-api-tests-numpy-dev.yml +++ b/.github/workflows/array-api-tests-numpy-dev.yml @@ -10,3 +10,5 @@ jobs: extra-requires: '--pre --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple' xfails-file-extra: '-dev' python-versions: '[''3.11'', ''3.13'']' + extra-env-vars: | + ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-latest.yml b/.github/workflows/array-api-tests-numpy-latest.yml index 4d3667f6..54b21a25 100644 --- a/.github/workflows/array-api-tests-numpy-latest.yml +++ b/.github/workflows/array-api-tests-numpy-latest.yml @@ -8,3 +8,5 @@ jobs: with: package-name: numpy python-versions: '[''3.10'', ''3.13'']' + extra-env-vars: | + ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-torch.yml b/.github/workflows/array-api-tests-torch.yml index ac20df25..4dcb3347 100644 --- a/.github/workflows/array-api-tests-torch.yml +++ b/.github/workflows/array-api-tests-torch.yml @@ -10,4 +10,5 @@ jobs: extra-requires: '--index-url https://download.pytorch.org/whl/cpu' extra-env-vars: | ARRAY_API_TESTS_SKIP_DTYPES=uint16,uint32,uint64 + ARRAY_API_TESTS_XFAIL_MARK=skip python-versions: '[''3.10'', ''3.13'']' From c5e76132cbdca73e315de818eb20ff5401577e1a Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Sun, 11 May 2025 13:00:59 +0200 Subject: [PATCH 2/6] WIP: checkout array_api_tests from a fork/branch --- .github/workflows/array-api-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/array-api-tests.yml b/.github/workflows/array-api-tests.yml index 31bedde6..ff4b8916 100644 --- a/.github/workflows/array-api-tests.yml +++ b/.github/workflows/array-api-tests.yml @@ -56,7 +56,8 @@ jobs: - name: Checkout array-api-tests uses: actions/checkout@v4 with: - repository: data-apis/array-api-tests + repository: ev-br/array-api-tests + ref: skip_xfails submodules: 'true' path: array-api-tests From 46e971172fa6cad3dc6fbceb370654077a36e3d9 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Sun, 11 May 2025 14:12:51 +0200 Subject: [PATCH 3/6] CI: run 500 examples on NumPy and PyTorch; 50 on Dask --- .github/workflows/array-api-tests-dask.yml | 2 +- .github/workflows/array-api-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/array-api-tests-dask.yml b/.github/workflows/array-api-tests-dask.yml index 964fb52d..a60b28a4 100644 --- a/.github/workflows/array-api-tests-dask.yml +++ b/.github/workflows/array-api-tests-dask.yml @@ -14,7 +14,7 @@ jobs: # workflow is barely more than a smoke test, and one should expect extreme # flakiness. Before changes to dask-xfails.txt or dask-skips.txt, please run # the full test suite with at least 200 examples. - pytest-extra-args: --max-examples=5 + pytest-extra-args: --max-examples=50 python-versions: '[''3.10'', ''3.13'']' extra-env-vars: | ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests.yml b/.github/workflows/array-api-tests.yml index ff4b8916..c661fe9b 100644 --- a/.github/workflows/array-api-tests.yml +++ b/.github/workflows/array-api-tests.yml @@ -37,7 +37,7 @@ on: description: Multiline string of environment variables to set for the test run. env: - PYTEST_ARGS: "--max-examples 200 -v -rxXfE --ci ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline --durations 10" + PYTEST_ARGS: "--max-examples 500 -v -rxXfE --ci ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline --durations 20" jobs: tests: From 232e70355c478884a5ab6c64676e73de6c1a167d Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Mon, 12 May 2025 14:18:30 +0200 Subject: [PATCH 4/6] CI: use 4 workers; bump the # of examples to 1000 (np/torch), 200 (dask) --- .github/workflows/array-api-tests-dask.yml | 2 +- .github/workflows/array-api-tests-numpy-1-22.yml | 1 + .github/workflows/array-api-tests-numpy-1-26.yml | 1 + .github/workflows/array-api-tests-numpy-dev.yml | 1 + .github/workflows/array-api-tests-numpy-latest.yml | 1 + .github/workflows/array-api-tests-torch.yml | 1 + .github/workflows/array-api-tests.yml | 3 ++- 7 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/array-api-tests-dask.yml b/.github/workflows/array-api-tests-dask.yml index a60b28a4..ef430d9c 100644 --- a/.github/workflows/array-api-tests-dask.yml +++ b/.github/workflows/array-api-tests-dask.yml @@ -14,7 +14,7 @@ jobs: # workflow is barely more than a smoke test, and one should expect extreme # flakiness. Before changes to dask-xfails.txt or dask-skips.txt, please run # the full test suite with at least 200 examples. - pytest-extra-args: --max-examples=50 + pytest-extra-args: --max-examples=200 -n 4 python-versions: '[''3.10'', ''3.13'']' extra-env-vars: | ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-1-22.yml b/.github/workflows/array-api-tests-numpy-1-22.yml index 1cf6e26d..83d4cf1d 100644 --- a/.github/workflows/array-api-tests-numpy-1-22.yml +++ b/.github/workflows/array-api-tests-numpy-1-22.yml @@ -10,5 +10,6 @@ jobs: package-version: '== 1.22.*' xfails-file-extra: '-1-22' python-versions: '[''3.10'']' + pytest-extra-args: -n 4 extra-env-vars: | ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-1-26.yml b/.github/workflows/array-api-tests-numpy-1-26.yml index a2788d2f..13124644 100644 --- a/.github/workflows/array-api-tests-numpy-1-26.yml +++ b/.github/workflows/array-api-tests-numpy-1-26.yml @@ -10,5 +10,6 @@ jobs: package-version: '== 1.26.*' xfails-file-extra: '-1-26' python-versions: '[''3.10'', ''3.12'']' + pytest-extra-args: -n 4 extra-env-vars: | ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-dev.yml b/.github/workflows/array-api-tests-numpy-dev.yml index dce0813f..dec4c7ae 100644 --- a/.github/workflows/array-api-tests-numpy-dev.yml +++ b/.github/workflows/array-api-tests-numpy-dev.yml @@ -10,5 +10,6 @@ jobs: extra-requires: '--pre --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple' xfails-file-extra: '-dev' python-versions: '[''3.11'', ''3.13'']' + pytest-extra-args: -n 4 extra-env-vars: | ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-numpy-latest.yml b/.github/workflows/array-api-tests-numpy-latest.yml index 54b21a25..65bbc9a2 100644 --- a/.github/workflows/array-api-tests-numpy-latest.yml +++ b/.github/workflows/array-api-tests-numpy-latest.yml @@ -8,5 +8,6 @@ jobs: with: package-name: numpy python-versions: '[''3.10'', ''3.13'']' + pytest-extra-args: -n 4 extra-env-vars: | ARRAY_API_TESTS_XFAIL_MARK=skip diff --git a/.github/workflows/array-api-tests-torch.yml b/.github/workflows/array-api-tests-torch.yml index 4dcb3347..4b4b945e 100644 --- a/.github/workflows/array-api-tests-torch.yml +++ b/.github/workflows/array-api-tests-torch.yml @@ -12,3 +12,4 @@ jobs: ARRAY_API_TESTS_SKIP_DTYPES=uint16,uint32,uint64 ARRAY_API_TESTS_XFAIL_MARK=skip python-versions: '[''3.10'', ''3.13'']' + pytest-extra-args: -n 4 diff --git a/.github/workflows/array-api-tests.yml b/.github/workflows/array-api-tests.yml index c661fe9b..f5352d32 100644 --- a/.github/workflows/array-api-tests.yml +++ b/.github/workflows/array-api-tests.yml @@ -37,7 +37,7 @@ on: description: Multiline string of environment variables to set for the test run. env: - PYTEST_ARGS: "--max-examples 500 -v -rxXfE --ci ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline --durations 20" + PYTEST_ARGS: "--max-examples 1000 -v -rxXfE --ci ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline --durations 20" jobs: tests: @@ -77,6 +77,7 @@ jobs: python -m pip install --upgrade pip python -m pip install '${{ inputs.package-name }} ${{ inputs.package-version }}' ${{ inputs.extra-requires }} python -m pip install -r ${GITHUB_WORKSPACE}/array-api-tests/requirements.txt + python -m pip install pytest-xdist - name: Dump pip environment run: pip freeze From ff0e22a8636c20213758f208d78de7429382e357 Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Mon, 12 May 2025 16:42:01 +0200 Subject: [PATCH 5/6] MAINT: update numpy-1.22 xfails --- numpy-1-22-xfails.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy-1-22-xfails.txt b/numpy-1-22-xfails.txt index cacb95b7..d4022b31 100644 --- a/numpy-1-22-xfails.txt +++ b/numpy-1-22-xfails.txt @@ -133,7 +133,7 @@ array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_sc array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[maximum] array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[copysign] array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[pow] -array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[pow] +array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[minimum] array_api_tests/test_operators_and_elementwise_functions.py::test_where_with_scalars From 12f5ff2e142f498b1088a4b6a13c59d58dfe58ff Mon Sep 17 00:00:00 2001 From: Evgeni Burovski Date: Mon, 12 May 2025 16:42:29 +0200 Subject: [PATCH 6/6] MAINT: remove --ci pytest switch The warning says it's deprecated. --- .github/workflows/array-api-tests.yml | 2 +- numpy-1-22-xfails.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/array-api-tests.yml b/.github/workflows/array-api-tests.yml index f5352d32..a77ee3d0 100644 --- a/.github/workflows/array-api-tests.yml +++ b/.github/workflows/array-api-tests.yml @@ -37,7 +37,7 @@ on: description: Multiline string of environment variables to set for the test run. env: - PYTEST_ARGS: "--max-examples 1000 -v -rxXfE --ci ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline --durations 20" + PYTEST_ARGS: "--max-examples 1000 -v -rxXfE ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline --durations 20" jobs: tests: diff --git a/numpy-1-22-xfails.txt b/numpy-1-22-xfails.txt index d4022b31..e1c4f832 100644 --- a/numpy-1-22-xfails.txt +++ b/numpy-1-22-xfails.txt @@ -134,6 +134,7 @@ array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_sc array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[copysign] array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[pow] array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[minimum] +array_api_tests/test_operators_and_elementwise_functions.py::test_binary_with_scalars_real[atan2] array_api_tests/test_operators_and_elementwise_functions.py::test_where_with_scalars