From 18cd7ba83f254195aadaae775b2c90e7d521eeac Mon Sep 17 00:00:00 2001 From: Aditya Agarwal <50960175+adi611@users.noreply.github.com> Date: Fri, 8 Sep 2023 23:34:00 +0530 Subject: [PATCH 01/13] Update testslurm.yml --- .github/workflows/testslurm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index dd4d153e60..dbd3ce38ae 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -10,7 +10,7 @@ jobs: build: runs-on: ubuntu-latest env: - DOCKER_IMAGE: giovtorres/docker-centos7-slurm:latest + DOCKER_IMAGE: giovtorres/docker-centos7-slurm:21.08.6 steps: - name: Disable etelemetry From d171f6b21fa20d5c2995d1e6f33d8103bed6aefa Mon Sep 17 00:00:00 2001 From: Aditya Agarwal <50960175+adi611@users.noreply.github.com> Date: Fri, 8 Sep 2023 23:42:10 +0530 Subject: [PATCH 02/13] Update testslurm.yml --- .github/workflows/testslurm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index dbd3ce38ae..629287e97d 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -10,7 +10,7 @@ jobs: build: runs-on: ubuntu-latest env: - DOCKER_IMAGE: giovtorres/docker-centos7-slurm:21.08.6 + DOCKER_IMAGE: giovtorres/docker-centos7-slurm:21.08.0 steps: - name: Disable etelemetry From d910c93c53ce1cb29e37a3b0eff2a77939aee9f4 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Mon, 11 Sep 2023 18:57:57 -0400 Subject: [PATCH 03/13] test with new image --- .github/workflows/testslurm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index 629287e97d..2e6a2dc8ac 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -10,7 +10,7 @@ jobs: build: runs-on: ubuntu-latest env: - DOCKER_IMAGE: giovtorres/docker-centos7-slurm:21.08.0 + DOCKER_IMAGE: adi611/docker-centos7-slurm:23.02.1 steps: - name: Disable etelemetry From e286273df9b4f9ba09e29f4653d6c0afea79d813 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Mon, 11 Sep 2023 19:01:31 -0400 Subject: [PATCH 04/13] remove adding cluster in slurm --- .github/workflows/testslurm.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index 2e6a2dc8ac..1e8e092c4d 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -24,10 +24,7 @@ jobs: - name: Display previous jobs with sacct run: | echo "Allowing ports/daemons time to start" && sleep 10 - docker exec slurm bash -c "sacctmgr -i add cluster name=linux \ - && supervisorctl restart slurmdbd \ - && supervisorctl restart slurmctld \ - && sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'" + sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'" docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null if [ $? -ne 0 ]; then echo "Slurm docker image error" From 549986197300a9ab9f5b8f70196462beb0221d28 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Mon, 11 Sep 2023 19:05:08 -0400 Subject: [PATCH 05/13] fix: execute slurm command inside container --- .github/workflows/testslurm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index 1e8e092c4d..6de33f8f93 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -24,7 +24,7 @@ jobs: - name: Display previous jobs with sacct run: | echo "Allowing ports/daemons time to start" && sleep 10 - sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'" + docker exec slurm bash -c "sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'" docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null if [ $? -ne 0 ]; then echo "Slurm docker image error" From 73b1cf3b7a0308d37a162fd08b8d5b8589716d1e Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Mon, 11 Sep 2023 19:25:06 -0400 Subject: [PATCH 06/13] fix: set python 3.9 to be global --- .github/workflows/testslurm.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index 6de33f8f93..0b71597075 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -35,6 +35,7 @@ jobs: docker exec slurm bash -c "echo $NO_ET" docker exec slurm bash -c "ls -la && echo list top level dir" docker exec slurm bash -c "ls -la /pydra && echo list pydra dir" + docker exec slurm bash -c "pyenv global 3.9.16" docker exec slurm bash -c "pip3.9 install --upgrade pip && pip3.9 install -e /pydra[test] && python3.9 -c 'import pydra; print(pydra.__version__)'" - name: Run pytest run: | From 368600cdce008a1ed0b2bc4a8b22140350c559c6 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Mon, 11 Sep 2023 19:30:53 -0400 Subject: [PATCH 07/13] trying running global in all commands. --- .github/workflows/testslurm.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index 0b71597075..a8ba0628f9 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -35,13 +35,12 @@ jobs: docker exec slurm bash -c "echo $NO_ET" docker exec slurm bash -c "ls -la && echo list top level dir" docker exec slurm bash -c "ls -la /pydra && echo list pydra dir" - docker exec slurm bash -c "pyenv global 3.9.16" - docker exec slurm bash -c "pip3.9 install --upgrade pip && pip3.9 install -e /pydra[test] && python3.9 -c 'import pydra; print(pydra.__version__)'" + docker exec slurm bash -c "pyenv global 3.9.16 && pip3.9 install --upgrade pip && pip3.9 install -e /pydra[test] && python3.9 -c 'import pydra; print(pydra.__version__)'" - name: Run pytest run: | - docker exec slurm bash -c "pytest --color=yes -vs -n auto --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'" + docker exec slurm bash -c "pyenv global 3.9.16 && pytest --color=yes -vs -n auto --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'" - name: Upload to codecov run: | - docker exec slurm bash -c "pip3.9 install urllib3==1.26.6" - docker exec slurm bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests" + docker exec slurm bash -c "pyenv global 3.9.16 && pip3.9 install urllib3==1.26.6" + docker exec slurm bash -c "pyenv global 3.9.16 && codecov --root /pydra -f /pydra/cov.xml -F unittests" docker rm -f slurm From 61fb7223200b846deba1e0bd91bad5fe5c98d418 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Mon, 11 Sep 2023 19:39:48 -0400 Subject: [PATCH 08/13] drop version specific call --- .github/workflows/testslurm.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index a8ba0628f9..09d7f719db 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -35,12 +35,12 @@ jobs: docker exec slurm bash -c "echo $NO_ET" docker exec slurm bash -c "ls -la && echo list top level dir" docker exec slurm bash -c "ls -la /pydra && echo list pydra dir" - docker exec slurm bash -c "pyenv global 3.9.16 && pip3.9 install --upgrade pip && pip3.9 install -e /pydra[test] && python3.9 -c 'import pydra; print(pydra.__version__)'" + docker exec slurm bash -c "pip3 install --upgrade pip && pip3 install -e /pydra[test] && python3 -c 'import pydra; print(pydra.__version__)'" - name: Run pytest run: | - docker exec slurm bash -c "pyenv global 3.9.16 && pytest --color=yes -vs -n auto --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'" + docker exec slurm bash -c "pytest --color=yes -vs -n auto --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'" - name: Upload to codecov run: | - docker exec slurm bash -c "pyenv global 3.9.16 && pip3.9 install urllib3==1.26.6" - docker exec slurm bash -c "pyenv global 3.9.16 && codecov --root /pydra -f /pydra/cov.xml -F unittests" + docker exec slurm bash -c "pip3 install urllib3==1.26.6" + docker exec slurm bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests" docker rm -f slurm From 5168155e1a188fadb8906a33ba7482df69586c4e Mon Sep 17 00:00:00 2001 From: Aditya Agarwal <50960175+adi611@users.noreply.github.com> Date: Tue, 12 Sep 2023 22:36:27 +0530 Subject: [PATCH 09/13] remove `-n auto` from pytest command --- .github/workflows/testslurm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index 09d7f719db..68a3dd17d4 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -38,7 +38,7 @@ jobs: docker exec slurm bash -c "pip3 install --upgrade pip && pip3 install -e /pydra[test] && python3 -c 'import pydra; print(pydra.__version__)'" - name: Run pytest run: | - docker exec slurm bash -c "pytest --color=yes -vs -n auto --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'" + docker exec slurm bash -c "pytest --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'" - name: Upload to codecov run: | docker exec slurm bash -c "pip3 install urllib3==1.26.6" From 10548b04718ee3a05c900aaa8edec28d9bc265da Mon Sep 17 00:00:00 2001 From: Aditya Agarwal <50960175+adi611@users.noreply.github.com> Date: Thu, 14 Sep 2023 20:33:14 +0530 Subject: [PATCH 10/13] check for all python versions for the container --- .github/workflows/testslurm.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index 68a3dd17d4..7cc95e24da 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -8,6 +8,10 @@ on: jobs: build: + strategy: + matrix: + python-version: [3.8.16, 3.9.16, 3.10.9, 3.11.1] + fail-fast: false runs-on: ubuntu-latest env: DOCKER_IMAGE: adi611/docker-centos7-slurm:23.02.1 @@ -35,12 +39,13 @@ jobs: docker exec slurm bash -c "echo $NO_ET" docker exec slurm bash -c "ls -la && echo list top level dir" docker exec slurm bash -c "ls -la /pydra && echo list pydra dir" - docker exec slurm bash -c "pip3 install --upgrade pip && pip3 install -e /pydra[test] && python3 -c 'import pydra; print(pydra.__version__)'" + docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}" + docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'" - name: Run pytest run: | docker exec slurm bash -c "pytest --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'" - name: Upload to codecov run: | - docker exec slurm bash -c "pip3 install urllib3==1.26.6" + docker exec slurm bash -c "pip install urllib3==1.26.6" docker exec slurm bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests" docker rm -f slurm From 06276d8b4d5d79b0ee240f87b35eb8e10c9469e3 Mon Sep 17 00:00:00 2001 From: Aditya Agarwal <50960175+adi611@users.noreply.github.com> Date: Fri, 15 Sep 2023 11:34:36 +0530 Subject: [PATCH 11/13] add python version 3.11.5 --- .github/workflows/testslurm.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index 7cc95e24da..1e974cd11b 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -10,7 +10,7 @@ jobs: build: strategy: matrix: - python-version: [3.8.16, 3.9.16, 3.10.9, 3.11.1] + python-version: [3.8.16, 3.9.16, 3.10.9, 3.11.1, 3.11.5] fail-fast: false runs-on: ubuntu-latest env: @@ -39,6 +39,9 @@ jobs: docker exec slurm bash -c "echo $NO_ET" docker exec slurm bash -c "ls -la && echo list top level dir" docker exec slurm bash -c "ls -la /pydra && echo list pydra dir" + if [[ "${{ matrix.python-version }}" == "3.11.5" ]]; then + docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v 3.11.5" + fi docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}" docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'" - name: Run pytest From 12c65d60583fb8957090339b34d5652857901fd7 Mon Sep 17 00:00:00 2001 From: Aditya Agarwal <50960175+adi611@users.noreply.github.com> Date: Fri, 15 Sep 2023 22:40:46 +0530 Subject: [PATCH 12/13] remove python 3.11.1 --- .github/workflows/testslurm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index 1e974cd11b..e4f4bddec2 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -10,7 +10,7 @@ jobs: build: strategy: matrix: - python-version: [3.8.16, 3.9.16, 3.10.9, 3.11.1, 3.11.5] + python-version: [3.8.16, 3.9.16, 3.10.9, 3.11.5] fail-fast: false runs-on: ubuntu-latest env: From afeb70597425d4554f6963d5a969abe6dc70c64b Mon Sep 17 00:00:00 2001 From: Aditya Agarwal <50960175+adi611@users.noreply.github.com> Date: Mon, 18 Sep 2023 01:23:01 +0530 Subject: [PATCH 13/13] exclude python 3.11.1 in `pyproject.toml` refer #697 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 188a219ea8..e40b98f693 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "flit_scm:buildapi" name = "pydra" description = "Pydra dataflow engine" readme = "README.rst" -requires-python = ">=3.8" +requires-python = ">=3.8, !=3.11.1" dependencies = [ "attrs >=19.1.0", "cloudpickle >=2.0.0",