From 01718b3e761ffde4d7198d1fb05a2c1f856335fd Mon Sep 17 00:00:00 2001 From: Mariana Montes Date: Sun, 17 Apr 2022 14:12:42 +0200 Subject: [PATCH 01/94] use cache_dir instead of output_dir --- pydra/engine/core.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 824fdf6869..37d350d1e8 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -1189,21 +1189,22 @@ def _collect_outputs(self): ) return attr.evolve(output, **output_wf) - def create_dotfile(self, type="simple", export=None, name=None): + def create_dotfile(self, type="simple", export=None, name=None, output_dir = None): """creating a graph - dotfile and optionally exporting to other formats""" + outdir = output_dir if output_dir is not None else self.cache_dir if not name: name = f"graph_{self.name}" if type == "simple": for task in self.graph.nodes: self.create_connections(task) dotfile = self.graph.create_dotfile_simple( - outdir=self.output_dir, name=name + outdir=outdir, name=name ) elif type == "nested": for task in self.graph.nodes: self.create_connections(task) dotfile = self.graph.create_dotfile_nested( - outdir=self.output_dir, name=name + outdir=outdir, name=name ) elif type == "detailed": # create connections with detailed=True @@ -1213,7 +1214,7 @@ def create_dotfile(self, type="simple", export=None, name=None): for (wf_out, lf) in self._connections: self.graph.add_edges_description((self.name, wf_out, lf.name, lf.field)) dotfile = self.graph.create_dotfile_detailed( - outdir=self.output_dir, name=name + outdir=outdir, name=name ) else: raise Exception( From 2e04ffc68a23c55aad16c4524165b5876ad0f47a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 17 Apr 2022 12:20:14 +0000 Subject: [PATCH 02/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/core.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 37d350d1e8..64dfd32195 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -1189,7 +1189,7 @@ def _collect_outputs(self): ) return attr.evolve(output, **output_wf) - def create_dotfile(self, type="simple", export=None, name=None, output_dir = None): + def create_dotfile(self, type="simple", export=None, name=None, output_dir=None): """creating a graph - dotfile and optionally exporting to other formats""" outdir = output_dir if output_dir is not None else self.cache_dir if not name: @@ -1197,15 +1197,11 @@ def create_dotfile(self, type="simple", export=None, name=None, output_dir = Non if type == "simple": for task in self.graph.nodes: self.create_connections(task) - dotfile = self.graph.create_dotfile_simple( - outdir=outdir, name=name - ) + dotfile = self.graph.create_dotfile_simple(outdir=outdir, name=name) elif type == "nested": for task in self.graph.nodes: self.create_connections(task) - dotfile = self.graph.create_dotfile_nested( - outdir=outdir, name=name - ) + dotfile = self.graph.create_dotfile_nested(outdir=outdir, name=name) elif type == "detailed": # create connections with detailed=True for task in self.graph.nodes: @@ -1213,9 +1209,7 @@ def create_dotfile(self, type="simple", export=None, name=None, output_dir = Non # adding wf outputs for (wf_out, lf) in self._connections: self.graph.add_edges_description((self.name, wf_out, lf.name, lf.field)) - dotfile = self.graph.create_dotfile_detailed( - outdir=outdir, name=name - ) + dotfile = self.graph.create_dotfile_detailed(outdir=outdir, name=name) else: raise Exception( f"type of the graph can be simple, detailed or nested, " From 572bdc38713d4a714b742c9b76bec97afef70a5e Mon Sep 17 00:00:00 2001 From: rcali21 Date: Sun, 18 Sep 2022 21:29:32 -0700 Subject: [PATCH 03/94] New branch for adding io tracking in PROV --- pydra/engine/audit.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 10fe02bcdb..c061d19a3e 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -185,6 +185,10 @@ def audit_task(self, task): "command": command, "startedAtTime": now(), } + + # new code to be added here for i/o tracking - WIP + + self.audit_message(start_message, AuditFlag.PROV) # add more fields according to BEP208 doc From bdf00af7d2f954fb3b76e8568905796de7765cc2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Sep 2022 04:31:52 +0000 Subject: [PATCH 04/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index c061d19a3e..fa83f86115 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -188,7 +188,6 @@ def audit_task(self, task): # new code to be added here for i/o tracking - WIP - self.audit_message(start_message, AuditFlag.PROV) # add more fields according to BEP208 doc From 11b9f0a111160c16129859b45fa63a196b77c0c4 Mon Sep 17 00:00:00 2001 From: rcali21 Date: Tue, 20 Sep 2022 22:33:17 -0700 Subject: [PATCH 05/94] Initial attempt to include input files by gathering from input_spec --- pydra/engine/audit.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index c061d19a3e..bda891e26b 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -175,15 +175,18 @@ def audit_task(self, task): command = task.cmdline # assume function task else: - # work on changing this to function name command = None + # initial attempt to gather files + in_file = task.input_spec.fields + start_message = { "@id": self.aid, "@type": "task", - "label": label, - "command": command, - "startedAtTime": now(), + "Label": label, + "Command": command, + "StartedAtTime": now(), + "Used": in_file } # new code to be added here for i/o tracking - WIP From 896ffffa5a9a756a371f39dd29948cf165868072 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Sep 2022 05:43:13 +0000 Subject: [PATCH 06/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 5745790c0b..aec590beba 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -177,7 +177,7 @@ def audit_task(self, task): else: command = None - # initial attempt to gather files + # initial attempt to gather files in_file = task.input_spec.fields start_message = { @@ -186,7 +186,7 @@ def audit_task(self, task): "Label": label, "Command": command, "StartedAtTime": now(), - "Used": in_file + "Used": in_file, } # new code to be added here for i/o tracking - WIP From f483b36e3cffc0693af252a6a1a174f5ccb395da Mon Sep 17 00:00:00 2001 From: rcali21 Date: Wed, 12 Oct 2022 07:59:33 -0700 Subject: [PATCH 07/94] First attempt at implementing version query --- pydra/engine/audit.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index aec590beba..dfff31794f 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -170,15 +170,28 @@ def audit_check(self, flag): return self.audit_flags & flag def audit_task(self, task): + import subprocess as sp label = task.name + + print('task input:', task.inputs) if hasattr(task.inputs, "executable"): command = task.cmdline # assume function task else: command = None + # if hasattr(task.inputs, "in_file"): + # input_file = task.inputs.in_file + # else: + # input_file = None + + if command is not None: + software = command.split()[0] + software = software + ' ' + '--version' + # take the first word of command as the name of the executable (this may not always be the case) + version_cmd = sp.run(software, shell=True, check=True, stdout=sp.PIPE).stdout.decode('utf-8').splitlines()[0] + else: + software = None - # initial attempt to gather files - in_file = task.input_spec.fields start_message = { "@id": self.aid, @@ -186,7 +199,8 @@ def audit_task(self, task): "Label": label, "Command": command, "StartedAtTime": now(), - "Used": in_file, + # "Used": input_file, + "AssociatedWith": version_cmd } # new code to be added here for i/o tracking - WIP From a297565b9b541437db39dc1513dcda55d49337fc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Oct 2022 15:00:34 +0000 Subject: [PATCH 08/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index dfff31794f..acd66f6b79 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -171,9 +171,10 @@ def audit_check(self, flag): def audit_task(self, task): import subprocess as sp + label = task.name - - print('task input:', task.inputs) + + print("task input:", task.inputs) if hasattr(task.inputs, "executable"): command = task.cmdline # assume function task @@ -186,13 +187,16 @@ def audit_task(self, task): if command is not None: software = command.split()[0] - software = software + ' ' + '--version' + software = software + " " + "--version" # take the first word of command as the name of the executable (this may not always be the case) - version_cmd = sp.run(software, shell=True, check=True, stdout=sp.PIPE).stdout.decode('utf-8').splitlines()[0] + version_cmd = ( + sp.run(software, shell=True, check=True, stdout=sp.PIPE) + .stdout.decode("utf-8") + .splitlines()[0] + ) else: software = None - start_message = { "@id": self.aid, "@type": "task", @@ -200,7 +204,7 @@ def audit_task(self, task): "Command": command, "StartedAtTime": now(), # "Used": input_file, - "AssociatedWith": version_cmd + "AssociatedWith": version_cmd, } # new code to be added here for i/o tracking - WIP From 475ec9072b6a8824ce75d6272efb30aab46a35cb Mon Sep 17 00:00:00 2001 From: rcali21 Date: Wed, 12 Oct 2022 08:01:42 -0700 Subject: [PATCH 09/94] Deleted print statements --- pydra/engine/audit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index dfff31794f..3533d12adc 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -173,7 +173,7 @@ def audit_task(self, task): import subprocess as sp label = task.name - print('task input:', task.inputs) + if hasattr(task.inputs, "executable"): command = task.cmdline # assume function task From 00c9573165645dbb6197cd464716bbfb27eff4f3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Oct 2022 15:03:35 +0000 Subject: [PATCH 10/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 3b8116ca08..033dcace00 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -171,6 +171,7 @@ def audit_check(self, flag): def audit_task(self, task): import subprocess as sp + label = task.name if hasattr(task.inputs, "executable"): command = task.cmdline From 321c00e9f7d67497ed9c75c89c6f022e1353db6a Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Fri, 14 Oct 2022 22:03:33 -0700 Subject: [PATCH 11/94] Implemented --version call in PROV as well as tests --- pydra/engine/audit.py | 17 +++++++------- pydra/engine/tests/test_task.py | 41 +++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 033dcace00..f94adf6018 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -171,7 +171,6 @@ def audit_check(self, flag): def audit_task(self, task): import subprocess as sp - label = task.name if hasattr(task.inputs, "executable"): command = task.cmdline @@ -187,13 +186,15 @@ def audit_task(self, task): software = command.split()[0] software = software + " " + "--version" # take the first word of command as the name of the executable (this may not always be the case) - version_cmd = ( - sp.run(software, shell=True, check=True, stdout=sp.PIPE) - .stdout.decode("utf-8") - .splitlines()[0] - ) + version_cmd = (sp.run(software, shell=True, stdout=sp.PIPE).stdout.decode("utf-8")) + try: + version_cmd = version_cmd.splitlines()[0] + + except IndexError: + version_cmd = None + else: - software = None + version_cmd = None start_message = { "@id": self.aid, @@ -202,7 +203,7 @@ def audit_task(self, task): "Command": command, "StartedAtTime": now(), # "Used": input_file, - "AssociatedWith": version_cmd, + "AssociatedWith": version_cmd } # new code to be added here for i/o tracking - WIP diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index bf0abbf623..611fd7b77f 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1003,9 +1003,10 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] for file in glob(str(message_path) + "/*.jsonld"): with open(file, "r") as f: data = json.load(f) - if "label" in data: + print(data) + if "Label" in data: json_content.append(True) - assert "testfunc" == data["label"] + assert "testfunc" == data["Label"] assert any(json_content) @@ -1031,16 +1032,46 @@ def test_audit_shellcommandtask(tmpdir): for file in glob(str(message_path) + "/*.jsonld"): with open(file, "r") as f: data = json.load(f) - if "label" in data: + print(data) + if "Label" in data: label_content.append(True) - if "command" in data: + if "Command" in data: command_content.append(True) - assert "ls -l" == data["command"] + assert "ls -l" == data["Command"] print(command_content) assert any(label_content) + + +def test_audit_shellcommandtask_version(tmpdir): + cmd = "mrcat" + shelly = ShellCommandTask( + name="shelly", + executable=cmd, + audit_flags=AuditFlag.PROV, + messengers=FileMessenger(), + ) + + import subprocess as sp + import glob + shelly.cache_dir = tmpdir + shelly() + message_path = tmpdir / shelly.checksum / "messages" + # go through each jsonld file in message_path and check if the label field exists + version_content = [] + for file in glob.glob(str(message_path) + "/*.jsonld"): + with open(file, "r") as f: + data = json.load(f) + if "AssociatedWith" in data: + if "== mrcat 3.0.3 ==" in data["AssociatedWith"]: + version_content.append(True) + + + assert any(version_content) + + def test_audit_prov_messdir_1(tmpdir, use_validator): """customized messenger dir""" From 262494291da3d81057ddd174a1d6746f8b00d47c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 15 Oct 2022 05:03:59 +0000 Subject: [PATCH 12/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 7 +++++-- pydra/engine/tests/test_task.py | 4 +--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index f94adf6018..3a58d8f7f3 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -171,6 +171,7 @@ def audit_check(self, flag): def audit_task(self, task): import subprocess as sp + label = task.name if hasattr(task.inputs, "executable"): command = task.cmdline @@ -186,7 +187,9 @@ def audit_task(self, task): software = command.split()[0] software = software + " " + "--version" # take the first word of command as the name of the executable (this may not always be the case) - version_cmd = (sp.run(software, shell=True, stdout=sp.PIPE).stdout.decode("utf-8")) + version_cmd = sp.run(software, shell=True, stdout=sp.PIPE).stdout.decode( + "utf-8" + ) try: version_cmd = version_cmd.splitlines()[0] @@ -203,7 +206,7 @@ def audit_task(self, task): "Command": command, "StartedAtTime": now(), # "Used": input_file, - "AssociatedWith": version_cmd + "AssociatedWith": version_cmd, } # new code to be added here for i/o tracking - WIP diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 611fd7b77f..23d86c17fd 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1043,8 +1043,6 @@ def test_audit_shellcommandtask(tmpdir): assert any(label_content) - - def test_audit_shellcommandtask_version(tmpdir): cmd = "mrcat" shelly = ShellCommandTask( @@ -1056,6 +1054,7 @@ def test_audit_shellcommandtask_version(tmpdir): import subprocess as sp import glob + shelly.cache_dir = tmpdir shelly() message_path = tmpdir / shelly.checksum / "messages" @@ -1068,7 +1067,6 @@ def test_audit_shellcommandtask_version(tmpdir): if "== mrcat 3.0.3 ==" in data["AssociatedWith"]: version_content.append(True) - assert any(version_content) From eb7b89434682dd9297d63e0748020845d09b2890 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Fri, 14 Oct 2022 22:18:13 -0700 Subject: [PATCH 13/94] Removed 'mrcat' call from test_task.py and replaced with 'less' --- pydra/engine/tests/test_task.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 23d86c17fd..421e386613 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1044,15 +1044,20 @@ def test_audit_shellcommandtask(tmpdir): def test_audit_shellcommandtask_version(tmpdir): - cmd = "mrcat" + import subprocess as sp + version_cmd = (sp.run("less --version", shell=True, + stdout=sp.PIPE).stdout.decode("utf-8")) + version_cmd = version_cmd.splitlines()[0] + cmd = "less" shelly = ShellCommandTask( name="shelly", executable=cmd, + args="test.txt", audit_flags=AuditFlag.PROV, messengers=FileMessenger(), ) - import subprocess as sp + import glob shelly.cache_dir = tmpdir @@ -1064,7 +1069,7 @@ def test_audit_shellcommandtask_version(tmpdir): with open(file, "r") as f: data = json.load(f) if "AssociatedWith" in data: - if "== mrcat 3.0.3 ==" in data["AssociatedWith"]: + if version_cmd in data["AssociatedWith"]: version_content.append(True) assert any(version_content) From d5468419bb0831c88e11942ec299f83db88e1f3c Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Fri, 14 Oct 2022 22:25:25 -0700 Subject: [PATCH 14/94] Removed 'mrcat' call from test_task.py and replaced with 'less' --- pydra/engine/tests/test_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 421e386613..4bd3eade1c 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1052,7 +1052,7 @@ def test_audit_shellcommandtask_version(tmpdir): shelly = ShellCommandTask( name="shelly", executable=cmd, - args="test.txt", + args="test_task.py", audit_flags=AuditFlag.PROV, messengers=FileMessenger(), ) From c0d62ffc7e9ed1f6d2d3e2e5ad300e307e0b1cdf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 15 Oct 2022 05:25:53 +0000 Subject: [PATCH 15/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/tests/test_task.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 4bd3eade1c..a78a6ea45f 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1045,8 +1045,10 @@ def test_audit_shellcommandtask(tmpdir): def test_audit_shellcommandtask_version(tmpdir): import subprocess as sp - version_cmd = (sp.run("less --version", shell=True, - stdout=sp.PIPE).stdout.decode("utf-8")) + + version_cmd = sp.run("less --version", shell=True, stdout=sp.PIPE).stdout.decode( + "utf-8" + ) version_cmd = version_cmd.splitlines()[0] cmd = "less" shelly = ShellCommandTask( @@ -1057,7 +1059,6 @@ def test_audit_shellcommandtask_version(tmpdir): messengers=FileMessenger(), ) - import glob shelly.cache_dir = tmpdir From 712994be211ed6d0023e78a47e5d085c3dd31945 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Fri, 14 Oct 2022 23:15:12 -0700 Subject: [PATCH 16/94] String formatting and flake8 issues resolved. --- pydra/engine/audit.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 3a58d8f7f3..b8b8ac468e 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -184,17 +184,16 @@ def audit_task(self, task): # input_file = None if command is not None: - software = command.split()[0] - software = software + " " + "--version" + cmd_name = command.split()[0] + software = f"{cmd_name} --version" # take the first word of command as the name of the executable (this may not always be the case) - version_cmd = sp.run(software, shell=True, stdout=sp.PIPE).stdout.decode( - "utf-8" - ) + version_cmd = sp.run(software, shell=True, + stdout=sp.PIPE).stdout.decode("utf-8") try: version_cmd = version_cmd.splitlines()[0] except IndexError: - version_cmd = None + version_cmd = f"{cmd_name} -- Version unknown" else: version_cmd = None From 2aff2500a1f4cc8fc9c85aaab1b41f0813494946 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 15 Oct 2022 06:16:47 +0000 Subject: [PATCH 17/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index b8b8ac468e..7457fbe264 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -187,8 +187,9 @@ def audit_task(self, task): cmd_name = command.split()[0] software = f"{cmd_name} --version" # take the first word of command as the name of the executable (this may not always be the case) - version_cmd = sp.run(software, shell=True, - stdout=sp.PIPE).stdout.decode("utf-8") + version_cmd = sp.run(software, shell=True, stdout=sp.PIPE).stdout.decode( + "utf-8" + ) try: version_cmd = version_cmd.splitlines()[0] From 08def18c68d76c74b308bcf02d0fe9cf7510b238 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Fri, 14 Oct 2022 23:19:15 -0700 Subject: [PATCH 18/94] String formatting and flake8 issues resolved(2). --- pydra/engine/audit.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 7457fbe264..405ab0d378 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -186,10 +186,11 @@ def audit_task(self, task): if command is not None: cmd_name = command.split()[0] software = f"{cmd_name} --version" - # take the first word of command as the name of the executable (this may not always be the case) - version_cmd = sp.run(software, shell=True, stdout=sp.PIPE).stdout.decode( - "utf-8" - ) + # take the first word of command as the + # name of the executable + # (this may not always be the case) + version_cmd = sp.run(software, shell=True, + stdout=sp.PIPE).stdout.decode("utf-8") try: version_cmd = version_cmd.splitlines()[0] From 2080652830422699be830b97fb12da753f6b5695 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 15 Oct 2022 06:20:30 +0000 Subject: [PATCH 19/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 405ab0d378..7216128470 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -186,11 +186,12 @@ def audit_task(self, task): if command is not None: cmd_name = command.split()[0] software = f"{cmd_name} --version" - # take the first word of command as the - # name of the executable + # take the first word of command as the + # name of the executable # (this may not always be the case) - version_cmd = sp.run(software, shell=True, - stdout=sp.PIPE).stdout.decode("utf-8") + version_cmd = sp.run(software, shell=True, stdout=sp.PIPE).stdout.decode( + "utf-8" + ) try: version_cmd = version_cmd.splitlines()[0] From 82f340dc0d7df4e1e54c497158d6b9971ba8a4a0 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Sat, 15 Oct 2022 13:43:12 -0700 Subject: [PATCH 20/94] Initial attempt at adding infile tracking --- pydra/engine/audit.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 10fe02bcdb..af51fbfb31 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -177,6 +177,10 @@ def audit_task(self, task): else: # work on changing this to function name command = None + if hasattr(task.inputs, "in_file"): + inputs = task.inputs.in_file + else: + inputs = None start_message = { "@id": self.aid, @@ -184,6 +188,7 @@ def audit_task(self, task): "label": label, "command": command, "startedAtTime": now(), + "Used": inputs } self.audit_message(start_message, AuditFlag.PROV) From 4af794445a920ef69881740cafcc9dd0312bcf29 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Thu, 27 Oct 2022 21:53:43 -0700 Subject: [PATCH 21/94] Added version check to test_audit_task --- pydra/engine/tests/test_task.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index a78a6ea45f..2b28afcd30 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1007,6 +1007,8 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] if "Label" in data: json_content.append(True) assert "testfunc" == data["Label"] + if "AssociatedWith" in data: + assert None == data["AssociatedWith"] assert any(json_content) From 4b9df088f070243610caa991ed27bc25b95e889a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 28 Oct 2022 05:17:59 +0000 Subject: [PATCH 22/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 0a44260a63..f85c5da8ad 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -6,6 +6,7 @@ from ..utils.messenger import send_message, make_message, gen_uuid, now, AuditFlag from .helpers import ensure_list, gather_runtime_info + class Audit: """Handle provenance tracking and resource utilization.""" From 5b05be5243792299042c324a3174d4946cb7025f Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Tue, 1 Nov 2022 20:12:19 -0700 Subject: [PATCH 23/94] Working on adding entity_message --- pydra/engine/audit.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index f85c5da8ad..b4368f9749 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -173,15 +173,21 @@ def audit_task(self, task): import subprocess as sp label = task.name + entity_label = type(label) + if hasattr(task.inputs, "executable"): + print(task.inputs) command = task.cmdline # assume function task else: command = None - # if hasattr(task.inputs, "in_file"): - # input_file = task.inputs.in_file - # else: - # input_file = None + if hasattr(task.inputs, "in_file"): + input_file = task.inputs.in_file + print(task.inputs) + at_location = os.path.abspath(input_file) + else: + at_location = None + input_file = None if command is not None: cmd_name = command.split()[0] @@ -201,6 +207,8 @@ def audit_task(self, task): else: version_cmd = None + + start_message = { "@id": self.aid, "@type": "task", @@ -210,9 +218,18 @@ def audit_task(self, task): "AssociatedWith": version_cmd, } + entity_message = { + "@id": self.aid, + "Label": print(entity_label), + "AtLocation": at_location, + "GeneratedBy": 'test', + "Type": "Task", + "digest": "checksum" + } + # new code to be added here for i/o tracking - WIP self.audit_message(start_message, AuditFlag.PROV) - + self.audit_message(entity_message, AuditFlag.PROV) # add more fields according to BEP208 doc # with every field, check in tests From 1f236f9e510209091417b068c45f9575e5aee6f5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Nov 2022 03:14:32 +0000 Subject: [PATCH 24/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index b4368f9749..aabd11b4f3 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -174,7 +174,7 @@ def audit_task(self, task): label = task.name entity_label = type(label) - + if hasattr(task.inputs, "executable"): print(task.inputs) command = task.cmdline @@ -207,8 +207,6 @@ def audit_task(self, task): else: version_cmd = None - - start_message = { "@id": self.aid, "@type": "task", @@ -222,9 +220,9 @@ def audit_task(self, task): "@id": self.aid, "Label": print(entity_label), "AtLocation": at_location, - "GeneratedBy": 'test', + "GeneratedBy": "test", "Type": "Task", - "digest": "checksum" + "digest": "checksum", } # new code to be added here for i/o tracking - WIP From 162b03b8384356c2a393c323b15440855eb52633 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Wed, 2 Nov 2022 00:07:10 -0700 Subject: [PATCH 25/94] Added GeneratedBy to entity using a path grabber --- pydra/engine/audit.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index aabd11b4f3..d78ea8f2c5 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -174,20 +174,28 @@ def audit_task(self, task): label = task.name entity_label = type(label) - + if hasattr(task.inputs, "executable"): - print(task.inputs) command = task.cmdline # assume function task else: command = None - if hasattr(task.inputs, "in_file"): - input_file = task.inputs.in_file - print(task.inputs) - at_location = os.path.abspath(input_file) - else: - at_location = None - input_file = None + + + # if hasattr(task.inputs, "in_file"): + # input_file = task.in_file + # at_location = os.path.abspath(input_file) + # else: + # at_location = None + # input_file = None + + # Next question for Dorota...Should we check if + # an output is generated by the task, and if so, + # map it to the entity message? (e.g., mrcat input output -- then get the responsible command if ran successfully) GeneratedBy: mrcat + # if hasattr(task.outputs, "out_file"): + + + if command is not None: cmd_name = command.split()[0] @@ -220,7 +228,7 @@ def audit_task(self, task): "@id": self.aid, "Label": print(entity_label), "AtLocation": at_location, - "GeneratedBy": "test", + "GeneratedBy": 'test', "Type": "Task", "digest": "checksum", } From 5a60569ef9944f939678e77dac9b20db8e256de6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Nov 2022 07:09:15 +0000 Subject: [PATCH 26/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index d78ea8f2c5..bea33ac906 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -174,29 +174,25 @@ def audit_task(self, task): label = task.name entity_label = type(label) - + if hasattr(task.inputs, "executable"): command = task.cmdline # assume function task else: command = None - # if hasattr(task.inputs, "in_file"): # input_file = task.in_file # at_location = os.path.abspath(input_file) # else: # at_location = None # input_file = None - + # Next question for Dorota...Should we check if # an output is generated by the task, and if so, # map it to the entity message? (e.g., mrcat input output -- then get the responsible command if ran successfully) GeneratedBy: mrcat # if hasattr(task.outputs, "out_file"): - - - if command is not None: cmd_name = command.split()[0] software = f"{cmd_name} --version" @@ -228,7 +224,7 @@ def audit_task(self, task): "@id": self.aid, "Label": print(entity_label), "AtLocation": at_location, - "GeneratedBy": 'test', + "GeneratedBy": "test", "Type": "Task", "digest": "checksum", } From 1cc91ef4db4fd412d323613135e992b40ac4e951 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Wed, 2 Nov 2022 00:19:15 -0700 Subject: [PATCH 27/94] GeneratedBy populated with path grabber --- pydra/engine/audit.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index bea33ac906..335f6d8416 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -172,6 +172,9 @@ def audit_check(self, flag): def audit_task(self, task): import subprocess as sp + args_list = [ item for item in task.inputs.args if os.path.isfile(item) ] + at_location = args_list[0] + label = task.name entity_label = type(label) From ca9d0ea8ac6c65af486ce2263f55082fb76095a9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Nov 2022 07:19:36 +0000 Subject: [PATCH 28/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 335f6d8416..e3affa22f1 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -172,7 +172,7 @@ def audit_check(self, flag): def audit_task(self, task): import subprocess as sp - args_list = [ item for item in task.inputs.args if os.path.isfile(item) ] + args_list = [item for item in task.inputs.args if os.path.isfile(item)] at_location = args_list[0] label = task.name From d0770b41ff7ef4be009e4f116e7d45c073dd2015 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Thu, 3 Nov 2022 00:30:11 -0700 Subject: [PATCH 29/94] Implemented AtLocation field, tested with input_spec --- pydra/engine/audit.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index e3affa22f1..7034f7acae 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -172,9 +172,6 @@ def audit_check(self, flag): def audit_task(self, task): import subprocess as sp - args_list = [item for item in task.inputs.args if os.path.isfile(item)] - at_location = args_list[0] - label = task.name entity_label = type(label) @@ -184,12 +181,12 @@ def audit_task(self, task): else: command = None - # if hasattr(task.inputs, "in_file"): - # input_file = task.in_file - # at_location = os.path.abspath(input_file) - # else: - # at_location = None - # input_file = None + if hasattr(task.inputs, "in_file"): + input_file = task.inputs.in_file + at_location = os.path.abspath(input_file) + else: + at_location = None + input_file = None # Next question for Dorota...Should we check if # an output is generated by the task, and if so, @@ -227,9 +224,9 @@ def audit_task(self, task): "@id": self.aid, "Label": print(entity_label), "AtLocation": at_location, - "GeneratedBy": "test", + "GeneratedBy": "test", # if not part of workflow, this will be none "Type": "Task", - "digest": "checksum", + "digest": "checksum" # hash value under helpers.py } # new code to be added here for i/o tracking - WIP From 34624665e8ab62cd4d9d411c606142081d66200e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 3 Nov 2022 07:33:24 +0000 Subject: [PATCH 30/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 7034f7acae..20af83f985 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -224,9 +224,9 @@ def audit_task(self, task): "@id": self.aid, "Label": print(entity_label), "AtLocation": at_location, - "GeneratedBy": "test", # if not part of workflow, this will be none + "GeneratedBy": "test", # if not part of workflow, this will be none "Type": "Task", - "digest": "checksum" # hash value under helpers.py + "digest": "checksum", # hash value under helpers.py } # new code to be added here for i/o tracking - WIP From 096cbb4ae93bc5952f3481a4fc2db874913474dc Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Fri, 4 Nov 2022 00:10:45 -0700 Subject: [PATCH 31/94] Updated tests --- pydra/engine/audit.py | 4 ++-- pydra/engine/tests/test_task.py | 17 +++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 20af83f985..f7f100ff16 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -214,7 +214,7 @@ def audit_task(self, task): start_message = { "@id": self.aid, "@type": "task", - "Label": label, + "Activity_Label": label, "Command": command, "StartedAtTime": now(), "AssociatedWith": version_cmd, @@ -222,7 +222,7 @@ def audit_task(self, task): entity_message = { "@id": self.aid, - "Label": print(entity_label), + "Entity_Label": print(entity_label), "AtLocation": at_location, "GeneratedBy": "test", # if not part of workflow, this will be none "Type": "Task", diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 2b28afcd30..7b5dceca1f 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -993,6 +993,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] return a + b from glob import glob + import json funky = testfunc(a=2, audit_flags=AuditFlag.PROV, messengers=FileMessenger()) funky.cache_dir = tmpdir @@ -1000,15 +1001,19 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] message_path = tmpdir / funky.checksum / "messages" # go through each jsonld file in message_path and check if the label field exists json_content = [] + for file in glob(str(message_path) + "/*.jsonld"): with open(file, "r") as f: data = json.load(f) print(data) - if "Label" in data: + if "Activity_Label" in data: json_content.append(True) - assert "testfunc" == data["Label"] - if "AssociatedWith" in data: - assert None == data["AssociatedWith"] + assert data["Activity_Label"] == "testfunc" + if "Entity_Label" in data: + assert data["Entity_Label"] == None + if "AssociatedWith" in data: + assert None == data["AssociatedWith"] + assert any(json_content) @@ -1035,7 +1040,7 @@ def test_audit_shellcommandtask(tmpdir): with open(file, "r") as f: data = json.load(f) print(data) - if "Label" in data: + if "Activity_Label" in data: label_content.append(True) if "Command" in data: command_content.append(True) @@ -1174,7 +1179,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] from glob import glob assert len(glob(str(tmpdir / funky.checksum / "proc*.log"))) == 1 - assert len(glob(str(message_path / "*.jsonld"))) == 7 + assert len(glob(str(message_path / "*.jsonld"))) == 8 # commented out to speed up testing collect_messages(tmpdir / funky.checksum, message_path, ld_op="compact") From ae543aed7b18f9d6665439a21770572f7fb7c32b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Nov 2022 07:11:12 +0000 Subject: [PATCH 32/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/tests/test_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 7b5dceca1f..7a76893ba6 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1001,7 +1001,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] message_path = tmpdir / funky.checksum / "messages" # go through each jsonld file in message_path and check if the label field exists json_content = [] - + for file in glob(str(message_path) + "/*.jsonld"): with open(file, "r") as f: data = json.load(f) From ef2505528a24c5c68d4830baed44f0b28a1e05c9 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Fri, 4 Nov 2022 00:14:51 -0700 Subject: [PATCH 33/94] Removed unused import --- pydra/engine/tests/test_task.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 7a76893ba6..bb99b50f22 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -993,7 +993,6 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] return a + b from glob import glob - import json funky = testfunc(a=2, audit_flags=AuditFlag.PROV, messengers=FileMessenger()) funky.cache_dir = tmpdir From 70f15eed8143c7f95d08c6870128b367058612a6 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Fri, 4 Nov 2022 00:17:21 -0700 Subject: [PATCH 34/94] flake8 --- pydra/engine/audit.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index f7f100ff16..3debd8b43d 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -188,11 +188,6 @@ def audit_task(self, task): at_location = None input_file = None - # Next question for Dorota...Should we check if - # an output is generated by the task, and if so, - # map it to the entity message? (e.g., mrcat input output -- then get the responsible command if ran successfully) GeneratedBy: mrcat - # if hasattr(task.outputs, "out_file"): - if command is not None: cmd_name = command.split()[0] software = f"{cmd_name} --version" From 7ab3ddfd8425c74a99c3eb6a71a5ed7b17d0041b Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Sun, 6 Nov 2022 22:55:54 -0800 Subject: [PATCH 35/94] Updated tests, accounting for duplicate key names --- pydra/engine/tests/test_task.py | 41 ++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index bb99b50f22..9229518683 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -998,22 +998,29 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] funky.cache_dir = tmpdir funky() message_path = tmpdir / funky.checksum / "messages" + print(message_path) # go through each jsonld file in message_path and check if the label field exists json_content = [] for file in glob(str(message_path) + "/*.jsonld"): with open(file, "r") as f: data = json.load(f) - print(data) - if "Activity_Label" in data: - json_content.append(True) - assert data["Activity_Label"] == "testfunc" - if "Entity_Label" in data: - assert data["Entity_Label"] == None + if "@type" in data: + if "AssociatedWith" in data: + assert "testfunc" in data["Label"] + + + + if "@type" in data: + if data["@type"] == "input": + assert None == data["Label"] + + #assert data["Type"] == "input" + if "AssociatedWith" in data: assert None == data["AssociatedWith"] - assert any(json_content) + # assert any(json_content) def test_audit_shellcommandtask(tmpdir): @@ -1032,21 +1039,29 @@ def test_audit_shellcommandtask(tmpdir): shelly() message_path = tmpdir / shelly.checksum / "messages" # go through each jsonld file in message_path and check if the label field exists - label_content = [] + command_content = [] for file in glob(str(message_path) + "/*.jsonld"): with open(file, "r") as f: data = json.load(f) - print(data) - if "Activity_Label" in data: - label_content.append(True) + if "@type" in data: + if "AssociatedWith" in data: + assert "shelly" in data["Label"] + + + + if "@type" in data: + if data["@type"] == "input": + assert None == data["Label"] + + if "Command" in data: command_content.append(True) assert "ls -l" == data["Command"] - print(command_content) - assert any(label_content) + + assert any(command_content) def test_audit_shellcommandtask_version(tmpdir): From c4f21afb2e600a6e7c56cc17a53673c2b64683e5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Nov 2022 06:56:15 +0000 Subject: [PATCH 36/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/tests/test_task.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 9229518683..a053900189 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1009,13 +1009,11 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] if "AssociatedWith" in data: assert "testfunc" in data["Label"] - - if "@type" in data: if data["@type"] == "input": assert None == data["Label"] - - #assert data["Type"] == "input" + + # assert data["Type"] == "input" if "AssociatedWith" in data: assert None == data["AssociatedWith"] @@ -1039,7 +1037,7 @@ def test_audit_shellcommandtask(tmpdir): shelly() message_path = tmpdir / shelly.checksum / "messages" # go through each jsonld file in message_path and check if the label field exists - + command_content = [] for file in glob(str(message_path) + "/*.jsonld"): @@ -1049,18 +1047,14 @@ def test_audit_shellcommandtask(tmpdir): if "AssociatedWith" in data: assert "shelly" in data["Label"] - - if "@type" in data: if data["@type"] == "input": assert None == data["Label"] - if "Command" in data: command_content.append(True) assert "ls -l" == data["Command"] - assert any(command_content) From fd625a9e6bcb4c761f18d244385178b94fc9f79f Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Sun, 6 Nov 2022 23:11:43 -0800 Subject: [PATCH 37/94] Updated tests, accounting for duplicate key names --- pydra/engine/tests/test_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index a053900189..4c5ff1be4d 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1043,8 +1043,8 @@ def test_audit_shellcommandtask(tmpdir): for file in glob(str(message_path) + "/*.jsonld"): with open(file, "r") as f: data = json.load(f) - if "@type" in data: - if "AssociatedWith" in data: + if "AssociatedWith" in data: + if "@type" in data: assert "shelly" in data["Label"] if "@type" in data: From 4d447d198aabfa379c7da5c3798c7dd0d627ef05 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Sun, 6 Nov 2022 23:14:39 -0800 Subject: [PATCH 38/94] Fixed test --- pydra/engine/audit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 3debd8b43d..db8b23c423 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -209,7 +209,7 @@ def audit_task(self, task): start_message = { "@id": self.aid, "@type": "task", - "Activity_Label": label, + "Label": label, "Command": command, "StartedAtTime": now(), "AssociatedWith": version_cmd, @@ -217,10 +217,10 @@ def audit_task(self, task): entity_message = { "@id": self.aid, - "Entity_Label": print(entity_label), + "Label": print(entity_label), "AtLocation": at_location, "GeneratedBy": "test", # if not part of workflow, this will be none - "Type": "Task", + "@type": "input", "digest": "checksum", # hash value under helpers.py } From 58e47679d1b8212eb78b9ded0a53346ec7e24439 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Sun, 6 Nov 2022 23:22:04 -0800 Subject: [PATCH 39/94] Fixed tests --- pydra/engine/tests/test_task.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 4c5ff1be4d..85bc07d3ee 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1043,13 +1043,14 @@ def test_audit_shellcommandtask(tmpdir): for file in glob(str(message_path) + "/*.jsonld"): with open(file, "r") as f: data = json.load(f) - if "AssociatedWith" in data: - if "@type" in data: + + if "@type" in data: + if "AssociatedWith" in data: assert "shelly" in data["Label"] if "@type" in data: if data["@type"] == "input": - assert None == data["Label"] + assert data["Label"] == None if "Command" in data: command_content.append(True) From e355f2121f39e9173c3a1041b2ea6621b65c4773 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Nov 2022 07:22:25 +0000 Subject: [PATCH 40/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/tests/test_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 85bc07d3ee..9368a6a27d 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1043,7 +1043,7 @@ def test_audit_shellcommandtask(tmpdir): for file in glob(str(message_path) + "/*.jsonld"): with open(file, "r") as f: data = json.load(f) - + if "@type" in data: if "AssociatedWith" in data: assert "shelly" in data["Label"] From a8c3a371954014ad2ab2962316879ac6a988d6bc Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Mon, 7 Nov 2022 10:40:14 -0800 Subject: [PATCH 41/94] Added atlocation in tests to meet coverage --- pydra/engine/tests/test_task.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 9368a6a27d..23e7e210e2 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1012,6 +1012,9 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] if "@type" in data: if data["@type"] == "input": assert None == data["Label"] + # placeholder for atlocation until + # new test is added + assert None == data["AtLocation"] # assert data["Type"] == "input" From 22ef750562fa44691add9828b56812e36d53a5f1 Mon Sep 17 00:00:00 2001 From: Antoine Blachair Date: Wed, 16 Nov 2022 10:12:10 +1100 Subject: [PATCH 42/94] Add unit test to check task_inputs_mandatory_with_xOR --- pydra/engine/tests/test_specs.py | 47 +++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/pydra/engine/tests/test_specs.py b/pydra/engine/tests/test_specs.py index a0af757546..33a6061ac3 100644 --- a/pydra/engine/tests/test_specs.py +++ b/pydra/engine/tests/test_specs.py @@ -14,10 +14,15 @@ DockerSpec, SingularitySpec, LazyField, + ShellOutSpec, +) +from ..task import ( + TaskBase, + ShellCommandTask ) from ..helpers import make_klass import pytest - +import attr def test_basespec(): spec = BaseSpec() @@ -366,3 +371,43 @@ def test_input_file_hash_5(tmpdir): f.write("hi") hash3 = inputs(in_file=[{"file": file_diffcontent, "int": 3}]).hash assert hash1 != hash3 + + + +def test_task_inputs_mandatory_with_xOR(): + """input spec with mandatory inputs""" + input_fields=[ + ( + "input_1", + bool, + { + "help_string": "help", + "argstr": "--i1", + "xor": ("input_1", "input_2"), + } + ), + ( + "input_2", + bool, + { + "help_string": "help", + "mandatory": True, + "argstr": "--i2", + "xor": ("input_1", "input_2"), + } + ) + ] + task_input_spec = SpecInfo(name="Input", fields=input_fields, bases=(ShellSpec,)) + task_output_fields = [] + task_output_spec = SpecInfo(name="Output", fields=task_output_fields, bases=(ShellOutSpec,)) + + class MyTask(ShellCommandTask): + input_spec = task_input_spec + output_spec = task_output_spec + executable = "task" + + task = MyTask() + task.inputs.input_1 = True + task.inputs.input_2 = True + task.inputs.check_fields_input_spec() + #task.cmdline \ No newline at end of file From c91b4534e62e315bcef1c88e3916c4509e08eee9 Mon Sep 17 00:00:00 2001 From: Antoine Blachair Date: Thu, 17 Nov 2022 10:35:20 +1100 Subject: [PATCH 43/94] Add another unit test to check both conditions --- pydra/engine/tests/test_specs.py | 44 +++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/pydra/engine/tests/test_specs.py b/pydra/engine/tests/test_specs.py index 33a6061ac3..ccc1b26d86 100644 --- a/pydra/engine/tests/test_specs.py +++ b/pydra/engine/tests/test_specs.py @@ -374,7 +374,7 @@ def test_input_file_hash_5(tmpdir): -def test_task_inputs_mandatory_with_xOR(): +def test_task_inputs_mandatory_with_xOR_TF(): """input spec with mandatory inputs""" input_fields=[ ( @@ -382,7 +382,44 @@ def test_task_inputs_mandatory_with_xOR(): bool, { "help_string": "help", - "argstr": "--i1", + "mandatory": True, + "xor": ("input_1", "input_2"), + } + ), + ( + "input_2", + bool, + { + "help_string": "help", + "mandatory": True, + "argstr": "--i2", + "xor": ("input_1", "input_2"), + } + ) + ] + task_input_spec = SpecInfo(name="Input", fields=input_fields, bases=(ShellSpec,)) + task_output_fields = [] + task_output_spec = SpecInfo(name="Output", fields=task_output_fields, bases=(ShellOutSpec,)) + + class MyTask(ShellCommandTask): + input_spec = task_input_spec + output_spec = task_output_spec + executable = "cmd" + + task = MyTask() + task.inputs.input_1 = True + task.inputs.input_2 = attr.NOTHING + task.inputs.check_fields_input_spec() + +def test_task_inputs_mandatory_with_xOR_TT(): + """input spec with mandatory inputs""" + input_fields=[ + ( + "input_1", + bool, + { + "help_string": "help", + "mandatory": True, "xor": ("input_1", "input_2"), } ), @@ -404,10 +441,9 @@ def test_task_inputs_mandatory_with_xOR(): class MyTask(ShellCommandTask): input_spec = task_input_spec output_spec = task_output_spec - executable = "task" + executable = "cmd" task = MyTask() task.inputs.input_1 = True task.inputs.input_2 = True task.inputs.check_fields_input_spec() - #task.cmdline \ No newline at end of file From 63399a9074b78a477c8111bebe6cc1a0de10b3b3 Mon Sep 17 00:00:00 2001 From: Antoine Blachair Date: Thu, 17 Nov 2022 11:37:29 +1100 Subject: [PATCH 44/94] Refactor test --- pydra/engine/tests/test_specs.py | 61 +++++++++++--------------------- 1 file changed, 21 insertions(+), 40 deletions(-) diff --git a/pydra/engine/tests/test_specs.py b/pydra/engine/tests/test_specs.py index ccc1b26d86..59684d485d 100644 --- a/pydra/engine/tests/test_specs.py +++ b/pydra/engine/tests/test_specs.py @@ -372,10 +372,7 @@ def test_input_file_hash_5(tmpdir): hash3 = inputs(in_file=[{"file": file_diffcontent, "int": 3}]).hash assert hash1 != hash3 - - -def test_task_inputs_mandatory_with_xOR_TF(): - """input spec with mandatory inputs""" +class SimpleTask(ShellCommandTask): input_fields=[ ( "input_1", @@ -401,49 +398,33 @@ def test_task_inputs_mandatory_with_xOR_TF(): task_output_fields = [] task_output_spec = SpecInfo(name="Output", fields=task_output_fields, bases=(ShellOutSpec,)) - class MyTask(ShellCommandTask): - input_spec = task_input_spec - output_spec = task_output_spec - executable = "cmd" + input_spec = task_input_spec + output_spec = task_output_spec + executable = "cmd" - task = MyTask() + +def test_task_inputs_mandatory_with_xOR_one_mandatory_is_enough(): + """input spec with mandatory inputs""" + task = SimpleTask() task.inputs.input_1 = True task.inputs.input_2 = attr.NOTHING task.inputs.check_fields_input_spec() -def test_task_inputs_mandatory_with_xOR_TT(): +def test_task_inputs_mandatory_with_xOR_zero_mandatory_raises_error(): """input spec with mandatory inputs""" - input_fields=[ - ( - "input_1", - bool, - { - "help_string": "help", - "mandatory": True, - "xor": ("input_1", "input_2"), - } - ), - ( - "input_2", - bool, - { - "help_string": "help", - "mandatory": True, - "argstr": "--i2", - "xor": ("input_1", "input_2"), - } - ) - ] - task_input_spec = SpecInfo(name="Input", fields=input_fields, bases=(ShellSpec,)) - task_output_fields = [] - task_output_spec = SpecInfo(name="Output", fields=task_output_fields, bases=(ShellOutSpec,)) + task = SimpleTask() + task.inputs.input_1 = attr.NOTHING + task.inputs.input_2 = attr.NOTHING + task.inputs.check_fields_input_spec() - class MyTask(ShellCommandTask): - input_spec = task_input_spec - output_spec = task_output_spec - executable = "cmd" - task = MyTask() +def test_task_inputs_mandatory_with_xOR_two_mandatories_raises_error(): + """input spec with mandatory inputs""" + task = SimpleTask() task.inputs.input_1 = True task.inputs.input_2 = True - task.inputs.check_fields_input_spec() + + with pytest.raises(Exception) as excinfo: + task.inputs.check_fields_input_spec() + assert "input_2 is mutually exclusive with ('input_1', 'input_2')" in str(excinfo.value) + assert excinfo.type is AttributeError From e412f103b7774715969ada6116eb8ff61f8e8f5d Mon Sep 17 00:00:00 2001 From: Antoine Blachair Date: Thu, 17 Nov 2022 13:05:21 +1100 Subject: [PATCH 45/94] Change test input type --- pydra/engine/tests/test_specs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pydra/engine/tests/test_specs.py b/pydra/engine/tests/test_specs.py index 59684d485d..21dac4c3c9 100644 --- a/pydra/engine/tests/test_specs.py +++ b/pydra/engine/tests/test_specs.py @@ -376,7 +376,7 @@ class SimpleTask(ShellCommandTask): input_fields=[ ( "input_1", - bool, + str, { "help_string": "help", "mandatory": True, @@ -403,10 +403,10 @@ class SimpleTask(ShellCommandTask): executable = "cmd" -def test_task_inputs_mandatory_with_xOR_one_mandatory_is_enough(): +def test_task_inputs_mandatory_with_xOR_one_mandatory_is_OK(): """input spec with mandatory inputs""" task = SimpleTask() - task.inputs.input_1 = True + task.inputs.input_1 = 'Input1' task.inputs.input_2 = attr.NOTHING task.inputs.check_fields_input_spec() @@ -421,7 +421,7 @@ def test_task_inputs_mandatory_with_xOR_zero_mandatory_raises_error(): def test_task_inputs_mandatory_with_xOR_two_mandatories_raises_error(): """input spec with mandatory inputs""" task = SimpleTask() - task.inputs.input_1 = True + task.inputs.input_1 = 'Input1' task.inputs.input_2 = True with pytest.raises(Exception) as excinfo: From 96d8d0f76c5df9a7cdfca1dcda6cedafcff3f96d Mon Sep 17 00:00:00 2001 From: Antoine Blachair Date: Thu, 17 Nov 2022 15:37:06 +1100 Subject: [PATCH 46/94] Add check for 'xor' concurrently with 'mandatory' --- pydra/engine/specs.py | 13 +++++++--- pydra/engine/tests/test_specs.py | 42 ++++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 554394d15e..249ceea138 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -162,10 +162,15 @@ def check_fields_input_spec(self): mdata = fld.metadata # checking if the mandatory field is provided if getattr(self, fld.name) is attr.NOTHING: - if mdata.get("mandatory"): - raise AttributeError( - f"{fld.name} is mandatory, but no value provided" - ) + if mdata.get("mandatory"): + # checking if the mandatory field is provided elsewhere in the xor list + alreday_populated = [getattr(self, el) for el in mdata["xor"] if (getattr(self, el) is not attr.NOTHING)] + if alreday_populated: #another input satisfies mandatory attribute via xor condition + continue + else: + raise AttributeError( + f"{fld.name} is mandatory, but no value provided" + ) else: continue names.append(fld.name) diff --git a/pydra/engine/tests/test_specs.py b/pydra/engine/tests/test_specs.py index 21dac4c3c9..8dc0a9cf39 100644 --- a/pydra/engine/tests/test_specs.py +++ b/pydra/engine/tests/test_specs.py @@ -380,7 +380,7 @@ class SimpleTask(ShellCommandTask): { "help_string": "help", "mandatory": True, - "xor": ("input_1", "input_2"), + "xor": ("input_1", "input_2", "input_3"), } ), ( @@ -390,9 +390,18 @@ class SimpleTask(ShellCommandTask): "help_string": "help", "mandatory": True, "argstr": "--i2", - "xor": ("input_1", "input_2"), + "xor": ("input_1", "input_2", "input_3"), } - ) + ), + ( + "input_3", + bool, + { + "help_string": "help", + "mandatory": True, + "xor": ("input_1", "input_2", "input_3"), + } + ) ] task_input_spec = SpecInfo(name="Input", fields=input_fields, bases=(ShellSpec,)) task_output_fields = [] @@ -410,13 +419,24 @@ def test_task_inputs_mandatory_with_xOR_one_mandatory_is_OK(): task.inputs.input_2 = attr.NOTHING task.inputs.check_fields_input_spec() -def test_task_inputs_mandatory_with_xOR_zero_mandatory_raises_error(): +def test_task_inputs_mandatory_with_xOR_one_mandatory_out_3_is_OK(): """input spec with mandatory inputs""" task = SimpleTask() task.inputs.input_1 = attr.NOTHING task.inputs.input_2 = attr.NOTHING + task.inputs.input_3 = True task.inputs.check_fields_input_spec() +def test_task_inputs_mandatory_with_xOR_zero_mandatory_raises_error(): + """input spec with mandatory inputs""" + task = SimpleTask() + task.inputs.input_1 = attr.NOTHING + task.inputs.input_2 = attr.NOTHING + with pytest.raises(Exception) as excinfo: + task.inputs.check_fields_input_spec() + assert "input_1 is mandatory, but no value provided" in str(excinfo.value) + assert excinfo.type is AttributeError + def test_task_inputs_mandatory_with_xOR_two_mandatories_raises_error(): """input spec with mandatory inputs""" @@ -426,5 +446,17 @@ def test_task_inputs_mandatory_with_xOR_two_mandatories_raises_error(): with pytest.raises(Exception) as excinfo: task.inputs.check_fields_input_spec() - assert "input_2 is mutually exclusive with ('input_1', 'input_2')" in str(excinfo.value) + assert "input_2 is mutually exclusive with ('input_1', 'input_2'" in str(excinfo.value) + assert excinfo.type is AttributeError + +def test_task_inputs_mandatory_with_xOR_3_mandatories_raises_error(): + """input spec with mandatory inputs""" + task = SimpleTask() + task.inputs.input_1 = 'Input1' + task.inputs.input_2 = True + task.inputs.input_3 = False + + with pytest.raises(Exception) as excinfo: + task.inputs.check_fields_input_spec() + assert "input_2 is mutually exclusive with ('input_1', 'input_2', 'input_3'" in str(excinfo.value) assert excinfo.type is AttributeError From 0a790804eec57346f8358d5624d5c91cd62c45ec Mon Sep 17 00:00:00 2001 From: Antoine Blachair Date: Thu, 17 Nov 2022 15:47:06 +1100 Subject: [PATCH 47/94] Clean up --- pydra/engine/tests/test_specs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pydra/engine/tests/test_specs.py b/pydra/engine/tests/test_specs.py index 8dc0a9cf39..40785ae9eb 100644 --- a/pydra/engine/tests/test_specs.py +++ b/pydra/engine/tests/test_specs.py @@ -419,6 +419,7 @@ def test_task_inputs_mandatory_with_xOR_one_mandatory_is_OK(): task.inputs.input_2 = attr.NOTHING task.inputs.check_fields_input_spec() + def test_task_inputs_mandatory_with_xOR_one_mandatory_out_3_is_OK(): """input spec with mandatory inputs""" task = SimpleTask() @@ -427,6 +428,7 @@ def test_task_inputs_mandatory_with_xOR_one_mandatory_out_3_is_OK(): task.inputs.input_3 = True task.inputs.check_fields_input_spec() + def test_task_inputs_mandatory_with_xOR_zero_mandatory_raises_error(): """input spec with mandatory inputs""" task = SimpleTask() @@ -449,6 +451,7 @@ def test_task_inputs_mandatory_with_xOR_two_mandatories_raises_error(): assert "input_2 is mutually exclusive with ('input_1', 'input_2'" in str(excinfo.value) assert excinfo.type is AttributeError + def test_task_inputs_mandatory_with_xOR_3_mandatories_raises_error(): """input spec with mandatory inputs""" task = SimpleTask() @@ -459,4 +462,4 @@ def test_task_inputs_mandatory_with_xOR_3_mandatories_raises_error(): with pytest.raises(Exception) as excinfo: task.inputs.check_fields_input_spec() assert "input_2 is mutually exclusive with ('input_1', 'input_2', 'input_3'" in str(excinfo.value) - assert excinfo.type is AttributeError + assert excinfo.type is AttributeError \ No newline at end of file From a2da63c27f64b93de5d0f1b0d0a091d7b183f787 Mon Sep 17 00:00:00 2001 From: Antoine Blachair Date: Thu, 17 Nov 2022 16:34:28 +1100 Subject: [PATCH 48/94] Check if input is in exlusion list before testing it --- pydra/engine/specs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 249ceea138..eb01aa5fe8 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -164,7 +164,8 @@ def check_fields_input_spec(self): if getattr(self, fld.name) is attr.NOTHING: if mdata.get("mandatory"): # checking if the mandatory field is provided elsewhere in the xor list - alreday_populated = [getattr(self, el) for el in mdata["xor"] if (getattr(self, el) is not attr.NOTHING)] + in_exclusion_list = mdata.get("xor") is not None + alreday_populated = in_exclusion_list and [getattr(self, el) for el in mdata["xor"] if (getattr(self, el) is not attr.NOTHING)] if alreday_populated: #another input satisfies mandatory attribute via xor condition continue else: From 5e0c24942a70f3e01703f16e4801280f324fa63e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 17 Nov 2022 06:31:39 +0000 Subject: [PATCH 49/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/specs.py | 12 +++-- pydra/engine/tests/test_specs.py | 89 +++++++++++++++++--------------- 2 files changed, 56 insertions(+), 45 deletions(-) diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index eb01aa5fe8..20d4661bda 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -162,11 +162,17 @@ def check_fields_input_spec(self): mdata = fld.metadata # checking if the mandatory field is provided if getattr(self, fld.name) is attr.NOTHING: - if mdata.get("mandatory"): + if mdata.get("mandatory"): # checking if the mandatory field is provided elsewhere in the xor list in_exclusion_list = mdata.get("xor") is not None - alreday_populated = in_exclusion_list and [getattr(self, el) for el in mdata["xor"] if (getattr(self, el) is not attr.NOTHING)] - if alreday_populated: #another input satisfies mandatory attribute via xor condition + alreday_populated = in_exclusion_list and [ + getattr(self, el) + for el in mdata["xor"] + if (getattr(self, el) is not attr.NOTHING) + ] + if ( + alreday_populated + ): # another input satisfies mandatory attribute via xor condition continue else: raise AttributeError( diff --git a/pydra/engine/tests/test_specs.py b/pydra/engine/tests/test_specs.py index 40785ae9eb..17cd7b0b82 100644 --- a/pydra/engine/tests/test_specs.py +++ b/pydra/engine/tests/test_specs.py @@ -16,14 +16,12 @@ LazyField, ShellOutSpec, ) -from ..task import ( - TaskBase, - ShellCommandTask -) +from ..task import TaskBase, ShellCommandTask from ..helpers import make_klass import pytest import attr + def test_basespec(): spec = BaseSpec() assert ( @@ -372,40 +370,43 @@ def test_input_file_hash_5(tmpdir): hash3 = inputs(in_file=[{"file": file_diffcontent, "int": 3}]).hash assert hash1 != hash3 + class SimpleTask(ShellCommandTask): - input_fields=[ - ( - "input_1", - str, - { - "help_string": "help", - "mandatory": True, - "xor": ("input_1", "input_2", "input_3"), - } - ), - ( - "input_2", - bool, - { - "help_string": "help", - "mandatory": True, - "argstr": "--i2", - "xor": ("input_1", "input_2", "input_3"), - } - ), - ( - "input_3", - bool, - { - "help_string": "help", - "mandatory": True, - "xor": ("input_1", "input_2", "input_3"), - } - ) + input_fields = [ + ( + "input_1", + str, + { + "help_string": "help", + "mandatory": True, + "xor": ("input_1", "input_2", "input_3"), + }, + ), + ( + "input_2", + bool, + { + "help_string": "help", + "mandatory": True, + "argstr": "--i2", + "xor": ("input_1", "input_2", "input_3"), + }, + ), + ( + "input_3", + bool, + { + "help_string": "help", + "mandatory": True, + "xor": ("input_1", "input_2", "input_3"), + }, + ), ] task_input_spec = SpecInfo(name="Input", fields=input_fields, bases=(ShellSpec,)) task_output_fields = [] - task_output_spec = SpecInfo(name="Output", fields=task_output_fields, bases=(ShellOutSpec,)) + task_output_spec = SpecInfo( + name="Output", fields=task_output_fields, bases=(ShellOutSpec,) + ) input_spec = task_input_spec output_spec = task_output_spec @@ -415,7 +416,7 @@ class SimpleTask(ShellCommandTask): def test_task_inputs_mandatory_with_xOR_one_mandatory_is_OK(): """input spec with mandatory inputs""" task = SimpleTask() - task.inputs.input_1 = 'Input1' + task.inputs.input_1 = "Input1" task.inputs.input_2 = attr.NOTHING task.inputs.check_fields_input_spec() @@ -443,23 +444,27 @@ def test_task_inputs_mandatory_with_xOR_zero_mandatory_raises_error(): def test_task_inputs_mandatory_with_xOR_two_mandatories_raises_error(): """input spec with mandatory inputs""" task = SimpleTask() - task.inputs.input_1 = 'Input1' + task.inputs.input_1 = "Input1" task.inputs.input_2 = True - + with pytest.raises(Exception) as excinfo: task.inputs.check_fields_input_spec() - assert "input_2 is mutually exclusive with ('input_1', 'input_2'" in str(excinfo.value) + assert "input_2 is mutually exclusive with ('input_1', 'input_2'" in str( + excinfo.value + ) assert excinfo.type is AttributeError def test_task_inputs_mandatory_with_xOR_3_mandatories_raises_error(): """input spec with mandatory inputs""" task = SimpleTask() - task.inputs.input_1 = 'Input1' + task.inputs.input_1 = "Input1" task.inputs.input_2 = True task.inputs.input_3 = False - + with pytest.raises(Exception) as excinfo: task.inputs.check_fields_input_spec() - assert "input_2 is mutually exclusive with ('input_1', 'input_2', 'input_3'" in str(excinfo.value) - assert excinfo.type is AttributeError \ No newline at end of file + assert "input_2 is mutually exclusive with ('input_1', 'input_2', 'input_3'" in str( + excinfo.value + ) + assert excinfo.type is AttributeError From 175662f549be2ea227c12d24ae5b6e14c548d6d2 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Thu, 17 Nov 2022 20:56:11 -0800 Subject: [PATCH 50/94] Added file test, atlocation test, hash test --- pydra/engine/audit.py | 6 ++-- pydra/engine/tests/test_task.py | 50 +++++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index db8b23c423..3a80a47609 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -4,7 +4,7 @@ import json import attr from ..utils.messenger import send_message, make_message, gen_uuid, now, AuditFlag -from .helpers import ensure_list, gather_runtime_info +from .helpers import ensure_list, gather_runtime_info, hash_file class Audit: @@ -183,8 +183,10 @@ def audit_task(self, task): if hasattr(task.inputs, "in_file"): input_file = task.inputs.in_file + file_hash = hash_file(input_file) at_location = os.path.abspath(input_file) else: + file_hash = None at_location = None input_file = None @@ -221,7 +223,7 @@ def audit_task(self, task): "AtLocation": at_location, "GeneratedBy": "test", # if not part of workflow, this will be none "@type": "input", - "digest": "checksum", # hash value under helpers.py + "digest": file_hash, # hash value under helpers.py } # new code to be added here for i/o tracking - WIP diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 23e7e210e2..c067e756e2 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -11,8 +11,9 @@ from ..core import Workflow from ..task import AuditFlag, ShellCommandTask, DockerTask, SingularityTask from ...utils.messenger import FileMessenger, PrintMessenger, collect_messages -from .utils import gen_basic_wf, use_validator -from ..specs import MultiInputObj, MultiOutputObj, SpecInfo, FunctionSpec, BaseSpec +from .utils import gen_basic_wf, use_validator, Submitter +from ..specs import MultiInputObj, MultiOutputObj, SpecInfo, FunctionSpec, BaseSpec, ShellSpec, File +from ..helpers import hash_file no_win = pytest.mark.skipif( sys.platform.startswith("win"), @@ -1021,6 +1022,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] if "AssociatedWith" in data: assert None == data["AssociatedWith"] + # assert any(json_content) @@ -1062,6 +1064,50 @@ def test_audit_shellcommandtask(tmpdir): assert any(command_content) +def test_audit_shellcommandtask_file(tmpdir): + # create test.txt file with "This is a test" in it in the tmpdir + with open(tmpdir / "test.txt", "w") as f: + f.write("This is a test.") + + cmd = "cat" + file_in = tmpdir / "test.txt" + test_file_hash = hash_file(file_in) + my_input_spec = SpecInfo( + name='Input', + fields=[ + ( + 'in_file', + attr.ib( + type=File, + metadata={ + 'position': 1, + 'argstr': '', + 'help_string': 'text', + 'mandatory': True, + }, + ), + ) + ], + bases=(ShellSpec,), +) + shelly = ShellCommandTask( + name='shelly', in_file=file_in, input_spec=my_input_spec, executable=cmd, audit_flags=AuditFlag.PROV, messengers=PrintMessenger() +) + shelly.cache_dir = tmpdir + shelly() + message_path = tmpdir / shelly.checksum / "messages" + for file in glob.glob(str(message_path) + "/*.jsonld"): + with open(file, "r") as f: + data = json.load(f) + print(file_in) + if "AtLocation" in data: + assert data["AtLocation"] == str(file_in) + if "digest" in data: + assert test_file_hash == data["digest"] + + + + def test_audit_shellcommandtask_version(tmpdir): import subprocess as sp From 0759ab1aa9f7a9beef22e6aca1f9ba1ad052850e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 18 Nov 2022 04:56:30 +0000 Subject: [PATCH 51/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/tests/test_task.py | 56 +++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index c067e756e2..14902355fd 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -12,7 +12,15 @@ from ..task import AuditFlag, ShellCommandTask, DockerTask, SingularityTask from ...utils.messenger import FileMessenger, PrintMessenger, collect_messages from .utils import gen_basic_wf, use_validator, Submitter -from ..specs import MultiInputObj, MultiOutputObj, SpecInfo, FunctionSpec, BaseSpec, ShellSpec, File +from ..specs import ( + MultiInputObj, + MultiOutputObj, + SpecInfo, + FunctionSpec, + BaseSpec, + ShellSpec, + File, +) from ..helpers import hash_file no_win = pytest.mark.skipif( @@ -1022,7 +1030,6 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] if "AssociatedWith" in data: assert None == data["AssociatedWith"] - # assert any(json_content) @@ -1073,26 +1080,31 @@ def test_audit_shellcommandtask_file(tmpdir): file_in = tmpdir / "test.txt" test_file_hash = hash_file(file_in) my_input_spec = SpecInfo( - name='Input', - fields=[ - ( - 'in_file', - attr.ib( - type=File, - metadata={ - 'position': 1, - 'argstr': '', - 'help_string': 'text', - 'mandatory': True, - }, - ), - ) - ], - bases=(ShellSpec,), -) + name="Input", + fields=[ + ( + "in_file", + attr.ib( + type=File, + metadata={ + "position": 1, + "argstr": "", + "help_string": "text", + "mandatory": True, + }, + ), + ) + ], + bases=(ShellSpec,), + ) shelly = ShellCommandTask( - name='shelly', in_file=file_in, input_spec=my_input_spec, executable=cmd, audit_flags=AuditFlag.PROV, messengers=PrintMessenger() -) + name="shelly", + in_file=file_in, + input_spec=my_input_spec, + executable=cmd, + audit_flags=AuditFlag.PROV, + messengers=PrintMessenger(), + ) shelly.cache_dir = tmpdir shelly() message_path = tmpdir / shelly.checksum / "messages" @@ -1106,8 +1118,6 @@ def test_audit_shellcommandtask_file(tmpdir): assert test_file_hash == data["digest"] - - def test_audit_shellcommandtask_version(tmpdir): import subprocess as sp From 6c3cded5834f44feba67284379aad99115f5c8e4 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Fri, 18 Nov 2022 13:27:45 -0800 Subject: [PATCH 52/94] Updated ent. uuid, input_file checks --- pydra/engine/audit.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 3a80a47609..c597843b0b 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -5,6 +5,7 @@ import attr from ..utils.messenger import send_message, make_message, gen_uuid, now, AuditFlag from .helpers import ensure_list, gather_runtime_info, hash_file +from .specs import attr_fields, File, Directory class Audit: @@ -180,15 +181,29 @@ def audit_task(self, task): # assume function task else: command = None + # only implementing for file or directories. Check "type" and return "File" or "Directory" + attr_list = attr_fields(task.inputs) + for attrs in attr_list: + if attrs.type in [File, Directory]: + input_name = attrs.name + input_path = os.path.abspath(getattr(task.inputs, input_name)) + file_hash = hash_file(input_path) - if hasattr(task.inputs, "in_file"): - input_file = task.inputs.in_file - file_hash = hash_file(input_file) - at_location = os.path.abspath(input_file) - else: - file_hash = None - at_location = None - input_file = None + else: + input_name = attrs.name + input_path = None + file_hash = None + # at_location = os.path.abspath(input_name) + + + # if hasattr(task.inputs, "in_file"): + # input_file = task.inputs.in_file + # file_hash = hash_file(input_file) + # at_location = os.path.abspath(input_file) + # else: + # file_hash = None + # at_location = None + # input_file = None if command is not None: cmd_name = command.split()[0] @@ -216,14 +231,14 @@ def audit_task(self, task): "StartedAtTime": now(), "AssociatedWith": version_cmd, } - + entity_id = f"uid:{gen_uuid()}" entity_message = { - "@id": self.aid, + "@id": entity_id, # add ID here "Label": print(entity_label), - "AtLocation": at_location, + "AtLocation": input_path, #at_location, "GeneratedBy": "test", # if not part of workflow, this will be none "@type": "input", - "digest": file_hash, # hash value under helpers.py + "digest": file_hash # hash value under helpers.py } # new code to be added here for i/o tracking - WIP From 78953194e017e310fda463c301531613550461d0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 18 Nov 2022 21:29:00 +0000 Subject: [PATCH 53/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index c597843b0b..64c826d209 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -194,8 +194,7 @@ def audit_task(self, task): input_path = None file_hash = None # at_location = os.path.abspath(input_name) - - + # if hasattr(task.inputs, "in_file"): # input_file = task.inputs.in_file # file_hash = hash_file(input_file) @@ -233,12 +232,12 @@ def audit_task(self, task): } entity_id = f"uid:{gen_uuid()}" entity_message = { - "@id": entity_id, # add ID here + "@id": entity_id, # add ID here "Label": print(entity_label), - "AtLocation": input_path, #at_location, + "AtLocation": input_path, # at_location, "GeneratedBy": "test", # if not part of workflow, this will be none "@type": "input", - "digest": file_hash # hash value under helpers.py + "digest": file_hash, # hash value under helpers.py } # new code to be added here for i/o tracking - WIP From 8a0d63c246cbe825f2b56f1f1353b5d7d70bf8cf Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Sun, 20 Nov 2022 15:09:35 -0800 Subject: [PATCH 54/94] Refactored, cleaned up old comments --- pydra/engine/audit.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 64c826d209..876d54854b 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -176,12 +176,8 @@ def audit_task(self, task): label = task.name entity_label = type(label) - if hasattr(task.inputs, "executable"): - command = task.cmdline - # assume function task - else: - command = None - # only implementing for file or directories. Check "type" and return "File" or "Directory" + command = task.cmdline if hasattr(task.inputs, "executable") else None + attr_list = attr_fields(task.inputs) for attrs in attr_list: if attrs.type in [File, Directory]: @@ -194,7 +190,8 @@ def audit_task(self, task): input_path = None file_hash = None # at_location = os.path.abspath(input_name) - + + # if hasattr(task.inputs, "in_file"): # input_file = task.inputs.in_file # file_hash = hash_file(input_file) @@ -232,17 +229,15 @@ def audit_task(self, task): } entity_id = f"uid:{gen_uuid()}" entity_message = { - "@id": entity_id, # add ID here + "@id": entity_id, # add ID here "Label": print(entity_label), - "AtLocation": input_path, # at_location, + "AtLocation": input_path, #at_location, "GeneratedBy": "test", # if not part of workflow, this will be none "@type": "input", - "digest": file_hash, # hash value under helpers.py + "digest": file_hash # hash value under helpers.py } - # new code to be added here for i/o tracking - WIP self.audit_message(start_message, AuditFlag.PROV) self.audit_message(entity_message, AuditFlag.PROV) - # add more fields according to BEP208 doc - # with every field, check in tests + From 1a5822c88f6ae862980d4a2aa81ced54a40c71e0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 20 Nov 2022 23:11:17 +0000 Subject: [PATCH 55/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 876d54854b..016d266d6a 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -190,8 +190,7 @@ def audit_task(self, task): input_path = None file_hash = None # at_location = os.path.abspath(input_name) - - + # if hasattr(task.inputs, "in_file"): # input_file = task.inputs.in_file # file_hash = hash_file(input_file) @@ -229,15 +228,13 @@ def audit_task(self, task): } entity_id = f"uid:{gen_uuid()}" entity_message = { - "@id": entity_id, # add ID here + "@id": entity_id, # add ID here "Label": print(entity_label), - "AtLocation": input_path, #at_location, + "AtLocation": input_path, # at_location, "GeneratedBy": "test", # if not part of workflow, this will be none "@type": "input", - "digest": file_hash # hash value under helpers.py + "digest": file_hash, # hash value under helpers.py } - self.audit_message(start_message, AuditFlag.PROV) self.audit_message(entity_message, AuditFlag.PROV) - From 704d52ec04aa41daad021c11acd6ddf855b0f5fc Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Sun, 20 Nov 2022 15:12:33 -0800 Subject: [PATCH 56/94] Refactored, cleaned up old comments --- pydra/engine/audit.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 016d266d6a..de25d93199 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -189,16 +189,6 @@ def audit_task(self, task): input_name = attrs.name input_path = None file_hash = None - # at_location = os.path.abspath(input_name) - - # if hasattr(task.inputs, "in_file"): - # input_file = task.inputs.in_file - # file_hash = hash_file(input_file) - # at_location = os.path.abspath(input_file) - # else: - # file_hash = None - # at_location = None - # input_file = None if command is not None: cmd_name = command.split()[0] @@ -228,12 +218,12 @@ def audit_task(self, task): } entity_id = f"uid:{gen_uuid()}" entity_message = { - "@id": entity_id, # add ID here + "@id": entity_id, "Label": print(entity_label), - "AtLocation": input_path, # at_location, - "GeneratedBy": "test", # if not part of workflow, this will be none + "AtLocation": input_path, + "GeneratedBy": "test", "@type": "input", - "digest": file_hash, # hash value under helpers.py + "digest": file_hash } self.audit_message(start_message, AuditFlag.PROV) From 2912ee603103adeb9ec2e1f8890acbb0890ecdfe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 20 Nov 2022 23:13:59 +0000 Subject: [PATCH 57/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index de25d93199..8016882e13 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -218,12 +218,12 @@ def audit_task(self, task): } entity_id = f"uid:{gen_uuid()}" entity_message = { - "@id": entity_id, + "@id": entity_id, "Label": print(entity_label), - "AtLocation": input_path, - "GeneratedBy": "test", + "AtLocation": input_path, + "GeneratedBy": "test", "@type": "input", - "digest": file_hash + "digest": file_hash, } self.audit_message(start_message, AuditFlag.PROV) From dfecb8a6666658b276f9ef1519718c1882e38e00 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Tue, 22 Nov 2022 12:49:36 -0800 Subject: [PATCH 58/94] Added dict for atlocation, digest tracking, added tests --- pydra/engine/audit.py | 27 ++++++++++++++++++--------- pydra/engine/tests/test_task.py | 27 ++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 8016882e13..77790aa31c 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -176,7 +176,13 @@ def audit_task(self, task): label = task.name entity_label = type(label) - command = task.cmdline if hasattr(task.inputs, "executable") else None + if hasattr(task.inputs, "executable"): + command = task.cmdline + # assume function task + else: + command = None + + path_hash_dict = {} attr_list = attr_fields(task.inputs) for attrs in attr_list: @@ -184,11 +190,11 @@ def audit_task(self, task): input_name = attrs.name input_path = os.path.abspath(getattr(task.inputs, input_name)) file_hash = hash_file(input_path) + path_hash_dict[input_path] = file_hash - else: - input_name = attrs.name - input_path = None - file_hash = None + # get the hash for the output + input_paths = list(path_hash_dict.keys()) + input_paths_hash = list(path_hash_dict.values()) if command is not None: cmd_name = command.split()[0] @@ -218,13 +224,16 @@ def audit_task(self, task): } entity_id = f"uid:{gen_uuid()}" entity_message = { - "@id": entity_id, + "@id": entity_id, "Label": print(entity_label), - "AtLocation": input_path, - "GeneratedBy": "test", + "AtLocation": input_paths, # + "GeneratedBy": "test", "@type": "input", - "digest": file_hash, + "digest": input_paths_hash } + + self.audit_message(start_message, AuditFlag.PROV) self.audit_message(entity_message, AuditFlag.PROV) + diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 14902355fd..696472eb0b 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1008,8 +1008,6 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] funky() message_path = tmpdir / funky.checksum / "messages" print(message_path) - # go through each jsonld file in message_path and check if the label field exists - json_content = [] for file in glob(str(message_path) + "/*.jsonld"): with open(file, "r") as f: @@ -1023,7 +1021,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] assert None == data["Label"] # placeholder for atlocation until # new test is added - assert None == data["AtLocation"] + assert [] == data["AtLocation"] # assert data["Type"] == "input" @@ -1072,13 +1070,19 @@ def test_audit_shellcommandtask(tmpdir): def test_audit_shellcommandtask_file(tmpdir): + import shutil # create test.txt file with "This is a test" in it in the tmpdir with open(tmpdir / "test.txt", "w") as f: f.write("This is a test.") + # make a copy of the test.txt file in the tmpdir and name it test2.txt + shutil.copy(tmpdir / "test.txt", tmpdir / "test2.txt") + cmd = "cat" file_in = tmpdir / "test.txt" + file_in_2 = tmpdir / "test2.txt" test_file_hash = hash_file(file_in) + test_file_hash_2 = hash_file(file_in_2) my_input_spec = SpecInfo( name="Input", fields=[ @@ -1093,6 +1097,18 @@ def test_audit_shellcommandtask_file(tmpdir): "mandatory": True, }, ), + ), + ( + "in_file_2", + attr.ib( + type=File, + metadata={ + "position": 2, + "argstr": "", + "help_string": "text", + "mandatory": True, + }, + ), ) ], bases=(ShellSpec,), @@ -1100,6 +1116,7 @@ def test_audit_shellcommandtask_file(tmpdir): shelly = ShellCommandTask( name="shelly", in_file=file_in, + in_file_2=file_in_2, input_spec=my_input_spec, executable=cmd, audit_flags=AuditFlag.PROV, @@ -1113,9 +1130,9 @@ def test_audit_shellcommandtask_file(tmpdir): data = json.load(f) print(file_in) if "AtLocation" in data: - assert data["AtLocation"] == str(file_in) + assert data["AtLocation"] == [file_in, file_in_2] if "digest" in data: - assert test_file_hash == data["digest"] + assert data["digest"] == [test_file_hash, test_file_hash_2] def test_audit_shellcommandtask_version(tmpdir): From 9be04d2e1fc90e841d2cb72297ac165c94b967a3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Nov 2022 20:52:57 +0000 Subject: [PATCH 59/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 11 ++++------- pydra/engine/tests/test_task.py | 6 +++--- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 77790aa31c..f86fad33ae 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -224,16 +224,13 @@ def audit_task(self, task): } entity_id = f"uid:{gen_uuid()}" entity_message = { - "@id": entity_id, + "@id": entity_id, "Label": print(entity_label), - "AtLocation": input_paths, # - "GeneratedBy": "test", + "AtLocation": input_paths, # + "GeneratedBy": "test", "@type": "input", - "digest": input_paths_hash + "digest": input_paths_hash, } - - self.audit_message(start_message, AuditFlag.PROV) self.audit_message(entity_message, AuditFlag.PROV) - diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 696472eb0b..49c84134d0 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1071,13 +1071,13 @@ def test_audit_shellcommandtask(tmpdir): def test_audit_shellcommandtask_file(tmpdir): import shutil + # create test.txt file with "This is a test" in it in the tmpdir with open(tmpdir / "test.txt", "w") as f: f.write("This is a test.") # make a copy of the test.txt file in the tmpdir and name it test2.txt shutil.copy(tmpdir / "test.txt", tmpdir / "test2.txt") - cmd = "cat" file_in = tmpdir / "test.txt" file_in_2 = tmpdir / "test2.txt" @@ -1098,7 +1098,7 @@ def test_audit_shellcommandtask_file(tmpdir): }, ), ), - ( + ( "in_file_2", attr.ib( type=File, @@ -1109,7 +1109,7 @@ def test_audit_shellcommandtask_file(tmpdir): "mandatory": True, }, ), - ) + ), ], bases=(ShellSpec,), ) From 7d3845e0a3455c5f492b11853dff5dff11d54c55 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Tue, 22 Nov 2022 23:30:17 -0800 Subject: [PATCH 60/94] Updated entity_message to be collected per input --- pydra/engine/audit.py | 31 +++++++++++++++---------------- pydra/engine/tests/test_task.py | 2 +- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index f86fad33ae..e01a43883f 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -182,19 +182,24 @@ def audit_task(self, task): else: command = None - path_hash_dict = {} - attr_list = attr_fields(task.inputs) for attrs in attr_list: if attrs.type in [File, Directory]: input_name = attrs.name input_path = os.path.abspath(getattr(task.inputs, input_name)) file_hash = hash_file(input_path) - path_hash_dict[input_path] = file_hash + entity_id = f"uid:{gen_uuid()}" + entity_message = { + "@id": entity_id, + "Label": print(entity_label), + "AtLocation": input_path, + "GeneratedBy": "test", + "@type": "input", + "digest": file_hash + } + self.audit_message(entity_message, AuditFlag.PROV) + - # get the hash for the output - input_paths = list(path_hash_dict.keys()) - input_paths_hash = list(path_hash_dict.values()) if command is not None: cmd_name = command.split()[0] @@ -222,15 +227,9 @@ def audit_task(self, task): "StartedAtTime": now(), "AssociatedWith": version_cmd, } - entity_id = f"uid:{gen_uuid()}" - entity_message = { - "@id": entity_id, - "Label": print(entity_label), - "AtLocation": input_paths, # - "GeneratedBy": "test", - "@type": "input", - "digest": input_paths_hash, - } + + self.audit_message(start_message, AuditFlag.PROV) - self.audit_message(entity_message, AuditFlag.PROV) + + diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 49c84134d0..cf71ed772e 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1264,7 +1264,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] from glob import glob assert len(glob(str(tmpdir / funky.checksum / "proc*.log"))) == 1 - assert len(glob(str(message_path / "*.jsonld"))) == 8 + assert len(glob(str(message_path / "*.jsonld"))) == 7 # commented out to speed up testing collect_messages(tmpdir / funky.checksum, message_path, ld_op="compact") From 230d0efb8e65b596a82c810dc436ccbf705dd046 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 23 Nov 2022 07:31:45 +0000 Subject: [PATCH 61/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index e01a43883f..97eb282341 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -190,17 +190,15 @@ def audit_task(self, task): file_hash = hash_file(input_path) entity_id = f"uid:{gen_uuid()}" entity_message = { - "@id": entity_id, + "@id": entity_id, "Label": print(entity_label), "AtLocation": input_path, - "GeneratedBy": "test", + "GeneratedBy": "test", "@type": "input", - "digest": file_hash + "digest": file_hash, } self.audit_message(entity_message, AuditFlag.PROV) - - if command is not None: cmd_name = command.split()[0] software = f"{cmd_name} --version" @@ -227,9 +225,5 @@ def audit_task(self, task): "StartedAtTime": now(), "AssociatedWith": version_cmd, } - - self.audit_message(start_message, AuditFlag.PROV) - - From e7583e4f4e70aea6d55034102639e5c11b30927e Mon Sep 17 00:00:00 2001 From: Ghislain Vaillant Date: Wed, 23 Nov 2022 09:30:12 +0100 Subject: [PATCH 62/94] MNT: Migrate project to flit-scm --- .pre-commit-config.yaml | 3 +- long_description.rst | 6 - pydra/__init__.py | 7 +- pydra/_version.py | 556 ------------ pyproject.toml | 93 +- setup.cfg | 105 --- setup.py | 24 - versioneer.py | 1885 --------------------------------------- 8 files changed, 98 insertions(+), 2581 deletions(-) delete mode 100644 long_description.rst delete mode 100644 pydra/_version.py delete mode 100755 setup.py delete mode 100644 versioneer.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1c0376116a..f0f606c598 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,8 @@ repos: rev: v2.2.2 hooks: - id: codespell - exclude: ^(pydra/_version\.py|versioneer\.py)$ + additional_dependencies: + - tomli - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: diff --git a/long_description.rst b/long_description.rst deleted file mode 100644 index 9234032e51..0000000000 --- a/long_description.rst +++ /dev/null @@ -1,6 +0,0 @@ -====================== -Pydra: Dataflow Engine -====================== - -Pydra is a rewrite of the Nipype engine with mapping and joining as -first-class operations. It forms the core of the Nipype 2.0 ecosystem. diff --git a/pydra/__init__.py b/pydra/__init__.py index fa944f6c79..cb6823dc86 100644 --- a/pydra/__init__.py +++ b/pydra/__init__.py @@ -12,10 +12,11 @@ import logging logger = logging.getLogger("pydra") -from ._version import get_versions -__version__ = get_versions()["version"] -del get_versions +try: + from ._version import __version__ +except ImportError: + pass from .engine import Submitter, Workflow, AuditFlag, ShellCommandTask, DockerTask, specs from . import mark diff --git a/pydra/_version.py b/pydra/_version.py deleted file mode 100644 index da2b4c4b22..0000000000 --- a/pydra/_version.py +++ /dev/null @@ -1,556 +0,0 @@ -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "" - cfg.parentdir_prefix = "" - cfg.versionfile_source = "pydra/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split("/"): - root = os.path.dirname(root) - except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } diff --git a/pyproject.toml b/pyproject.toml index 34597341e2..bbf4003a21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,97 @@ [build-system] -requires = ["setuptools==62", "wheel"] +requires = ["flit_scm"] +build-backend = "flit_scm:buildapi" + +[project] +name = "pydra" +description = "Pydra dataflow engine" +readme = "README.rst" +requires-python = ">=3.7" +dependencies = [ + "attrs >=19.1.0", + "cloudpickle >=2.0.0", + "etelemetry >=0.2.2", + "filelock >=3.0.0", +] +license = {file = "LICENSE"} +authors = [ + {name = "Nipype developers", email = "neuroimaging@python.org"}, +] +maintainers = [ + {name = "Nipype developers", email = "neuroimaging@python.org"}, +] +keywords = [ + "brainweb", + "dataflow", + "neuroimaging", + "pydra", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering", +] +dynamic = ["version"] + +[project.optional-dependencies] +dask = [ + "dask", + "distributed", +] +dev = [ + "black", + "pre-commit", +] +doc = [ + "attrs >=19.1.0", + "cloudpickle", + "filelock", + "packaging", + "sphinx >=2.1.2", + "sphinx_rtd_theme", + "sphinxcontrib-apidoc ~=0.3.0", + "sphinxcontrib-napoleon", + "sphinxcontrib-versioning", +] +test = [ + "pytest >=6.2.5", + "pytest-cov", + "pytest-env", + "pytest-xdist <2.0", + "pytest-rerunfailures", + "pytest-timeout", + "codecov", + "numpy", + "pyld", + "psutil", + "python-dateutil", + "tornado", + "boutiques", + "pympler", +] + +[project.urls] +documentation = "https://nipype.github.io/pydra/" +homepage = "https://nipype.github.io/pydra/" +repository = "https://github.com/nipype/pydra.git" + +[tool.flit.module] +name = "pydra" + +[tool.flit.sdist] +exclude = [".gitignore"] + +[tool.setuptools_scm] +write_to = "pydra/_version.py" [tool.black] target-version = ['py37', 'py38'] exclude = "pydra/_version.py" + +[tool.codespell] +ignore-words = ".codespell-ignorewords" diff --git a/setup.cfg b/setup.cfg index 89dc37dd10..a0478b8f62 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,105 +1,3 @@ -[metadata] -url = https://github.com/nipype/pydra -author = Nipype developers -author_email = neuroimaging@python.org -maintainer = Nipype developers -maintainer_email = neuroimaging@python.org -description = Pydra dataflow engine -long_description = file:README.rst -long_description_content_type = text/x-rst; charset=UTF-8 -license = Apache License, 2.0 -provides = - pydra -classifiers = - Development Status :: 3 - Alpha - Environment :: Console - Intended Audience :: Science/Research - License :: OSI Approved :: Apache Software License - Operating System :: MacOS :: MacOS X - Operating System :: POSIX :: Linux - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Topic :: Scientific/Engineering - -[options] -python_requires = >= 3.7 -install_requires = - attrs >= 19.1.0 - cloudpickle >= 2.0.0 - filelock >= 3.0.0 - etelemetry >= 0.2.2 - -test_requires = - pytest >= 6.2.5 - pytest-cov - pytest-env - pytest-xdist < 2.0 - pytest-rerunfailures - pytest-timeout - codecov - numpy - psutil - python-dateutil - tornado - boutiques - pympler -packages = find: -include_package_data = True - -[options.package_data] -pydra = - schema/context.jsonld - -[options.extras_require] -doc = - attrs >= 19.1.0 - cloudpickle - filelock - packaging - sphinx >= 2.1.2 - sphinx_rtd_theme - sphinxcontrib-apidoc ~= 0.3.0 - sphinxcontrib-napoleon - sphinxcontrib-versioning -docs = - %(doc)s -test = - pytest >= 6.2.5 - pytest-cov - pytest-env - pytest-xdist < 2.0 - pytest-rerunfailures - pytest-timeout - codecov - numpy - pyld - psutil - python-dateutil - tornado - boutiques - pympler -tests = - %(test)s -dev = - %(test)s - black==21.4b2 - pre-commit -dask = - %(test)s - dask - distributed -all = - %(doc)s - %(dev)s - -[versioneer] -VCS = git -style = pep440 -versionfile_source = pydra/_version.py -versionfile_build = pydra/_version.py -tag_prefix = -parentdir_prefix = - [flake8] doctests = True exclude = @@ -113,6 +11,3 @@ exclude = versioneer.py max-line-length=105 ignore = E203,W503,F541 - -[codespell] -ignore-words = .codespell-ignorewords diff --git a/setup.py b/setup.py deleted file mode 100755 index e03e00f4e3..0000000000 --- a/setup.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- -# vi: set ft=python sts=4 ts=4 sw=4 et: -"""Pydra: Dataflow Engine - -""" -import sys -from setuptools import setup -import versioneer - -# Give setuptools a hint to complain if it's too old a version -# 30.3.0 allows us to put most metadata in setup.cfg -# Should match pyproject.toml -SETUP_REQUIRES = ["setuptools >= 30.3.0"] -# This enables setuptools to install wheel on-the-fly -SETUP_REQUIRES += ["wheel"] if "bdist_wheel" in sys.argv else [] - -if __name__ == "__main__": - setup( - name="pydra", - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), - setup_requires=SETUP_REQUIRES, - ) diff --git a/versioneer.py b/versioneer.py deleted file mode 100644 index 2b54540510..0000000000 --- a/versioneer.py +++ /dev/null @@ -1,1885 +0,0 @@ -# Version: 0.18 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer -* Brian Warner -* License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) - -This is a tool for managing a recorded version number in distutils-based -python projects. The goal is to remove the tedious and error-prone "update -the embedded version string" step from your release process. Making a new -release should be as easy as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) -* run `versioneer install` in your source tree, commit the results - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -See [INSTALL.md](./INSTALL.md) for detailed installation instructions. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the - commit date in ISO 8601 format. This will be None if the date is not - available. - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from ._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See [details.md](details.md) in the Versioneer -source tree for descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Known Limitations - -Some situations are known to cause problems for Versioneer. This details the -most significant ones. More can be found on Github -[issues page](https://github.com/warner/python-versioneer/issues). - -### Subprojects - -Versioneer has limited support for source trees in which `setup.py` is not in -the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are -two common reasons why `setup.py` might not be in the root: - -* Source trees which contain multiple subprojects, such as - [Buildbot](https://github.com/buildbot/buildbot), which contains both - "master" and "slave" subprojects, each with their own `setup.py`, - `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI - distributions (and upload multiple independently-installable tarballs). -* Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other langauges) in subdirectories. - -Versioneer will look for `.git` in parent directories, and most operations -should get the right version string. However `pip` and `setuptools` have bugs -and implementation details which frequently cause `pip install .` from a -subproject directory to fail to find a correct version string (so it usually -defaults to `0+unknown`). - -`pip install --editable .` should work correctly. `setup.py install` might -work too. - -Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in -some later version. - -[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking -this issue. The discussion in -[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the -issue from the Versioneer side in more detail. -[pip PR#3176](https://github.com/pypa/pip/pull/3176) and -[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve -pip to let Versioneer work correctly. - -Versioneer-0.16 and earlier only looked for a `.git` directory next to the -`setup.cfg`, so subprojects were completely unsupported with those releases. - -### Editable installs with setuptools <= 18.5 - -`setup.py develop` and `pip install --editable .` allow you to install a -project into a virtualenv once, then continue editing the source code (and -test) without re-installing after every change. - -"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a -convenient way to specify executable scripts that should be installed along -with the python package. - -These both work as expected when using modern setuptools. When using -setuptools-18.5 or earlier, however, certain operations will cause -`pkg_resources.DistributionNotFound` errors when running the entrypoint -script, which must be resolved by re-installing the package. This happens -when the install happens with one version, then the egg_info data is -regenerated while a different version is checked out. Many setup.py commands -cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into -a different virtualenv), so this can be surprising. - -[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes -this one, but upgrading to a newer version of setuptools should probably -resolve it. - -### Unicode version strings - -While Versioneer works (and is continually tested) with both Python 2 and -Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. -Newer releases probably generate unicode version strings on py2. It's not -clear that this is wrong, but it may be surprising for applications when then -write these strings to a network connection or include them in bytes-oriented -APIs like cryptographic checksums. - -[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates -this question. - - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . - -""" - -from __future__ import print_function - -try: - import configparser -except ImportError: - import ConfigParser as configparser -import errno -import json -import os -import re -import subprocess -import sys - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ( - "Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND')." - ) - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(me)[0]) - vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir: - print( - "Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py) - ) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: - parser.readfp(f) - VCS = parser.get("versioneer", "VCS") # mandatory - - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None - - cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") - if cfg.tag_prefix in ("''", '""'): - cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -LONG_VERSION_PY[ - "git" -] = ''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - print("stdout was %%s" %% stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %%s but none started with prefix %%s" %% - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs - tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %%s not under git control" %% root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def do_vcs_install(manifest_in, versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-subst keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source] - if ipy: - files.append(ipy) - try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - f = open(".gitattributes", "r") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except EnvironmentError: - pass - if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.18) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except EnvironmentError: - raise NotThisMethod("unable to read _version.py") - mo = re.search( - r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S - ) - if not mo: - mo = re.search( - r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S - ) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set %s to '%s'" % (filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert ( - cfg.versionfile_source is not None - ), "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if verbose: - print("unable to compute version") - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 - - cmds = {} - - # we add "version" to both distutils and setuptools - from distutils.core import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - print(" date: %s" % vers.get("date")) - if vers["error"]: - print(" error: %s" % vers["error"]) - - cmds["version"] = cmd_version - - # we override "build_py" in both distutils and setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - # pip install: - # copies source tree to a tempdir before running egg_info/etc - # if .git isn't copied too, 'git describe' will fail - # then does setup.py bdist_wheel, or sometimes setup.py install - # setup.py egg_info -> ? - - # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.build_py import build_py as _build_py - else: - from distutils.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_py"] = cmd_build_py - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - - # nczeczulin reports that py2exe won't like the pep440-style string - # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. - # setup(console=[{ - # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION - # "product_version": versioneer.get_version(), - # ... - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - if "py2exe" in sys.modules: # py2exe enabled? - try: - from py2exe.distutils_buildexe import py2exe as _py2exe # py3 - except ImportError: - from py2exe.build_exe import py2exe as _py2exe # py2 - - class cmd_py2exe(_py2exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _py2exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["py2exe"] = cmd_py2exe - - # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.sdist import sdist as _sdist - else: - from distutils.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file( - target_versionfile, self._versioneer_generated_versions - ) - - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -INIT_PY_SNIPPET = """ -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - - -def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except ( - EnvironmentError, - configparser.NoSectionError, - configparser.NoOptionError, - ) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", file=sys.stderr) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy, "r") as f: - old = f.read() - except EnvironmentError: - old = "" - if INIT_PY_SNIPPET not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(root, "MANIFEST.in") - simple_includes = set() - try: - with open(manifest_in, "r") as f: - for line in f: - if line.startswith("include "): - for include in line.split()[1:]: - simple_includes.add(include) - except EnvironmentError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - with open(manifest_in, "a") as f: - f.write("include versioneer.py\n") - else: - print(" 'versioneer.py' already in MANIFEST.in") - if cfg.versionfile_source not in simple_includes: - print( - " appending versionfile_source ('%s') to MANIFEST.in" - % cfg.versionfile_source - ) - with open(manifest_in, "a") as f: - f.write("include %s\n" % cfg.versionfile_source) - else: - print(" versionfile_source already in MANIFEST.in") - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-subst keyword - # substitution. - do_vcs_install(manifest_in, cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py", "r") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1) From 611ae10a72dbcfba39bed39b9fad6ce60dec2f99 Mon Sep 17 00:00:00 2001 From: Ghislain Vaillant Date: Wed, 23 Nov 2022 09:32:23 +0100 Subject: [PATCH 63/94] MNT: Rename setup.cfg to .flake8 Flake8 does not support pyproject.toml yet (if ever?). Rename setup.cfg to .flake8 to make it more explicit that the only concern left is flake8 configuration. --- setup.cfg => .flake8 | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename setup.cfg => .flake8 (100%) diff --git a/setup.cfg b/.flake8 similarity index 100% rename from setup.cfg rename to .flake8 From abfd9aabbda743d67590bb94f1748b6b99627654 Mon Sep 17 00:00:00 2001 From: Ghislain Vaillant Date: Wed, 23 Nov 2022 09:44:20 +0100 Subject: [PATCH 64/94] CI: Update build steps in workflows --- .circleci/config.yml | 4 +++- .github/workflows/publish.yml | 4 ++-- .github/workflows/testsingularity.yml | 2 +- .github/workflows/testslurm.yml | 2 +- docs/requirements.txt | 9 --------- 5 files changed, 7 insertions(+), 14 deletions(-) delete mode 100644 docs/requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index e025c480f0..8c88c538d0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,7 +46,9 @@ jobs: - checkout - run: name: Install deps - command: pip install --no-cache-dir -r docs/requirements.txt + command: | + pip install --upgrade pip + pip install --no-cache-dir .[doc] - run: name: Build only this commit command: make -C docs SPHINXOPTS="-W" BUILDDIR="_build/no_version_html" SPHINX_APIDOC_OPTIONS="members,undoc-members,show-inheritance,noindex" html diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 84f3ac6b2e..0f71d0ad2e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,12 +20,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel twine + pip install build twine - name: Build and publish env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: | - python setup.py bdist_wheel + python -m build twine upload dist/* diff --git a/.github/workflows/testsingularity.yml b/.github/workflows/testsingularity.yml index b0f8a9bcb0..5e2d5362ac 100644 --- a/.github/workflows/testsingularity.yml +++ b/.github/workflows/testsingularity.yml @@ -53,7 +53,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Update build tools - run: python -m pip install --upgrade pip setuptools + run: python -m pip install --upgrade pip - name: Checkout Pydra repo diff --git a/.github/workflows/testslurm.yml b/.github/workflows/testslurm.yml index 7e33e247ac..a0bb516607 100644 --- a/.github/workflows/testslurm.yml +++ b/.github/workflows/testslurm.yml @@ -38,7 +38,7 @@ jobs: docker exec slurm bash -c "echo $NO_ET" docker exec slurm bash -c "ls -la && echo list top level dir" docker exec slurm bash -c "ls -la /pydra && echo list pydra dir" - docker exec slurm bash -c "pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'" + docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'" - name: Run pytest run: docker exec slurm bash -c "pytest --color=yes -vs -n auto --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra" - name: Upload to codecov diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index e507dcf5ca..0000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -attrs >= 19.1.0 -cloudpickle -filelock -git+https://github.com/AleksandarPetrov/napoleon.git@0dc3f28a309ad602be5f44a9049785a1026451b3#egg=sphinxcontrib-napoleon -git+https://github.com/effigies/sphinxcontrib-versioning.git@master#egg=sphinxcontrib-versioning -packaging -sphinx>=2.1.2 -sphinx_rtd_theme -sphinxcontrib-apidoc ~= 0.3.0 From 36bb2de1b7665ec1b03d2edc0cf87a5cceedb06d Mon Sep 17 00:00:00 2001 From: Ghislain Vaillant Date: Wed, 23 Nov 2022 11:25:36 +0100 Subject: [PATCH 65/94] CI: Add configuration for readthedocs --- .readthedocs.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000000..0512ede649 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,24 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build with Python 3.9 on the latest version of Ubuntu. +build: + os: ubuntu-22.04 + tools: + python: '3.9' + +# Build documentation in the docs/ directory with Sphinx. +sphinx: + configuration: docs/conf.py + +# Install extra requirements for the documentation. +python: + install: + - method: pip + path: '.' + extra_requirements: + - doc From d1e3c58df41e407a0cd4820cb62aad2be76ec810 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Wed, 23 Nov 2022 22:19:20 -0800 Subject: [PATCH 66/94] Addressed all issues in audit --- pydra/engine/audit.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 97eb282341..51d259b86f 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -174,31 +174,30 @@ def audit_task(self, task): import subprocess as sp label = task.name - entity_label = type(label) - - if hasattr(task.inputs, "executable"): - command = task.cmdline - # assume function task - else: - command = None + + command = task.cmdline if hasattr(task.inputs, "executable") else None attr_list = attr_fields(task.inputs) for attrs in attr_list: if attrs.type in [File, Directory]: input_name = attrs.name + entity_label = attrs.name input_path = os.path.abspath(getattr(task.inputs, input_name)) file_hash = hash_file(input_path) entity_id = f"uid:{gen_uuid()}" entity_message = { - "@id": entity_id, - "Label": print(entity_label), + "@id": entity_id, + "Label": entity_label, "AtLocation": input_path, - "GeneratedBy": "test", + "GeneratedBy": None, "@type": "input", "digest": file_hash, } self.audit_message(entity_message, AuditFlag.PROV) + + + if command is not None: cmd_name = command.split()[0] software = f"{cmd_name} --version" @@ -226,4 +225,6 @@ def audit_task(self, task): "AssociatedWith": version_cmd, } + + self.audit_message(start_message, AuditFlag.PROV) From 9d037df07c8b20a60200943f1141fb6f6dcccdf3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 24 Nov 2022 06:21:11 +0000 Subject: [PATCH 67/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 51d259b86f..17c32b84fe 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -174,7 +174,6 @@ def audit_task(self, task): import subprocess as sp label = task.name - command = task.cmdline if hasattr(task.inputs, "executable") else None attr_list = attr_fields(task.inputs) @@ -186,18 +185,15 @@ def audit_task(self, task): file_hash = hash_file(input_path) entity_id = f"uid:{gen_uuid()}" entity_message = { - "@id": entity_id, + "@id": entity_id, "Label": entity_label, "AtLocation": input_path, - "GeneratedBy": None, + "GeneratedBy": None, "@type": "input", "digest": file_hash, } self.audit_message(entity_message, AuditFlag.PROV) - - - if command is not None: cmd_name = command.split()[0] software = f"{cmd_name} --version" @@ -225,6 +221,4 @@ def audit_task(self, task): "AssociatedWith": version_cmd, } - - self.audit_message(start_message, AuditFlag.PROV) From e73b15b85d5bb13845877aa9f19fb38bad780175 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Wed, 23 Nov 2022 22:23:17 -0800 Subject: [PATCH 68/94] Addressed all issues in audit --- pydra/engine/audit.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 17c32b84fe..ce330526e4 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -180,13 +180,12 @@ def audit_task(self, task): for attrs in attr_list: if attrs.type in [File, Directory]: input_name = attrs.name - entity_label = attrs.name input_path = os.path.abspath(getattr(task.inputs, input_name)) file_hash = hash_file(input_path) entity_id = f"uid:{gen_uuid()}" entity_message = { - "@id": entity_id, - "Label": entity_label, + "@id": entity_id, + "Label": input_name, "AtLocation": input_path, "GeneratedBy": None, "@type": "input", From bf593de26f5dafec57c50d239064dd2945006ad4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 24 Nov 2022 06:24:06 +0000 Subject: [PATCH 69/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/audit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index ce330526e4..ece23239e6 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -184,7 +184,7 @@ def audit_task(self, task): file_hash = hash_file(input_path) entity_id = f"uid:{gen_uuid()}" entity_message = { - "@id": entity_id, + "@id": entity_id, "Label": input_name, "AtLocation": input_path, "GeneratedBy": None, From 118b7cdd3e99435c077e57cbba8d09969bd45930 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Fri, 25 Nov 2022 23:12:18 -0800 Subject: [PATCH 70/94] Fixed tests --- pydra/engine/tests/test_task.py | 57 ++++++++++++--------------------- 1 file changed, 20 insertions(+), 37 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index cf71ed772e..09a5a5c1b1 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1007,7 +1007,7 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] funky.cache_dir = tmpdir funky() message_path = tmpdir / funky.checksum / "messages" - print(message_path) + for file in glob(str(message_path) + "/*.jsonld"): with open(file, "r") as f: @@ -1019,12 +1019,6 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] if "@type" in data: if data["@type"] == "input": assert None == data["Label"] - # placeholder for atlocation until - # new test is added - assert [] == data["AtLocation"] - - # assert data["Type"] == "input" - if "AssociatedWith" in data: assert None == data["AssociatedWith"] @@ -1070,19 +1064,21 @@ def test_audit_shellcommandtask(tmpdir): def test_audit_shellcommandtask_file(tmpdir): + # sourcery skip: use-fstring-for-concatenation + import glob import shutil - # create test.txt file with "This is a test" in it in the tmpdir - with open(tmpdir / "test.txt", "w") as f: - f.write("This is a test.") - # make a copy of the test.txt file in the tmpdir and name it test2.txt - shutil.copy(tmpdir / "test.txt", tmpdir / "test2.txt") + # create txt file in cwd + with open("test.txt", "w") as f: + f.write("This is a test") + + # copy the test.txt file to the tmpdir + shutil.copy("test.txt", tmpdir) + cmd = "cat" file_in = tmpdir / "test.txt" - file_in_2 = tmpdir / "test2.txt" test_file_hash = hash_file(file_in) - test_file_hash_2 = hash_file(file_in_2) my_input_spec = SpecInfo( name="Input", fields=[ @@ -1098,43 +1094,30 @@ def test_audit_shellcommandtask_file(tmpdir): }, ), ), - ( - "in_file_2", - attr.ib( - type=File, - metadata={ - "position": 2, - "argstr": "", - "help_string": "text", - "mandatory": True, - }, - ), - ), ], bases=(ShellSpec,), ) shelly = ShellCommandTask( name="shelly", in_file=file_in, - in_file_2=file_in_2, input_spec=my_input_spec, executable=cmd, audit_flags=AuditFlag.PROV, - messengers=PrintMessenger(), + messengers=FileMessenger(), ) shelly.cache_dir = tmpdir shelly() message_path = tmpdir / shelly.checksum / "messages" for file in glob.glob(str(message_path) + "/*.jsonld"): - with open(file, "r") as f: - data = json.load(f) - print(file_in) - if "AtLocation" in data: - assert data["AtLocation"] == [file_in, file_in_2] - if "digest" in data: - assert data["digest"] == [test_file_hash, test_file_hash_2] - - + with open(file, "r") as x: + data = json.load(x) + if "@type" in data: + if data["@type"] == "input": + if "AtLocation" in data: + assert data["AtLocation"] == str(file_in) + if "digest" in data: + assert data["digest"] == test_file_hash + def test_audit_shellcommandtask_version(tmpdir): import subprocess as sp From 15a754269c8bbb2c7bff8114dca57ac2d40271ee Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 26 Nov 2022 07:12:44 +0000 Subject: [PATCH 71/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/tests/test_task.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 09a5a5c1b1..8aebd8b1d5 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1007,7 +1007,6 @@ def testfunc(a: int, b: float = 0.1) -> ty.NamedTuple("Output", [("out", float)] funky.cache_dir = tmpdir funky() message_path = tmpdir / funky.checksum / "messages" - for file in glob(str(message_path) + "/*.jsonld"): with open(file, "r") as f: @@ -1067,6 +1066,7 @@ def test_audit_shellcommandtask_file(tmpdir): # sourcery skip: use-fstring-for-concatenation import glob import shutil + # create test.txt file with "This is a test" in it in the tmpdir # create txt file in cwd with open("test.txt", "w") as f: @@ -1075,7 +1075,6 @@ def test_audit_shellcommandtask_file(tmpdir): # copy the test.txt file to the tmpdir shutil.copy("test.txt", tmpdir) - cmd = "cat" file_in = tmpdir / "test.txt" test_file_hash = hash_file(file_in) @@ -1117,7 +1116,8 @@ def test_audit_shellcommandtask_file(tmpdir): assert data["AtLocation"] == str(file_in) if "digest" in data: assert data["digest"] == test_file_hash - + + def test_audit_shellcommandtask_version(tmpdir): import subprocess as sp From 662ade1d2866e08f7479a6c2f716d16150893d25 Mon Sep 17 00:00:00 2001 From: Ryan Cali Date: Sat, 26 Nov 2022 23:45:58 -0800 Subject: [PATCH 72/94] Added two file test back --- pydra/engine/tests/test_task.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 8aebd8b1d5..82f5287970 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1072,12 +1072,19 @@ def test_audit_shellcommandtask_file(tmpdir): with open("test.txt", "w") as f: f.write("This is a test") + with open("test2.txt", "w") as f: + f.write("This is a test") + # copy the test.txt file to the tmpdir shutil.copy("test.txt", tmpdir) + shutil.copy("test2.txt", tmpdir) + cmd = "cat" file_in = tmpdir / "test.txt" + file_in_2 = tmpdir / "test2.txt" test_file_hash = hash_file(file_in) + test_file_hash_2 = hash_file(file_in_2) my_input_spec = SpecInfo( name="Input", fields=[ @@ -1092,6 +1099,18 @@ def test_audit_shellcommandtask_file(tmpdir): "mandatory": True, }, ), + ), + ( + "in_file_2", + attr.ib( + type=File, + metadata={ + "position": 2, + "argstr": "", + "help_string": "text", + "mandatory": True, + }, + ), ), ], bases=(ShellSpec,), @@ -1099,6 +1118,7 @@ def test_audit_shellcommandtask_file(tmpdir): shelly = ShellCommandTask( name="shelly", in_file=file_in, + in_file_2=file_in_2, input_spec=my_input_spec, executable=cmd, audit_flags=AuditFlag.PROV, @@ -1112,11 +1132,12 @@ def test_audit_shellcommandtask_file(tmpdir): data = json.load(x) if "@type" in data: if data["@type"] == "input": - if "AtLocation" in data: + if data["Label"] == "in_file": assert data["AtLocation"] == str(file_in) - if "digest" in data: - assert data["digest"] == test_file_hash - + assert data["digest"] == test_file_hash + if data["Label"] == "in_file_2": + assert data["AtLocation"] == str(file_in_2) + assert data["digest"] == test_file_hash_2 def test_audit_shellcommandtask_version(tmpdir): import subprocess as sp From 19beeba10a52c0026485b5333900b7f9f92a4bf1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 27 Nov 2022 07:47:17 +0000 Subject: [PATCH 73/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pydra/engine/tests/test_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index 82f5287970..7ef386f348 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -1079,7 +1079,6 @@ def test_audit_shellcommandtask_file(tmpdir): shutil.copy("test.txt", tmpdir) shutil.copy("test2.txt", tmpdir) - cmd = "cat" file_in = tmpdir / "test.txt" file_in_2 = tmpdir / "test2.txt" @@ -1100,7 +1099,7 @@ def test_audit_shellcommandtask_file(tmpdir): }, ), ), - ( + ( "in_file_2", attr.ib( type=File, @@ -1139,6 +1138,7 @@ def test_audit_shellcommandtask_file(tmpdir): assert data["AtLocation"] == str(file_in_2) assert data["digest"] == test_file_hash_2 + def test_audit_shellcommandtask_version(tmpdir): import subprocess as sp From 7cb881541a4efa3543d27146e4b39fe6627d1732 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 17:27:30 +0000 Subject: [PATCH 74/94] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.3.0 → v4.4.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.3.0...v4.4.0) - [github.com/PyCQA/flake8: 5.0.4 → 6.0.0](https://github.com/PyCQA/flake8/compare/5.0.4...6.0.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1c0376116a..ad0af40dae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -18,6 +18,6 @@ repos: - id: codespell exclude: ^(pydra/_version\.py|versioneer\.py)$ - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 + rev: 6.0.0 hooks: - id: flake8 From 22264b7ec9fb805572284432484e51ea0b58414e Mon Sep 17 00:00:00 2001 From: Ghislain Vaillant Date: Mon, 28 Nov 2022 20:36:00 +0100 Subject: [PATCH 75/94] MNT: Add compatibility aliases Co-authored-by: Chris Markiewicz --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index bbf4003a21..197dfe1810 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,10 @@ test = [ "boutiques", "pympler", ] +# Aliases +tests = ["pydra[test]"] +docs = ["pydra[doc]"] +all = ["pydra[doc,dev]"] [project.urls] documentation = "https://nipype.github.io/pydra/" From cd95d0da9aeca6432130f20366b851c1ce31405e Mon Sep 17 00:00:00 2001 From: Ghislain Vaillant Date: Mon, 28 Nov 2022 20:37:13 +0100 Subject: [PATCH 76/94] MNT: Include test dependencies in dev group Co-authored-by: Chris Markiewicz --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 197dfe1810..692b5a8bcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ dask = [ dev = [ "black", "pre-commit", + "pydra[test]", ] doc = [ "attrs >=19.1.0", From cd5b38a9c53c1b374653f16178269cc79caa06ba Mon Sep 17 00:00:00 2001 From: Ghislain Vaillant Date: Tue, 29 Nov 2022 00:13:50 +0100 Subject: [PATCH 77/94] MNT: Remove .gitattributes file No longer required since the migration to Flit --- .gitattributes | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index a29f8b362d..0000000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -pydra/_version.py export-subst From 59219acb31d523b0da656e3e98648bb51108fd23 Mon Sep 17 00:00:00 2001 From: Ghislain Vaillant Date: Tue, 29 Nov 2022 00:15:13 +0100 Subject: [PATCH 78/94] MNT: Remove MANIFEST.in file No longer required since the migration to Flit --- MANIFEST.in | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 01c2de4725..0000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include versioneer.py -include pydra/_version.py -include pydra/schema/context.jsonld From 1cde34ec1210a4dad3ed874a3bffdfea0b33d8c6 Mon Sep 17 00:00:00 2001 From: Ghislain Vaillant Date: Mon, 28 Nov 2022 23:55:07 +0100 Subject: [PATCH 79/94] CI: Add Python 3.11 to test matrix --- .github/workflows/testpydra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 5c0bb16643..51ce5555f8 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: os: [macos-latest, ubuntu-latest, windows-latest] - python-version: [3.7, 3.8, 3.9, "3.10"] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] fail-fast: false runs-on: ${{ matrix.os }} From 161e7393e9cf2f97c94f489474a4693029b699bb Mon Sep 17 00:00:00 2001 From: Ghislain Vaillant Date: Tue, 29 Nov 2022 15:22:16 +0100 Subject: [PATCH 80/94] FIX: Ensure coroutines are converted to tasks prior to asyncio.wait --- pydra/engine/workers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index 3e87ccacce..54b8129232 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -58,7 +58,11 @@ async def fetch_finished(self, futures): done = set() try: done, pending = await asyncio.wait( - futures, return_when=asyncio.FIRST_COMPLETED + [ + asyncio.create_task(f) if not isinstance(f, asyncio.Task) else f + for f in futures + ], + return_when=asyncio.FIRST_COMPLETED, ) except ValueError: # nothing pending! @@ -105,7 +109,11 @@ async def fetch_finished(self, futures): try: self._jobs += len(futures) done, pending = await asyncio.wait( - futures, return_when=asyncio.FIRST_COMPLETED + [ + asyncio.create_task(f) if not isinstance(f, asyncio.Task) else f + for f in futures + ], + return_when=asyncio.FIRST_COMPLETED, ) except ValueError: # nothing pending! From 95dbe7fb9371b6f95b335e7e2e9ac4f8813405e8 Mon Sep 17 00:00:00 2001 From: Ghislain Vaillant Date: Tue, 29 Nov 2022 18:56:22 +0100 Subject: [PATCH 81/94] CI: Use wheel packages when installing --- .github/workflows/testpydra.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index 51ce5555f8..d738875aec 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -30,7 +30,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Update build tools - run: python -m pip install --upgrade pip build + run: | + python -m pip install --upgrade pip + python -m pip install build wheel - name: Build pydra run: python -m build From 4bf6ca1d2f1fb51b511c1f521bc04f8a776f3541 Mon Sep 17 00:00:00 2001 From: Ghislain Vaillant Date: Tue, 29 Nov 2022 19:08:56 +0100 Subject: [PATCH 82/94] CI: Do not test Python 3.11 on Windows Requires an update to lxml. See: https://github.com/lxml/lxml/pull/360 Co-authored-by: Chris Markiewicz --- .github/workflows/testpydra.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/testpydra.yml b/.github/workflows/testpydra.yml index d738875aec..49e26b9940 100644 --- a/.github/workflows/testpydra.yml +++ b/.github/workflows/testpydra.yml @@ -17,6 +17,9 @@ jobs: matrix: os: [macos-latest, ubuntu-latest, windows-latest] python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + exclude: + - os: 'windows-latest' + python-version: '3.11' fail-fast: false runs-on: ${{ matrix.os }} From d31d842bd3c9c5a29b0a98382a20aaa9806f4155 Mon Sep 17 00:00:00 2001 From: Ghislain Vaillant Date: Wed, 30 Nov 2022 17:04:36 +0100 Subject: [PATCH 83/94] MNT: Add myself to Zenodo --- .zenodo.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.zenodo.json b/.zenodo.json index 078c7a644a..12cac06da0 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -74,6 +74,11 @@ "affiliation": "MIT, HMS", "name": "Ghosh, Satrajit", "orcid": "0000-0002-5312-6729" + }, + { + "affiliation": "Paris Brain Institute", + "name": "Vaillant, Ghislain", + "orcid": "0000-0003-0267-3033" } ], "keywords": [ From 738552fe066af928cdedbbe0b9ddada76d02ae97 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 30 Nov 2022 09:42:04 -0500 Subject: [PATCH 84/94] MNT: Advertise Python and Windows compatibility --- pyproject.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 692b5a8bcf..085f350def 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,8 +32,13 @@ classifiers = [ "Intended Audience :: Science/Research", "License :: OSI Approved :: Apache Software License", "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering", ] dynamic = ["version"] From cf4e543f920d63be9bbe548a3224cc94237c7832 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 30 Nov 2022 09:43:50 -0500 Subject: [PATCH 85/94] MNT: Remove redundant requirements in [doc] extra --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 085f350def..adce6b24aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,9 +54,6 @@ dev = [ "pydra[test]", ] doc = [ - "attrs >=19.1.0", - "cloudpickle", - "filelock", "packaging", "sphinx >=2.1.2", "sphinx_rtd_theme", From b49039c7b0367b5c57588c74000eaff55cd67199 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 30 Nov 2022 09:50:10 -0500 Subject: [PATCH 86/94] FIX: Restore docs/requirements.txt --- .circleci/config.yml | 1 + docs/requirements.txt | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 docs/requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index 8c88c538d0..741e44e024 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,6 +48,7 @@ jobs: name: Install deps command: | pip install --upgrade pip + pip install --no-cache-dir -r docs/requirements.txt pip install --no-cache-dir .[doc] - run: name: Build only this commit diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000000..3196fdb2d3 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +# This requirements file includes unreleased dependencies above and beyond those +# specified in the project.optional-dependencies.doc table of pyproject.toml +git+https://github.com/AleksandarPetrov/napoleon.git@0dc3f28a309ad602be5f44a9049785a1026451b3#egg=sphinxcontrib-napoleon +git+https://github.com/effigies/sphinxcontrib-versioning.git@master#egg=sphinxcontrib-versioning From 3ce7eeb57490add048601b313df6854023b57668 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 30 Nov 2022 09:51:36 -0500 Subject: [PATCH 87/94] MNT: Remove setuptools/versioneer debris --- .flake8 | 1 - codecov.yml | 2 -- 2 files changed, 3 deletions(-) diff --git a/.flake8 b/.flake8 index a0478b8f62..ed35f02a1b 100644 --- a/.flake8 +++ b/.flake8 @@ -8,6 +8,5 @@ exclude = docs/tools/ docs/conf.py pydra/_version.py - versioneer.py max-line-length=105 ignore = E203,W503,F541 diff --git a/codecov.yml b/codecov.yml index 18a9bfa3ab..5da784c892 100644 --- a/codecov.yml +++ b/codecov.yml @@ -3,8 +3,6 @@ coverage: ignore: # files and folders that will be removed during processing - "**/tests" - "pydra/_version.py" - - "setup.py" - - "versioneer.py" status: project: default: From 020b95950af0b3ae7d41e26a698fa0ddf3a8b943 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 30 Nov 2022 10:00:54 -0500 Subject: [PATCH 88/94] DOC: Use built-in napoleon --- docs/conf.py | 2 +- docs/requirements.txt | 1 - pyproject.toml | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index c155320f0f..3e7e15d1e1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -45,8 +45,8 @@ "sphinx.ext.ifconfig", "sphinx.ext.linkcode", "sphinx.ext.githubpages", + "sphinx.ext.napoleon", "sphinxcontrib.apidoc", - "sphinxcontrib.napoleon", ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/requirements.txt b/docs/requirements.txt index 3196fdb2d3..fd5bbc2c82 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,3 @@ # This requirements file includes unreleased dependencies above and beyond those # specified in the project.optional-dependencies.doc table of pyproject.toml -git+https://github.com/AleksandarPetrov/napoleon.git@0dc3f28a309ad602be5f44a9049785a1026451b3#egg=sphinxcontrib-napoleon git+https://github.com/effigies/sphinxcontrib-versioning.git@master#egg=sphinxcontrib-versioning diff --git a/pyproject.toml b/pyproject.toml index adce6b24aa..ce7eb465dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,6 @@ doc = [ "sphinx >=2.1.2", "sphinx_rtd_theme", "sphinxcontrib-apidoc ~=0.3.0", - "sphinxcontrib-napoleon", "sphinxcontrib-versioning", ] test = [ From 65e1be356021c268077e84ed11dd409d0a33fd9e Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 30 Nov 2022 10:09:40 -0500 Subject: [PATCH 89/94] MNT: .gitignore pydra/_version.py --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index bced49b1bb..1263cb93e9 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ cov.xml .idea .DS_Store + +# This can be generated in-tree. We never want to commit it. +pydra/_version.py From b7b4f5eae20c56cdc03123674a2a3bbf4547a4bf Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 30 Nov 2022 10:20:43 -0500 Subject: [PATCH 90/94] DOC: Update Python intersphinx inventory --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 3e7e15d1e1..b3a1d23acc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -89,7 +89,7 @@ napoleon_custom_sections = [("Inputs", "Parameters"), ("Outputs", "Parameters")] # Intersphinx -intersphinx_mapping = {"https://docs.python.org/": None} +intersphinx_mapping = {"https://docs.python.org/3/": None} # Linkcode # The following is used by sphinx.ext.linkcode to provide links to github From b026b7873ad8e18d00c66dbd804f42733d1f8e92 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 30 Nov 2022 11:09:10 -0500 Subject: [PATCH 91/94] MNT: Keep Satra as last author --- .zenodo.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.zenodo.json b/.zenodo.json index 12cac06da0..7d81b12ac7 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -70,15 +70,15 @@ "name": "Close, Thomas G.", "orcid": "0000-0002-4160-2134" }, - { - "affiliation": "MIT, HMS", - "name": "Ghosh, Satrajit", - "orcid": "0000-0002-5312-6729" - }, { "affiliation": "Paris Brain Institute", "name": "Vaillant, Ghislain", "orcid": "0000-0003-0267-3033" + }, + { + "affiliation": "MIT, HMS", + "name": "Ghosh, Satrajit", + "orcid": "0000-0002-5312-6729" } ], "keywords": [ From fbc8c495d6fb283f6f9e4b8b7a57b59403de4c38 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 1 Dec 2022 00:46:38 -0500 Subject: [PATCH 92/94] adding test for wf graph when wf has aplitter; small fix of Docker/Singularity tasks; marking one test as flaky for slurm --- pydra/engine/task.py | 3 ++- pydra/engine/tests/test_workflow.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pydra/engine/task.py b/pydra/engine/task.py index 09d173171b..431ce9539c 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -686,7 +686,7 @@ class DockerTask(ContainerTask): def __init__( self, - name, + name=None, audit_flags: AuditFlag = AuditFlag.NONE, cache_dir=None, input_spec: ty.Optional[SpecInfo] = None, @@ -769,6 +769,7 @@ class SingularityTask(ContainerTask): def __init__( self, + name=None, audit_flags: AuditFlag = AuditFlag.NONE, cache_dir=None, input_spec: ty.Optional[SpecInfo] = None, diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index 553092ca87..bdae9e135b 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -4192,6 +4192,7 @@ def test_wf_upstream_error2(plugin, tmpdir): assert "raised an error" in str(excinfo.value) +@pytest.mark.flaky(reruns=2) # when slurm def test_wf_upstream_error3(plugin, tmpdir): """task2 dependent on task1, task1 errors, task-level split on task 1 goal - workflow finish running, one output errors but the other doesn't @@ -4455,13 +4456,15 @@ def exporting_graphs(wf, name): print("\n pdf of the detailed graph in: ", formatted_dot[0]) -def test_graph_1(tmpdir): +@pytest.mark.parametrize("splitter", [None, "x"]) +def test_graph_1(tmpdir, splitter): """creating a set of graphs, wf with two nodes""" wf = Workflow(name="wf", input_spec=["x", "y"], cache_dir=tmpdir) wf.add(multiply(name="mult_1", x=wf.lzin.x, y=wf.lzin.y)) wf.add(multiply(name="mult_2", x=wf.lzin.x, y=wf.lzin.x)) wf.add(add2(name="add2", x=wf.mult_1.lzout.out)) wf.set_output([("out", wf.add2.lzout.out)]) + wf.split(splitter) # simple graph dotfile_s = wf.create_dotfile() From 986d53941004bfc145d3d5220f67ba36dbffe328 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Thu, 1 Dec 2022 07:49:16 -0500 Subject: [PATCH 93/94] small fix --- pydra/engine/task.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pydra/engine/task.py b/pydra/engine/task.py index 431ce9539c..135af50796 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -809,6 +809,7 @@ def __init__( name="Inputs", fields=[], bases=(SingularitySpec,) ) super().__init__( + name=name, input_spec=input_spec, output_spec=output_spec, audit_flags=audit_flags, From f74ee502f78d4daa029430476c16a44dc2bdcee6 Mon Sep 17 00:00:00 2001 From: Dorota Jarecka Date: Fri, 2 Dec 2022 17:43:18 -0500 Subject: [PATCH 94/94] moving tests with xor to test_shelltask_inputspec.py --- .../engine/tests/test_shelltask_inputspec.py | 102 ++++++++++++++++++ pydra/engine/tests/test_specs.py | 102 ------------------ 2 files changed, 102 insertions(+), 102 deletions(-) diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index 240502dc1d..cbb9a28e4b 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -2133,3 +2133,105 @@ def test_shell_cmd_inputs_di(tmpdir, use_validator): image_dimensionality=5, ) assert "value of image_dimensionality" in str(excinfo.value) + + +# tests with XOR in input metadata + + +class SimpleTaskXor(ShellCommandTask): + input_fields = [ + ( + "input_1", + str, + { + "help_string": "help", + "mandatory": True, + "xor": ("input_1", "input_2", "input_3"), + }, + ), + ( + "input_2", + bool, + { + "help_string": "help", + "mandatory": True, + "argstr": "--i2", + "xor": ("input_1", "input_2", "input_3"), + }, + ), + ( + "input_3", + bool, + { + "help_string": "help", + "mandatory": True, + "xor": ("input_1", "input_2", "input_3"), + }, + ), + ] + task_input_spec = SpecInfo(name="Input", fields=input_fields, bases=(ShellSpec,)) + task_output_fields = [] + task_output_spec = SpecInfo( + name="Output", fields=task_output_fields, bases=(ShellOutSpec,) + ) + + input_spec = task_input_spec + output_spec = task_output_spec + executable = "cmd" + + +def test_task_inputs_mandatory_with_xOR_one_mandatory_is_OK(): + """input spec with mandatory inputs""" + task = SimpleTaskXor() + task.inputs.input_1 = "Input1" + task.inputs.input_2 = attr.NOTHING + task.inputs.check_fields_input_spec() + + +def test_task_inputs_mandatory_with_xOR_one_mandatory_out_3_is_OK(): + """input spec with mandatory inputs""" + task = SimpleTaskXor() + task.inputs.input_1 = attr.NOTHING + task.inputs.input_2 = attr.NOTHING + task.inputs.input_3 = True + task.inputs.check_fields_input_spec() + + +def test_task_inputs_mandatory_with_xOR_zero_mandatory_raises_error(): + """input spec with mandatory inputs""" + task = SimpleTaskXor() + task.inputs.input_1 = attr.NOTHING + task.inputs.input_2 = attr.NOTHING + with pytest.raises(Exception) as excinfo: + task.inputs.check_fields_input_spec() + assert "input_1 is mandatory, but no value provided" in str(excinfo.value) + assert excinfo.type is AttributeError + + +def test_task_inputs_mandatory_with_xOR_two_mandatories_raises_error(): + """input spec with mandatory inputs""" + task = SimpleTaskXor() + task.inputs.input_1 = "Input1" + task.inputs.input_2 = True + + with pytest.raises(Exception) as excinfo: + task.inputs.check_fields_input_spec() + assert "input_2 is mutually exclusive with ('input_1', 'input_2'" in str( + excinfo.value + ) + assert excinfo.type is AttributeError + + +def test_task_inputs_mandatory_with_xOR_3_mandatories_raises_error(): + """input spec with mandatory inputs""" + task = SimpleTaskXor() + task.inputs.input_1 = "Input1" + task.inputs.input_2 = True + task.inputs.input_3 = False + + with pytest.raises(Exception) as excinfo: + task.inputs.check_fields_input_spec() + assert "input_2 is mutually exclusive with ('input_1', 'input_2', 'input_3'" in str( + excinfo.value + ) + assert excinfo.type is AttributeError diff --git a/pydra/engine/tests/test_specs.py b/pydra/engine/tests/test_specs.py index 17cd7b0b82..a0af757546 100644 --- a/pydra/engine/tests/test_specs.py +++ b/pydra/engine/tests/test_specs.py @@ -14,12 +14,9 @@ DockerSpec, SingularitySpec, LazyField, - ShellOutSpec, ) -from ..task import TaskBase, ShellCommandTask from ..helpers import make_klass import pytest -import attr def test_basespec(): @@ -369,102 +366,3 @@ def test_input_file_hash_5(tmpdir): f.write("hi") hash3 = inputs(in_file=[{"file": file_diffcontent, "int": 3}]).hash assert hash1 != hash3 - - -class SimpleTask(ShellCommandTask): - input_fields = [ - ( - "input_1", - str, - { - "help_string": "help", - "mandatory": True, - "xor": ("input_1", "input_2", "input_3"), - }, - ), - ( - "input_2", - bool, - { - "help_string": "help", - "mandatory": True, - "argstr": "--i2", - "xor": ("input_1", "input_2", "input_3"), - }, - ), - ( - "input_3", - bool, - { - "help_string": "help", - "mandatory": True, - "xor": ("input_1", "input_2", "input_3"), - }, - ), - ] - task_input_spec = SpecInfo(name="Input", fields=input_fields, bases=(ShellSpec,)) - task_output_fields = [] - task_output_spec = SpecInfo( - name="Output", fields=task_output_fields, bases=(ShellOutSpec,) - ) - - input_spec = task_input_spec - output_spec = task_output_spec - executable = "cmd" - - -def test_task_inputs_mandatory_with_xOR_one_mandatory_is_OK(): - """input spec with mandatory inputs""" - task = SimpleTask() - task.inputs.input_1 = "Input1" - task.inputs.input_2 = attr.NOTHING - task.inputs.check_fields_input_spec() - - -def test_task_inputs_mandatory_with_xOR_one_mandatory_out_3_is_OK(): - """input spec with mandatory inputs""" - task = SimpleTask() - task.inputs.input_1 = attr.NOTHING - task.inputs.input_2 = attr.NOTHING - task.inputs.input_3 = True - task.inputs.check_fields_input_spec() - - -def test_task_inputs_mandatory_with_xOR_zero_mandatory_raises_error(): - """input spec with mandatory inputs""" - task = SimpleTask() - task.inputs.input_1 = attr.NOTHING - task.inputs.input_2 = attr.NOTHING - with pytest.raises(Exception) as excinfo: - task.inputs.check_fields_input_spec() - assert "input_1 is mandatory, but no value provided" in str(excinfo.value) - assert excinfo.type is AttributeError - - -def test_task_inputs_mandatory_with_xOR_two_mandatories_raises_error(): - """input spec with mandatory inputs""" - task = SimpleTask() - task.inputs.input_1 = "Input1" - task.inputs.input_2 = True - - with pytest.raises(Exception) as excinfo: - task.inputs.check_fields_input_spec() - assert "input_2 is mutually exclusive with ('input_1', 'input_2'" in str( - excinfo.value - ) - assert excinfo.type is AttributeError - - -def test_task_inputs_mandatory_with_xOR_3_mandatories_raises_error(): - """input spec with mandatory inputs""" - task = SimpleTask() - task.inputs.input_1 = "Input1" - task.inputs.input_2 = True - task.inputs.input_3 = False - - with pytest.raises(Exception) as excinfo: - task.inputs.check_fields_input_spec() - assert "input_2 is mutually exclusive with ('input_1', 'input_2', 'input_3'" in str( - excinfo.value - ) - assert excinfo.type is AttributeError