8000 Add Requirement parsing into correct type · phenoflow/python-cwlgen@acdda69 · GitHub
[go: up one dir, main page]
8000 Skip to content

Commit acdda69

Browse files
illusionalillusional
committed
Add Requirement parsing into correct type
1 parent f46bdfe commit acdda69

File tree

7 files changed

+188
-31
lines changed

7 files changed

+188
-31
lines changed

‎cwlgen/commandlinetool.py

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ class CommandInputParameter(Parameter):
4141
An input parameter for a :class:`cwlgen.CommandLineTool`.
4242
"""
4343

44-
parse_types = [("inputBinding", [CommandLineBinding])]
44+
parse_types = {
45+
"inputBinding": [CommandLineBinding]
46+
}
4547

4648
def __init__(
4749
self,
@@ -96,7 +98,7 @@ class CommandOutputParameter(Parameter):
9698
An output parameter for a :class:`cwlgen.CommandLineTool`.
9799
"""
98100

99-
parse_types = [("outputBinding", [CommandOutputBinding])]
101+
parse_types = {"outputBinding": [CommandOutputBinding]}
100102

101103
def __init__(
102104
self,
@@ -149,10 +151,10 @@ class CommandLineTool(Serializable):
149151

150152
__CLASS__ = "CommandLineTool"
151153

152-
parse_types = [
153-
("inputs", [[CommandInputParameter]]),
154-
("outputs", [[CommandOutputParameter]]),
155-
]
154+
parse_types = {
155+
"inputs": [[CommandInputParameter]],
156+
"outputs": [[CommandOutputParameter]],
157+
}
156158
ignore_fields_on_parse = ["namespaces", "class"]
157159

158160
def __init__(

‎cwlgen/import_cwl.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,11 @@ def parse_cwl(cwl_path):
3232
return parse_cwl_dict(cwl_dict)
3333

3434

35+
def parse_cwl_string(cwlstr):
36+
cwl_dict = ryaml.load(cwlstr, Loader=ryaml.Loader)
37+
return parse_cwl_dict(cwl_dict)
38+
39+
3540
def parse_cwl_dict(cwl_dict):
3641
cl = cwl_dict['class']
3742

‎cwlgen/requirements.py

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
import six
2+
from cwlgen.commandlinebinding import CommandLineBinding
23

34
from .common import parse_type, get_type_dict
45
from .utils import Serializable
56

67

78
class Requirement(Serializable):
9+
10+
ignore_fields_on_parse = ["class"]
11+
812
'''
913
Requirement that must be met in order to execute the process.
1014
'''
11-
1215
def __init__(self, req_class):
1316
'''
1417
:param req_class: requirement class
@@ -20,6 +23,23 @@ def __init__(self, req_class):
2023
def get_class(self):
2124
return self._req_class
2225

26+
@classmethod
27+
def parse_dict(cls, d):
28+
29+
c = d["class"]
30+
requirements = [
31+
InlineJavascriptReq, SchemaDefRequirement, SoftwareRequirement, InitialWorkDirRequirement,
32+
SubworkflowFeatureRequirement, ScatterFeatureRequirement, MultipleInputFeatureRequirement,
33+
StepInputExpressionRequirement, DockerRequirement, EnvVarRequirement, ShellCommandRequirement,
34+
ResourceRequirement
35+
]
36+
37+
for Req in requirements:
38+
if Req.__name__ == c:
39+
return Req.parse_dict_generic(Req, d)
40+
41+
return None
42+
2343

2444
class InlineJavascriptReq(Requirement):
2545
"""
@@ -73,6 +93,11 @@ def __init__(self, label=None, name=None):
7393
self.name = name
7494
self.type = "record"
7595

96+
def parse_dict(cls, d):
97+
if d["type"] != "record":
98+
return None
99+
return cls.parse_dict_generic(cls, d)
100+
76101
class InputRecordField(Serializable):
77102
"""
78103
Documentation: https://www.commonwl.org/v1.0/Workflow.html#InputRecordField
@@ -99,6 +124,10 @@ def get_dict(self):
99124
d["type"] = get_type_dict(self.type)
100125
return d
101126

127+
parse_types = {
128+
"fields": [InputRecordField]
129+
}
130+
102131
class InputEnumSchema(Serializable):
103132
"""
104133
Documentation: https://www.commonwl.org/v1.0/Workflow.html#InputEnumSchema
@@ -120,6 +149,15 @@ def __init__(self, symbols, label=None, name=None, input_binding=None):
120149
self.name = name
121150
self.inputBinding = input_binding
122151

152+
def parse_dict(cls, d):
153+
if d["type"] != "enum":
154+
return None
155+
return cls.parse_dict_generic(cls, d)
156+
157+
parse_types = {
158+
"inputBinding": CommandLineBinding
159+
}
160+
123161
class InputArraySchema(Serializable):
124162
"""
125163
Documentation: https://www.commonwl.org/v1.0/Workflow.html#InputArraySchema
@@ -140,6 +178,15 @@ def __init__(self, items, label=None, input_binding=None):
140178
self.label = label
141179
self.inputBinding = input_binding
142180

181+
def parse_dict(cls, d):
182+
if d["type"] != "record":
183+
return None
184+
return cls.parse_dict_generic(cls, d)
185+
186+
parse_types = {
187+
"fields": [InputRecordSchema, InputEnumSchema, InputArraySchema]
188+
}
189+
143190

144191
class SoftwareRequirement(Requirement):
145192
"""
@@ -166,6 +213,10 @@ def __init__(self, package, version=None, specs=None):
166213
self.version = version
167214
self.specs = specs
168215

216+
parse_types = {
217+
"package": [SoftwarePackage]
218+
}
219+
169220

170221
class InitialWorkDirRequirement(Requirement):
171222
"""
@@ -211,6 +262,10 @@ def __init__(self, entry, entryname=None, writable=None):
211262
self.entryname = entryname
212263
self.writable = writable
213264

265+
parse_types = {
266+
"listing": [Dirent, str]
267+
}
268+
214269

215270
class SubworkflowFeatureRequirement(Requirement):
216271
"""

‎cwlgen/utils.py

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,17 +14,27 @@ def literal_presenter(dumper, data):
1414

1515
class Serializable(object):
1616
"""
17-
Serializable docstring
17+
The Serializable class contains logic to automatically serialize a class based on
18+
its attributes. This behaviour can be overridden via the ``get_dict`` method on its
19+
subclasses with a call to super. Fields can be ignored by the base converter through
20+
the ``ignore_field_on_convert`` static attribute on your subclass.
21+
22+
The parsing behaviour (beta) is similar, however it will attempt to set all attributes
23+
from the dictionary onto a newly initialised class. If your initialiser has required
24+
arguments, this converter will do its best to pull the id out of the dictionary to provide
25+
to your initializer (or pull it from the { $id: value } dictionary). Typing hints can be
26+
provided by the ``parse_types`` static attribute, and required attributes can be tagged
27+
the ``required_fields`` attribute.
1828
"""
1929

2030

2131
"""
22-
This is a special field, with format: [(fieldName: str, [Serializable])]
32+
This is a special field, with format: {fieldName: str, [Serializable]}
2333
If the field name is present in the dict, then it will call the parse_dict(cls, d)
24-
method on that type. It should return None if it can't parse that dictionary. This means
25-
the type will need to override the parse_dict method.
34+
method on that type. It should return None if it can't parse that dictionary. This
35+
means the type will need to override the ``parse_dict`` method.
2636
"""
27-
parse_types = [] # type: [(str, [type])]
37+
parse_types = {} # type: {str, [type]}
2838
ignore_fields_on_parse = []
2939
ignore_fields_on_convert = []
3040
required_fields = [] # type: str
@@ -67,18 +77,33 @@ def get_dict(self):
6777

6878
@classmethod
6979
def parse_dict(cls, d):
70-
pts = {t[0]: t[1] for t in cls.parse_types}
71-
req = {r: False for r in cls.required_fields}
72-
ignore = set(cls.ignore_fields_on_parse)
80+
return cls.parse_dict_generic(cls, d)
81+
82+
@staticmethod
83+
def parse_dict_generic(T, d, parse_types=None, required_fields=None, ignore_fields_on_parse=None):
84+
85+
if parse_types is None and hasattr(T, "parse_types"):
86+
parse_types = T.parse_types
87+
if required_fields is None and hasattr(T, "required_fields"):
88+
required_fields = T.required_fields
89+
if ignore_fields_on_parse is None and hasattr(T, "ignore_fields_on_parse"):
90+
ignore_fields_on_parse = T.ignore_fields_on_parse
91+
92+
pts = parse_types
93+
req = {r: False for r in required_fields}
94+
ignore = set(ignore_fields_on_parse)
7395

7496
# may not be able to just initialise blank class
7597
# but we can use inspect to get required params and init using **kwargs
76-
required_init_kwargs = cls.get_required_input_params_for_cls(cls, d)
77-
self = cls(**required_init_kwargs)
98+
try:
99+
required_init_kwargs = T.get_required_input_params_for_cls(T, d)
100+
self = T(**required_init_kwargs)
101+
except:
102+
return None
78103

79104
for k, v in d.items():
80105
if k in ignore: continue
81-
val = cls.try_parse(v, pts.get(k))
106+
val = T.try_parse(v, pts.get(k))
82107
if val is None: continue
83108
if not hasattr(self, k):
84109
raise KeyError("Key '%s' does not exist on type '%s'" % (k, type(self)))
@@ -88,9 +113,9 @@ def parse_dict(cls, d):
88113
if not all(req.values()):
89114
# There was a required field that wasn't mapped
90115
req_fields = ", ".join(r for r in req if not req[r])
91-
clsname = cls.__name__
116+
clsname = T.__name__
92117

93-
raise Exception("The fields %s were not found when parsing type '%",format(req_fields, clsname))
118+
raise Exception("The fields %s were not found when parsing type '%", format(req_fields, clsname))
94119

95120
return self
96121

@@ -110,7 +135,7 @@ def get_required_input_params_for_cls(cls, valuesdict):
110135
argspec = inspect.getargspec(cls.__init__)
111136

112137
args, defaults = argspec.args, argspec.defaults
113-
required_param_keys = set(args[1:-len(defaults)]) if len(defaults) > 0 else args[1:]
138+
required_param_keys = set(args[1:-len(defaults)]) if defaults is not None and len(defaults) > 0 else args[1:]
114139

115140
inspect_ignore_keys = {"self", "args", "kwargs"}
116141
# Params can't shadow the built in 'id', so we'll put in a little hack
@@ -150,6 +175,8 @@ def try_parse_type(value, T):
150175
# if T is a primitive (str, bool, int, float), just return the T representation of retval
151176
elif T in _unparseable_types:
152177
try:
178+
if isinstance(value, list):
179+
return [T(v) for v in value]
153180
return T(value)
154181
except:
155182
return None

‎cwlgen/workflow.py

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,9 @@
77
import ruamel.yaml
88
import six
99

10-
from .version import __version__
11-
1210
# Internal libraries
1311

12+
from .requirements import Requirement
1413
from .utils import literal, literal_presenter, Serializable
1514
from .common import Parameter, CWL_SHEBANG
1615
from .workflowdeps import InputParameter, WorkflowOutputParameter, WorkflowStep
@@ -37,12 +36,12 @@ class Workflow(Serializable):
3736
"""
3837
__CLASS__ = 'Workflow'
3938
ignore_fields_on_parse = ["class"]
40-
ignore_fields_on_convert = ["inputs", "outputs"]
41-
parse_types = [
42-
("inputs", [[InputParameter]]),
43-
("outputs", [[WorkflowOutputParameter]]),
44-
("steps", [[WorkflowStep]])
45-
]
39+
ignore_fields_on_convert = ["inputs", "outputs", "requirements"]
40+
parse_types = {
41+
"inputs": [[InputParameter]],
42+
"outputs": [[WorkflowOutputParameter]],
43+
"steps": [[WorkflowStep]],
44+
}
4645

4746
def __init__(self, workflow_id=None, label=None, doc=None, cwl_version='v1.0'):
4847
"""
@@ -85,6 +84,19 @@ def get_dict(self):
8584

8685
return cwl_workflow
8786

87+
@classmethod
88+
def parse_dict(cls, d):
89+
wf = super(Workflow, cls).parse_dict(d)
90+
91+
reqs = d.get("requirements")
92+
if reqs:
93+
if isinstance(reqs, list):
94+
wf.requirements = [Requirement.parse_dict(r) for r in reqs]
95+
elif isinstance(reqs, dict):
96+
wf.requirements = [Requirement.parse_dict({**r, "class": c}) for c, r in reqs.items()]
97+
98+
return wf
99+
88100
def export_string(self):
89101
ruamel.yaml.add_representer(literal, literal_presenter)
90102
cwl_tool = self.get_dict()

‎cwlgen/workflowdeps.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,9 @@ class WorkflowStep(Serializable):
143143
Documentation: https://www.commonwl.org/v1.0/Workflow.html#WorkflowStep
144144
"""
145145

146-
parse_types = [("inputs", [[WorkflowStepInput]])]
146+
parse_types = {
147+
"inputs": [[WorkflowStepInput]]
148+
}
147149

148150
def __init__(self, step_id, run, label=None, doc=None, scatter=None, scatter_method=None):
149151
"""

‎test/test_unit_import_workflow.py

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313
import ruamel.yaml as ryaml
1414

1515
# External libraries
16-
from cwlgen.import_cwl import parse_cwl_dict
16+
import cwlgen.requirements as Requirements
17+
from cwlgen.import_cwl import parse_cwl_dict, parse_cwl_string
1718

1819
# Class(es) ------------------------------
1920

@@ -109,3 +110,56 @@ def test_load_id(self):
109110
def test_load_source(self):
110111
self.assertEqual(self.wf.steps[0].inputs[0].source, "untar/extracted_file")
111112

113+
114+
class TestIntegrationImportWorkflow(unittest.TestCase):
115+
def test_issue_25_requirements(self):
116+
wfstr = """\
117+
class: Workflow
118+
cwlVersion: v1.0
119+
120+
label: joint calling workflow
121+
doc: Perform joint calling on multiple sets aligned reads from the same family.
122+
123+
requirements:
124+
- class: MultipleInputFeatureRequirement
125+
- class: StepInputExpressionRequirement
126+
- class: SubworkflowFeatureRequirement"""
127+
wf = parse_cwl_string(wfstr)
128+
self.assertEqual(3, len(wf.requirements))
129+
self.assertIsInstance(wf.requirements[0], Requirements.MultipleInputFeatureRequirement)
130+
self.assertIsInstance(wf.requirements[1], Requirements.StepInputExpressionRequirement)
131+
self.assertIsInstance(wf.requirements[2], Requirements.SubworkflowFeatureRequirement)
132+
133+
wfd = wf.get_dict()
134+
expected = {
135+
'MultipleInputFeatureRequirement': {},
136+
'StepInputExpressionRequirement': {},
137+
'SubworkflowFeatureRequirement': {}
138+
}
139+
self.assertDictEqual(expected, wfd["requirements"])
140+
141+
def test_issue_25_requirements_dict(self):
142+
wfstr = """\
143+
class: Workflow
144+
cwlVersion: v1.0
145+
146+
label: joint calling workflow
147+
doc: Perform joint calling on multiple sets aligned reads from the same family.
148+
149+
requirements:
150+
MultipleInputFeatureRequirement: {}
151+
StepInputExpressionRequirement: {}
152+
SubworkflowFeatureRequirement: {}"""
153+
wf = parse_cwl_string(wfstr)
154+
self.assertEqual(3, len(wf.requirements))
155+
self.assertIsInstance(wf.requirements[0], Requirements.MultipleInputFeatureRequirement)
156+
self.assertIsInstance(wf.requirements[1], Requirements.StepInputExpressionRequirement)
157+
self.assertIsInstance(wf.requirements[2], Requirements.SubworkflowFeatureRequirement)
158+
159+
wfd = wf.get_dict()
160+
expected = {
161+
'MultipleInputFeatureRequirement': {},
162+
'StepInputExpressionRequirement': {},
163+
'SubworkflowFeatureRequirement': {}
164+
}
165+
self.assertDictEqual(expected, wfd["requirements"])

0 commit comments

Comments
 (0)
0