8000 Implement dependency mapping for environment modules, galaxy packages… · galaxyproject/galaxy@495802d · GitHub
[go: up one dir, main page]
Skip to content

Commit 495802d

Browse files
jmchiltonjmchilton
committed
Implement dependency mapping for environment modules, galaxy packages, and Conda.
1 parent 4a544e9 commit 495802d

File tree

5 files changed

+172
-14
lines changed

5 files changed

+172
-14
lines changed

‎lib/galaxy/tools/deps/resolvers/__init__.py

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
abstractproperty,
66
)
77

8+
import yaml
9+
10+
from galaxy.util import listify
811
from galaxy.util.dictifiable import Dictifiable
912

1013
from ..requirements import ToolRequirement
@@ -65,6 +68,72 @@ def _to_requirement(self, name, version=None):
6568
return ToolRequirement(name=name, type="package", version=version)
6669

6770

71+
class MappableDependencyResolver:
72+
"""Mix this into a ``DependencyResolver`` to allow mapping files.
73+
74+
Mapping files allow adapting generic requirements to specific local implementations.
75+
"""
76+
77+
def _setup_mapping(self, dependency_manager, **kwds):
78+
mapping_files = dependency_manager.get_resolver_option(self, "mapping_file", explicit_resolver_options=kwds)
79+
mappings = []
80+
if mapping_files:
81+
mapping_files = listify(mapping_files)
82+
for mapping_file in mapping_files:
83+
mappings.extend(MappableDependencyResolver._mapping_file_to_list(mapping_file))
84+
self._mappings = mappings
85+
86+
@staticmethod
87+
def _mapping_file_to_list(mapping_file):
88+
with open(mapping_file, "r") as f:
89+
raw_mapping = yaml.load(f)
90+
return map(RequirementMapping.from_dict, raw_mapping)
91+
92+
def _expand_mappings(self, requirement):
93+
for mapping in self._mappings:
94+
if requirement.name == mapping.from_name:
95+
if mapping.from_version is not None and mapping.from_version != requirement.version:
96+
continue
97+
98+
requirement = requirement.copy()
99+
requirement.name = mapping.to_name
100+
if mapping.to_version is not None:
101+
requirement.version = mapping.to_version
102+
103+
break
104+
105+
return requirement
106+
107+
108+
class RequirementMapping(object):
109+
110+
def __init__(self, from_name, from_version, to_name, to_version):
111+
self.from_name = from_name
112+
self.from_version = from_version
113+
self.to_name = to_name
114+
self.to_version = to_version
115+
116+
@staticmethod
117+
def from_dict(raw_mapping):
118+
from_raw = raw_mapping.get("from")
119+
if isinstance(from_raw, dict):
120+
from_name = from_raw.get("name")
121+
from_version = str(from_raw.get("version"))
122+
else:
123+
from_name = from_raw
124+
from_version = None
125+
126+
to_raw = raw_mapping.get("to")
127+
if isinstance(to_raw, dict):
128+
to_name = to_raw.get("name", from_name)
129+
to_version = str(to_raw.get("version"))
130+
else:
131+
to_name = to_raw
132+
to_version = None
133+
134+
return RequirementMapping(from_name, from_version, to_name, to_version)
135+
136+
68137
class SpecificationAwareDependencyResolver:
69138
"""Mix this into a :class:`DependencyResolver` to implement URI specification matching.
70139

‎lib/galaxy/tools/deps/resolvers/conda.py

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
DependencyResolver,
3030
InstallableDependencyResolver,
3131
ListableDependencyResolver,
32+
MappableDependencyResolver,
3233
MultipleDependencyResolver,
3334
NullDependency,
3435
SpecificationPatternDependencyResolver,
@@ -42,7 +43,7 @@
4243
log = logging.getLogger(__name__)
4344

4445

45-
class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver):
46+
class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver, MappableDependencyResolver):
4647
dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install']
4748
resolver_type = "conda"
4849
config_options = {
@@ -57,6 +58,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li
5758
_specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+")
5859

5960
def __init__(self, dependency_manager, **kwds):
61+
self._setup_mapping(dependency_manager, **kwds)
6062
self.versionless = _string_as_bool(kwds.get('versionless', 'false'))
6163
self.dependency_manager = dependency_manager
6264

@@ -146,7 +148,7 @@ def resolve_all(self, requirements, **kwds):
146148

147149
conda_targets = []
148150
for requirement in requirements:
149-
requirement = self._expand_specs(requirement)
151+
requirement = self._expand_requirement(requirement)
150152

151153
version = requirement.version
152154
if self.versionless:
@@ -186,7 +188,7 @@ def merged_environment_name(self, conda_targets):
186188
return conda_targets[0].install_environment
187189

188190
def resolve(self, requirement, **kwds):
189-
requirement = self._expand_specs(requirement)
191+
requirement = self._expand_requirement(requirement)
190192
name, version, type = requirement.name, requirement.version, requirement.type
191193

192194
# Check for conda just not being there, this way we can enable
@@ -236,6 +238,9 @@ def resolve(self, requirement, **kwds):
236238
preserve_python_environment=preserve_python_environment,
237239
)
238240

241+
def _expand_requirement(self, requirement):
242+
return self._expand_specs(self._expand_mappings(requirement))
243+
239244
def list_dependencies(self):
240245
for install_target in installed_conda_targets(self.conda_context):
241246
name = install_target.package

‎lib/galaxy/tools/deps/resolvers/galaxy_packages.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
Dependency,
1717
DependencyResolver,
1818
ListableDependencyResolver,
19+
MappableDependencyResolver,
1920
NullDependency,
2021
)
2122

@@ -103,9 +104,17 @@ def _galaxy_package_dep( self, path, version, name, exact ):
103104
return NullDependency(version=version, name=name)
104105

105106

106-
class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver):
107+
class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver, MappableDependencyResolver):
107108
resolver_type = "galaxy_packages"
108109

110+
def __init__(self, dependency_manager, **kwds):
111+
super(GalaxyPackageDependencyResolver, self).__init__(dependency_manager, **kwds)
112+
self._setup_mapping(dependency_manager, **kwds)
113+
114+
def resolve(self, requirement, **kwds):
115+
requirement = self._expand_mappings(requirement)
116+
return super(GalaxyPackageDependencyResolver, self).resolve(requirement, **kwds)
117+
109118
def list_dependencies(self):
110119
base_path = self.base_path
111120
for package_name in listdir(base_path):

‎lib/galaxy/tools/deps/resolvers/modules.py

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,12 @@
1313

1414
from six import StringIO
1515

16-
from ..resolvers import Dependency, DependencyResolver, NullDependency
16+
from ..resolvers import (
17+
Dependency,
18+
DependencyResolver,
19+
MappableDependencyResolver,
20+
NullDependency,
21+
)
1722

1823
log = logging.getLogger( __name__ )
1924

@@ -24,11 +29,12 @@
2429
UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s"
2530

2631

27-
class ModuleDependencyResolver(DependencyResolver):
32+
class ModuleDependencyResolver(DependencyResolver, MappableDependencyResolver):
2833
dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'modulepath']
2934
resolver_type = "modules"
3035

3136
def __init__(self, dependency_manager, **kwds):
37+
self._setup_mapping(dependency_manager, **kwds)
3238
self.versionless = _string_as_bool(kwds.get('versionless', 'false'))
3339
find_by = kwds.get('find_by', 'avail')
3440
prefetch = _string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH))
@@ -52,6 +58,7 @@ def __default_modulespath(self):
5258
return module_path
5359

5460
def resolve(self, requirement, **kwds):
61+
requirement = self._expand_mappings(requirement)
5562
name, version, type = requirement.name, requirement.version, requirement.type
5663

5764
if type != "package":

‎test/unit/tools/test_tool_deps.py

Lines changed: 76 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -115,13 +115,7 @@ def __build_ts_test_package(base_path, script_contents=''):
115115

116116
def test_module_dependency_resolver():
117117
with __test_base_path() as temp_directory:
118-
module_script = os.path.join(temp_directory, "modulecmd")
119-
__write_script(module_script, '''#!/bin/sh
120-
cat %s/example_output 1>&2;
121-
''' % temp_directory)
122-
with open(os.path.join(temp_directory, "example_output"), "w") as f:
123-
# Subset of module avail from MSI cluster.
124-
f.write('''
118+
module_script = _setup_module_command(temp_directory, '''
125119
-------------------------- /soft/modules/modulefiles ---------------------------
126120
JAGS/3.2.0-gcc45
127121
JAGS/3.3.0-gcc4.7.2
@@ -141,7 +135,7 @@ def test_module_dependency_resolver():
141135
advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035
142136
advisor/2013/update3 intel/12.0 mkl/10.2.7.041
143137
''')
144-
resolver = ModuleDependencyResolver(None, modulecmd=module_script)
138+
resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script)
145139
module = resolver.resolve( ToolRequirement( name="R", version=None, type="package" ) )
146140
assert module.module_name == "R"
147141
assert module.module_version is None
@@ -154,6 +148,74 @@ def test_module_dependency_resolver():
154148
assert isinstance(module, NullDependency)
155149

156150

151+
def test_module_resolver_with_mapping():
152+
with __test_base_path() as temp_directory:
153+
module_script = _setup_module_command(temp_directory, '''
154+
-------------------------- /soft/modules/modulefiles ---------------------------
155+
blast/2.24
156+
''')
157+
mapping_file = os.path.join(temp_directory, "mapping")
158+
with open(mapping_file, "w") as f:
159+
f.write('''
160+
- from: blast+
161+
to: blast
162+
''')
163+
164+
resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_file=mapping_file)
165+
module = resolver.resolve( ToolRequirement( name="blast+", version="2.24", type="package" ) )
166+
assert module.module_name == "blast"
167+
assert module.module_version == "2.24", module.module_version
168+
169+
170+
def test_module_resolver_with_mapping_versions():
171+
with __test_base_path() as temp_directory:
172+
module_script = _setup_module_command(temp_directory, '''
173+
-------------------------- /soft/modules/modulefiles ---------------------------
174+
blast/2.22.0-mpi
175+
blast/2.23
176+
blast/2.24.0-mpi
177+
''')
178+
mapping_file = os.path.join(temp_directory, "mapping")
179+
with open(mapping_file, "w") as f:
180+
f.write('''
181+
- from:
182+
name: blast+
183+
version: 2.24
184+
to:
185+
name: blast
186+
version: 2.24.0-mpi
187+
- from:
188+
name: blast
189+
version: 2.22
190+
to:
191+
version: 2.22.0-mpi
192+
''')
193+
194+
resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_file=mapping_file)
195+
module = resolver.resolve( ToolRequirement( name="blast+", version="2.24", type="package" ) )
196+
assert module.module_name == "blast"
197+
assert module.module_version == "2.24.0-mpi", module.module_version
198+
199+
resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_file=mapping_file)
200+
module = resolver.resolve( ToolRequirement( name="blast+", version="2.23", type="package" ) )
201+
assert isinstance(module, NullDependency)
202+
203+
module = resolver.resolve( ToolRequirement( name="blast", version="2.22", type="package" ) )
204+
assert module.module_name == "blast"
205+
assert module.module_version == "2.22.0-mpi", module.module_version
206+
207+
208+
def _setup_module_command(temp_directory, contents):
209+
module_script = os.path.join(temp_directory, "modulecmd")
210+
__write_script(module_script, '''#!/bin/sh
211+
cat %s/example_output 1>&2;
212+
''' % temp_directory)
213+
with open(os.path.join(temp_directory, "example_output"), "w") as f:
214+
# Subset of module avail from MSI cluster.
215+
f.write(contents)
216+
return module_script
217+
218+
157219
def test_module_dependency():
158220
with __test_base_path() as temp_directory:
159221
# Create mock modulecmd script that just exports a variable
@@ -386,3 +448,9 @@ def __dependency_manager(xml_content):
386448
f.flush()
387449
dm = DependencyManager( default_base_path=base_path, conf_file=f.name )
388450
yield dm
451+
452+
453+
class _SimpleDependencyManager(object):
454+
455+
def get_resolver_option(self, resolver, key, explicit_resolver_options={}):
456+
return explicit_resolver_options.get(key)

0 commit comments

Comments
 (0)
0