8000 joblib 0.13.0 (#12531) · thoo/scikit-learn@a68e7ca · GitHub
[go: up one dir, main page]
Skip to content

Commit a68e7ca

Browse files
ogriselthoo
ogrisel
authored and
thoo
committed
joblib 0.13.0 (scikit-learn#12531)
1 parent 5e66970 commit a68e7ca

18 files changed

+496
-322
lines changed

‎sklearn/externals/joblib/__init__.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,11 @@
1414
==================== ===============================================
1515
**Documentation:** https://joblib.readthedocs.io
1616
17-
**Download:** https://pypi.org/project/joblib/#files
17+
**Download:** http://pypi.python.org/pypi/joblib#downloads
1818
19-
**Source code:** https://github.com/joblib/joblib
19+
**Source code:** http://github.com/joblib/joblib
2020
21-
**Report issues:** https://github.com/joblib/joblib/issues
21+
**Report issues:** http://github.com/joblib/joblib/issues
2222
==================== ===============================================
2323
2424
@@ -106,7 +106,7 @@
106106
# Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
107107
# 'X.Y.dev0' is the canonical version of 'X.Y.dev'
108108
#
109-
__version__ = '0.12.5'
109+
__version__ = '0.13.0'
110110

111111

112112
from .memory import Memory, MemorizedResult, register_store_backend
@@ -123,8 +123,11 @@
123123
from .parallel import parallel_backend
124124
from .parallel import effective_n_jobs
125125

126+
from .externals.loky import wrap_non_picklable_objects
127+
126128

127129
__all__ = ['Memory', 'MemorizedResult', 'PrintTime', 'Logger', 'hash', 'dump',
128130
'load', 'Parallel', 'delayed', 'cpu_count', 'effective_n_jobs',
129131
'register_parallel_backend', 'parallel_backend',
130-
'register_store_backend', 'register_compressor']
132+
'register_store_backend', 'register_compressor',
133+
'wrap_non_picklable_objects']

‎sklearn/externals/joblib/_dask.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ def __reduce__(self):
145145
return (DaskDistributedBackend, ())
146146

147147
def get_nested_backend(self):
148-
return DaskDistributedBackend(client=self.client)
148+
return DaskDistributedBackend(client=self.client), -1
149149

150150
def configure(self, n_jobs=1, parallel=None, **backend_args):
151151
return self.effective_n_jobs(n_jobs)

‎sklearn/externals/joblib/_parallel_backends.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,9 +126,9 @@ def get_nested_backend(self):
126126
"""
127127
nesting_level = getattr(self, 'nesting_level', 0) + 1
128128
if nesting_level > 1:
129-
return SequentialBackend(nesting_level=nesting_level)
129+
return SequentialBackend(nesting_level=nesting_level), None
130130
else:
131-
return ThreadingBackend(nesting_level=nesting_level)
131+
return ThreadingBackend(nesting_level=nesting_level), None
132132

133133
@contextlib.contextmanager
134134
def retrieval_context(self):
@@ -185,8 +185,12 @@ def apply_async(self, func, callback=None):
185185
return result
186186

187187
def get_nested_backend(self):
188-
nested_level = getattr(self, 'nesting_level', 0) + 1
189-
return SequentialBackend(nesting_level=nested_level)
188+
# import is not top level to avoid cyclic import errors.
189+
from .parallel import get_active_backend
190+
191+
# SequentialBackend should neither change the nesting level, the
192+
# default backend or the number of jobs. Just return the current one.
193+
return get_active_backend()
190194

191195

192196
class PoolManagerMixin(object):

‎sklearn/externals/joblib/compressor.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
lz4 = None
3131

3232
LZ4_NOT_INSTALLED_ERROR = ('LZ4 is not installed. Install it with pip: '
33-
'https://python-lz4.readthedocs.io/')
33+
'http://python-lz4.readthedocs.io/')
3434

3535
# Registered compressors
3636
_COMPRESSORS = {}

‎sklearn/externals/joblib/externals/cloudpickle/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22

33
from .cloudpickle import *
44

5-
__version__ = '0.5.6'
5+
__version__ = '0.6.1'

‎sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py

Lines changed: 72 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -42,20 +42,20 @@
4242
"""
4343
from __future__ import print_function
4444

45-
import dis
46-
from functools import partial
47-
import imp
4845
import io
49-
import itertools
50-
import logging
46+
import dis
47+
import sys
48+
import types
5149
import opcode
52-
import operator
5350
import pickle
5451
import struct
55-
import sys
56-
import traceback
57-
import types
52+
import logging
5853
import weakref
54+
import operator
55+
import importlib
56+
import itertools
57+
import traceback
58+
from functools import partial
5959

6060

6161
# cloudpickle is meant for inter process communication: we expect all
@@ -78,6 +78,22 @@
7878
PY3 = True
7979

8080

81+
# Container for the global namespace to ensure consistent unpickling of
82+
# functions defined in dynamic modules (modules not registed in sys.modules).
83+
_dynamic_modules_globals = weakref.WeakValueDictionary()
84+
85+
86+
class _DynamicModuleFuncGlobals(dict):
87+
"""Global variables referenced by a function defined in a dynamic module
88+
89+
To avoid leaking references we store such context in a WeakValueDictionary
90+
instance. However instances of python builtin types such as dict cannot
91+
be used directly as values in such a construct, hence the need for a
92+
derived class.
93+
"""
94+
pass
95+
96+
8197
def _make_cell_set_template_code():
8298
"""Get the Python compiler to emit LOAD_FAST(arg); STORE_DEREF
8399
@@ -288,20 +304,10 @@ def save_module(self, obj):
288304
"""
289305
Save a module as an import
290306
"""
291-
mod_name = obj.__name__
292-
# If module is successfully found then it is not a dynamically created module
293-
if hasattr(obj, '__file__'):
294-
is_dynamic = False
295-
else:
296-
try:
297-
_find_module(mod_name)
298-
is_dynamic = False
299-
except ImportError:
300-
is_dynamic = True
301-
302307
self.modules.add(obj)
303-
if is_dynamic:
304-
self.save_reduce(dynamic_subimport, (obj.__name__, vars(obj)), obj=obj)
308+
if _is_dynamic(obj):
309+
self.save_reduce(dynamic_subimport, (obj.__name__, vars(obj)),
310+
obj=obj)
305311
else:
306312
self.save_reduce(subimport, (obj.__name__,), obj=obj)
307313

@@ -566,7 +572,7 @@ def save_function_tuple(self, func):
566572
'name': func.__name__,
567573
'doc': func.__doc__,
568574
}
569-
if hasattr(func, '__annotations__'):
575+
if hasattr(func, '__annotations__') and sys.version_info >= (3, 7):
570576
state['annotations'] = func.__annotations__
571577
if hasattr(func, '__qualname__'):
572578
state['qualname'] = func.__qualname__
@@ -661,6 +667,13 @@ def save_global(self, obj, name=None, pack=struct.pack):
661667
The name of this method is somewhat misleading: all types get
662668
dispatched here.
663669
"""
670+
if obj is type(None):
671+
return self.save_reduce(type, (None,), obj=obj)
672+
elif obj is type(Ellipsis):
673+
return self.save_reduce(type, (Ellipsis,), obj=obj)
674+
elif obj is type(NotImplemented):
675+
return self.save_reduce(type, (NotImplemented,), obj=obj)
676+
664677
if obj.__module__ == "__main__":
665678
return self.save_dynamic_class(obj)
666679

@@ -933,7 +946,7 @@ def subimport(name):
933946

934947

935948
def dynamic_subimport(name, vars):
936-
mod = imp.new_module(name)
949+
mod = types.ModuleType(name)
937950
mod.__dict__.update(vars)
938951
return mod
939952

@@ -1090,12 +1103,18 @@ def _make_skel_func(code, cell_count, base_globals=None):
10901103
if base_globals is None:
10911104
base_globals = {}
10921105
elif isinstance(base_globals, str):
1093-
if sys.modules.get(base_globals, None) is not None:
1094-
# this checks if we can import the previous environment the object
1095-
# lived in
1096-
base_globals = vars(sys.modules[base_globals])
1097-
else:
1098-
base_globals = {}
1106+
base_globals_name = base_globals
1107+
try:
1108+
# First try to reuse the globals from the module containing the
1109+
# function. If it is not possible to retrieve it, fallback to an
1110+
# empty dictionary.
1111+
base_globals = vars(importlib.import_module(base_globals))
1112+
except ImportError:
1113+
base_globals = _dynamic_modules_globals.get(
1114+
base_globals_name, None)
1115+
if base_globals is None:
1116+
base_globals = _DynamicModuleFuncGlobals()
1117+
_dynamic_modules_globals[base_globals_name] = base_globals
10991118

11001119
base_globals['__builtins__'] = __builtins__
11011120

@@ -1125,19 +1144,31 @@ def _rehydrate_skeleton_class(skeleton_class, class_dict):
11251144
return skeleton_class
11261145

11271146

1128-
def _find_module(mod_name):
1147+
def _is_dynamic(module):
11291148
"""
1130-
Iterate over each part instead of calling imp.find_module directly.
1131-
This function is able to find submodules (e.g. scikit.tree)
1149+
Return True if the module is special module that cannot be imported by its
1150+
name.
11321151
"""
1133-
path = None
1134-
for part in mod_name.split('.'):
1135-
if path is not None:
1136-
path = [path]
1137-
file, path, description = imp.find_module(part, path)
1138-
if file is not None:
1139-
file.close()
1140-
return path, description
1152+
# Quick check: module that have __file__ attribute are not dynamic modules.
1153+
if hasattr(module, '__file__'):
1154+
return False
1155+
1156+
if hasattr(module, '__spec__'):
1157+
return module.__spec__ is None
1158+
else:
1159+
# Backward compat for Python 2
1160+
import imp
1161+
try:
1162+
path = None
1163+
for part in module.__name__.split('.'):
1164+
if path is not None:
1165+
path = [path]
1166+
f, path, description = imp.find_module(part, path)
1167+
if f is not None:
1168+
f.close()
1169+
except ImportError:
1170+
return True
1171+
return False
11411172

11421173

11431174
"""Constructors for 3rd party libraries

‎sklearn/externals/joblib/externals/loky/__init__.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,17 @@
99
from ._base import ALL_COMPLETED, FIRST_COMPLETED, FIRST_EXCEPTION
1010

1111
from .backend.context import cpu_count
12+
from .backend.reduction import set_loky_pickler
1213
from .reusable_executor import get_reusable_executor
14+
from .cloudpickle_wrapper import wrap_non_picklable_objects
1315
from .process_executor import BrokenProcessPool, ProcessPoolExecutor
1416

1517

1618
__all__ = ["get_reusable_executor", "cpu_count", "wait", "as_completed",
1719
"Future", "Executor", "ProcessPoolExecutor",
1820
"BrokenProcessPool", "CancelledError", "TimeoutError",
19-
"FIRST_COMPLETED", "FIRST_EXCEPTION", "ALL_COMPLETED", ]
21+
"FIRST_COMPLETED", "FIRST_EXCEPTION", "ALL_COMPLETED",
22+
"wrap_non_picklable_objects", "set_loky_pickler"]
2023

2124

22-
__version__ = '2.3.1'
25+
__version__ = '2.4.2'

‎sklearn/externals/joblib/externals/loky/backend/__init__.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33

44
from .context import get_context
55

6-
LOKY_PICKLER = os.environ.get("LOKY_PICKLER")
7-
86
if sys.version_info > (3, 4):
97

108
def _make_name():

‎sklearn/externals/joblib/externals/loky/backend/compat.py

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
# flake8: noqa
21
###############################################################################
32
# Compat file to import the correct modules for each platform and python
43
# version.
@@ -7,20 +6,36 @@
76
#
87
import sys
98

10-
if sys.version_info[:2] >= (3, 3):
9+
PY3 = sys.version_info[:2] >= (3, 3)
10+
11+
if PY3:
1112
import queue
1213
else:
1314
import Queue as queue
1415

15-
from pickle import PicklingError
16-
1716
if sys.version_info >= (3, 4):
1817
from multiprocessing.process import BaseProcess
1918
else:
2019
from multiprocessing.process import Process as BaseProcess
2120

2221
# Platform specific compat
2322
if sys.platform == "win32":
24-
from .compat_win32 import *
23+
from .compat_win32 import wait
2524
else:
26-
from .compat_posix import *
25+
from .compat_posix import wait
26+
27+
28+
def set_cause(exc, cause):
29+
exc.__cause__ = cause
30+
31+
if not PY3:
32+
# Preformat message here.
33+
if exc.__cause__ is not None:
34+
exc.args = ("{}\n\nThis was caused directly by {}".format(
35+
exc.args if len(exc.args) != 1 else exc.args[0],
36+
str(exc.__cause__)),)
37+
38+
return exc
39+
40+
41+
__all__ = ["queue", "BaseProcess", "set_cause", "wait"]

‎sklearn/externals/joblib/externals/loky/backend/queues.py

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
from multiprocessing.queues import _sentinel, Queue as mp_Queue
2323
from multiprocessing.queues import SimpleQueue as mp_SimpleQueue
2424

25-
from .reduction import CustomizableLokyPickler
25+
from .reduction import loads, dumps
2626
from .context import assert_spawning, get_context
2727

2828

@@ -147,8 +147,7 @@ def _feed(buffer, notempty, send_bytes, writelock, close, reducers,
147147
return
148148

149149
# serialize the data before acquiring the lock
150-
obj_ = CustomizableLokyPickler.dumps(
151-
obj, reducers=reducers)
150+
obj_ = dumps(obj, reducers=reducers)
152151
if wacquire is None:
153152
send_bytes(obj_)
154153
else:
@@ -227,12 +226,12 @@ def get(self):
227226
with self._rlock:
228227
res = self._reader.recv_bytes()
229228
# unserialize the data after having released the lock
230-
return CustomizableLokyPickler.loads(res)
229+
return loads(res)
231230

232231
# Overload put to use our customizable reducer
233232
def put(self, obj):
234233
# serialize the data before acquiring the lock
235-
obj = CustomizableLokyPickler.dumps(obj, reducers=self._reducers)
234+
obj = dumps(obj, reducers=self._reducers)
236235
if self._wlock is None:
237236
# writes to a message oriented win32 pipe are atomic
238237
self._writer.send_bytes(obj)

0 commit comments

Comments
 (0)
0