8000 MAINT joblib 0.12.5 (#12066) · scikit-learn/scikit-learn@c2f9206 · GitHub
[go: up one dir, main page]
Skip to content

Commit c2f9206

Browse files
ogriselogrisel
authored
MAINT joblib 0.12.5 (#12066)
1 parent dfdf605 commit c2f9206

File tree

5 files changed

+48
-28
lines changed

5 files changed

+48
-28
lines changed

‎sklearn/externals/joblib/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@
106106
# Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
107107
# 'X.Y.dev0' is the canonical version of 'X.Y.dev'
108108
#
109-
__version__ = '0.12.4'
109+
__version__ = '0.12.5'
110110

111111

112112
from .memory import Memory, MemorizedResult, register_store_backend

‎sklearn/externals/joblib/externals/cloudpickle/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22

33
from .cloudpickle import *
44

5-
__version__ = '0.5.5'
5+
__version__ = '0.5.6'

‎sklearn/externals/joblib/externals/cloudpickle/cloudpickle.py

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -635,11 +635,12 @@ def extract_func_data(self, func):
635635

636636
base_globals = self.globals_ref.get(id(func.__globals__), None)
637637
if base_globals is None:
638-
# For functions defined in __main__, use vars(__main__) for
639-
# base_global. This is necessary to share the global variables
640-
# across multiple functions in this module.
641-
if func.__module__ == "__main__":
642-
base_globals = "__main__"
638+
# For functions defined in a well behaved module use
639+
# vars(func.__module__) for base_globals. This is necessary to
640+
# share the global variables across multiple pickled functions from
641+
# this module.
642+
if hasattr(func, '__module__') and func.__module__ is not None:
643+
base_globals = func.__module__
643644
else:
644645
base_globals = {}
645646
self.globals_ref[id(func.__globals__)] = base_globals
@@ -934,7 +935,6 @@ def subimport(name):
934935
def dynamic_subimport(name, vars):
935936
mod = imp.new_module(name)
936937
mod.__dict__.update(vars)
937-
sys.modules[name] = mod
938938
return mod
939939

940940

@@ -1090,7 +1090,13 @@ def _make_skel_func(code, cell_count, base_globals=None):
10901090
if base_globals is None:
10911091
base_globals = {}
10921092
elif isinstance(base_globals, str):
1093-
base_globals = vars(sys.modules[base_globals])
1093+
if sys.modules.get(base_globals, None) is not None:
1094+
# this checks if we can import the previous environment the object
1095+
# lived in
1096+
base_globals = vars(sys.modules[base_globals])
1097+
else:
1098+
base_globals = {}
1099+
10941100
base_globals['__builtins__'] = __builtins__
10951101

10961102
closure = (

‎sklearn/externals/joblib/externals/loky/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,4 @@
1919
"FIRST_COMPLETED", "FIRST_EXCEPTION", "ALL_COMPLETED", ]
2020

2121

22-
__version__ = '2.3.0'
22+
__version__ = '2.3.1'

‎sklearn/externals/joblib/externals/loky/process_executor.py

Lines changed: 32 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@
6262
import os
6363
import gc
6464
import sys
65-
import types
6665
import struct
6766
import weakref
6867
import warnings
@@ -438,7 +437,6 @@ def _process_worker(call_queue, result_queue, initializer, initargs,
438437
continue
439438
if time() - _last_memory_leak_check > _MEMORY_LEAK_CHECK_DELAY:
440439
mem_usage = _get_memory_usage(pid)
441-
print(mem_usage)
442440
_last_memory_leak_check = time()
443441
if mem_usage - _process_reference_size < _MAX_MEMORY_LEAK_SIZE:
444442
# Memory usage stays within bounds: everything is fine.
@@ -618,34 +616,41 @@ def shutdown_all_workers():
618616
worker_sentinels = [p.sentinel for p in processes.values()]
619617
ready = wait(readers + worker_sentinels)
620618

621-
broken = ("A process in the executor was terminated abruptly", None)
619+
broken = ("A worker process managed by the executor was unexpectedly "
620+
"terminated. This could be caused by a segmentation fault "
621+
"while calling the function or by an excessive memory usage "
622+
"causing the Operating System to kill the worker.", None,
623+
TerminatedWorkerError)
622624
if result_reader in ready:
623625
try:
624626
result_item = result_reader.recv()
625627
broken = None
626628
if isinstance(result_item, _RemoteTraceback):
627-
cause = result_item.tb
628-
broken = ("A task has failed to un-serialize", cause)
629+
broken = ("A task has failed to un-serialize. Please "
630+
"ensure that the arguments of the function are "
631+
"all picklable.", result_item.tb,
632+
BrokenProcessPool)
629633
except BaseException as e:
630634
tb = getattr(e, "__traceback__", None)
631635
if tb is None:
632636
_, _, tb = sys.exc_info()
633-
broken = ("A result has failed to un-serialize",
634-
traceback.format_exception(type(e), e, tb))
637+
broken = ("A result has failed to un-serialize. Please "
638+
"ensure that the objects returned by the function "
639+
"are always picklable.",
640+
traceback.format_exception(type(e), e, tb),
641+
BrokenProcessPool)
635642
elif wakeup_reader in ready:
636643
broken = None
637644
result_item = None
638645
thread_wakeup.clear()
639-
if broken:
640-
msg, cause = broken
641-
# Mark the process pool broken so that submits fail right now.
642-
executor_flags.flag_as_broken(
643-
msg + ", the pool is not usable anymore.")
644-
bpe = BrokenProcessPool(
645-
msg + " while the future was running or pending.")
646-
if cause is not None:
646+
if broken is not None:
647+
msg, cause_tb, exc_type = broken
648+
bpe = exc_type(msg)
649+
if cause_tb is not None:
647650
bpe.__cause__ = _RemoteTraceback(
648-
"\n'''\n{}'''".format(''.join(cause)))
651+
"\n'''\n{}'''".format(''.join(cause_tb)))
652+
# Mark the process pool broken so that submits fail right now.
653+
executor_flags.flag_as_broken(bpe)
649654

650655
# All futures in flight must be marked failed
651656
for work_id, work_item in pending_work_items.items():
@@ -808,6 +813,15 @@ class LokyRecursionError(RuntimeError):
808813

809814

810815
class BrokenProcessPool(_BPPException):
816+
"""
817+
Raised when the executor is broken while a future was in the running state.
818+
The cause can an error raised when unpickling the task in the worker
819+
process or when unpickling the result value in the parent process. It can
820+
also be caused by a worker process being terminated unexpectedly.
821+
"""
822+
823+
824+
class TerminatedWorkerError(BrokenProcessPool):
811825
"""
812826
Raised when a process in a ProcessPoolExecutor terminated abruptly
813827
while a future was in the running state.
@@ -998,8 +1012,8 @@ def _ensure_executor_running(self):
9981012

9991013
def submit(self, fn, *args, **kwargs):
10001014
with self._flags.shutdown_lock:
1001-
if self._flags.broken:
1002-
raise BrokenProcessPool(self._flags.broken)
1015+
if self._flags.broken is not None:
1016+
raise self._flags.broken
10031017
if self._flags.shutdown:
10041018
raise ShutdownExecutorError(
10051019
'cannot schedule new futures after shutdown')

0 commit comments

Comments
 (0)
0