8000 Bugfix release of joblib that also restores PyPy support by ogrisel · Pull Request #12012 · scikit-learn/scikit-learn · GitHub
[go: up one dir, main page]

Skip to content

Bugfix release of joblib that also restores PyPy support #12012

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 5, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion sklearn/externals/copy_joblib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ else
JOBLIB=$1
fi

pip install $JOBLIB --target $INSTALL_FOLDER
pip install --no-cache $JOBLIB --target $INSTALL_FOLDER
cp -r $INSTALL_FOLDER/joblib joblib
rm -rf $INSTALL_FOLDER

Expand Down
4 changes: 2 additions & 2 deletions sklearn/externals/joblib/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@


==================== ===============================================
**Documentation:** http://pythonhosted.org/joblib
**Documentation:** https://joblib.readthedocs.io

**Download:** http://pypi.python.org/pypi/joblib#downloads

Expand Down Expand Up @@ -106,7 +106,7 @@
# Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
# 'X.Y.dev0' is the canonical version of 'X.Y.dev'
#
__version__ = '0.12.2'
__version__ = '0.12.4'


from .memory import Memory, MemorizedResult, register_store_backend
Expand Down
12 changes: 11 additions & 1 deletion sklearn/externals/joblib/_multiprocessing_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
circular dependencies (for instance for the assert_spawning name).
"""
import os
import sys
import warnings


Expand All @@ -21,7 +22,16 @@
# issue a warning if not
if mp is not None:
try:
_sem = mp.Semaphore()
# Use the spawn context
if sys.version_info < (3, 3):
Semaphore = mp.Semaphore
else:
# Using mp.Semaphore has a border effect and set the default
# backend for multiprocessing. To avoid that, we use the 'spawn'
# context which is available on all supported platforms.
ctx = mp.get_context('spawn')
Semaphore = ctx.Semaphore
_sem = Semaphore()
del _sem # cleanup
except (ImportError, OSError) as e:
mp = None
Expand Down
10 changes: 6 additions & 4 deletions sklearn/externals/joblib/_parallel_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,6 @@ def get_nested_backend(self):
else:
return ThreadingBackend(nesting_level=nesting_level)


@contextlib.contextmanager
def retrieval_context(self):
"""Context manager to 8000 manage an execution context.
Expand Down Expand Up @@ -348,7 +347,8 @@ def configure(self, n_jobs=1, parallel=None, **backend_args):
n_jobs = self.effective_n_jobs(n_jobs)
if n_jobs == 1:
# Avoid unnecessary overhead and use sequential backend instead.
raise FallbackToBackend(SequentialBackend())
raise FallbackToBackend(
SequentialBackend(nesting_level=self.nesting_level))
self.parallel = parallel
self._n_jobs = n_jobs
return n_jobs
Expand Down Expand Up @@ -421,7 +421,8 @@ def configure(self, n_jobs=1, parallel=None, prefer=None, require=None,
"""Build a process or thread pool and return the number of workers"""
n_jobs = self.effective_n_jobs(n_jobs)
if n_jobs == 1:
raise FallbackToBackend(SequentialBackend())
raise FallbackToBackend(
SequentialBackend(nesting_level=self.nesting_level))

already_forked = int(os.environ.get(self.JOBLIB_SPAWNED_PROCESS, 0))
if already_forked:
Expand Down Expand Up @@ -462,7 +463,8 @@ def configure(self, n_jobs=1, parallel=None, prefer=None, require=None,
"""Build a process executor and return the number of workers"""
n_jobs = self.effective_n_jobs(n_jobs)
if n_jobs == 1:
raise FallbackToBackend(SequentialBackend())
raise FallbackToBackend(
SequentialBackend(nesting_level=self.nesting_level))

self._workers = get_memmapping_executor(
n_jobs, timeout=idle_worker_timeout,
Expand Down
23 changes: 13 additions & 10 deletions sklearn/externals/joblib/_store_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ class StoreBackendBase(with_metaclass(ABCMeta)):
"""Helper Abstract Base Class which defines all methods that
a StorageBackend must implement."""

location = None

@abstractmethod
def _open_item(self, f, mode):
"""Opens an item on the store and return a file-like object.
Expand Down Expand Up @@ -327,7 +329,8 @@ def _concurrency_safe_write(self, to_write, filename, write_func):

def __repr__(self):
"""Printable representation of the store location."""
return self.location
return '{class_name}(location="{location}")'.format(
class_name=self.__class__.__name__, location=self.location)


class FileSystemStoreBackend(StoreBackendBase, StoreBackendMixin):
Expand Down Expand Up @@ -384,29 +387,29 @@ def get_items(self):

return items

def configure(self, location, verbose=1, backend_options={}):
def configure(self, location, verbose=1, backend_options=None):
"""Configure the store backend.

For this backend, valid store options are 'compress' and 'mmap_mode'
"""
if backend_options is None:
backend_options = {}

# setup location directory
self.location = location
if not os.path.exists(self.location):
mkdirp(self.location)

# item can be stored compressed for faster I/O
self.compress = backend_options['compress']
self.compress = backend_options.get('compress', False)

# FileSystemStoreBackend can be used with mmap_mode options under
# certain conditions.
mmap_mode = None
if 'mmap_mode' in backend_options:
mmap_mode = backend_options['mmap_mode']
if self.compress and mmap_mode is not None:
warnings.warn('Compressed items cannot be memmapped in a '
'filesystem store. Option will be ignored.',
stacklevel=2)
mmap_mode = backend_options.get('mmap_mode')
if self.compress and mmap_mode is not None:
warnings.warn('Compressed items cannot be memmapped in a '
'filesystem store. Option will be ignored.',
stacklevel=2)

self.mmap_mode = mmap_mode
self.verbose = verbose
2 changes: 1 addition & 1 deletion sklearn/externals/joblib/externals/cloudpickle/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@

from .cloudpickle import *

__version__ = '0.5.2'
__version__ = '0.5.5'
Loading
0