Added parallelization to slices

SeanApplebaum · SeanApplebaum · commit 77d96cbe8a67 · 2021-04-01T17:37:14.000-04:00
diff --git a/sklearn/metrics/pairwise.py b/sklearn/metrics/pairwise.py
@@ -18,6 +18,7 @@
 from scipy.sparse import csr_matrix
 from scipy.sparse import issparse
 from joblib import Parallel, effective_n_jobs
+from multiprocessing import cpu_count
 
 from ..utils.validation import _num_samples
 from ..utils.validation import check_non_negative
@@ -1644,7 +1645,7 @@ def pairwise_distances_chunked(X, Y=None, *, reduce_func=None,
     params = _precompute_metric_params(X, Y, metric=metric, **kwds)
     kwds.update(**params)
 
-    for sl in slices:
+    def _process_slice(sl, reduce_func):
         if sl.start == 0 and sl.stop == n_samples_X:
             X_chunk = X  # enable optimised paths for X is Y
         else:
@@ -1661,8 +1662,12 @@ def pairwise_distances_chunked(X, Y=None, *, reduce_func=None,
             chunk_size = D_chunk.shape[0]
             D_chunk = reduce_func(D_chunk, sl.start)
             _check_chunk_size(D_chunk, chunk_size)
-        yield D_chunk
+        return D_chunk
 
+    generator = (delayed(_process_slice)(sl, reduce_func) for sl in slices)
+    par_res=Parallel(n_jobs, backend='threading')(generator)
+    for res in par_res:
+        yield res
 
 @_deprecate_positional_args
 def pairwise_distances(X, Y=None, metric="euclidean", *, n_jobs=None,
diff --git a/sklearn/preprocessing/_data.py b/sklearn/preprocessing/_data.py
@@ -1643,6 +1643,7 @@ def __init__(self, max_degree=None, *, interaction_only=False, include_bias=True
         self.interaction_only = interaction_only
         self.include_bias = include_bias
         self.order = order
+        self.degree = max_degree or degree # lazy evaluation to handle deprecaition
 
     @staticmethod
     @_deprecate_positional_args
