ENH: allow non-consolidation in constructors

pandas-dev · jreback · Mar 31, 2021 · Oct 5, 2020 · Oct 5, 2020 · Oct 5, 2020
commit d653c5435729c6bb4a9e6d51445d85932ec70e9a
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
@@ -421,6 +421,8 @@ class DataFrame(NDFrame, OpsMixin):
         Data type to force. Only a single dtype is allowed. If None, infer.
     copy : bool, default False
         Copy data from inputs. Only affects DataFrame / 2d ndarray input.
+    consolidate : bool or None, default None
+        Whether to consolidate the arrays in the new DataFrame.
 
     See Also
     --------
@@ -508,12 +510,16 @@ def __init__(
         columns: Optional[Axes] = None,
         dtype: Optional[Dtype] = None,
         copy: bool = False,
+        consolidate=None,
     ):
         if data is None:
             data = {}
         if dtype is not None:
             dtype = self._validate_dtype(dtype)
 
+        if consolidate is None:
+            consolidate = not copy
+
         if isinstance(data, DataFrame):
             data = data._mgr
 
@@ -528,7 +534,7 @@ def __init__(
             )
 
         elif isinstance(data, dict):
-            mgr = init_dict(data, index, columns, dtype=dtype)
+            mgr = init_dict(data, index, columns, dtype=dtype, consolidate=consolidate)
         elif isinstance(data, ma.MaskedArray):
             import numpy.ma.mrecords as mrecords
 
@@ -545,19 +551,41 @@ def __init__(
                     data[mask] = fill_value
                 else:
                     data = data.copy()
-                mgr = init_ndarray(data, index, columns, dtype=dtype, copy=copy)
+                mgr = init_ndarray(
+                    data,
+                    index,
+                    columns,
+                    dtype=dtype,
+                    copy=copy,
+                    consolidate=consolidate,
+                )
 
         elif isinstance(data, (np.ndarray, Series, Index)):
             if data.dtype.names:
                 data_columns = list(data.dtype.names)
                 data = {k: data[k] for k in data_columns}
                 if columns is None:
                     columns = data_columns
-                mgr = init_dict(data, index, columns, dtype=dtype)
+                mgr = init_dict(
+                    data, index, columns, dtype=dtype, consolidate=consolidate
+                )
             elif getattr(data, "name", None) is not None:
-                mgr = init_dict({data.name: data}, index, columns, dtype=dtype)
+                mgr = init_dict(
+                    {data.name: data},
+                    index,
+                    columns,
+                    dtype=dtype,
+                    consolidate=consolidate,
+                )
             else:
-                mgr = init_ndarray(data, index, columns, dtype=dtype, copy=copy)
+                mgr = init_ndarray(
+                    data,
+                    index,
+                    columns,
+                    dtype=dtype,
+                    copy=copy,
+                    consolidate=consolidate,
+                )
 
         # For data is list-like, or Iterable (will consume into list)
         elif isinstance(data, abc.Iterable) and not isinstance(data, (str, bytes)):
@@ -581,11 +609,27 @@ def __init__(
                         else:
                             index = ibase.default_index(len(data))
 
-                    mgr = arrays_to_mgr(arrays, columns, index, columns, dtype=dtype)
+                    mgr = arrays_to_mgr(
+                        arrays,
+                        columns,
+                        index,
+                        columns,
+                        dtype=dtype,
+                        consolidate=consolidate,
+                    )
                 else:
-                    mgr = init_ndarray(data, index, columns, dtype=dtype, copy=copy)
+                    mgr = init_ndarray(
+                        data,
+                        index,
+                        columns,
+                        dtype=dtype,
+                        copy=copy,
+                        consolidate=consolidate,
+                    )
             else:
-                mgr = init_dict({}, index, columns, dtype=dtype)
+                mgr = init_dict(
+                    {}, index, columns, dtype=dtype, consolidate=consolidate
+                )
         # For data is scalar
         else:
             if index is None or columns is None:
@@ -601,7 +645,9 @@ def __init__(
                     construct_1d_arraylike_from_scalar(data, len(index), dtype)
                     for _ in range(len(columns))
                 ]
-                mgr = arrays_to_mgr(values, columns, index, columns, dtype=None)
+                mgr = arrays_to_mgr(
+                    values, columns, index, columns, dtype=None, consolidate=consolidate
+                )
             else:
                 # Attempt to coerce to a numpy array
                 try:
@@ -621,7 +667,12 @@ def __init__(
                 )
 
                 mgr = init_ndarray(
-                    values, index, columns, dtype=values.dtype, copy=False
+                    values,
+                    index,
+                    columns,
+                    dtype=values.dtype,
+                    copy=False,
+                    consolidate=consolidate,
                 )
 
         NDFrame.__init__(self, mgr)
@@ -1733,6 +1784,7 @@ def from_records(
         columns=None,
         coerce_float=False,
         nrows=None,
+        consolidate: bool = True,
     ) -> DataFrame:
         """
         Convert structured or record ndarray to DataFrame.
@@ -1760,6 +1812,8 @@ def from_records(
             decimal.Decimal) to floating point, useful for SQL result sets.
         nrows : int, default None
             Number of rows to read if data is an iterator.
+        consolidate: bool, default True
+            Whether to consolidate the arrays in the new DataFrame.
 
         Returns
         -------
@@ -1895,7 +1949,9 @@ def from_records(
             arr_columns = arr_columns.drop(arr_exclude)
             columns = columns.drop(exclude)
 
-        mgr = arrays_to_mgr(arrays, arr_columns, result_index, columns)
+        mgr = arrays_to_mgr(
+            arrays, arr_columns, result_index, columns, consolidate=consolidate
+        )
 
         return cls(mgr)
 
@@ -2074,6 +2130,7 @@ def _from_arrays(
         index,
         dtype: Optional[Dtype] = None,
         verify_integrity: bool = True,
+        consolidate: bool = True,
     ) -> DataFrame:
         """
         Create DataFrame from a list of arrays corresponding to the columns.
@@ -2094,6 +2151,8 @@ def _from_arrays(
             stored in a block (numpy ndarray or ExtensionArray), have the same
             length as and are aligned with the index, and that `columns` and
             `index` are ensured to be an Index object.
+        consolidate: bool, default True
+            Whether to consolidate the passed arrays in the new DataFrame.
 
         Returns
         -------
@@ -2109,6 +2168,7 @@ def _from_arrays(
             columns,
             dtype=dtype,
             verify_integrity=verify_integrity,
+            consolidate=consolidate,
         )
         return cls(mgr)
 
@@ -6047,7 +6107,7 @@ def _dispatch_frame_op(self, right, func, axis: Optional[int] = None):
             raise NotImplementedError(right)
 
         return type(self)._from_arrays(
-            arrays, self.columns, self.index, verify_integrity=False
+            arrays, self.columns, self.index, verify_integrity=False, consolidate=False
         )
 
     def _combine_frame(self, other: DataFrame, func, fill_value=None):

diff --git a/pandas/core/internals/construction.py b/pandas/core/internals/construction.py
@@ -66,6 +66,7 @@ def arrays_to_mgr(
     columns,
     dtype: Optional[DtypeObj] = None,
     verify_integrity: bool = True,
+    consolidate: bool = True,
 ):
     """
     Segregate Series based on type and coerce into matrices.
@@ -92,7 +93,9 @@ def arrays_to_mgr(
     # from BlockManager perspective
     axes = [columns, index]
 
-    return create_block_manager_from_arrays(arrays, arr_names, axes)
+    return create_block_manager_from_arrays(
+        arrays, arr_names, axes, consolidate=consolidate
+    )
 
 
 def masked_rec_array_to_mgr(
@@ -131,7 +134,9 @@ def masked_rec_array_to_mgr(
     if columns is None:
         columns = arr_columns
 
-    mgr = arrays_to_mgr(arrays, arr_columns, index, columns, dtype)
+    mgr = arrays_to_mgr(
+        arrays, arr_columns, index, columns, dtype, consolidate=True
+    )  # FIXME: dont hardcode
 
     if copy:
         mgr = mgr.copy()
@@ -142,7 +147,14 @@ def masked_rec_array_to_mgr(
 # DataFrame Constructor Interface
 
 
-def init_ndarray(values, index, columns, dtype: Optional[DtypeObj], copy: bool):
+def init_ndarray(
+    values,
+    index,
+    columns,
+    dtype: Optional[DtypeObj],
+    copy: bool,
+    consolidate: bool = True,
+):
     # input must be a ndarray, list, Series, index
 
     if isinstance(values, ABCSeries):
@@ -171,7 +183,9 @@ def init_ndarray(values, index, columns, dtype: Optional[DtypeObj], copy: bool):
             values = values.copy()
 
         index, columns = _get_axes(len(values), 1, index, columns)
-        return arrays_to_mgr([values], columns, index, columns, dtype=dtype)
+        return arrays_to_mgr(
+            [values], columns, index, columns, dtype=dtype, consolidate=consolidate
+        )
     elif is_extension_array_dtype(values) or is_extension_array_dtype(dtype):
         # GH#19157
 
@@ -185,7 +199,9 @@ def init_ndarray(values, index, columns, dtype: Optional[DtypeObj], copy: bool):
         if columns is None:
             columns = Index(range(len(values)))
 
-        return arrays_to_mgr(values, columns, index, columns, dtype=dtype)
+        return arrays_to_mgr(
+            values, columns, index, columns, dtype=dtype, consolidate=consolidate
+        )
 
     # by definition an array here
     # the dtypes will be coerced to a single dtype
@@ -235,10 +251,18 @@ def init_ndarray(values, index, columns, dtype: Optional[DtypeObj], copy: bool):
     else:
         block_values = [values]
 
-    return create_block_manager_from_blocks(block_values, [columns, index])
+    return create_block_manager_from_blocks(
+        block_values, [columns, index], consolidate=consolidate
+    )
 
 
-def init_dict(data: Dict, index, columns, dtype: Optional[DtypeObj] = None):
+def init_dict(
+    data: Dict,
+    index,
+    columns,
+    dtype: Optional[DtypeObj] = None,
+    consolidate: bool = True,
+):
     """
     Segregate Series based on type and coerce into matrices.
     Needs to handle a lot of exceptional cases.
@@ -284,7 +308,9 @@ def init_dict(data: Dict, index, columns, dtype: Optional[DtypeObj] = None):
         arrays = [
             arr if not is_datetime64tz_dtype(arr) else arr.copy() for arr in arrays
         ]
-    return arrays_to_mgr(arrays, data_names, index, columns, dtype=dtype)
+    return arrays_to_mgr(
+        arrays, data_names, index, columns, dtype=dtype, consolidate=consolidate
+    )
 
 
 # ---------------------------------------------------------------------

diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py
@@ -1648,7 +1648,9 @@ def fast_xs(self, loc):
 # Constructor Helpers
 
 
-def create_block_manager_from_blocks(blocks, axes: List[Index]) -> BlockManager:
+def create_block_manager_from_blocks(
+    blocks, axes: List[Index], consolidate: bool = True
+) -> BlockManager:
     try:
         if len(blocks) == 1 and not isinstance(blocks[0], Block):
             # if blocks[0] is of length 0, return empty blocks
@@ -1665,7 +1667,8 @@ def create_block_manager_from_blocks(blocks, axes: List[Index]) -> BlockManager:
                 ]
 
         mgr = BlockManager(blocks, axes)
-        mgr._consolidate_inplace()
+        if consolidate:
+            mgr._consolidate_inplace()
         return mgr
 
     except ValueError as e:
@@ -1675,7 +1678,10 @@ def create_block_manager_from_blocks(blocks, axes: List[Index]) -> BlockManager:
 
 
 def create_block_manager_from_arrays(
-    arrays, names: Index, axes: List[Index]
+    arrays,
+    names: Index,
+    axes: List[Index],
+    consolidate: bool = True,
 ) -> BlockManager:
     assert isinstance(names, Index)
     assert isinstance(axes, list)
@@ -1687,10 +1693,11 @@ def create_block_manager_from_arrays(
     try:
         blocks = _form_blocks(arrays, names, axes)
         mgr = BlockManager(blocks, axes)
-        mgr._consolidate_inplace()
-        return mgr
     except ValueError as e:
         raise construction_error(len(arrays), arrays[0].shape, axes, e)
+    if consolidate:
+        mgr._consolidate_inplace()
+    return mgr
 
 
 def construction_error(tot_items, block_shape, axes, e=None):