10000 Array Interface and Categorical internals Refactor by TomAugspurger · Pull Request #19268 · pandas-dev/pandas · GitHub
[go: up one dir, main page]

Skip to content

Array Interface and Categorical internals Refactor #19268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
2ef5216
REF: Define extension base classes
TomAugspurger Jan 15, 2018
57e8b0f
Updated for comments
TomAugspurger Jan 18, 2018
01bd42f
Remove metaclasses from PeriodDtype and IntervalDtype
TomAugspurger Jan 18, 2018
ce81706
Fixup form_blocks rebase
TomAugspurger Jan 18, 2018
87a70e3
Restore concat casting cat -> object
TomAugspurger Jan 18, 2018
8c61886
Remove _slice, clarify semantics around __getitem__
TomAugspurger Jan 19, 2018
cb41803
Document and use take.
TomAugspurger Jan 19, 2018
65d5a61
Clarify type, kind, init
TomAugspurger Jan 19, 2018
57c749b
Remove base
TomAugspurger Jan 19, 2018
6736b0f
API: Remove unused __iter__ and get_values
TomAugspurger Jan 21, 2018
e4acb59
API: Implement repr and str
TomAugspurger Jan 21, 2018
0e9337b
Merge remote-tracking branch 'upstream/master' into pandas-array-inte…
TomAugspurger Jan 26, 2018
df68f3b
Remove default value_counts for now
TomAugspurger Jan 26, 2018
2746a43
Fixed merge conflicts
TomAugspurger Jan 27, 2018
34d2b99
Remove implementation of construct_from_string
TomAugspurger Jan 27, 2018
a484d61
Example implementation of take
TomAugspurger Jan 27, 2018
04b2e72
Cleanup ExtensionBlock
TomAugspurger Jan 27, 2018
df0fa12
Merge remote-tracking branch 'upstream/master' into pandas-array-inte…
TomAugspurger Jan 27, 2018
e778053
Pass through ndim
TomAugspurger Jan 27, 2018
d15a722
Use series._values
TomAugspurger Jan 27, 2018
b5f736d
Removed repr, updated take doc
TomAugspurger Jan 27, 2018
240e8f6
Various cleanups
TomAugspurger Jan 28, 2018
f9b0b49
Handle get_values, to_dense, is_view
TomAugspurger Jan 28, 2018
7913186
Docs
TomAugspurger Jan 30, 2018
df18c3b
Remove is_extension, is_bool
TomAugspurger Jan 30, 2018
ab2f045
Sparse formatter
TomAugspurger Jan 30, 2018
520876f
Revert "Sparse formatter"
TomAugspurger Jan 30, 2018
4dfa39c
Unbox SparseSeries
TomAugspurger Jan 30, 2018
e252103
Added test for sparse consolidation
TomAugspurger Jan 30, 2018
7110b2a
Docs
TomAugspurger Jan 30, 2018
c59dca0
Merge remote-tracking branch 'upstream/master' into pandas-array-inte…
TomAugspurger Jan 31, 2018
fc688a5
Moved to errors
TomAugspurger Jan 31, 2018
fbc8466
Handle classmethods, properties
TomAugspurger Jan 31, 2018
030bb19
Use our AbstractMethodError
TomAugspurger Jan 31, 2018
0f4c2d7
Lint
TomAugspurger Jan 31, 2018
f9316e0
Cleanup
TomAugspurger Feb 1, 2018
9c06b13
Move ndim validation to a method.
TomAugspurger Feb 1, 2018
7d2cf9c
Try this
TomAugspurger Feb 1, 2018
afae8ae
Make ExtensionBlock._holder a property
TomAugspurger Feb 1, 2018
cd0997e
Make _holder a property for all
TomAugspurger Feb 1, 2018
1d6eb04
Refactored validate_ndim
TomAugspurger Feb 1, 2018
92aed49
fixup! Refactored validate_ndim
TomAugspurger Feb 1, 2018
34134f2
lint
TomAugspurger Feb 1, 2018
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Make _holder a property for all
  • Loading branch information
TomAugspurger committed Feb 1, 2018
commit cd0997e354121bc0414ef1675cb5b8241944a9b0
35 changes: 28 additions & 7 deletions pandas/core/internals.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
from pandas.core.indexing import maybe_convert_indices, length_of_indexer
from pandas.core.arrays import Categorical
from pandas.core.indexes.datetimes import DatetimeIndex
from pandas.core.indexes.timedeltas import TimedeltaIndex
from pandas.io.formats.printing import pprint_thing

import pandas.core.missing as missing
Expand Down Expand Up @@ -104,7 +105,6 @@ class Block(PandasObject):
_verify_integrity = True
_validate_ndim = True
_ftype = 'dense'
_holder = None
_concatenator = staticmethod(np.concatenate)

def __init__(self, values, placement, ndim=None):
Expand Down Expand Up @@ -135,6 +135,15 @@ def _maybe_validate_ndim(self, values, ndim):
if self._validate_ndim and values.ndim != ndim:
raise ValueError('Wrong number of dimensions')

@property
def _holder(self):
"""The array-like that can hold the underlying values.

None for 'Block', overridden by subclasses that don't
use an ndarray.
"""
return None

@property
def _consolidate_key(self):
return (self._can_consolidate, self.dtype.name)
Expand Down Expand Up @@ -1689,7 +1698,6 @@ class NonConsolidatableMixIn(object):
_can_consolidate = False
_verify_integrity = False
_validate_ndim = False
_holder = None

def __init__(self, values, placement, ndim=None):
"""Initialize a non-consolidatable block.
Expand Down Expand Up @@ -1832,8 +1840,7 @@ class ExtensionBlock(NonConsolidatableMixIn, Block):
"""
@property
def _holder(self):
# For extension blocks, the holder is values-dependent so we
# use a property.
# For extension blocks, the holder is values-dependent.
return type(self.values)

@property
Expand Down Expand Up @@ -2012,6 +2019,11 @@ def should_store(self, value):


class DatetimeLikeBlockMixin(object):
"""Mixin class for DatetimeBlock and DatetimeTZBlock."""

@property
def _holder(self):
return DatetimeIndex

@property
def _na_value(self):
Expand Down Expand Up @@ -2044,6 +2056,10 @@ def __init__(self, values, placement, ndim=None):
super(TimeDeltaBlock, self).__init__(values,
placement=placement, ndim=ndim)

@property
def _holder(self):
return TimedeltaIndex

@property
def _box_func(self):
return lambda x: tslib.Timedelta(x, unit='ns')
Expand Down Expand Up @@ -2424,7 +2440,6 @@ class CategoricalBlock(ExtensionBlock):
is_categorical = True
_verify_integrity = True
_can_hold_na = True
_holder = Categorical
_concatenator = staticmethod(_concat._concat_categorical)

def __init__(self, values, placement, ndim=None):
Expand All @@ -2435,6 +2450,10 @@ def __init__(self, values, placement, ndim=None):
placement=placement,
ndim=ndim)

@property
def _holder(self):
return Categorical

@property
def array_dtype(self):
""" the dtype to return if I want to construct this block as an
Expand Down Expand Up @@ -2673,7 +2692,6 @@ def set(self, locs, values, check=False):
class DatetimeTZBlock(NonConsolidatableMixIn, DatetimeBlock):
""" implement a datetime64 block with a tz attribute """
__slots__ = ()
_holder = DatetimeIndex
_concatenator = staticmethod(_concat._concat_datetime)
is_datetimetz = True

Expand Down Expand Up @@ -2856,7 +2874,6 @@ class SparseBlock(NonConsolidatableMixIn, Block):
_box_to_block_values = False
_can_hold_na = True
_ftype = 'sparse'
_holder = SparseArray
_concatenator = staticmethod(_concat._concat_sparse)

def __init__(self, values, placement, ndim=None):
Expand All @@ -2866,6 +2883,10 @@ def __init__(self, values, placement, ndim=None):
assert isinstance(values, SparseArray)
super(SparseBlock, self).__init__(values, placement, ndim=ndim)

@property
def _holder(self):
return SparseArray

@property
def shape(self):
return (len(self.mgr_locs), self.sp_index.length)
Expand Down
14 changes: 13 additions & 1 deletion pandas/tests/internals/test_internals.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from distutils.version import LooseVersion
import itertools
from pandas import (Index, MultiIndex, DataFrame, DatetimeIndex,
Series, Categorical)
Series, Categorical, TimedeltaIndex, SparseArray)
from pandas.compat import OrderedDict, lrange
from pandas.core.sparse.array import SparseArray
from pandas.core.internals import (BlockPlacement, SingleBlockManager,
Expand Down Expand Up @@ -1263,6 +1263,18 @@ def test_binop_other(self, op, value, dtype):
assert_series_equal(result, expected)


@pytest.mark.parametrize('typestr, holder', [
('category', Categorical),
('M8[ns]', DatetimeIndex),
('M8[ns, US/Central]', DatetimeIndex),
('m8[ns]', TimedeltaIndex),
('sparse', SparseArray),
])
def test_holder(typestr, holder):
blk = create_block(typestr, [1])
assert blk._holder is holder


def test_deprecated_fastpath():
# GH#19265
values = np.random.rand(3, 3)
Expand Down
0