wip:constructor

pandas-dev · jreback · Jan 2, 2019 · Nov 30, 2018 · Nov 30, 2018 · Nov 30, 2018
commit d4c9521cf9ad9ce4bc5688cc6ca138279198bb12
diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py
@@ -23,6 +23,7 @@
     is_dtype_equal, is_extension_array_dtype, is_float_dtype, is_integer_dtype,
     is_list_like, is_object_dtype, is_offsetlike, is_period_dtype,
     is_string_dtype, is_timedelta64_dtype, needs_i8_conversion, pandas_dtype)
+from pandas.core.dtypes.dtypes import DatetimeTZDtype
 from pandas.core.dtypes.generic import ABCDataFrame, ABCIndexClass, ABCSeries
 from pandas.core.dtypes.missing import isna
 
@@ -938,7 +939,7 @@ def _add_nat(self):
         result.fill(iNaT)
         if is_timedelta64_dtype(self):
             return type(self)(result, freq=None)
-        return type(self)(result, tz=self.tz, freq=None)
+        return type(self)(result, dtype=self.dtype, freq=None)
 
     def _sub_nat(self):
         """
@@ -1074,8 +1075,8 @@ def _time_shift(self, periods, freq=None):
                 freq = frequencies.to_offset(freq)
             offset = periods * freq
             result = self + offset
-            if hasattr(self, 'tz'):
-                result._tz = self.tz
+            if getattr(self, 'tz'):
+                result._dtype = DatetimeTZDtype(tz=self.tz)
             return result
 
         if periods == 0:

diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py
@@ -159,6 +159,15 @@ def wrapper(self, other):
     return compat.set_function_name(wrapper, opname, cls)
 
 
+def validate_values_freq(values, freq):
+    # type: (Union[DatetimeArrayMixin, TimedeltaArrayMixin], Freq) -> Freq
+    if freq:
+        freq = to_offset(freq)
+        if values.freq != freq:
+            raise ValueError("'freq' does not match.")
+    return values.freq
+
+
 class DatetimeArrayMixin(dtl.DatetimeLikeArrayMixin,
                          dtl.TimelikeOps,
                          dtl.DatelikeOps):
@@ -204,23 +213,21 @@ class DatetimeArrayMixin(dtl.DatetimeLikeArrayMixin,
     _attributes = ["freq", "tz"]
     _freq = None
 
-    @classmethod
-    def _simple_new(cls, values, freq=None, tz=None):
-        """
-        we require the we have a dtype compat for the values
-        if we are passed a non-dtype compat, then coerce using the constructor
-        """
-        if isinstance(values, cls):
-            # todo: validate
-            if freq and values.freq:
-                assert freq == values.freq
-            freq = freq or values.freq
-
-            if tz and values.tz:
-                assert timezones.tz_compare(tz, values.tz)
-
-            tz = tz or values.tz
+    def __init__(self, values, dtype=_NS_DTYPE, freq=None, copy=False):
+        if isinstance(values, (ABCSeries, ABCIndexClass)):
+            values = values._values
+
+        if isinstance(values, type(self)):
+            # validation
+            if getattr(dtype, 'tz', None) and values.tz is None:
+                dtype = DatetimeTZDtype(tz=dtype.tz)
+            elif values.tz:
+                dtype = values.dtype
+            # freq = validate_values_freq(values, freq)
+            if freq is None:
+                freq = values.freq
             values = values._data
+
         assert isinstance(values, np.ndarray), type(values)
         if values.dtype == 'i8':
             # for compat with datetime/timedelta/period shared methods,
@@ -229,22 +236,26 @@ def _simple_new(cls, values, freq=None, tz=None):
             values = values.view('M8[ns]')
 
         assert values.dtype == 'M8[ns]', values.dtype
+        assert isinstance(dtype, (np.dtype, DatetimeTZDtype)), dtype
+        assert freq != "infer"
+        if copy:
+            values = values.copy()
+        if freq:
+            freq = to_offset(freq)
 
-        result = object.__new__(cls)
-        result._data = values
-        result._freq = freq
-        tz = timezones.maybe_get_tz(tz)
-        if tz:
-            result._dtype = DatetimeTZDtype('ns', tz)
-        else:
-            result._dtype = values.dtype  # M8[ns]
-        return result
+        self._data = values
+        self._dtype = dtype
+        self._freq = freq
 
-    def __new__(cls, values, freq=None, tz=None, dtype=None, copy=False,
-                dayfirst=False, yearfirst=False, ambiguous='raise'):
-        return cls._from_sequence(
-            values, freq=freq, tz=tz, dtype=dtype, copy=copy,
-            dayfirst=dayfirst, yearfirst=yearfirst, ambiguous=ambiguous)
+    @classmethod
+    def _simple_new(cls, values, freq=None, tz=None):
+        """
+        we require the we have a dtype compat for the values
+        if we are passed a non-dtype compat, then coerce using the constructor
+        """
+        dtype = DatetimeTZDtype(tz=tz) if tz else _NS_DTYPE
+
+        return cls(values, freq=freq, dtype=dtype)
 
     @classmethod
     def _from_sequence(cls, data, dtype=None, copy=False,
@@ -476,11 +487,8 @@ def __iter__(self):
                 yield v
 
     def copy(self, deep=False):
-        # have to use simple_new, else we raise a freq validation error?
-        # Can't use simple_new in the parent, since the function signature
-        # doesn't match.
         values = self.asi8.copy()
-        return type(self)._simple_new(values, tz=self.tz, freq=self.freq)
+        return type(self)(values, dtype=self.dtype, freq=self.freq)
 
     # ----------------------------------------------------------------
     # ExtensionArray Interface
@@ -591,7 +599,7 @@ def _add_offset(self, offset):
                           "or DatetimeIndex", PerformanceWarning)
             result = self.astype('O') + offset
 
-        return type(self)(result, freq='infer')
+        return type(self)._from_sequence(result, freq='infer')
 
     def _sub_datetimelike_scalar(self, other):
         # subtract a datetime from myself, yielding a ndarray[timedelta64[ns]]
@@ -627,7 +635,9 @@ def _add_delta(self, delta):
         result : DatetimeArray
         """
         new_values = dtl.DatetimeLikeArrayMixin._add_delta(self, delta)
-        return type(self)(new_values, tz=self.tz, freq='infer')
+        return type(self)._from_sequence(new_values,
+                                         dtype=self.dtype,
+                                         freq="infer")
 
     # -----------------------------------------------------------------
     # Timezone Conversion and Localization Methods
@@ -930,14 +940,15 @@ def normalize(self):
                        dtype='datetime64[ns, Asia/Calcutta]', freq=None)
         """
         if self.tz is None or timezones.is_utc(self.tz):
-            not_null = self.notna()
+            not_null = ~self.isna()
             DAY_NS = ccalendar.DAY_SECONDS * 1000000000
             new_values = self.asi8.copy()
             adjustment = (new_values[not_null] % DAY_NS)
             new_values[not_null] = new_values[not_null] - adjustment
         else:
             new_values = conversion.normalize_i8_timestamps(self.asi8, self.tz)
-        return type(self)(new_values, freq='infer').tz_localize(self.tz)
+        return type(self)._from_sequence(new_values,
+                                         freq='infer').tz_localize(self.tz)
 
     def to_period(self, freq=None):
         """

diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py
@@ -350,7 +350,7 @@ def to_timestamp(self, freq=None, how='start'):
         new_data = self.asfreq(freq, how=how)
 
         new_data = libperiod.periodarr_to_dt64arr(new_data.asi8, base)
-        return DatetimeArrayMixin(new_data, freq='infer')
+        return DatetimeArrayMixin._from_sequence(new_data, freq='infer')
 
     # --------------------------------------------------------------------
     # Array-like / EA-Interface Methods

diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py
@@ -15,10 +15,11 @@
 from pandas.util._decorators import Appender
 
 from pandas.core.dtypes.common import (
-    _TD_DTYPE, ensure_int64, is_datetime64_dtype, is_float_dtype,
+    _NS_DTYPE, _TD_DTYPE, ensure_int64, is_datetime64_dtype, is_float_dtype,
     is_int64_dtype, is_integer_dtype, is_list_like, is_object_dtype, is_scalar,
     is_string_dtype, is_timedelta64_dtype, is_timedelta64_ns_dtype,
     pandas_dtype)
+from pandas.core.dtypes.dtypes import DatetimeTZDtype
 from pandas.core.dtypes.generic import (
     ABCDataFrame, ABCIndexClass, ABCSeries, ABCTimedeltaIndex)
 from pandas.core.dtypes.missing import isna
@@ -317,7 +318,8 @@ def _add_datetimelike_scalar(self, other):
         result = checked_add_with_arr(i8, other.value,
                                       arr_mask=self._isnan)
         result = self._maybe_mask_results(result)
-        return DatetimeArrayMixin(result, tz=other.tz, freq=self.freq)
+        dtype = DatetimeTZDtype(tz=other.tz) if other.tz else _NS_DTYPE
+        return DatetimeArrayMixin(result, dtype=dtype, freq=self.freq)
 
     def _addsub_offset_array(self, other, op):
         # Add or subtract Array-like of DateOffset objects

diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
@@ -25,9 +25,8 @@
     is_signed_integer_dtype, is_timedelta64_dtype, is_unsigned_integer_dtype)
 import pandas.core.dtypes.concat as _concat
 from pandas.core.dtypes.generic import (
-    ABCDataFrame, ABCDateOffset, ABCDatetimeIndex, ABCIndexClass,
-    ABCMultiIndex, ABCPeriodIndex, ABCSeries, ABCTimedeltaArray,
-    ABCTimedeltaIndex)
+    ABCDataFrame, ABCDateOffset, ABCIndexClass, ABCMultiIndex, ABCPeriodIndex,
+    ABCSeries, ABCTimedeltaArray, ABCTimedeltaIndex)
 from pandas.core.dtypes.missing import array_equivalent, isna
 
 from pandas.core import ops
@@ -549,6 +548,8 @@ def _get_attributes_dict(self):
 
     @Appender(_index_shared_docs['_shallow_copy'])
     def _shallow_copy(self, values=None, **kwargs):
+        from pandas.core.arrays import DatetimeArrayMixin
+
         if values is None:
             values = self.values
         attributes = self._get_attributes_dict()
@@ -557,8 +558,8 @@ def _shallow_copy(self, values=None, **kwargs):
             attributes['dtype'] = self.dtype
 
         # _simple_new expects an ndarray
-        values = getattr(values, 'values', values)
-        if isinstance(values, ABCDatetimeIndex):
+        values = getattr(values, '_values', values)
+        if isinstance(values, DatetimeArrayMixin):
             # `self.values` returns `self` for tz-aware, so we need to unwrap
             #  more specifically
             values = values.asi8

diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py
@@ -37,7 +37,6 @@ class DatetimeIndexOpsMixin(ExtensionOpsMixin):
     """
 
     # override DatetimeLikeArrayMixin method
-    copy = Index.copy
     unique = Index.unique
     take = Index.take
 

diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
@@ -64,21 +64,29 @@ def _new_DatetimeIndex(cls, d):
 
 class DatetimeDelegateMixin(DatetimelikeDelegateMixin):
     _extra_methods = [
+        'normalize',
+    ]
+    _extra_raw_methods = [
         'to_pydatetime',
            '_box_func',
-        '_box_values',
         '_local_timestamps',
     ]
-    _delegated_properties = DatetimeArray._datetimelike_ops
+    _extra_raw_properties = [
+        '_box_func',
+    ]
+    _delegated_properties = (
+        DatetimeArray._datetimelike_ops + _extra_raw_properties
+    )
     _delegated_methods = (
         DatetimeArray._datetimelike_methods + _extra_methods
     )
     _raw_properties = {
         'date',
         'time',
         'timetz',
+        '_box_func',
     }
-    _raw_methods = set(_extra_methods)
+    _raw_methods = set(_extra_raw_methods)
     _delegate_class = DatetimeArray
 
 
@@ -1160,12 +1168,15 @@ def slice_indexer(self, start=None, end=None, step=None, kind=None):
     is_year_end = wrap_field_accessor(DatetimeArray.is_year_end)
     is_leap_year = wrap_field_accessor(DatetimeArray.is_leap_year)
 
+    _local_timestamps = wrap_array_method(DatetimeArray._local_timestamps,
+                                          box=False)
     tz_localize = wrap_array_method(DatetimeArray.tz_localize, True)
     tz_convert = wrap_array_method(DatetimeArray.tz_convert, True)
     to_perioddelta = wrap_array_method(DatetimeArray.to_perioddelta,
                                        False)
+    to_pydatetime = wrap_array_method(DatetimeArray.to_pydatetime,
+                                      box=False)
     to_period = wrap_array_method(DatetimeArray.to_period, True)
-    normalize = wrap_array_method(DatetimeArray.normalize, True)
     to_julian_date = wrap_array_method(DatetimeArray.to_julian_date,
                                        False)
     month_name = wrap_array_method(DatetimeArray.month_name, True)

diff --git a/pandas/tests/extension/test_datetime.py b/pandas/tests/extension/test_datetime.py
@@ -17,15 +17,15 @@ def dtype(request):
 @pytest.fixture
 def data(dtype):
     data = DatetimeArray(pd.date_range("2000", periods=100, tz=dtype.tz),
-                         tz=dtype.tz)
+                         dtype=dtype)
     return data
 
 
 @pytest.fixture
 def data_missing(dtype):
     return DatetimeArray(
         np.array(['NaT', '2000-01-01'], dtype='datetime64[ns]'),
-        tz=dtype.tz
+        dtype=dtype
     )
 
 
@@ -35,15 +35,15 @@ def data_for_sorting(dtype):
     b = pd.Timestamp('2000-01-02')
     c = pd.Timestamp('2000-01-03')
     return DatetimeArray(np.array([b, c, a], dtype='datetime64[ns]'),
-                         tz=dtype.tz)
+                         dtype=dtype)
 
 
 @pytest.fixture
 def data_missing_for_sorting(dtype):
     a = pd.Timestamp('2000-01-01')
     b = pd.Timestamp('2000-01-02')
     return DatetimeArray(np.array([b, 'NaT', a], dtype='datetime64[ns]'),
-                         tz=dtype.tz)
+                         dtype=dtype)
 
 
 @pytest.fixture
@@ -59,7 +59,7 @@ def data_for_grouping(dtype):
     na = 'NaT'
     return DatetimeArray(np.array([b, b, na, na, a, a, b, c],
                                   dtype='datetime64[ns]'),
-                         tz=dtype.tz)
+                         dtype=dtype)
 
 
 @pytest.fixture

diff --git a/pandas/tests/indexes/datetimes/test_ops.py b/pandas/tests/indexes/datetimes/test_ops.py
@@ -451,7 +451,6 @@ def test_comparison(self):
         assert comp[11]
         assert not comp[9]
 
-    @pytest.mark.xfail(reason="TODO-pickle", strict=True)
     def test_pickle_unpickle(self):
         unpickled = tm.round_trip_pickle(self.rng)
         assert unpickled.freq is not None
@@ -461,7 +460,6 @@ def test_copy(self):
         repr(cp)
         tm.assert_index_equal(cp, self.rng)
 
-    @pytest.mark.xfail(reason="TODO-constructor")
     def test_shift(self):
         shifted = self.rng.shift(5)
         assert shifted[0] == self.rng[5]
@@ -516,7 +514,6 @@ def test_copy(self):
         repr(cp)
         tm.assert_index_equal(cp, self.rng)
 
-    @pytest.mark.xfail(reason="TODO-constructor")
     def test_shift(self):
 
         shifted = self.rng.shift(5)
@@ -546,7 +543,6 @@ def test_shift_periods(self):
                                         check_stacklevel=True):
             tm.assert_index_equal(idx.shift(n=0), idx)
 
-    @pytest.mark.xfail(reason="TODO-pickle", strict=True)
     def test_pickle_unpickle(self):
         unpickled = tm.round_trip_pickle(self.rng)
         assert unpickled.freq is not None