ENH - incompatibility checks and incompatible type unions

pandas-dev · TomAugspurger · May 21, 2019 · Nov 7, 2018 · Nov 8, 2018 · Nov 8, 2018
commit cdaa5b0fb56a4bcee2803f96029841b080858386
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
@@ -2756,6 +2756,15 @@ def _get_reconciled_name_object(self, other):
             return self._shallow_copy(name=name)
         return self
 
+    def _union_inconsistent_dtypes(self, other):
+        this = self.astype('O')
+        other = Index(other).astype('O')
+        return Index.union(this, other).astype('O')
+
+    def _is_inconsistent(self, other):
+        return (type(self) is not type(other)
+                or not is_dtype_equal(self.dtype, other.dtype))
+
     def union(self, other):
         """
         Form the union of two Index objects and sorts if possible.
@@ -2780,27 +2789,30 @@ def union(self, other):
         self._assert_can_do_setop(other)
         other = ensure_index(other)
 
-        if self.equals(other):
+        if self._is_inconsistent(other):
+            return self._union_inconsistent_dtypes(other)
+
+        # if is_dtype_equal(self.dtype, other.dtype):
+        if len(self) == 0:
+            return other._get_reconciled_name_object(self)
+        elif len(other) == 0:
             return self._get_reconciled_name_object(other)
 
-        # Don't allow empty index to move on. If `other` is not monotonic-incr
-        # ...will fail
-        if is_dtype_equal(self.dtype, other.dtype):
-            if len(self) == 0:
-                return other._get_reconciled_name_object(self)
-            elif len(other) == 0:
-                return self._get_reconciled_name_object(other)
+        if self.equals(other):
+            return self._get_reconciled_name_object(other)
 
 
         # TODO: is_dtype_union_equal is a hack around
         # 1. buggy set ops with duplicates (GH #13432)
         # 2. CategoricalIndex lacking setops (GH #10186)
         # Once those are fixed, this workaround can be removed
-        if not is_dtype_union_equal(self.dtype, other.dtype):
-            this = self.astype('O')
-            other = other.astype('O')
-            # force object dtype, for empty index cases
-            return this.union(other).astype('O')
+
+        # remove? this might be too lenient
+        # if not is_dtype_union_equal(self.dtype, other.dtype):
+        #     this = self.astype('O')
+        #     other = other.astype('O')
+        #     # force object dtype, for empty index cases
+        #     return this.union(other).astype('O')
 
         # TODO(EA): setops-refactor, clean all this up
         if is_period_dtype(self) or is_datetime64tz_dtype(self):

diff --git a/pandas/core/indexes/category.py b/pandas/core/indexes/category.py
@@ -872,6 +872,14 @@ def _delegate_method(self, name, *args, **kwargs):
             return res
         return CategoricalIndex(res, name=self.name)
 
+    def _is_inconsistent(self, other):
+        if type(self) is not type(other):
+            return True
+        elif type(self.dtype) is not type(other.dtype):
+            return True
+        else:
+            return False
+
 
 CategoricalIndex._add_numeric_methods_add_sub_disabled()
 CategoricalIndex._add_numeric_methods_disabled()

diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
@@ -578,6 +578,17 @@ def unique(self, level=None):
         result = super(DatetimeIndex, naive).unique(level=level)
         return self._shallow_copy(result.values)
 
+    def _is_inconsistent(self, other):
+        is_inconsistent = super(DatetimeIndex, self)._is_inconsistent(other)
+        if is_inconsistent:
+            if hasattr(other, 'dtype'):
+                # If same base, consider consistent, let UTC logic takeover
+                return self.dtype.base != other.dtype.base
+            else:
+                return True
+        else:
+            return is_inconsistent
+
     def union(self, other):
         """
         Specialized union for DatetimeIndex objects. If combine
@@ -594,6 +605,9 @@ def union(self, other):
         """
         self._assert_can_do_setop(other)
 
+        if self._is_inconsistent(other):
+            return self._union_inconsistent_dtypes(other)
+
         if len(other) == 0 or self.equals(other) or len(self) == 0:
             return super(DatetimeIndex, self).union(other)
 

diff --git a/pandas/core/indexes/numeric.py b/pandas/core/indexes/numeric.py
@@ -228,6 +228,17 @@ def _assert_safe_casting(cls, data, subarr):
                 raise TypeError('Unsafe NumPy casting, you must '
                                 'explicitly cast')
 
+    def _is_inconsistent(self, other):
+        from pandas.core.indexes.range import RangeIndex
+        is_inconsistent = super(Int64Index, self)._is_inconsistent(other)
+        if is_inconsistent:
+            if type(self) is Int64Index and isinstance(other, RangeIndex):
+                return False
+            else:
+                return True
+        else:
+            return is_inconsistent
+
 
 Int64Index._add_numeric_methods()
 Int64Index._add_logical_methods()

diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py
@@ -416,6 +416,13 @@ def _extended_gcd(self, a, b):
             old_t, t = t, old_t - quotient * t
         return old_r, old_s, old_t
 
+    def _is_inconsistent(self, other):
+        is_inconsistent = super(RangeIndex, self)._is_inconsistent(other)
+        if is_inconsistent:
+            return not isinstance(other, Int64Index)
+        else:
+            return is_inconsistent
+
     def union(self, other):
         """
         Form the union of two Index objects and sorts if possible
@@ -429,6 +436,10 @@ def union(self, other):
         union : Index
         """
         self._assert_can_do_setop(other)
+
+        if self._is_inconsistent(other):
+            return self._union_inconsistent_dtypes(other)
+
         if len(other) == 0 or self.equals(other) or len(self) == 0:
             return super(RangeIndex, self).union(other)
 

diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py
@@ -283,6 +283,9 @@ def union(self, other):
         """
         self._assert_can_do_setop(other)
 
+        if self._is_inconsistent(other):
+            return self._union_inconsistent_dtypes(other)
+
         if len(other) == 0 or self.equals(other) or len(self) == 0:
             return super(TimedeltaIndex, self).union(other)