From 76247c142893c710e970c4cf8a25d73121aa5a2b Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 22 Jul 2019 10:03:56 -0700 Subject: [PATCH] BUG: Fix inserting of wrong-dtyped NaT, closes #27297 (#27311) --- doc/source/whatsnew/v1.0.0.rst | 2 +- pandas/_libs/index.pyx | 3 ++ pandas/core/internals/blocks.py | 3 +- pandas/tests/series/indexing/test_indexing.py | 32 +++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 9caf127553e05..87d3e47cbefe5 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -87,7 +87,7 @@ Categorical Datetimelike ^^^^^^^^^^^^ - +- Bug in :meth:`Series.__setitem__` incorrectly casting ``np.timedelta64("NaT")`` to ``np.datetime64("NaT")`` when inserting into a :class:`Series` with datetime64 dtype (:issue:`27311`) - - diff --git a/pandas/_libs/index.pyx b/pandas/_libs/index.pyx index 7000c07b1f5a6..13812663dd907 100644 --- a/pandas/_libs/index.pyx +++ b/pandas/_libs/index.pyx @@ -533,6 +533,9 @@ cpdef convert_scalar(ndarray arr, object value): pass elif isinstance(value, (datetime, np.datetime64, date)): return Timestamp(value).value + elif util.is_timedelta64_object(value): + # exclude np.timedelta64("NaT") from value != value below + pass elif value is None or value != value: return NPY_NAT elif isinstance(value, str): diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index ace57938f948c..e352444a98bbe 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -2589,8 +2589,9 @@ def setitem(self, indexer, value): try: return super().setitem(indexer, value) except (ValueError, TypeError): + obj_vals = self.values.astype(object) newb = make_block( - self.values.astype(object), placement=self.mgr_locs, klass=ObjectBlock + obj_vals, placement=self.mgr_locs, klass=ObjectBlock, ndim=self.ndim ) return newb.setitem(indexer, value) diff --git a/pandas/tests/series/indexing/test_indexing.py b/pandas/tests/series/indexing/test_indexing.py index d73be76795c88..2d36bfdb93a17 100644 --- a/pandas/tests/series/indexing/test_indexing.py +++ b/pandas/tests/series/indexing/test_indexing.py @@ -654,6 +654,38 @@ def test_timedelta_assignment(): tm.assert_series_equal(s, expected) +@pytest.mark.parametrize( + "nat_val,should_cast", + [ + (pd.NaT, True), + (np.timedelta64("NaT", "ns"), False), + (np.datetime64("NaT", "ns"), True), + ], +) +@pytest.mark.parametrize("tz", [None, "UTC"]) +def test_dt64_series_assign_nat(nat_val, should_cast, tz): + # some nat-like values should be cast to datetime64 when inserting + # into a datetime64 series. Others should coerce to object + # and retain their dtypes. + dti = pd.date_range("2016-01-01", periods=3, tz=tz) + base = pd.Series(dti) + expected = pd.Series([pd.NaT] + list(dti[1:]), dtype=dti.dtype) + if not should_cast: + expected = expected.astype(object) + + ser = base.copy(deep=True) + ser[0] = nat_val + tm.assert_series_equal(ser, expected) + + ser = base.copy(deep=True) + ser.loc[0] = nat_val + tm.assert_series_equal(ser, expected) + + ser = base.copy(deep=True) + ser.iloc[0] = nat_val + tm.assert_series_equal(ser, expected) + + @pytest.mark.parametrize( "nat_val,should_cast", [