API: to_datetime, required unit with numerical

All numerical types now need a unit in to_datetime call.

* Update tests
* Add bool verification in tslib.pyx
* Bug correction with bool and unit in ['ms', 'us', 's', 'D']
  * to_datetime(True, unit='ms') did run without error
  * to_datetime(True, unit='ms', errors='ignore') returned None

Issue: #15836

Author: atbd

doc/source/whatsnew/v0.21.0.txt

@@ -40,6 +40,7 @@ Other Enhancements
 - :func:`DataFrame.clip()` and :func:`Series.clip()` have gained an ``inplace`` argument. (:issue:`15388`)
 - :func:`crosstab` has gained a ``margins_name`` parameter to define the name of the row / column that will contain the totals when ``margins=True``. (:issue:`15972`)
 - :func:`Dataframe.select_dtypes` now accepts scalar values for include/exclude as well as list-like. (:issue:`16855`)
+- :func:`to_datetime` requires an unit with numerical arg (:issue:`15836`)
 
 .. _whatsnew_0210.api_breaking:
 

pandas/_libs/tslib.pyx

@@ -33,7 +33,7 @@ from datetime cimport cmp_pandas_datetimestruct
 from libc.stdlib cimport free
 
 from util cimport (is_integer_object, is_float_object, is_datetime64_object,
-                   is_timedelta64_object, INT64_MAX)
+                   is_bool_object, is_timedelta64_object, INT64_MAX)
 cimport util
 
 from datetime cimport *
@@ -2230,6 +2230,11 @@ cpdef array_with_unit_to_datetime(ndarray values, unit, errors='coerce'):
     m = cast_from_unit(None, unit)
 
     if is_raise:
+        from pandas.core.dtypes.common import is_bool_dtype
+
+        if is_bool_dtype(values):
+            raise TypeError("{0} is not convertible to datetime"
+                            .format(values.dtype))
 
         # try a quick conversion to i8
         # if we have nulls that are not type-compat
@@ -2265,6 +2270,16 @@ cpdef array_with_unit_to_datetime(ndarray values, unit, errors='coerce'):
             if _checknull_with_nat(val):
                 iresult[i] = NPY_NAT
 
+            elif is_bool_object(val):
+                if is_raise:
+                    raise TypeError(
+                        "{0} is not convertible to datetime"
+                        .format(values.dtype)
+                    )
+                elif is_ignore:
+                    raise AssertionError
+                iresult[i] = NPY_NAT
+
             elif is_integer_object(val) or is_float_object(val):
 
                 if val != val or val == NPY_NAT:
@@ -2308,7 +2323,7 @@ cpdef array_with_unit_to_datetime(ndarray values, unit, errors='coerce'):
             else:
 
                 if is_raise:
-                    raise ValueError("non convertible value {0}"
+                    raise ValueError("non convertible value {0} "
                                      "with the unit '{1}'".format(
                                          val,
                                          unit))
@@ -2332,6 +2347,8 @@ cpdef array_with_unit_to_datetime(ndarray values, unit, errors='coerce'):
 
         if _checknull_with_nat(val):
             oresult[i] = NaT
+        elif is_bool_object(val):
+            oresult[i] = val
         elif is_integer_object(val) or is_float_object(val):
 
             if val != val or val == NPY_NAT:

pandas/core/dtypes/cast.py

@@ -154,7 +154,7 @@ def trans(x):  # noqa
                 if dtype.tz:
                     # convert to datetime and change timezone
                     from pandas import to_datetime
-                    result = to_datetime(result).tz_localize('utc')
+                    result = to_datetime(result, unit='ns').tz_localize('utc')
                     result = result.tz_convert(dtype.tz)
 
     except:
@@ -929,11 +929,11 @@ def maybe_cast_to_datetime(value, dtype, errors='raise'):
                                                                 dtype):
                     try:
                         if is_datetime64:
-                            value = to_datetime(value, errors=errors)._values
+                            value = to_datetime(value, unit='ns', errors=errors)._values
                         elif is_datetime64tz:
                             # input has to be UTC at this point, so just
                             # localize
-                            value = (to_datetime(value, errors=errors)
+                            value = (to_datetime(value, unit='ns', errors=errors)
                                      .tz_localize('UTC')
                                      .tz_convert(dtype.tz)
                                      )

pandas/core/indexes/datetimes.py

@@ -277,6 +277,7 @@ def __new__(cls, data=None,
 
         dayfirst = kwargs.pop('dayfirst', None)
         yearfirst = kwargs.pop('yearfirst', None)
+        unit = kwargs.pop('unit', None)
 
         freq_infer = False
         if not isinstance(freq, DateOffset):
@@ -333,7 +334,7 @@ def __new__(cls, data=None,
         if not (is_datetime64_dtype(data) or is_datetimetz(data) or
                 is_integer_dtype(data)):
             data = tools.to_datetime(data, dayfirst=dayfirst,
-                                     yearfirst=yearfirst)
+                                     unit=unit, yearfirst=yearfirst)
 
         if issubclass(data.dtype.type, np.datetime64) or is_datetimetz(data):
 

pandas/core/tools/datetimes.py

@@ -234,7 +234,7 @@ def to_datetime(arg, errors='raise', dayfirst=False, yearfirst=False,
         - If True, require an exact format match.
         - If False, allow the format to match anywhere in the target string.
 
-    unit : string, default 'ns'
+    unit : string
         unit of the arg (D,s,ms,us,ns) denote the unit, which is an
         integer or float number. This will be based off the origin.
         Example, with unit='ms' and origin='unix' (the default), this
@@ -337,6 +337,7 @@ def to_datetime(arg, errors='raise', dayfirst=False, yearfirst=False,
 
     """
     from pandas.core.indexes.datetimes import DatetimeIndex
+    from pandas.core.frame import DataFrame
 
     tz = 'utc' if utc else None
 
@@ -446,8 +447,15 @@ def _convert_listlike(arg, box, format, name=None, tz=tz):
             except (ValueError, TypeError):
                 raise e
 
+    def check_numerical_arg():
+        return ((is_scalar(arg) and (is_integer(arg) or is_float(arg))) or
+                (is_numeric_dtype(np.asarray(arg)) and np.asarray(arg).size))
+
     if arg is None:
         return None
+    elif ((not isinstance(arg, DataFrame)) and
+          (check_numerical_arg() and unit is None and format is None)):
+        raise ValueError("a unit is required in case of numerical arg")
 
     # handle origin
     if origin == 'julian':
@@ -474,8 +482,7 @@ def _convert_listlike(arg, box, format, name=None, tz=tz):
 
         # arg must be a numeric
         original = arg
-        if not ((is_scalar(arg) and (is_integer(arg) or is_float(arg))) or
-                is_numeric_dtype(np.asarray(arg))):
+        if not check_numerical_arg():
             raise ValueError(
                 "'{arg}' is not compatible with origin='{origin}'; "
                 "it must be numeric with a unit specified ".format(

pandas/tests/indexes/datetimes/test_tools.py

@@ -25,6 +25,35 @@
                     compat)
 
 
+@pytest.fixture(params=['D', 's', 'ms', 'us', 'ns'])
+def units(request):
+    return request.param
+
+
+@pytest.fixture
+def epoch_1960():
+    # for origin as 1960-01-01
+    return Timestamp('1960-01-01')
+
+
+@pytest.fixture
+def units_from_epochs():
+    return list(range(5))
+
+
+@pytest.fixture(params=[epoch_1960(),
+                        epoch_1960().to_pydatetime(),
+                        epoch_1960().to_datetime64(),
+                        str(epoch_1960())])
+def epochs(request):
+    return request.param
+
+
+@pytest.fixture
+def julian_dates():
+    return pd.date_range('2014-1-1', periods=10).to_julian_date().values
+
+
 class TimeConversionFormats(object):
 
     def test_to_datetime_format(self):
@@ -306,25 +335,6 @@ def test_to_datetime_tz_psycopg2(self):
                                     dtype='datetime64[ns, UTC]')
         tm.assert_index_equal(result, expected)
 
-    def test_datetime_bool(self):
-        # GH13176
-        with pytest.raises(TypeError):
-            to_datetime(False)
-        assert to_datetime(False, errors="coerce") is NaT
-        assert to_datetime(False, errors="ignore") is False
-        with pytest.raises(TypeError):
-            to_datetime(True)
-        assert to_datetime(True, errors="coerce") is NaT
-        assert to_datetime(True, errors="ignore") is True
-        with pytest.raises(TypeError):
-            to_datetime([False, datetime.today()])
-        with pytest.raises(TypeError):
-            to_datetime(['20130101', True])
-        tm.assert_index_equal(to_datetime([0, False, NaT, 0.0],
-                                          errors="coerce"),
-                              DatetimeIndex([to_datetime(0), NaT,
-                                             NaT, to_datetime(0)]))
-
     def test_datetime_invalid_datatype(self):
         # GH13176
 
@@ -334,7 +344,28 @@ def test_datetime_invalid_datatype(self):
             pd.to_datetime(pd.to_datetime)
 
 
-class ToDatetimeUnit(object):
+class TestToDatetimeUnit(object):
+
+    def test_datetime_bool(self, units):
+        # GH13176
+        #import pdb; pdb.set_trace()
+        with pytest.raises(TypeError):
+            to_datetime(False, unit=units)
+        assert to_datetime(False, unit=units, errors="coerce") is NaT
+        assert to_datetime(False, unit=units, errors="ignore") == False
+        with pytest.raises(TypeError):
+            to_datetime(True, unit=units)
+        assert to_datetime(True, unit=units, errors="coerce") is NaT
+        assert to_datetime(True, unit=units, errors="ignore") == True
+        with pytest.raises(TypeError):
+            to_datetime([False, datetime.today()], unit=units)
+        with pytest.raises(TypeError):
+            to_datetime([True, '20130101'], unit=units)
+
+        tm.assert_index_equal(to_datetime([0, False, NaT, 0.0],
+                                          errors="coerce"),
+                              DatetimeIndex([to_datetime(0, unit=units), NaT,
+                                             NaT, to_datetime(0, unit=units)]))
 
     def test_unit(self):
         # GH 11758
@@ -409,10 +440,10 @@ def test_unit_with_numeric(self):
         arr1 = [1.434692e+18, 1.432766e+18]
         arr2 = np.array(arr1).astype('int64')
         for errors in ['ignore', 'raise', 'coerce']:
-            result = pd.to_datetime(arr1, errors=errors)
+            result = pd.to_datetime(arr1, unit='ns', errors=errors)
             tm.assert_index_equal(result, expected)
 
-            result = pd.to_datetime(arr2, errors=errors)
+            result = pd.to_datetime(arr2, unit='ns', errors=errors)
             tm.assert_index_equal(result, expected)
 
         # but we want to make sure that we are coercing
@@ -421,37 +452,37 @@ def test_unit_with_numeric(self):
                                   '2015-06-19 05:33:20',
                                   '2015-05-27 22:33:20'])
         arr = ['foo', 1.434692e+18, 1.432766e+18]
-        result = pd.to_datetime(arr, errors='coerce')
+        result = pd.to_datetime(arr, unit='ns', errors='coerce')
         tm.assert_index_equal(result, expected)
 
         expected = DatetimeIndex(['2015-06-19 05:33:20',
                                   '2015-05-27 22:33:20',
                                   'NaT',
                                   'NaT'])
         arr = [1.434692e+18, 1.432766e+18, 'foo', 'NaT']
-        result = pd.to_datetime(arr, errors='coerce')
+        result = pd.to_datetime(arr, unit='ns', errors='coerce')
         tm.assert_index_equal(result, expected)
 
     def test_unit_mixed(self):
 
         # mixed integers/datetimes
         expected = DatetimeIndex(['2013-01-01', 'NaT', 'NaT'])
         arr = [pd.Timestamp('20130101'), 1.434692e+18, 1.432766e+18]
-        result = pd.to_datetime(arr, errors='coerce')
+        result = pd.to_datetime(arr, unit='ns', errors='coerce')
         tm.assert_index_equal(result, expected)
 
         with pytest.raises(ValueError):
-            pd.to_datetime(arr, errors='raise')
+            pd.to_datetime(arr, unit='ns', errors='raise')
 
         expected = DatetimeIndex(['NaT',
                                   'NaT',
                                   '2013-01-01'])
         arr = [1.434692e+18, 1.432766e+18, pd.Timestamp('20130101')]
-        result = pd.to_datetime(arr, errors='coerce')
+        result = pd.to_datetime(arr, unit='ns', errors='coerce')
         tm.assert_index_equal(result, expected)
 
         with pytest.raises(ValueError):
-            pd.to_datetime(arr, errors='raise')
+            pd.to_datetime(arr, unit='ns', errors='raise')
 
     def test_dataframe(self):
 
@@ -1488,35 +1519,6 @@ def test_normalize_date():
     assert (result == datetime(2012, 9, 7))
 
 
-@pytest.fixture(params=['D', 's', 'ms', 'us', 'ns'])
-def units(request):
-    return request.param
-
-
-@pytest.fixture
-def epoch_1960():
-    # for origin as 1960-01-01
-    return Timestamp('1960-01-01')
-
-
-@pytest.fixture
-def units_from_epochs():
-    return list(range(5))
-
-
-@pytest.fixture(params=[epoch_1960(),
-                        epoch_1960().to_pydatetime(),
-                        epoch_1960().to_datetime64(),
-                        str(epoch_1960())])
-def epochs(request):
-    return request.param
-
-
-@pytest.fixture
-def julian_dates():
-    return pd.date_range('2014-1-1', periods=10).to_julian_date().values
-
-
 class TestOrigin(object):
 
     def test_to_basic(self, julian_dates):

pandas/tests/io/json/test_json_table_schema.py

@@ -349,15 +349,15 @@ def test_make_field_float(self):
 
     def test_make_field_datetime(self):
         data = [1., 2., 3.]
-        kinds = [pd.Series(pd.to_datetime(data), name='values'),
-                 pd.to_datetime(data)]
+        kinds = [pd.Series(pd.to_datetime(data, unit='ns'), name='values'),
+                 pd.to_datetime(data, unit='ns')]
         for kind in kinds:
             result = make_field(kind)
             expected = {"name": "values", "type": 'datetime'}
             assert result == expected
 
-        kinds = [pd.Series(pd.to_datetime(data, utc=True), name='values'),
-                 pd.to_datetime(data, utc=True)]
+        kinds = [pd.Series(pd.to_datetime(data, unit='ns', utc=True), name='values'),
+                 pd.to_datetime(data, unit='ns', utc=True)]
         for kind in kinds:
             result = make_field(kind)
             expected = {"name": "values", "type": 'datetime', "tz": "UTC"}

pandas/tests/io/json/test_pandas.py

@@ -181,7 +181,7 @@ def _check_orient(df, orient, dtype=None, numpy=False,
 
             if not convert_axes and df.index.dtype.type == np.datetime64:
                 unser.index = DatetimeIndex(
-                    unser.index.values.astype('i8') * 1e6)
+                    unser.index.values.astype('i8') * 1e6, unit='ns')
             if orient == "records":
                 # index is not captured in this orientation
                 tm.assert_almost_equal(df.values, unser.values,
@@ -832,7 +832,7 @@ def test_timedelta(self):
 
         result = pd.read_json(frame.to_json(date_unit='ns'))
         result['a'] = pd.to_timedelta(result.a, unit='ns')
-        result['c'] = pd.to_datetime(result.c)
+        result['c'] = pd.to_datetime(result.c, unit='ns')
         assert_frame_equal(frame, result)
 
     def test_mixed_timedelta_datetime(self):

pandas/tests/test_internals.py

@@ -98,7 +98,7 @@ def create_block(typestr, placement, item_shape=None, num_offset=0):
         assert m is not None, "incompatible typestr -> {0}".format(typestr)
         tz = m.groups()[0]
         assert num_items == 1, "must have only 1 num items for a tz-aware"
-        values = DatetimeIndex(np.arange(N) * 1e9, tz=tz)
+        values = DatetimeIndex(np.arange(N) * 1e9, unit='ns', tz=tz)
     elif typestr in ('timedelta', 'td', 'm8[ns]'):
         values = (mat * 1).astype('m8[ns]')
     elif typestr in ('category', ):

pandas/tests/test_resample.py

@@ -1720,7 +1720,7 @@ def test_nanosecond_resample_error(self):
         start = 1443707890427
         exp_start = 1443707890400
         indx = pd.date_range(
-            start=pd.to_datetime(start),
+            start=pd.to_datetime(start, unit='ns'),
             periods=10,
             freq='100n'
         )
@@ -1729,7 +1729,7 @@ def test_nanosecond_resample_error(self):
         result = r.agg('mean')
 
         exp_indx = pd.date_range(
-            start=pd.to_datetime(exp_start),
+            start=pd.to_datetime(exp_start, unit='ns'),
             periods=10,
             freq='100n'
         )