API: add DatetimeBlockTZ pandas-dev#8260

fix scalar comparisons vs None generally

fix NaT formattting in Series

TST: skip postgresql test with tz's

update for msgpack

Conflicts:
	pandas/core/base.py
	pandas/core/categorical.py
	pandas/core/format.py
	pandas/tests/test_base.py
	pandas/util/testing.py

full interop for tz-aware Series & timedeltas pandas-dev#10763

Author: jreback

doc/source/basics.rst

@@ -1549,9 +1549,10 @@ dtypes
 ------
 
 The main types stored in pandas objects are ``float``, ``int``, ``bool``,
-``datetime64[ns]``, ``timedelta[ns]`` and ``object``. In addition these dtypes
-have item sizes, e.g. ``int64`` and ``int32``. A convenient :attr:`~DataFrame.dtypes``
-attribute for DataFrames returns a Series with the data type of each column.
+``datetime64[ns]`` and ``datetime64[ns, tz]`` (in >= 0.17.0), ``timedelta[ns]``, ``category`` (in >= 0.15.0), and ``object``. In addition these dtypes
+have item sizes, e.g. ``int64`` and ``int32``. See :ref:`Series with TZ <timeseries.timezone_series>` for more detail on ``datetime64[ns, tz]`` dtypes.
+
+A convenient :attr:`~DataFrame.dtypes`` attribute for DataFrames returns a Series with the data type of each column.
 
 .. ipython:: python
 
@@ -1773,8 +1774,14 @@ dtypes:
    df['tdeltas'] = df.dates.diff()
    df['uint64'] = np.arange(3, 6).astype('u8')
    df['other_dates'] = pd.date_range('20130101', periods=3).values
+   df['tz_aware_dates'] = pd.date_range('20130101', periods=3, tz='US/Eastern')
    df
 
+And the dtypes
+
+.. ipython:: python
+
+   df.dtypes
 
 :meth:`~DataFrame.select_dtypes` has two parameters ``include`` and ``exclude`` that allow you to
 say "give me the columns WITH these dtypes" (``include``) and/or "give the
@@ -1827,7 +1834,7 @@ All numpy dtypes are subclasses of ``numpy.generic``:
 
 .. note::
 
-    Pandas also defines an additional ``category`` dtype, which is not integrated into the normal
+    Pandas also defines an types `category``, and ``datetime64[ns, tz]``, which are not integrated into the normal
     numpy hierarchy and wont show up with the above function.
 
 .. note::

doc/source/timeseries.rst

@@ -1734,3 +1734,30 @@ constructor as well as ``tz_localize``.
 
    # tz_convert(None) is identical with tz_convert('UTC').tz_localize(None)
    didx.tz_convert('UCT').tz_localize(None)
+
+.. _timeseries.timezone_series:
+
+TZ aware Dtypes
+~~~~~~~~~~~~~~~
+
+.. versionadded:: 0.17.0
+
+``Series/DatetimeIndex`` with a timezone naive value are represented with a dtype of ``datetime64[ns]``.
+
+.. ipython:: python
+
+   dr = pd.date_range('20130101',periods=3)
+   dr
+   s = Series(dr)
+   s
+
+``Series/DatetimeIndex`` with a timezone aware value are represented with a dtype of ``datetime64[ns, tz]``.
+
+.. ipython:: python
+
+   dr = pd.date_range('20130101',periods=3,tz='US/Eastern')
+   dr
+   s = Series(dr)
+   s
+
+Both of these ``Series`` can be manipulated via the ``.dt`` accessor, see the :ref:`docs <basics.dt_accessors>` as well.

doc/source/whatsnew/v0.17.0.txt

@@ -14,6 +14,7 @@ users upgrade to this version.
 Highlights include:
 
 - Release the Global Interpreter Lock (GIL) on some cython operations, see :ref:`here <whatsnew_0170.gil>`
+- Support for a ``datetime64[ns]`` with timezones as a first-class dtype, see :ref:`here <whatsnew_0170.tz>`
 - The default for ``to_datetime`` will now be to ``raise`` when presented with unparseable formats,
   previously this would return the original input, see :ref:`here <whatsnew_0170.api_breaking.to_datetime>`
 - The default for ``dropna`` in ``HDFStore`` has changed to ``False``, to store by default all rows even
@@ -564,6 +565,84 @@ Removal of prior version deprecations/changes
 
 - Remove use of some deprecated numpy comparison operations, mainly in tests. (:issue:`10569`)
 
+.. _dask: https://dask.readthedocs.org/en/latest/
+
+.. _whatsnew_0170.tz:
+
+Datetime with TZ
+~~~~~~~~~~~~~~~~
+
+We are adding an implementation that natively supports datetime with timezones. A ``Series`` or a ``DataFrame`` column previously
+*could* be assigned a datetime with timezones, and would work as an ``object`` dtype. This had performance issues with a large
+number rows. (:issue:`8260`, :issue:`10763`)
+
+The new implementation allows for having a single-timezone across all rows, and operating on it in a performant manner.
+
+.. ipython:: python
+
+   df = DataFrame({'A' : date_range('20130101',periods=3),
+                   'B' : date_range('20130101',periods=3,tz='US/Eastern'),
+                   'C' : date_range('20130101',periods=3,tz='CET')})
+   df
+   df.dtypes
+
+.. ipython:: python
+
+   df.B
+   df.B.dt.tz_localize(None)
+
+This uses a new-dtype representation as well, that is very similar in look-and-feel to its numpy cousin ``datetime64[ns]``
+
+.. ipython:: python
+
+   df['B'].dtype
+   type(df['B']).dtype
+
+.. note::
+
+   There is a slightly different string repr for the underlying ``DatetimeIndex`` as a result of the dtype changes, but
+   functionaily these are the same.
+
+   Previously
+
+   .. code-block:: python
+
+      In [1]: pd.date_range('20130101',periods=3,tz='US/Eastern')
+      Out[1]: DatetimeIndex(['2013-01-01 00:00:00-05:00', '2013-01-02 00:00:00-05:00',
+                             '2013-01-03 00:00:00-05:00'],
+                            dtype='datetime64[ns]', freq='D', tz='US/Eastern')
+
+      In [2]: pd.date_range('20130101',periods=3,tz='US/Eastern').dtype
+      Out[2]: dtype('<M8[ns]')
+
+   .. ipython:: python
+
+      pd.date_range('20130101',periods=3,tz='US/Eastern')
+      pd.date_range('20130101',periods=3,tz='US/Eastern').dtype
+
+explain ``DatetimeIndex`` repr change & dtype display
+
+
+.. _whatsnew_0170.gil:
+
+Releasing the GIL
+~~~~~~~~~~~~~~~~~
+
+We are releasing the global-interpreter-lock (GIL) on some cython operations.
+This will allow other threads to run simultaneously during computation, potentially allowing performance improvements
+from multi-threading. Notably ``groupby`` and some indexing operations are a benefit from this. (:issue:`8882`)
+
+For example the groupby expression in the following code will have the GIL released during the factorization step, e.g. ``df.groupby('key')``
+as well as the ``.sum()`` operation.
+
+.. code-block:: python
+
+   N = 1e6
+   df = DataFrame({'key' : np.random.randint(0,ngroups,size=N),
+                   'data' : np.random.randn(N) })
+   df.groupby('key')['data'].sum()
+
+Releasing of the GIL could benefit an application that uses threads for user interactions (e.g. ``QT``), or performaning multi-threaded computations. A nice example of a library that can handle these types of computation-in-parallel is the dask_ library.
 
 .. _whatsnew_0170.performance:
 
@@ -587,6 +666,9 @@ Bug Fixes
 
 
 - Bug in ``DataFrame.to_html(index=False)`` renders unnecessary ``name`` row (:issue:`10344`)
+- Bug in ``DatetimeIndex`` when localizing with ``NaT`` (:issue:`10477`)
+- Bug in ``Series.dt`` ops in preserving meta-data (:issue:`10477`)
+- Bug in preserving ``NaT`` when passed in an otherwise invalid ``to_datetime`` construction (:issue:`10477`)
 - Bug in ``DataFrame.apply`` when function returns categorical series. (:issue:`9573`)
 - Bug in ``to_datetime`` with invalid dates and formats supplied (:issue:`10154`)
 - Bug in ``Index.drop_duplicates`` dropping name(s) (:issue:`10115`)

pandas/core/algorithms.py

@@ -206,7 +206,7 @@ def value_counts(values, sort=True, ascending=False, normalize=False,
     """
     from pandas.core.series import Series
     from pandas.tools.tile import cut
-    from pandas.tseries.period import PeriodIndex
+    from pandas import Index, PeriodIndex, DatetimeIndex
 
     name = getattr(values, 'name', None)
     values = Series(values).values
@@ -225,11 +225,15 @@ def value_counts(values, sort=True, ascending=False, normalize=False,
 
         dtype = values.dtype
         is_period = com.is_period_arraylike(values)
+        is_datetimetz = com.is_datetimetz(values)
 
-        if com.is_datetime_or_timedelta_dtype(dtype) or is_period:
+        if com.is_datetime_or_timedelta_dtype(dtype) or is_period or is_datetimetz:
 
             if is_period:
-                values = PeriodIndex(values, name=name)
+                values = PeriodIndex(values)
+            elif is_datetimetz:
+                tz = getattr(values, 'tz', None)
+                values = DatetimeIndex(values).tz_localize(None)
 
             values = values.view(np.int64)
             keys, counts = htable.value_count_int64(values)
@@ -239,8 +243,14 @@ def value_counts(values, sort=True, ascending=False, normalize=False,
                 msk = keys != iNaT
                 keys, counts = keys[msk], counts[msk]
 
+            # localize to the original tz if necessary
+            if is_datetimetz:
+                keys = DatetimeIndex(keys).tz_localize(tz)
+
             # convert the keys back to the dtype we came in
-            keys = keys.astype(dtype)
+            else:
+                keys = keys.astype(dtype)
+
 
         elif com.is_integer_dtype(dtype):
             values = com._ensure_int64(values)
@@ -254,7 +264,9 @@ def value_counts(values, sort=True, ascending=False, normalize=False,
                 keys = np.insert(keys, 0, np.NaN)
                 counts = np.insert(counts, 0, mask.sum())
 
-        result = Series(counts, index=com._values_from_object(keys), name=name)
+        if not isinstance(keys, Index):
+            keys = Index(keys)
+        result = Series(counts, index=keys, name=name)
 
         if bins is not None:
             # TODO: This next line should be more efficient

pandas/core/base.py

@@ -396,6 +396,14 @@ def hasnans(self):
         """ return if I have any nans; enables various perf speedups """
         return com.isnull(self).any()
 
+    def _reduce(self, op, name, axis=0, skipna=True, numeric_only=None,
+                filter_type=None, **kwds):
+        """ perform the reduction type operation if we can """
+        func = getattr(self,name,None)
+        if func is None:
+            raise TypeError("{klass} cannot perform the operation {op}".format(klass=self.__class__.__name__,op=name))
+        return func(**kwds)
+
     def value_counts(self, normalize=False, sort=True, ascending=False,
                      bins=None, dropna=True):
         """
@@ -585,7 +593,7 @@ def drop_duplicates(self, keep='first', inplace=False):
     @deprecate_kwarg('take_last', 'keep', mapping={True: 'last', False: 'first'})
     @Appender(_shared_docs['duplicated'] % _indexops_doc_kwargs)
     def duplicated(self, keep='first'):
-        keys = com._ensure_object(self.values)
+        keys = com._values_from_object(com._ensure_object(self.values))
         duplicated = lib.duplicated(keys, keep=keep)
         try:
             return self._constructor(duplicated,

pandas/core/categorical.py

@@ -12,13 +12,14 @@
 import pandas.core.common as com
 from pandas.util.decorators import cache_readonly, deprecate_kwarg
 
-from pandas.core.common import (CategoricalDtype, ABCSeries, ABCIndexClass, ABCCategoricalIndex,
+from pandas.core.common import (ABCSeries, ABCIndexClass, ABCPeriodIndex, ABCCategoricalIndex,
                                 isnull, notnull, is_dtype_equal,
                                 is_categorical_dtype, is_integer_dtype, is_object_dtype,
                                 _possibly_infer_to_datetimelike, get_dtype_kinds,
                                 is_list_like, is_sequence, is_null_slice, is_bool,
                                 _ensure_platform_int, _ensure_object, _ensure_int64,
                                 _coerce_indexer_dtype, take_1d)
+from pandas.core.dtypes import CategoricalDtype
 from pandas.util.terminal import get_terminal_size
 from pandas.core.config import get_option