Merge remote-tracking branch 'upstream/master' into read_excel-docstring

* upstream/master:
  REF/TST: replace capture_stdout with pytest capsys fixture (pandas-dev#24501)
  BUG: fix .iat assignment creates a new column (pandas-dev#24495)
  DOC: add checks on the returns section in the docstrings (pandas-dev#23138) (pandas-dev#23432)
  ENH: Add strings_as_fixed_length parameter for df.to_records() (pandas-dev#18146) (pandas-dev#22229)
  TST: Skip db tests unless explicitly specified in -m pattern (pandas-dev#24492)
  Mix EA into DTA/TDA; part of 24024 (pandas-dev#24502)
  DOC: Fix building of a single API document (pandas-dev#24506)

Author: thoo

.travis.yml

@@ -34,23 +34,23 @@ matrix:
     include:
     - dist: trusty
       env:
-        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="not slow and not network and not db"
+        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="not slow and not network"
 
     - dist: trusty
       env:
-        - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" PATTERN="not slow"
+        - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" PATTERN="not slow and db"
       addons:
         apt:
           packages:
           - python-gtk2
 
     - dist: trusty
       env:
-        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="not slow and not network" LOCALE_OVERRIDE="zh_CN.UTF-8"
+        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="not slow and not network and db" LOCALE_OVERRIDE="zh_CN.UTF-8"
 
     - dist: trusty
       env:
-        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36.yaml" PATTERN="not slow and not network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true
+        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36.yaml" PATTERN="not slow and not network and db" PANDAS_TESTING_MODE="deprecate" COVERAGE=true
 
     # In allow_failures
     - dist: trusty

ci/azure/posix.yml

@@ -12,37 +12,37 @@ jobs:
         py35_np_120:
           ENV_FILE: ci/deps/azure-macos-35.yaml
           CONDA_PY: "35"
-          PATTERN: "not slow and not network and not db"
+          PATTERN: "not slow and not network"
 
       ${{ if eq(parameters.name, 'Linux') }}:
         py27_np_120:
           ENV_FILE: ci/deps/azure-27-compat.yaml
           CONDA_PY: "27"
-          PATTERN: "not slow and not network and not db"
+          PATTERN: "not slow and not network"
 
         py27_locale_slow_old_np:
           ENV_FILE: ci/deps/azure-27-locale.yaml
           CONDA_PY: "27"
-          PATTERN: "slow and not db"
+          PATTERN: "slow"
           LOCALE_OVERRIDE: "zh_CN.UTF-8"
           EXTRA_APT: "language-pack-zh-hans"
 
         py36_locale_slow:
           ENV_FILE: ci/deps/azure-36-locale_slow.yaml
           CONDA_PY: "36"
-          PATTERN: "not slow and not network and not db"
+          PATTERN: "not slow and not network"
           LOCALE_OVERRIDE: "it_IT.UTF-8"
 
         py37_locale:
           ENV_FILE: ci/deps/azure-37-locale.yaml
           CONDA_PY: "37"
-          PATTERN: "not slow and not network and not db"
+          PATTERN: "not slow and not network"
           LOCALE_OVERRIDE: "zh_CN.UTF-8"
 
         py37_np_dev:
           ENV_FILE: ci/deps/azure-37-numpydev.yaml
           CONDA_PY: "37"
-          PATTERN: "not slow and not network and not db"
+          PATTERN: "not slow and not network"
           TEST_ARGS: "-W error"
           PANDAS_TESTING_MODE: "deprecate"
           EXTRA_APT: "xsel"

ci/azure/windows.yml

@@ -38,7 +38,7 @@ jobs:
       displayName: 'Build'
     - script: |
         call activate pandas-dev
-        pytest -m "not slow and not network and not db" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %*
+        pytest -m "not slow and not network" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %*
       displayName: 'Test'
     - task: PublishTestResults@2
       inputs:

doc/make.py

@@ -50,7 +50,7 @@ def __init__(self, num_jobs=0, include_api=True, single_doc=None,
         if single_doc and single_doc.endswith('.rst'):
             self.single_doc_html = os.path.splitext(single_doc)[0] + '.html'
         elif single_doc:
-            self.single_doc_html = 'generated/pandas.{}.html'.format(
+            self.single_doc_html = 'api/generated/pandas.{}.html'.format(
                 single_doc)
 
     def _process_single_doc(self, single_doc):

doc/source/conf.py

@@ -98,8 +98,7 @@
                 if (fname == 'index.rst'
                         and os.path.abspath(dirname) == source_path):
                     continue
-                elif (pattern == '-api'
-                        and (fname == 'api.rst' or dirname == 'generated')):
+                elif pattern == '-api' and dirname == 'api':
                     exclude_patterns.append(fname)
                 elif fname != pattern:
                     exclude_patterns.append(fname)

doc/source/index.rst.template

@@ -113,7 +113,7 @@ See the package overview for more detail about what's in the library.
     {{ single_doc[:-4] }}
 {% elif single_doc %}
 .. autosummary::
-    :toctree: generated/
+    :toctree: api/generated/
 
     {{ single_doc }}
 {% else -%}

doc/source/whatsnew/v0.24.0.rst

@@ -411,6 +411,7 @@ Other Enhancements
 - :meth:`DataFrame.to_stata` and :class:`pandas.io.stata.StataWriter117` can write mixed sting columns to Stata strl format (:issue:`23633`)
 - :meth:`DataFrame.between_time` and :meth:`DataFrame.at_time` have gained the ``axis`` parameter (:issue:`8839`)
 - The ``scatter_matrix``, ``andrews_curves``, ``parallel_coordinates``, ``lag_plot``, ``autocorrelation_plot``, ``bootstrap_plot``, and ``radviz`` plots from the ``pandas.plotting`` module are now accessible from calling :meth:`DataFrame.plot` (:issue:`11978`)
+- :meth:`DataFrame.to_records` now accepts ``index_dtypes`` and ``column_dtypes`` parameters to allow different data types in stored column and index records (:issue:`18146`)
 - :class:`IntervalIndex` has gained the :attr:`~IntervalIndex.is_overlapping` attribute to indicate if the ``IntervalIndex`` contains any overlapping intervals (:issue:`23309`)
 - :func:`pandas.DataFrame.to_sql` has gained the ``method`` argument to control SQL insertion clause. See the :ref:`insertion method <io.sql.method>` section in the documentation. (:issue:`8953`)
 
@@ -1494,6 +1495,7 @@ Indexing
 - Bug in :class:`Index` slicing with boolean :class:`Index` may raise ``TypeError`` (:issue:`22533`)
 - Bug in ``PeriodArray.__setitem__`` when accepting slice and list-like value (:issue:`23978`)
 - Bug in :class:`DatetimeIndex`, :class:`TimedeltaIndex` where indexing with ``Ellipsis`` would lose their ``freq`` attribute (:issue:`21282`)
+- Bug in ``iat`` where using it to assign an incompatible value would create a new column (:issue:`23236`)
 
 Missing
 ^^^^^^^

pandas/conftest.py

@@ -1,3 +1,4 @@
+import collections
 from datetime import date, time, timedelta
 from decimal import Decimal
 import importlib
@@ -54,14 +55,22 @@ def pytest_runtest_setup(item):
     if 'network' in item.keywords and item.config.getoption("--skip-network"):
         pytest.skip("skipping due to --skip-network")
 
-    if 'db' in item.keywords and item.config.getoption("--skip-db"):
-        pytest.skip("skipping due to --skip-db")
-
     if 'high_memory' in item.keywords and not item.config.getoption(
             "--run-high-memory"):
         pytest.skip(
             "skipping high memory test since --run-high-memory was not set")
 
+    # if "db" not explicitly set in the -m pattern, we skip the db tests
+    if 'db' in item.keywords:
+        pattern = item.config.getoption('-m')
+        markers = collections.defaultdict(bool)
+        for marker in item.iter_markers():
+            markers[marker.name] = True
+        markers['db'] = False
+        db_in_pattern = not eval(pattern, {}, markers)
+        if not db_in_pattern:
+            pytest.skip('skipping db unless -m "db" is specified')
+
 
 # Configurations for all tests and all test modules
 

pandas/core/arrays/datetimelike.py

@@ -28,13 +28,14 @@
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import nanops
-from pandas.core.algorithms import checked_add_with_arr, take, unique1d
+from pandas.core.algorithms import (
+    checked_add_with_arr, take, unique1d, value_counts)
 import pandas.core.common as com
 
 from pandas.tseries import frequencies
 from pandas.tseries.offsets import DateOffset, Tick
 
-from .base import ExtensionOpsMixin
+from .base import ExtensionArray, ExtensionOpsMixin
 
 
 def _make_comparison_op(cls, op):
@@ -343,7 +344,9 @@ def ceil(self, freq, ambiguous='raise', nonexistent='raise'):
         return self._round(freq, RoundTo.PLUS_INFTY, ambiguous, nonexistent)
 
 
-class DatetimeLikeArrayMixin(ExtensionOpsMixin, AttributesMixin):
+class DatetimeLikeArrayMixin(ExtensionOpsMixin,
+                             AttributesMixin,
+                             ExtensionArray):
     """
     Shared Base/Mixin class for DatetimeArray, TimedeltaArray, PeriodArray
 
@@ -701,6 +704,43 @@ def repeat(self, repeats, *args, **kwargs):
         values = self._data.repeat(repeats)
         return type(self)(values.view('i8'), dtype=self.dtype)
 
+    def value_counts(self, dropna=False):
+        """
+        Return a Series containing counts of unique values.
+
+        Parameters
+        ----------
+        dropna : boolean, default True
+            Don't include counts of NaT values.
+
+        Returns
+        -------
+        Series
+        """
+        from pandas import Series, Index
+
+        if dropna:
+            values = self[~self.isna()]._data
+        else:
+            values = self._data
+
+        cls = type(self)
+
+        result = value_counts(values, sort=False, dropna=dropna)
+        index = Index(cls(result.index.view('i8'), dtype=self.dtype),
+                      name=result.index.name)
+        return Series(result.values, index=index, name=result.name)
+
+    def map(self, mapper):
+        # TODO(GH-23179): Add ExtensionArray.map
+        # Need to figure out if we want ExtensionArray.map first.
+        # If so, then we can refactor IndexOpsMixin._map_values to
+        # a standalone function and call from here..
+        # Else, just rewrite _map_infer_values to do the right thing.
+        from pandas import Index
+
+        return Index(self).map(mapper).array
+
     # ------------------------------------------------------------------
     # Null Handling
 
@@ -1357,10 +1397,9 @@ def _reduce(self, name, axis=0, skipna=True, **kwargs):
         if op:
             return op(axis=axis, skipna=skipna, **kwargs)
         else:
-            raise TypeError("cannot perform {name} with type {dtype}"
-                            .format(name=name, dtype=self.dtype))
-            # TODO: use super(DatetimeLikeArrayMixin, self)._reduce
-            #  after we subclass ExtensionArray
+            return super(DatetimeLikeArrayMixin, self)._reduce(
+                name, skipna, **kwargs
+            )
 
     def min(self, axis=None, skipna=True, *args, **kwargs):
         """

pandas/core/arrays/period.py

@@ -22,7 +22,7 @@
 from pandas.core.dtypes.missing import isna, notna
 
 import pandas.core.algorithms as algos
-from pandas.core.arrays import ExtensionArray, datetimelike as dtl
+from pandas.core.arrays import datetimelike as dtl
 import pandas.core.common as com
 from pandas.core.missing import backfill_1d, pad_1d
 
@@ -92,9 +92,7 @@ def wrapper(self, other):
     return compat.set_function_name(wrapper, opname, cls)
 
 
-class PeriodArray(dtl.DatetimeLikeArrayMixin,
-                  dtl.DatelikeOps,
-                  ExtensionArray):
+class PeriodArray(dtl.DatetimeLikeArrayMixin, dtl.DatelikeOps):
     """
     Pandas ExtensionArray for storing Period data.
 
@@ -418,21 +416,6 @@ def fillna(self, value=None, method=None, limit=None):
             new_values = self.copy()
         return new_values
 
-    def value_counts(self, dropna=False):
-        from pandas import Series, PeriodIndex
-
-        if dropna:
-            values = self[~self.isna()]._data
-        else:
-            values = self._data
-
-        cls = type(self)
-
-        result = algos.value_counts(values, sort=False)
-        index = PeriodIndex(cls(result.index, freq=self.freq),
-                            name=result.index.name)
-        return Series(result.values, index=index, name=result.name)
-
     # --------------------------------------------------------------------
 
     def _time_shift(self, n, freq=None):

pandas/core/arrays/timedeltas.py

@@ -16,7 +16,7 @@
 
 from pandas.core.dtypes.common import (
     _NS_DTYPE, _TD_DTYPE, ensure_int64, is_datetime64_dtype, is_float_dtype,
-    is_integer_dtype, is_list_like, is_object_dtype, is_scalar,
+    is_int64_dtype, is_integer_dtype, is_list_like, is_object_dtype, is_scalar,
     is_string_dtype, is_timedelta64_dtype, is_timedelta64_ns_dtype,
     pandas_dtype)
 from pandas.core.dtypes.dtypes import DatetimeTZDtype
@@ -244,6 +244,16 @@ def _maybe_clear_freq(self):
     # ----------------------------------------------------------------
     # Array-Like / EA-Interface Methods
 
+    def __array__(self, dtype=None):
+        # TODO(https://github.com/pandas-dev/pandas/pull/23593)
+        # Maybe push to parent once datetimetz __array__ is figured out.
+        if is_object_dtype(dtype):
+            return np.array(list(self), dtype=object)
+        elif is_int64_dtype(dtype):
+            return self.asi8
+
+        return self._data
+
     @Appender(dtl.DatetimeLikeArrayMixin._validate_fill_value.__doc__)
     def _validate_fill_value(self, fill_value):
         if isna(fill_value):

pandas/core/dtypes/inference.py

@@ -398,8 +398,11 @@ def is_dict_like(obj):
     >>> is_dict_like([1, 2, 3])
     False
     """
+    for attr in ("__getitem__", "keys", "__contains__"):
+        if not hasattr(obj, attr):
+            return False
 
-    return hasattr(obj, '__getitem__') and hasattr(obj, 'keys')
+    return True
 
 
 def is_named_tuple(obj):