Merge remote-tracking branch 'upstream/master' into multiindex_at_fix

# Conflicts:
#	doc/source/whatsnew/v1.1.0.rst

Author: rhshadrach

doc/source/development/internals.rst

@@ -89,16 +89,10 @@ pandas extends NumPy's type system with custom types, like ``Categorical`` or
 datetimes with a timezone, so we have multiple notions of "values". For 1-D
 containers (``Index`` classes and ``Series``) we have the following convention:
 
-* ``cls._ndarray_values`` is *always* a NumPy ``ndarray``. Ideally,
-  ``_ndarray_values`` is cheap to compute. For example, for a ``Categorical``,
-  this returns the codes, not the array of objects.
 * ``cls._values`` refers is the "best possible" array. This could be an
-  ``ndarray``, ``ExtensionArray``, or in ``Index`` subclass (note: we're in the
-  process of removing the index subclasses here so that it's always an
-  ``ndarray`` or ``ExtensionArray``).
+  ``ndarray`` or ``ExtensionArray``.
 
-So, for example, ``Series[category]._values`` is a ``Categorical``, while
-``Series[category]._ndarray_values`` is the underlying codes.
+So, for example, ``Series[category]._values`` is a ``Categorical``.
 
 .. _ref-subclassing-pandas:
 

doc/source/reference/extensions.rst

@@ -37,7 +37,6 @@ objects.
       api.extensions.ExtensionArray._from_factorized
       api.extensions.ExtensionArray._from_sequence
       api.extensions.ExtensionArray._from_sequence_of_strings
-      api.extensions.ExtensionArray._ndarray_values
       api.extensions.ExtensionArray._reduce
       api.extensions.ExtensionArray._values_for_argsort
       api.extensions.ExtensionArray._values_for_factorize

doc/source/whatsnew/v1.1.0.rst

@@ -303,6 +303,7 @@ Indexing
 - Bug in :meth:`Series.loc` and :meth:`DataFrame.loc` when indexing with an integer key on a object-dtype :class:`Index` that is not all-integers (:issue:`31905`)
 - Bug in :meth:`DataFrame.iloc.__setitem__` on a :class:`DataFrame` with duplicate columns incorrectly setting values for all matching columns (:issue:`15686`, :issue:`22036`)
 - Bug in :meth:`DataFrame.loc:` and :meth:`Series.loc` with a :class:`DatetimeIndex`, :class:`TimedeltaIndex`, or :class:`PeriodIndex` incorrectly allowing lookups of non-matching datetime-like dtypes (:issue:`32650`)
+- Bug in :meth:`Series.__getitem__` indexing with non-standard scalars, e.g. ``np.dtype`` (:issue:`32684`)
 - Bug in :meth:`Series.at` when used with a :class:`MultiIndex` would raise an exception on valid inputs (:issue:`26989`)
 -
 
@@ -344,10 +345,12 @@ I/O
   timestamps with ``version="2.0"`` (:issue:`31652`).
 - Bug in :meth:`read_csv` was raising `TypeError` when `sep=None` was used in combination with `comment` keyword (:issue:`31396`)
 - Bug in :class:`HDFStore` that caused it to set to ``int64`` the dtype of a ``datetime64`` column when reading a DataFrame in Python 3 from fixed format written in Python 2 (:issue:`31750`)
+- Bug in :meth:`DataFrame.to_json` where ``Timedelta`` objects would not be serialized correctly with ``date_format="iso"`` (:issue:`28256`)
 - :func:`read_csv` will raise a ``ValueError`` when the column names passed in `parse_dates` are missing in the Dataframe (:issue:`31251`)
 - Bug in :meth:`read_excel` where a UTF-8 string with a high surrogate would cause a segmentation violation (:issue:`23809`)
 - Bug in :meth:`read_csv` was causing a file descriptor leak on an empty file (:issue:`31488`)
 - Bug in :meth:`read_csv` was causing a segfault when there were blank lines between the header and data rows (:issue:`28071`)
+- Bug in :meth:`read_csv` was raising a misleading exception on a permissions issue (:issue:`23784`)
 
 
 Plotting
@@ -404,6 +407,7 @@ Other
 - Fixed :func:`pandas.testing.assert_series_equal` to correctly raise if left object is a different subclass with ``check_series_type=True`` (:issue:`32670`).
 - :meth:`IntegerArray.astype` now supports ``datetime64`` dtype (:issue:32538`)
 - Fixed bug in :func:`pandas.testing.assert_series_equal` where dtypes were checked for ``Interval`` and ``ExtensionArray`` operands when ``check_dtype`` was ``False`` (:issue:`32747`)
+- Bug in :meth:`DataFrame.__dir__` caused a segfault when using unicode surrogates in a column name (:issue:`25509`)
 
 .. ---------------------------------------------------------------------------
 

pandas/_libs/groupby.pyx

@@ -869,7 +869,9 @@ def group_last(rank_t[:, :] out,
 
     assert min_count == -1, "'min_count' only used in add and prod"
 
-    if not len(values) == len(labels):
+    # TODO(cython 3.0):
+    # Instead of `labels.shape[0]` use `len(labels)`
+    if not len(values) == labels.shape[0]:
         raise AssertionError("len(index) != len(labels)")
 
     nobs = np.zeros((<object>out).shape, dtype=np.int64)
@@ -960,7 +962,9 @@ def group_nth(rank_t[:, :] out,
 
     assert min_count == -1, "'min_count' only used in add and prod"
 
-    if not len(values) == len(labels):
+    # TODO(cython 3.0):
+    # Instead of `labels.shape[0]` use `len(labels)`
+    if not len(values) == labels.shape[0]:
         raise AssertionError("len(index) != len(labels)")
 
     nobs = np.zeros((<object>out).shape, dtype=np.int64)
@@ -1254,7 +1258,9 @@ def group_max(groupby_t[:, :] out,
 
     assert min_count == -1, "'min_count' only used in add and prod"
 
-    if not len(values) == len(labels):
+    # TODO(cython 3.0):
+    # Instead of `labels.shape[0]` use `len(labels)`
+    if not len(values) == labels.shape[0]:
         raise AssertionError("len(index) != len(labels)")
 
     nobs = np.zeros((<object>out).shape, dtype=np.int64)
@@ -1327,7 +1333,9 @@ def group_min(groupby_t[:, :] out,
 
     assert min_count == -1, "'min_count' only used in add and prod"
 
-    if not len(values) == len(labels):
+    # TODO(cython 3.0):
+    # Instead of `labels.shape[0]` use `len(labels)`
+    if not len(values) == labels.shape[0]:
         raise AssertionError("len(index) != len(labels)")
 
     nobs = np.zeros((<object>out).shape, dtype=np.int64)

pandas/_libs/hashtable_class_helper.pxi.in

@@ -12,6 +12,9 @@ WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
 from pandas._libs.tslibs.util cimport get_c_string
 from pandas._libs.missing cimport C_NA
 
+cdef extern from "Python.h":
+    void PyErr_Clear()
+
 {{py:
 
 # name, dtype, c_type
@@ -790,6 +793,9 @@ cdef class StringHashTable(HashTable):
             else:
                 # if ignore_na is False, we also stringify NaN/None/etc.
                 v = get_c_string(<str>val)
+                if v == NULL:
+                    PyErr_Clear()
+                    v = get_c_string(<str>repr(val))
                 vecs[i] = v
 
         # compute

pandas/_libs/internals.pyx

@@ -308,7 +308,10 @@ cdef slice_getitem(slice slc, ind):
             return slice(s_start, s_stop, s_step)
 
     else:
-        return np.arange(s_start, s_stop, s_step, dtype=np.int64)[ind]
+        # NOTE:
+        # this is the C-optimized equivalent of
+        # `np.arange(s_start, s_stop, s_step, dtype=np.int64)[ind]`
+        return cnp.PyArray_Arange(s_start, s_stop, s_step, NPY_INT64)[ind]
 
 
 @cython.boundscheck(False)

pandas/_libs/parsers.pyx

@@ -241,9 +241,9 @@ cdef extern from "parser/io.h":
     void* buffer_mmap_bytes(void *source, size_t nbytes,
                             size_t *bytes_read, int *status)
 
-    void *new_file_source(char *fname, size_t buffer_size)
+    void *new_file_source(char *fname, size_t buffer_size) except NULL
 
-    void *new_rd_source(object obj)
+    void *new_rd_source(object obj) except NULL
 
     int del_file_source(void *src)
     int del_rd_source(void *src)
@@ -667,26 +667,12 @@ cdef class TextReader:
                 ptr = new_file_source(source, self.parser.chunksize)
                 self.parser.cb_io = &buffer_file_bytes
                 self.parser.cb_cleanup = &del_file_source
-
-            if ptr == NULL:
-                if not os.path.exists(source):
-
-                    raise FileNotFoundError(
-                        ENOENT,
-                        f'File {usource} does not exist',
-                        usource)
-                raise IOError('Initializing from file failed')
-
             self.parser.source = ptr
 
         elif hasattr(source, 'read'):
             # e.g., StringIO
 
             ptr = new_rd_source(source)
-            if ptr == NULL:
-                raise IOError('Initializing parser from file-like '
-                              'object failed')
-
             self.parser.source = ptr
             self.parser.cb_io = &buffer_rd_bytes
             self.parser.cb_cleanup = &del_rd_source

pandas/_libs/src/parser/io.c

@@ -28,6 +28,7 @@ The full license is in the LICENSE file, distributed with this software.
 void *new_file_source(char *fname, size_t buffer_size) {
     file_source *fs = (file_source *)malloc(sizeof(file_source));
     if (fs == NULL) {
+        PyErr_NoMemory();
         return NULL;
     }
 
@@ -41,17 +42,20 @@ void *new_file_source(char *fname, size_t buffer_size) {
         int required = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);
         if (required == 0) {
             free(fs);
+            PyErr_SetFromWindowsErr(0);
             return NULL;
         }
         wname = (wchar_t*)malloc(required * sizeof(wchar_t));
         if (wname == NULL) {
             free(fs);
+            PyErr_NoMemory();
             return NULL;
         }
         if (MultiByteToWideChar(CP_UTF8, 0, fname, -1, wname, required) <
                                                                 required) {
             free(wname);
             free(fs);
+            PyErr_SetFromWindowsErr(0);
             return NULL;
         }
         fs->fd = _wopen(wname, O_RDONLY | O_BINARY);
@@ -62,6 +66,7 @@ void *new_file_source(char *fname, size_t buffer_size) {
 #endif
     if (fs->fd == -1) {
         free(fs);
+        PyErr_SetFromErrnoWithFilename(PyExc_OSError, fname);
         return NULL;
     }
 
@@ -71,6 +76,7 @@ void *new_file_source(char *fname, size_t buffer_size) {
     if (fs->buffer == NULL) {
         close(fs->fd);
         free(fs);
+        PyErr_NoMemory();
         return NULL;
     }
 
@@ -83,6 +89,10 @@ void *new_file_source(char *fname, size_t buffer_size) {
 void *new_rd_source(PyObject *obj) {
     rd_source *rds = (rd_source *)malloc(sizeof(rd_source));
 
+    if (rds == NULL) {
+        PyErr_NoMemory();
+        return NULL;
+    }
     /* hold on to this object */
     Py_INCREF(obj);
     rds->obj = obj;
@@ -220,20 +230,15 @@ void *new_mmap(char *fname) {
 
     mm = (memory_map *)malloc(sizeof(memory_map));
     if (mm == NULL) {
-        fprintf(stderr, "new_file_buffer: malloc() failed.\n");
-        return (NULL);
+        return NULL;
     }
     mm->fd = open(fname, O_RDONLY | O_BINARY);
     if (mm->fd == -1) {
-        fprintf(stderr, "new_file_buffer: open(%s) failed. errno =%d\n",
-          fname, errno);
         free(mm);
         return NULL;
     }
 
     if (fstat(mm->fd, &stat) == -1) {
-        fprintf(stderr, "new_file_buffer: fstat() failed. errno =%d\n",
-          errno);
         close(mm->fd);
         free(mm);
         return NULL;
@@ -242,8 +247,6 @@ void *new_mmap(char *fname) {
 
     mm->memmap = mmap(NULL, filesize, PROT_READ, MAP_SHARED, mm->fd, 0);
     if (mm->memmap == MAP_FAILED) {
-        /* XXX Eventually remove this print statement. */
-        fprintf(stderr, "new_file_buffer: mmap() failed.\n");
         close(mm->fd);
         free(mm);
         return NULL;

pandas/_libs/src/ujson/python/date_conversions.c

@@ -116,3 +116,29 @@ npy_datetime PyDateTimeToEpoch(PyDateTime_Date *dt, NPY_DATETIMEUNIT base) {
     npy_datetime npy_dt = npy_datetimestruct_to_datetime(NPY_FR_ns, &dts);
     return NpyDateTimeToEpoch(npy_dt, base);
 }
+
+/* Converts the int64_t representation of a duration to ISO; mutates len */
+char *int64ToIsoDuration(int64_t value, size_t *len) {
+    pandas_timedeltastruct tds;
+    int ret_code;
+
+    pandas_timedelta_to_timedeltastruct(value, NPY_FR_ns, &tds);
+
+    // Max theoretical length of ISO Duration with 64 bit day
+    // as the largest unit is 70 characters + 1 for a null terminator
+    char *result = PyObject_Malloc(71);
+    if (result == NULL) {
+        PyErr_NoMemory();
+        return NULL;
+    }
+
+    ret_code = make_iso_8601_timedelta(&tds, result, len);
+    if (ret_code == -1) {
+        PyErr_SetString(PyExc_ValueError,
+                        "Could not convert timedelta value to string");
+        PyObject_Free(result);
+        return NULL;
+    }
+
+    return result;
+}

pandas/_libs/src/ujson/python/date_conversions.h

@@ -28,4 +28,6 @@ char *PyDateTimeToIso(PyDateTime_Date *obj, NPY_DATETIMEUNIT base, size_t *len);
 // Convert a Python Date/Datetime to Unix epoch with resolution base
 npy_datetime PyDateTimeToEpoch(PyDateTime_Date *dt, NPY_DATETIMEUNIT base);
 
+char *int64ToIsoDuration(int64_t value, size_t *len);
+
 #endif