Merge pull request #4657 from jreback/bool

API: GH4633, bool(obj) behavior, raise on __nonzero__ always

Author: jreback

doc/source/10min.rst

@@ -269,7 +269,6 @@ A ``where`` operation for getting.
 
    df[df > 0]
 
-
 Setting
 ~~~~~~~
 
@@ -708,3 +707,20 @@ Reading from an excel file
    :suppress:
 
    os.remove('foo.xlsx')
+
+Gotchas
+-------
+
+If you are trying an operation and you see an exception like:
+
+.. code-block:: python
+
+    >>> if pd.Series([False, True, False]):
+        print("I was true")
+    Traceback
+        ...
+    ValueError: The truth value of an array is ambiguous. Use a.empty, a.any() or a.all().
+
+See :ref:`Comparisons<basics.compare>` for an explanation and what to do.
+
+See :ref:`Gotachas<gotchas>` as well.

doc/source/basics.rst

@@ -8,7 +8,7 @@
    from pandas import *
    randn = np.random.randn
    np.set_printoptions(precision=4, suppress=True)
-   from pandas.compat import lrange 
+   from pandas.compat import lrange
 
 ==============================
  Essential Basic Functionality
@@ -198,16 +198,62 @@ replace NaN with some other value using ``fillna`` if you wish).
 
 Flexible Comparisons
 ~~~~~~~~~~~~~~~~~~~~
+
+.. _basics.compare:
+
 Starting in v0.8, pandas introduced binary comparison methods eq, ne, lt, gt,
 le, and ge to Series and DataFrame whose behavior is analogous to the binary
 arithmetic operations described above:
 
 .. ipython:: python
 
    df.gt(df2)
-
    df2.ne(df)
 
+These operations produce a pandas object the same type as the left-hand-side input
+that if of dtype ``bool``. These ``boolean`` objects can be used in indexing operations,
+see :ref:`here<indexing.boolean>`
+
+Furthermore, you can apply the reduction functions: ``any()`` and ``all()`` to provide a
+way to summarize these results.
+
+.. ipython:: python
+
+   (df>0).all()
+   (df>0).any()
+
+Finally you can test if a pandas object is empty, via the ``empty`` property.
+
+.. ipython:: python
+
+   df.empty
+   DataFrame(columns=list('ABC')).empty
+
+.. warning::
+
+   You might be tempted to do the following:
+
+   .. code-block:: python
+
+       >>>if df:
+            ...
+
+   Or
+
+   .. code-block:: python
+
+       >>> df and df2
+
+   These both will raise as you are trying to compare multiple values.
+
+   .. code-block:: python
+
+       ValueError: The truth value of an array is ambiguous. Use a.empty, a.any() or a.all().
+
+
+See :ref:`gotchas<gotchas.truth>` for a more detailed discussion.
+
+
 Combining overlapping data sets
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/source/gotchas.rst

@@ -15,6 +15,58 @@
 Caveats and Gotchas
 *******************
 
+Using If/Truth Statements with Pandas
+-------------------------------------
+
+.. _gotchas.truth:
+
+Pandas follows the numpy convention of raising an error when you try to convert something to a ``bool``.
+This happens in a ``if`` or when using the boolean operations, ``and``, ``or``, or ``not``.  It is not clear
+what the result of
+
+.. code-block:: python
+
+    >>> if Series([False, True, False]):
+         ...
+
+should be. Should it be ``True`` because it's not zero-length? ``False`` because there are ``False`` values?
+It is unclear, so instead, pandas raises a ``ValueError``:
+
+.. code-block:: python
+
+    >>> if pd.Series([False, True, False]):
+        print("I was true")
+    Traceback
+        ...
+    ValueError: The truth value of an array is ambiguous. Use a.empty, a.any() or a.all().
+
+
+If you see that, you need to explicitly choose what you want to do with it (e.g., use `any()`, `all()` or `empty`).
+or, you might want to compare if the pandas object is ``None``
+
+.. code-block:: python
+
+    >>> if pd.Series([False, True, False]) is not None:
+           print("I was not None")
+    >>> I was not None
+
+Bitwise boolean
+~~~~~~~~~~~~~~~
+
+Bitwise boolean operators like ``==`` and ``!=`` will return a boolean ``Series``,
+which is almost always what you want anyways.
+
+.. code-block:: python
+
+   >>> s = pd.Series(range(5))
+   >>> s == 4
+   0    False
+   1    False
+   2    False
+   3    False
+   4     True
+   dtype: bool
+
 ``NaN``, Integer ``NA`` values and ``NA`` type promotions
 ---------------------------------------------------------
 
@@ -428,7 +480,7 @@ parse HTML tables in the top-level pandas io function ``read_html``.
       lxml will work correctly:
 
       .. code-block:: sh
-         
+
          # remove the included version
          conda remove lxml
 

doc/source/release.rst

@@ -134,6 +134,9 @@ pandas 0.13
       now returns a ``MultiIndex`` rather than an ``Index``. (:issue:`4039`)
 
   - Infer and downcast dtype if ``downcast='infer'`` is passed to ``fillna/ffill/bfill`` (:issue:`4604`)
+  - Factored out excel_value_to_python_value from ExcelFile::_parse_excel (:issue:`4589`)
+  - ``__nonzero__`` for all NDFrame objects, will now raise a ``ValueError``, this reverts back to (:issue:`1073`, :issue:`4633`)
+    behavior.
 
 **Internal Refactoring**
 

doc/source/v0.13.0.txt

@@ -121,6 +121,18 @@ API changes
         index.set_names(["bob", "cranberry"], inplace=True)
 
   - Infer and downcast dtype if ``downcast='infer'`` is passed to ``fillna/ffill/bfill`` (:issue:`4604`)
+  - ``__nonzero__`` for all NDFrame objects, will now raise a ``ValueError``, this reverts back to (:issue:`1073`, :issue:`4633`)
+    behavior.
+
+    This prevent behaviors like (which will now all raise ``ValueError``)
+
+    ..code-block ::
+
+        if df:
+           ....
+
+        df1 and df2
+        s1 and s2
 
 Enhancements
 ~~~~~~~~~~~~

pandas/core/generic.py

@@ -531,7 +531,8 @@ def empty(self):
         return not all(len(self._get_axis(a)) > 0 for a in self._AXIS_ORDERS)
 
     def __nonzero__(self):
-        return not self.empty
+        raise ValueError("The truth value of an array is ambiguous. Use a.empty, a.any() or a.all().")
+
     __bool__ = __nonzero__
 
     #----------------------------------------------------------------------

pandas/core/groupby.py

@@ -2101,9 +2101,22 @@ def filter(self, func, dropna=True, *args, **kwargs):
             else:
                 res = path(group)
 
-            if res:
+            def add_indexer():
                 indexers.append(self.obj.index.get_indexer(group.index))
 
+            # interpret the result of the filter
+            if isinstance(res,(bool,np.bool_)):
+                if res:
+                    add_indexer()
+            else:
+                if getattr(res,'ndim',None) == 1:
+                    if res.ravel()[0]:
+                        add_indexer()
+                else:
+
+                    # in theory you could do .all() on the boolean result ?
+                    raise TypeError("the filter must return a boolean result")
+
         if len(indexers) == 0:
             filtered = self.obj.take([]) # because np.concatenate would fail
         else:

pandas/core/series.py

@@ -798,13 +798,6 @@ def __contains__(self, key):
     __long__ = _coerce_method(int)
     __int__ = _coerce_method(int)
 
-    def __nonzero__(self):
-        # special case of a single element bool series degenerating to a scalar
-        if self.dtype == np.bool_ and len(self) == 1:
-            return bool(self.iloc[0])
-        return not self.empty
-    __bool__ = __nonzero__
-
     # we are preserving name here
     def __getstate__(self):
         return dict(_data=self._data, name=self.name)

pandas/io/tests/test_pytables.py

@@ -1593,19 +1593,19 @@ def test_table_values_dtypes_roundtrip(self):
         with ensure_clean(self.path) as store:
             df1 = DataFrame({'a': [1, 2, 3]}, dtype='f8')
             store.append('df_f8', df1)
-            assert df1.dtypes == store['df_f8'].dtypes
+            assert_series_equal(df1.dtypes,store['df_f8'].dtypes)
 
             df2 = DataFrame({'a': [1, 2, 3]}, dtype='i8')
             store.append('df_i8', df2)
-            assert df2.dtypes == store['df_i8'].dtypes
+            assert_series_equal(df2.dtypes,store['df_i8'].dtypes)
 
             # incompatible dtype
             self.assertRaises(ValueError, store.append, 'df_i8', df1)
 
             # check creation/storage/retrieval of float32 (a bit hacky to actually create them thought)
             df1 = DataFrame(np.array([[1],[2],[3]],dtype='f4'),columns = ['A'])
             store.append('df_f4', df1)
-            assert df1.dtypes == store['df_f4'].dtypes
+            assert_series_equal(df1.dtypes,store['df_f4'].dtypes)
             assert df1.dtypes[0] == 'float32'
 
             # check with mixed dtypes

pandas/tests/test_frame.py

@@ -10607,13 +10607,10 @@ def test_index_namedtuple(self):
         df = DataFrame([(1, 2), (3, 4)], index=index, columns=["A", "B"])
         self.assertEqual(df.ix[IndexType("foo", "bar")]["A"], 1)
 
-    def test_bool_empty_nonzero(self):
+    def test_empty_nonzero(self):
         df = DataFrame([1, 2, 3])
-        self.assertTrue(bool(df))
         self.assertFalse(df.empty)
         df = DataFrame(index=['a', 'b'], columns=['c', 'd']).dropna()
-        self.assertFalse(bool(df))
-        self.assertFalse(bool(df.T))
         self.assertTrue(df.empty)
         self.assertTrue(df.T.empty)