Expand tests

Author: kotamatsuoka

pandas/core/algorithms.py

@@ -727,6 +727,8 @@ def value_counts(
         if not isinstance(counts, np.ndarray):
             counts = counts.to_numpy()
         result = result / float(counts.sum())
+    else:
+        result = result.astype("Int64")
 
     return result
 

pandas/core/arrays/masked.py

@@ -259,7 +259,7 @@ def value_counts(self, dropna: bool = True) -> "Series":
 
         # if we want nans, count the mask
         if dropna:
-            counts = value_counts._values
+            counts = np.array(value_counts._values, dtype=int)
         else:
             counts = np.empty(len(value_counts) + 1, dtype="int64")
             counts[:-1] = value_counts

pandas/core/groupby/generic.py

@@ -731,7 +731,12 @@ def value_counts(
             )
 
             if is_integer_dtype(out):
-                out = ensure_int64(out)
+                return Series(
+                    ensure_int64(out),
+                    index=mi,
+                    name=self._selection_name,
+                    dtype="Int64",
+                )
             return Series(out, index=mi, name=self._selection_name)
 
         # for compat. with libgroupby.value_counts need to ensure every
@@ -763,7 +768,10 @@ def build_codes(lev_codes: np.ndarray) -> np.ndarray:
         mi = MultiIndex(levels=levels, codes=codes, names=names, verify_integrity=False)
 
         if is_integer_dtype(out):
-            out = ensure_int64(out)
+            return Series(
+                ensure_int64(out), index=mi, name=self._selection_name, dtype="Int64"
+            )
+
         return Series(out, index=mi, name=self._selection_name)
 
     def count(self) -> Series:

pandas/tests/arrays/integer/test_function.py

@@ -93,7 +93,7 @@ def test_stat_method(pandasmethname, kwargs):
 
 
 def test_value_counts_na():
-    arr = pd.array([1, 2, 1, pd.NA], dtype="Int64")
+    arr = pd.array([1, 2, 1, pd.NA])
     result = arr.value_counts(dropna=False)
     expected = pd.Series([2, 1, 1], index=[1, 2, pd.NA], dtype="Int64")
     tm.assert_series_equal(result, expected)
@@ -105,10 +105,10 @@ def test_value_counts_na():
 
 def test_value_counts_empty():
     # https://github.com/pandas-dev/pandas/issues/33317
-    s = pd.Series([], dtype="Int64")
+    s = pd.Series([])
     result = s.value_counts()
     # TODO: The dtype of the index seems wrong (it's int64 for non-empty)
-    idx = pd.Index([], dtype="object")
+    idx = pd.Float64Index([], dtype="float64")
     expected = pd.Series([], index=idx, dtype="Int64")
     tm.assert_series_equal(result, expected)
 

pandas/tests/arrays/string_/test_string.py

@@ -277,13 +277,3 @@ def test_value_counts_na():
     result = arr.value_counts(dropna=True)
     expected = pd.Series([2, 1], index=["a", "b"], dtype="Int64")
     tm.assert_series_equal(result, expected)
-
-
-def test_normalize_value_counts():
-    result = (
-        pd.Series(list("abcd"), dtype="string")
-        .value_counts(normalize=True)
-        .sort_index()
-    )
-    expected = pd.Series([0.25, 0.25, 0.25, 0.25], index=["a", "b", "c", "d"])
-    tm.assert_series_equal(expected, result)

pandas/tests/arrays/test_datetimes.py

@@ -248,7 +248,7 @@ def test_value_counts_preserves_tz(self):
 
         arr[-2] = pd.NaT
         result = arr.value_counts()
-        expected = pd.Series([1, 4, 2], index=[pd.NaT, dti[0], dti[1]])
+        expected = pd.Series([1, 4, 2], index=[pd.NaT, dti[0], dti[1]], dtype="Int64")
         tm.assert_series_equal(result, expected)
 
     @pytest.mark.parametrize("method", ["pad", "backfill"])

pandas/tests/base/test_value_counts.py

@@ -30,7 +30,7 @@ def test_value_counts(index_or_series_obj):
     result = obj.value_counts()
 
     counter = collections.Counter(obj)
-    expected = pd.Series(dict(counter.most_common()), dtype=np.int64, name=obj.name)
+    expected = pd.Series(dict(counter.most_common()), dtype="Int64", name=obj.name)
     expected.index = expected.index.astype(obj.dtype)
     if isinstance(obj, pd.MultiIndex):
         expected.index = pd.Index(expected.index)
@@ -67,7 +67,7 @@ def test_value_counts_null(null_obj, index_or_series_obj):
     # because np.nan == np.nan is False, but None == None is True
     # np.nan would be duplicated, whereas None wouldn't
     counter = collections.Counter(obj.dropna())
-    expected = pd.Series(dict(counter.most_common()), dtype=np.int64)
+    expected = pd.Series(dict(counter.most_common()), dtype="Int64")
     expected.index = expected.index.astype(obj.dtype)
 
     result = obj.value_counts()
@@ -80,7 +80,7 @@ def test_value_counts_null(null_obj, index_or_series_obj):
 
     # can't use expected[null_obj] = 3 as
     # IntervalIndex doesn't allow assignment
-    new_entry = pd.Series({np.nan: 3}, dtype=np.int64)
+    new_entry = pd.Series({np.nan: 3}, dtype="Int64")
     expected = expected.append(new_entry)
 
     result = obj.value_counts(dropna=False)
@@ -96,7 +96,7 @@ def test_value_counts_inferred(index_or_series):
     klass = index_or_series
     s_values = ["a", "b", "b", "b", "b", "c", "d", "d", "a", "a"]
     s = klass(s_values)
-    expected = Series([4, 3, 2, 1], index=["b", "a", "d", "c"])
+    expected = Series([4, 3, 2, 1], index=["b", "a", "d", "c"], dtype="Int64")
     tm.assert_series_equal(s.value_counts(), expected)
 
     if isinstance(s, Index):
@@ -110,17 +110,17 @@ def test_value_counts_inferred(index_or_series):
     # don't sort, have to sort after the fact as not sorting is
     # platform-dep
     hist = s.value_counts(sort=False).sort_values()
-    expected = Series([3, 1, 4, 2], index=list("acbd")).sort_values()
+    expected = Series([3, 1, 4, 2], index=list("acbd"), dtype="Int64").sort_values()
     tm.assert_series_equal(hist, expected)
 
     # sort ascending
     hist = s.value_counts(ascending=True)
-    expected = Series([1, 2, 3, 4], index=list("cdab"))
+    expected = Series([1, 2, 3, 4], index=list("cdab"), dtype="Int64")
     tm.assert_series_equal(hist, expected)
 
     # relative histogram.
     hist = s.value_counts(normalize=True)
-    expected = Series([0.4, 0.3, 0.2, 0.1], index=["b", "a", "d", "c"])
+    expected = Series([0.4, 0.3, 0.2, 0.1], index=["b", "a", "d", "c"], dtype="float64")
     tm.assert_series_equal(hist, expected)
 
 
@@ -136,39 +136,41 @@ def test_value_counts_bins(index_or_series):
 
     s1 = Series([1, 1, 2, 3])
     res1 = s1.value_counts(bins=1)
-    exp1 = Series({Interval(0.997, 3.0): 4})
+    exp1 = Series({Interval(0.997, 3.0): 4}, dtype="Int64")
     tm.assert_series_equal(res1, exp1)
     res1n = s1.value_counts(bins=1, normalize=True)
-    exp1n = Series({Interval(0.997, 3.0): 1.0})
+    exp1n = Series({Interval(0.997, 3.0): 1.0}, dtype="float64")
     tm.assert_series_equal(res1n, exp1n)
 
     if isinstance(s1, Index):
         tm.assert_index_equal(s1.unique(), Index([1, 2, 3]))
     else:
-        exp = np.array([1, 2, 3], dtype=np.int64)
+        exp = np.array([1, 2, 3], dtype="Int64")
         tm.assert_numpy_array_equal(s1.unique(), exp)
 
     assert s1.nunique() == 3
 
     # these return the same
     res4 = s1.value_counts(bins=4, dropna=True)
     intervals = IntervalIndex.from_breaks([0.997, 1.5, 2.0, 2.5, 3.0])
-    exp4 = Series([2, 1, 1, 0], index=intervals.take([0, 3, 1, 2]))
+    exp4 = Series([2, 1, 1, 0], index=intervals.take([0, 3, 1, 2]), dtype="Int64")
     tm.assert_series_equal(res4, exp4)
 
     res4 = s1.value_counts(bins=4, dropna=False)
     intervals = IntervalIndex.from_breaks([0.997, 1.5, 2.0, 2.5, 3.0])
-    exp4 = Series([2, 1, 1, 0], index=intervals.take([0, 3, 1, 2]))
+    exp4 = Series([2, 1, 1, 0], index=intervals.take([0, 3, 1, 2]), dtype="Int64")
     tm.assert_series_equal(res4, exp4)
 
     res4n = s1.value_counts(bins=4, normalize=True)
-    exp4n = Series([0.5, 0.25, 0.25, 0], index=intervals.take([0, 3, 1, 2]))
+    exp4n = Series(
+        [0.5, 0.25, 0.25, 0], index=intervals.take([0, 3, 1, 2]), dtype="float64"
+    )
     tm.assert_series_equal(res4n, exp4n)
 
     # handle NA's properly
     s_values = ["a", "b", "b", "b", np.nan, np.nan, "d", "d", "a", "a", "b"]
     s = klass(s_values)
-    expected = Series([4, 3, 2], index=["b", "a", "d"])
+    expected = Series([4, 3, 2], index=["b", "a", "d"], dtype="Int64")
     tm.assert_series_equal(s.value_counts(), expected)
 
     if isinstance(s, Index):
@@ -180,7 +182,7 @@ def test_value_counts_bins(index_or_series):
     assert s.nunique() == 3
 
     s = klass({}) if klass is dict else klass({}, dtype=object)
-    expected = Series([], dtype=np.int64)
+    expected = Series([], dtype="Int64")
     tm.assert_series_equal(s.value_counts(), expected, check_index_type=False)
     # returned dtype differs depending on original
     if isinstance(s, Index):
@@ -216,7 +218,7 @@ def test_value_counts_datetime64(index_or_series):
     idx = pd.to_datetime(
         ["2010-01-01 00:00:00", "2008-09-09 00:00:00", "2009-01-01 00:00:00"]
     )
-    expected_s = Series([3, 2, 1], index=idx)
+    expected_s = Series([3, 2, 1], index=idx, dtype="Int64")
     tm.assert_series_equal(s.value_counts(), expected_s)
 
     expected = np_array_datetime64_compat(
@@ -240,7 +242,7 @@ def test_value_counts_datetime64(index_or_series):
 
     result = s.value_counts(dropna=False)
     expected_s[pd.NaT] = 1
-    tm.assert_series_equal(result, expected_s)
+    tm.assert_series_equal(result, expected_s.astype("Int64"))
 
     unique = s.unique()
     assert unique.dtype == "datetime64[ns]"
@@ -261,7 +263,7 @@ def test_value_counts_datetime64(index_or_series):
     td = klass(td, name="dt")
 
     result = td.value_counts()
-    expected_s = Series([6], index=[Timedelta("1day")], name="dt")
+    expected_s = Series([6], index=[Timedelta("1day")], name="dt", dtype="Int64")
     tm.assert_series_equal(result, expected_s)
 
     expected = TimedeltaIndex(["1 days"], name="dt")

pandas/tests/extension/base/methods.py

@@ -3,12 +3,10 @@
 import numpy as np
 import pytest
 
-from pandas.core.dtypes.common import is_bool_dtype
-
 import pandas as pd
 import pandas._testing as tm
+from pandas.core.dtypes.common import is_bool_dtype
 from pandas.core.sorting import nargsort
-
 from .base import BaseExtensionTests
 
 
@@ -17,7 +15,7 @@ class BaseMethodsTests(BaseExtensionTests):
 
     @pytest.mark.parametrize("dropna", [True, False])
     def test_value_counts(self, all_data, dropna):
-        all_data = all_data[:10]
+        all_data = all_data[:5]
         if dropna:
             other = np.array(all_data[~all_data.isna()])
         else:
@@ -28,6 +26,15 @@ def test_value_counts(self, all_data, dropna):
 
         self.assert_series_equal(result, expected)
 
+        result = (
+            pd.Series(all_data, dtype=all_data.dtype)
+            .value_counts(dropna=dropna, normalize=True)
+            .sort_index()
+        )
+
+        expected = pd.Series([1 / len(other)] * len(other), index=result.index)
+        tm.assert_series_equal(expected, result)
+
     def test_count(self, data_missing):
         df = pd.DataFrame({"A": data_missing})
         result = df.count(axis="columns")

pandas/tests/extension/test_datetime.py

@@ -90,10 +90,6 @@ class TestGetitem(BaseDatetimeTests, base.BaseGetitemTests):
 
 
 class TestMethods(BaseDatetimeTests, base.BaseMethodsTests):
-    @pytest.mark.skip(reason="Incorrect expected")
-    def test_value_counts(self, all_data, dropna):
-        pass
-
     def test_combine_add(self, data_repeated):
         # Timestamp.__add__(Timestamp) not defined
         pass

pandas/tests/extension/test_string.py

@@ -81,9 +81,7 @@ class TestNoReduce(base.BaseNoReduceTests):
 
 
 class TestMethods(base.BaseMethodsTests):
-    @pytest.mark.skip(reason="returns nullable")
-    def test_value_counts(self, all_data, dropna):
-        return super().test_value_counts(all_data, dropna)
+    pass
 
 
 class TestCasting(base.BaseCastingTests):

pandas/tests/frame/test_api.py

@@ -443,7 +443,7 @@ def test_with_datetimelikes(self):
         t = df.T
 
         result = t.dtypes.value_counts()
-        expected = Series({np.dtype("object"): 10})
+        expected = Series({np.dtype("object"): 10}, dtype="Int64")
         tm.assert_series_equal(result, expected)
 
     def test_values(self, float_frame):

pandas/tests/frame/test_arithmetic.py

@@ -325,7 +325,8 @@ def test_df_flex_cmp_constant_return_types(self, opname):
         const = 2
 
         result = getattr(df, opname)(const).dtypes.value_counts()
-        tm.assert_series_equal(result, pd.Series([2], index=[np.dtype(bool)]))
+        expected = pd.Series([2], index=[np.dtype(bool)], dtype="Int64")
+        tm.assert_series_equal(result, expected)
 
     @pytest.mark.parametrize("opname", ["eq", "ne", "gt", "lt", "ge", "le"])
     def test_df_flex_cmp_constant_return_types_empty(self, opname):
@@ -335,7 +336,8 @@ def test_df_flex_cmp_constant_return_types_empty(self, opname):
 
         empty = df.iloc[:0]
         result = getattr(empty, opname)(const).dtypes.value_counts()
-        tm.assert_series_equal(result, pd.Series([2], index=[np.dtype(bool)]))
+        expected = pd.Series([2], index=[np.dtype(bool)], dtype="Int64")
+        tm.assert_series_equal(result, expected)
 
 
 # -------------------------------------------------------------------

pandas/tests/indexes/datetimes/test_ops.py

@@ -133,7 +133,7 @@ def test_value_counts_unique(self, tz_naive_fixture):
         idx = DatetimeIndex(np.repeat(idx.values, range(1, len(idx) + 1)), tz=tz)
 
         exp_idx = pd.date_range("2011-01-01 18:00", freq="-1H", periods=10, tz=tz)
-        expected = Series(range(10, 0, -1), index=exp_idx, dtype="int64")
+        expected = Series(range(10, 0, -1), index=exp_idx, dtype="Int64")
 
         for obj in [idx, Series(idx)]:
             tm.assert_series_equal(obj.value_counts(), expected)
@@ -154,13 +154,13 @@ def test_value_counts_unique(self, tz_naive_fixture):
         )
 
         exp_idx = DatetimeIndex(["2013-01-01 09:00", "2013-01-01 08:00"], tz=tz)
-        expected = Series([3, 2], index=exp_idx)
+        expected = Series([3, 2], index=exp_idx, dtype="Int64")
 
         for obj in [idx, Series(idx)]:
             tm.assert_series_equal(obj.value_counts(), expected)
 
         exp_idx = DatetimeIndex(["2013-01-01 09:00", "2013-01-01 08:00", pd.NaT], tz=tz)
-        expected = Series([3, 2, 1], index=exp_idx)
+        expected = Series([3, 2, 1], index=exp_idx, dtype="Int64")
 
         for obj in [idx, Series(idx)]:
             tm.assert_series_equal(obj.value_counts(dropna=False), expected)

pandas/tests/indexes/period/test_ops.py

@@ -47,7 +47,7 @@ def test_value_counts_unique(self):
             ],
             freq="H",
         )
-        expected = Series(range(10, 0, -1), index=exp_idx, dtype="int64")
+        expected = Series(range(10, 0, -1), index=exp_idx, dtype="Int64")
 
         for obj in [idx, Series(idx)]:
             tm.assert_series_equal(obj.value_counts(), expected)
@@ -68,13 +68,13 @@ def test_value_counts_unique(self):
         )
 
         exp_idx = PeriodIndex(["2013-01-01 09:00", "2013-01-01 08:00"], freq="H")
-        expected = Series([3, 2], index=exp_idx)
+        expected = Series([3, 2], index=exp_idx, dtype="Int64")
 
         for obj in [idx, Series(idx)]:
             tm.assert_series_equal(obj.value_counts(), expected)
 
         exp_idx = PeriodIndex(["2013-01-01 09:00", "2013-01-01 08:00", NaT], freq="H")
-        expected = Series([3, 2, 1], index=exp_idx)
+        expected = Series([3, 2, 1], index=exp_idx, dtype="Int64")
 
         for obj in [idx, Series(idx)]:
             tm.assert_series_equal(obj.value_counts(dropna=False), expected)

pandas/tests/indexes/timedeltas/test_ops.py

@@ -18,7 +18,7 @@ def test_value_counts_unique(self):
         idx = TimedeltaIndex(np.repeat(idx.values, range(1, len(idx) + 1)))
 
         exp_idx = timedelta_range("1 days 18:00:00", freq="-1H", periods=10)
-        expected = Series(range(10, 0, -1), index=exp_idx, dtype="int64")
+        expected = Series(range(10, 0, -1), index=exp_idx, dtype="Int64")
 
         for obj in [idx, Series(idx)]:
             tm.assert_series_equal(obj.value_counts(), expected)
@@ -38,13 +38,13 @@ def test_value_counts_unique(self):
         )
 
         exp_idx = TimedeltaIndex(["1 days 09:00:00", "1 days 08:00:00"])
-        expected = Series([3, 2], index=exp_idx)
+        expected = Series([3, 2], index=exp_idx, dtype="Int64")
 
         for obj in [idx, Series(idx)]:
             tm.assert_series_equal(obj.value_counts(), expected)
 
         exp_idx = TimedeltaIndex(["1 days 09:00:00", "1 days 08:00:00", pd.NaT])
-        expected = Series([3, 2, 1], index=exp_idx)
+        expected = Series([3, 2, 1], index=exp_idx, dtype="Int64")
 
         for obj in [idx, Series(idx)]:
             tm.assert_series_equal(obj.value_counts(dropna=False), expected)