Add id_gaps and id_gaps_index functions to times.py.

    gh-20

Author: ray310

CHANGELOG.md

@@ -7,7 +7,7 @@
 ## 0.1.1 - Unreleased
 ### Added
 - SeriesProfile now reports gaps in pd.Series with type `datetime64` or for Series with `DatetimeIndex`. [gh-20](https://github.com/ray310/Panda-Helper/issues/20)
-- `times.py` module has been added with public functions `time_diffs` and `time_diffs_index`. [gh-20](https://github.com/ray310/Panda-Helper/issues/20)
+- `times.py` module has been added with public functions `time_diffs`, `time_diffs_index`, `id_gaps`, `id_gaps_index`. [gh-20](https://github.com/ray310/Panda-Helper/issues/20)
 - [`freq_most_least` default parameter for SeriesProfile has been changed to `(10, 5)`.](https://github.com/ray310/Panda-Helper/commit/9ea7a4108996422eaa433e3b86ed20dbbb3c0bdb)
 
 ____

mkdocs.yml

@@ -101,6 +101,7 @@ plugins:
       python:
         paths: [src]
         options:
+          members_order: alphabetical
           docstring_style: google
           docstring_section_style: list
           docstring_options:

src/pandahelper/__init__.py

@@ -4,7 +4,7 @@
 
 from pandahelper.profiles import DataFrameProfile, SeriesProfile
 from pandahelper.stats import distribution_stats, frequency_table
-from pandahelper.times import time_diffs, time_diffs_index
+from pandahelper.times import time_diffs, time_diffs_index, id_gaps, id_gaps_index
 
 __version__ = "0.1.1"
 __all__ = [
@@ -14,4 +14,6 @@
     "SeriesProfile",
     "time_diffs",
     "time_diffs_index",
+    "id_gaps",
+    "id_gaps_index",
 ]

src/pandahelper/times.py

@@ -6,7 +6,7 @@
 
 
 def time_diffs(series: Union[pd.Series, pd.DatetimeIndex]) -> pd.Series(pd.Timedelta):
-    """Calculate time diffs (gaps) for Pandas Series or Index of timestamps.
+    """Calculate time difference between subsequent observations.
 
     Sorts input by time before calculating diffs.
 
@@ -19,19 +19,39 @@ def time_diffs(series: Union[pd.Series, pd.DatetimeIndex]) -> pd.Series(pd.Timed
 
     Raises:
         TypeError: If input is not Series of type datetime64 or DatetimeIndex.
+
+    Examples:
+        Calculate time differences between observations on Series of timestamps after
+        it has been randomized:
+
+        >>> import pandahelper as ph
+        >>> import pandas as pd
+        >>>
+        >>> start = pd.Timestamp(year=1999, month=1, day=1)
+        >>> rng = pd.date_range(start, periods=10, freq="D").delete([3, 4, 5, 8])
+        >>> series = pd.Series(rng).sample(frac=1, random_state=3)  # randomize order
+
+        >>> ph.time_diffs(series)
+        1999-01-01      NaT
+        1999-01-02   1 days
+        1999-01-03   1 days
+        1999-01-07   4 days
+        1999-01-08   1 days
+        1999-01-10   2 days
+        Name: diffs, dtype: timedelta64[ns]
     """
     if not pat.is_datetime64_any_dtype(series.dtype):
-        raise TypeError("Should be Series of datetime64 dtype.")
+        raise TypeError("Should be of datetime64 dtype.")
     series = series.sort_values()
     diffs = pd.Series(series.diff(), name="diffs")
     diffs.index = series
     return diffs
 
 
-def time_diffs_index(df: Union[pd.Series, pd.DatetimeIndex]) -> pd.Series(pd.Timedelta):
-    """Calculate time diffs (gaps) for time-indexed Pandas Series or Dataframe.
+def time_diffs_index(df: Union[pd.Series, pd.DataFrame]) -> pd.Series(pd.Timedelta):
+    """Calculate time difference between subsequent time-indexed observations.
 
-    Sorts input by time before calculating diffs.
+    Sorts input by time index before calculating diffs.
 
     Args:
         df (pd.Series or pd.DataFrame): Pandas Series or DataFrame with DateTimeIndex
@@ -42,10 +62,102 @@ def time_diffs_index(df: Union[pd.Series, pd.DatetimeIndex]) -> pd.Series(pd.Tim
 
     Raises:
         TypeError: If input does not have a DatetimeIndex.
+
+    Examples:
+        Calculate time differences between observations on time-indexed DataFrame after
+        it has been randomized:
+
+        >>> import pandahelper as ph
+        >>> import pandas as pd
+        >>>
+        >>> start = pd.Timestamp(year=1999, month=1, day=1)
+        >>> rng = pd.date_range(start, periods=10, freq="D").delete([3, 4, 5, 8])
+        >>> # index by time then randomize order
+        >>> df = pd.DataFrame(range(len(rng)), index=rng).sample(frac=1, random_state=3)
+
+        >>> ph.time_diffs_index(df)
+        1999-01-01      NaT
+        1999-01-02   1 days
+        1999-01-03   1 days
+        1999-01-07   4 days
+        1999-01-08   1 days
+        1999-01-10   2 days
+        Name: diffs, dtype: timedelta64[ns]
     """
     if isinstance(df.index, pd.DatetimeIndex):
         df = df.sort_index()
         diffs = pd.Series(df.index.diff(), name="diffs")
         diffs.index = df.index
         return diffs
     raise TypeError(f"Index should be of type {pd.DatetimeIndex}")
+
+
+def id_gaps(
+    series: Union[pd.Series, pd.DatetimeIndex], threshold: pd.Timedelta
+) -> pd.DataFrame:
+    """Identify time gaps above `threshold` in datetime64 Series or DatetimeIndex.
+
+    Sorts input by time before calculating gaps.
+
+    Args:
+        series (pd.Series or pd.DatetimeIndex): `datetime64` Series or DatetimeIndex.
+        threshold (pd.Timedelta): Threshold to identify gaps
+            (and not expected time differences).
+
+    Returns:
+        One-column Pandas DataFrame of gaps indexed by when gap was calculated.
+
+    Examples:
+        Identify time gaps on Series of timestamps with a 2 and 4 hour
+        gap after it has been randomized:
+
+        >>> import pandahelper as ph
+        >>> import pandas as pd
+        >>>
+        >>> start = pd.Timestamp(year=1999, month=1, day=1)
+        >>> rng = pd.date_range(start, periods=24, freq="1h").delete([3, 4, 8, 9, 10])
+        >>> series = pd.Series(rng).sample(frac=1, random_state=3)  # randomize order
+
+        >>> ph.id_gaps(series, pd.Timedelta(hours=1))
+                                      diffs
+        1999-01-01 11:00:00 0 days 04:00:00
+        1999-01-01 04:00:00 0 days 02:00:00
+    """
+    diffs = time_diffs(series)
+    return diffs[diffs > threshold].sort_values(ascending=False).to_frame()
+
+
+def id_gaps_index(
+    df: Union[pd.Series, pd.DataFrame], threshold: pd.Timedelta
+) -> pd.DataFrame:
+    """Identify time gaps above `threshold` in time-indexed Series or DataFrame.
+
+    Sorts input by time index before calculating diffs.
+
+    Args:
+        df (pd.Series or pd.DataFrame): Time-indexed Series or DataFrame.
+        threshold (pd.Timedelta): Threshold to identify gaps
+            (and not expected time differences).
+
+    Returns:
+        One-column Pandas DataFrame of gaps indexed by when gap was calculated.
+
+    Examples:
+        Identify time gaps on an hourly, time-indexed Series with a 2 and 4 hour
+        gap after it has been randomized:
+
+        >>> import pandahelper as ph
+        >>> import pandas as pd
+        >>>
+        >>> start = pd.Timestamp(year=1999, month=1, day=1)
+        >>> rng = pd.date_range(start, periods=24, freq="1h").delete([3, 8, 9, 10])
+        >>> # index by time then randomize order
+        >>> df = pd.DataFrame(range(len(rng)), index=rng).sample(frac=1, random_state=3)
+
+        >>> ph.id_gaps_index(df, pd.Timedelta(hours=1))
+                                      diffs
+        1999-01-01 11:00:00 0 days 04:00:00
+        1999-01-01 04:00:00 0 days 02:00:00
+    """
+    diffs = time_diffs_index(df)
+    return diffs[diffs > threshold].sort_values(ascending=False).to_frame()

tests/conftest.py

@@ -4,7 +4,6 @@
 cached value.
 """
 
-from datetime import datetime
 import os
 import numpy as np
 import pandas as pd
@@ -19,17 +18,28 @@
 
 @pytest.fixture
 def cat_df(scope="package"):  # pylint: disable=W0613
-    """Return test pd.DataFrame."""
-    start = datetime(year=1999, month=1, day=1, hour=0, minute=0)
+    """Return test pd.DataFrame with DatetimeIndex."""
+    start = pd.Timestamp(year=1999, month=1, day=1)
     end = start + pd.Timedelta(hours=10)
     df = make_category_data("Springfield", start, end, freq="h")
     df = df.sample(frac=1, random_state=2)  # index is out of order
     return df
 
 
+@pytest.fixture
+def ts_timeindex(scope="package"):  # pylint: disable=W0613
+    """Return pd.Series of type datetime64 with DatetimeIndex."""
+    start = pd.Timestamp(year=1999, month=1, day=1)
+    end = start + pd.Timedelta(hours=40)
+    time_series = pd.Series(pd.date_range(start, end, freq="4h", inclusive="left"))
+    index_end = start + pd.Timedelta(hours=10)
+    time_series.index = pd.date_range(start, index_end, freq="h", inclusive="left")
+    return time_series
+
+
 @pytest.fixture
 def test_df(scope="package"):  # pylint: disable=W0613
-    """Return test pd.DataFrame."""
+    """Return test pd.DataFrame from sample of NYC collisions dataset."""
     return pd.read_csv(os.path.join(TEST_DATA_DIR, TEST_DATA_FILE))
 
 

tests/test_profiles.py

@@ -218,17 +218,6 @@ def test_series_profile_time_index_false(cat_df):
     assert profile.time_diffs is None
 
 
-@pytest.fixture
-def ts_timeindex(scope="module"):  # pylint: disable=W0613
-    """Return pd.Series of type datetime64 with DatetimeIndex."""
-    start = datetime(year=1999, month=1, day=1, hour=0, minute=0)
-    end = start + pd.Timedelta(hours=40)
-    time_series = pd.Series(pd.date_range(start, end, freq="4h", inclusive="left"))
-    index_end = start + pd.Timedelta(hours=10)
-    time_series.index = pd.date_range(start, index_end, freq="h", inclusive="left")
-    return time_series
-
-
 def test_series_profile_ts_range_index_true(ts_timeindex):  # pylint: disable=W0621
     """time_index=True does not calculate time diffs for Series with RangeIndex."""
     series = ts_timeindex

tests/test_times.py

@@ -5,6 +5,23 @@
 import pandahelper.times as pht
 
 
+def test_time_diffs(cat_df):
+    """time_diffs should work on shuffled pd.Series or Index of timestamps."""
+    valid = [cat_df.index, pd.Series(cat_df.index)]
+    for v in valid:
+        result = pht.time_diffs(v)
+        assert result.iloc[0] is pd.NaT
+        assert all(result[1:] == pd.Timedelta(hours=1))
+
+
+def test_time_diffs_exception():
+    """Non-datetime64 pd.Series raises exception."""
+    invalid = [pd.Series(list(range(5))), pd.Series([pd.Timedelta(hours=1)] * 2)]
+    for tipo in invalid:
+        with pytest.raises(TypeError):
+            pht.time_diffs(tipo)
+
+
 def test_time_diffs_index(cat_df):
     """time_diffs_index should work on shuffled pd.Series or pd.DataFrame."""
     # test DF
@@ -27,18 +44,40 @@ def test_time_diffs_index_exception():
         assert str(pd.DatetimeIndex) in str(exc)
 
 
-def test_time_diffs(cat_df):
-    """time_diffs should work on shuffled pd.Series or Index of timestamps."""
-    valid = [cat_df.index, pd.Series(cat_df.index)]
-    for v in valid:
-        result = pht.time_diffs(v)
-        assert result.iloc[0] is pd.NaT
-        assert all(result[1:] == pd.Timedelta(hours=1))
+def test_id_gaps_index(ts_timeindex):
+    """id_gap_index returns expected gap from time-Series with DatetimeIndex."""
+    result = pht.id_gaps_index(
+        ts_timeindex, pd.Timedelta(minutes=59, microseconds=999999)
+    )
+    expected = pd.DataFrame(
+        [pd.Timedelta(hours=1)] * 9,
+        index=pd.date_range(pd.Timestamp(1999, 1, 1, 1), periods=9, freq="h"),
+        columns=["diffs"],
+    )
+    pd.testing.assert_frame_equal(expected, result, check_index_type=True)
 
 
-def test_time_diffs_exception():
-    """Non-datetime64 pd.Series raises exception."""
-    invalid = [pd.Series(list(range(5))), pd.Series([pd.Timedelta(hours=1)] * 2)]
-    for tipo in invalid:
-        with pytest.raises(TypeError):
-            pht.time_diffs(tipo)
+def test_id_gaps_index_no_gaps(ts_timeindex):
+    """id_gap_index returns empty Dataframe when threshold exceeds diffs."""
+    result = pht.id_gaps_index(ts_timeindex, pd.Timedelta(minutes=60, microseconds=1))
+    assert len(result) == 0
+
+
+def test_id_gaps_(ts_timeindex):
+    """id_gap returns expected gap from time-Series with DatetimeIndex."""
+    result = pht.id_gaps(
+        ts_timeindex, pd.Timedelta(hours=3, minutes=59, microseconds=999999)
+    )
+    expected = pd.DataFrame(
+        [pd.Timedelta(hours=4)] * 9,
+        index=pd.date_range(pd.Timestamp(1999, 1, 1, 4), periods=9, freq="4h"),
+        columns=["diffs"],
+    )
+    expected.index.freq = None  # diffs won't have freq set
+    pd.testing.assert_frame_equal(expected, result, check_index_type=True)
+
+
+def test_id_gaps_no_gaps(ts_timeindex):
+    """id_gap_index returns empty Dataframe when threshold exceeds diffs."""
+    result = pht.id_gaps(ts_timeindex, pd.Timedelta(hours=4, microseconds=1))
+    assert len(result) == 0