fix data reduction issue

flxdot · May 8, 2024 · d942606 · d942606
1 parent 54d6da8
commit d942606
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 2 deletions.
diff --git a/services/api/carlos/api/utils/data_reduction.py b/services/api/carlos/api/utils/data_reduction.py
@@ -28,6 +28,7 @@ def optimize_timeseries(
     timestamps = [timeseries.timestamps[0]]
     values = [timeseries.values[0]]
     prev_ts = timeseries.timestamps[0]
+    last_added_idx = 0
     for idx, (ts, val) in enumerate(
         zip(timeseries.timestamps[1:], timeseries.values[1:])
     ):
@@ -42,8 +43,16 @@ def optimize_timeseries(
             )
             or idx + 1 == ts_len
         ):
+            # if we skipped at least one datapoint, we need to add the right edge
+            # of the
+            if idx - last_added_idx > 1:
+                # we use idx instead of idx - 1, we would need to use (idx - 1 + 1)
+                # due to the loop overset
+                timestamps.append(timeseries.timestamps[idx])
+                values.append(timeseries.values[idx])
             timestamps.append(ts)
             values.append(val)
+            last_added_idx = idx
 
         prev_ts = ts
 

diff --git a/services/api/carlos/api/utils/data_reduction_test.py b/services/api/carlos/api/utils/data_reduction_test.py
@@ -87,8 +87,9 @@ def ts(offset: int) -> datetime:
             DEFAULT_SPLIT_THRESHOLD,
             TimeseriesData(
                 timeseries_id=42,
-                timestamps=[ts(0), ts(1), ts(7), ts(9)],
-                values=[1, 2, 3, 4],
+                # we want the first and last value of the steady state
+                timestamps=[ts(0), ts(1), ts(6), ts(7), ts(8), ts(9)],
+                values=[1, 2, 2, 3, 3, 4],
             ),
             id="duplicate in the middle beginning",
         ),