fixing chunk issue and using TimeSlideEventSets

projects/sandbox/mdc/mdc/data/batcher.py

@@ -26,7 +26,7 @@ def batch_chunks(
     # but will need to see how this bears out in
     # production, and if it's a problem we'll just
     # have to relax the test constraints
-    inf_per_second = throughput / 2 * inference_sampling_rate
+    inf_per_second = throughput / inference_sampling_rate
     batches_per_second = inf_per_second / batch_size
 
     max_calls = 2

projects/sandbox/mdc/mdc/data/loader.py

@@ -1,3 +1,4 @@
+from math import ceil
 from pathlib import Path
 from typing import List
 
@@ -13,7 +14,7 @@ def load_dataset(
     fs, dataset, channels: List[str], chunk_size: int
 ) -> np.ndarray:
     size = len(fs[0][channels[0]][dataset])
-    num_steps = int(size // chunk_size)
+    num_steps = int(ceil(size / chunk_size))
     for i in range(num_steps):
         slc = slice(i * chunk_size, (i + 1) * chunk_size)
         xs = [[i[j][dataset][slc] for j in channels] for i in fs]

projects/sandbox/mdc/mdc/deploy.py

@@ -2,15 +2,17 @@
 import re
 import shutil
 import time
+from collections import defaultdict
 from pathlib import Path
 from textwrap import dedent
 from typing import List
 
 import h5py
+import numpy as np
 import psutil
 from typeo import scriptify
 
-from aframe.analysis.ledger.events import EventSet, RecoveredInjectionSet
+from aframe.analysis.ledger.events import TimeSlideEventSet
 from aframe.deploy import condor
 from aframe.logging import configure_logging
 from aframe.utils.timeslides import calc_shifts_required
@@ -22,21 +24,19 @@
 
 
 def aggregate_results(output_directory: Path):
-    """
-    Combine results from across segments into a single
-    background file and foreground file. Remove the directory
-    containing the individual segment results.
-    """
-    background, foreground = EventSet(), RecoveredInjectionSet()
+    results = {k: defaultdict(list) for k in ["background", "foreground"]}
+    var = 1 / 8
     for data_dir in (output_directory / "tmp").iterdir():
-        bckground = EventSet.read(data_dir / "background.h5")
-        frground = RecoveredInjectionSet.read(data_dir / "foreground.h5")
-
-        background.append(bckground)
-        foreground.append(frground)
-
-    background.write(output_directory / "background.h5")
-    foreground.write(output_directory / "foreground.h5")
+        for key, value in results.items():
+            events = TimeSlideEventSet.read(data_dir / f"{key}.hdf")
+            value["time"].append(events.time)
+            value["stat"].append(events.detection_statistic)
+    for key, value in results.items():
+        with h5py.File(output_directory / f"{key}.hdf", "w") as f:
+            for k, v in value.items():
+                v = np.concatenate(v)
+                f[k] = v
+            f["var"] = var * np.ones_like(v)
     shutil.rmtree(output_directory / "tmp")
 
 

projects/sandbox/mdc/mdc/main.py

@@ -7,7 +7,7 @@
 from mdc.data import batch_chunks, load_datasets
 from typeo import scriptify
 
-from aframe.analysis.ledger.events import EventSet, RecoveredInjectionSet
+from aframe.analysis.ledger.events import TimeSlideEventSet
 from aframe.logging import configure_logging
 from hermes.aeriel.client import InferenceClient
 
@@ -238,8 +238,8 @@ def main(
     )
     chunk_size = int(chunk_size * sample_rate)
     with client:
-        background_events = EventSet()
-        foreground_events = RecoveredInjectionSet()
+        background_events = TimeSlideEventSet()
+        foreground_events = TimeSlideEventSet()
 
         logging.info(f"Iterating through data from directory {data_dir}")
         loader = load_datasets(data_dir, datasets, ifos, chunk_size)