Check for missing fovs after run is complete

src/toffy/fov_watcher.py

@@ -1,6 +1,6 @@
 import logging
 import os
-import platform
+import threading
 import time
 import warnings
 from datetime import datetime
@@ -29,16 +29,16 @@ class RunStructure:
  fov_progress (dict): Whether or not an expected file has been created
  """
 
- def __init__(self, run_folder: str, timeout: int = 10 * 60):
+ def __init__(self, run_folder: str, fov_timeout: int = 7800):
  """initializes RunStructure by parsing run json within provided run folder
 
  Args:
  run_folder (str):
  path to run folder
- timeout (int):
+ fov_timeout (int):
  number of seconds to wait for non-null filesize before raising an error
  """
- self.timeout = timeout
+ self.timeout = fov_timeout
  self.fov_progress = {}
  self.processed_fovs = []
  self.moly_points = []
@@ -183,7 +183,8 @@ def __init__(
  fov_callback: Callable[[str, str], None],
  run_callback: Callable[[str], None],
  intermediate_callback: Callable[[str], None] = None,
- timeout: int = 1.03 * 60 * 60,
+ fov_timeout: int = 7800,
+ watcher_timeout: int = 3 * 7800,
  ):
  """Initializes FOV_EventHandler
 
@@ -198,12 +199,21 @@ def __init__(
  callback to run over the entire run
  intermediate_callback (Callable[[None], None]):
  run callback overriden to run on each fov
- timeout (int):
+ fov_timeout (int):
  number of seconds to wait for non-null filesize before raising an error
+ watcher_timeout (int):
+ length to wait for new file generation before timing out
  """
  super().__init__()
  self.run_folder = run_folder
 
+ self.last_event_time = datetime.now()
+ self.timer_thread = threading.Thread(
+ target=self.file_timer, args=(fov_timeout, watcher_timeout)
+ )
+ self.timer_thread.daemon = True
+ self.timer_thread.start()
+
  self.log_path = os.path.join(log_folder, f"{Path(run_folder).parts[-1]}_log.txt")
  if not os.path.exists(log_folder):
  os.makedirs(log_folder)
@@ -215,7 +225,7 @@ def __init__(
  )
 
  # create run structure
- self.run_structure = RunStructure(run_folder, timeout=timeout)
+ self.run_structure = RunStructure(run_folder, fov_timeout=fov_timeout)
 
  self.fov_func = fov_callback
  self.run_func = run_callback
@@ -471,6 +481,10 @@ def on_created(self, event: FileCreatedEvent, check_last_fov: bool = True):
  event (FileCreatedEvent):
  file creation event
  """
+ # reset event creation time
+ current_time = datetime.now()
+ self.last_event_time = current_time
+
  # this happens if _check_last_fov gets called by a prior FOV, no need to reprocess
  if self.last_fov_num_processed == self.run_structure.highest_fov:
  return
@@ -479,6 +493,42 @@ def on_created(self, event: FileCreatedEvent, check_last_fov: bool = True):
  super().on_created(event)
  self._run_callbacks(event, check_last_fov)
 
+ def file_timer(self, fov_timeout, watcher_timeout):
+ """Checks time since last file was generated
+ Args:
+
+ fov_timeout (int):
+ how long to wait for fov data to be generated once file detected
+ watcher_timeout (int):
+ length to wait for new file generation before timing out
+ """
+ while True:
+ current_time = datetime.now()
+ time_elapsed = (current_time - self.last_event_time).total_seconds()
+
+ # 3 fov cycles and no new files --> timeout
+ if time_elapsed > watcher_timeout:
+ fov_num = self.last_fov_num_processed
+ fov_name = list(self.run_structure.fov_progress.keys())[fov_num]
+ print(f"Timed out waiting for {fov_name} files to be generated.")
+ logging.info(
+ f'{datetime.now().strftime("%d/%m/%Y %H:%M:%S")} -- Timed out'
+ f"waiting for {fov_name} files to be generated.\n"
+ )
+ logging.info(
+ f'{datetime.now().strftime("%d/%m/%Y %H:%M:%S")} -- '
+ f"Running {self.run_func.__name__} on FOVs\n"
+ )
+
+ # mark remaining fovs as completed to exit watcher
+ for fov_name in list(self.run_structure.fov_progress.keys()):
+ self.run_structure.fov_progress[fov_name] = {"json": True, "bin": True}
+
+ # trigger run callbacks
+ self.run_func(self.run_folder)
+ break
+ time.sleep(fov_timeout)
+
  def on_moved(self, event: FileMovedEvent, check_last_fov: bool = True):
  """Handles file renaming events
 
@@ -526,6 +576,7 @@ def start_watcher(
  run_folder_timeout: int = 5400,
  completion_check_time: int = 30,
  zero_size_timeout: int = 7800,
+ watcher_timeout: int = 3 * 7800,
 ):
  """Passes bin files to provided callback functions as they're created
 
@@ -577,7 +628,13 @@ def start_watcher(
 
  observer = Observer()
  event_handler = FOV_EventHandler(
- run_folder, log_folder, fov_callback, run_callback, intermediate_callback, zero_size_timeout
+ run_folder,
+ log_folder,
+ fov_callback,
+ run_callback,
+ intermediate_callback,
+ zero_size_timeout,
+ watcher_timeout,
  )
  observer.schedule(event_handler, run_folder, recursive=True)
  observer.start()

src/toffy/json_utils.py

@@ -258,3 +258,46 @@ def check_fov_resolutions(bin_file_dir, run_name, save_path=None):
  resolution_data.to_csv(save_path, index=False)
 
  return resolution_data
+
+
+def missing_fov_check(bin_file_dir, run_name):
+ """Use the run metadata to check if all FOV data was generated
+ Args:
+ bin_file_dir (str): directory containing the run json file
+ run_name (str): name of the run and corresponding run file
+
+ Raises:
+ Warning if any FOVs specified in the run file doesn't have associated bin/json files
+ """
+
+ # read in run file
+ run_file_path = os.path.join(bin_file_dir, run_name + ".json")
+ io_utils.validate_paths([run_file_path])
+ run_metadata = read_json_file(run_file_path, encoding="utf-8")
+
+ missing_fovs = {}
+ for fov in run_metadata.get("fovs", ()):
+ if fov.get("standardTarget") == "Molybdenum Foil":
+ continue
+
+ # get fov names
+ fov_number = fov.get("runOrder")
+ default_name = f"fov-{fov_number}-scan-1"
+ custom_name = fov.get("name")
+
+ # add default name and custom name for missing data
+ if not os.path.exists(
+ os.path.join(bin_file_dir, default_name + ".bin")
+ ) or not os.path.exists(os.path.join(bin_file_dir, default_name + ".json")):
+ missing_fovs[default_name] = [custom_name]
+
+ elif os.path.getsize(os.path.join(bin_file_dir, default_name + ".json")) == 0:
+ missing_fovs[default_name] = [custom_name]
+
+ if missing_fovs:
+ missing_fovs = pd.DataFrame(missing_fovs, index=[0]).T
+ missing_fovs.columns = ["fov_name"]
+ warnings.warn(
+ "The following FOVs were not processed due to missing/empty/late files: \n"
+ f"{missing_fovs}"
+ )

src/toffy/watcher_callbacks.py

@@ -17,6 +17,7 @@
 
 from toffy.bin_extraction import incomplete_fov_check
 from toffy.image_stitching import stitch_images
+from toffy.json_utils import missing_fov_check
 from toffy.mph_comp import combine_mph_metrics, compute_mph_metrics, visualize_mph
 from toffy.normalize import write_mph_per_mass
 from toffy.panel_utils import modify_panel_ranges
@@ -129,6 +130,13 @@ def check_incomplete_fovs(self, tiff_out_dir, **kwargs):
  """
  incomplete_fov_check(self.run_folder, tiff_out_dir)
 
+ def check_missing_fovs(self, **kwargs):
+ """Checks for associated bin/json files per FOV
+ Raises:
+ Warning if any fov data is missing
+ """
+ missing_fov_check(self.run_folder, os.path.basename(self.run_folder))
+
 
 @dataclass
 class FovCallbacks:

templates/3a_monitor_MIBI_run.ipynb

@@ -91,15 +91,19 @@
  "* The `generate_pulse_heights` FOV callback computes the median pulse heights for each mass specified in the `panel`. \n",
  "<br/> (See [4b_normalize_image_data](./4b_normalize_image_data.ipynb) for more details.)\n",
  "\n",
- "Callbacks listed in the `run_callbacks` argument will be run only once all expected FOV's have been discovered and processed. \n",
+ "Callbacks listed in the `intermediate_callbacks` argument run and update as each new FOV is generated.\n",
+ "\n",
+ "* The `plot_qc_metrics` intermediate callback will run all currently available qc metrics on each FOV and plot the results. <br/> (See [3c_generate_qc_metrics](./3c_generate_qc_metrics.ipynb) for more details.)\n",
  "\n",
- "* The `plot_qc_metrics` run callback will run all currently available qc metrics on each FOV, and plot the results once the run has completed. <br/> (See [3c_generate_qc_metrics](./3c_generate_qc_metrics.ipynb) for more details.)\n",
+ "* The `plot_mph_metrics` intermediate callback will compute the median pulse height data for each  FOV and plot the results. Additional arguments are: `regression` which when set to True will also plot the linear regression line for the data. <br/> (See [3d_compute_median_pulse_height](./3d_compute_median_pulse_height.ipynb) for more details.)\n",
  "\n",
- "* The `plot_mph_metrics` run callback will compute the median pulse height data for each FOV, and plot the results once the run has completed. Additional arguments are: `regression` which when set to True will also plot the linear regression line for the data. <br/> (See [3d_compute_median_pulse_height](./3d_compute_median_pulse_height.ipynb) for more details.)\n",
+ "Callbacks listed in the `run_callbacks` argument will be run only once all expected FOV's have been discovered and processed. \n",
  "\n",
  "* The `image_stitching` run callback will create a single image, which stitched together all FOV images for a specific channel. Additional arguments are: `channels`. <br/> (See [3e_stitch_images](./3e_stitch_images.ipynb) for more details.)\n",
  "\n",
- "* The `check_incomplete_fovs` run callback will check the run for any partially generated images. <br/> (See [3b_extract_images_from_bin](./3b_extract_images_from_bin.ipynb) for more details.)"
+ "* The `check_incomplete_fovs` run callback will check the run for any partially generated images. <br/> (See [3b_extract_images_from_bin](./3b_extract_images_from_bin.ipynb) for more details.)\n",
+ " \n",
+ "* The `check_missing_fovs` run callback checks that the run produces the appropriate .bin and .json all files for all FOVs included in the run file. <br/> (See [3f_FOV_checks](./3f_FOV_checks.ipynb) for more details.)"
  ]
  },
  {
@@ -109,7 +113,7 @@
  "outputs": [],
  "source": [
  "fov_callback, run_callback, intermediate_callback = build_callbacks(\n",
- " run_callbacks = ['image_stitching', 'check_incomplete_fovs'],\n",
+ " run_callbacks = ['image_stitching', 'check_incomplete_fovs', 'check_missing_fovs'],\n",
  " intermediate_callbacks = ['plot_qc_metrics', 'plot_mph_metrics'],\n",
  " fov_callbacks = ['extract_tiffs', 'generate_pulse_heights'],\n",
  " extract_prof=extract_prof,\n",

templates/3b_extract_images_from_bin.ipynb

@@ -115,17 +115,17 @@
  },
  {
  "cell_type": "markdown",
- "id": "3c7985e3-7f9a-499c-bb42-13e72cd340e5",
+ "id": "4f366be5-5ee7-4ecf-8db1-397fd7f6c0b9",
  "metadata": {},
  "source": [
- "## Check for any incomplete FOVs\n",
+ "## Check for any incomplete FOVs \n",
  "If the instrument is shut off part way through a run, this can result in output FOVs which are generated, but missing counts in parts of the images. The cell below will check for any incompete FOVs and warn you if any exist."
  ]
  },
  {
  "cell_type": "code",
  "execution_count": null,
- "id": "5cdf46cf-edf3-4889-b1b5-08601b5b3496",
+ "id": "6ba3af2f-d926-44cd-b688-5dab7e45513c",
  "metadata": {},
  "outputs": [],
  "source": [

templates/3f_check_resolutions.ipynb → templates/3f_FOV_checks.ipynb

@@ -2,11 +2,11 @@
  "cells": [
  {
  "cell_type": "markdown",
- "id": "4857c47b-f476-4494-9a0d-742ef35c0a85",
+ "id": "f8a1d486-4576-464a-bfe3-7e1f584f9a78",
  "metadata": {},
  "source": [
- "# Check FOV Resolutions\n",
- "This notebook will use the run file to check the image resolution for each FOV in the run. Consistent resolution level is important for downstream processing, so it is advised to change the image size of any problematic FOVs."
+ "# FOV checks\n",
+ "This notebook will perform two checks on your completed run which ensure all FOVs are the same resolution and verify there are no missing FOV data files."
  ]
  },
  {
@@ -17,7 +17,7 @@
  "outputs": [],
  "source": [
  "import os\n",
- "from toffy.json_utils import check_fov_resolutions\n",
+ "from toffy.json_utils import check_fov_resolutions, missing_fov_check\n",
  "from toffy.image_stitching import fix_image_resolutions"
  ]
  },
@@ -45,6 +45,15 @@
  "extraction_dir = os.path.join('D:\\\\Extracted_Images', run_name) "
  ]
  },
+ {
+ "cell_type": "markdown",
+ "id": "adba5c6c-7b6b-4a7f-8723-b589df680317",
+ "metadata": {},
+ "source": [
+ "## FOV resolutions\n",
+ "This section will use the run file to check the image resolution for each FOV in the run. Consistent resolution level is important for downstream processing, so it is advised to change the image size of any problematic FOVs."
+ ]
+ },
  {
  "cell_type": "markdown",
  "id": "b4edefe7-7f3a-4883-bcb3-47158753c837",
@@ -78,7 +87,7 @@
  "tags": []
  },
  "source": [
- "**Step 2: Change image sizes**"
+ "**Step 2: Change image sizes (if necessary)**"
  ]
  },
  {
@@ -91,6 +100,26 @@
  "# change image size for any FOVs with inconsistent resolutions\n",
  "fix_image_resolutions(resolution_data, extraction_dir)"
  ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "23bac7b6-e67e-41ac-aee2-903cc6ab7e27",
+ "metadata": {},
+ "source": [
+ "## Missing Fovs\n",
+ "Sometimes FOVs will fail to be generated due to instrument issues, so the function below checks that each FOV specified in the run json has the corresponding (non-empty) output files."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b3f3b134-2f6d-43b7-bba9-4eb4303555ad",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# check for .bin and .json files\n",
+ "missing_fov_check(bin_file_dir, run_name)"
+ ]
  }
  ],
  "metadata": {