Add ability to override run callbacks to process per FOV (#313)

* Support intermediate callbacks for build step

* Add intermediate callback params to notebook

* Add intermediate callback functionality to fov_watcher

* PYCODESTYLE

* Add inter callbacks to fov watcher test cases

* Ensure inter_cbs passed to fov_watcher test

* Add replace parameter to intermediate callback test

* Testing

* Swap order of run callbacks and intermediate callbacks

* added runcallbacks plot refresh on new FOV

* set matplotlib backend to agg

* agg only on mac

* removed threading debug import

* swapped cbs and ibs in case_inter_callback

* build_callbacks in test_watcher fix

* clear plot in `fov_watcher.FOV_EventHandler.on_created`

* remove plt clear in watcher_callbacks

* removed fig.show()

* removed plot clearing

* show plot in on_created

* set fig to _

* save plots

* save plots

* removed plot_dir argument

* barplot -> catplot

* removed misc plt.show()

* directly save the qc_metric figure

* do not use dark_background

* Fix intermediate callback tests

* visualize_qc_metrics adjustments

* updated a test

* visualize qc_metrics arg fix

* removed commented line

---------

Co-authored-by: Sricharan Reddy Varra <srivarra@stanford.edu>

.gitignore

@@ -134,3 +134,6 @@ dmypy.json
 
 # .DS_Store
 **/.DS_Store
+
+data/
+scripts/

templates/3a_monitor_MIBI_run.ipynb

@@ -112,8 +112,9 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "fov_callback, run_callback = build_callbacks(\n",
- " run_callbacks = ['plot_qc_metrics', 'plot_mph_metrics', 'image_stitching'],\n",
+ "fov_callback, run_callback, intermediate_callback = build_callbacks(\n",
+ " run_callbacks = ['image_stitching'],\n",
+ " intermediate_callbacks = ['plot_qc_metrics', 'plot_mph_metrics'],\n",
  " fov_callbacks = ['extract_tiffs', 'generate_pulse_heights'],\n",
  " tiff_out_dir=extraction_dir,\n",
  " qc_out_dir=metrics_data_dir,\n",
@@ -131,13 +132,13 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "start_watcher(base_dir, log_path, fov_callback, run_callback)"
+ "start_watcher(base_dir, log_path, fov_callback, run_callback, intermediate_callback)"
  ]
  }
  ],
  "metadata": {
  "kernelspec": {
- "display_name": "Python 3 (ipykernel)",
+ "display_name": "toffy38",
  "language": "python",
  "name": "python3"
  },
@@ -151,7 +152,12 @@
  "name": "python",
  "nbconvert_exporter": "python",
  "pygments_lexer": "ipython3",
- "version": "3.8.13"
+ "version": "3.8.16"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "f6f196d8c15ef51fc3b92e6b5462f515514162e36d9ad988b37a3636f71a6ec3"
+ }
  }
  },
  "nbformat": 4,

templates/3c_generate_qc_metrics.ipynb

@@ -91,7 +91,6 @@
  "outputs": [],
  "source": [
  "# set up directories\n",
- "bin_file_path = os.path.join('D:\\\\Data', run_name)\n",
  "extracted_imgs_path = os.path.join('D:\\\\Extracted_Images', run_name)\n",
  "qc_out_dir = os.path.join('C:\\\\Users\\\\Customer.ION\\\\Documents\\\\run_metrics', run_name, 'fov_data')\n",
  "\n",
@@ -132,7 +131,6 @@
  " if not os.path.exists(os.path.join(qc_out_dir, '%s_nonzero_mean_stats.csv' % fov)):\n",
  " print(\"Extracting QC metrics for fov %s\" % fov)\n",
  " qc_comp.compute_qc_metrics(\n",
- " bin_file_path,\n",
  " extracted_imgs_path,\n",
  " fov, \n",
  " save_csv=qc_out_dir\n",
@@ -234,7 +232,7 @@
  ],
  "metadata": {
  "kernelspec": {
- "display_name": "Python 3 (ipykernel)",
+ "display_name": "toffy38",
  "language": "python",
  "name": "python3"
  },
@@ -248,7 +246,12 @@
  "name": "python",
  "nbconvert_exporter": "python",
  "pygments_lexer": "ipython3",
- "version": "3.8.13"
+ "version": "3.8.16"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "f6f196d8c15ef51fc3b92e6b5462f515514162e36d9ad988b37a3636f71a6ec3"
+ }
  }
  },
  "nbformat": 4,

toffy/fov_watcher.py

@@ -4,6 +4,7 @@
 from datetime import datetime
 from pathlib import Path
 from typing import Callable, Tuple
+from matplotlib import pyplot as plt
 
 from watchdog.events import FileCreatedEvent, FileSystemEventHandler
 from watchdog.observers import Observer
@@ -156,7 +157,9 @@ class FOV_EventHandler(FileSystemEventHandler):
  """
  def __init__(self, run_folder: str, log_folder: str,
  fov_callback: Callable[[str, str], None],
- run_callback: Callable[[str], None], timeout: int = 1.03 * 60 * 60):
+ run_callback: Callable[[str], None],
+ intermediate_callback: Callable[[str], None] = None,
+ timeout: int = 1.03 * 60 * 60):
  """Initializes FOV_EventHandler
 
  Args:
@@ -168,6 +171,8 @@ def __init__(self, run_folder: str, log_folder: str,
  callback to run on each fov
  run_callback (Callable[[None], None]):
  callback to run over the entire run
+ intermediate_callback (Callable[[None], None]):
+ run callback overriden to run on each fov
  timeout (int):
  number of seconds to wait for non-null filesize before raising an error
  """
@@ -183,6 +188,7 @@ def __init__(self, run_folder: str, log_folder: str,
 
  self.fov_func = fov_callback
  self.run_func = run_callback
+ self.inter_func = intermediate_callback
 
  for root, dirs, files in os.walk(run_folder):
  for name in files:
@@ -231,6 +237,9 @@ def on_created(self, event: FileCreatedEvent):
  self.fov_func(self.run_folder, point_name)
  self.run_structure.processed(point_name)
 
+ if self.inter_func:
+ self.inter_func(self.run_folder)
+
  logf.close()
  self.check_complete()
 
@@ -257,7 +266,9 @@ def check_complete(self):
 
 
 def start_watcher(run_folder: str, log_folder: str, fov_callback: Callable[[str, str], None],
- run_callback: Callable[[None], None], completion_check_time: int = 30,
+ run_callback: Callable[[None], None],
+ intermediate_callback: Callable[[str, str], None] = None,
+ completion_check_time: int = 30,
  zero_size_timeout: int = 1.03 * 60 * 60):
  """ Passes bin files to provided callback functions as they're created
 
@@ -272,6 +283,9 @@ def start_watcher(run_folder: str, log_folder: str, fov_callback: Callable[[str,
  run_callback (Callable[[None], None]):
  function ran once the run has completed. assemble this using
  `watcher_callbacks.build_callbacks`
+ intermediate_callback (Callable[[None], None]):
+ function defined as run callback overriden as fov callback. assemble this using
+ `watcher_callbacks.build_callbacks`
  completion_check_time (int):
  how long to wait before checking watcher completion, in seconds.
  note, this doesn't effect the watcher itself, just when this wrapper function exits.
@@ -280,7 +294,7 @@ def start_watcher(run_folder: str, log_folder: str, fov_callback: Callable[[str,
  """
  observer = Observer()
  event_handler = FOV_EventHandler(run_folder, log_folder, fov_callback, run_callback,
- zero_size_timeout)
+ intermediate_callback, zero_size_timeout)
  observer.schedule(event_handler, run_folder, recursive=True)
  observer.start()
 

toffy/fov_watcher_test.py

@@ -75,8 +75,9 @@ def test_run_structure(run_json, expected_files):
 @patch('toffy.watcher_callbacks.visualize_qc_metrics', side_effect=mock_visualize_qc_metrics)
 @patch('toffy.watcher_callbacks.visualize_mph', side_effect=mock_visualize_mph)
 @pytest.mark.parametrize('add_blank', [False, True])
-@parametrize_with_cases('run_cbs, fov_cbs, kwargs, validators', cases=WatcherCases)
-def test_watcher(mock_viz_qc, mock_viz_mph, run_cbs, fov_cbs, kwargs, validators, add_blank):
+@parametrize_with_cases('run_cbs, int_cbs, fov_cbs, kwargs, validators', cases=WatcherCases)
+def test_watcher(mock_viz_qc, mock_viz_mph, run_cbs, int_cbs, fov_cbs,
+ kwargs, validators, add_blank):
  try:
  with tempfile.TemporaryDirectory() as tmpdir:
 
@@ -97,8 +98,8 @@ def test_watcher(mock_viz_qc, mock_viz_mph, run_cbs, fov_cbs, kwargs, validators
  run_data = os.path.join(tmpdir, 'test_run')
  log_out = os.path.join(tmpdir, 'log_output')
  os.makedirs(run_data)
-
- fov_callback, run_callback = build_callbacks(run_cbs, fov_cbs, **kwargs)
+ fov_callback, run_callback, intermediate_callback = \
+  build_callbacks(run_cbs, int_cbs, fov_cbs, **kwargs)
  write_json_file(json_path=os.path.join(run_data, 'test_run.json'),
  json_object=COMBINED_RUN_JSON_SPOOF)
 
@@ -116,7 +117,8 @@ def test_watcher(mock_viz_qc, mock_viz_mph, run_cbs, fov_cbs, kwargs, validators
  # zero-size files are halted for 1 second or until they have non zero-size
  res_scan = pool.apply_async(
  start_watcher,
- (run_data, log_out, fov_callback, run_callback, 1, SLOW_COPY_INTERVAL_S)
+ (run_data, log_out, fov_callback, run_callback,
+ intermediate_callback, 1, SLOW_COPY_INTERVAL_S)
  )
 
  res_scan.get()

toffy/mph_comp.py

@@ -152,8 +152,7 @@ def visualize_mph(mph_df, out_dir, regression: bool = False):
  io_utils.validate_paths(out_dir)
 
  # visualize the median pulse heights
- plt.style.use('dark_background')
- # plt.title('FOV total counts vs median pulse height')
+ plt.title('FOV total counts vs median pulse height')
  fig = plt.figure()
  ax1 = fig.add_subplot(111)
  x = mph_df['cum_total_count']/1000000
@@ -182,7 +181,7 @@ def visualize_mph(mph_df, out_dir, regression: bool = False):
  ax1.plot(x2, m * x2 + b)
 
  # save figure
- file_path = os.path.join(out_dir, 'fov_vs_mph.jpg')
+ file_path = os.path.join(out_dir, 'fov_vs_mph.png')
  if os.path.exists(file_path):
  os.remove(file_path)
  plt.savefig(file_path)

toffy/mph_comp_test.py

@@ -127,4 +127,4 @@ def test_visualize_mph():
  with tempfile.TemporaryDirectory() as temp_dir:
  # test for saving to directory
  mph.visualize_mph(mph_data, out_dir=temp_dir, regression=True)
- assert os.path.exists(os.path.join(temp_dir, 'fov_vs_mph.jpg'))
+ assert os.path.exists(os.path.join(temp_dir, 'fov_vs_mph.png'))