Squashed commit of the following:

commit fac2c13 Author: Ivan Ivanov <ivan.ivanov@czbiohub.org> Date: Tue Apr 9 11:25:36 2024 -0700 Fix bug reading dragonfly acquisitions (#215) * fix bug reading dragonfly acquisitions * fix typo * style * bugfix commit 0c6984e Author: Ivan Ivanov <ivan.ivanov@czbiohub.org> Date: Mon Mar 11 12:35:51 2024 -0700 Fix bug determining number of rows and cols (#214) * fix bug determining number of rows and cols * add another XY Stage variation * add docs and fix style commit 3ab89ba Author: Ziwen Liu <67518483+ziw-liu@users.noreply.github.com> Date: Mon Mar 4 11:02:49 2024 -0800 Universal API implementations for Micro-Manager OME-TIFF and NDTiff (#185) * wip: draft mmstack ome-tiff fov * MM FOV base class * tests * bump tifffile * comment * fix indent after rebase * use get default * test pixel indexing * set MM metadata * style * update dependencies * add xarray * move old readers to the `_deprecated` namespace * uapi for ndtiff * refactor test setup to parametrize by dataset use globals instead of fixtures since parametrization happens before fixture evaluation * convert mmstack * fix and test chunking * fix metadata conversion and test ndtiff * update cli * fix scaling * test 1.4 and incomplete ome-tiffs * move reader tests * deprecate reader tests * update deprecated tests * update ngff tests * isort * update black target to 3.10 * lint * fix download paths * update docs references and theme * untrack autogenerated file * ignore execution time file * add github icon * update docstring * update docstring * show channel names and chunk size in info * print plate chunk size if verbose * fallback for pixel size * remove log level setting * do not filter logs and warnings in reader * avoid root logger * isort * set default logging level to INFO * format docstring * improve conversion messages * black * fix ome-tiff channel name indexing * fix ndtiff channel name indexing * update converter test * remove use of os.path in `reader` * expand _check_ndtiff checks * fix iteration * fix python 3.10 using `Path.glob(*/)` to get subdirs was added in 3.11 * bump zarr version to include resizing fix zarr-developers/zarr-python#1540 * fix cli default * set log level with an environment variable * fix unset * catch non-existent page * implement fallback for incomplete channel names workaround for the Dragonfly microscope where the multi-camera setup only has one channel name written * add debug logs * handle virtual frames * try reading pages from TiffFile directly * filter error logs about ImageJ metadata being broken this is a known MM limitation when writing OME-TIFFs * fix regex * remove use of os.path in `convert.py` * better channel indexing in `_get_summary_metadata` * style * safer NoneType check * private default axis names for NDTiff * update documentation to reflect new entry point * add repr to MM FOV and dataset types * rename mm_meta and expose summary metadata * add MicroManagerFOVMapping.root * add MicroManagerFOVMapping.zyx_scale * add warning log for failed position grid * fix grid layout * suppress hypothesis flakiness * different health check suppression --------- Co-authored-by: Ivan Ivanov <ivan.ivanov@czbiohub.org>
czbiohub-sf · Apr 15, 2024 · 3f00215 · 3f00215
1 parent 0468367
commit 3f00215
Show file tree

Hide file tree

Showing 15 changed files with 140 additions and 71 deletions.
diff --git a/README.md b/README.md
@@ -100,9 +100,9 @@ and the [example scripts](https://github.com/czbiohub-sf/iohub/tree/main/example
 Read a directory containing a TIFF dataset:
 
 ```py
-from iohub import read_micromanager
+from iohub import read_images
 
-reader = read_micromanager("/path/to/data/")
+reader = read_images("/path/to/data/")
 print(reader.shape)
 ```
 

diff --git a/iohub/_deprecated/reader_base.py b/iohub/_deprecated/reader_base.py
@@ -30,17 +30,21 @@ def shape(self):
         return self.frames, self.channels, self.slices, self.height, self.width
 
     @property
-    def mm_meta(self):
+    def micromanager_metadata(self) -> dict | None:
         return self._mm_meta
 
-    @mm_meta.setter
-    def mm_meta(self, value):
+    @micromanager_metadata.setter
+    def micromanager_metadata(self, value):
         if not isinstance(value, dict):
             raise TypeError(
                 f"Type of `mm_meta` should be `dict`, got `{type(value)}`."
             )
         self._mm_meta = value
 
+    @property
+    def micromanager_summary(self) -> dict | None:
+        return self._mm_meta.get("Summary", None)
+
     @property
     def stage_positions(self):
         return self._stage_positions

diff --git a/iohub/_deprecated/singlepagetiff.py b/iohub/_deprecated/singlepagetiff.py
@@ -91,9 +91,9 @@ def _set_mm_meta(self, one_pos):
         # pull one metadata sample and extract experiment dimensions
         metadata_path = os.path.join(one_pos, "metadata.txt")
         with open(metadata_path, "r") as f:
-            self.mm_meta = json.load(f)
+            self._mm_meta = json.load(f)
 
-        mm_version = self.mm_meta["Summary"]["MicroManagerVersion"]
+        mm_version = self._mm_meta["Summary"]["MicroManagerVersion"]
         if mm_version == "1.4.22":
             self._mm1_meta_parser()
         elif "beta" in mm_version:
@@ -402,12 +402,12 @@ def _mm1_meta_parser(self):
         -------
 
         """
-        self.z_step_size = self.mm_meta["Summary"]["z-step_um"]
-        self.width = self.mm_meta["Summary"]["Width"]
-        self.height = self.mm_meta["Summary"]["Height"]
-        self.frames = self.mm_meta["Summary"]["Frames"]
-        self.slices = self.mm_meta["Summary"]["Slices"]
-        self.channels = self.mm_meta["Summary"]["Channels"]
+        self.z_step_size = self._mm_meta["Summary"]["z-step_um"]
+        self.width = self._mm_meta["Summary"]["Width"]
+        self.height = self._mm_meta["Summary"]["Height"]
+        self.frames = self._mm_meta["Summary"]["Frames"]
+        self.slices = self._mm_meta["Summary"]["Slices"]
+        self.channels = self._mm_meta["Summary"]["Channels"]
 
     def _mm2beta_meta_parser(self):
         """
@@ -418,14 +418,14 @@ def _mm2beta_meta_parser(self):
         -------
 
         """
-        self.z_step_size = self.mm_meta["Summary"]["z-step_um"]
+        self.z_step_size = self._mm_meta["Summary"]["z-step_um"]
         self.width = int(
-            self.mm_meta["Summary"]["UserData"]["Width"]["PropVal"]
+            self._mm_meta["Summary"]["UserData"]["Width"]["PropVal"]
         )
         self.height = int(
-            self.mm_meta["Summary"]["UserData"]["Height"]["PropVal"]
+            self._mm_meta["Summary"]["UserData"]["Height"]["PropVal"]
         )
-        self.time_stamp = self.mm_meta["Summary"]["StartTime"]
+        self.time_stamp = self._mm_meta["Summary"]["StartTime"]
 
     def _mm2gamma_meta_parser(self):
         """
@@ -436,18 +436,18 @@ def _mm2gamma_meta_parser(self):
         -------
 
         """
-        keys_list = list(self.mm_meta.keys())
+        keys_list = list(self._mm_meta.keys())
         if "FrameKey-0-0-0" in keys_list[1]:
-            roi_string = self.mm_meta[keys_list[1]]["ROI"]
+            roi_string = self._mm_meta[keys_list[1]]["ROI"]
             self.width = int(roi_string.split("-")[2])
             self.height = int(roi_string.split("-")[3])
         elif "Metadata-" in keys_list[2]:
-            self.width = self.mm_meta[keys_list[2]]["Width"]
-            self.height = self.mm_meta[keys_list[2]]["Height"]
+            self.width = self._mm_meta[keys_list[2]]["Width"]
+            self.height = self._mm_meta[keys_list[2]]["Height"]
         else:
             raise ValueError("Metadata file incompatible with metadata reader")
 
-        self.z_step_size = self.mm_meta["Summary"]["z-step_um"]
-        self.frames = self.mm_meta["Summary"]["Frames"]
-        self.slices = self.mm_meta["Summary"]["Slices"]
-        self.channels = self.mm_meta["Summary"]["Channels"]
+        self.z_step_size = self._mm_meta["Summary"]["z-step_um"]
+        self.frames = self._mm_meta["Summary"]["Frames"]
+        self.slices = self._mm_meta["Summary"]["Slices"]
+        self.channels = self._mm_meta["Summary"]["Channels"]
diff --git a/iohub/_deprecated/zarrfile.py b/iohub/_deprecated/zarrfile.py
@@ -95,7 +95,7 @@ def __init__(
         try:
             self._set_mm_meta()
         except TypeError:
-            self.mm_meta = dict()
+            self.micromanager_metadata = {}
 
         self._generate_hcs_meta()
 
@@ -188,37 +188,37 @@ def _set_mm_meta(self):
         -------
 
         """
-        self.mm_meta = self.root.attrs.get("Summary")
-        mm_version = self.mm_meta["MicroManagerVersion"]
+        self._mm_meta = self.root.attrs.get("Summary")
+        mm_version = self._mm_meta["MicroManagerVersion"]
 
         if mm_version != "pycromanager":
             if "beta" in mm_version:
-                if self.mm_meta["Positions"] > 1:
+                if self._mm_meta["Positions"] > 1:
                     self.stage_positions = []
 
-                    for p in range(len(self.mm_meta["StagePositions"])):
+                    for p in range(len(self._mm_meta["StagePositions"])):
                         pos = self._simplify_stage_position_beta(
-                            self.mm_meta["StagePositions"][p]
+                            self._mm_meta["StagePositions"][p]
                         )
                         self.stage_positions.append(pos)
 
             # elif mm_version == '1.4.22':
-            #     for ch in self.mm_meta['ChNames']:
+            #     for ch in self._mm_meta['ChNames']:
             #         self.channel_names.append(ch)
             else:
-                if self.mm_meta["Positions"] > 1:
+                if self._mm_meta["Positions"] > 1:
                     self.stage_positions = []
 
-                    for p in range(self.mm_meta["Positions"]):
+                    for p in range(self._mm_meta["Positions"]):
                         pos = self._simplify_stage_position(
-                            self.mm_meta["StagePositions"][p]
+                            self._mm_meta["StagePositions"][p]
                         )
                         self.stage_positions.append(pos)
 
-                # for ch in self.mm_meta['ChNames']:
+                # for ch in self._mm_meta['ChNames']:
                 #     self.channel_names.append(ch)
 
-        self.z_step_size = self.mm_meta["z-step_um"]
+        self.z_step_size = self._mm_meta["z-step_um"]
 
     def _get_channel_names(self):
         well = self.hcs_meta["plate"]["wells"][0]["path"]

diff --git a/iohub/convert.py b/iohub/convert.py
@@ -118,9 +118,7 @@ def __init__(
                 f"Reader type {reader_type} not supported for conversion."
             )
         _logger.debug("Finished initializing data.")
-        self.summary_metadata = (
-            self.reader.mm_meta["Summary"] if self.reader.mm_meta else None
-        )
+        self.summary_metadata = self.reader.micromanager_summary
         self.save_name = output_dir.name
         _logger.debug("Getting dataset summary information.")
         self.coord_map = dict()
@@ -152,7 +150,8 @@ def __init__(
                 self.position_grid = _create_grid_from_coordinates(
                     *self._get_position_coords()
                 )
-            except ValueError:
+            except ValueError as e:
+                _logger.warning(f"Failed to generate grid layout: {e}")
                 self._make_default_grid()
         else:
             self._make_default_grid()
@@ -181,27 +180,41 @@ def _make_default_grid(self):
             )
 
     def _get_position_coords(self):
-        row_max = 0
-        col_max = 0
-        coords_list = []
+        """Get the position coordinates from the reader metadata.
+
+        Raises:
+            ValueError: If stage positions are not available.
+
+        Returns:
+            list: XY stage position coordinates.
+            int: Number of grid rows.
+            int: Number of grid columns.
+        """
+        rows = set()
+        cols = set()
+        xy_coords = []
 
-        # TODO: read rows, cols directly from XY corods
         # TODO: account for non MM2gamma meta?
         if not self.reader.stage_positions:
             raise ValueError("Stage positions not available.")
         for idx, pos in enumerate(self.reader.stage_positions):
-            stage_pos = pos.get("XYStage")
+            stage_pos = (
+                pos.get("XYStage") or pos.get("XY") or pos.get("XY Stage")
+            )
             if stage_pos is None:
                 raise ValueError(
                     f"Stage position is not available for position {idx}"
                 )
-            coords_list.append(pos["XYStage"])
-            row = pos["GridRow"]
-            col = pos["GridCol"]
-            row_max = row if row > row_max else row_max
-            col_max = col if col > col_max else col_max
+            xy_coords.append(stage_pos)
+            try:
+                rows.add(pos["GridRow"])
+                cols.add(pos["GridCol"])
+            except KeyError:
+                raise ValueError(
+                    f"Grid indices not available for position {idx}"
+                )
 
-        return coords_list, row_max + 1, col_max + 1
+        return xy_coords, len(rows), len(cols)
 
     def _get_pos_names(self):
         """Append a list of pos names in ascending order

diff --git a/iohub/mm_fov.py b/iohub/mm_fov.py
@@ -2,6 +2,8 @@
 
 from pathlib import Path
 
+from xarray import DataArray
+
 from iohub.fov import BaseFOV, BaseFOVMapping
 
 
@@ -10,6 +12,14 @@ def __init__(self, parent: MicroManagerFOVMapping, key: int) -> None:
         self._position = key
         self._parent = parent
 
+    def __repr__(self) -> str:
+        return (
+            f"Type: {type(self)}\n"
+            f"Parent: {self.parent}\n"
+            f"FOV key: {self._position}\n"
+            f"Data:\n"
+        ) + self.xdata.__repr__()
+
     @property
     def parent(self) -> MicroManagerFOVMapping:
         return self._parent
@@ -26,6 +36,10 @@ def zyx_scale(self) -> tuple[float, float, float]:
     def channel_names(self) -> list[str]:
         return self.parent.channel_names
 
+    @property
+    def xdata(self) -> DataArray:
+        raise NotImplementedError
+
     def frame_metadata(self, t: int, z: int, c: int) -> dict | None:
         """
         Return image plane metadata for a given camera frame.
@@ -49,22 +63,36 @@ def frame_metadata(self, t: int, z: int, c: int) -> dict | None:
 
 class MicroManagerFOVMapping(BaseFOVMapping):
     def __init__(self):
+        self._root: Path = None
         self._mm_meta: dict = None
         self._stage_positions: list[dict[str, str | float]] = []
         self.channel_names: list[str] = None
 
+    def __repr__(self) -> str:
+        return (f"Type: {type(self)}\nData:\n") + self.xdata.__repr__()
+
     @property
-    def mm_meta(self):
+    def root(self) -> Path:
+        """Root directory of the dataset."""
+        return self._root
+
+    @property
+    def micromanager_metadata(self) -> dict | None:
         return self._mm_meta
 
-    @mm_meta.setter
-    def mm_meta(self, value):
+    @micromanager_metadata.setter
+    def micromanager_metadata(self, value):
         if not isinstance(value, dict):
             raise TypeError(
                 f"Type of `mm_meta` should be `dict`, got `{type(value)}`."
             )
         self._mm_meta = value
 
+    @property
+    def micromanager_summary(self) -> dict | None:
+        """Micro-manager summary metadata."""
+        return self._mm_meta.get("Summary", None)
+
     @property
     def stage_positions(self):
         return self._stage_positions
@@ -117,3 +145,8 @@ def hcs_position_labels(self):
                     "'A1-Site_0', 'H12-Site_1', or '1-Pos000_000' "
                     f"Got labels {labels}"
                 )
+
+    @property
+    def zyx_scale(self) -> tuple[float, float, float]:
+        """ZXY pixel size in micrometers."""
+        raise NotImplementedError
diff --git a/iohub/mmstack.py b/iohub/mmstack.py
@@ -9,6 +9,7 @@
 import dask.array as da
 import numpy as np
 import zarr
+from natsort import natsorted
 from numpy.typing import ArrayLike
 from tifffile import TiffFile
 from xarray import DataArray
@@ -94,8 +95,8 @@ def __init__(self, data_path: StrOrBytesPath):
         super().__init__()
         data_path = Path(data_path)
         first_file = find_first_ome_tiff_in_mmstack(data_path)
-        self.root = first_file.parent
-        self.dirname = self.root.name
+        self._root = first_file.parent
+        self.dirname = self._root.name
         self._first_tif = TiffFile(first_file, is_mmstack=True)
         _logger.debug(f"Parsing {first_file} as MMStack.")
         with catch_warnings():
@@ -140,6 +141,8 @@ def _parse_data(self):
                 "smaller than the number of channels. "
                 f"Completing with fallback names: {self.channel_names}."
             )
+        # not all positions in the position list may have been acquired
+        xarr = xarr[: self.positions]
         xarr.coords["C"] = self.channel_names
         xset = xarr.to_dataset(dim="R")
         self._xdata = xset
@@ -150,7 +153,7 @@ def xdata(self):
         return self._xdata
 
     def __len__(self) -> int:
-        return self.positions
+        return len(self.xdata.keys())
 
     def __getitem__(self, key: str | int) -> MMOmeTiffFOV:
         key = _normalize_mm_pos_key(key)
@@ -218,15 +221,14 @@ def _set_mm_meta(self, mm_meta: dict) -> None:
             mm_version == "2.0.1 20220920"
             and self._mm_meta["Summary"]["Prefix"] == "raw_data"
         ):
-            file_names = set(
-                [(key[0], val[0]) for key, val in self.coord_map.items()]
-            )
+            files = natsorted(self.root.glob("*.ome.tif"))
+            self.positions = len(files)  # not all positions are saved
 
             if self._mm_meta["Summary"]["Positions"] > 1:
                 self._stage_positions = [None] * self.positions
 
-                for p_idx, file_name in file_names:
-                    site_idx = int(file_name.split("_")[-1].split("-")[0])
+                for p_idx, file_name in enumerate(files):
+                    site_idx = int(str(file_name).split("_")[-1].split("-")[0])
                     pos = self._simplify_stage_position(
                         self._mm_meta["Summary"]["StagePositions"][site_idx]
                     )
@@ -380,7 +382,6 @@ def _infer_image_meta(self) -> None:
 
     @property
     def zyx_scale(self) -> tuple[float, float, float]:
-        """ZXY pixel size in micrometers."""
         return (
             self._z_step_size,
             self._xy_pixel_size,