From 6fcb776f9d4e9d51252921161a8851106fa3edea Mon Sep 17 00:00:00 2001
From: Melissa Linkert <melissa@glencoesoftware.com>
Date: Wed, 26 Jul 2023 16:38:42 -0500
Subject: [PATCH 1/2] Add --split-planes option to write one plane per file

Fixes #107
---
 .../pyramid/PyramidFromDirectoryWriter.java   | 177 ++++++++++++++----
 .../pyramid/PyramidSeries.java                |   2 +-
 2 files changed, 142 insertions(+), 37 deletions(-)

diff --git a/src/main/java/com/glencoesoftware/pyramid/PyramidFromDirectoryWriter.java b/src/main/java/com/glencoesoftware/pyramid/PyramidFromDirectoryWriter.java
index 9665304..318614b 100755
--- a/src/main/java/com/glencoesoftware/pyramid/PyramidFromDirectoryWriter.java
+++ b/src/main/java/com/glencoesoftware/pyramid/PyramidFromDirectoryWriter.java
@@ -135,6 +135,7 @@ public class PyramidFromDirectoryWriter implements Callable<Void> {
   private List<PyramidSeries> series = new ArrayList<PyramidSeries>();
   private Map<String, TiffSaver> tiffSavers = new HashMap<String, TiffSaver>();
   boolean splitBySeries = false;
+  boolean splitByPlane = false;
 
   private ZarrGroup reader = null;
 
@@ -306,6 +307,20 @@ public void setSplitTIFFs(boolean split) {
     splitBySeries = split;
   }
 
+  /**
+   * Configure whether to write one OME-TIFF per plane.
+   *
+   * @param split true if output should be split into one OME-TIFF per plane
+   */
+  @Option(
+      names = "--split-planes",
+      description =
+        "Split output into one OME-TIFF file per plane"
+  )
+  public void setSplitSinglePlaneTIFFs(boolean split) {
+    splitByPlane = split;
+  }
+
   /**
    * Set the maximum number of workers to use for converting tiles.
    * Defaults to the number of detected CPUs.
@@ -407,6 +422,13 @@ public boolean getSplitTIFFs() {
     return splitBySeries;
   }
 
+  /**
+   * @return true if output will be split into one OME-TIFF per plane
+   */
+  public boolean getSplitSinglePlaneTIFFs() {
+    return splitByPlane;
+  }
+
   /**
    * @return maximum number of worker threads
    */
@@ -475,8 +497,9 @@ public Void call() throws Exception {
     setupLogger();
 
     // we could support this case later, just keeping it simple for now
-    if (splitBySeries && legacy) {
-      throw new IllegalArgumentException("--split not supported with --legacy");
+    if ((splitBySeries || splitByPlane) && legacy) {
+      throw new IllegalArgumentException(
+        "--split and --split-planes not supported with --legacy");
     }
 
     try {
@@ -1011,12 +1034,9 @@ else if (rgb) {
       }
 
       s.planeCount *= effectiveChannels;
-
       s.describePyramid(reader, metadata);
 
       metadata.setTiffDataIFD(new NonNegativeInteger(totalPlanes), s.index, 0);
-      metadata.setTiffDataPlaneCount(
-        new NonNegativeInteger(s.planeCount), s.index, 0);
 
       for (ResolutionDescriptor descriptor : s.resolutions) {
         LOG.info("Adding metadata for resolution: {}",
@@ -1049,22 +1069,39 @@ else if (rgb) {
       series.add(s);
       totalPlanes += s.planeCount;
 
-      if (splitBySeries) {
+      if (splitBySeries && !splitByPlane) {
         String basePath = getOutputPathPrefix();
         // append the series index and file extension
         basePath += "_s";
         seriesPaths.add(Paths.get(basePath + s.index + ".ome.tiff"));
         // generate one UUID per file
-        s.uuid = "urn:uuid:" + UUID.randomUUID().toString();
+        s.uuid.add("urn:uuid:" + UUID.randomUUID().toString());
+      }
+      else if (splitByPlane) {
+        String basePath = getOutputPathPrefix();
+        // append the series index and file extension
+        basePath += "_s";
+        basePath += s.index;
+        basePath += "_z";
+        for (int t=0; t<s.t; t++) {
+          for (int c=0; c<effectiveChannels; c++) {
+            for (int z=0; z<s.z; z++) {
+              seriesPaths.add(
+                Paths.get(basePath + z + "_c" + c + "_t" + t + ".ome.tiff"));
+              // generate one UUID per file
+              s.uuid.add("urn:uuid:" + UUID.randomUUID().toString());
+            }
+          }
+        }
       }
       else {
         seriesPaths.add(outputFilePath);
         // use the same UUID everywhere since we're only writing one file
         if (seriesIndex == 0) {
-          s.uuid = "urn:uuid:" + UUID.randomUUID().toString();
+          s.uuid.add("urn:uuid:" + UUID.randomUUID().toString());
         }
         else {
-          s.uuid = series.get(0).uuid;
+          s.uuid.add(series.get(0).uuid.get(0));
         }
       }
     }
@@ -1072,7 +1109,7 @@ else if (rgb) {
     populateTiffData();
     populateOriginalMetadata(service);
 
-    if (splitBySeries) {
+    if (splitBySeries || splitByPlane) {
       // splitting into separate OME-TIFFs results in a companion OME-XML file
       // the OME-TIFFs then use the BinaryOnly element to reference the OME-XML
       // for large plates in particular, this is useful as it reduces the
@@ -1137,7 +1174,19 @@ private TiffSaver createTiffSaver(RandomAccessOutputStream out, String file) {
     return w;
   }
 
-  private String getSeriesPathName(PyramidSeries s) {
+  private String getPathName(PyramidSeries s, int plane) {
+    if (splitByPlane) {
+      int index = 0;
+      for (int i=0; i<series.size(); i++) {
+        if (!series.get(i).equals(s)) {
+          index += series.get(i).uuid.size();
+        }
+        else {
+          break;
+        }
+      }
+      return seriesPaths.get(index + plane).toString();
+    }
     return seriesPaths.get(s.index).toString();
   }
 
@@ -1292,7 +1341,7 @@ private void writeIFDs() throws FormatException, IOException {
     long[][][] subs = new long[series.size()][][];
     for (PyramidSeries s : series) {
       StopWatch t1 = new Slf4JStopWatch("subifd-" + s.index);
-      String path = getSeriesPathName(s);
+      String path = getPathName(s, 0);
       try (RandomAccessOutputStream out = new RandomAccessOutputStream(path)) {
         out.order(s.littleEndian);
         out.seek(out.length());
@@ -1314,9 +1363,25 @@ private void writeIFDs() throws FormatException, IOException {
         else {
           subs[s.index] = new long[s.planeCount][s.numberOfResolutions - 1];
           for (int plane=0; plane<s.planeCount; plane++) {
-            for (int r=1; r<s.numberOfResolutions; r++) {
-              subs[s.index][plane][r - 1] = out.getFilePointer();
-              writeIFD(out, s, r, plane, r < s.numberOfResolutions - 1);
+            if (splitByPlane) {
+              String planePath = getPathName(s, plane);
+              try (RandomAccessOutputStream p =
+                new RandomAccessOutputStream(planePath))
+              {
+                p.order(s.littleEndian);
+                p.seek(p.length());
+                for (int r=1; r<s.numberOfResolutions; r++) {
+                  subs[s.index][plane][r - 1] = p.getFilePointer();
+                  writeIFD(p, s, r, plane, r < s.numberOfResolutions - 1);
+                }
+              }
+              tiffSavers.remove(planePath);
+            }
+            else {
+              for (int r=1; r<s.numberOfResolutions; r++) {
+                subs[s.index][plane][r - 1] = out.getFilePointer();
+                writeIFD(out, s, r, plane, r < s.numberOfResolutions - 1);
+              }
             }
           }
         }
@@ -1332,7 +1397,7 @@ private void writeIFDs() throws FormatException, IOException {
       firstIFDOffsets.clear();
       for (PyramidSeries s : series) {
         StopWatch t1 = new Slf4JStopWatch("fullResolution-" + s.index);
-        String path = getSeriesPathName(s);
+        String path = getPathName(s, 0);
         try (RandomAccessOutputStream out =
           new RandomAccessOutputStream(path))
         {
@@ -1345,14 +1410,33 @@ private void writeIFDs() throws FormatException, IOException {
 
           for (int plane=0; plane<s.planeCount; plane++) {
             s.ifds[0].get(plane).put(IFD.SUB_IFD, subs[s.index][plane]);
-            boolean overwrite = plane < s.planeCount - 1;
-            if (!splitBySeries) {
-              overwrite = overwrite || s.index < series.size() - 1;
+            // if splitting by plane, the first IFD is also the last in the file
+            if (!splitByPlane) {
+              boolean overwrite = plane < s.planeCount - 1;
+              if (!splitBySeries) {
+                overwrite = overwrite || s.index < series.size() - 1;
+              }
+              writeIFD(out, s, 0, plane, overwrite);
+            }
+            else {
+              String planePath = getPathName(s, plane);
+              try (RandomAccessOutputStream p =
+                new RandomAccessOutputStream(planePath))
+              {
+                p.order(s.littleEndian);
+                long offset = p.length();
+                p.seek(offset);
+                writeIFD(p, s, 0, plane, false);
+                p.seek(FIRST_IFD_OFFSET);
+                p.writeLong(offset);
+              }
+              tiffSavers.remove(planePath);
             }
-            writeIFD(out, s, 0, plane, overwrite);
           }
-          out.seek(FIRST_IFD_OFFSET);
-          out.writeLong(firstIFDOffsets.get(path));
+          if (!splitByPlane) {
+            out.seek(FIRST_IFD_OFFSET);
+            out.writeLong(firstIFDOffsets.get(path));
+          }
         }
         tiffSavers.remove(path);
         t1.stop();
@@ -1451,18 +1535,18 @@ else if (FormatTools.isSigned(s.pixelType)) {
     }
 
     // only write the OME-XML to the first full-resolution IFD
-    if (resolution == 0 && plane == 0) {
-      if (splitBySeries) {
+    if (resolution == 0 && (plane == 0 || splitByPlane)) {
+      if (splitBySeries || splitByPlane) {
         // if each series is in a separate OME-TIFF, store BinaryOnly OME-XML
         // that references the companion OME-XML file
 
-        String omexml = getBinaryOnlyOMEXML(s);
+        String omexml = getBinaryOnlyOMEXML(s, plane);
         ifd.put(IFD.IMAGE_DESCRIPTION, omexml);
       }
       else if (s.index == 0) {
         // if everything is in one OME-TIFF file, store the complete OME-XML
         try {
-          metadata.setUUID(s.uuid);
+          metadata.setUUID(s.uuid.get(0));
           OMEXMLService service = getService();
           String omexml = service.getOMEXML(metadata);
           ifd.put(IFD.IMAGE_DESCRIPTION, omexml);
@@ -1562,7 +1646,7 @@ private synchronized void writeToDisk(
     long[] byteCounts = ifd.getIFDLongArray(IFD.TILE_BYTE_COUNTS);
     byteCounts[tileIndex] = (long) realTile.length;
 
-    String file = getSeriesPathName(s);
+    String file = getPathName(s, imageNumber);
     try (RandomAccessOutputStream out = new RandomAccessOutputStream(file)) {
       out.seek(out.length());
       offsets[tileIndex] = out.getFilePointer();
@@ -1601,7 +1685,7 @@ private void writeIFD(
     PyramidSeries s, int resolution, int plane, boolean overwrite)
     throws FormatException, IOException
   {
-    TiffSaver writer = createTiffSaver(outStream, getSeriesPathName(s));
+    TiffSaver writer = createTiffSaver(outStream, getPathName(s, plane));
     int ifdSize = getIFDSize(s.ifds[resolution].get(plane));
     long offsetPointer = outStream.getFilePointer() + ifdSize;
     writer.writeIFD(s.ifds[resolution].get(plane), 0);
@@ -1646,11 +1730,31 @@ private void printVersion() {
 
   private void populateTiffData() {
     for (PyramidSeries s : series) {
-      metadata.setUUIDFileName(
-        Paths.get(getSeriesPathName(s)).getFileName().toString(), s.index, 0);
-      metadata.setUUIDValue(s.uuid, s.index, 0);
-      if (splitBySeries) {
-        metadata.setTiffDataIFD(new NonNegativeInteger(0), s.index, 0);
+      for (int tiffData=0; tiffData<s.uuid.size(); tiffData++) {
+        String path = getPathName(s, tiffData);
+        metadata.setUUIDFileName(
+          Paths.get(path).getFileName().toString(), s.index, tiffData);
+        metadata.setUUIDValue(s.uuid.get(tiffData), s.index, tiffData);
+
+        if (splitByPlane) {
+          int[] zct = s.getZCTCoords(tiffData);
+          metadata.setTiffDataFirstZ(
+            new NonNegativeInteger(zct[0]), s.index, tiffData);
+          metadata.setTiffDataFirstC(
+            new NonNegativeInteger(zct[1]), s.index, tiffData);
+          metadata.setTiffDataFirstT(
+            new NonNegativeInteger(zct[2]), s.index, tiffData);
+          metadata.setTiffDataPlaneCount(
+            new NonNegativeInteger(1), s.index, tiffData);
+        }
+        else {
+          metadata.setTiffDataPlaneCount(
+            new NonNegativeInteger(s.planeCount), s.index, tiffData);
+        }
+
+        if (splitBySeries || splitByPlane) {
+          metadata.setTiffDataIFD(new NonNegativeInteger(0), s.index, tiffData);
+        }
       }
     }
   }
@@ -1662,9 +1766,10 @@ private void populateTiffData() {
    * to have BinaryOnly OME-XML that references the companion OME-XML file.
    *
    * @param s pyramid series for UUID retrieval
+   * @param plane plane index for UUID retrieval
    * @return corresponding BinaryOnly OME-XML string
    */
-  private String getBinaryOnlyOMEXML(PyramidSeries s) {
+  private String getBinaryOnlyOMEXML(PyramidSeries s, int plane) {
     try {
       OMEXMLService service = getService();
       if (binaryOnly == null) {
@@ -1675,11 +1780,11 @@ private String getBinaryOnlyOMEXML(PyramidSeries s) {
           companion.getName(companion.getNameCount() - 1).toString());
         binaryOnly.setBinaryOnlyUUID(companionUUID);
       }
-      binaryOnly.setUUID(s.uuid);
+      binaryOnly.setUUID(s.uuid.get(plane));
       return service.getOMEXML(binaryOnly);
     }
     catch (DependencyException | ServiceException e) {
-      LOG.warn("Could not create OME-XML for " + getSeriesPathName(s), e);
+      LOG.warn("Could not create OME-XML for " + getPathName(s, plane), e);
     }
     return null;
   }
diff --git a/src/main/java/com/glencoesoftware/pyramid/PyramidSeries.java b/src/main/java/com/glencoesoftware/pyramid/PyramidSeries.java
index 6cc3038..3cfd26e 100644
--- a/src/main/java/com/glencoesoftware/pyramid/PyramidSeries.java
+++ b/src/main/java/com/glencoesoftware/pyramid/PyramidSeries.java
@@ -28,7 +28,7 @@ public class PyramidSeries {
   /** Path to series. */
   String path;
 
-  String uuid;
+  List<String> uuid = new ArrayList<String>();
 
   int index = -1;
 

From 697cf3a69c3ea5bf4e24e93e8e2f695c94d0f26d Mon Sep 17 00:00:00 2001
From: Melissa Linkert <melissa@glencoesoftware.com>
Date: Tue, 1 Aug 2023 14:04:59 -0500
Subject: [PATCH 2/2] Add split plane test cases

---
 .../raw2ometiff/test/ConversionTest.java      | 81 ++++++++++++++++++-
 1 file changed, 77 insertions(+), 4 deletions(-)

diff --git a/src/test/java/com/glencoesoftware/raw2ometiff/test/ConversionTest.java b/src/test/java/com/glencoesoftware/raw2ometiff/test/ConversionTest.java
index 7ff9fbd..ce1b99e 100644
--- a/src/test/java/com/glencoesoftware/raw2ometiff/test/ConversionTest.java
+++ b/src/test/java/com/glencoesoftware/raw2ometiff/test/ConversionTest.java
@@ -98,17 +98,20 @@ void assertBioFormats2Raw(String...additionalArgs) throws IOException {
    * @param additionalArgs CLI arguments as needed beyond "input output"
    */
   void assertTool(String...additionalArgs) throws IOException {
-    assertTool(0, additionalArgs);
+    assertTool(0, 0, additionalArgs);
   }
 
   /**
    * Run the PyramidFromDirectoryWriter main method and check for success or
    * failure.
    *
+   * @param seriesCount number of series to expect, if splitting
    * @param fileCount number of files to expect, if splitting
    * @param additionalArgs CLI arguments as needed beyond "input output"
    */
-  void assertTool(int fileCount, String...additionalArgs) throws IOException {
+  void assertTool(int seriesCount, int fileCount, String...additionalArgs)
+    throws IOException
+  {
     List<String> args = new ArrayList<String>();
     for (String arg : additionalArgs) {
       args.add(arg);
@@ -122,7 +125,7 @@ void assertTool(int fileCount, String...additionalArgs) throws IOException {
       if (fileCount == 0) {
         Assert.assertTrue(Files.exists(outputOmeTiff));
       }
-      else {
+      else if (seriesCount == fileCount) {
         String prefix = output.resolve("output").toString();
         for (int i=0; i<fileCount; i++) {
           Assert.assertTrue(Files.exists(
@@ -132,6 +135,14 @@ void assertTool(int fileCount, String...additionalArgs) throws IOException {
             Paths.get(prefix + "_s" +
             fileCount + ".ome.tiff")));
       }
+      else {
+        String prefix = output.resolve("output").toString();
+        for (int i=0; i<seriesCount; i++) {
+          Assert.assertTrue(Files.exists(
+            Paths.get(prefix + "_s" + i + "_z0_c0_t0.ome.tiff")));
+        }
+        Assert.assertFalse(Files.exists(Paths.get(prefix + "_s0.ome.tiff")));
+      }
     }
     catch (RuntimeException rt) {
       throw rt;
@@ -629,7 +640,7 @@ public void testSplitFiles() throws Exception {
     input =
       fake("plateRows", "2", "plateCols", "3", "fields", "4", "sizeC", "3");
     assertBioFormats2Raw();
-    assertTool(24, "--split");
+    assertTool(24, 24, "--split");
 
     try (ImageReader reader = new ImageReader()) {
       ServiceFactory sf = new ServiceFactory();
@@ -651,6 +662,68 @@ public void testSplitFiles() throws Exception {
     }
   }
 
+  /**
+   * Test splitting planes into separate files.
+   */
+  @Test
+  public void testSplitPlanes() throws Exception {
+    input =
+      fake("plateRows", "2", "plateCols", "3", "fields", "4", "sizeC", "3");
+    assertBioFormats2Raw();
+    assertTool(24, 72, "--split-planes");
+
+    try (ImageReader reader = new ImageReader()) {
+      ServiceFactory sf = new ServiceFactory();
+      OMEXMLService xmlService = sf.getInstance(OMEXMLService.class);
+      OMEXMLMetadata metadata = xmlService.createOMEXMLMetadata();
+      reader.setMetadataStore(metadata);
+      reader.setFlattenedResolutions(false);
+
+      // --split-planes should always produce a companion OME-XML file
+      // with BinaryOnly OME-TIFFs
+      reader.setId(output.resolve("output").toString() + ".companion.ome");
+
+      Assert.assertEquals(reader.getUsedFiles().length, 73);
+      Assert.assertEquals(reader.getSeriesCount(), 24);
+      Assert.assertEquals(1, metadata.getPlateCount());
+      Assert.assertEquals(24, metadata.getImageCount());
+
+      iteratePixels(reader);
+    }
+  }
+
+  /**
+   * Test what happens when the split-by-series and split-by-plane
+   * options are used together.
+   */
+  @Test
+  public void testBothSplitOptions() throws Exception {
+    input =
+      fake("plateRows", "2", "plateCols", "3", "fields", "4", "sizeC", "3");
+    assertBioFormats2Raw();
+    // expect --split-planes to take precedence
+    assertTool(24, 72, "--split", "--split-planes");
+
+    try (ImageReader reader = new ImageReader()) {
+      ServiceFactory sf = new ServiceFactory();
+      OMEXMLService xmlService = sf.getInstance(OMEXMLService.class);
+      OMEXMLMetadata metadata = xmlService.createOMEXMLMetadata();
+      reader.setMetadataStore(metadata);
+      reader.setFlattenedResolutions(false);
+
+      // --split-planes should always produce a companion OME-XML file
+      // with BinaryOnly OME-TIFFs
+      reader.setId(output.resolve("output").toString() + ".companion.ome");
+
+      Assert.assertEquals(reader.getUsedFiles().length, 73);
+      Assert.assertEquals(reader.getSeriesCount(), 24);
+      Assert.assertEquals(1, metadata.getPlateCount());
+      Assert.assertEquals(24, metadata.getImageCount());
+
+      iteratePixels(reader);
+    }
+  }
+
   /**
    * Test RGB with multiple channels using API instead of command line.
    */