Update tensorstore zarr writer to write sequences

.github/workflows/publish_pypi.yml

@@ -54,17 +54,16 @@ jobs:
                           bash ci-utils/install_prereq_linux.sh &&
                           mkdir -p /tmp/bfiocpp_bld &&
                           cp -r local_install /tmp/bfiocpp_bld
-          CIBW_BEFORE_ALL_LINUX: yum -y install wget &&
-                                   wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 &&
-                                   tar -xjf nasm-2.15.05.tar.bz2 &&
-                                   cd nasm-2.15.05 &&
-                                   ./configure && 
-                                   make && 
-                                   make install &&
-                                   cd .. &&
-                                   bash ci-utils/install_prereq_linux.sh &&
-                                   mkdir -p /tmp/bfiocpp_bld &&
-                                   cp -r local_install /tmp/bfiocpp_bld
+          CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 -o nasm-2.15.05.tar.bz2 &&
+                          tar -xjf nasm-2.15.05.tar.bz2 &&
+                          cd nasm-2.15.05 &&
+                          ./configure && 
+                          make && 
+                          make install &&
+                          cd .. &&
+                          bash ci-utils/install_prereq_linux.sh &&
+                          mkdir -p /tmp/bfiocpp_bld &&
+                          cp -r local_install /tmp/bfiocpp_bld
           CIBW_BEFORE_ALL_WINDOWS: ci-utils\install_prereq_win.bat &&
                                    xcopy /E /I /y local_install C:\TEMP\bfiocpp_bld\local_install 
           CIBW_ENVIRONMENT_MACOS: REPAIR_LIBRARY_PATH="/tmp/bfiocpp_bld/local_install/lib:/tmp/bfiocpp_bld/local_install/lib64" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="/tmp/bfiocpp_bld/local_install"
@@ -73,8 +72,11 @@ jobs:
           CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\bfiocpp\\bin;$PATH" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="C:\\TEMP\\bfiocpp_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja"
           CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
-          CIBW_BEFORE_TEST_LINUX: yum -y install maven java
-          CIBW_TEST_REQUIRES: bfio requests numpy ome_zarr
+          CIBW_BEFORE_TEST_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo &&
+                                  sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo &&
+                                  sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && 
+                                  yum -y install maven java
+          CIBW_TEST_REQUIRES: bfio requests numpy==1.24.0 ome_zarr
           CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v
 
       - name: Install Dependencies

.github/workflows/wheel_build.yml

@@ -51,9 +51,8 @@ jobs:
           CIBW_BEFORE_ALL_MACOS: brew install nasm &&
                           bash ci-utils/install_prereq_linux.sh &&
                           mkdir -p /tmp/bfiocpp_bld &&
-                          cp -r local_install /tmp/bfiocpp_bld
-          CIBW_BEFORE_ALL_LINUX: yum -y install wget &&
-                                   wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 &&
+                          cp -r local_install /tmp/bfiocpp_bld              
+          CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 -o nasm-2.15.05.tar.bz2 &&
                                    tar -xjf nasm-2.15.05.tar.bz2 &&
                                    cd nasm-2.15.05 &&
                                    ./configure && 
@@ -71,8 +70,11 @@ jobs:
           CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\bfiocpp\\bin;$PATH" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="C:\\TEMP\\bfiocpp_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja"
           CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
-          CIBW_BEFORE_TEST_LINUX: yum -y install maven java
-          CIBW_TEST_REQUIRES: bfio requests numpy ome_zarr
+          CIBW_BEFORE_TEST_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo &&
+                                  sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo &&
+                                  sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && 
+                                  yum -y install maven java
+          CIBW_TEST_REQUIRES: bfio requests numpy==1.24.0 ome_zarr
           CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v
 
       - name: Upload Artifact

src/cpp/interface/interface.cpp

@@ -4,7 +4,7 @@
 #include <pybind11/numpy.h>
 #include <tuple>
 #include "../reader/tsreader.h"
-#include "../reader/sequence.h"
+#include "../utilities/sequence.h"
 #include "../utilities/utilities.h"
 #include "../writer/tswriter.h"
 
@@ -122,6 +122,6 @@ PYBIND11_MODULE(libbfiocpp, m) {
 
     // Writer class
     py::class_<bfiocpp::TsWriterCPP, std::shared_ptr<bfiocpp::TsWriterCPP>>(m, "TsWriterCPP") 
-    .def(py::init<const std::string&, const std::vector<std::int64_t>&, const std::vector<std::int64_t>&, const std::string&>())
+    .def(py::init<const std::string&, const std::vector<std::int64_t>&, const std::vector<std::int64_t>&, const std::string&, const std::string&>())
     .def("write_image_data", &bfiocpp::TsWriterCPP::WriteImageData);
 }

src/cpp/reader/tsreader.h

@@ -9,7 +9,7 @@
 #include <optional>
 #include <unordered_map>
 #include "tensorstore/tensorstore.h"
-#include "sequence.h"
+#include "../utilities/sequence.h"
 using image_data = std::variant<std::vector<std::uint8_t>,
                                 std::vector<std::uint16_t>, 
                                 std::vector<std::uint32_t>, 
@@ -58,6 +58,7 @@ class TsReaderCPP{
                     _num_tsteps;
     std::uint16_t _data_type_code;
     FileType _file_type;
+
     std::optional<int>_z_index, _c_index, _t_index;
 
     tensorstore::TensorStore<void, -1, tensorstore::ReadWriteMode::dynamic> source;

src/cpp/writer/tswriter.cpp

@@ -2,6 +2,7 @@
 
 #include "tensorstore/array.h"
 #include "tensorstore/open.h"
+#include "tensorstore/index_space/dim_expression.h"
 
 #include "tswriter.h"
 #include "../utilities/utilities.h"
@@ -10,126 +11,177 @@
 namespace bfiocpp {
 
 TsWriterCPP::TsWriterCPP(
-    const std::string& fname, 
-    const std::vector<std::int64_t>& image_shape, 
+    const std::string& fname,
+    const std::vector<std::int64_t>& image_shape,
     const std::vector<std::int64_t>& chunk_shape,
-    const std::string& dtype_str
-  ): _filename(fname), 
-     _image_shape(image_shape), 
-     _chunk_shape(chunk_shape),  
+    const std::string& dtype_str,
+    const std::string& dimension_order
+  ): _filename(fname),
+     _image_shape(image_shape),
+     _chunk_shape(chunk_shape),
      _dtype_code(GetDataTypeCode(dtype_str)) {
-
-  TENSORSTORE_CHECK_OK_AND_ASSIGN(_source, tensorstore::Open(
-                            GetZarrSpecToWrite(_filename, _image_shape, _chunk_shape, GetEncodedType(_dtype_code)),
-                            tensorstore::OpenMode::create |
-                            tensorstore::OpenMode::delete_existing,
-                            tensorstore::ReadWriteMode::write).result());
-}
 
+
+    TENSORSTORE_CHECK_OK_AND_ASSIGN(_source, tensorstore::Open(
+        GetZarrSpecToWrite(_filename, _image_shape, _chunk_shape, GetEncodedType(_dtype_code)),
+        tensorstore::OpenMode::create |
+        tensorstore::OpenMode::delete_existing,
+        tensorstore::ReadWriteMode::write).result()
+    );
 
-void TsWriterCPP::WriteImageData(py::array& py_image) {
+    auto position = dimension_order.find("X");
+    if (position != std::string::npos) _x_index = position;
 
-  // use switch instead of template to avoid creating functions for each datatype
-  switch(_dtype_code)
-  {
-    case (1): {
-        auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::uint8_t, 1>().data(0), _image_shape, tensorstore::c_order);
+    position = dimension_order.find("Y");
+    if (position != std::string::npos) _y_index = position;
 
-        // Write data array to TensorStore
-        auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result();
+    position = dimension_order.find("C");
+    if (position != std::string::npos) _c_index.emplace(position);
 
-        if (!write_result.ok()) {
-            std::cerr << "Error writing image: " << write_result.status() << std::endl;
-        }
+    position = dimension_order.find("T");
+    if (position != std::string::npos) _t_index.emplace(position);
 
-        break;
-    }
-    case (2): {
-        auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::uint16_t, 1>().data(0), _image_shape, tensorstore::c_order);
+    position = dimension_order.find("Z");
+    if (position != std::string::npos) _z_index.emplace(position);
+}
 
-        auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result();
-        if (!write_result.ok()) {
-            std::cerr << "Error writing image: " << write_result.status() << std::endl;
-        }
-        break;  
-    }  
-    case (4): {
-        auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::uint32_t, 1>().data(0), _image_shape, tensorstore::c_order);
-
-        auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result();
-        if (!write_result.ok()) {
-            std::cerr << "Error writing image: " << write_result.status() << std::endl;
-        }
-        break;
+void TsWriterCPP::WriteImageData(
+    py::array& py_image,
+    const Seq& rows,
+    const Seq& cols,
+    const std::optional<Seq>& layers,
+    const std::optional<Seq>& channels,
+    const std::optional<Seq>& tsteps) {
+
+    std::vector<std::int64_t> shape;
+
+    auto output_transform = tensorstore::IdentityTransform(_source.domain());
+
+    if (_t_index.has_value() && tsteps.has_value()) {
+        output_transform = (std::move(output_transform) | tensorstore::Dims(_t_index.value()).ClosedInterval(tsteps.value().Start(), tsteps.value().Stop())).value();
+        shape.emplace_back(tsteps.value().Stop() - tsteps.value().Start()+1);
     }
-    case (8): {
-        auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::uint64_t, 1>().data(0), _image_shape, tensorstore::c_order);
 
-        auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result();
-        if (!write_result.ok()) {
-            std::cerr << "Error writing image: " << write_result.status() << std::endl;
-        }
-        break;
+    if (_c_index.has_value() && channels.has_value()) {
+        output_transform = (std::move(output_transform) | tensorstore::Dims(_c_index.value()).ClosedInterval(channels.value().Start(), channels.value().Stop())).value();
+        shape.emplace_back(channels.value().Stop() - channels.value().Start()+1);
     }
-    case (16): {
-        auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::int8_t, 1>().data(0), _image_shape, tensorstore::c_order);
 
-        auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result();
-        if (!write_result.ok()) {
-            std::cerr << "Error writing image: " << write_result.status() << std::endl;
-        }
-        break;
+    if (_z_index.has_value() && layers.has_value()) {
+        output_transform = (std::move(output_transform) | tensorstore::Dims(_z_index.value()).ClosedInterval(layers.value().Start(), layers.value().Stop())).value();
+        shape.emplace_back(layers.value().Stop() - layers.value().Start()+1);
     }
-    case (32): {
-        auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::int16_t, 1>().data(0), _image_shape, tensorstore::c_order);
 
-        auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result();
-        if (!write_result.ok()) {
-            std::cerr << "Error writing image: " << write_result.status() << std::endl;
+    output_transform = (std::move(output_transform) | tensorstore::Dims(_y_index).ClosedInterval(rows.Start(), rows.Stop()) |
+                                                      tensorstore::Dims(_x_index).ClosedInterval(cols.Start(), cols.Stop())).value();
+
+
+
+    shape.emplace_back(rows.Stop() - rows.Start()+1);
+    shape.emplace_back(cols.Stop() - cols.Start()+1);
+
+    // use switch instead of template to avoid creating functions for each datatype
+    switch(_dtype_code)
+    {
+        case (1): {
+            auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::uint8_t, 1>().data(0), shape, tensorstore::c_order);
+
+            // Write data array to TensorStore
+            auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result();
+
+            if (!write_result.ok()) {
+                std::cerr << "Error writing image: " << write_result.status() << std::endl;
+            }
+
+            break;
         }
-        break;   
-    } 
-    case (64): {
-        auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::int32_t, 1>().data(0), _image_shape, tensorstore::c_order);
-
-        auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result();
-        if (!write_result.ok()) {
-            std::cerr << "Error writing image: " << write_result.status() << std::endl;
+        case (2): {
+            auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::uint16_t, 1>().data(0), shape, tensorstore::c_order);
+
+            auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result();
+            if (!write_result.ok()) {
+                std::cerr << "Error writing image: " << write_result.status() << std::endl;
+            }
+            break;
         }
-        break;
-    }
-    case (128): {
-        auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::int64_t, 1>().data(0), _image_shape, tensorstore::c_order);
-
-        // Write data array to TensorStore
-        auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result();
-        if (!write_result.ok()) {
-            std::cerr << "Error writing image: " << write_result.status() << std::endl;
+        case (4): {
+            auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::uint32_t, 1>().data(0), shape, tensorstore::c_order);
+
+            auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result();
+            if (!write_result.ok()) {
+                std::cerr << "Error writing image: " << write_result.status() << std::endl;
+            }
+            break;
         }
-        break;
-    }
-    case (256): {
-        auto data_array = tensorstore::Array(py_image.mutable_unchecked<float, 1>().data(0), _image_shape, tensorstore::c_order);
-
-        auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result();
-        if (!write_result.ok()) {
-            std::cerr << "Error writing image: " << write_result.status() << std::endl;
+        case (8): {
+            auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::uint64_t, 1>().data(0), shape, tensorstore::c_order);
+
+            auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result();
+            if (!write_result.ok()) {
+                std::cerr << "Error writing image: " << write_result.status() << std::endl;
+            }
+            break;
         }
-        break;
-    }
-    case (512): {
-        auto data_array = tensorstore::Array(py_image.mutable_unchecked<double, 1>().data(0), _image_shape, tensorstore::c_order);
-
-        auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result();
-        if (!write_result.ok()) {
-            std::cerr << "Error writing image: " << write_result.status() << std::endl;
+        case (16): {
+            auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::int8_t, 1>().data(0), shape, tensorstore::c_order);
+
+            auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result();
+            if (!write_result.ok()) {
+                std::cerr << "Error writing image: " << write_result.status() << std::endl;
+            }
+            break;
+        }
+        case (32): {
+            auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::int16_t, 1>().data(0), shape, tensorstore::c_order);
+
+            auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result();
+            if (!write_result.ok()) {
+                std::cerr << "Error writing image: " << write_result.status() << std::endl;
+            }
+            break;
+        }
+        case (64): {
+            auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::int32_t, 1>().data(0), shape, tensorstore::c_order);
+
+            auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result();
+            if (!write_result.ok()) {
+                std::cerr << "Error writing image: " << write_result.status() << std::endl;
+            }
+            break;
+        }
+        case (128): {
+            auto data_array = tensorstore::Array(py_image.mutable_unchecked<std::int64_t, 1>().data(0), shape, tensorstore::c_order);
+
+            // Write data array to TensorStore
+            auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result();
+            if (!write_result.ok()) {
+                std::cerr << "Error writing image: " << write_result.status() << std::endl;
+            }
+            break;
+        }
+        case (256): {
+            auto data_array = tensorstore::Array(py_image.mutable_unchecked<float, 1>().data(0), shape, tensorstore::c_order);
+
+            auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result();
+            if (!write_result.ok()) {
+                std::cerr << "Error writing image: " << write_result.status() << std::endl;
+            }
+            break;
+        }
+        case (512): {
+            auto data_array = tensorstore::Array(py_image.mutable_unchecked<double, 1>().data(0), shape, tensorstore::c_order);
+
+            auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result();
+            if (!write_result.ok()) {
+                std::cerr << "Error writing image: " << write_result.status() << std::endl;
+            }
+            break;
+        }
+        default: {
+            // should not be reached
+            std::cerr << "Error writing image: unsupported data type" << std::endl;
         }
-        break;
-    }
-    default: {
-        // should not be reached
-        std::cerr << "Error writing image: unsupported data type" << std::endl;
     }
-  }
-  }
-} 
+}
+
+} // end ns bfiocpp