From 97bc1c78dd6ebc155592e6a96709a458623bdb5c Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 2 Jul 2024 07:45:23 -0600 Subject: [PATCH 01/22] Update tensorstore zarr writer to writer sequences --- src/cpp/writer/tswriter.cpp | 207 ++++++++++++++++++--------------- src/cpp/writer/tswriter.h | 22 +++- src/python/bfiocpp/tswriter.py | 4 +- tests/test_write.py | 13 ++- 4 files changed, 140 insertions(+), 106 deletions(-) diff --git a/src/cpp/writer/tswriter.cpp b/src/cpp/writer/tswriter.cpp index ce3e234..1b59049 100644 --- a/src/cpp/writer/tswriter.cpp +++ b/src/cpp/writer/tswriter.cpp @@ -2,6 +2,7 @@ #include "tensorstore/array.h" #include "tensorstore/open.h" +#include "tensorstore/index_space/dim_expression.h" #include "tswriter.h" #include "../utilities/utilities.h" @@ -18,118 +19,134 @@ TsWriterCPP::TsWriterCPP( _image_shape(image_shape), _chunk_shape(chunk_shape), _dtype_code(GetDataTypeCode(dtype_str)) { - - TENSORSTORE_CHECK_OK_AND_ASSIGN(_source, tensorstore::Open( - GetZarrSpecToWrite(_filename, _image_shape, _chunk_shape, GetEncodedType(_dtype_code)), - tensorstore::OpenMode::create | - tensorstore::OpenMode::delete_existing, - tensorstore::ReadWriteMode::write).result()); + + + TENSORSTORE_CHECK_OK_AND_ASSIGN(_source, tensorstore::Open( + GetZarrSpecToWrite(_filename, _image_shape, _chunk_shape, GetEncodedType(_dtype_code)), + tensorstore::OpenMode::create | + tensorstore::OpenMode::delete_existing, + tensorstore::ReadWriteMode::write).result() + ); } -void TsWriterCPP::WriteImageData(py::array& py_image) { +void TsWriterCPP::WriteImageData( + py::array& py_image, + const Seq& rows, + const Seq& cols, + const Seq& layers, + const Seq& channels, + const Seq& tsteps) { - // use switch instead of template to avoid creating functions for each datatype - switch(_dtype_code) - { - case (1): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + tensorstore::IndexTransform<> output_transform = tensorstore::IdentityTransform(_source.domain()); - // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); + output_transform = (std::move(output_transform) | tensorstore::Dims(_t_index).ClosedInterval(tsteps.Start(), tsteps.Stop())).value(); + output_transform = (std::move(output_transform) | tensorstore::Dims(_c_index).ClosedInterval(channels.Start(), channels.Stop())).value(); + output_transform = (std::move(output_transform) | tensorstore::Dims(_z_index).ClosedInterval(layers.Start(), layers.Stop())).value(); + output_transform = (std::move(output_transform) | tensorstore::Dims(_y_index).ClosedInterval(rows.Start(), rows.Stop()) | + tensorstore::Dims(_x_index).ClosedInterval(cols.Start(), cols.Stop())).value(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; - } + // use switch instead of template to avoid creating functions for each datatype + switch(_dtype_code) + { + case (1): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - break; - } - case (2): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + break; } - break; - } - case (4): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + case (2): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; + } + case (4): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; } - break; - } - case (8): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + case (8): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; } - break; - } - case (16): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + case (16): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; } - break; - } - case (32): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + case (32): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; + } + case (64): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; } - break; - } - case (64): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + case (128): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; } - break; - } - case (128): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - - // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + case (256): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; } - break; - } - case (256): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + case (512): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; } - break; - } - case (512): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + default: { + // should not be reached + std::cerr << "Error writing image: unsupported data type" << std::endl; } - break; } - default: { - // should not be reached - std::cerr << "Error writing image: unsupported data type" << std::endl; - } - } } } diff --git a/src/cpp/writer/tswriter.h b/src/cpp/writer/tswriter.h index c245ff2..63cc7b4 100644 --- a/src/cpp/writer/tswriter.h +++ b/src/cpp/writer/tswriter.h @@ -12,14 +12,30 @@ namespace bfiocpp{ class TsWriterCPP{ public: - TsWriterCPP(const std::string& fname, const std::vector& image_shape, const std::vector& chunk_shape, const std::string& dtype); - - void WriteImageData(py::array& py_image); + TsWriterCPP( + const std::string& fname, + const std::vector& image_shape, + const std::vector& chunk_shape, + const std::string& dtype); + + void WriteImageData( + py::array& py_image, + const Seq& rows, + const Seq& cols, + const Seq& layers, + const Seq& channels, + const Seq& tsteps); private: std::string _filename; std::vector _image_shape, _chunk_shape; + + static constexpr int _t_index = 0, + _c_index = 1, + _z_index = 2, + _y_index = 3, + _x_index = 4; uint16_t _dtype_code; diff --git a/src/python/bfiocpp/tswriter.py b/src/python/bfiocpp/tswriter.py index 261416c..f9f6d84 100644 --- a/src/python/bfiocpp/tswriter.py +++ b/src/python/bfiocpp/tswriter.py @@ -16,7 +16,7 @@ def __init__( file_name, image_shape, chunk_shape, str(dtype) ) - def write_image_data(self, image_data: np.ndarray): + def write_image_data(self, image_data: np.ndarray, rows: int, cols: int, layers: int, channels: int, tsteps: int): """Write image data to file image_data: 5d numpy array containing image data @@ -27,7 +27,7 @@ def write_image_data(self, image_data: np.ndarray): raise ValueError("Image data must be a 5d numpy array") try: - self._image_writer.write_image_data(image_data.flatten()) + self._image_writer.write_image_data(image_data.flatten(), rows, cols, layers, channels, tsteps) except Exception as e: raise RuntimeError(f"Error writing image data: {e.what}") diff --git a/tests/test_write.py b/tests/test_write.py index 00ea27c..de4560a 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -77,8 +77,14 @@ def test_write_zarr_2d(self): # Use the temporary directory test_file_path = os.path.join(dir, 'out/test.ome.zarr') + rows = Seq(0, br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(0, 0, 1) + tsteps = Seq(0, 0, 1) + bw = TSWriter(test_file_path, tmp.shape, tmp.shape, str(tmp.dtype)) - bw.write_image_data(tmp) + bw.write_image_data(tmp, rows, cols, layers, channels, tsteps) bw.close() br = TSReader( @@ -87,11 +93,6 @@ def test_write_zarr_2d(self): "", ) - rows = Seq(0, br._Y - 1, 1) - cols = Seq(0, br._X - 1, 1) - layers = Seq(0, 0, 1) - channels = Seq(0, 0, 1) - tsteps = Seq(0, 0, 1) tmp = br.data(rows, cols, layers, channels, tsteps) assert tmp.dtype == np.uint8 From 8c6fb5b3d101dc19fb4ae27484e26e1e0d2ef913 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 2 Jul 2024 07:48:42 -0600 Subject: [PATCH 02/22] Formatting --- src/python/bfiocpp/tswriter.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/python/bfiocpp/tswriter.py b/src/python/bfiocpp/tswriter.py index f9f6d84..6ac2c83 100644 --- a/src/python/bfiocpp/tswriter.py +++ b/src/python/bfiocpp/tswriter.py @@ -16,7 +16,15 @@ def __init__( file_name, image_shape, chunk_shape, str(dtype) ) - def write_image_data(self, image_data: np.ndarray, rows: int, cols: int, layers: int, channels: int, tsteps: int): + def write_image_data( + self, + image_data: np.ndarray, + rows: int, + cols: int, + layers: int, + channels: int, + tsteps: int, + ): """Write image data to file image_data: 5d numpy array containing image data @@ -27,7 +35,9 @@ def write_image_data(self, image_data: np.ndarray, rows: int, cols: int, layers raise ValueError("Image data must be a 5d numpy array") try: - self._image_writer.write_image_data(image_data.flatten(), rows, cols, layers, channels, tsteps) + self._image_writer.write_image_data( + image_data.flatten(), rows, cols, layers, channels, tsteps + ) except Exception as e: raise RuntimeError(f"Error writing image data: {e.what}") From ebffd4770e3bdef01e00118419c5152eef7452b4 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Mon, 8 Jul 2024 14:14:48 -0600 Subject: [PATCH 03/22] Update tensorstore zarr writer to use sequences --- src/cpp/writer/tswriter.cpp | 79 +++++++++++++++++++++++++++---------- src/cpp/writer/tswriter.h | 24 ++++++----- 2 files changed, 74 insertions(+), 29 deletions(-) diff --git a/src/cpp/writer/tswriter.cpp b/src/cpp/writer/tswriter.cpp index 1b59049..dfbfe19 100644 --- a/src/cpp/writer/tswriter.cpp +++ b/src/cpp/writer/tswriter.cpp @@ -38,13 +38,7 @@ void TsWriterCPP::WriteImageData( const Seq& channels, const Seq& tsteps) { - tensorstore::IndexTransform<> output_transform = tensorstore::IdentityTransform(_source.domain()); - - output_transform = (std::move(output_transform) | tensorstore::Dims(_t_index).ClosedInterval(tsteps.Start(), tsteps.Stop())).value(); - output_transform = (std::move(output_transform) | tensorstore::Dims(_c_index).ClosedInterval(channels.Start(), channels.Stop())).value(); - output_transform = (std::move(output_transform) | tensorstore::Dims(_z_index).ClosedInterval(layers.Start(), layers.Stop())).value(); - output_transform = (std::move(output_transform) | tensorstore::Dims(_y_index).ClosedInterval(rows.Start(), rows.Stop()) | - tensorstore::Dims(_x_index).ClosedInterval(cols.Start(), cols.Stop())).value(); + SetOutputTransform(rows, cols, layers, channels, tsteps); // use switch instead of template to avoid creating functions for each datatype switch(_dtype_code) @@ -53,7 +47,7 @@ void TsWriterCPP::WriteImageData( auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; @@ -64,7 +58,7 @@ void TsWriterCPP::WriteImageData( case (2): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -73,7 +67,7 @@ void TsWriterCPP::WriteImageData( case (4): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -82,7 +76,7 @@ void TsWriterCPP::WriteImageData( case (8): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -91,7 +85,7 @@ void TsWriterCPP::WriteImageData( case (16): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -100,7 +94,7 @@ void TsWriterCPP::WriteImageData( case (32): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -109,7 +103,7 @@ void TsWriterCPP::WriteImageData( case (64): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -119,7 +113,7 @@ void TsWriterCPP::WriteImageData( auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -128,7 +122,7 @@ void TsWriterCPP::WriteImageData( case (256): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -137,7 +131,7 @@ void TsWriterCPP::WriteImageData( case (512): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -146,7 +140,52 @@ void TsWriterCPP::WriteImageData( default: { // should not be reached std::cerr << "Error writing image: unsupported data type" << std::endl; - } + } + } +} + +void TsWriterCPP::SetOutputTransform( + const Seq& rows, + const Seq& cols, + const Seq& layers, + const Seq& channels, + const Seq& tsteps +) { + + output_transform_ = tensorstore::IdentityTransform(_source.domain()); + + int z_index=0, + c_index=1, t_index=2, + y_index=3, + x_index=4; + + if (layers.IsValidDimension()) { + output_transform_ = (std::move(output_transform_) | tensorstore::Dims(z_index).ClosedInterval(layers.Start(), layers.Stop())).value(); + } else { + --c_index; + --t_index; + --y_index; + --x_index; + } + + if (channels.IsValidDimension()) { + output_transform_ = (std::move(output_transform_) | tensorstore::Dims(c_index).ClosedInterval(channels.Start(), channels.Stop())).value(); + } else { + --t_index; + --y_index; + --x_index; + } + + if (tsteps.IsValidDimension()) { + output_transform_ = (std::move(output_transform_) | tensorstore::Dims(t_index).ClosedInterval(tsteps.Start(), tsteps.Stop())).value(); + } else { + --y_index; + --x_index; } - } -} + + output_transform_ = (std::move(output_transform_) | tensorstore::Dims(y_index).ClosedInterval(rows.Start(), rows.Stop()) | + tensorstore::Dims(x_index).ClosedInterval(cols.Start(), cols.Stop())).value(); +} + + +} // end ns bfiocpp diff --git a/src/cpp/writer/tswriter.h b/src/cpp/writer/tswriter.h index 63cc7b4..78b18f2 100644 --- a/src/cpp/writer/tswriter.h +++ b/src/cpp/writer/tswriter.h @@ -3,7 +3,7 @@ #include #include #include "tensorstore/tensorstore.h" -#include "../reader/sequence.h" +#include "../utilities/sequence.h" #include namespace py = pybind11; @@ -16,7 +16,8 @@ class TsWriterCPP{ const std::string& fname, const std::vector& image_shape, const std::vector& chunk_shape, - const std::string& dtype); + const std::string& dtype + ); void WriteImageData( py::array& py_image, @@ -24,23 +25,28 @@ class TsWriterCPP{ const Seq& cols, const Seq& layers, const Seq& channels, - const Seq& tsteps); + const Seq& tsteps + ); private: std::string _filename; std::vector _image_shape, _chunk_shape; - - static constexpr int _t_index = 0, - _c_index = 1, - _z_index = 2, - _y_index = 3, - _x_index = 4; uint16_t _dtype_code; tensorstore::TensorStore _source; + tensorstore::IndexTransform<> output_transform_; + + void SetOutputTransform( + const Seq& rows, + const Seq& cols, + const Seq& layers, + const Seq& channels, + const Seq& tsteps + ); + }; } From d884c015ec05e0999cb2eed70a9a52985eedb3cb Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Mon, 8 Jul 2024 14:15:05 -0600 Subject: [PATCH 04/22] Move Seq to utilities --- src/cpp/interface/interface.cpp | 5 +++-- src/cpp/reader/tsreader.h | 4 +++- src/cpp/{reader => utilities}/sequence.h | 3 +++ 3 files changed, 9 insertions(+), 3 deletions(-) rename src/cpp/{reader => utilities}/sequence.h (69%) diff --git a/src/cpp/interface/interface.cpp b/src/cpp/interface/interface.cpp index 5b5113c..d477164 100644 --- a/src/cpp/interface/interface.cpp +++ b/src/cpp/interface/interface.cpp @@ -4,7 +4,7 @@ #include #include #include "../reader/tsreader.h" -#include "../reader/sequence.h" +#include "../utilities/sequence.h" #include "../utilities/utilities.h" #include "../writer/tswriter.h" @@ -71,7 +71,8 @@ py::array get_iterator_requested_tile_data(bfiocpp::TsReaderCPP& tl, std::int64 PYBIND11_MODULE(libbfiocpp, m) { py::class_>(m, "Seq") - .def(py::init()); + .def(py::init()) + .def("set_valid_dimension", &bfiocpp::Seq::SetValidDimension); py::class_>(m, "TsReaderCPP") .def(py::init()) diff --git a/src/cpp/reader/tsreader.h b/src/cpp/reader/tsreader.h index 1029a33..e44d485 100644 --- a/src/cpp/reader/tsreader.h +++ b/src/cpp/reader/tsreader.h @@ -9,7 +9,7 @@ #include #include #include "tensorstore/tensorstore.h" -#include "sequence.h" +#include "../utilities/sequence.h" using image_data = std::variant, std::vector, std::vector, @@ -58,7 +58,9 @@ class TsReaderCPP{ _num_tsteps; std::uint16_t _data_type_code; FileType _file_type; + std::optional_z_index, _c_index, _t_index; + int _x_index, _y_index; tensorstore::TensorStore source; diff --git a/src/cpp/reader/sequence.h b/src/cpp/utilities/sequence.h similarity index 69% rename from src/cpp/reader/sequence.h rename to src/cpp/utilities/sequence.h index e0766c8..169f7c2 100644 --- a/src/cpp/reader/sequence.h +++ b/src/cpp/utilities/sequence.h @@ -7,10 +7,13 @@ class Seq { private: long start_index_, stop_index_, step_; + bool is_valid_dimension_ = true; public: inline Seq(const long start, const long stop, const long step=1):start_index_(start), stop_index_(stop), step_(step){} inline long Start() const {return start_index_;} inline long Stop() const {return stop_index_;} inline long Step() const {return step_;} + inline bool IsValidDimension() const {return is_valid_dimension_;} + inline void SetValidDimension(bool valid) {is_valid_dimension_ = valid;} }; } //ns bfiocpp \ No newline at end of file From 6f1e194bb518edae34f710cf5243401f7aeb98ac Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 9 Jul 2024 07:18:33 -0600 Subject: [PATCH 05/22] Fix linux build --- .github/workflows/publish_pypi.yml | 25 ++++++++++++++----------- .github/workflows/wheel_build.yml | 25 ++++++++++++++----------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index b3124a6..59561f3 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -54,17 +54,20 @@ jobs: bash ci-utils/install_prereq_linux.sh && mkdir -p /tmp/bfiocpp_bld && cp -r local_install /tmp/bfiocpp_bld - CIBW_BEFORE_ALL_LINUX: yum -y install wget && - wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && - tar -xjf nasm-2.15.05.tar.bz2 && - cd nasm-2.15.05 && - ./configure && - make && - make install && - cd .. && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/bfiocpp_bld && - cp -r local_install /tmp/bfiocpp_bld + CIBW_BEFORE_ALL_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && + sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && + sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && + yum -y install wget && + wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && + tar -xjf nasm-2.15.05.tar.bz2 && + cd nasm-2.15.05 && + ./configure && + make && + make install && + cd .. && + bash ci-utils/install_prereq_linux.sh && + mkdir -p /tmp/bfiocpp_bld && + cp -r local_install /tmp/bfiocpp_bld CIBW_BEFORE_ALL_WINDOWS: ci-utils\install_prereq_win.bat && xcopy /E /I /y local_install C:\TEMP\bfiocpp_bld\local_install CIBW_ENVIRONMENT_MACOS: REPAIR_LIBRARY_PATH="/tmp/bfiocpp_bld/local_install/lib:/tmp/bfiocpp_bld/local_install/lib64" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="/tmp/bfiocpp_bld/local_install" diff --git a/.github/workflows/wheel_build.yml b/.github/workflows/wheel_build.yml index 35b801b..61bfcce 100644 --- a/.github/workflows/wheel_build.yml +++ b/.github/workflows/wheel_build.yml @@ -52,17 +52,20 @@ jobs: bash ci-utils/install_prereq_linux.sh && mkdir -p /tmp/bfiocpp_bld && cp -r local_install /tmp/bfiocpp_bld - CIBW_BEFORE_ALL_LINUX: yum -y install wget && - wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && - tar -xjf nasm-2.15.05.tar.bz2 && - cd nasm-2.15.05 && - ./configure && - make && - make install && - cd .. && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/bfiocpp_bld && - cp -r local_install /tmp/bfiocpp_bld + CIBW_BEFORE_ALL_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && + sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && + sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && + yum -y install wget && yum -y install wget && + wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && + tar -xjf nasm-2.15.05.tar.bz2 && + cd nasm-2.15.05 && + ./configure && + make && + make install && + cd .. && + bash ci-utils/install_prereq_linux.sh && + mkdir -p /tmp/bfiocpp_bld && + cp -r local_install /tmp/bfiocpp_bld CIBW_BEFORE_ALL_WINDOWS: ci-utils\install_prereq_win.bat && xcopy /E /I /y local_install C:\TEMP\bfiocpp_bld\local_install CIBW_ENVIRONMENT_MACOS: REPAIR_LIBRARY_PATH="/tmp/bfiocpp_bld/local_install/lib:/tmp/bfiocpp_bld/local_install/lib64" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="/tmp/bfiocpp_bld/local_install" From cd19a69fd806a0ed24f1a4c3e93a194ed0d49405 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 9 Jul 2024 07:22:28 -0600 Subject: [PATCH 06/22] Update linux build --- .github/workflows/publish_pypi.yml | 28 ++++++++++++++-------------- .github/workflows/wheel_build.yml | 29 +++++++++++++++-------------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 59561f3..935ff79 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -54,20 +54,20 @@ jobs: bash ci-utils/install_prereq_linux.sh && mkdir -p /tmp/bfiocpp_bld && cp -r local_install /tmp/bfiocpp_bld - CIBW_BEFORE_ALL_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && - sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && - sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && - yum -y install wget && - wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && - tar -xjf nasm-2.15.05.tar.bz2 && - cd nasm-2.15.05 && - ./configure && - make && - make install && - cd .. && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/bfiocpp_bld && - cp -r local_install /tmp/bfiocpp_bld + CIBW_BEFORE_ALL_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && + sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && + sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && + yum -y install wget && + wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && + tar -xjf nasm-2.15.05.tar.bz2 && + cd nasm-2.15.05 && + ./configure && + make && + make install && + cd .. && + bash ci-utils/install_prereq_linux.sh && + mkdir -p /tmp/bfiocpp_bld && + cp -r local_install /tmp/bfiocpp_bld CIBW_BEFORE_ALL_WINDOWS: ci-utils\install_prereq_win.bat && xcopy /E /I /y local_install C:\TEMP\bfiocpp_bld\local_install CIBW_ENVIRONMENT_MACOS: REPAIR_LIBRARY_PATH="/tmp/bfiocpp_bld/local_install/lib:/tmp/bfiocpp_bld/local_install/lib64" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="/tmp/bfiocpp_bld/local_install" diff --git a/.github/workflows/wheel_build.yml b/.github/workflows/wheel_build.yml index 61bfcce..80d66d4 100644 --- a/.github/workflows/wheel_build.yml +++ b/.github/workflows/wheel_build.yml @@ -52,20 +52,21 @@ jobs: bash ci-utils/install_prereq_linux.sh && mkdir -p /tmp/bfiocpp_bld && cp -r local_install /tmp/bfiocpp_bld - CIBW_BEFORE_ALL_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && - sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && - sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && - yum -y install wget && yum -y install wget && - wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && - tar -xjf nasm-2.15.05.tar.bz2 && - cd nasm-2.15.05 && - ./configure && - make && - make install && - cd .. && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/bfiocpp_bld && - cp -r local_install /tmp/bfiocpp_bld + + CIBW_BEFORE_ALL_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && + sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && + sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && + yum -y install wget && + wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && + tar -xjf nasm-2.15.05.tar.bz2 && + cd nasm-2.15.05 && + ./configure && + make && + make install && + cd .. && + bash ci-utils/install_prereq_linux.sh && + mkdir -p /tmp/bfiocpp_bld && + cp -r local_install /tmp/bfiocpp_bld CIBW_BEFORE_ALL_WINDOWS: ci-utils\install_prereq_win.bat && xcopy /E /I /y local_install C:\TEMP\bfiocpp_bld\local_install CIBW_ENVIRONMENT_MACOS: REPAIR_LIBRARY_PATH="/tmp/bfiocpp_bld/local_install/lib:/tmp/bfiocpp_bld/local_install/lib64" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="/tmp/bfiocpp_bld/local_install" From c74a232e953da72f14c8d24c92cc5bebff9de8de Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 9 Jul 2024 08:19:49 -0600 Subject: [PATCH 07/22] Switch to curl --- .github/workflows/publish_pypi.yml | 29 ++++++++++++++--------------- .github/workflows/wheel_build.yml | 14 ++++++-------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 935ff79..66f7122 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -54,20 +54,16 @@ jobs: bash ci-utils/install_prereq_linux.sh && mkdir -p /tmp/bfiocpp_bld && cp -r local_install /tmp/bfiocpp_bld - CIBW_BEFORE_ALL_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && - sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && - sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && - yum -y install wget && - wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && - tar -xjf nasm-2.15.05.tar.bz2 && - cd nasm-2.15.05 && - ./configure && - make && - make install && - cd .. && - bash ci-utils/install_prereq_linux.sh && - mkdir -p /tmp/bfiocpp_bld && - cp -r local_install /tmp/bfiocpp_bld + CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && + tar -xjf nasm-2.15.05.tar.bz2 && + cd nasm-2.15.05 && + ./configure && + make && + make install && + cd .. && + bash ci-utils/install_prereq_linux.sh && + mkdir -p /tmp/bfiocpp_bld && + cp -r local_install /tmp/bfiocpp_bld CIBW_BEFORE_ALL_WINDOWS: ci-utils\install_prereq_win.bat && xcopy /E /I /y local_install C:\TEMP\bfiocpp_bld\local_install CIBW_ENVIRONMENT_MACOS: REPAIR_LIBRARY_PATH="/tmp/bfiocpp_bld/local_install/lib:/tmp/bfiocpp_bld/local_install/lib64" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="/tmp/bfiocpp_bld/local_install" @@ -76,7 +72,10 @@ jobs: CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\bfiocpp\\bin;$PATH" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="C:\\TEMP\\bfiocpp_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" CIBW_ARCHS: ${{ matrix.cibw_archs }} - CIBW_BEFORE_TEST_LINUX: yum -y install maven java + CIBW_BEFORE_TEST_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && + sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && + sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && + yum -y install maven java CIBW_TEST_REQUIRES: bfio requests numpy==1.24.0 ome_zarr CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v diff --git a/.github/workflows/wheel_build.yml b/.github/workflows/wheel_build.yml index 80d66d4..49f0e37 100644 --- a/.github/workflows/wheel_build.yml +++ b/.github/workflows/wheel_build.yml @@ -51,13 +51,8 @@ jobs: CIBW_BEFORE_ALL_MACOS: brew install nasm && bash ci-utils/install_prereq_linux.sh && mkdir -p /tmp/bfiocpp_bld && - cp -r local_install /tmp/bfiocpp_bld - - CIBW_BEFORE_ALL_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && - sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && - sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && - yum -y install wget && - wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && + cp -r local_install /tmp/bfiocpp_bld + CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && tar -xjf nasm-2.15.05.tar.bz2 && cd nasm-2.15.05 && ./configure && @@ -75,7 +70,10 @@ jobs: CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\bfiocpp\\bin;$PATH" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="C:\\TEMP\\bfiocpp_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" CIBW_ARCHS: ${{ matrix.cibw_archs }} - CIBW_BEFORE_TEST_LINUX: yum -y install maven java + CIBW_BEFORE_TEST_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && + sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && + sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && + yum -y install maven java CIBW_TEST_REQUIRES: bfio requests numpy==1.24.0 ome_zarr CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v From 2bb371812d453d6edf102f70bd400afd74d92475 Mon Sep 17 00:00:00 2001 From: sameeul Date: Tue, 9 Jul 2024 13:45:11 -0400 Subject: [PATCH 08/22] Update curl command --- .github/workflows/publish_pypi.yml | 2 +- .github/workflows/wheel_build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 2b130b2..1f79d15 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -54,7 +54,7 @@ jobs: bash ci-utils/install_prereq_linux.sh && mkdir -p /tmp/bfiocpp_bld && cp -r local_install /tmp/bfiocpp_bld - CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && + CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 -o nasm-2.15.05.tar.bz2 && tar -xjf nasm-2.15.05.tar.bz2 && cd nasm-2.15.05 && ./configure && diff --git a/.github/workflows/wheel_build.yml b/.github/workflows/wheel_build.yml index 24d5079..06dec61 100644 --- a/.github/workflows/wheel_build.yml +++ b/.github/workflows/wheel_build.yml @@ -52,7 +52,7 @@ jobs: bash ci-utils/install_prereq_linux.sh && mkdir -p /tmp/bfiocpp_bld && cp -r local_install /tmp/bfiocpp_bld - CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 && + CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 -o nasm-2.15.05.tar.bz2 && tar -xjf nasm-2.15.05.tar.bz2 && cd nasm-2.15.05 && ./configure && From 85bdb5056bcf8bc25691450eb24e1f82397fc2af Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 9 Jul 2024 12:07:22 -0600 Subject: [PATCH 09/22] Remove unused members --- src/cpp/reader/tsreader.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cpp/reader/tsreader.h b/src/cpp/reader/tsreader.h index e44d485..4062856 100644 --- a/src/cpp/reader/tsreader.h +++ b/src/cpp/reader/tsreader.h @@ -60,7 +60,6 @@ class TsReaderCPP{ FileType _file_type; std::optional_z_index, _c_index, _t_index; - int _x_index, _y_index; tensorstore::TensorStore source; From aa439d12ab028da1d646535b80915ceddc2c66b0 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Thu, 11 Jul 2024 07:27:00 -0600 Subject: [PATCH 10/22] Update write method --- src/cpp/writer/tswriter.cpp | 106 ++++++++++++++++-------------------- src/cpp/writer/tswriter.h | 26 ++++----- 2 files changed, 56 insertions(+), 76 deletions(-) diff --git a/src/cpp/writer/tswriter.cpp b/src/cpp/writer/tswriter.cpp index dfbfe19..e491384 100644 --- a/src/cpp/writer/tswriter.cpp +++ b/src/cpp/writer/tswriter.cpp @@ -14,19 +14,34 @@ TsWriterCPP::TsWriterCPP( const std::string& fname, const std::vector& image_shape, const std::vector& chunk_shape, - const std::string& dtype_str + const std::string& dtype_str, + const std::string& dimension_order ): _filename(fname), _image_shape(image_shape), _chunk_shape(chunk_shape), _dtype_code(GetDataTypeCode(dtype_str)) { - TENSORSTORE_CHECK_OK_AND_ASSIGN(_source, tensorstore::Open( GetZarrSpecToWrite(_filename, _image_shape, _chunk_shape, GetEncodedType(_dtype_code)), tensorstore::OpenMode::create | tensorstore::OpenMode::delete_existing, tensorstore::ReadWriteMode::write).result() ); + + auto position = dimension_order.find("X"); + if (position != std::string::npos) _x_index = position; + + position = dimension_order.find("Y"); + if (position != std::string::npos) _y_index = position; + + position = dimension_order.find("C"); + if (position != std::string::npos) _c_index.emplace(position); + + position = dimension_order.find("T"); + if (position != std::string::npos) _t_index.emplace(position); + + position = dimension_order.find("Z"); + if (position != std::string::npos) _z_index.emplace(position); } @@ -34,11 +49,26 @@ void TsWriterCPP::WriteImageData( py::array& py_image, const Seq& rows, const Seq& cols, - const Seq& layers, - const Seq& channels, - const Seq& tsteps) { + const std::optional& layers, + const std::optional& channels, + const std::optional& tsteps) { + + auto output_transform = tensorstore::IdentityTransform(_source.domain()); + + if (_z_index.has_value() && layers.has_value()) { + output_transform = (std::move(output_transform) | tensorstore::Dims(_z_index.value()).ClosedInterval(layers.value().Start(), layers.value().Stop())).value(); + } + + if (_c_index.has_value() && channels.has_value()) { + output_transform = (std::move(output_transform) | tensorstore::Dims(_c_index.value()).ClosedInterval(channels.value().Start(), channels.value().Stop())).value(); + } - SetOutputTransform(rows, cols, layers, channels, tsteps); + if (_t_index.has_value() && tsteps.has_value()) { + output_transform = (std::move(output_transform) | tensorstore::Dims(_t_index.value()).ClosedInterval(tsteps.value().Start(), tsteps.value().Stop())).value(); + } + + output_transform = (std::move(output_transform) | tensorstore::Dims(_y_index).ClosedInterval(rows.Start(), rows.Stop()) | + tensorstore::Dims(_x_index).ClosedInterval(cols.Start(), cols.Stop())).value(); // use switch instead of template to avoid creating functions for each datatype switch(_dtype_code) @@ -47,7 +77,7 @@ void TsWriterCPP::WriteImageData( auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; @@ -58,7 +88,7 @@ void TsWriterCPP::WriteImageData( case (2): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -67,7 +97,7 @@ void TsWriterCPP::WriteImageData( case (4): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -76,7 +106,7 @@ void TsWriterCPP::WriteImageData( case (8): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -85,7 +115,7 @@ void TsWriterCPP::WriteImageData( case (16): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -94,7 +124,7 @@ void TsWriterCPP::WriteImageData( case (32): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -103,7 +133,7 @@ void TsWriterCPP::WriteImageData( case (64): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -113,7 +143,7 @@ void TsWriterCPP::WriteImageData( auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -122,7 +152,7 @@ void TsWriterCPP::WriteImageData( case (256): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -131,7 +161,7 @@ void TsWriterCPP::WriteImageData( case (512): { auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform_).result(); + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } @@ -144,48 +174,4 @@ void TsWriterCPP::WriteImageData( } } -void TsWriterCPP::SetOutputTransform( - const Seq& rows, - const Seq& cols, - const Seq& layers, - const Seq& channels, - const Seq& tsteps -) { - - output_transform_ = tensorstore::IdentityTransform(_source.domain()); - - int z_index=0, - c_index=1, t_index=2, - y_index=3, - x_index=4; - - if (layers.IsValidDimension()) { - output_transform_ = (std::move(output_transform_) | tensorstore::Dims(z_index).ClosedInterval(layers.Start(), layers.Stop())).value(); - } else { - --c_index; - --t_index; - --y_index; - --x_index; - } - - if (channels.IsValidDimension()) { - output_transform_ = (std::move(output_transform_) | tensorstore::Dims(c_index).ClosedInterval(channels.Start(), channels.Stop())).value(); - } else { - --t_index; - --y_index; - --x_index; - } - - if (tsteps.IsValidDimension()) { - output_transform_ = (std::move(output_transform_) | tensorstore::Dims(t_index).ClosedInterval(tsteps.Start(), tsteps.Stop())).value(); - } else { - --y_index; - --x_index; - } - - output_transform_ = (std::move(output_transform_) | tensorstore::Dims(y_index).ClosedInterval(rows.Start(), rows.Stop()) | - tensorstore::Dims(x_index).ClosedInterval(cols.Start(), cols.Stop())).value(); -} - - } // end ns bfiocpp diff --git a/src/cpp/writer/tswriter.h b/src/cpp/writer/tswriter.h index 78b18f2..92dbaa8 100644 --- a/src/cpp/writer/tswriter.h +++ b/src/cpp/writer/tswriter.h @@ -12,20 +12,21 @@ namespace bfiocpp{ class TsWriterCPP{ public: - TsWriterCPP( + TsWriterCPP ( const std::string& fname, const std::vector& image_shape, - const std::vector& chunk_shape, - const std::string& dtype + const std::vector& chunk_shape, + const std::string& dtype_str, + const std::string& dimension_order ); - void WriteImageData( + void WriteImageData ( py::array& py_image, const Seq& rows, const Seq& cols, - const Seq& layers, - const Seq& channels, - const Seq& tsteps + const std::optional& layers, + const std::optional& channels, + const std::optional& tsteps ); private: @@ -37,15 +38,8 @@ class TsWriterCPP{ tensorstore::TensorStore _source; - tensorstore::IndexTransform<> output_transform_; - - void SetOutputTransform( - const Seq& rows, - const Seq& cols, - const Seq& layers, - const Seq& channels, - const Seq& tsteps - ); + std::optional_z_index, _c_index, _t_index; + int _x_index, _y_index; }; } From 5f6297cc49391ed0a9c8193c7801608a7b40c5db Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Thu, 11 Jul 2024 07:27:09 -0600 Subject: [PATCH 11/22] Remove unused methods --- src/cpp/utilities/sequence.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/cpp/utilities/sequence.h b/src/cpp/utilities/sequence.h index 169f7c2..e0766c8 100644 --- a/src/cpp/utilities/sequence.h +++ b/src/cpp/utilities/sequence.h @@ -7,13 +7,10 @@ class Seq { private: long start_index_, stop_index_, step_; - bool is_valid_dimension_ = true; public: inline Seq(const long start, const long stop, const long step=1):start_index_(start), stop_index_(stop), step_(step){} inline long Start() const {return start_index_;} inline long Stop() const {return stop_index_;} inline long Step() const {return step_;} - inline bool IsValidDimension() const {return is_valid_dimension_;} - inline void SetValidDimension(bool valid) {is_valid_dimension_ = valid;} }; } //ns bfiocpp \ No newline at end of file From f0185f638c012fdfb50230846d9b7501ea0415f6 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Thu, 11 Jul 2024 07:27:25 -0600 Subject: [PATCH 12/22] Update tensorstore zarr writer constructor arguments --- src/cpp/interface/interface.cpp | 5 ++--- src/python/bfiocpp/tswriter.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/cpp/interface/interface.cpp b/src/cpp/interface/interface.cpp index d477164..ed7916f 100644 --- a/src/cpp/interface/interface.cpp +++ b/src/cpp/interface/interface.cpp @@ -71,8 +71,7 @@ py::array get_iterator_requested_tile_data(bfiocpp::TsReaderCPP& tl, std::int64 PYBIND11_MODULE(libbfiocpp, m) { py::class_>(m, "Seq") - .def(py::init()) - .def("set_valid_dimension", &bfiocpp::Seq::SetValidDimension); + .def(py::init()); py::class_>(m, "TsReaderCPP") .def(py::init()) @@ -123,6 +122,6 @@ PYBIND11_MODULE(libbfiocpp, m) { // Writer class py::class_>(m, "TsWriterCPP") - .def(py::init&, const std::vector&, const std::string&>()) + .def(py::init&, const std::vector&, const std::string&, const std::string&>()) .def("write_image_data", &bfiocpp::TsWriterCPP::WriteImageData); } diff --git a/src/python/bfiocpp/tswriter.py b/src/python/bfiocpp/tswriter.py index 6ac2c83..f673adf 100644 --- a/src/python/bfiocpp/tswriter.py +++ b/src/python/bfiocpp/tswriter.py @@ -5,7 +5,7 @@ class TSWriter: def __init__( - self, file_name: str, image_shape: list, chunk_shape: list, dtype: np.dtype + self, file_name: str, image_shape: list, chunk_shape: list, dtype: np.dtype, dimension_order: str, ): """Initialize tensorstore Zarr writer @@ -13,7 +13,7 @@ def __init__( """ self._image_writer: TsWriterCPP = TsWriterCPP( - file_name, image_shape, chunk_shape, str(dtype) + file_name, image_shape, chunk_shape, str(dtype), dimension_order ) def write_image_data( From 749abb411fa8e2a5bf61af153cb2d21b4f852ca5 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Thu, 11 Jul 2024 07:27:54 -0600 Subject: [PATCH 13/22] Update tensorstore zarr writer test to use new api --- tests/test_write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_write.py b/tests/test_write.py index de4560a..6301522 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -83,7 +83,7 @@ def test_write_zarr_2d(self): channels = Seq(0, 0, 1) tsteps = Seq(0, 0, 1) - bw = TSWriter(test_file_path, tmp.shape, tmp.shape, str(tmp.dtype)) + bw = TSWriter(test_file_path, tmp.shape, tmp.shape, str(tmp.dtype), "ZCTYX") bw.write_image_data(tmp, rows, cols, layers, channels, tsteps) bw.close() From fa67f793e40bf4158be5343d6d3ea27b969c53b8 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Thu, 11 Jul 2024 07:29:23 -0600 Subject: [PATCH 14/22] Reformat --- src/python/bfiocpp/tswriter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/python/bfiocpp/tswriter.py b/src/python/bfiocpp/tswriter.py index f673adf..7d81f55 100644 --- a/src/python/bfiocpp/tswriter.py +++ b/src/python/bfiocpp/tswriter.py @@ -5,7 +5,12 @@ class TSWriter: def __init__( - self, file_name: str, image_shape: list, chunk_shape: list, dtype: np.dtype, dimension_order: str, + self, + file_name: str, + image_shape: list, + chunk_shape: list, + dtype: np.dtype, + dimension_order: str, ): """Initialize tensorstore Zarr writer From d3c0e811c8fe79952a1e048f3238e832f2016171 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 16 Jul 2024 09:14:09 -0600 Subject: [PATCH 15/22] Update zarr writer to write chunks --- src/cpp/writer/tswriter.cpp | 68 +++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/src/cpp/writer/tswriter.cpp b/src/cpp/writer/tswriter.cpp index e491384..65850bf 100644 --- a/src/cpp/writer/tswriter.cpp +++ b/src/cpp/writer/tswriter.cpp @@ -11,16 +11,17 @@ namespace bfiocpp { TsWriterCPP::TsWriterCPP( - const std::string& fname, - const std::vector& image_shape, + const std::string& fname, + const std::vector& image_shape, const std::vector& chunk_shape, const std::string& dtype_str, const std::string& dimension_order - ): _filename(fname), - _image_shape(image_shape), - _chunk_shape(chunk_shape), + ): _filename(fname), + _image_shape(image_shape), + _chunk_shape(chunk_shape), _dtype_code(GetDataTypeCode(dtype_str)) { + TENSORSTORE_CHECK_OK_AND_ASSIGN(_source, tensorstore::Open( GetZarrSpecToWrite(_filename, _image_shape, _chunk_shape, GetEncodedType(_dtype_code)), tensorstore::OpenMode::create | @@ -44,37 +45,46 @@ TsWriterCPP::TsWriterCPP( if (position != std::string::npos) _z_index.emplace(position); } - void TsWriterCPP::WriteImageData( - py::array& py_image, - const Seq& rows, - const Seq& cols, - const std::optional& layers, - const std::optional& channels, + py::array& py_image, + const Seq& rows, + const Seq& cols, + const std::optional& layers, + const std::optional& channels, const std::optional& tsteps) { + std::vector shape; + auto output_transform = tensorstore::IdentityTransform(_source.domain()); if (_z_index.has_value() && layers.has_value()) { output_transform = (std::move(output_transform) | tensorstore::Dims(_z_index.value()).ClosedInterval(layers.value().Start(), layers.value().Stop())).value(); + shape.emplace_back(layers.value().Stop() - layers.value().Start()+1); } if (_c_index.has_value() && channels.has_value()) { output_transform = (std::move(output_transform) | tensorstore::Dims(_c_index.value()).ClosedInterval(channels.value().Start(), channels.value().Stop())).value(); - } + shape.emplace_back(channels.value().Stop() - channels.value().Start()+1); + } if (_t_index.has_value() && tsteps.has_value()) { output_transform = (std::move(output_transform) | tensorstore::Dims(_t_index.value()).ClosedInterval(tsteps.value().Start(), tsteps.value().Stop())).value(); - } + shape.emplace_back(tsteps.value().Stop() - tsteps.value().Start()+1); + } output_transform = (std::move(output_transform) | tensorstore::Dims(_y_index).ClosedInterval(rows.Start(), rows.Stop()) | - tensorstore::Dims(_x_index).ClosedInterval(cols.Start(), cols.Stop())).value(); + tensorstore::Dims(_x_index).ClosedInterval(cols.Start(), cols.Stop())).value(); + + + + shape.emplace_back(rows.Stop() - rows.Start()+1); + shape.emplace_back(cols.Stop() - cols.Start()+1); // use switch instead of template to avoid creating functions for each datatype switch(_dtype_code) { case (1): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), shape, tensorstore::c_order); // Write data array to TensorStore auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); @@ -86,16 +96,16 @@ void TsWriterCPP::WriteImageData( break; } case (2): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), shape, tensorstore::c_order); auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } - break; - } + break; + } case (4): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), shape, tensorstore::c_order); auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { @@ -104,7 +114,7 @@ void TsWriterCPP::WriteImageData( break; } case (8): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), shape, tensorstore::c_order); auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { @@ -113,7 +123,7 @@ void TsWriterCPP::WriteImageData( break; } case (16): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), shape, tensorstore::c_order); auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { @@ -122,16 +132,16 @@ void TsWriterCPP::WriteImageData( break; } case (32): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), shape, tensorstore::c_order); auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { std::cerr << "Error writing image: " << write_result.status() << std::endl; } - break; - } + break; + } case (64): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), shape, tensorstore::c_order); auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { @@ -140,7 +150,7 @@ void TsWriterCPP::WriteImageData( break; } case (128): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), shape, tensorstore::c_order); // Write data array to TensorStore auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); @@ -150,7 +160,7 @@ void TsWriterCPP::WriteImageData( break; } case (256): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), shape, tensorstore::c_order); auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { @@ -159,7 +169,7 @@ void TsWriterCPP::WriteImageData( break; } case (512): { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), shape, tensorstore::c_order); auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source | output_transform).result(); if (!write_result.ok()) { @@ -170,7 +180,7 @@ void TsWriterCPP::WriteImageData( default: { // should not be reached std::cerr << "Error writing image: unsupported data type" << std::endl; - } + } } } From 66dfea66a822d59fda7d668db6c47bf0db6063e1 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 16 Jul 2024 09:14:20 -0600 Subject: [PATCH 16/22] Update api to write 2D to 5D images --- src/python/bfiocpp/tswriter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/python/bfiocpp/tswriter.py b/src/python/bfiocpp/tswriter.py index 7d81f55..2426a8f 100644 --- a/src/python/bfiocpp/tswriter.py +++ b/src/python/bfiocpp/tswriter.py @@ -1,4 +1,5 @@ import numpy as np +from typing import Optional from .libbfiocpp import TsWriterCPP @@ -26,9 +27,9 @@ def write_image_data( image_data: np.ndarray, rows: int, cols: int, - layers: int, - channels: int, - tsteps: int, + layers: Optional[int] = None, + channels: Optional[int] = None, + tsteps: Optional[int] = None, ): """Write image data to file From bc4cdd8c4e7999f5b4e4c5db2ac157727236bc9c Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 16 Jul 2024 09:14:30 -0600 Subject: [PATCH 17/22] Add tests for writing 2D and 3D chunks --- tests/test_write.py | 235 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 216 insertions(+), 19 deletions(-) diff --git a/tests/test_write.py b/tests/test_write.py index 6301522..0bb72dd 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -22,32 +22,32 @@ logger.setLevel(logging.INFO) -def setUpModule(): - """Download images for testing""" - TEST_DIR.mkdir(exist_ok=True) +# def setUpModule(): +# """Download images for testing""" +# TEST_DIR.mkdir(exist_ok=True) - for file, url in TEST_IMAGES.items(): - logger.info(f"setup - Downloading: {file}") +# for file, url in TEST_IMAGES.items(): +# logger.info(f"setup - Downloading: {file}") - if not file.endswith(".zarr"): - if TEST_DIR.joinpath(file).exists(): - continue +# if not file.endswith(".zarr"): +# if TEST_DIR.joinpath(file).exists(): +# continue - r = requests.get(url) +# r = requests.get(url) - with open(TEST_DIR.joinpath(file), "wb") as fw: - fw.write(r.content) - else: - if TEST_DIR.joinpath(file).exists(): - shutil.rmtree(TEST_DIR.joinpath(file)) - zarr_download(url, str(TEST_DIR)) +# with open(TEST_DIR.joinpath(file), "wb") as fw: +# fw.write(r.content) +# else: +# if TEST_DIR.joinpath(file).exists(): +# shutil.rmtree(TEST_DIR.joinpath(file)) +# zarr_download(url, str(TEST_DIR)) -def tearDownModule(): - """Remove test images""" +# def tearDownModule(): +# """Remove test images""" - logger.info("teardown - Removing test images...") - shutil.rmtree(TEST_DIR) +# logger.info("teardown - Removing test images...") +# shutil.rmtree(TEST_DIR) class TestZarrWrite(unittest.TestCase): @@ -60,6 +60,7 @@ def test_write_zarr_2d(self): FileType.OmeZarr, "", ) + assert br._X == 2702 assert br._Y == 2700 assert br._Z == 1 @@ -99,3 +100,199 @@ def test_write_zarr_2d(self): assert tmp.sum() == 183750394 assert tmp.shape == (1, 1, 1, 2700, 2702) + def test_write_zarr_chunk_2d(self): + """test_write_zarr_2d - Write zarr using TSWrtier""" + + br = TSReader( + str(TEST_DIR.joinpath("5025551.zarr/0")), + FileType.OmeZarr, + "", + ) + + assert br._X == 2702 + assert br._Y == 2700 + assert br._Z == 1 + assert br._C == 27 + assert br._T == 1 + + rows = Seq(0, br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(0, 0, 1) + tsteps = Seq(0, 0, 1) + tmp = br.data(rows, cols, layers, channels, tsteps) + + with tempfile.TemporaryDirectory() as dir: + # Use the temporary directory + test_file_path = os.path.join(dir, 'out/test.ome.zarr') + + print(tmp.shape) + chunk_shape = [1, 1, 1, 1350, 1351] + #chunk_shape = tmp.shape + print("dtype: " + str(tmp.dtype)) + bw = TSWriter(test_file_path, tmp.shape, chunk_shape, str(tmp.dtype), "ZCTYX") + + rows = Seq(0, br._Y//2, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(0, 0, 1) + tsteps = Seq(0, 0, 1) + tmp = br.data(rows, cols, layers, channels, tsteps) + bw.write_image_data(tmp, rows, cols, layers, channels, tsteps) + + rows = Seq(br._Y//2 , br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(0, 0, 1) + tsteps = Seq(0, 0, 1) + tmp = br.data(rows, cols, layers, channels, tsteps) + bw.write_image_data(tmp, rows, cols, layers, channels, tsteps) + + bw.close() + + br = TSReader( + str(test_file_path), + FileType.OmeZarr, + "", + ) + + rows = Seq(0, br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(0, 0, 1) + tsteps = Seq(0, 0, 1) + + tmp = br.data(rows, cols, layers, channels, tsteps) + + print(tmp) + + assert tmp.dtype == np.uint8 + assert tmp.shape == (1, 1, 1, 2700, 2702) + assert tmp.sum() == 183750394 + + def test_write_zarr_3d(self): + """test_write_zarr_2d - Write zarr using TSWrtier""" + + br = TSReader( + str(TEST_DIR.joinpath("5025551.zarr/0")), + FileType.OmeZarr, + "", + ) + assert br._X == 2702 + assert br._Y == 2700 + assert br._Z == 1 + assert br._C == 27 + assert br._T == 1 + + rows = Seq(0, br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(0, 0, 1) + tsteps = Seq(0, 0, 1) + tmp = br.data(rows, cols, layers, channels, tsteps) + + # Update shape to (1, 3, 1, 2700, 2702) + tmp = np.repeat(tmp, 3, axis=1) + + with tempfile.TemporaryDirectory() as dir: + # Use the temporary directory + test_file_path = os.path.join(dir, 'out/test.ome.zarr') + + rows = Seq(0, br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(0, 2, 1) + tsteps = Seq(0, 0, 1) + + bw = TSWriter(test_file_path, tmp.shape, tmp.shape, str(tmp.dtype), "ZCTYX") + bw.write_image_data(tmp, rows, cols, layers, channels, tsteps) + bw.close() + + br = TSReader( + str(test_file_path), + FileType.OmeZarr, + "", + ) + + tmp = br.data(rows, cols, layers, channels, tsteps) + + assert tmp.dtype == np.uint8 + assert tmp.sum() == 3*183750394 + assert tmp.shape == (1, 3, 1, 2700, 2702) + + + def test_write_zarr_chunk_3d(self): + """test_write_zarr_2d - Write zarr using TSWrtier""" + + br = TSReader( + str(TEST_DIR.joinpath("5025551.zarr/0")), + FileType.OmeZarr, + "", + ) + assert br._X == 2702 + assert br._Y == 2700 + assert br._Z == 1 + assert br._C == 27 + assert br._T == 1 + + rows = Seq(0, br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(0, 0, 1) + tsteps = Seq(0, 0, 1) + tmp = br.data(rows, cols, layers, channels, tsteps) + + # Update shape to (1, 3, 1, 2700, 2702) + tmp = np.repeat(tmp, 3, axis=1) + + with tempfile.TemporaryDirectory() as dir: + # Use the temporary directory + test_file_path = os.path.join(dir, 'out/test.ome.zarr') + + chunk_size = (1,1,1,2700,2702) + + bw = TSWriter(test_file_path, tmp.shape, chunk_size, str(tmp.dtype), "ZCTYX") + + # write first channel + rows = Seq(0, br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(0, 0, 1) + tsteps = Seq(0, 0, 1) + bw.write_image_data(tmp, rows, cols, layers, channels, tsteps) + + # write second channel + rows = Seq(0, br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(1, 1, 1) + tsteps = Seq(0, 0, 1) + bw.write_image_data(tmp, rows, cols, layers, channels, tsteps) + + # write third channel + rows = Seq(0, br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(2, 2, 1) + tsteps = Seq(0, 0, 1) + bw.write_image_data(tmp, rows, cols, layers, channels, tsteps) + + bw.close() + + br = TSReader( + str(test_file_path), + FileType.OmeZarr, + "", + ) + + rows = Seq(0, br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(0, 2, 1) + tsteps = Seq(0, 0, 1) + + tmp = br.data(rows, cols, layers, channels, tsteps) + + assert tmp.dtype == np.uint8 + assert tmp.sum() == 3*183750394 + assert tmp.shape == (1, 3, 1, 2700, 2702) From b4550ac2ab9904ad6e140dd773bfaa9281bf977b Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 16 Jul 2024 09:52:49 -0600 Subject: [PATCH 18/22] Update test_write.py --- tests/test_write.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/test_write.py b/tests/test_write.py index 0bb72dd..762e7c9 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -22,32 +22,32 @@ logger.setLevel(logging.INFO) -# def setUpModule(): -# """Download images for testing""" -# TEST_DIR.mkdir(exist_ok=True) +def setUpModule(): + """Download images for testing""" + TEST_DIR.mkdir(exist_ok=True) -# for file, url in TEST_IMAGES.items(): -# logger.info(f"setup - Downloading: {file}") + for file, url in TEST_IMAGES.items(): + logger.info(f"setup - Downloading: {file}") -# if not file.endswith(".zarr"): -# if TEST_DIR.joinpath(file).exists(): -# continue + if not file.endswith(".zarr"): + if TEST_DIR.joinpath(file).exists(): + continue -# r = requests.get(url) + r = requests.get(url) -# with open(TEST_DIR.joinpath(file), "wb") as fw: -# fw.write(r.content) -# else: -# if TEST_DIR.joinpath(file).exists(): -# shutil.rmtree(TEST_DIR.joinpath(file)) -# zarr_download(url, str(TEST_DIR)) + with open(TEST_DIR.joinpath(file), "wb") as fw: + fw.write(r.content) + else: + if TEST_DIR.joinpath(file).exists(): + shutil.rmtree(TEST_DIR.joinpath(file)) + zarr_download(url, str(TEST_DIR)) -# def tearDownModule(): -# """Remove test images""" +def tearDownModule(): + """Remove test images""" -# logger.info("teardown - Removing test images...") -# shutil.rmtree(TEST_DIR) + logger.info("teardown - Removing test images...") + shutil.rmtree(TEST_DIR) class TestZarrWrite(unittest.TestCase): From 9f34e04779c9a3017cc90f55a83f4fb32f916a61 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 16 Jul 2024 10:31:02 -0600 Subject: [PATCH 19/22] Fix typo --- tests/test_write.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_write.py b/tests/test_write.py index 762e7c9..1c1630d 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -53,7 +53,7 @@ def tearDownModule(): class TestZarrWrite(unittest.TestCase): def test_write_zarr_2d(self): - """test_write_zarr_2d - Write zarr using TSWrtier""" + """test_write_zarr_2d - Write zarr using TSWriter""" br = TSReader( str(TEST_DIR.joinpath("5025551.zarr/0")), @@ -101,7 +101,7 @@ def test_write_zarr_2d(self): assert tmp.shape == (1, 1, 1, 2700, 2702) def test_write_zarr_chunk_2d(self): - """test_write_zarr_2d - Write zarr using TSWrtier""" + """test_write_zarr_2d - Write zarr using TsWriter""" br = TSReader( str(TEST_DIR.joinpath("5025551.zarr/0")), @@ -171,7 +171,7 @@ def test_write_zarr_chunk_2d(self): assert tmp.sum() == 183750394 def test_write_zarr_3d(self): - """test_write_zarr_2d - Write zarr using TSWrtier""" + """test_write_zarr_2d - Write zarr using TsWriter""" br = TSReader( str(TEST_DIR.joinpath("5025551.zarr/0")), @@ -222,7 +222,7 @@ def test_write_zarr_3d(self): def test_write_zarr_chunk_3d(self): - """test_write_zarr_2d - Write zarr using TSWrtier""" + """test_write_zarr_2d - Write zarr using TsWriter""" br = TSReader( str(TEST_DIR.joinpath("5025551.zarr/0")), From 15f9aa29b17cccbddcc7f793341b5f895023a08c Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Wed, 17 Jul 2024 08:38:42 -0600 Subject: [PATCH 20/22] Update dimension order --- src/cpp/writer/tswriter.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/cpp/writer/tswriter.cpp b/src/cpp/writer/tswriter.cpp index 65850bf..844a751 100644 --- a/src/cpp/writer/tswriter.cpp +++ b/src/cpp/writer/tswriter.cpp @@ -57,9 +57,9 @@ void TsWriterCPP::WriteImageData( auto output_transform = tensorstore::IdentityTransform(_source.domain()); - if (_z_index.has_value() && layers.has_value()) { - output_transform = (std::move(output_transform) | tensorstore::Dims(_z_index.value()).ClosedInterval(layers.value().Start(), layers.value().Stop())).value(); - shape.emplace_back(layers.value().Stop() - layers.value().Start()+1); + if (_t_index.has_value() && tsteps.has_value()) { + output_transform = (std::move(output_transform) | tensorstore::Dims(_t_index.value()).ClosedInterval(tsteps.value().Start(), tsteps.value().Stop())).value(); + shape.emplace_back(tsteps.value().Stop() - tsteps.value().Start()+1); } if (_c_index.has_value() && channels.has_value()) { @@ -67,9 +67,9 @@ void TsWriterCPP::WriteImageData( shape.emplace_back(channels.value().Stop() - channels.value().Start()+1); } - if (_t_index.has_value() && tsteps.has_value()) { - output_transform = (std::move(output_transform) | tensorstore::Dims(_t_index.value()).ClosedInterval(tsteps.value().Start(), tsteps.value().Stop())).value(); - shape.emplace_back(tsteps.value().Stop() - tsteps.value().Start()+1); + if (_z_index.has_value() && layers.has_value()) { + output_transform = (std::move(output_transform) | tensorstore::Dims(_z_index.value()).ClosedInterval(layers.value().Start(), layers.value().Stop())).value(); + shape.emplace_back(layers.value().Stop() - layers.value().Start()+1); } output_transform = (std::move(output_transform) | tensorstore::Dims(_y_index).ClosedInterval(rows.Start(), rows.Stop()) | From d529bed240326f99cd4e0711eb071dee9be9559f Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Wed, 17 Jul 2024 08:38:53 -0600 Subject: [PATCH 21/22] Update dimensions and add test for 3D image --- tests/test_write.py | 68 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/tests/test_write.py b/tests/test_write.py index 1c1630d..4833369 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -52,8 +52,8 @@ def tearDownModule(): class TestZarrWrite(unittest.TestCase): - def test_write_zarr_2d(self): - """test_write_zarr_2d - Write zarr using TSWriter""" + def test_write_zarr_5d(self): + """test_write_zarr_5d - Write zarr using TSWriter""" br = TSReader( str(TEST_DIR.joinpath("5025551.zarr/0")), @@ -84,7 +84,7 @@ def test_write_zarr_2d(self): channels = Seq(0, 0, 1) tsteps = Seq(0, 0, 1) - bw = TSWriter(test_file_path, tmp.shape, tmp.shape, str(tmp.dtype), "ZCTYX") + bw = TSWriter(test_file_path, tmp.shape, tmp.shape, str(tmp.dtype), "TCZYX") bw.write_image_data(tmp, rows, cols, layers, channels, tsteps) bw.close() @@ -100,8 +100,56 @@ def test_write_zarr_2d(self): assert tmp.sum() == 183750394 assert tmp.shape == (1, 1, 1, 2700, 2702) - def test_write_zarr_chunk_2d(self): - """test_write_zarr_2d - Write zarr using TsWriter""" + def test_write_zarr_3d(self): + """test_write_zarr_5d - Write zarr using TSWriter""" + + br = TSReader( + str(TEST_DIR.joinpath("5025551.zarr/0")), + FileType.OmeZarr, + "", + ) + + assert br._X == 2702 + assert br._Y == 2700 + assert br._Z == 1 + assert br._C == 27 + assert br._T == 1 + + rows = Seq(0, br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(0, 0, 1) + tsteps = Seq(0, 0, 1) + tmp = br.data(rows, cols, layers, channels, tsteps) + + with tempfile.TemporaryDirectory() as dir: + # Use the temporary directory + test_file_path = os.path.join(dir, 'out/test.ome.zarr') + + rows = Seq(0, br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + + tmp = np.expand_dims(np.squeeze(tmp), axis=0) # modify image to be 3D instead of 5D + + bw = TSWriter(test_file_path, tmp.shape, tmp.shape, str(tmp.dtype), "ZYX") + bw.write_image_data(tmp, rows, cols, layers) + bw.close() + + br = TSReader( + str(test_file_path), + FileType.OmeZarr, + "", + ) + + tmp = br.data(rows, cols, layers) + + assert tmp.dtype == np.uint8 + assert tmp.sum() == 183750394 + assert tmp.shape == (1, 1, 1, 2700, 2702) + + def test_write_zarr_chunk_5d(self): + """test_write_zarr_5d - Write zarr using TsWriter""" br = TSReader( str(TEST_DIR.joinpath("5025551.zarr/0")), @@ -130,7 +178,7 @@ def test_write_zarr_chunk_2d(self): chunk_shape = [1, 1, 1, 1350, 1351] #chunk_shape = tmp.shape print("dtype: " + str(tmp.dtype)) - bw = TSWriter(test_file_path, tmp.shape, chunk_shape, str(tmp.dtype), "ZCTYX") + bw = TSWriter(test_file_path, tmp.shape, chunk_shape, str(tmp.dtype), "TCZYX") rows = Seq(0, br._Y//2, 1) cols = Seq(0, br._X - 1, 1) @@ -171,7 +219,7 @@ def test_write_zarr_chunk_2d(self): assert tmp.sum() == 183750394 def test_write_zarr_3d(self): - """test_write_zarr_2d - Write zarr using TsWriter""" + """test_write_zarr_5d - Write zarr using TsWriter""" br = TSReader( str(TEST_DIR.joinpath("5025551.zarr/0")), @@ -204,7 +252,7 @@ def test_write_zarr_3d(self): channels = Seq(0, 2, 1) tsteps = Seq(0, 0, 1) - bw = TSWriter(test_file_path, tmp.shape, tmp.shape, str(tmp.dtype), "ZCTYX") + bw = TSWriter(test_file_path, tmp.shape, tmp.shape, str(tmp.dtype), "TCZYX") bw.write_image_data(tmp, rows, cols, layers, channels, tsteps) bw.close() @@ -222,7 +270,7 @@ def test_write_zarr_3d(self): def test_write_zarr_chunk_3d(self): - """test_write_zarr_2d - Write zarr using TsWriter""" + """test_write_zarr_5d - Write zarr using TsWriter""" br = TSReader( str(TEST_DIR.joinpath("5025551.zarr/0")), @@ -251,7 +299,7 @@ def test_write_zarr_chunk_3d(self): chunk_size = (1,1,1,2700,2702) - bw = TSWriter(test_file_path, tmp.shape, chunk_size, str(tmp.dtype), "ZCTYX") + bw = TSWriter(test_file_path, tmp.shape, chunk_size, str(tmp.dtype), "TCZYX") # write first channel rows = Seq(0, br._Y - 1, 1) From fdc83f3cfe80e8bd439de0bb227ba93fff1f93ec Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Wed, 17 Jul 2024 08:55:48 -0600 Subject: [PATCH 22/22] Update 3d zarr write test --- tests/test_write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_write.py b/tests/test_write.py index 4833369..eced474 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -146,7 +146,7 @@ def test_write_zarr_3d(self): assert tmp.dtype == np.uint8 assert tmp.sum() == 183750394 - assert tmp.shape == (1, 1, 1, 2700, 2702) + assert tmp.shape == (1, 2700, 2702) def test_write_zarr_chunk_5d(self): """test_write_zarr_5d - Write zarr using TsWriter"""