From 6780144f5b1fd9b7fb7ae31cba2276315e011d96 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Thu, 20 Jun 2024 12:38:51 -0600 Subject: [PATCH 01/21] Add tensorstore zarr writer --- src/cpp/writer/tswriter.cpp | 127 ++++++++++++++ src/cpp/writer/tswriter.h | 330 ++++++++++++++++++++++++++++++++++++ 2 files changed, 457 insertions(+) create mode 100644 src/cpp/writer/tswriter.cpp create mode 100644 src/cpp/writer/tswriter.h diff --git a/src/cpp/writer/tswriter.cpp b/src/cpp/writer/tswriter.cpp new file mode 100644 index 0000000..087ab2e --- /dev/null +++ b/src/cpp/writer/tswriter.cpp @@ -0,0 +1,127 @@ +#include "tswriter.h" + +#include "../utilities/utilities.h" + +#include +#include + +using ::tensorstore::internal_zarr::ChooseBaseDType; + +namespace bfiocpp { + +TsWriterCPP::TsWriterCPP(const std::string& fname): _filename(fname) { + +} + +void TsWriterCPP::SetImageHeight(std::int64_t image_height) {_image_height = image_height;} +void TsWriterCPP::SetImageWidth (std::int64_t image_width) {_image_width = image_width;} +void TsWriterCPP::SetImageDepth (std::int64_t image_depth) {_image_depth = image_depth;} +void TsWriterCPP::SetTileHeight (std::int64_t tile_height) {_tile_height = tile_height;} +void TsWriterCPP::SetTileWidth (std::int64_t tile_width) {_tile_width = tile_width;} +void TsWriterCPP::SetTileDepth (std::int64_t tile_depth) {_tile_depth = tile_depth;} +void TsWriterCPP::SetChannelCount (std::int64_t num_channels) {_num_channels = num_channels;} +void TsWriterCPP::SetTstepCount (std::int64_t num_tsteps) {_num_tsteps = num_tsteps;} +void TsWriterCPP::SetDataType(const std::string& data_type) {_data_type = data_type;} + +void TsWriterCPP::write_image(py::array& py_image, const std::vector& image_shape, const std::vector& chunk_shape) { + + auto dt = py_image.dtype(); + auto dt_str = py::str(dt).cast(); + + dt_str = (dt_str == "float64") ? "double" : dt_str; // change float64 numpy type to double + + auto dtype = GetTensorStoreDataType(dt_str); + + std::string dtype_str = ChooseBaseDType(dtype).value().encoded_dtype; + + auto spec = GetZarrSpecToWrite(_filename, image_shape, chunk_shape, dtype_str); + + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( + spec, + tensorstore::OpenMode::create | + tensorstore::OpenMode::delete_existing, + tensorstore::ReadWriteMode::write).result()); + + // use if-else statement instead of template to avoid creating functions for each datatype + if (dt_str == "uint8") { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); + + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } else if (dt_str == "uint16") { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + + } else if (dt_str == "uint32") { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } else if (dt_str == "uint64") { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } else if (dt_str == "int8") { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + + } else if (dt_str == "int16") { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } else if (dt_str == "int32") { + + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } else if (dt_str == "int64") { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); + + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } else if (dt_str == "float") { + + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } else if (dt_str == "double") { + + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } else { + throw std::runtime_error("Unsupported data type: " + dt_str); + } + } +} \ No newline at end of file diff --git a/src/cpp/writer/tswriter.h b/src/cpp/writer/tswriter.h new file mode 100644 index 0000000..2269546 --- /dev/null +++ b/src/cpp/writer/tswriter.h @@ -0,0 +1,330 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include "../reader/sequence.h" + +#include "tensorstore/tensorstore.h" +#include "tensorstore/context.h" +#include "tensorstore/array.h" +#include "tensorstore/driver/zarr/dtype.h" +#include "tensorstore/index_space/dim_expression.h" +#include "tensorstore/kvstore/kvstore.h" +#include "tensorstore/open.h" + +#include +#include +#include + +namespace py = pybind11; + + +using image_data = std::variant, + std::vector, + std::vector, + std::vector, + std::vector, + std::vector, + std::vector, + std::vector, + std::vector, + std::vector>; + + + +using iter_indicies = std::tuple; + +namespace bfiocpp{ + +/* +struct VisitImage { + void operator()(const std::vector& image_vec, + const std::vector& image_shape, + const std::vector& chunk_shape, + const std::string& filename, + const std::string& dtype_str) { + + auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); + + auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); + + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( + spec, + tensorstore::OpenMode::create | + tensorstore::OpenMode::delete_existing, + tensorstore::ReadWriteMode::write).result()); + + + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } + void operator()(const std::vector& image_vec, + const std::vector& image_shape, + const std::vector& chunk_shape, + const std::string& filename, + const std::string& dtype_str) { + + auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); + + auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); + + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( + spec, + tensorstore::OpenMode::create | + tensorstore::OpenMode::delete_existing, + tensorstore::ReadWriteMode::write).result()); + + + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } + void operator()(const std::vector& image_vec, + const std::vector& image_shape, + const std::vector& chunk_shape, + const std::string& filename, + const std::string& dtype_str) { + + auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); + + auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); + + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( + spec, + tensorstore::OpenMode::create | + tensorstore::OpenMode::delete_existing, + tensorstore::ReadWriteMode::write).result()); + + + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } + void operator()(const std::vector& image_vec, + const std::vector& image_shape, + const std::vector& chunk_shape, + const std::string& filename, + const std::string& dtype_str) { + + auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); + + auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); + + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( + spec, + tensorstore::OpenMode::create | + tensorstore::OpenMode::delete_existing, + tensorstore::ReadWriteMode::write).result()); + + + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } + + void operator()(const std::vector& image_vec, + const std::vector& image_shape, + const std::vector& chunk_shape, + const std::string& filename, + const std::string& dtype_str) { + + auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); + + auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); + + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( + spec, + tensorstore::OpenMode::create | + tensorstore::OpenMode::delete_existing, + tensorstore::ReadWriteMode::write).result()); + + + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } + void operator()(const std::vector& image_vec, + const std::vector& image_shape, + const std::vector& chunk_shape, + const std::string& filename, + const std::string& dtype_str) { + + auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); + + auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); + + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( + spec, + tensorstore::OpenMode::create | + tensorstore::OpenMode::delete_existing, + tensorstore::ReadWriteMode::write).result()); + + + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } + void operator()(const std::vector& image_vec, + const std::vector& image_shape, + const std::vector& chunk_shape, + const std::string& filename, + const std::string& dtype_str) { + + auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); + + auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); + + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( + spec, + tensorstore::OpenMode::create | + tensorstore::OpenMode::delete_existing, + tensorstore::ReadWriteMode::write).result()); + + + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } + void operator()(const std::vector& image_vec, + const std::vector& image_shape, + const std::vector& chunk_shape, + const std::string& filename, + const std::string& dtype_str) { + + auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); + + auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); + + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( + spec, + tensorstore::OpenMode::create | + tensorstore::OpenMode::delete_existing, + tensorstore::ReadWriteMode::write).result()); + + + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } + void operator()(const std::vector& image_vec, + const std::vector& image_shape, + const std::vector& chunk_shape, + const std::string& filename, + const std::string& dtype_str) { + + auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); + + auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); + + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( + spec, + tensorstore::OpenMode::create | + tensorstore::OpenMode::delete_existing, + tensorstore::ReadWriteMode::write).result()); + + + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } + void operator()(const std::vector& image_vec, + const std::vector& image_shape, + const std::vector& chunk_shape, + const std::string& filename, + const std::string& dtype_str) { + + auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); + + auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); + + TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( + spec, + tensorstore::OpenMode::create | + tensorstore::OpenMode::delete_existing, + tensorstore::ReadWriteMode::write).result()); + + + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + } +}; +*/ + + +class TsWriterCPP{ +public: + TsWriterCPP(const std::string& fname); + + void SetImageHeight(std::int64_t image_height); + void SetImageWidth (std::int64_t image_width); + void SetImageDepth (std::int64_t image_depth); + void SetTileHeight (std::int64_t tile_height); + void SetTileWidth (std::int64_t tile_width); + void SetTileDepth (std::int64_t tile_depth); + void SetChannelCount (std::int64_t num_channels); + void SetTstepCount (std::int64_t num_steps); + void SetDataType(const std::string& data_type); + + void write_image(py::array& py_image, const std::vector& image_shape, const std::vector& chunk_shape); + +private: + std::string _filename, _data_type; + std::int64_t _image_height, + _image_width, + _image_depth, + _tile_height, + _tile_width, + _tile_depth, + _num_channels, + _num_tsteps; + std::uint16_t _data_type_code; + std::optional_z_index, _c_index, _t_index; + + std::string get_variant_type(const image_data& image) { + return std::visit([](auto&& arg) -> std::string { + using T = std::decay_t; + + if constexpr (std::is_same_v>) return "uint8"; + else if constexpr (std::is_same_v>) return "uint16"; + else if constexpr (std::is_same_v>) return "uint32"; + else if constexpr (std::is_same_v>) return "uint64"; + else if constexpr (std::is_same_v>) return "int8"; + else if constexpr (std::is_same_v>) return "int16"; + else if constexpr (std::is_same_v>) return "int32"; + else if constexpr (std::is_same_v>) return "int64"; + else if constexpr (std::is_same_v>) return "float"; + else if constexpr (std::is_same_v>) return "double"; + else return "Unknown type"; + }, image); + } +}; +} + From 35f48cc521f198257b95ac6f189c51f835f1bf24 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Thu, 20 Jun 2024 12:39:09 -0600 Subject: [PATCH 02/21] Add tensorstore writer --- CMakeLists.txt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 52f7856..43dc4e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,14 +7,14 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) include(GNUInstallDirs) # this is a workaround for GitHub Action for wheelbuiling -if(DEFINED ENV{BFIO_CPP_DEP_DIR}) - set(CMAKE_PREFIX_PATH $ENV{BFIO_CPP_DEP_DIR}) - link_directories($ENV{BFIO_CPP_DEP_DIR}/${CMAKE_INSTALL_LIBDIR}) +if(DEFINED ENV{BFIOCPP_DEP_DIR}) + set(CMAKE_PREFIX_PATH $ENV{BFIOCPP_DEP_DIR}) + link_directories($ENV{BFIOCPP_DEP_DIR}/${CMAKE_INSTALL_LIBDIR}) endif() message(STATUS "CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH}") if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) - endif() +endif() set(CMAKE_CXX_FLAGS_RELEASE "-O2") set(SOURCE @@ -24,7 +24,8 @@ set(SOURCE src/cpp/ts_driver/ometiff/driver.cc src/cpp/interface/interface.cpp src/cpp/reader/tsreader.cpp - src/cpp/reader/utilities.cpp + src/cpp/utilities/utilities.cpp + src/cpp/writer/tswriter.cpp ) include(FetchContent) @@ -68,4 +69,4 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") endif() target_link_libraries(libbfiocpp PRIVATE tensorstore::tensorstore tensorstore::all_drivers) -target_link_libraries(libbfiocpp PRIVATE ${Build_LIBRARIES}) +target_link_libraries(libbfiocpp PRIVATE ${Build_LIBRARIES}) \ No newline at end of file From 375b77d2dfdc190403d879ef884abc260e80e78d Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Thu, 20 Jun 2024 12:39:27 -0600 Subject: [PATCH 03/21] Add python interface for zarr writer --- src/cpp/interface/interface.cpp | 12 +++++++-- src/python/bfiocpp/tswriter.py | 47 +++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 src/python/bfiocpp/tswriter.py diff --git a/src/cpp/interface/interface.cpp b/src/cpp/interface/interface.cpp index a6a6f15..a27165e 100644 --- a/src/cpp/interface/interface.cpp +++ b/src/cpp/interface/interface.cpp @@ -1,10 +1,12 @@ #include #include +#include #include #include #include "../reader/tsreader.h" #include "../reader/sequence.h" -#include "../reader/utilities.h" +#include "../utilities/utilities.h" +#include "../writer/tswriter.h" namespace py = pybind11; using bfiocpp::Seq; @@ -116,4 +118,10 @@ PYBIND11_MODULE(libbfiocpp, m) { .export_values(); m.def("get_ome_xml", &bfiocpp::GetOmeXml); -} \ No newline at end of file + + + // Writer class + py::class_>(m, "TsWriterCPP") + .def(py::init()) + .def("write", &bfiocpp::TsWriterCPP::write_image); +} diff --git a/src/python/bfiocpp/tswriter.py b/src/python/bfiocpp/tswriter.py new file mode 100644 index 0000000..ae202af --- /dev/null +++ b/src/python/bfiocpp/tswriter.py @@ -0,0 +1,47 @@ +import numpy as np +from typing import Tuple +from .libbfiocpp import TsWriterCPP, FileType + + +class TSWriter: + + def __init__(self, file_name: str): + self._image_writer: TsWriterCPP = TsWriterCPP(file_name) + + def write_image(self, image_data: np.ndarray, image_shape:list, chunk_shape: list): + """ Write image data to file + + image_data: 5d numpy array containing image data + """ + if (not isinstance(image_data, np.ndarray)): + raise ValueError("Image data must be a 5d numpy array") + + try: + self._image_writer.write(image_data.flatten(), image_shape, chunk_shape) + except Exception as e: + raise RuntimeError(f"Error writing image data: {e.what}") + + def close(self): + pass + + def __enter__(self) -> "TSWriter": + """Handle entrance to a context manager. + + This code is called when a `with` statement is used. + ... + """ + return self + + def __del__(self) -> None: + """Handle file deletion. + + This code runs when an object is deleted.. + """ + self.close() + + def __exit__(self, type_class, value, traceback) -> None: + """Handle exit from the context manager. + + This code runs when exiting a `with` statement. + """ + self.close() From 6872d8213f8999ef6fd2dc41484d6f2a58003462 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Thu, 20 Jun 2024 12:39:58 -0600 Subject: [PATCH 04/21] Move utilities to separate directory --- src/cpp/reader/tsreader.cpp | 2 +- src/cpp/{reader => utilities}/utilities.cpp | 51 ++++++++++++++++++++- src/cpp/{reader => utilities}/utilities.h | 6 +++ 3 files changed, 57 insertions(+), 2 deletions(-) rename src/cpp/{reader => utilities}/utilities.cpp (60%) rename src/cpp/{reader => utilities}/utilities.h (59%) diff --git a/src/cpp/reader/tsreader.cpp b/src/cpp/reader/tsreader.cpp index 2dc36b2..6171426 100644 --- a/src/cpp/reader/tsreader.cpp +++ b/src/cpp/reader/tsreader.cpp @@ -7,7 +7,7 @@ #include "tensorstore/open.h" #include "tsreader.h" -#include "utilities.h" +#include "../utilities/utilities.h" #include "type_info.h" diff --git a/src/cpp/reader/utilities.cpp b/src/cpp/utilities/utilities.cpp similarity index 60% rename from src/cpp/reader/utilities.cpp rename to src/cpp/utilities/utilities.cpp index b88c67f..f360512 100644 --- a/src/cpp/reader/utilities.cpp +++ b/src/cpp/utilities/utilities.cpp @@ -4,6 +4,7 @@ #include "utilities.h" #include #include +#include namespace bfiocpp { tensorstore::Spec GetOmeTiffSpecToRead(const std::string& filename){ @@ -29,7 +30,6 @@ tensorstore::Spec GetZarrSpecToRead(const std::string& filename){ } - uint16_t GetDataTypeCode (std::string_view type_name){ if (type_name == std::string_view{"uint8"}) {return 1;} @@ -108,4 +108,53 @@ std::string GetOmeXml(const std::string& file_path){ return OmeXmlInfo; } +tensorstore::Spec GetZarrSpecToWrite( const std::string& filename, + const std::vector& image_shape, + const std::vector& chunk_shape, + const std::string& dtype){ + return tensorstore::Spec::FromJson({{"driver", "zarr"}, + {"kvstore", {{"driver", "file"}, + {"path", filename}} + }, + {"context", { + {"cache_pool", {{"total_bytes_limit", 1000000000}}}, + {"data_copy_concurrency", {{"limit", std::thread::hardware_concurrency()}}}, + {"file_io_concurrency", {{"limit", std::thread::hardware_concurrency()}}}, + }}, + {"metadata", { + {"zarr_format", 2}, + {"shape", image_shape}, + {"chunks", chunk_shape}, + {"dtype", dtype}, + }, + }}).value(); + } + +// Function to get the TensorStore DataType based on a string identifier +tensorstore::DataType GetTensorStoreDataType(const std::string& type_str) { + if (type_str == "uint8") { + return tensorstore::dtype_v; + } else if (type_str == "uint16") { + return tensorstore::dtype_v; + } else if (type_str == "uint32") { + return tensorstore::dtype_v; + } else if (type_str == "uint64") { + return tensorstore::dtype_v; + } else if (type_str == "int8") { + return tensorstore::dtype_v; + } else if (type_str == "int16") { + return tensorstore::dtype_v; + } else if (type_str == "int32") { + return tensorstore::dtype_v; + } else if (type_str == "int64") { + return tensorstore::dtype_v; + } else if (type_str == "float") { + return tensorstore::dtype_v; + } else if (type_str == "double" || type_str == "float64") { // handle float64 from numpy + return tensorstore::dtype_v; + } else { + throw std::invalid_argument("Unknown data type string: " + type_str); + } +} + } // ns bfiocpp \ No newline at end of file diff --git a/src/cpp/reader/utilities.h b/src/cpp/utilities/utilities.h similarity index 59% rename from src/cpp/reader/utilities.h rename to src/cpp/utilities/utilities.h index 29ed949..0da9ba1 100644 --- a/src/cpp/reader/utilities.h +++ b/src/cpp/utilities/utilities.h @@ -18,4 +18,10 @@ uint16_t GetDataTypeCode (std::string_view type_name); std::string GetUTCString(); std::string GetOmeXml(const std::string& file_path); std::tuple, std::optional, std::optional>ParseMultiscaleMetadata(const std::string& axes_list, int len); +tensorstore::DataType GetTensorStoreDataType(const std::string& type_str); +uint16_t GetDataTypeCode (std::string_view type_name); +tensorstore::Spec GetZarrSpecToWrite(const std::string& filename, + const std::vector& image_shape, + const std::vector& chunk_shape, + const std::string& dtype); } // ns bfiocpp \ No newline at end of file From d7e79f4e802389892c566161b79e3878c92b41b6 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Thu, 20 Jun 2024 12:40:14 -0600 Subject: [PATCH 05/21] Add python interface for TSWriter --- src/python/bfiocpp/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/bfiocpp/__init__.py b/src/python/bfiocpp/__init__.py index 955243a..3b035aa 100644 --- a/src/python/bfiocpp/__init__.py +++ b/src/python/bfiocpp/__init__.py @@ -1,4 +1,5 @@ from .tsreader import TSReader, Seq, FileType, get_ome_xml # NOQA: F401 +from .tswriter import TSWriter from . import _version __version__ = _version.get_versions()["version"] From 2ecf5cc365924880c115fe05757eb56da18db859 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Thu, 20 Jun 2024 13:37:39 -0600 Subject: [PATCH 06/21] Move module setup to conftest.py --- tests/__init__.py | 1 + tests/conftest.py | 55 ++++++++++++++++++++++++++++++++++++++++ tests/test_read.py | 63 +--------------------------------------------- 3 files changed, 57 insertions(+), 62 deletions(-) create mode 100644 tests/conftest.py diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..950237f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1 @@ +from .conftest import TEST_DIR \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..4fad7c1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,55 @@ +import requests, pathlib, shutil, logging, sys +from ome_zarr.utils import download as zarr_download + +TEST_IMAGES = { + "5025551.zarr": "https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.4/idr0054A/5025551.zarr", + "p01_x01_y01_wx0_wy0_c1.ome.tif": "https://raw.githubusercontent.com/sameeul/polus-test-data/main/bfio/p01_x01_y01_wx0_wy0_c1.ome.tif", + "Plate1-Blue-A-12-Scene-3-P3-F2-03.czi": "https://downloads.openmicroscopy.org/images/Zeiss-CZI/idr0011/Plate1-Blue-A_TS-Stinger/Plate1-Blue-A-12-Scene-3-P3-F2-03.czi", +} + +TEST_DIR = pathlib.Path(__file__).with_name("data") + +logging.basicConfig( + format="%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) +logger = logging.getLogger("bfio.test") + +if "-v" in sys.argv: + logger.setLevel(logging.INFO) + + +def setUpModule(): + """Download images for testing""" + TEST_DIR.mkdir(exist_ok=True) + + for file, url in TEST_IMAGES.items(): + logger.info(f"setup - Downloading: {file}") + + if not file.endswith(".zarr"): + if TEST_DIR.joinpath(file).exists(): + continue + + r = requests.get(url) + + with open(TEST_DIR.joinpath(file), "wb") as fw: + fw.write(r.content) + else: + if TEST_DIR.joinpath(file).exists(): + shutil.rmtree(TEST_DIR.joinpath(file)) + zarr_download(url, str(TEST_DIR)) + + """Load the czi image, and save as a npy file for further testing.""" + with bfio.BioReader( + TEST_DIR.joinpath("Plate1-Blue-A-12-Scene-3-P3-F2-03.czi") + ) as br: + with bfio.BioWriter( + TEST_DIR.joinpath("4d_array.ome.tif"), + metadata=br.metadata, + X=br.X, + Y=br.Y, + Z=br.Z, + C=br.C, + T=br.T, + ) as bw: + bw[:] = br[:] \ No newline at end of file diff --git a/tests/test_read.py b/tests/test_read.py index dff293b..1c8cf70 100644 --- a/tests/test_read.py +++ b/tests/test_read.py @@ -1,71 +1,10 @@ from bfiocpp import TSReader, Seq, FileType import unittest -import requests, pathlib, shutil, logging, sys import bfio import numpy as np import random -from ome_zarr.utils import download as zarr_download - -TEST_IMAGES = { - "5025551.zarr": "https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.4/idr0054A/5025551.zarr", - "p01_x01_y01_wx0_wy0_c1.ome.tif": "https://raw.githubusercontent.com/sameeul/polus-test-data/main/bfio/p01_x01_y01_wx0_wy0_c1.ome.tif", - "Plate1-Blue-A-12-Scene-3-P3-F2-03.czi": "https://downloads.openmicroscopy.org/images/Zeiss-CZI/idr0011/Plate1-Blue-A_TS-Stinger/Plate1-Blue-A-12-Scene-3-P3-F2-03.czi", -} - -TEST_DIR = pathlib.Path(__file__).with_name("data") - -logging.basicConfig( - format="%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s", - datefmt="%d-%b-%y %H:%M:%S", -) -logger = logging.getLogger("bfio.test") - -if "-v" in sys.argv: - logger.setLevel(logging.INFO) - - -def setUpModule(): - """Download images for testing""" - TEST_DIR.mkdir(exist_ok=True) - - for file, url in TEST_IMAGES.items(): - logger.info(f"setup - Downloading: {file}") - - if not file.endswith(".zarr"): - if TEST_DIR.joinpath(file).exists(): - continue - - r = requests.get(url) - - with open(TEST_DIR.joinpath(file), "wb") as fw: - fw.write(r.content) - else: - if TEST_DIR.joinpath(file).exists(): - shutil.rmtree(TEST_DIR.joinpath(file)) - zarr_download(url, str(TEST_DIR)) - - """Load the czi image, and save as a npy file for further testing.""" - with bfio.BioReader( - TEST_DIR.joinpath("Plate1-Blue-A-12-Scene-3-P3-F2-03.czi") - ) as br: - with bfio.BioWriter( - TEST_DIR.joinpath("4d_array.ome.tif"), - metadata=br.metadata, - X=br.X, - Y=br.Y, - Z=br.Z, - C=br.C, - T=br.T, - ) as bw: - bw[:] = br[:] - - -def tearDownModule(): - """Remove test images""" - - logger.info("teardown - Removing test images...") - # shutil.rmtree(TEST_DIR) +from . import TEST_DIR class TestOmeTiffRead(unittest.TestCase): From 182e03c7a1e0db3303b9073d86c315ee52d8282d Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Thu, 20 Jun 2024 13:37:46 -0600 Subject: [PATCH 07/21] Add test for zarr writer --- tests/test_write.py | 48 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tests/test_write.py diff --git a/tests/test_write.py b/tests/test_write.py new file mode 100644 index 0000000..c4e0596 --- /dev/null +++ b/tests/test_write.py @@ -0,0 +1,48 @@ +import unittest +import zarr +import tempfile +import os +import numpy as np + +from bfiocpp import TSWriter, TSReader, Seq, FileType +from . import TEST_DIR + +class TestZarrWrite(unittest.TestCase): + + def test_write_zarr_2d(self): + """test_write_zarr_2d - Write zarr using TSWrtier""" + + br = TSReader( + str(TEST_DIR.joinpath("5025551.zarr/0")), + FileType.OmeZarr, + "", + ) + assert br._X == 2702 + assert br._Y == 2700 + assert br._Z == 1 + assert br._C == 27 + assert br._T == 1 + + rows = Seq(0, br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(0, 0, 1) + tsteps = Seq(0, 0, 1) + tmp = br.data(rows, cols, layers, channels, tsteps) + + with tempfile.TemporaryDirectory() as test_dir: + # Use the temporary directory + test_file_path = os.path.join(test_dir, 'out/test.ome.zarr') + writer = TSWriter('out/test.ome.zarr') + writer.write(tmp, tmp.shape, tmp.shape) + + bw = TSWriter(test_file_path) + bw.write(tmp, tmp.shape, tmp.shape) + bw.close() + + # read zarr back into memory to check data + z2 = zarr.open(test_file_path, mode='r')[:] + + assert z2.dtype == np.uint8 + assert z2.sum() == 183750394 + assert z2.shape == (1, 1, 1, 2700, 2702) From f360bfa6624c737749745539889ae4470a5db365 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Thu, 20 Jun 2024 13:57:19 -0600 Subject: [PATCH 08/21] Add zarr requirement for unit tests --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 2996b62..a4f12b6 100644 --- a/setup.py +++ b/setup.py @@ -109,4 +109,7 @@ def build_extension(self, ext): install_requires=[ "numpy", ], + tests_require=[ + "zarr", + ], ) From 217f94613f34fc0dfc99f5b1b8d2a43211c5af6e Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Thu, 20 Jun 2024 13:58:08 -0600 Subject: [PATCH 09/21] Add zarr to TEST_REQUIRES --- .github/workflows/publish_pypi.yml | 2 +- .github/workflows/wheel_build.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 47d3d4c..ebc2ba3 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -65,7 +65,7 @@ jobs: CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\bfiocpp\\bin;$PATH" ON_GITHUB="TRUE" BFIO_CPP_DEP_DIR="C:\\TEMP\\bfiocpp_bld\\local_install" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" CIBW_ARCHS: ${{ matrix.cibw_archs }} - CIBW_TEST_REQUIRES: bfio requests numpy ome_zarr + CIBW_TEST_REQUIRES: bfio requests numpy ome_zarr zarr CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v - name: Install Dependencies diff --git a/.github/workflows/wheel_build.yml b/.github/workflows/wheel_build.yml index 43d6094..3f38b66 100644 --- a/.github/workflows/wheel_build.yml +++ b/.github/workflows/wheel_build.yml @@ -63,7 +63,7 @@ jobs: CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\bfiocpp\\bin;$PATH" ON_GITHUB="TRUE" BFIO_CPP_DEP_DIR="C:\\TEMP\\bfiocpp_bld\\local_install" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" CIBW_ARCHS: ${{ matrix.cibw_archs }} - CIBW_TEST_REQUIRES: bfio requests numpy ome_zarr + CIBW_TEST_REQUIRES: bfio requests numpy ome_zarr zarr CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v build_wheels_apple_arm64: @@ -107,7 +107,7 @@ jobs: CIBW_ENVIRONMENT_MACOS: REPAIR_LIBRARY_PATH="/tmp/bfiocpp_bld/local_install/lib:/tmp/bfiocpp_bld/local_install/lib64" ON_GITHUB="TRUE" BFIO_CPP_DEP_DIR="/tmp/bfiocpp_bld/local_install" CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON" CIBW_REPAIR_WHEEL_COMMAND_MACOS: DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} CIBW_ARCHS: ${{ matrix.cibw_archs }} - CIBW_TEST_REQUIRES: bfio requests numpy ome_zarr + CIBW_TEST_REQUIRES: bfio requests numpy ome_zarr zarr CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v - name: Upload Artifact From 39fa10160f09a873f89e9d39b62eeaae2b8ff28a Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Fri, 21 Jun 2024 08:51:55 -0600 Subject: [PATCH 10/21] Remove unused methods --- src/cpp/writer/tswriter.h | 257 -------------------------------------- 1 file changed, 257 deletions(-) diff --git a/src/cpp/writer/tswriter.h b/src/cpp/writer/tswriter.h index 2269546..bc0c527 100644 --- a/src/cpp/writer/tswriter.h +++ b/src/cpp/writer/tswriter.h @@ -42,243 +42,6 @@ using iter_indicies = std::tuple& image_vec, - const std::vector& image_shape, - const std::vector& chunk_shape, - const std::string& filename, - const std::string& dtype_str) { - - auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); - - auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); - - TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( - spec, - tensorstore::OpenMode::create | - tensorstore::OpenMode::delete_existing, - tensorstore::ReadWriteMode::write).result()); - - - // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; - } - } - void operator()(const std::vector& image_vec, - const std::vector& image_shape, - const std::vector& chunk_shape, - const std::string& filename, - const std::string& dtype_str) { - - auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); - - auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); - - TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( - spec, - tensorstore::OpenMode::create | - tensorstore::OpenMode::delete_existing, - tensorstore::ReadWriteMode::write).result()); - - - // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; - } - } - void operator()(const std::vector& image_vec, - const std::vector& image_shape, - const std::vector& chunk_shape, - const std::string& filename, - const std::string& dtype_str) { - - auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); - - auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); - - TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( - spec, - tensorstore::OpenMode::create | - tensorstore::OpenMode::delete_existing, - tensorstore::ReadWriteMode::write).result()); - - - // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; - } - } - void operator()(const std::vector& image_vec, - const std::vector& image_shape, - const std::vector& chunk_shape, - const std::string& filename, - const std::string& dtype_str) { - - auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); - - auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); - - TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( - spec, - tensorstore::OpenMode::create | - tensorstore::OpenMode::delete_existing, - tensorstore::ReadWriteMode::write).result()); - - - // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; - } - } - - void operator()(const std::vector& image_vec, - const std::vector& image_shape, - const std::vector& chunk_shape, - const std::string& filename, - const std::string& dtype_str) { - - auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); - - auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); - - TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( - spec, - tensorstore::OpenMode::create | - tensorstore::OpenMode::delete_existing, - tensorstore::ReadWriteMode::write).result()); - - - // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; - } - } - void operator()(const std::vector& image_vec, - const std::vector& image_shape, - const std::vector& chunk_shape, - const std::string& filename, - const std::string& dtype_str) { - - auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); - - auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); - - TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( - spec, - tensorstore::OpenMode::create | - tensorstore::OpenMode::delete_existing, - tensorstore::ReadWriteMode::write).result()); - - - // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; - } - } - void operator()(const std::vector& image_vec, - const std::vector& image_shape, - const std::vector& chunk_shape, - const std::string& filename, - const std::string& dtype_str) { - - auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); - - auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); - - TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( - spec, - tensorstore::OpenMode::create | - tensorstore::OpenMode::delete_existing, - tensorstore::ReadWriteMode::write).result()); - - - // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; - } - } - void operator()(const std::vector& image_vec, - const std::vector& image_shape, - const std::vector& chunk_shape, - const std::string& filename, - const std::string& dtype_str) { - - auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); - - auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); - - TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( - spec, - tensorstore::OpenMode::create | - tensorstore::OpenMode::delete_existing, - tensorstore::ReadWriteMode::write).result()); - - - // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; - } - } - void operator()(const std::vector& image_vec, - const std::vector& image_shape, - const std::vector& chunk_shape, - const std::string& filename, - const std::string& dtype_str) { - - auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); - - auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); - - TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( - spec, - tensorstore::OpenMode::create | - tensorstore::OpenMode::delete_existing, - tensorstore::ReadWriteMode::write).result()); - - - // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; - } - } - void operator()(const std::vector& image_vec, - const std::vector& image_shape, - const std::vector& chunk_shape, - const std::string& filename, - const std::string& dtype_str) { - - auto data_array = tensorstore::Array(image_vec.data(), image_shape, tensorstore::c_order); - - auto spec = GetZarrSpecToWrite(filename, image_shape, chunk_shape, dtype_str); - - TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( - spec, - tensorstore::OpenMode::create | - tensorstore::OpenMode::delete_existing, - tensorstore::ReadWriteMode::write).result()); - - - // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; - } - } -}; -*/ - - class TsWriterCPP{ public: TsWriterCPP(const std::string& fname); @@ -305,26 +68,6 @@ class TsWriterCPP{ _tile_depth, _num_channels, _num_tsteps; - std::uint16_t _data_type_code; - std::optional_z_index, _c_index, _t_index; - - std::string get_variant_type(const image_data& image) { - return std::visit([](auto&& arg) -> std::string { - using T = std::decay_t; - - if constexpr (std::is_same_v>) return "uint8"; - else if constexpr (std::is_same_v>) return "uint16"; - else if constexpr (std::is_same_v>) return "uint32"; - else if constexpr (std::is_same_v>) return "uint64"; - else if constexpr (std::is_same_v>) return "int8"; - else if constexpr (std::is_same_v>) return "int16"; - else if constexpr (std::is_same_v>) return "int32"; - else if constexpr (std::is_same_v>) return "int64"; - else if constexpr (std::is_same_v>) return "float"; - else if constexpr (std::is_same_v>) return "double"; - else return "Unknown type"; - }, image); - } }; } From 61e85d813399cf2588a17bf589782b394dc0cf61 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Fri, 21 Jun 2024 08:54:53 -0600 Subject: [PATCH 11/21] Update test to use TSReader --- setup.py | 5 +---- tests/test_write.py | 26 +++++++++++++++++--------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/setup.py b/setup.py index a4f12b6..19c3806 100644 --- a/setup.py +++ b/setup.py @@ -108,8 +108,5 @@ def build_extension(self, ext): python_requires=">=3.8", install_requires=[ "numpy", - ], - tests_require=[ - "zarr", - ], + ], ) diff --git a/tests/test_write.py b/tests/test_write.py index c4e0596..6f7e068 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -1,5 +1,4 @@ import unittest -import zarr import tempfile import os import numpy as np @@ -33,16 +32,25 @@ def test_write_zarr_2d(self): with tempfile.TemporaryDirectory() as test_dir: # Use the temporary directory test_file_path = os.path.join(test_dir, 'out/test.ome.zarr') - writer = TSWriter('out/test.ome.zarr') - writer.write(tmp, tmp.shape, tmp.shape) bw = TSWriter(test_file_path) - bw.write(tmp, tmp.shape, tmp.shape) + bw.write_image(tmp, tmp.shape, tmp.shape) bw.close() - # read zarr back into memory to check data - z2 = zarr.open(test_file_path, mode='r')[:] + br = TSReader( + str(test_file_path), + FileType.OmeZarr, + "", + ) + + rows = Seq(0, br._Y - 1, 1) + cols = Seq(0, br._X - 1, 1) + layers = Seq(0, 0, 1) + channels = Seq(0, 0, 1) + tsteps = Seq(0, 0, 1) + tmp = br.data(rows, cols, layers, channels, tsteps) + + assert tmp.dtype == np.uint8 + assert tmp.sum() == 183750394 + assert tmp.shape == (1, 1, 1, 2700, 2702) - assert z2.dtype == np.uint8 - assert z2.sum() == 183750394 - assert z2.shape == (1, 1, 1, 2700, 2702) From 23ad5dd9b1de6948446164e1528d751157719d3d Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Mon, 24 Jun 2024 07:04:53 -0600 Subject: [PATCH 12/21] Fix flake8 formatting --- setup.py | 2 +- src/python/bfiocpp/__init__.py | 2 +- src/python/bfiocpp/tswriter.py | 22 ++++++++++++++++------ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 419f3de..09f4936 100644 --- a/setup.py +++ b/setup.py @@ -105,5 +105,5 @@ def build_extension(self, ext): python_requires=">=3.8", install_requires=[ "numpy", - ], + ], ) diff --git a/src/python/bfiocpp/__init__.py b/src/python/bfiocpp/__init__.py index 3b035aa..5e89052 100644 --- a/src/python/bfiocpp/__init__.py +++ b/src/python/bfiocpp/__init__.py @@ -1,5 +1,5 @@ from .tsreader import TSReader, Seq, FileType, get_ome_xml # NOQA: F401 -from .tswriter import TSWriter +from .tswriter import TSWriter # NOQA: F401 from . import _version __version__ = _version.get_versions()["version"] diff --git a/src/python/bfiocpp/tswriter.py b/src/python/bfiocpp/tswriter.py index ae202af..4261d16 100644 --- a/src/python/bfiocpp/tswriter.py +++ b/src/python/bfiocpp/tswriter.py @@ -1,27 +1,34 @@ import numpy as np -from typing import Tuple -from .libbfiocpp import TsWriterCPP, FileType +from .libbfiocpp import TsWriterCPP class TSWriter: def __init__(self, file_name: str): + """ Initialize tensorstore Zarr writer + + file_name: Path to write file to + """ + self._image_writer: TsWriterCPP = TsWriterCPP(file_name) - - def write_image(self, image_data: np.ndarray, image_shape:list, chunk_shape: list): + + def write_image(self, image_data: np.ndarray, image_shape: list, chunk_shape: list): """ Write image data to file - + image_data: 5d numpy array containing image data """ + if (not isinstance(image_data, np.ndarray)): + raise ValueError("Image data must be a 5d numpy array") - + try: self._image_writer.write(image_data.flatten(), image_shape, chunk_shape) except Exception as e: raise RuntimeError(f"Error writing image data: {e.what}") def close(self): + pass def __enter__(self) -> "TSWriter": @@ -30,6 +37,7 @@ def __enter__(self) -> "TSWriter": This code is called when a `with` statement is used. ... """ + return self def __del__(self) -> None: @@ -37,6 +45,7 @@ def __del__(self) -> None: This code runs when an object is deleted.. """ + self.close() def __exit__(self, type_class, value, traceback) -> None: @@ -44,4 +53,5 @@ def __exit__(self, type_class, value, traceback) -> None: This code runs when exiting a `with` statement. """ + self.close() From 772326a5de05c1f849405b6c4f6660c364de213b Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Mon, 24 Jun 2024 07:08:19 -0600 Subject: [PATCH 13/21] Reformat --- src/python/bfiocpp/tswriter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python/bfiocpp/tswriter.py b/src/python/bfiocpp/tswriter.py index 4261d16..9713375 100644 --- a/src/python/bfiocpp/tswriter.py +++ b/src/python/bfiocpp/tswriter.py @@ -5,7 +5,7 @@ class TSWriter: def __init__(self, file_name: str): - """ Initialize tensorstore Zarr writer + """Initialize tensorstore Zarr writer file_name: Path to write file to """ @@ -13,12 +13,12 @@ def __init__(self, file_name: str): self._image_writer: TsWriterCPP = TsWriterCPP(file_name) def write_image(self, image_data: np.ndarray, image_shape: list, chunk_shape: list): - """ Write image data to file + """Write image data to file image_data: 5d numpy array containing image data """ - if (not isinstance(image_data, np.ndarray)): + if not isinstance(image_data, np.ndarray): raise ValueError("Image data must be a 5d numpy array") From 15949d6dad1076155d8af9d6e8f947c27cf916c7 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Mon, 24 Jun 2024 08:37:24 -0600 Subject: [PATCH 14/21] Update data directory path --- tests/test_read.py | 3 ++- tests/test_write.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_read.py b/tests/test_read.py index 1c8cf70..9c34b16 100644 --- a/tests/test_read.py +++ b/tests/test_read.py @@ -3,8 +3,9 @@ import bfio import numpy as np import random +import pathlib -from . import TEST_DIR +TEST_DIR = pathlib.Path(__file__).with_name("data") class TestOmeTiffRead(unittest.TestCase): diff --git a/tests/test_write.py b/tests/test_write.py index 6f7e068..1d981d2 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -2,9 +2,11 @@ import tempfile import os import numpy as np +import pathlib from bfiocpp import TSWriter, TSReader, Seq, FileType -from . import TEST_DIR + +TEST_DIR = pathlib.Path(__file__).with_name("data") class TestZarrWrite(unittest.TestCase): From 6d827e09d28f0e7b5f13d2e7378bf81b7db8e5bb Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 25 Jun 2024 06:47:58 -0600 Subject: [PATCH 15/21] Update module setup for tests --- tests/__init__.py | 1 - tests/conftest.py | 55 ---------------------------------------- tests/test_read.py | 61 ++++++++++++++++++++++++++++++++++++++++++++- tests/test_write.py | 54 ++++++++++++++++++++++++++++++++++----- 4 files changed, 108 insertions(+), 63 deletions(-) delete mode 100644 tests/conftest.py diff --git a/tests/__init__.py b/tests/__init__.py index 950237f..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +0,0 @@ -from .conftest import TEST_DIR \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 4fad7c1..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,55 +0,0 @@ -import requests, pathlib, shutil, logging, sys -from ome_zarr.utils import download as zarr_download - -TEST_IMAGES = { - "5025551.zarr": "https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.4/idr0054A/5025551.zarr", - "p01_x01_y01_wx0_wy0_c1.ome.tif": "https://raw.githubusercontent.com/sameeul/polus-test-data/main/bfio/p01_x01_y01_wx0_wy0_c1.ome.tif", - "Plate1-Blue-A-12-Scene-3-P3-F2-03.czi": "https://downloads.openmicroscopy.org/images/Zeiss-CZI/idr0011/Plate1-Blue-A_TS-Stinger/Plate1-Blue-A-12-Scene-3-P3-F2-03.czi", -} - -TEST_DIR = pathlib.Path(__file__).with_name("data") - -logging.basicConfig( - format="%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s", - datefmt="%d-%b-%y %H:%M:%S", -) -logger = logging.getLogger("bfio.test") - -if "-v" in sys.argv: - logger.setLevel(logging.INFO) - - -def setUpModule(): - """Download images for testing""" - TEST_DIR.mkdir(exist_ok=True) - - for file, url in TEST_IMAGES.items(): - logger.info(f"setup - Downloading: {file}") - - if not file.endswith(".zarr"): - if TEST_DIR.joinpath(file).exists(): - continue - - r = requests.get(url) - - with open(TEST_DIR.joinpath(file), "wb") as fw: - fw.write(r.content) - else: - if TEST_DIR.joinpath(file).exists(): - shutil.rmtree(TEST_DIR.joinpath(file)) - zarr_download(url, str(TEST_DIR)) - - """Load the czi image, and save as a npy file for further testing.""" - with bfio.BioReader( - TEST_DIR.joinpath("Plate1-Blue-A-12-Scene-3-P3-F2-03.czi") - ) as br: - with bfio.BioWriter( - TEST_DIR.joinpath("4d_array.ome.tif"), - metadata=br.metadata, - X=br.X, - Y=br.Y, - Z=br.Z, - C=br.C, - T=br.T, - ) as bw: - bw[:] = br[:] \ No newline at end of file diff --git a/tests/test_read.py b/tests/test_read.py index 9c34b16..c45c556 100644 --- a/tests/test_read.py +++ b/tests/test_read.py @@ -1,12 +1,71 @@ from bfiocpp import TSReader, Seq, FileType import unittest +import requests, pathlib, shutil, logging, sys import bfio import numpy as np import random -import pathlib +from ome_zarr.utils import download as zarr_download + +TEST_IMAGES = { + "5025551.zarr": "https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.4/idr0054A/5025551.zarr", + "p01_x01_y01_wx0_wy0_c1.ome.tif": "https://raw.githubusercontent.com/sameeul/polus-test-data/main/bfio/p01_x01_y01_wx0_wy0_c1.ome.tif", + "Plate1-Blue-A-12-Scene-3-P3-F2-03.czi": "https://downloads.openmicroscopy.org/images/Zeiss-CZI/idr0011/Plate1-Blue-A_TS-Stinger/Plate1-Blue-A-12-Scene-3-P3-F2-03.czi", +} TEST_DIR = pathlib.Path(__file__).with_name("data") +logging.basicConfig( + format="%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) +logger = logging.getLogger("bfio.test") + +if "-v" in sys.argv: + logger.setLevel(logging.INFO) + + +def setUpModule(): + """Download images for testing""" + TEST_DIR.mkdir(exist_ok=True) + + for file, url in TEST_IMAGES.items(): + logger.info(f"setup - Downloading: {file}") + + if not file.endswith(".zarr"): + if TEST_DIR.joinpath(file).exists(): + continue + + r = requests.get(url) + + with open(TEST_DIR.joinpath(file), "wb") as fw: + fw.write(r.content) + else: + if TEST_DIR.joinpath(file).exists(): + shutil.rmtree(TEST_DIR.joinpath(file)) + zarr_download(url, str(TEST_DIR)) + + """Load the czi image, and save as a npy file for further testing.""" + with bfio.BioReader( + TEST_DIR.joinpath("Plate1-Blue-A-12-Scene-3-P3-F2-03.czi") + ) as br: + with bfio.BioWriter( + TEST_DIR.joinpath("4d_array.ome.tif"), + metadata=br.metadata, + X=br.X, + Y=br.Y, + Z=br.Z, + C=br.C, + T=br.T, + ) as bw: + bw[:] = br[:] + +def tearDownModule(): + """Remove test images""" + + logger.info("teardown - Removing test images...") + shutil.rmtree(TEST_DIR) + + class TestOmeTiffRead(unittest.TestCase): def test_read_ome_tif_full(self): diff --git a/tests/test_write.py b/tests/test_write.py index 1d981d2..e333bd0 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -1,13 +1,55 @@ +from bfiocpp import TSReader, TSWriter, Seq, FileType import unittest -import tempfile -import os +import requests, pathlib, shutil, logging, sys +import bfio import numpy as np -import pathlib +import tempfile, os +from ome_zarr.utils import download as zarr_download -from bfiocpp import TSWriter, TSReader, Seq, FileType +TEST_IMAGES = { + "5025551.zarr": "https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.4/idr0054A/5025551.zarr", +} TEST_DIR = pathlib.Path(__file__).with_name("data") +logging.basicConfig( + format="%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) +logger = logging.getLogger("bfio.test") + +if "-v" in sys.argv: + logger.setLevel(logging.INFO) + + +def setUpModule(): + """Download images for testing""" + TEST_DIR.mkdir(exist_ok=True) + + for file, url in TEST_IMAGES.items(): + logger.info(f"setup - Downloading: {file}") + + if not file.endswith(".zarr"): + if TEST_DIR.joinpath(file).exists(): + continue + + r = requests.get(url) + + with open(TEST_DIR.joinpath(file), "wb") as fw: + fw.write(r.content) + else: + if TEST_DIR.joinpath(file).exists(): + shutil.rmtree(TEST_DIR.joinpath(file)) + zarr_download(url, str(TEST_DIR)) + + +def tearDownModule(): + """Remove test images""" + + logger.info("teardown - Removing test images...") + shutil.rmtree(TEST_DIR) + + class TestZarrWrite(unittest.TestCase): def test_write_zarr_2d(self): @@ -31,9 +73,9 @@ def test_write_zarr_2d(self): tsteps = Seq(0, 0, 1) tmp = br.data(rows, cols, layers, channels, tsteps) - with tempfile.TemporaryDirectory() as test_dir: + with tempfile.TemporaryDirectory() as dir: # Use the temporary directory - test_file_path = os.path.join(test_dir, 'out/test.ome.zarr') + test_file_path = os.path.join(dir, 'out/test.ome.zarr') bw = TSWriter(test_file_path) bw.write_image(tmp, tmp.shape, tmp.shape) From e5a0faee4c8c2ec685c944c50cd77b02e3aaf3ec Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 25 Jun 2024 08:30:55 -0600 Subject: [PATCH 16/21] Update TSWriter interface --- src/cpp/interface/interface.cpp | 2 +- src/cpp/writer/tswriter.cpp | 203 +++++++++++++++++--------------- src/cpp/writer/tswriter.h | 47 ++------ src/python/bfiocpp/tswriter.py | 11 +- 4 files changed, 126 insertions(+), 137 deletions(-) diff --git a/src/cpp/interface/interface.cpp b/src/cpp/interface/interface.cpp index a27165e..2911755 100644 --- a/src/cpp/interface/interface.cpp +++ b/src/cpp/interface/interface.cpp @@ -122,6 +122,6 @@ PYBIND11_MODULE(libbfiocpp, m) { // Writer class py::class_>(m, "TsWriterCPP") - .def(py::init()) + .def(py::init&, const std::vector&, const std::string&>()) .def("write", &bfiocpp::TsWriterCPP::write_image); } diff --git a/src/cpp/writer/tswriter.cpp b/src/cpp/writer/tswriter.cpp index 087ab2e..f907ef6 100644 --- a/src/cpp/writer/tswriter.cpp +++ b/src/cpp/writer/tswriter.cpp @@ -9,119 +9,134 @@ using ::tensorstore::internal_zarr::ChooseBaseDType; namespace bfiocpp { -TsWriterCPP::TsWriterCPP(const std::string& fname): _filename(fname) { - -} - -void TsWriterCPP::SetImageHeight(std::int64_t image_height) {_image_height = image_height;} -void TsWriterCPP::SetImageWidth (std::int64_t image_width) {_image_width = image_width;} -void TsWriterCPP::SetImageDepth (std::int64_t image_depth) {_image_depth = image_depth;} -void TsWriterCPP::SetTileHeight (std::int64_t tile_height) {_tile_height = tile_height;} -void TsWriterCPP::SetTileWidth (std::int64_t tile_width) {_tile_width = tile_width;} -void TsWriterCPP::SetTileDepth (std::int64_t tile_depth) {_tile_depth = tile_depth;} -void TsWriterCPP::SetChannelCount (std::int64_t num_channels) {_num_channels = num_channels;} -void TsWriterCPP::SetTstepCount (std::int64_t num_tsteps) {_num_tsteps = num_tsteps;} -void TsWriterCPP::SetDataType(const std::string& data_type) {_data_type = data_type;} - -void TsWriterCPP::write_image(py::array& py_image, const std::vector& image_shape, const std::vector& chunk_shape) { - - auto dt = py_image.dtype(); - auto dt_str = py::str(dt).cast(); - - dt_str = (dt_str == "float64") ? "double" : dt_str; // change float64 numpy type to double +TsWriterCPP::TsWriterCPP( + const std::string& fname, + const std::vector& image_shape, + const std::vector& chunk_shape, + const std::string& dtype_str + ): _filename(fname), _image_shape(image_shape), _chunk_shape(chunk_shape) { + + _dtype_code = GetDataTypeCode(dtype_str); - auto dtype = GetTensorStoreDataType(dt_str); + std::string dtype_str_converted = (dtype_str == "float64") ? "double" : dtype_str; // change float64 numpy type to double + + auto dtype = GetTensorStoreDataType(dtype_str_converted); - std::string dtype_str = ChooseBaseDType(dtype).value().encoded_dtype; + auto dtype_base = ChooseBaseDType(dtype).value().encoded_dtype; - auto spec = GetZarrSpecToWrite(_filename, image_shape, chunk_shape, dtype_str); + auto spec = GetZarrSpecToWrite(_filename, image_shape, chunk_shape, dtype_base); - TENSORSTORE_CHECK_OK_AND_ASSIGN(auto store, tensorstore::Open( + TENSORSTORE_CHECK_OK_AND_ASSIGN(_source, tensorstore::Open( spec, tensorstore::OpenMode::create | tensorstore::OpenMode::delete_existing, tensorstore::ReadWriteMode::write).result()); +} - // use if-else statement instead of template to avoid creating functions for each datatype - if (dt_str == "uint8") { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); - // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; - } - } else if (dt_str == "uint16") { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); +void TsWriterCPP::write_image(py::array& py_image) { - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; - } + // use switch instead of template to avoid creating functions for each datatype + switch(_dtype_code) + { + case (1): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); - } else if (dt_str == "uint32") { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; - } - } else if (dt_str == "uint64") { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + break; } - } else if (dt_str == "int8") { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); - - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + case (2): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; + } + case (4): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; } - - } else if (dt_str == "int16") { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); - - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + case (8): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; } - } else if (dt_str == "int32") { - - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); - - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + case (16): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; } - } else if (dt_str == "int64") { - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); - - // Write data array to TensorStore - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + case (32): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; + } + case (64): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; } - } else if (dt_str == "float") { - - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); - - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + case (128): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + // Write data array to TensorStore + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; } - } else if (dt_str == "double") { - - auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), image_shape, tensorstore::c_order); - - auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), store).result(); - if (!write_result.ok()) { - std::cerr << "Error writing image: " << write_result.status() << std::endl; + case (256): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; + } + case (512): { + auto data_array = tensorstore::Array(py_image.mutable_unchecked().data(0), _image_shape, tensorstore::c_order); + + auto write_result = tensorstore::Write(tensorstore::UnownedToShared(data_array), _source).result(); + if (!write_result.ok()) { + std::cerr << "Error writing image: " << write_result.status() << std::endl; + } + break; } - } else { - throw std::runtime_error("Unsupported data type: " + dt_str); - } + default: { + // should not be reached + std::cerr << "Error writing image: unsupported data type" << std::endl; + } + } } -} \ No newline at end of file +} diff --git a/src/cpp/writer/tswriter.h b/src/cpp/writer/tswriter.h index bc0c527..63e59e2 100644 --- a/src/cpp/writer/tswriter.h +++ b/src/cpp/writer/tswriter.h @@ -24,50 +24,23 @@ namespace py = pybind11; - -using image_data = std::variant, - std::vector, - std::vector, - std::vector, - std::vector, - std::vector, - std::vector, - std::vector, - std::vector, - std::vector>; - - - -using iter_indicies = std::tuple; - namespace bfiocpp{ class TsWriterCPP{ public: - TsWriterCPP(const std::string& fname); + TsWriterCPP(const std::string& fname, const std::vector& image_shape, const std::vector& chunk_shape, const std::string& dtype); - void SetImageHeight(std::int64_t image_height); - void SetImageWidth (std::int64_t image_width); - void SetImageDepth (std::int64_t image_depth); - void SetTileHeight (std::int64_t tile_height); - void SetTileWidth (std::int64_t tile_width); - void SetTileDepth (std::int64_t tile_depth); - void SetChannelCount (std::int64_t num_channels); - void SetTstepCount (std::int64_t num_steps); - void SetDataType(const std::string& data_type); - - void write_image(py::array& py_image, const std::vector& image_shape, const std::vector& chunk_shape); + void write_image(py::array& py_image); private: - std::string _filename, _data_type; - std::int64_t _image_height, - _image_width, - _image_depth, - _tile_height, - _tile_width, - _tile_depth, - _num_channels, - _num_tsteps; + std::string _filename; + + std::vector _image_shape, _chunk_shape; + + uint16_t _dtype_code; + + tensorstore::TensorStore _source; + }; } diff --git a/src/python/bfiocpp/tswriter.py b/src/python/bfiocpp/tswriter.py index 9713375..3cf69c6 100644 --- a/src/python/bfiocpp/tswriter.py +++ b/src/python/bfiocpp/tswriter.py @@ -4,15 +4,15 @@ class TSWriter: - def __init__(self, file_name: str): + def __init__(self, file_name: str, image_shape: list, chunk_shape: list, dtype: np.dtype): """Initialize tensorstore Zarr writer file_name: Path to write file to """ + + self._image_writer: TsWriterCPP = TsWriterCPP(file_name, image_shape, chunk_shape, str(dtype)) - self._image_writer: TsWriterCPP = TsWriterCPP(file_name) - - def write_image(self, image_data: np.ndarray, image_shape: list, chunk_shape: list): + def write_image(self, image_data: np.ndarray): """Write image data to file image_data: 5d numpy array containing image data @@ -23,7 +23,8 @@ def write_image(self, image_data: np.ndarray, image_shape: list, chunk_shape: li raise ValueError("Image data must be a 5d numpy array") try: - self._image_writer.write(image_data.flatten(), image_shape, chunk_shape) + self._image_writer.write(image_data.flatten()) + except Exception as e: raise RuntimeError(f"Error writing image data: {e.what}") From 18462fc1768fbbc8418614ff7406b1e46b993b2b Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 25 Jun 2024 08:31:03 -0600 Subject: [PATCH 17/21] Update writer test --- tests/test_write.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_write.py b/tests/test_write.py index e333bd0..55a6af4 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -77,8 +77,8 @@ def test_write_zarr_2d(self): # Use the temporary directory test_file_path = os.path.join(dir, 'out/test.ome.zarr') - bw = TSWriter(test_file_path) - bw.write_image(tmp, tmp.shape, tmp.shape) + bw = TSWriter(test_file_path, tmp.shape, tmp.shape, str(tmp.dtype)) + bw.write_image(tmp) bw.close() br = TSReader( From 3098e58fa93f5e312e74fb7f14ce7fd6af7af179 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 25 Jun 2024 08:36:28 -0600 Subject: [PATCH 18/21] Reformat tswriter --- src/python/bfiocpp/tswriter.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/python/bfiocpp/tswriter.py b/src/python/bfiocpp/tswriter.py index 3cf69c6..f43a402 100644 --- a/src/python/bfiocpp/tswriter.py +++ b/src/python/bfiocpp/tswriter.py @@ -4,13 +4,17 @@ class TSWriter: - def __init__(self, file_name: str, image_shape: list, chunk_shape: list, dtype: np.dtype): + def __init__( + self, file_name: str, image_shape: list, chunk_shape: list, dtype: np.dtype + ): """Initialize tensorstore Zarr writer file_name: Path to write file to """ - - self._image_writer: TsWriterCPP = TsWriterCPP(file_name, image_shape, chunk_shape, str(dtype)) + + self._image_writer: TsWriterCPP = TsWriterCPP( + file_name, image_shape, chunk_shape, str(dtype) + ) def write_image(self, image_data: np.ndarray): """Write image data to file From 8908c90ab2bd92fa7c5e0bad2c02126d2bdbb7d9 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 25 Jun 2024 10:14:29 -0600 Subject: [PATCH 19/21] Freeze numpy version --- .github/workflows/publish_pypi.yml | 4 ++-- .github/workflows/wheel_build.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index b130ea3..b3124a6 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -74,7 +74,7 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_BEFORE_TEST_LINUX: yum -y install maven java - CIBW_TEST_REQUIRES: bfio requests numpy ome_zarr + CIBW_TEST_REQUIRES: bfio requests numpy==1.24.0 ome_zarr CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v - name: Install Dependencies @@ -128,7 +128,7 @@ jobs: CIBW_ENVIRONMENT_MACOS: REPAIR_LIBRARY_PATH="/tmp/bfiocpp_bld/local_install/lib:/tmp/bfiocpp_bld/local_install/lib64" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="/tmp/bfiocpp_bld/local_install" CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON" CIBW_REPAIR_WHEEL_COMMAND_MACOS: DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} CIBW_ARCHS: ${{ matrix.cibw_archs }} - CIBW_TEST_REQUIRES: bfio requests numpy ome_zarr + CIBW_TEST_REQUIRES: bfio requests numpy==1.24.0 ome_zarr CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v - name: Install Dependencies diff --git a/.github/workflows/wheel_build.yml b/.github/workflows/wheel_build.yml index 111b4cd..35b801b 100644 --- a/.github/workflows/wheel_build.yml +++ b/.github/workflows/wheel_build.yml @@ -72,7 +72,7 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_BEFORE_TEST_LINUX: yum -y install maven java - CIBW_TEST_REQUIRES: bfio requests numpy ome_zarr + CIBW_TEST_REQUIRES: bfio requests numpy==1.24.0 ome_zarr CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v - name: Upload Artifact @@ -124,7 +124,7 @@ jobs: CIBW_ENVIRONMENT_MACOS: REPAIR_LIBRARY_PATH="/tmp/bfiocpp_bld/local_install/lib:/tmp/bfiocpp_bld/local_install/lib64" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="/tmp/bfiocpp_bld/local_install" CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON" CIBW_REPAIR_WHEEL_COMMAND_MACOS: DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} CIBW_ARCHS: ${{ matrix.cibw_archs }} - CIBW_TEST_REQUIRES: bfio requests numpy ome_zarr zarr + CIBW_TEST_REQUIRES: bfio requests numpy==1.24.0 ome_zarr zarr CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v - name: Upload Artifact From a9743bf34b34fe1c0d585287ccbe062170a237f2 Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 25 Jun 2024 10:14:43 -0600 Subject: [PATCH 20/21] Restrict numpy version to less than 2.0.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 09f4936..fed64e8 100644 --- a/setup.py +++ b/setup.py @@ -104,6 +104,6 @@ def build_extension(self, ext): zip_safe=False, python_requires=">=3.8", install_requires=[ - "numpy", + "numpy<2.0.0", ], ) From 2887ecb24728e738308d9b60a22e5e373ea4da8d Mon Sep 17 00:00:00 2001 From: JesseMckinzie Date: Tue, 25 Jun 2024 10:47:45 -0600 Subject: [PATCH 21/21] Remove duplicate definition --- src/cpp/utilities/utilities.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cpp/utilities/utilities.h b/src/cpp/utilities/utilities.h index 0da9ba1..047b49c 100644 --- a/src/cpp/utilities/utilities.h +++ b/src/cpp/utilities/utilities.h @@ -19,7 +19,6 @@ std::string GetUTCString(); std::string GetOmeXml(const std::string& file_path); std::tuple, std::optional, std::optional>ParseMultiscaleMetadata(const std::string& axes_list, int len); tensorstore::DataType GetTensorStoreDataType(const std::string& type_str); -uint16_t GetDataTypeCode (std::string_view type_name); tensorstore::Spec GetZarrSpecToWrite(const std::string& filename, const std::vector& image_shape, const std::vector& chunk_shape,