From 0453801d029aa8a121b1ecbc97dc0e4777e68813 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Sat, 20 Nov 2021 09:18:41 +0100 Subject: [PATCH 01/15] Add deepimagej config generation WIP --- bioimageio/core/build_spec/build_model.py | 169 ++++++++++++++++++++++ tests/build_spec/test_build_spec.py | 14 +- 2 files changed, 182 insertions(+), 1 deletion(-) diff --git a/bioimageio/core/build_spec/build_model.py b/bioimageio/core/build_spec/build_model.py index 98109a80..a4f1850b 100644 --- a/bioimageio/core/build_spec/build_model.py +++ b/bioimageio/core/build_spec/build_model.py @@ -5,7 +5,9 @@ from shutil import copyfile from typing import Any, Dict, List, Optional, Union, Tuple +import imageio import numpy as np +import requests import bioimageio.spec as spec import bioimageio.spec.model as model_spec @@ -258,6 +260,146 @@ def _get_dependencies(dependencies, root): return model_spec.raw_nodes.Dependencies(manager=manager, file=_process_uri(path, root)) +def _get_deepimagej_preprocessing(name, kwargs, export_folder): + # these are the only preprocessings we currently use + assert name in ("scale_linear", "scale_range", "zero_mean_unit_variance") + if name == "scale_linear": + macro = "scale_linear.ijm" + + replace = {"gain": kwargs["gain"], "offset": kwargs["offset"]} + elif name == "scale_range": + macro = "per_sample_scale_range.ijm" + replace = {"min_precentile": kwargs["min_percentile"], "max_percentile": kwargs["max_percentile"]} + + elif name == "zero_mean_unit_variance": + mode = kwargs["mode"] + if mode == "fixed": + macro = "fixed_zero_mean_unit_variance.ijm" + replace = {"paramMean": kwargs["mean"], "paramStd": kwargs["std"]} + else: + macro = "zero_mean_unit_variance.ijm" + replace = {} + + macro = f"{name}.ijm" + url = f"https://raw.githubusercontent.com/deepimagej/imagej-macros/master/bioimage.io/{macro}" + + path = os.path.join(export_folder, macro) + # TODO do we use requests? + with requests.get(url, stream=True) as r: + with open(path, "w") as f: + f.write(r.text) + + # replace the kwargs in the macro file + if replace: + lines = [] + with open(path) as f: + for line in f: + kwarg = [kwarg for kwarg in replace if line.startswith(kwarg)] + if kwarg: + assert len(kwarg) == 1 + kwarg = kwarg[0] + # each kwarg should only be replaced ones + val = replace.pop(kwarg) + lines.append(f"{kwarg} = {val};\n") + else: + lines.append(line) + + with open(path, "w") as f: + for line in lines: + f.write(line) + + preprocess = [ + {"spec": "ij.IJ::runMacroFile", + "kwargs": macro} + ] + + return preprocess, {"files": [macro]} + + +def _get_deepimagej_config(export_folder, + sample_inputs, sample_outputs, + test_in_path, test_out_path, + preprocessing): + if preprocessing: + assert len(preprocessing) == 1 + name = list(preprocessing[0].keys())[0] + kwargs = preprocessing[0][name] + preprocess, attachments = _get_deepimagej_preprocessing(name, kwargs, export_folder) + else: + preprocess = [{"spec": None}] + attachments = None + + # we currently don"t implement any postprocessing + postprocess = [{"spec": None}] + + def _get_size(path): + # load shape and get rid of batchdim + shape = np.load(path).shape[1:] + # reverse the shape; deepij expexts xyzc + shape = shape[::-1] + # add singleton z axis if we have 2d data + if len(shape) == 3: + shape = shape[:2] + (1,) + shape[-1:] + assert len(shape) == 4 + return " x ".join(map(str, shape)) + + # TODO get the pixel size info from somewhere + test_info = { + "inputs": [ + { + "name": in_path, + "size": _get_size(in_path), + "pixel_size": {"x": 1.0, "y": 1.0, "z": 1.0} + } for in_path in sample_inputs + ], + "outputs": [ + { + "name": out_path, + "type": "image", + "size": _get_size(out_path) + } for out_path in sample_outputs + ], + "memory_peak": None, + "runtime": None + } + + config = { + "prediction": { + "preprocess": preprocess, + "postprocess": postprocess, + }, + "test_information": test_info, + # other stuff deepimagej needs + "pyramidal_model": False, + "allow_tiling": True, + "model_keys": None + } + return {"deepimagej": config}, attachments + + +def _write_sample_data(input_paths, output_paths, export_folder): + + for i, in_path in enumerate(input_paths): + inp = np.load(in_path).squeeze() + sample_in_path = os.path.join(export_folder, f"sample_input_{i}.tif") + imageio.imwrite(sample_in_path, inp) if inp.ndim == 2 else imageio.volwrite(sample_in_path, inp) + + for i, out_path in enumerate(output_paths): + outp = np.load(out_path).squeeze() + sample_out_path = os.path.join(export_folder, f"sample_output_{i}.tif") + if outp.ndim == 2: + imageio.imwrite(sample_out_path, outp) + elif outp.ndim == 3: + imageio.volwrite(sample_out_path, outp) + elif outp.ndim == 4: + # we need to have channel last to write 4d tifs + imageio.volwrite(sample_out_path, outp.T) + else: + raise RuntimeError("Only support wrting up to 4d sample data, got {outp.ndim}d.") + + return os.path.split(sample_in_path)[1], os.path.split(sample_out_path)[1] + + def build_model( weight_uri: str, test_inputs: List[Union[str, Path]], @@ -303,6 +445,7 @@ def build_model( dependencies: Optional[str] = None, links: Optional[List[str]] = None, root: Optional[Union[Path, str]] = None, + add_deepimagej_config: bool = False, **weight_kwargs, ): """Create a zipped bioimage.io model. @@ -368,6 +511,7 @@ def build_model( config: custom configuration for this model. dependencies: relative path to file with dependencies for this model. root: optional root path for relative paths. This can be helpful when building a spec from another model spec. + add_deepimagej_config: add the deepimagej config to the model. weight_kwargs: keyword arguments for this weight type, e.g. "tensorflow_version". """ if root is None: @@ -462,6 +606,31 @@ def build_model( assert len(parent) == 2 kwargs["parent"] = {"uri": parent[0], "sha256": parent[1]} + if add_deepimagej_config: + sample_in_path, sample_out_path = _write_sample_data(test_inputs, + test_outputs, + root) + ij_config, attachments = _get_deepimagej_config(root, + sample_in_path, sample_out_path, + test_inputs, test_outputs, + preprocessing) + config.update(ij_config) + kwargs.update({ + "sample_inputs": [sample_in_path], + "sample_outputs": [sample_out_path], + "config": config + }) + if attachments is not None: + kwargs.update({"attachments": attachments}) + + if links is None: + links = ["deepimagej/deepimagej"] + else: + links.append("deepimagej/deepimagej") + + # make sure links are unique + if links is not None: + links = list(set(links)) try: model = model_spec.raw_nodes.Model( format_version=format_version, diff --git a/tests/build_spec/test_build_spec.py b/tests/build_spec/test_build_spec.py index 520b8762..f2086da6 100644 --- a/tests/build_spec/test_build_spec.py +++ b/tests/build_spec/test_build_spec.py @@ -3,7 +3,14 @@ from bioimageio.core.resource_io.io_ import load_raw_resource_description -def _test_build_spec(spec_path, out_path, weight_type, tensorflow_version=None, use_implicit_output_shape=False): +def _test_build_spec( + spec_path, + out_path, + weight_type, + tensorflow_version=None, + use_implicit_output_shape=False, + for_deepimagej=True +): from bioimageio.core.build_spec import build_model model_spec = load_raw_resource_description(spec_path) @@ -45,6 +52,7 @@ def _test_build_spec(spec_path, out_path, weight_type, tensorflow_version=None, root=model_spec.root_path, weight_type=weight_type_, output_path=out_path, + for_deepimagej=for_deepimagej, ) if tensorflow_version is not None: kwargs["tensorflow_version"] = tensorflow_version @@ -89,3 +97,7 @@ def test_build_spec_tf(any_tensorflow_model, tmp_path): def test_build_spec_tfjs(any_tensorflow_js_model, tmp_path): _test_build_spec(any_tensorflow_js_model, tmp_path / "model.zip", "tensorflow_js", tensorflow_version="1.12") + + +def test_build_spec_deepimagej(any_torchscript_model, tmp_path): + _test_build_spec(any_torchscript_model, tmp_path / "model.zip", "pytorch_script", for_deepimagej=True) From a53b4c75dc5cb56c730a314f4b42157995a1ec8f Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Sat, 20 Nov 2021 09:34:51 +0100 Subject: [PATCH 02/15] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- bioimageio/core/build_spec/build_model.py | 46 ++++++----------------- tests/build_spec/test_build_spec.py | 7 +--- 2 files changed, 13 insertions(+), 40 deletions(-) diff --git a/bioimageio/core/build_spec/build_model.py b/bioimageio/core/build_spec/build_model.py index a4f1850b..c442f2ed 100644 --- a/bioimageio/core/build_spec/build_model.py +++ b/bioimageio/core/build_spec/build_model.py @@ -308,18 +308,12 @@ def _get_deepimagej_preprocessing(name, kwargs, export_folder): for line in lines: f.write(line) - preprocess = [ - {"spec": "ij.IJ::runMacroFile", - "kwargs": macro} - ] + preprocess = [{"spec": "ij.IJ::runMacroFile", "kwargs": macro}] return preprocess, {"files": [macro]} -def _get_deepimagej_config(export_folder, - sample_inputs, sample_outputs, - test_in_path, test_out_path, - preprocessing): +def _get_deepimagej_config(export_folder, sample_inputs, sample_outputs, test_in_path, test_out_path, preprocessing): if preprocessing: assert len(preprocessing) == 1 name = list(preprocessing[0].keys())[0] @@ -346,21 +340,12 @@ def _get_size(path): # TODO get the pixel size info from somewhere test_info = { "inputs": [ - { - "name": in_path, - "size": _get_size(in_path), - "pixel_size": {"x": 1.0, "y": 1.0, "z": 1.0} - } for in_path in sample_inputs - ], - "outputs": [ - { - "name": out_path, - "type": "image", - "size": _get_size(out_path) - } for out_path in sample_outputs + {"name": in_path, "size": _get_size(in_path), "pixel_size": {"x": 1.0, "y": 1.0, "z": 1.0}} + for in_path in sample_inputs ], + "outputs": [{"name": out_path, "type": "image", "size": _get_size(out_path)} for out_path in sample_outputs], "memory_peak": None, - "runtime": None + "runtime": None, } config = { @@ -372,7 +357,7 @@ def _get_size(path): # other stuff deepimagej needs "pyramidal_model": False, "allow_tiling": True, - "model_keys": None + "model_keys": None, } return {"deepimagej": config}, attachments @@ -607,19 +592,12 @@ def build_model( kwargs["parent"] = {"uri": parent[0], "sha256": parent[1]} if add_deepimagej_config: - sample_in_path, sample_out_path = _write_sample_data(test_inputs, - test_outputs, - root) - ij_config, attachments = _get_deepimagej_config(root, - sample_in_path, sample_out_path, - test_inputs, test_outputs, - preprocessing) + sample_in_path, sample_out_path = _write_sample_data(test_inputs, test_outputs, root) + ij_config, attachments = _get_deepimagej_config( + root, sample_in_path, sample_out_path, test_inputs, test_outputs, preprocessing + ) config.update(ij_config) - kwargs.update({ - "sample_inputs": [sample_in_path], - "sample_outputs": [sample_out_path], - "config": config - }) + kwargs.update({"sample_inputs": [sample_in_path], "sample_outputs": [sample_out_path], "config": config}) if attachments is not None: kwargs.update({"attachments": attachments}) diff --git a/tests/build_spec/test_build_spec.py b/tests/build_spec/test_build_spec.py index f2086da6..4eb79322 100644 --- a/tests/build_spec/test_build_spec.py +++ b/tests/build_spec/test_build_spec.py @@ -4,12 +4,7 @@ def _test_build_spec( - spec_path, - out_path, - weight_type, - tensorflow_version=None, - use_implicit_output_shape=False, - for_deepimagej=True + spec_path, out_path, weight_type, tensorflow_version=None, use_implicit_output_shape=False, for_deepimagej=True ): from bioimageio.core.build_spec import build_model From 9e446503a09659680cc89560589c509c743b50b7 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Fri, 26 Nov 2021 00:03:03 +0100 Subject: [PATCH 03/15] Update deepimagej config in build model --- bioimageio/core/build_spec/build_model.py | 235 +++++++++++++++------- tests/build_spec/test_build_spec.py | 21 +- 2 files changed, 171 insertions(+), 85 deletions(-) diff --git a/bioimageio/core/build_spec/build_model.py b/bioimageio/core/build_spec/build_model.py index a4f1850b..e27772fe 100644 --- a/bioimageio/core/build_spec/build_model.py +++ b/bioimageio/core/build_spec/build_model.py @@ -5,7 +5,6 @@ from shutil import copyfile from typing import Any, Dict, List, Optional, Union, Tuple -import imageio import numpy as np import requests @@ -19,6 +18,13 @@ except ImportError: from typing_extensions import get_args # type: ignore +# need tifffile for writing the deepimagej config +# we probably always have this, but wrap into an ImportGuard just in case +try: + import tifffile +except ImportError: + tifffile = None + # # utility functions to build the spec from python # @@ -260,13 +266,17 @@ def _get_dependencies(dependencies, root): return model_spec.raw_nodes.Dependencies(manager=manager, file=_process_uri(path, root)) -def _get_deepimagej_preprocessing(name, kwargs, export_folder): - # these are the only preprocessings we currently use +def _get_deepimagej_macro(name, kwargs, export_folder): + + # macros available in deepimagej assert name in ("scale_linear", "scale_range", "zero_mean_unit_variance") + # TODO deep imagej has a binarize macro but I have no idea what it is doing + # assert name in ("binarize", "scale_linear", "scale_range", "zero_mean_unit_variance") + if name == "scale_linear": macro = "scale_linear.ijm" - replace = {"gain": kwargs["gain"], "offset": kwargs["offset"]} + elif name == "scale_range": macro = "per_sample_scale_range.ijm" replace = {"min_precentile": kwargs["min_percentile"], "max_percentile": kwargs["max_percentile"]} @@ -280,6 +290,10 @@ def _get_deepimagej_preprocessing(name, kwargs, export_folder): macro = "zero_mean_unit_variance.ijm" replace = {} + # elif name == "binarize": + # macro = "binarize.ijm" + # replace = {"", kwargs["threshold"]} + macro = f"{name}.ijm" url = f"https://raw.githubusercontent.com/deepimagej/imagej-macros/master/bioimage.io/{macro}" @@ -308,55 +322,57 @@ def _get_deepimagej_preprocessing(name, kwargs, export_folder): for line in lines: f.write(line) - preprocess = [ - {"spec": "ij.IJ::runMacroFile", - "kwargs": macro} - ] + preprocess = {"spec": "ij.IJ::runMacroFile", "kwargs": macro} + return preprocess - return preprocess, {"files": [macro]} +def _get_deepimagej_config(export_folder, sample_inputs, sample_outputs, pixel_sizes, preprocessing, postprocessing): + assert len(sample_inputs) == len(sample_outputs) == 1, "deepimagej config only valid for single input/output" -def _get_deepimagej_config(export_folder, - sample_inputs, sample_outputs, - test_in_path, test_out_path, - preprocessing): - if preprocessing: + if any(preproc is not None for preproc in preprocessing): assert len(preprocessing) == 1 - name = list(preprocessing[0].keys())[0] - kwargs = preprocessing[0][name] - preprocess, attachments = _get_deepimagej_preprocessing(name, kwargs, export_folder) + preprocess_ij = [_get_deepimagej_macro(name, kwargs) for name, kwargs in preprocessing[0].items()] + attachments = [preproc["kwargs"] for preproc in preprocess_ij] else: - preprocess = [{"spec": None}] + preprocess_ij = [{"spec": None}] attachments = None - # we currently don"t implement any postprocessing - postprocess = [{"spec": None}] + if any(postproc is not None for postproc in postprocessing): + assert len(postprocessing) == 1 + postprocess_ij = [_get_deepimagej_macro(name, kwargs) for name, kwargs in postprocessing[0].items()] + if attachments is None: + attachments = [postproc["kwargs"] for postproc in postprocess_ij] + else: + attachments.extend([postproc["kwargs"] for postproc in postprocess_ij]) + else: + postprocess_ij = [{"spec": None}] - def _get_size(path): - # load shape and get rid of batchdim - shape = np.load(path).shape[1:] - # reverse the shape; deepij expexts xyzc - shape = shape[::-1] + def get_size(path): + assert tifffile is not None, "need tifffile for writing deepimagej config" + with tifffile.TiffFile(path) as f: + shape = f.asarray().shape # add singleton z axis if we have 2d data if len(shape) == 3: shape = shape[:2] + (1,) + shape[-1:] assert len(shape) == 4 return " x ".join(map(str, shape)) - # TODO get the pixel size info from somewhere + # deepimagej always expexts a pixel size for the z axis + pixel_sizes_ = [pix_size if "z" in pix_size else dict(z=1.0, **pix_size) for pix_size in pixel_sizes] + test_info = { "inputs": [ { "name": in_path, - "size": _get_size(in_path), - "pixel_size": {"x": 1.0, "y": 1.0, "z": 1.0} - } for in_path in sample_inputs + "size": get_size(in_path), + "pixel_size": pix_size + } for in_path, pix_size in zip(sample_inputs, pixel_sizes_) ], "outputs": [ { "name": out_path, "type": "image", - "size": _get_size(out_path) + "size": get_size(out_path) } for out_path in sample_outputs ], "memory_peak": None, @@ -365,8 +381,8 @@ def _get_size(path): config = { "prediction": { - "preprocess": preprocess, - "postprocess": postprocess, + "preprocess": preprocess_ij, + "postprocess": postprocess_ij, }, "test_information": test_info, # other stuff deepimagej needs @@ -377,27 +393,42 @@ def _get_size(path): return {"deepimagej": config}, attachments -def _write_sample_data(input_paths, output_paths, export_folder): +def _write_sample_data(input_paths, output_paths, input_axes, output_axes, export_folder): + + def write_im(path, im, axes): + assert tifffile is not None, "need tifffile for writing deepimagej config" + assert len(axes) == im.ndim + assert im.ndim in (3, 4) - for i, in_path in enumerate(input_paths): - inp = np.load(in_path).squeeze() + # deepimagej expects xyzc axis order + if im.ndim == 3: + assert set(axes) == {"x", "y", "c"} + axes_ij = "xyc" + else: + assert set(axes) == {"x", "y", "z", "c"} + axes_ij = "xyzc" + + axis_permutation = tuple(axes_ij.index(ax) for ax in axes) + im = im.transpose(axis_permutation) + + with tifffile.TiffWriter(path) as f: + f.write(im) + + sample_in_paths = [] + for i, (in_path, axes) in enumerate(zip(input_paths, input_axes)): + inp = np.load(in_path)[0] sample_in_path = os.path.join(export_folder, f"sample_input_{i}.tif") - imageio.imwrite(sample_in_path, inp) if inp.ndim == 2 else imageio.volwrite(sample_in_path, inp) + write_im(sample_in_path, inp, axes) + sample_in_paths.append(sample_in_path) - for i, out_path in enumerate(output_paths): - outp = np.load(out_path).squeeze() + sample_out_paths = [] + for i, (out_path, axes) in enumerate(zip(output_paths, output_axes)): + outp = np.load(out_path)[0] sample_out_path = os.path.join(export_folder, f"sample_output_{i}.tif") - if outp.ndim == 2: - imageio.imwrite(sample_out_path, outp) - elif outp.ndim == 3: - imageio.volwrite(sample_out_path, outp) - elif outp.ndim == 4: - # we need to have channel last to write 4d tifs - imageio.volwrite(sample_out_path, outp.T) - else: - raise RuntimeError("Only support wrting up to 4d sample data, got {outp.ndim}d.") + write_im(sample_out_path, outp, axes) + sample_out_paths.append(sample_out_path) - return os.path.split(sample_in_path)[1], os.path.split(sample_out_path)[1] + return sample_in_paths, sample_out_paths def build_model( @@ -418,8 +449,8 @@ def build_model( source: Optional[str] = None, model_kwargs: Optional[Dict[str, Union[int, float, str]]] = None, weight_type: Optional[str] = None, - sample_inputs: Optional[str] = None, - sample_outputs: Optional[str] = None, + sample_inputs: Optional[List[str]] = None, + sample_outputs: Optional[List[str]] = None, # tensor specific input_name: Optional[List[str]] = None, input_step: Optional[List[List[int]]] = None, @@ -435,6 +466,7 @@ def build_model( halo: Optional[List[List[int]]] = None, preprocessing: Optional[List[Dict[str, Dict[str, Union[int, float, str]]]]] = None, postprocessing: Optional[List[Dict[str, Dict[str, Union[int, float, str]]]]] = None, + pixel_sizes: Optional[List[Dict[str, float]]] = None, # general optional git_repo: Optional[str] = None, attachments: Optional[Dict[str, Union[str, List[str]]]] = None, @@ -503,6 +535,8 @@ def build_model( halo: halo to be cropped from the output tensor. preprocessing: list of preprocessing operations for the input. postprocessing: list of postprocessing operations for the output. + pixel_sizes: the pixel sizes for the input tensors, only for spatial axes. + This information is currently only used by deepimagej, but will be added to the spec soon. git_repo: reference git repository for this model. attachments: list of additional files to package with the model. packaged_by: list of authors that have packaged this model. @@ -567,6 +601,16 @@ def build_model( ) ] + # validate the pixel sizes (currently only used by deepimagej) + spatial_axes = [[ax for ax in inp.axes if ax in "xyz"] for inp in inputs] + if pixel_sizes is None: + pixel_sizes = [{ax: 1.0 for ax in axes} for axes in spatial_axes] + else: + assert len(pixel_sizes) == n_inputs + for pix_size, axes in zip(pixel_sizes, spatial_axes): + assert isinstance(pixel_sizes, dict) + assert set(pix_size.keys()) == set(axes) + # # generate general fields # @@ -583,13 +627,63 @@ def build_model( weight_uri, weight_type, source, root, **weight_kwargs ) + # validate the sample inputs and outputs (if given) + if sample_inputs is not None: + assert sample_outputs is not None + assert len(sample_inputs) == n_inputs + assert len(sample_outputs) == n_outputs + + # add the deepimagej config if specified + if add_deepimagej_config: + if sample_inputs is None: + input_axes_ij = [inp.axes[1:] for inp in inputs] + output_axes_ij = [out.axes[1:] for out in outputs] + sample_inputs, sample_outputs = _write_sample_data( + test_inputs, test_outputs, input_axes_ij, output_axes_ij, root + ) + # deepimagej expect tifs as sample data + assert all(os.path.splitext(path)[1] in (".tif", ".tiff") for path in sample_inputs) + assert all(os.path.splitext(path)[1] in (".tif", ".tiff") for path in sample_outputs) + + ij_config, ij_attachments = _get_deepimagej_config( + root, sample_inputs, sample_outputs, pixel_sizes, preprocessing, postprocessing + ) + + if config is None: + config = ij_config + else: + config.update(ij_config) + + if ij_attachments is not None: + if attachments is None: + attachments = {"files": ij_attachments} + elif "files" not in attachments: + attachments["files"] = ij_attachments + else: + attachments["files"].extend(ij_attachments) + + if links is None: + links = ["deepimagej/deepimagej"] + else: + links.append("deepimagej/deepimagej") + + # make sure links are unique + if links is not None: + links = list(set(links)) + + # cast sample inputs / outputs to uri + if sample_inputs is not None: + sample_inputs = [_process_uri(uri, root) for uri in sample_inputs] + if sample_outputs is not None: + sample_outputs = [_process_uri(uri, root) for uri in sample_outputs] + # optional kwargs, don't pass them if none optional_kwargs = { - "git_repo": git_repo, "attachments": attachments, + "config": config, + "git_repo": git_repo, "packaged_by": packaged_by, "run_mode": run_mode, - "config": config, "sample_inputs": sample_inputs, "sample_outputs": sample_outputs, "framework": framework, @@ -600,37 +694,13 @@ def build_model( "links": links, } kwargs = {k: v for k, v in optional_kwargs.items() if v is not None} + if dependencies is not None: kwargs["dependencies"] = _get_dependencies(dependencies, root) if parent is not None: assert len(parent) == 2 kwargs["parent"] = {"uri": parent[0], "sha256": parent[1]} - if add_deepimagej_config: - sample_in_path, sample_out_path = _write_sample_data(test_inputs, - test_outputs, - root) - ij_config, attachments = _get_deepimagej_config(root, - sample_in_path, sample_out_path, - test_inputs, test_outputs, - preprocessing) - config.update(ij_config) - kwargs.update({ - "sample_inputs": [sample_in_path], - "sample_outputs": [sample_out_path], - "config": config - }) - if attachments is not None: - kwargs.update({"attachments": attachments}) - - if links is None: - links = ["deepimagej/deepimagej"] - else: - links.append("deepimagej/deepimagej") - - # make sure links are unique - if links is not None: - links = list(set(links)) try: model = model_spec.raw_nodes.Model( format_version=format_version, @@ -657,6 +727,17 @@ def build_model( if tmp_source is not None: os.remove(tmp_source) + # breakpoint() + # debugging: for some reason this causes a validation error for the sample_input / output, + # although they are passed in the same format as test inputs / outputs: + # (Pdb) model.sample_inputs + # [PosixPath('/tmp/bioimageio_cache/extracted_packages/efae734f36ff8634977c9174f01c854c4e2cfc799bc92751a4558a0edd963da6/sample_input_0.tif')] + # (Pdb) model.sample_outputs + # [PosixPath('/tmp/bioimageio_cache/extracted_packages/efae734f36ff8634977c9174f01c854c4e2cfc799bc92751a4558a0edd963da6/sample_output_0.tif')] + # (Pdb) model.test_inputs + # [PosixPath('/tmp/bioimageio_cache/extracted_packages/efae734f36ff8634977c9174f01c854c4e2cfc799bc92751a4558a0edd963da6/test_input.npy')] + # (Pdb) model.test_outputs + # [PosixPath('/tmp/bioimageio_cache/extracted_packages/efae734f36ff8634977c9174f01c854c4e2cfc799bc92751a4558a0edd963da6/test_output.npy')] model = load_raw_resource_description(model_package) return model diff --git a/tests/build_spec/test_build_spec.py b/tests/build_spec/test_build_spec.py index f2086da6..c62a895a 100644 --- a/tests/build_spec/test_build_spec.py +++ b/tests/build_spec/test_build_spec.py @@ -1,6 +1,6 @@ -from marshmallow import missing import bioimageio.spec as spec -from bioimageio.core.resource_io.io_ import load_raw_resource_description +from bioimageio.core import load_raw_resource_description, load_resource_description +from marshmallow import missing def _test_build_spec( @@ -9,7 +9,7 @@ def _test_build_spec( weight_type, tensorflow_version=None, use_implicit_output_shape=False, - for_deepimagej=True + add_deepimagej_config=False ): from bioimageio.core.build_spec import build_model @@ -52,7 +52,7 @@ def _test_build_spec( root=model_spec.root_path, weight_type=weight_type_, output_path=out_path, - for_deepimagej=for_deepimagej, + add_deepimagej_config=add_deepimagej_config, ) if tensorflow_version is not None: kwargs["tensorflow_version"] = tensorflow_version @@ -61,8 +61,13 @@ def _test_build_spec( kwargs["output_reference"] = ["input"] kwargs["output_scale"] = [[1.0, 1.0, 1.0, 1.0]] kwargs["output_offset"] = [[0.0, 0.0, 0.0, 0.0]] - raw_model = build_model(**kwargs) - spec.model.schema.Model().dump(raw_model) + + build_model(**kwargs) + assert out_path.exists() + loaded_model = load_resource_description(out_path) + if add_deepimagej_config: + loaded_config = loaded_model.config + assert "deepimagej" in loaded_config def test_build_spec_pytorch(any_torch_model, tmp_path): @@ -99,5 +104,5 @@ def test_build_spec_tfjs(any_tensorflow_js_model, tmp_path): _test_build_spec(any_tensorflow_js_model, tmp_path / "model.zip", "tensorflow_js", tensorflow_version="1.12") -def test_build_spec_deepimagej(any_torchscript_model, tmp_path): - _test_build_spec(any_torchscript_model, tmp_path / "model.zip", "pytorch_script", for_deepimagej=True) +def test_build_spec_deepimagej(unet2d_nuclei_broad_model, tmp_path): + _test_build_spec(unet2d_nuclei_broad_model, tmp_path / "model.zip", "pytorch_script", add_deepimagej_config=True) From 1865255b6fb8f2d1468fb89d234797289f5ac409 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Fri, 26 Nov 2021 00:19:20 +0100 Subject: [PATCH 04/15] Add tifffile to dependencies --- dev/environment-base.yaml | 1 + dev/environment-tf.yaml | 1 + dev/environment-torch.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/dev/environment-base.yaml b/dev/environment-base.yaml index 552981de..83cd24fb 100644 --- a/dev/environment-base.yaml +++ b/dev/environment-base.yaml @@ -15,5 +15,6 @@ dependencies: - pytorch - onnxruntime - tensorflow >=1.12,<2.0 + - tifffile - pip: - keras==1.2.2 diff --git a/dev/environment-tf.yaml b/dev/environment-tf.yaml index b3c5d52b..f7349451 100644 --- a/dev/environment-tf.yaml +++ b/dev/environment-tf.yaml @@ -13,5 +13,6 @@ dependencies: - python >=3.7,<3.8 # this environment is only available for python 3.7 - xarray - tensorflow >=1.12,<2.0 + - tifffile - pip: - keras==1.2.2 diff --git a/dev/environment-torch.yaml b/dev/environment-torch.yaml index 96eeb03a..0851442a 100644 --- a/dev/environment-torch.yaml +++ b/dev/environment-torch.yaml @@ -14,3 +14,4 @@ dependencies: - xarray - pytorch <1.10 - onnxruntime + - tifffile From 6e5cd1da84f9f16a980b98b721f2bbccb346a31b Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Fri, 26 Nov 2021 00:23:55 +0100 Subject: [PATCH 05/15] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- bioimageio/core/build_spec/build_model.py | 1 - tests/build_spec/test_build_spec.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bioimageio/core/build_spec/build_model.py b/bioimageio/core/build_spec/build_model.py index 574a7f07..3bf25eff 100644 --- a/bioimageio/core/build_spec/build_model.py +++ b/bioimageio/core/build_spec/build_model.py @@ -384,7 +384,6 @@ def get_size(path): def _write_sample_data(input_paths, output_paths, input_axes, output_axes, export_folder): - def write_im(path, im, axes): assert tifffile is not None, "need tifffile for writing deepimagej config" assert len(axes) == im.ndim diff --git a/tests/build_spec/test_build_spec.py b/tests/build_spec/test_build_spec.py index 1905e420..950443b3 100644 --- a/tests/build_spec/test_build_spec.py +++ b/tests/build_spec/test_build_spec.py @@ -9,7 +9,7 @@ def _test_build_spec( weight_type, tensorflow_version=None, use_implicit_output_shape=False, - add_deepimagej_config=True + add_deepimagej_config=True, ): from bioimageio.core.build_spec import build_model From 2d1e096fab650b8de71f62640cd50f23efa34d4c Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Fri, 26 Nov 2021 12:34:45 +0100 Subject: [PATCH 06/15] Fix test issues --- tests/build_spec/test_build_spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/build_spec/test_build_spec.py b/tests/build_spec/test_build_spec.py index 879f5a37..712dc988 100644 --- a/tests/build_spec/test_build_spec.py +++ b/tests/build_spec/test_build_spec.py @@ -10,7 +10,7 @@ def _test_build_spec( weight_type, tensorflow_version=None, use_implicit_output_shape=False, - add_deepimagej_config=True, + add_deepimagej_config=False, ): from bioimageio.core.build_spec import build_model From 1942df038eea92e1a29170921c422fe1e43adf3e Mon Sep 17 00:00:00 2001 From: fynnbe Date: Fri, 26 Nov 2021 13:04:42 +0100 Subject: [PATCH 07/15] make sure sample inputs/outputs are relative paths --- bioimageio/core/build_spec/build_model.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bioimageio/core/build_spec/build_model.py b/bioimageio/core/build_spec/build_model.py index 03fe655c..bcb241cc 100644 --- a/bioimageio/core/build_spec/build_model.py +++ b/bioimageio/core/build_spec/build_model.py @@ -649,11 +649,12 @@ def build_model( if links is not None: links = list(set(links)) - # cast sample inputs / outputs to uri + # make sure sample inputs / outputs are relative paths + # todo: currently this will fail for absolute paths that are not under root. should we copy the samples then? if sample_inputs is not None: - sample_inputs = resolve_local_source(sample_inputs, root) + sample_inputs = [p.relative_to(root) for p in resolve_local_source(sample_inputs, root)] if sample_outputs is not None: - sample_outputs = resolve_local_source(sample_outputs, root) + sample_outputs = [p.relative_to(root) for p in resolve_local_source(sample_outputs, root)] # optional kwargs, don't pass them if none optional_kwargs = { From f84640f6bf254b0ac1fdb7345fb73d8454a95124 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Fri, 26 Nov 2021 16:50:16 +0100 Subject: [PATCH 08/15] Use binarize macro in deepimagej --- bioimageio/core/build_spec/build_model.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bioimageio/core/build_spec/build_model.py b/bioimageio/core/build_spec/build_model.py index bcb241cc..ebdf3822 100644 --- a/bioimageio/core/build_spec/build_model.py +++ b/bioimageio/core/build_spec/build_model.py @@ -258,10 +258,7 @@ def _get_dependencies(dependencies, root): def _get_deepimagej_macro(name, kwargs, export_folder): # macros available in deepimagej - assert name in ("scale_linear", "scale_range", "zero_mean_unit_variance") - # TODO deep imagej has a binarize macro but I have no idea what it is doing - # assert name in ("binarize", "scale_linear", "scale_range", "zero_mean_unit_variance") - + macro_names = ("binarize", "scale_linear", "scale_range", "zero_mean_unit_variance") if name == "scale_linear": macro = "scale_linear.ijm" replace = {"gain": kwargs["gain"], "offset": kwargs["offset"]} @@ -279,9 +276,12 @@ def _get_deepimagej_macro(name, kwargs, export_folder): macro = "zero_mean_unit_variance.ijm" replace = {} - # elif name == "binarize": - # macro = "binarize.ijm" - # replace = {"", kwargs["threshold"]} + elif name == "binarize": + macro = "binarize.ijm" + replace = {"optimalThreshold", kwargs["threshold"]} + + else: + raise ValueError(f"Macro {name} is not available, must be one of {macro_names}.") macro = f"{name}.ijm" url = f"https://raw.githubusercontent.com/deepimagej/imagej-macros/master/bioimage.io/{macro}" From 2ffc41880de26b45c2bc166b7ba78357fdb5585e Mon Sep 17 00:00:00 2001 From: fynnbe Date: Fri, 26 Nov 2021 14:40:05 +0100 Subject: [PATCH 09/15] add output arg to resolve_source --- bioimageio/core/resource_io/utils.py | 86 ++++++++++++++++++---------- 1 file changed, 55 insertions(+), 31 deletions(-) diff --git a/bioimageio/core/resource_io/utils.py b/bioimageio/core/resource_io/utils.py index bc5df317..0050ae94 100644 --- a/bioimageio/core/resource_io/utils.py +++ b/bioimageio/core/resource_io/utils.py @@ -2,6 +2,7 @@ import importlib.util import os import pathlib +import shutil import sys import typing import warnings @@ -137,15 +138,17 @@ def generic_transformer(self, node: GenericRawNode) -> GenericResolvedNode: @singledispatch # todo: fix type annotations -def resolve_source(source, root_path: os.PathLike = pathlib.Path()): +def resolve_source(source, root_path: os.PathLike = pathlib.Path(), output: typing.Optional[os.PathLike] = None): raise TypeError(type(source)) @resolve_source.register -def _resolve_source_uri_node(source: raw_nodes.URI, root_path: os.PathLike = pathlib.Path()) -> pathlib.Path: - path_or_remote_uri = resolve_local_source(source, root_path) +def _resolve_source_uri_node( + source: raw_nodes.URI, root_path: os.PathLike = pathlib.Path(), output: typing.Optional[os.PathLike] = None +) -> pathlib.Path: + path_or_remote_uri = resolve_local_source(source, root_path, output) if isinstance(path_or_remote_uri, raw_nodes.URI): - local_path = _download_url_to_local_path(path_or_remote_uri) + local_path = _download_url(path_or_remote_uri, output) elif isinstance(path_or_remote_uri, pathlib.Path): local_path = path_or_remote_uri else: @@ -155,47 +158,66 @@ def _resolve_source_uri_node(source: raw_nodes.URI, root_path: os.PathLike = pat @resolve_source.register -def _resolve_source_str(source: str, root_path: os.PathLike = pathlib.Path()) -> pathlib.Path: - return resolve_source(fields.Union([fields.URI(), fields.Path()]).deserialize(source), root_path) +def _resolve_source_str( + source: str, root_path: os.PathLike = pathlib.Path(), output: typing.Optional[os.PathLike] = None +) -> pathlib.Path: + return resolve_source(fields.Union([fields.URI(), fields.Path()]).deserialize(source), root_path, output) @resolve_source.register -def _resolve_source_path(source: pathlib.Path, root_path: os.PathLike = pathlib.Path()) -> pathlib.Path: +def _resolve_source_path( + source: pathlib.Path, root_path: os.PathLike = pathlib.Path(), output: typing.Optional[os.PathLike] = None +) -> pathlib.Path: if not source.is_absolute(): source = pathlib.Path(root_path).absolute() / source - return source + if output is None: + return source + else: + try: + shutil.copy(source, output) + except shutil.SameFileError: + pass + return pathlib.Path(output) @resolve_source.register def _resolve_source_resolved_importable_path( - source: nodes.ResolvedImportableSourceFile, root_path: os.PathLike = pathlib.Path() + source: nodes.ResolvedImportableSourceFile, + root_path: os.PathLike = pathlib.Path(), + output: typing.Optional[os.PathLike] = None, ) -> nodes.ResolvedImportableSourceFile: return nodes.ResolvedImportableSourceFile( - callable_name=source.callable_name, source_file=resolve_source(source.source_file, root_path) + callable_name=source.callable_name, source_file=resolve_source(source.source_file, root_path, output) ) @resolve_source.register def _resolve_source_importable_path( - source: raw_nodes.ImportableSourceFile, root_path: os.PathLike = pathlib.Path() + source: raw_nodes.ImportableSourceFile, + root_path: os.PathLike = pathlib.Path(), + output: typing.Optional[os.PathLike] = None, ) -> nodes.ResolvedImportableSourceFile: return nodes.ResolvedImportableSourceFile( - callable_name=source.callable_name, source_file=resolve_source(source.source_file, root_path) + callable_name=source.callable_name, source_file=resolve_source(source.source_file, root_path, output) ) @resolve_source.register -def _resolve_source_list(source: list, root_path: os.PathLike = pathlib.Path()) -> typing.List[pathlib.Path]: - return [resolve_source(el, root_path) for el in source] +def _resolve_source_list( + source: list, root_path: os.PathLike = pathlib.Path(), output: typing.Optional[os.PathLike] = None +) -> typing.List[pathlib.Path]: + return [resolve_source(el, root_path, output) for el in source] # todo: write as singledispatch with type annotations def resolve_local_source( - source: typing.Union[str, os.PathLike, raw_nodes.URI, tuple, list], root_path: os.PathLike + source: typing.Union[str, os.PathLike, raw_nodes.URI, tuple, list], + root_path: os.PathLike, + output: typing.Optional[os.PathLike] = None, ) -> typing.Union[pathlib.Path, raw_nodes.URI, list, tuple]: if isinstance(source, (tuple, list)): - return type(source)([resolve_local_source(s, root_path) for s in source]) + return type(source)([resolve_local_source(s, root_path, output) for s in source]) elif isinstance(source, os.PathLike) or isinstance(source, str): try: # source as path from cwd is_path_cwd = pathlib.Path(source).exists() @@ -212,7 +234,16 @@ def resolve_local_source( source = path_from_root if is_path_cwd or is_path_rp: - return pathlib.Path(source) + source = pathlib.Path(source) + if output is None: + return source + else: + try: + shutil.copy(source, output) + except shutil.SameFileError: + pass + return pathlib.Path(output) + elif isinstance(source, os.PathLike): raise FileNotFoundError(f"Could neither find {source} nor {pathlib.Path(root_path) / source}") @@ -222,15 +253,7 @@ def resolve_local_source( uri = source assert isinstance(uri, raw_nodes.URI), uri - if not uri.scheme: # relative path - if uri.authority or uri.query or uri.fragment: - raise ValidationError(f"Invalid Path/URI: {uri}") - - local_path_or_remote_uri: typing.Union[pathlib.Path, raw_nodes.URI] = pathlib.Path(root_path) / uri.path - elif uri.scheme == "file": - if uri.authority or uri.query or uri.fragment: - raise NotImplementedError(uri) - + if uri.scheme == "file": local_path_or_remote_uri = pathlib.Path(url2pathname(uri.path)) elif uri.scheme in ("https", "https"): local_path_or_remote_uri = uri @@ -264,12 +287,13 @@ def all_sources_available( return True -def download_uri_to_local_path(uri: typing.Union[raw_nodes.URI, str]) -> pathlib.Path: - return resolve_source(uri) - +def _download_url(uri: raw_nodes.URI, output: typing.Optional[os.PathLike] = None) -> pathlib.Path: + if output is not None: + local_path = pathlib.Path(output) + else: + # todo: proper caching + local_path = BIOIMAGEIO_CACHE_PATH / uri.scheme / uri.authority / uri.path.strip("/") / uri.query -def _download_url_to_local_path(uri: raw_nodes.URI) -> pathlib.Path: - local_path = BIOIMAGEIO_CACHE_PATH / uri.scheme / uri.authority / uri.path.strip("/") / uri.query if local_path.exists(): warnings.warn(f"found cached {local_path}. Skipping download of {uri}.") else: From d8d359bfae5d80590e9039aea11cc7ff738572ce Mon Sep 17 00:00:00 2001 From: fynnbe Date: Fri, 26 Nov 2021 21:14:41 +0100 Subject: [PATCH 10/15] ensure local paths in root --- bioimageio/core/build_spec/build_model.py | 56 +++++++++++++++-------- bioimageio/core/resource_io/utils.py | 29 ++++++++---- 2 files changed, 58 insertions(+), 27 deletions(-) diff --git a/bioimageio/core/build_spec/build_model.py b/bioimageio/core/build_spec/build_model.py index ebdf3822..e29856b7 100644 --- a/bioimageio/core/build_spec/build_model.py +++ b/bioimageio/core/build_spec/build_model.py @@ -1,6 +1,7 @@ import datetime import hashlib import os +import shutil from pathlib import Path from shutil import copyfile from typing import Any, Dict, List, Optional, Tuple, Union @@ -11,6 +12,7 @@ import bioimageio.spec as spec import bioimageio.spec.model as model_spec from bioimageio.core import export_resource_package, load_raw_resource_description +from bioimageio.core.resource_io.nodes import URI from bioimageio.core.resource_io.utils import resolve_local_source, resolve_source try: @@ -52,8 +54,8 @@ def _infer_weight_type(path): raise ValueError(f"Could not infer weight type from extension {ext} for weight file {path}") -def _get_weights(weight_uri, weight_type, source, root, **kwargs): - weight_path = resolve_source(weight_uri, root) +def _get_weights(original_weight_source, weight_type, source, root, **kwargs): + weight_path = resolve_source(original_weight_source, root) if weight_type is None: weight_type = _infer_weight_type(weight_path) weight_hash = _get_hash(weight_path) @@ -65,7 +67,7 @@ def _get_weights(weight_uri, weight_type, source, root, **kwargs): source_file, source_class = source.replace("::", ":").split(":") # get the source path - source_file = resolve_source(source_file, root) + source_file = _ensure_local(source_file, root) source_hash = _get_hash(source_file) # if not relative, create local copy (otherwise this will not work) @@ -85,7 +87,9 @@ def _get_weights(weight_uri, weight_type, source, root, **kwargs): attachments = {} weight_types = model_spec.raw_nodes.WeightsFormat - weight_source = resolve_local_source(weight_uri, root) + weight_source = _ensure_local_or_url(original_weight_source, root) + + assert isinstance(weight_source, URI) or weight_source.is_relative_to(root) if weight_type == "pytorch_state_dict": # pytorch-state-dict -> we need a source assert source is not None @@ -135,7 +139,7 @@ def _get_weights(weight_uri, weight_type, source, root, **kwargs): elif weight_type == "tensorflow_js": weights = model_spec.raw_nodes.TensorflowJsWeightsEntry( - source=weight_uri, + source=weight_source, sha256=weight_hash, tensorflow_version=kwargs.get("tensorflow_version", "1.15"), **attachments, @@ -249,10 +253,11 @@ def _build_cite(cite: Dict[str, str]): def _get_dependencies(dependencies, root): if isinstance(dependencies, Path) or ":" not in dependencies: manager = "conda" - path = Path(dependencies) + path = dependencies else: manager, path = dependencies.split(":") - return model_spec.raw_nodes.Dependencies(manager=manager, file=resolve_source(path, root)) + + return model_spec.raw_nodes.Dependencies(manager=manager, file=_ensure_local(path, root)) def _get_deepimagej_macro(name, kwargs, export_folder): @@ -359,10 +364,7 @@ def get_size(path): } config = { - "prediction": { - "preprocess": preprocess_ij, - "postprocess": postprocess_ij, - }, + "prediction": {"preprocess": preprocess_ij, "postprocess": postprocess_ij}, "test_information": test_info, # other stuff deepimagej needs "pyramidal_model": False, @@ -409,6 +411,24 @@ def write_im(path, im, axes): return sample_in_paths, sample_out_paths +def _ensure_local(source: Union[Path, URI, str, list], root: Path) -> Union[Path, URI, list]: + """ensure source is local relative path in root""" + if isinstance(source, list): + return [_ensure_local(s, root) for s in source] + + local_source = resolve_source(source, root) + return resolve_source(local_source, root, root / local_source.name) + + +def _ensure_local_or_url(source: Union[Path, URI, str, list], root: Path) -> Union[Path, URI, list]: + """ensure source is remote URI or local relative path in root""" + if isinstance(source, list): + return [_ensure_local_or_url(s, root) for s in source] + + local_source = resolve_local_source(source, root) + return resolve_local_source(local_source, root, None if isinstance(local_source, URI) else root / local_source.name) + + def build_model( weight_uri: str, test_inputs: List[Union[str, Path]], @@ -536,8 +556,8 @@ def build_model( assert len(test_inputs) assert len(test_outputs) - test_inputs = resolve_local_source(test_inputs, root) - test_outputs = resolve_local_source(test_outputs, root) + test_inputs = _ensure_local_or_url(test_inputs, root) + test_outputs = _ensure_local_or_url(test_outputs, root) n_inputs = len(test_inputs) input_name = n_inputs * [None] if input_name is None else input_name @@ -597,8 +617,8 @@ def build_model( authors = _build_authors(authors) cite = _build_cite(cite) - documentation = resolve_source(documentation, root) - covers = resolve_source(covers, root) + documentation = _ensure_local(documentation, root) + covers = _ensure_local(covers, root) # parse the weights weights, language, framework, source, source_hash, tmp_source = _get_weights( @@ -650,11 +670,11 @@ def build_model( links = list(set(links)) # make sure sample inputs / outputs are relative paths - # todo: currently this will fail for absolute paths that are not under root. should we copy the samples then? if sample_inputs is not None: - sample_inputs = [p.relative_to(root) for p in resolve_local_source(sample_inputs, root)] + sample_inputs = _ensure_local_or_url(sample_inputs, root) + if sample_outputs is not None: - sample_outputs = [p.relative_to(root) for p in resolve_local_source(sample_outputs, root)] + sample_outputs = _ensure_local_or_url(sample_outputs, root) # optional kwargs, don't pass them if none optional_kwargs = { diff --git a/bioimageio/core/resource_io/utils.py b/bioimageio/core/resource_io/utils.py index 0050ae94..1f805993 100644 --- a/bioimageio/core/resource_io/utils.py +++ b/bioimageio/core/resource_io/utils.py @@ -138,7 +138,7 @@ def generic_transformer(self, node: GenericRawNode) -> GenericResolvedNode: @singledispatch # todo: fix type annotations -def resolve_source(source, root_path: os.PathLike = pathlib.Path(), output: typing.Optional[os.PathLike] = None): +def resolve_source(source, root_path: os.PathLike = pathlib.Path(), output=None): raise TypeError(type(source)) @@ -175,8 +175,8 @@ def _resolve_source_path( return source else: try: - shutil.copy(source, output) - except shutil.SameFileError: + shutil.copyfile(source, output) + except shutil.SameFileError: # source and output are identical pass return pathlib.Path(output) @@ -205,17 +205,28 @@ def _resolve_source_importable_path( @resolve_source.register def _resolve_source_list( - source: list, root_path: os.PathLike = pathlib.Path(), output: typing.Optional[os.PathLike] = None + source: list, + root_path: os.PathLike = pathlib.Path(), + output: typing.Optional[typing.Sequence[typing.Optional[os.PathLike]]] = None, ) -> typing.List[pathlib.Path]: - return [resolve_source(el, root_path, output) for el in source] + assert output is None or len(output) == len(source) + return [resolve_source(el, root_path, out) for el, out in zip(source, output or [None] * len(source))] + + +def resolve_local_sources( + sources: typing.Sequence[typing.Union[str, os.PathLike, raw_nodes.URI]], + root_path: os.PathLike, + outputs: typing.Optional[typing.Sequence[os.PathLike]] = None, +) -> typing.List[typing.Union[pathlib.Path, raw_nodes.URI]]: + assert outputs is None or len(outputs) == len(sources) + return [resolve_local_source(src, root_path, out) for src, out in zip(sources, outputs)] -# todo: write as singledispatch with type annotations def resolve_local_source( - source: typing.Union[str, os.PathLike, raw_nodes.URI, tuple, list], + source: typing.Union[str, os.PathLike, raw_nodes.URI], root_path: os.PathLike, output: typing.Optional[os.PathLike] = None, -) -> typing.Union[pathlib.Path, raw_nodes.URI, list, tuple]: +) -> typing.Union[pathlib.Path, raw_nodes.URI]: if isinstance(source, (tuple, list)): return type(source)([resolve_local_source(s, root_path, output) for s in source]) elif isinstance(source, os.PathLike) or isinstance(source, str): @@ -239,7 +250,7 @@ def resolve_local_source( return source else: try: - shutil.copy(source, output) + shutil.copyfile(source, output) except shutil.SameFileError: pass return pathlib.Path(output) From c6054e6889967025eb8d1d432b855594be3487e9 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Fri, 26 Nov 2021 21:19:11 +0100 Subject: [PATCH 11/15] remove check only available in py3.9 --- bioimageio/core/build_spec/build_model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bioimageio/core/build_spec/build_model.py b/bioimageio/core/build_spec/build_model.py index e29856b7..ca66a600 100644 --- a/bioimageio/core/build_spec/build_model.py +++ b/bioimageio/core/build_spec/build_model.py @@ -89,7 +89,6 @@ def _get_weights(original_weight_source, weight_type, source, root, **kwargs): weight_types = model_spec.raw_nodes.WeightsFormat weight_source = _ensure_local_or_url(original_weight_source, root) - assert isinstance(weight_source, URI) or weight_source.is_relative_to(root) if weight_type == "pytorch_state_dict": # pytorch-state-dict -> we need a source assert source is not None From aa399568be6bfab2852f32ba4295986bdb10854c Mon Sep 17 00:00:00 2001 From: fynnbe Date: Fri, 26 Nov 2021 21:33:22 +0100 Subject: [PATCH 12/15] various small fixes --- bioimageio/core/build_spec/build_model.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/bioimageio/core/build_spec/build_model.py b/bioimageio/core/build_spec/build_model.py index ca66a600..1431fbb8 100644 --- a/bioimageio/core/build_spec/build_model.py +++ b/bioimageio/core/build_spec/build_model.py @@ -341,7 +341,7 @@ def _get_deepimagej_config(export_folder, sample_inputs, sample_outputs, pixel_s def get_size(path): assert tifffile is not None, "need tifffile for writing deepimagej config" - with tifffile.TiffFile(path) as f: + with tifffile.TiffFile(export_folder / path) as f: shape = f.asarray().shape # add singleton z axis if we have 2d data if len(shape) == 3: @@ -373,7 +373,7 @@ def get_size(path): return {"deepimagej": config}, attachments -def _write_sample_data(input_paths, output_paths, input_axes, output_axes, export_folder): +def _write_sample_data(input_paths, output_paths, input_axes, output_axes, export_folder: Path): def write_im(path, im, axes): assert tifffile is not None, "need tifffile for writing deepimagej config" assert len(axes) == im.ndim @@ -396,18 +396,18 @@ def write_im(path, im, axes): sample_in_paths = [] for i, (in_path, axes) in enumerate(zip(input_paths, input_axes)): inp = np.load(in_path)[0] - sample_in_path = os.path.join(export_folder, f"sample_input_{i}.tif") + sample_in_path = export_folder / f"sample_input_{i}.tif" write_im(sample_in_path, inp, axes) sample_in_paths.append(sample_in_path) sample_out_paths = [] for i, (out_path, axes) in enumerate(zip(output_paths, output_axes)): outp = np.load(out_path)[0] - sample_out_path = os.path.join(export_folder, f"sample_output_{i}.tif") + sample_out_path = export_folder / f"sample_output_{i}.tif" write_im(sample_out_path, outp, axes) sample_out_paths.append(sample_out_path) - return sample_in_paths, sample_out_paths + return [Path(p.name) for p in sample_in_paths], [Path(p.name) for p in sample_out_paths] def _ensure_local(source: Union[Path, URI, str, list], root: Path) -> Union[Path, URI, list]: @@ -416,7 +416,8 @@ def _ensure_local(source: Union[Path, URI, str, list], root: Path) -> Union[Path return [_ensure_local(s, root) for s in source] local_source = resolve_source(source, root) - return resolve_source(local_source, root, root / local_source.name) + local_source = resolve_source(local_source, root, root / local_source.name) + return local_source.relative_to(root) def _ensure_local_or_url(source: Union[Path, URI, str, list], root: Path) -> Union[Path, URI, list]: @@ -425,7 +426,10 @@ def _ensure_local_or_url(source: Union[Path, URI, str, list], root: Path) -> Uni return [_ensure_local_or_url(s, root) for s in source] local_source = resolve_local_source(source, root) - return resolve_local_source(local_source, root, None if isinstance(local_source, URI) else root / local_source.name) + local_source = resolve_local_source( + local_source, root, None if isinstance(local_source, URI) else root / local_source.name + ) + return local_source.relative_to(root) def build_model( From a23a2d9c280fe7b5c68636458136c47cf2e7dcf0 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Fri, 26 Nov 2021 21:35:56 +0100 Subject: [PATCH 13/15] account for relative paths --- bioimageio/core/build_spec/build_model.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bioimageio/core/build_spec/build_model.py b/bioimageio/core/build_spec/build_model.py index 1431fbb8..d42e956b 100644 --- a/bioimageio/core/build_spec/build_model.py +++ b/bioimageio/core/build_spec/build_model.py @@ -395,14 +395,14 @@ def write_im(path, im, axes): sample_in_paths = [] for i, (in_path, axes) in enumerate(zip(input_paths, input_axes)): - inp = np.load(in_path)[0] + inp = np.load(export_folder / in_path)[0] sample_in_path = export_folder / f"sample_input_{i}.tif" write_im(sample_in_path, inp, axes) sample_in_paths.append(sample_in_path) sample_out_paths = [] for i, (out_path, axes) in enumerate(zip(output_paths, output_axes)): - outp = np.load(out_path)[0] + outp = np.load(export_folder / out_path)[0] sample_out_path = export_folder / f"sample_output_{i}.tif" write_im(sample_out_path, outp, axes) sample_out_paths.append(sample_out_path) @@ -571,7 +571,7 @@ def build_model( preprocessing = n_inputs * [None] if preprocessing is None else preprocessing inputs = [ - _get_input_tensor(test_in, name, step, min_shape, data_range, axes, preproc) + _get_input_tensor(root / test_in, name, step, min_shape, data_range, axes, preproc) for test_in, name, step, min_shape, axes, data_range, preproc in zip( test_inputs, input_name, input_step, input_min_shape, input_axes, input_data_range, preprocessing ) @@ -588,7 +588,7 @@ def build_model( halo = n_outputs * [None] if halo is None else halo outputs = [ - _get_output_tensor(test_out, name, reference, scale, offset, axes, data_range, postproc, hal) + _get_output_tensor(root / test_out, name, reference, scale, offset, axes, data_range, postproc, hal) for test_out, name, reference, scale, offset, axes, data_range, postproc, hal in zip( test_outputs, output_name, From 289f71abd93af5c37696de6fd97c162cb90c6350 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Sat, 27 Nov 2021 12:38:14 +0100 Subject: [PATCH 14/15] instantiate raw_nodes.Model with root_path needed to resolve relative paths --- bioimageio/core/build_spec/build_model.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/bioimageio/core/build_spec/build_model.py b/bioimageio/core/build_spec/build_model.py index d42e956b..aabba090 100644 --- a/bioimageio/core/build_spec/build_model.py +++ b/bioimageio/core/build_spec/build_model.py @@ -68,7 +68,7 @@ def _get_weights(original_weight_source, weight_type, source, root, **kwargs): # get the source path source_file = _ensure_local(source_file, root) - source_hash = _get_hash(source_file) + source_hash = _get_hash(root / source_file) # if not relative, create local copy (otherwise this will not work) if os.path.isabs(source_file): @@ -705,21 +705,22 @@ def build_model( try: model = model_spec.raw_nodes.Model( - format_version=format_version, - name=name, - description=description, authors=authors, cite=cite, - tags=tags, - license=license, - documentation=documentation, covers=covers, - timestamp=timestamp, - weights=weights, + description=description, + documentation=documentation, + format_version=format_version, inputs=inputs, + license=license, + name=name, outputs=outputs, + root_path=root, + tags=tags, test_inputs=test_inputs, test_outputs=test_outputs, + timestamp=timestamp, + weights=weights, **kwargs, ) model_package = export_resource_package(model, output_path=output_path) From 871cfafc4314317710e028eae4ffe46bcfd40372 Mon Sep 17 00:00:00 2001 From: Constantin Pape Date: Sun, 28 Nov 2021 12:14:38 +0100 Subject: [PATCH 15/15] Minor cleanup in build_spec --- bioimageio/core/build_spec/build_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bioimageio/core/build_spec/build_model.py b/bioimageio/core/build_spec/build_model.py index aabba090..6ce16b21 100644 --- a/bioimageio/core/build_spec/build_model.py +++ b/bioimageio/core/build_spec/build_model.py @@ -1,7 +1,6 @@ import datetime import hashlib import os -import shutil from pathlib import Path from shutil import copyfile from typing import Any, Dict, List, Optional, Tuple, Union @@ -291,7 +290,8 @@ def _get_deepimagej_macro(name, kwargs, export_folder): url = f"https://raw.githubusercontent.com/deepimagej/imagej-macros/master/bioimage.io/{macro}" path = os.path.join(export_folder, macro) - # TODO do we use requests? + # use https://github.com/bioimage-io/core-bioimage-io-python/blob/main/bioimageio/core/resource_io/utils.py#L267 + # instead if the implementation is update s.t. an output path is accepted with requests.get(url, stream=True) as r: with open(path, "w") as f: f.write(r.text)