diff --git a/.gitignore b/.gitignore index 583ecc4..0afa1d7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ **/test-output-actual +settings.json + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/Dockerfile b/Dockerfile index de93be1..7a11b9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,11 @@ # We just need to use --file to point at it, instead of assuming it is in context. # Using Conda because pyarrow did not install easily on python base images. -FROM continuumio/miniconda3 +FROM continuumio/miniconda3:4.7.12 +# For tiff packages +RUN apt-get update &&\ + apt-get install -y gcc python3-dev COPY requirements-freeze.txt . RUN pip install -r ./requirements-freeze.txt diff --git a/README.md b/README.md index dd63da0..bbe396f 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,9 @@ Docker containers to pre-process data for visualization in the portal. The subdirectories in this repo all have the same structure: + - `context/`: A Docker context, including at least -`main.py`, `requirements.txt`, and `requirements-freeze.txt`. + `main.py`, `requirements.txt`, and `requirements-freeze.txt`. - `test-input/`, `test-output-actual/`, `test-output-expected/`: Test fixtures. - `VERSION`: contains a semantic version number - and a `README.md`. @@ -15,6 +16,7 @@ Images are named by the containing directory. Running `test.sh` will build (and test!) all the images. You can then define `$INPUT_DIR`, `$OUTPUT_DIR`, and `$IMAGE` to run an image with your own data: + ``` docker run \ --mount type=bind,source=$INPUT_DIR,target=/input \ @@ -23,6 +25,7 @@ docker run \ ``` To push the latest versions to dockerhub just run: + ``` test.sh push ``` diff --git a/containers/ome-tiff-offsets/README.md b/containers/ome-tiff-offsets/README.md new file mode 100644 index 0000000..69ad545 --- /dev/null +++ b/containers/ome-tiff-offsets/README.md @@ -0,0 +1,4 @@ +# ome-tiff-offsets + +This docker container adds a structured annotation to the OMEXML +which contains the `IFD_Offsets` in bytes diff --git a/containers/ome-tiff-offsets/VERSION b/containers/ome-tiff-offsets/VERSION new file mode 100644 index 0000000..8acdd82 --- /dev/null +++ b/containers/ome-tiff-offsets/VERSION @@ -0,0 +1 @@ +0.0.1 diff --git a/containers/ome-tiff-offsets/context/main.py b/containers/ome-tiff-offsets/context/main.py new file mode 100644 index 0000000..1a94d31 --- /dev/null +++ b/containers/ome-tiff-offsets/context/main.py @@ -0,0 +1,65 @@ +import argparse +from glob import glob +from pathlib import Path +from os import makedirs + +from aicsimageio import AICSImage +from aicsimageio.writers import ome_tiff_writer +from tifffile import TiffFile +import xml.dom.minidom + +def get_offsets(tiff_filepath): + with TiffFile(tiff_filepath) as tif: + offsets = [page.offset for page in tif.pages] + return offsets + + + +def main(input_dir, output_dir): + makedirs(output_dir, exist_ok=True) + for input in glob(input_dir + '/*.ome.tif*') + glob(input_dir + '/*.ome.tiff'): + # Get image metadata and image data. + with AICSImage(input) as input_image: + image_data_from_input = input_image.get_image_data()[0] + omexml = input_image.metadata + # Create the output path for the compressed ome tiff. + input_path = Path(input) + compressed_dir = Path('/compressed/') + compressed_ome_tiff = compressed_dir / input_path.name + makedirs(compressed_dir) + with ome_tiff_writer.OmeTiffWriter(compressed_ome_tiff) as ome_writer: + ome_writer.save( + image_data_from_input, + ome_xml = omexml + ) + # Read in the newly compressed file. + with AICSImage(compressed_ome_tiff) as compressed_image: + image_data_from_compressed = compressed_image.get_image_data()[0] + omexml_compressed = compressed_image.metadata + # Get the offsets of said compressed file and add them to the omexml as structured annotations. + offsets = get_offsets(compressed_ome_tiff) + structured_annotations = omexml_compressed.structured_annotations + structured_annotations.add_original_metadata(key='IFD_Offsets', value=str(offsets)) + # Write the file out to the bound output directory. + with open(Path(output_dir) / 'ome.xml', 'w') as xml_write: + xml_write.write(xml.dom.minidom.parseString(str(omexml_compressed)).toprettyxml()) + new_ome_tiff_path = Path(output_dir) / input_path.name + with ome_tiff_writer.OmeTiffWriter(new_ome_tiff_path, overwrite_file=True) as ome_writer: + ome_writer.save( + image_data_from_compressed, + ome_xml = omexml_compressed + ) + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Add offsets to the OMEXML Metadata') + parser.add_argument( + '--input_dir', required=True, + help='Directory containing ome-tiff files to read') + parser.add_argument( + '--output_dir', required=True, + help='Directory where ome-tiff files should be written') + args = parser.parse_args() + main(args.input_dir, args.output_dir) diff --git a/containers/ome-tiff-offsets/context/requirements-freeze.txt b/containers/ome-tiff-offsets/context/requirements-freeze.txt new file mode 100644 index 0000000..e6e1bd0 --- /dev/null +++ b/containers/ome-tiff-offsets/context/requirements-freeze.txt @@ -0,0 +1,48 @@ +aicsimageio==3.1.4 +aicspylibczi==2.5.1 +asn1crypto==1.0.1 +certifi==2019.9.11 +cffi==1.12.3 +chardet==3.0.4 +click==7.1.1 +cloudpickle==1.3.0 +conda==4.7.12 +conda-package-handling==1.6.0 +cryptography==2.7 +cycler==0.10.0 +dask==2.13.0 +decorator==4.4.2 +distributed==2.13.0 +HeapDict==1.0.1 +idna==2.8 +imagecodecs==2020.2.18 +imageio==2.8.0 +kiwisolver==1.1.0 +lxml==4.5.0 +matplotlib==3.2.1 +msgpack==1.0.0 +networkx==2.4 +numpy==1.18.2 +Pillow==7.0.0 +psutil==5.7.0 +pycosat==0.6.3 +pycparser==2.19 +pyOpenSSL==19.0.0 +pyparsing==2.4.6 +PySocks==1.7.1 +python-dateutil==2.8.1 +PyWavelets==1.1.1 +PyYAML==5.3.1 +requests==2.22.0 +ruamel-yaml==0.15.46 +scikit-image==0.16.2 +scipy==1.4.1 +six==1.14.0 +sortedcontainers==2.1.0 +tblib==1.6.0 +tifffile==2020.2.16 +toolz==0.10.0 +tornado==6.0.4 +tqdm==4.36.1 +urllib3==1.24.2 +zict==2.0.0 diff --git a/containers/ome-tiff-offsets/context/requirements.txt b/containers/ome-tiff-offsets/context/requirements.txt new file mode 100644 index 0000000..3c6d210 --- /dev/null +++ b/containers/ome-tiff-offsets/context/requirements.txt @@ -0,0 +1,2 @@ +aicsimageio==3.1.4 +tifffile==2020.2.16 \ No newline at end of file diff --git a/containers/ome-tiff-offsets/test-input/multi-channel.ome.tif b/containers/ome-tiff-offsets/test-input/multi-channel.ome.tif new file mode 100644 index 0000000..3d32d85 Binary files /dev/null and b/containers/ome-tiff-offsets/test-input/multi-channel.ome.tif differ diff --git a/containers/ome-tiff-offsets/test-output-expected/multi-channel.ome.tif b/containers/ome-tiff-offsets/test-output-expected/multi-channel.ome.tif new file mode 100644 index 0000000..162fd7e Binary files /dev/null and b/containers/ome-tiff-offsets/test-output-expected/multi-channel.ome.tif differ diff --git a/containers/ome-tiff-offsets/test-output-expected/ome.xml b/containers/ome-tiff-offsets/test-output-expected/ome.xml new file mode 100644 index 0000000..f3c4353 --- /dev/null +++ b/containers/ome-tiff-offsets/test-output-expected/ome.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + IFD_Offsets + [8, 2712, 4394] + + + + + diff --git a/test.sh b/test.sh index 76dbed9..4f69cef 100755 --- a/test.sh +++ b/test.sh @@ -11,6 +11,7 @@ die() { set +v; echo "$red$*$reset" 1>&2 ; exit 1; } build_test() { TAG=$1 + BASENAME=$2 docker build --file ../../Dockerfile --tag $TAG context PWD_BASE=`basename $PWD` docker rm -f $PWD_BASE || echo "No container to stop" @@ -25,11 +26,12 @@ build_test() { # hexdump -C test-output-expected/2x2.arrow > test-output-expected/2x2.arrow.hex.txt # hexdump -C test-output-actual/2x2.arrow > test-output-actual/2x2.arrow.hex.txt - + if [ "$BASENAME" == "ome-tiff-offsets" ]; then + sed -i.bak 's/XMLAnnotation ID="[^"]*"/XMLAnnotation ID="PLACEHOLDER"/g' test-output-actual/ome.xml + fi diff -w -r test-output-expected test-output-actual \ - --exclude=.DS_Store --exclude=*.arrow \ - | head -n100 | cut -c 1-100 - + --exclude=.DS_Store --exclude=*.arrow --exclude=*.ome.tif* \ + --exclude=ome.xml.bak | head -n100 | cut -c 1-100 diff <( docker run $TAG pip freeze ) context/requirements-freeze.txt \ || die "Update dependencies: docker run $TAG pip freeze > $TAG/context/requirements-freeze.txt" @@ -46,7 +48,7 @@ for DIR in containers/*; do # Neither underscores nor double dash is allowed: # Don't get too creative! TAG="hubmap/portal-container-$BASENAME:$VERSION" - build_test $TAG + build_test $TAG $BASENAME if [ "$1" == 'push' ]; then COMMAND="docker push $TAG" echo "$green$COMMAND$reset"