diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 735ec3ad..26dd8cba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,8 @@ jobs: channels: conda-forge,ome environment-file: environment.yml python-version: ${{ matrix.python-version }} + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true - name: Run tests shell: bash -l {0} diff --git a/.github/workflows/zarr-dev.yml b/.github/workflows/zarr-dev.yml new file mode 100644 index 00000000..d2858a2d --- /dev/null +++ b/.github/workflows/zarr-dev.yml @@ -0,0 +1,38 @@ +--- +name: Build with Zarr dev + +on: [push, pull_request] + +jobs: + test: + name: ${{ matrix.platform }} ${{ matrix.python-version }} + runs-on: ${{ matrix.platform }} + + strategy: + fail-fast: true + matrix: + platform: [ubuntu-latest] + python-version: [3.7] + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + channels: conda-forge,ome + environment-file: environment.yml + python-version: ${{ matrix.python-version }} + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + + - name: Install zarr dev + shell: bash -l {0} + run: | + python -m pip install git+https://github.com/zarr-developers/zarr-python.git@master + + - name: Run tests + shell: bash -l {0} + run: make diff --git a/environment.yml b/environment.yml index 5854f629..60309009 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,7 @@ dependencies: - openjdk - maven - make - - z5py + - z5py >= 2.0.8 - python == 3.7.9 - scikit-image - pytest diff --git a/generate_data/generate_z5py.py b/generate_data/generate_z5py.py index 353a4d6f..8106cf71 100644 --- a/generate_data/generate_z5py.py +++ b/generate_data/generate_z5py.py @@ -5,28 +5,32 @@ CHUNKS = (100, 100, 1) # options for the different compressors -COMPRESSION_OPTIONS = {'blosc': {'filter_id': 'lz4'}} +COMPRESSION_OPTIONS = {"blosc": {"codec": "lz4"}} # TODO support more compressors: # - more compressors in numcodecs # - more blosc codecs def generate_zarr_format(compressors=['gzip', 'blosc', 'zlib', 'raw']): - path = '../data/z5py.zr' + path = 'data/z5py.zr' im = astronaut() - f = z5py.File(path) + f = z5py.File(path, mode='w') for compressor in compressors: copts = COMPRESSION_OPTIONS.get(compressor, {}) - name = compressor if compressor != 'blosc' else '%s/%s' % (compressor, copts.get('filter_id')) + name = ( + compressor + if compressor != "blosc" + else "%s/%s" % (compressor, copts.get("codec")) + ) f.create_dataset(name, data=im, compression=compressor, chunks=CHUNKS, **copts) def generate_n5_format(compressors=['gzip', 'raw']): - path = '../data/z5py.n5' + path = 'data/z5py.n5' im = astronaut() - f = z5py.File(path) + f = z5py.File(path, mode='w') for compressor in compressors: name = compressor f.create_dataset(name, data=im, chunks=CHUNKS, compression=compressor) diff --git a/generate_data/generate_zarr.py b/generate_data/generate_zarr.py index 27225ca4..7ad03ea8 100644 --- a/generate_data/generate_zarr.py +++ b/generate_data/generate_zarr.py @@ -4,24 +4,26 @@ # choose chunks s.t. we do have overhanging edge-chunks CHUNKS = (100, 100, 1) -STR_TO_COMPRESSOR = {'gzip': numcodecs.GZip, - 'blosc': numcodecs.Blosc, - 'zlib': numcodecs.Zlib} -COMPRESSION_OPTIONS = {'blosc': {'filter_id': 'lz4'}} +STR_TO_COMPRESSOR = { + "gzip": numcodecs.GZip, + "blosc": numcodecs.Blosc, + "zlib": numcodecs.Zlib, +} +COMPRESSION_OPTIONS = {"blosc": {"cname": "lz4"}} # TODO use more compressors from numcodecs and more blosc filter_ids def generate_zarr_format(compressors=['gzip', 'blosc', 'zlib', None]): - path = '../data/zarr.zr' + path = 'data/zarr.zr' im = astronaut() - f = zarr.open(path) + f = zarr.open(path, mode='w') for compressor in compressors: copts = COMPRESSION_OPTIONS.get(compressor, {}) if compressor is None: - name = 'taw' - elif compressor == 'blosc': - name = '%s/%s' % (compressor, copts.get('filter_id')) + name = "raw" + elif compressor == "blosc": + name = "%s/%s" % (compressor, copts.get("cname")) else: name = compressor compressor_impl = STR_TO_COMPRESSOR[compressor](**copts) if compressor is not None else None @@ -29,15 +31,14 @@ def generate_zarr_format(compressors=['gzip', 'blosc', 'zlib', None]): compressor=compressor_impl) -# this needs PR https://github.com/zarr-developers/zarr/pull/309 def generate_n5_format(compressors=['gzip', None]): - path = '../data/zarr.n5' + path = 'data/zarr.n5' im = astronaut() - f = zarr.open(path) + f = zarr.open(path, mode='w') for compressor in compressors: name = compressor if compressor is not None else 'raw' - compressor_impl = STR_TO_COMPRESSOR[compressor] if compressor is not None else None + compressor_impl = STR_TO_COMPRESSOR[compressor]() if compressor is not None else None f.create_dataset(name, data=im, chunks=CHUNKS, compressor=compressor_impl) diff --git a/test/test_read_all.py b/test/test_read_all.py index 47b6f503..be9a557b 100644 --- a/test/test_read_all.py +++ b/test/test_read_all.py @@ -40,6 +40,8 @@ def read_with_zarr(fpath, ds_name): import zarr + if ds_name == "blosc": + ds_name = "blosc/lz4" return zarr.open(os.fspath(fpath))[ds_name][:] @@ -50,6 +52,8 @@ def read_with_pyn5(fpath, ds_name): def read_with_z5py(fpath, ds_name): import z5py + if ds_name == "blosc": + ds_name = "blosc/lz4" return z5py.File(fpath)[ds_name][:]