Skip to content

Commit

Permalink
support overhanging and incomplete blocks for cell images
Browse files Browse the repository at this point in the history 
only the intersection of block size and cell size will be filled
among other use-cases, this supports zarr-like trailing blocks
  • Loading branch information
@axtimwalde
axtimwalde committed Feb 13, 2019
1 parent c121f78 commit dd26251
Show file tree
Hide file tree
Showing 4 changed files with 202 additions and 52 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<parent>
<groupId>org.scijava</groupId>
<artifactId>pom-scijava</artifactId>
<version>23.2.0</version>
<version>25.0.0</version>
<relativePath />
</parent>

Expand Down
149 changes: 117 additions & 32 deletions src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5CellLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,14 @@
import org.janelia.saalfeldlab.n5.N5Reader;

import net.imglib2.Cursor;
import net.imglib2.FinalInterval;
import net.imglib2.Interval;
import net.imglib2.IterableInterval;
import net.imglib2.RandomAccessibleInterval;
import net.imglib2.cache.img.CellLoader;
import net.imglib2.cache.img.SingleCellArrayImg;
import net.imglib2.img.array.ArrayImg;
import net.imglib2.img.array.ArrayImgs;
import net.imglib2.type.NativeType;
import net.imglib2.type.Type;
import net.imglib2.type.numeric.integer.GenericByteType;
Expand All @@ -49,6 +53,8 @@
import net.imglib2.type.numeric.integer.GenericShortType;
import net.imglib2.type.numeric.real.DoubleType;
import net.imglib2.type.numeric.real.FloatType;
import net.imglib2.util.Intervals;
import net.imglib2.view.IntervalView;
import net.imglib2.view.Views;

/**
Expand All @@ -69,7 +75,7 @@ public class N5CellLoader<T extends NativeType<T>> implements CellLoader<T> {

private final DatasetAttributes attributes;

private final BiConsumer<IterableInterval<T>, DataBlock<?>> copyFromBlock;
private final BiConsumer<SingleCellArrayImg<T, ?>, DataBlock<?>> copyFromBlock;

private final Consumer<IterableInterval<T>> blockNotFoundHandler;

Expand Down Expand Up @@ -170,61 +176,79 @@ public static <T extends Type<T>> boolean burnInTestAllEqual(
return equal;
}

public static <T extends NativeType<T>> BiConsumer<IterableInterval<T>, DataBlock<?>> createCopy(
public static <T extends NativeType<T>, I extends RandomAccessibleInterval<T> & IterableInterval<T>> BiConsumer<I, DataBlock<?>> createCopy(
final DataType dataType) {

switch (dataType) {
case INT8:
case UINT8:
return (a, b) -> {
final byte[] data = (byte[])b.getData();
@SuppressWarnings("unchecked")
final Cursor<? extends GenericByteType<?>> c = (Cursor<? extends GenericByteType<?>>)a.cursor();
for (int i = 0; i < data.length; ++i)
c.next().setByte(data[i]);
if (sizeEquals(a, b)) {
final byte[] data = (byte[])b.getData();
@SuppressWarnings("unchecked")
final Cursor<? extends GenericByteType<?>> c = (Cursor<? extends GenericByteType<?>>)a.cursor();
for (int i = 0; i < data.length; ++i)
c.next().setByte(data[i]);
} else
copyIntersection(a, b, dataType);
};
case INT16:
case UINT16:
return (a, b) -> {
final short[] data = (short[])b.getData();
@SuppressWarnings("unchecked")
final Cursor<? extends GenericShortType<?>> c = (Cursor<? extends GenericShortType<?>>)a.cursor();
for (int i = 0; i < data.length; ++i)
c.next().setShort(data[i]);
if (sizeEquals(a, b)) {
final short[] data = (short[])b.getData();
@SuppressWarnings("unchecked")
final Cursor<? extends GenericShortType<?>> c = (Cursor<? extends GenericShortType<?>>)a.cursor();
for (int i = 0; i < data.length; ++i)
c.next().setShort(data[i]);
} else
copyIntersection(a, b, dataType);
};
case INT32:
case UINT32:
return (a, b) -> {
final int[] data = (int[])b.getData();
@SuppressWarnings("unchecked")
final Cursor<? extends GenericIntType<?>> c = (Cursor<? extends GenericIntType<?>>)a.cursor();
for (int i = 0; i < data.length; ++i)
c.next().setInt(data[i]);
if (sizeEquals(a, b)) {
final int[] data = (int[])b.getData();
@SuppressWarnings("unchecked")
final Cursor<? extends GenericIntType<?>> c = (Cursor<? extends GenericIntType<?>>)a.cursor();
for (int i = 0; i < data.length; ++i)
c.next().setInt(data[i]);
} else
copyIntersection(a, b, dataType);
};
case INT64:
case UINT64:
return (a, b) -> {
final long[] data = (long[])b.getData();
@SuppressWarnings("unchecked")
final Cursor<? extends GenericLongType<?>> c = (Cursor<? extends GenericLongType<?>>)a.cursor();
for (int i = 0; i < data.length; ++i)
c.next().setLong(data[i]);
if (sizeEquals(a, b)) {
final long[] data = (long[])b.getData();
@SuppressWarnings("unchecked")
final Cursor<? extends GenericLongType<?>> c = (Cursor<? extends GenericLongType<?>>)a.cursor();
for (int i = 0; i < data.length; ++i)
c.next().setLong(data[i]);
} else
copyIntersection(a, b, dataType);
};
case FLOAT32:
return (a, b) -> {
final float[] data = (float[])b.getData();
@SuppressWarnings("unchecked")
final Cursor<? extends FloatType> c = (Cursor<? extends FloatType>)a.cursor();
for (int i = 0; i < data.length; ++i)
c.next().set(data[i]);
if (sizeEquals(a, b)) {
final float[] data = (float[])b.getData();
@SuppressWarnings("unchecked")
final Cursor<? extends FloatType> c = (Cursor<? extends FloatType>)a.cursor();
for (int i = 0; i < data.length; ++i)
c.next().set(data[i]);
} else
copyIntersection(a, b, dataType);
};
case FLOAT64:
return (a, b) -> {
final double[] data = (double[])b.getData();
@SuppressWarnings("unchecked")
final Cursor<? extends DoubleType> c = (Cursor<? extends DoubleType>)a.cursor();
for (int i = 0; i < data.length; ++i)
c.next().set(data[i]);
if (sizeEquals(a, b)) {
final double[] data = (double[])b.getData();
@SuppressWarnings("unchecked")
final Cursor<? extends DoubleType> c = (Cursor<? extends DoubleType>)a.cursor();
for (int i = 0; i < data.length; ++i)
c.next().set(data[i]);
} else
copyIntersection(a, b, dataType);
};
default:
throw new IllegalArgumentException("Type " + dataType.name() + " not supported!");
Expand All @@ -242,4 +266,65 @@ public static <T extends Type<T>, I extends IterableInterval<T>> Consumer<I> set

return rai -> rai.forEach(pixel -> pixel.set(defaultValue));
}

private static boolean sizeEquals(final Interval a, final DataBlock<?> b) {

final int[] dataBlockSize = b.getSize();
for (int d = 0; d < dataBlockSize.length; ++d) {
if (a.dimension(d) != dataBlockSize[d])
return false;
}
return true;
}

@SuppressWarnings("rawtypes")
private static ArrayImg dataBlock2ArrayImg(
final DataBlock<?> dataBlock,
final DataType dataType) {

final int[] dataBlockSize = dataBlock.getSize();
final long[] dims = new long[dataBlockSize.length];
for (int d = 0; d < dataBlockSize.length; ++d)
dims[d] = dataBlockSize[d];

switch (dataType) {
case INT8:
return ArrayImgs.bytes((byte[])dataBlock.getData(), dims);
case UINT8:
return ArrayImgs.unsignedBytes((byte[])dataBlock.getData(), dims);
case INT16:
return ArrayImgs.shorts((short[])dataBlock.getData(), dims);
case UINT16:
return ArrayImgs.unsignedShorts((short[])dataBlock.getData(), dims);
case INT32:
return ArrayImgs.ints((int[])dataBlock.getData(), dims);
case UINT32:
return ArrayImgs.unsignedInts((int[])dataBlock.getData(), dims);
case INT64:
return ArrayImgs.longs((long[])dataBlock.getData(), dims);
case UINT64:
return ArrayImgs.unsignedLongs((long[])dataBlock.getData(), dims);
case FLOAT32:
return ArrayImgs.floats((float[])dataBlock.getData(), dims);
case FLOAT64:
return ArrayImgs.doubles((double[])dataBlock.getData(), dims);
default:
return null;
}
}

private static <T extends NativeType<T>, I extends RandomAccessibleInterval<T> & IterableInterval<T>> void copyIntersection(
final I a,
final DataBlock<?> b,
final DataType dataType) {

@SuppressWarnings("unchecked")
final ArrayImg<T, ?> block = dataBlock2ArrayImg(b, dataType);
final IntervalView<T> za = Views.zeroMin(a);
final FinalInterval intersection = Intervals.intersect(block, za);
final Cursor<T> c = Views.interval(za, intersection).cursor();
final Cursor<T> d = Views.interval(block, intersection).cursor();
while (c.hasNext())
c.next().set(d.next());
}
}
15 changes: 8 additions & 7 deletions src/main/java/org/janelia/saalfeldlab/n5/imglib2/N5Utils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,6 @@
import java.util.function.Function;
import java.util.function.IntFunction;

import net.imglib2.cache.LoaderCache;
import net.imglib2.cache.ref.BoundedSoftRefLoaderCache;
import net.imglib2.img.basictypeaccess.array.ArrayDataAccess;
import org.janelia.saalfeldlab.n5.Compression;
import org.janelia.saalfeldlab.n5.DataBlock;
import org.janelia.saalfeldlab.n5.DataType;
Expand All @@ -50,14 +47,17 @@
import net.imglib2.IterableInterval;
import net.imglib2.RandomAccessibleInterval;
import net.imglib2.cache.Cache;
import net.imglib2.cache.LoaderCache;
import net.imglib2.cache.img.CachedCellImg;
import net.imglib2.cache.img.DiskCachedCellImgFactory;
import net.imglib2.cache.img.DiskCachedCellImgOptions;
import net.imglib2.cache.img.LoadedCellCacheLoader;
import net.imglib2.cache.ref.BoundedSoftRefLoaderCache;
import net.imglib2.cache.ref.SoftRefLoaderCache;
import net.imglib2.img.array.ArrayImgs;
import net.imglib2.img.basictypeaccess.AccessFlags;
import net.imglib2.img.basictypeaccess.ArrayDataAccessFactory;
import net.imglib2.img.basictypeaccess.array.ArrayDataAccess;
import net.imglib2.img.basictypeaccess.volatiles.VolatileAccess;
import net.imglib2.img.cell.Cell;
import net.imglib2.img.cell.CellGrid;
Expand Down Expand Up @@ -119,6 +119,7 @@ public static final <T extends NativeType<T>> DataType dataType(final T type) {
return null;
}

@SuppressWarnings("unchecked")
public static final <T extends NativeType<T>> T type(final DataType dataType) {

switch (dataType) {
Expand Down Expand Up @@ -647,7 +648,7 @@ public static final <T extends NativeType<T>> RandomAccessibleInterval<T> open(

final DatasetAttributes attributes = n5.getDatasetAttributes(dataset);
final LoaderCache loaderCache = loaderCacheFactory.apply(attributes.getDataType());
T type = type(attributes.getDataType());
final T type = type(attributes.getDataType());
return type == null
? null
: open(n5, dataset, blockNotFoundHandler, loaderCache, accessFlags, type);
Expand All @@ -667,8 +668,8 @@ public static final <T extends NativeType<T>, A extends ArrayDataAccess<A>> Cach
final N5Reader n5,
final String dataset,
final Consumer<IterableInterval<T>> blockNotFoundHandler,
LoaderCache<Long, Cell<A>> loaderCache,
Set<AccessFlags> accessFlags,
final LoaderCache<Long, Cell<A>> loaderCache,
final Set<AccessFlags> accessFlags,
final T type) throws IOException {

final DatasetAttributes attributes = n5.getDatasetAttributes(dataset);
Expand All @@ -679,7 +680,7 @@ public static final <T extends NativeType<T>, A extends ArrayDataAccess<A>> Cach

final CellGrid grid = new CellGrid(dimensions, blockSize);

Cache<Long, Cell<A>> cache = loaderCache.withLoader(LoadedCellCacheLoader.get(grid, loader, type, accessFlags));
final Cache<Long, Cell<A>> cache = loaderCache.withLoader(LoadedCellCacheLoader.get(grid, loader, type, accessFlags));
final CachedCellImg img = new CachedCellImg(grid, type, cache, ArrayDataAccessFactory.get(type, accessFlags));
return img;
}
Expand Down
Loading

0 comments on commit dd26251

Please sign in to comment.