Skip to content

NdArray Initializer API #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions ndarray/src/main/java/org/tensorflow/ndarray/NdArrays.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
*/
package org.tensorflow.ndarray;

import java.util.function.Consumer;
import org.tensorflow.ndarray.buffer.BooleanDataBuffer;
import org.tensorflow.ndarray.buffer.ByteDataBuffer;
import org.tensorflow.ndarray.buffer.DataBuffer;
Expand All @@ -25,6 +26,9 @@
import org.tensorflow.ndarray.buffer.IntDataBuffer;
import org.tensorflow.ndarray.buffer.LongDataBuffer;
import org.tensorflow.ndarray.buffer.ShortDataBuffer;
import org.tensorflow.ndarray.hydrator.DoubleNdArrayHydrator;
import org.tensorflow.ndarray.hydrator.NdArrayHydrator;
import org.tensorflow.ndarray.impl.dense.AbstractDenseNdArray;
import org.tensorflow.ndarray.impl.dense.BooleanDenseNdArray;
import org.tensorflow.ndarray.impl.dense.ByteDenseNdArray;
import org.tensorflow.ndarray.impl.dense.DenseNdArray;
Expand All @@ -33,14 +37,19 @@
import org.tensorflow.ndarray.impl.dense.IntDenseNdArray;
import org.tensorflow.ndarray.impl.dense.LongDenseNdArray;
import org.tensorflow.ndarray.impl.dense.ShortDenseNdArray;
import org.tensorflow.ndarray.impl.dense.hydrator.DenseNdArrayHydrator;
import org.tensorflow.ndarray.impl.dense.hydrator.DoubleDenseNdArrayHydrator;
import org.tensorflow.ndarray.impl.dimension.DimensionalSpace;
import org.tensorflow.ndarray.impl.sparse.AbstractSparseNdArray;
import org.tensorflow.ndarray.impl.sparse.BooleanSparseNdArray;
import org.tensorflow.ndarray.impl.sparse.ByteSparseNdArray;
import org.tensorflow.ndarray.impl.sparse.DoubleSparseNdArray;
import org.tensorflow.ndarray.impl.sparse.FloatSparseNdArray;
import org.tensorflow.ndarray.impl.sparse.IntSparseNdArray;
import org.tensorflow.ndarray.impl.sparse.LongSparseNdArray;
import org.tensorflow.ndarray.impl.sparse.ShortSparseNdArray;
import org.tensorflow.ndarray.impl.sparse.hydrator.DoubleSparseNdArrayHydrator;
import org.tensorflow.ndarray.impl.sparse.hydrator.SparseNdArrayHydrator;

/** Utility class for instantiating {@link NdArray} objects. */
public final class NdArrays {
Expand Down Expand Up @@ -555,6 +564,20 @@ public static DoubleNdArray ofDoubles(Shape shape) {
return wrap(shape, DataBuffers.ofDoubles(shape.size()));
}

/**
* Creates an N-dimensional array of doubles of the given shape, hydrating it with data after its allocation
*
* @param shape shape of the array
* @param hydrate initialize the data of the created array, using a hydrator
* @return new double N-dimensional array
* @throws IllegalArgumentException if shape is null or has unknown dimensions
*/
public static DoubleNdArray ofDoubles(Shape shape, Consumer<DoubleNdArrayHydrator> hydrate) {
DoubleDenseNdArray array = (DoubleDenseNdArray)ofDoubles(shape);
hydrate.accept(new DoubleDenseNdArrayHydrator(array));
return array;
}

/**
* Wraps a buffer in a double N-dimensional array of a given shape.
*
Expand All @@ -568,6 +591,23 @@ public static DoubleNdArray wrap(Shape shape, DoubleDataBuffer buffer) {
return DoubleDenseNdArray.create(buffer, shape);
}

/**
* Creates an Sparse array of doubles of the given shape, hydrating it with data after its allocation
*
* @param shape shape of the array
* @param numValues number of double value actually set in the array, others defaulting to the zero value
* @param hydrate initialize the data of the created array, using a hydrator
* @return new double N-dimensional array
* @throws IllegalArgumentException if shape is null or has unknown dimensions
*/
public static DoubleSparseNdArray sparseOfDoubles(Shape shape, long numValues, Consumer<DoubleNdArrayHydrator> hydrate) {
LongNdArray indices = ofLongs(Shape.of(numValues, shape.numDimensions()));
DoubleNdArray values = ofDoubles(Shape.of(numValues));
DoubleSparseNdArray array = DoubleSparseNdArray.create(indices, values, DimensionalSpace.create(shape));
hydrate.accept(new DoubleSparseNdArrayHydrator(array));
return array;
}

/**
* Creates a Sparse array of double values with a default value of zero
*
Expand Down Expand Up @@ -756,6 +796,21 @@ public static <T> NdArray<T> ofObjects(Class<T> clazz, Shape shape) {
return wrap(shape, DataBuffers.ofObjects(clazz, shape.size()));
}

/**
* Creates an N-dimensional array of objects of the given shape, hydrating it with data after its allocation
*
* @param clazz class of the data to be stored in this array
* @param shape shape of the array
* @param hydrate initialize the data of the created array, using a hydrator
* @return new N-dimensional array
* @throws IllegalArgumentException if shape is null or has unknown dimensions
*/
public static <T> NdArray<T> ofObjects(Class<T> clazz, Shape shape, Consumer<NdArrayHydrator<T>> hydrate) {
AbstractDenseNdArray<T, ?> array = (AbstractDenseNdArray<T, ?>)ofObjects(clazz, shape);
hydrate.accept(new DenseNdArrayHydrator<T>(array));
return array;
}

/**
* Wraps a buffer in an N-dimensional array of a given shape.
*
Expand All @@ -770,6 +825,24 @@ public static <T> NdArray<T> wrap(Shape shape, DataBuffer<T> buffer) {
return DenseNdArray.wrap(buffer, shape);
}

/**
* Creates an Sparse array of objects of the given shape, hydrating it with data after its allocation
*
* @param type the class type represented by this sparse array.
* @param shape shape of the array
* @param numValues number of values actually set in the array, others defaulting to the zero value
* @param hydrate initialize the data of the created array, using a hydrator
* @return new N-dimensional array
* @throws IllegalArgumentException if shape is null or has unknown dimensions
*/
public static <T> NdArray<T> sparseOfObjects(Class<T> type, Shape shape, long numValues, Consumer<NdArrayHydrator<T>> hydrate) {
LongNdArray indices = ofLongs(Shape.of(numValues, shape.numDimensions()));
NdArray<T> values = ofObjects(type, Shape.of(numValues));
AbstractSparseNdArray<T, ?> array = (AbstractSparseNdArray<T, ?>)sparseOfObjects(type, indices, values, shape);
hydrate.accept(new SparseNdArrayHydrator<T>(array));
return array;
}

/**
* Creates a Sparse array of values with a null default value
*
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
package org.tensorflow.ndarray.hydrator;

import org.tensorflow.ndarray.DoubleNdArray;

/**
* Specialization of the {@link NdArrayHydrator} API for hydrating arrays of doubles.
*
* @see NdArrayHydrator
*/
public interface DoubleNdArrayHydrator {

/**
* An API for hydrate an {@link DoubleNdArray} using scalar values
*/
interface Scalars {

/**
* Position the hydrator to the given {@code coordinates} to write the next scalars.
*
* @param coordinates position in the hydrated array
* @return this API
* @throws IllegalArgumentException if {@code coordinates} are empty or are not one of a scalar
*/
Scalars at(long... coordinates);

/**
* Set a double value as the next scalar value in the hydrated array.
*
* @param scalar next scalar value
* @return this API
* @throws IllegalArgumentException if {@code scalar} is null
*/
Scalars put(double scalar);
}

/**
* An API for hydrate an {@link DoubleNdArray} using vectors, i.e. a list of scalars
*/
interface Vectors {

/**
* Position the hydrator to the given {@code coordinates} to write the next vectors.
*
* @param coordinates position in the hydrated array
* @return this API
* @throws IllegalArgumentException if {@code coordinates} are empty or are not one of a vector
*/
Vectors at(long... coordinates);

/**
* Set a list of doubles as the next vector in the hydrated array.
*
* @param vector next vector values
* @return this API
* @throws IllegalArgumentException if {@code vector} is empty or its length is greater than the size of the dimension
* {@code n-1}, given {@code n} the rank of the hydrated array
*/
Vectors put(double... vector);
}

/**
* An API for hydrate an {@link DoubleNdArray} using n-dimensional elements (sub-arrays).
*/
interface Elements {

/**
* Position the hydrator to the given {@code coordinates} to write the next elements.
*
* @param coordinates position in the hydrated array
* @return this API
* @throws IllegalArgumentException if {@code coordinates} are empty or are not one of an element of the hydrated array
*/
Elements at(long... coordinates);

/**
* Set a n-dimensional array of doubles as the next element in the hydrated array.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So maybe this should be "next elements" rather than "next element", but also it's not entirely clear to me what happens if I pass in a rectangular ndarray into a larger ndarray where the values are set. Are they read off in row-major order and placed into the next elements in row-major order of the target array? Or is it doing broadcasting/shape matching of some kind?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I made the contract more strict when it comes to per-element initialization: an initialization pass must always be done with elements of the same rank. So basically you target a dimension and then provide the elements for it.

*
* @param element array containing the next element values
* @return this API
* @throws IllegalArgumentException if {@code element} is null or its shape is incompatible with the current hydrator position
*/
Elements put(DoubleNdArray element);
}

/**
* Start to hydrate the targeted array with scalars.
*
* If no {@code coordinates} are provided, the start position is the current one relatively to any previous hydration that occured or if none,
* defaults to the first scalar of this array.
*
* Example of usage:
* <pre>{@code
* DoubleNdArray array = NdArrays.ofDoubles(Shape.of(3, 2), hydrator -> {
* hydrator.byScalars()
* .put(10.0)
* .put(20.0)
* .put(30.0)
* .at(2, 1)
* .put(40.0);
* });
* // -> [[10.0, 20.0], [30.0, 0.0], [0.0, 40.0]]
* }</pre>
*
* @param coordinates position in the hydrated array to start from
* @return a {@link Scalars} instance
* @throws IllegalArgumentException if {@code coordinates} are set but are not one of a scalar
*/
Scalars byScalars(long... coordinates);

/**
* Start to hydrate the targeted array with vectors.
*
* If no {@code coordinates} are provided, the start position is the current one relatively to any previous hydration that occured or if none,
* defaults to the first scalar of the first vector of this array.
*
* Example of usage:
* <pre>{@code
* DoubleNdArray array = NdArrays.ofDoubles(Shape.of(3, 2), hydrator -> {
* hydrator.byVectors()
* .put(10.0, 20.0)
* .put(30.0)
* .at(2)
* .put(40.0, 50.0);
* });
* // -> [[10.0, 20.0], [30.0, null], [40.0, 50.0]]
* }</pre>
*
* @param coordinates position in the hydrated array to start from
* @return a {@link Vectors} instance
* @throws IllegalArgumentException if hydrated array is of rank-0 or if {@code coordinates} are set but are not one of a vector
*/
Vectors byVectors(long... coordinates);

/**
* Start to hydrate the targeted array with n-dimensional elements.
*
* If no {@code coordinates} are provided, the start position is the current one relatively to any previous hydration that occured or if none,
* defaults to the first element in the first (0) dimension of the hydrated array.
*
* Example of usage:
* <pre>{@code
* DoubleNdArray vector = NdArrays.vectorOf(10.0, 20.0);
* DoubleNdArray scalar = NdArrays.scalarOf(30.0);
*
* DoubleNdArray array = NdArrays.ofDoubles(Shape.of(4, 2), hydrator -> {
* hydrator.byElements()
* .put(vector)
* .put(vector)
* .at(2, 1)
* .put(scalar)
* .at(3)
* .put(vector);
* });
* // -> [[10.0, 20.0], [10.0, 20.0], [0.0, 30.0], [10.0, 20.0]]
* }</pre>
*
* @param coordinates position in the hydrated array to start from
* @return a {@link Elements} instance
* @throws IllegalArgumentException if {@code coordinates} are set but are not one of an element of the hydrated array
*/
Elements byElements(long... coordinates);

/**
* Creates an API to hydrate the targeted array with {@code Double} boxed type.
*
* Note that sticking to primitive types improve I/O performances overall, so only rely boxed types if the data is already
* available in that format.
*
* @return a hydrator supporting {@code Double} boxed type
*/
NdArrayHydrator<Double> boxed();
}
Loading