Add Activations (#123)

* Initial checkin of Keras Optimzers and helper classes.
Fixed dependencies in pom.xml

* Added static final NAME to replace hardcoded String in the create method. This allows the NAME to be used elsewhere instead of hardcoding the string.

* Changed of method to use the DataType NAME attribute rather than hardcoding the string.
added methods isFloating(), isInteger(), isNUmeric(), isBoolean() and isString()

* Added method WriteFieldWithInitializer to output a "final static String OP_NAME" to each generated operation.

* Added tf.nn.softmaxCrossEntropyWitLogits() and tf.nn.raw.softmaxCrossEntropyWitLogits()
Added tf.nn.sparesSoftmaxCrossEntropyWithLogits() and
tf.nn.raw.sparesSoftmaxCrossEntropyWithLogits()

Added tf.nn.sigmoidCrossEntropyWithLogits()

* Moved SoftmaxCrossEntropyWithLogits and  SparseSoftmaxCrossEntropyWithLogits to org.tensorflow.op.nn.raw

* Generated classes now have public static final String OP_NAME = "XXXXXXXX";

* Generated classes now have public static final String OP_NAME = "XXXXXXXX";

* fix dependencies for other Tensorflow Java modules

* formatting fix

* Fix ctors with name to properly pass the name to the the super ctor.

* change asserts to IllegalArgumentException
fix javadoc, fix casts

* change asserts to IllegalArgumentException

* Moved back to tests

* Moved SoftmaxCrossEntropyWithLogits.java and SparseSoftmaxCrossEntropyWithLogits.java to nn.raw,
added new versions of these to NnOps

* Deleted files that are not necessary yet

* Added nn.raw group for softmaxCrossEntropyWithLogits() and sparseSoftmaxCrossEntropyWithLogits()

* Added nn.raw group for softmaxCrossEntropyWithLogits() and sparseSoftmaxCrossEntropyWithLogits()

* Refactor NN into individual operations under org.tensorflow.op.nn. Fix JavaDoc. Change from snake case to camel case.

* Refactor NN into individual operations under org.tensorflow.op.nn. Fix JavaDoc. Change from snake case to camel case.

* Reformatted code

* Added sub scope

* Miscellaneous fixes based on review comments.

* Fixed op_generator.cc to remove a spurious new line in the generated Java files for some Ops. This also  resulted in new generated  source that are also committed.

* Changed back to non-generic Operand until we resolve how to handle generics.

* Regenerated due to creation of SoftmaxCrossEntropyWithLogits.java,  SigmoidCrossEntropyWithLogits.java, and SparseSoftmaxCrossEntropyWithLogits.java under package org.tensorflow.op.nn in

* change snake case to camel case. format code

* clean upd warning,  format code

* Added Adamax, Ftrl, and Nadam Optimizers. Added Optimizers enum for easy inclusion of a default optimizer. Cleaned up JavaDoc

* Removed optimize classes from tensorflow-keras, moved optimizer test cases to framework. Created Tests for GradientDescent and Momentum

* Fixed generics

* Fixed from Unit test results

* added @SuppressWarnings("unchecked") on Variable array

* Added Support for evaluating TFloat16

* Add Activations

* Remove no-arg CTORs

* Fix Unit Tests to include positive and negative numbers on input.

* Modify JavaDoc indicating Linear activation is also known as Identity activation

* Changed DEFAULT values from private to public

* Fixed last sum to be over 'e' instead of 'input'

* Added tests for various parameter constructs.

* added tests for 1D and 3D input

* Change snake case to camel case

* JavaDoc fixes

* Add TFloating family

* Add JavaDoc

* Changed to TFloating where appropriate.
Misc fixes to JavaDoc.
In ReLU, change to assign to new variable 'lInput' rather than change the 'input' parameter.

* Remove the test of int arguments for those classes changed to TFloating type, as they would no longer compile.

* Remove the test of int arguments for those classes changed to TFloating type, as they would no longer compile.

* Make LeakyRelu visible so that it is included in tf.nn.

* Remove TNumber import

* Add tf.nn.leakyRelu operation

Author: JimClarke5

tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_LeakyRelu.pbtxt

@@ -1,5 +1,6 @@
 op {
   graph_op_name: "LeakyRelu"
+  visibility: VISIBLE
   endpoint {
     name: "nn.LeakyRelu"
   }

tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/NnOps.java

@@ -59,6 +59,7 @@
 import org.tensorflow.op.nn.FusedResizeAndPadConv2d;
 import org.tensorflow.op.nn.InTopK;
 import org.tensorflow.op.nn.L2Loss;
+import org.tensorflow.op.nn.LeakyRelu;
 import org.tensorflow.op.nn.LearnedUnigramCandidateSampler;
 import org.tensorflow.op.nn.LocalResponseNormalization;
 import org.tensorflow.op.nn.LogSoftmax;
@@ -1226,6 +1227,19 @@ public <T extends TNumber> L2Loss<T> l2Loss(Operand<T> t) {
     return L2Loss.create(scope, t);
   }
 
+  /**
+   * Computes rectified linear: `max(features, features * alpha)`.
+   *
+   * @param <T> data type for {@code activations()} output
+   * @param features
+   * @param options carries optional attributes values
+   * @return a new instance of LeakyRelu
+   */
+  public <T extends TNumber> LeakyRelu<T> leakyRelu(Operand<T> features,
+      LeakyRelu.Options... options) {
+    return LeakyRelu.create(scope, features, options);
+  }
+
   /**
    * Generates labels for candidate sampling with a learned unigram distribution.
    *  <p>

tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/op/nn/LeakyRelu.java

@@ -33,6 +33,7 @@
  * 
  * @param <T> data type for {@code activations()} output
  */
+@Operator(group = "nn")
 public final class LeakyRelu<T extends TNumber> extends RawOp implements Operand<T> {
 
   /**

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/TBfloat16.java

@@ -30,7 +30,7 @@
 import org.tensorflow.ndarray.NdArray;
 import org.tensorflow.ndarray.StdArrays;
 import org.tensorflow.ndarray.impl.dense.FloatDenseNdArray;
-import org.tensorflow.types.family.TNumber;
+import org.tensorflow.types.family.TFloating;
 
 /**
  * Brain 16-bit float tensor type.
@@ -48,7 +48,7 @@
  * <p>Note that some CPUs support the bfloat16 format natively, which can result in faster
  * computation compared to {@link TFloat16} when GPUs are not used.
  */
-public interface TBfloat16 extends FloatNdArray, TNumber {
+public interface TBfloat16 extends FloatNdArray, TFloating {
   /** readable-name for the data type */
   static final String NAME = "BFLOAT16";
 

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/TFloat16.java

@@ -30,7 +30,7 @@
 import org.tensorflow.ndarray.NdArray;
 import org.tensorflow.ndarray.StdArrays;
 import org.tensorflow.ndarray.impl.dense.FloatDenseNdArray;
-import org.tensorflow.types.family.TNumber;
+import org.tensorflow.types.family.TFloating;
 
 /**
  * IEEE-754 half-precision 16-bit float tensor type.
@@ -45,7 +45,7 @@
  * most CPUs do not support this format natively. For CPU computation on 16-bit floats, the {@link
  * TBfloat16} tensor type might be a better option.
  */
-public interface TFloat16 extends FloatNdArray, TNumber {
+public interface TFloat16 extends FloatNdArray, TFloating {
 
   /** readable-name for the data type */
   static final String NAME = "FLOAT16";

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/TFloat32.java

@@ -29,10 +29,10 @@
 import org.tensorflow.ndarray.NdArray;
 import org.tensorflow.ndarray.StdArrays;
 import org.tensorflow.ndarray.impl.dense.FloatDenseNdArray;
-import org.tensorflow.types.family.TNumber;
+import org.tensorflow.types.family.TFloating;
 
 /** IEEE-754 single-precision 32-bit float tensor type. */
-public interface TFloat32 extends FloatNdArray, TNumber {
+public interface TFloat32 extends FloatNdArray, TFloating {
 
   /** readable-name for the data type */
   static final String NAME = "FLOAT";

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/TFloat64.java

@@ -29,10 +29,11 @@
 import org.tensorflow.ndarray.NdArray;
 import org.tensorflow.ndarray.StdArrays;
 import org.tensorflow.ndarray.impl.dense.DoubleDenseNdArray;
-import org.tensorflow.types.family.TNumber;
+import org.tensorflow.types.family.TFloating;
+
 
 /** IEEE-754 double-precision 64-bit float tensor type. */
-public interface TFloat64 extends DoubleNdArray, TNumber {
+public interface TFloat64 extends DoubleNdArray, TFloating {
 
   /** readable-name for the data type */
   static final String NAME = "DOUBLE";

tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/types/family/TFloating.java

@@ -0,0 +1,19 @@
+package org.tensorflow.types.family;
+
+/**
+ * Marker interface for floating point tensor types.
+ *
+ * <p>Operations that only accepts floating point values as some of their operands enforce that the tensor
+ * types for these operands to be bound to this interface. For example:
+ *
+ * <pre>{@code
+ * TFloat32 tensor1 = TFloat32.vectorOf(1, 2, 3);
+ * TBool tensor2 = TBool.vectorOf(true, false, true);
+ *
+ * Ops tf = Ops.create();
+ * Exponential<TFloat32> exp = new Exponential<>(tf);
+ * exp.call(tf.constant(tensor1));  // OK
+ * exp.call(tf.constant(tensor2));  // Compilation failure
+ * }</pre>
+ */
+public interface TFloating extends TNumber {}

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/Activation.java

@@ -0,0 +1,68 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.activations;
+
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.family.TNumber;
+
+/**
+ * Abstract base class for Activations
+ *
+ * <p><b>Note:</b> The {@link #tf} attribute must be set prior to invoking the call method. See
+ * {@link #setTF(Ops)} and the constructor {@link #Activation(Ops)}.
+ *
+ * @param <T> the data type of the activation
+ */
+public abstract class Activation<T extends TNumber> {
+
+  /** The TensorFlow Ops */
+  protected Ops tf;
+
+  /**
+   * Creates the abstract class for an Activation
+   *
+   * @param tf the TensorFlow Ops
+   */
+  protected Activation(Ops tf) {
+    this.tf = tf;
+  }
+
+  /**
+   * Sets the TensorFlow Ops
+   *
+   * @param tf the TensorFlow Ops
+   */
+  protected void setTF(Ops tf) {
+    this.tf = tf;
+  }
+
+  /**
+   * Gets the TensorFlow Ops
+   *
+   * @return the TensorFlow Ops
+   */
+  protected Ops getTF() {
+    return this.tf;
+  }
+
+  /**
+   * Gets the calculation operation for the activation.
+   *
+   * @param input the input tensor
+   * @return The operand for the activation
+   */
+  public abstract Operand<T> call(Operand<T> input);
+}

tensorflow-framework/src/main/java/org/tensorflow/framework/activations/ELU.java

@@ -0,0 +1,98 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.activations;
+
+import org.tensorflow.DataType;
+import org.tensorflow.Operand;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.TBool;
+import org.tensorflow.types.family.TFloating;
+
+/**
+ * Exponential linear unit.
+ *
+ * <p>The exponential linear unit (ELU) with <code>alpha &gt; 0</code> is:
+ *
+ * <p><code>x</code> if <code>x &gt; 0</code> and <code>alpha * (exp(x) -
+ * 1)</code> if <code>x &lt; 0</code>.
+ *
+ * <p>The ELU hyperparameter <code>alpha</code> controls the value to which an ELU saturates for
+ * negative net inputs. ELUs diminish the vanishing gradient effect.
+ *
+ * <p>ELUs have negative values which pushes the mean of the activations closer to zero. Mean
+ * activations that are closer to zero enable faster learning as they bring the gradient closer to
+ * the natural gradient. ELUs saturate to a negative value when the argument gets smaller.
+ * Saturation means a small derivative which decreases the variation and the information that is
+ * propagated to the next layer.
+ *
+ * <p>Example Usage:
+ *
+ * <pre>
+ *     Operand&lt;TFloat32&gt; input = &#46;&#46;&#46;;
+ *     ELU&lt;TFloat32&gt; elu = new ELU&lt;&gt;(tf, 2.0f);
+ *     Operand&lt;TFloat32&gt; result = elu.call(input);
+ * </pre>
+ *
+ * @param <T> the data type of the activation
+ * @see <a href="https://arxiv.org/abs/1511.07289">Clevert et al, 2016, Fast and Accurate Deep
+ *     Network Learning by Exponential Linear Units (ELUs)</a>
+ */
+public class ELU<T extends TFloating> extends Activation<T> {
+
+  private static final double ALPHA_DEFAULT = 1.0;
+
+  /** A scalar, slope of negative section. */
+  private final double alpha;
+
+  /**
+   * Creates a new ELU with alpha={@link #ALPHA_DEFAULT}.
+   *
+   * @param tf the TensorFlow Ops
+   */
+  public ELU(Ops tf) {
+    this(tf, ALPHA_DEFAULT);
+  }
+
+  /**
+   * Creates a new ELU
+   *
+   * @param tf the TensorFlow Ops
+   * @param alpha A scalar, slope of negative section. It controls the value to which an ELU
+   *     saturates for negative net inputs.
+   */
+  public ELU(Ops tf, double alpha) {
+    super(tf);
+    this.alpha = alpha;
+  }
+
+  /**
+   * Gets the calculation operation for the activation.
+   *
+   * @param input the input tensor
+   * @return The operand for the activation
+   */
+  @Override
+  public Operand<T> call(Operand<T> input) {
+
+    Operand<T> result = tf.nn.elu(input);
+    if (alpha == 1.0) return result;
+    else {
+      DataType<T> dataType = input.asOutput().dataType();
+      Operand<T> y = tf.math.mul(result, tf.dtypes.cast(tf.constant(alpha), dataType));
+      Operand<TBool> cond = tf.math.greater(result, tf.dtypes.cast(tf.constant(0), dataType));
+      return tf.select(cond, result, y);
+    }
+  }
+}