Merged PR 1138: Add activation functions: Elu, LeakyRelu, ThresholdedRelu

Add activation functions: Elu, LeakyRelu, ThresholdedRelu

Related work items: #30

Author: Ke Deng

lotus/core/framework/kernel_def_builder.h

@@ -95,6 +95,7 @@ class KernelDef {
     return *this;
   }
 
+  // allowing output j to reuse memory of input i
   KernelDef& MayInplace(int i, int j) {
     // TODO: validate inputs.
     inplace_map_.push_back({i, j});

lotus/core/providers/cpu/activation/activations.cc

@@ -0,0 +1,21 @@
+#include "core/providers/cpu/activation/activations.h"
+
+namespace Lotus {
+
+#define REGISTER_UNARY_ELEMENTWISE_KERNEL(x)                                      \
+  REGISTER_KERNEL(KernelDef(#x)                                                   \
+                      .Domain(LotusIR::kOnnxDomain)                               \
+                      .SinceVersion(1, 2)                                         \
+                      .Provider(LotusIR::kCpuExecutionProvider)                   \
+                      .MayInplace(0, 0)                                           \
+                      .TypeConstraint("T", DataTypeImpl::GetTensorType<float>()), \
+                  x<float>)
+
+REGISTER_UNARY_ELEMENTWISE_KERNEL(Elu);
+REGISTER_UNARY_ELEMENTWISE_KERNEL(LeakyRelu);
+REGISTER_UNARY_ELEMENTWISE_KERNEL(Relu);
+REGISTER_UNARY_ELEMENTWISE_KERNEL(Sigmoid);
+REGISTER_UNARY_ELEMENTWISE_KERNEL(Tanh);
+REGISTER_UNARY_ELEMENTWISE_KERNEL(ThresholdedRelu);
+
+}  // namespace Lotus

lotus/core/providers/cpu/activation/activations.h

@@ -0,0 +1,47 @@
+#ifndef CORE_PROVIDERS_CPU_ACTIVATION_RELU_H
+#define CORE_PROVIDERS_CPU_ACTIVATION_RELU_H
+
+#include "core/common/common.h"
+#include "core/framework/op_kernel.h"
+#include "core/util/math_cpuonly.h"
+
+namespace Lotus {
+
+DECLARE_EIGEN_UNARY_ELEMENTWISE_KERNEL(Elu,
+                                       {
+                                         EIGEN_X_VAR(xm);
+                                         EIGEN_Y = (xm >= 0).select(xm, Attr("alpha") * (xm.exp() - 1));
+                                       },
+                                       {"alpha"})
+
+DECLARE_EIGEN_UNARY_ELEMENTWISE_KERNEL(LeakyRelu,
+                                       {
+                                         EIGEN_X_VAR(xm);
+                                         EIGEN_Y = (xm >= 0).select(xm, Attr("alpha") * xm);
+                                       },
+                                       {"alpha"})
+
+DECLARE_EIGEN_UNARY_ELEMENTWISE_KERNEL(Relu,
+                                       { EIGEN_Y = EIGEN_X.cwiseMax(0); },
+                                       {})
+
+DECLARE_EIGEN_UNARY_ELEMENTWISE_KERNEL(Sigmoid,
+                                       {
+                                         EIGEN_X_VAR(xm);
+                                         EIGEN_Y_VAR(ym);
+                                         ym = (xm >= 0).select(1 / (1. + (-xm.abs()).exp()), 1 - 1 / (1. + (-xm.abs()).exp()));
+                                       },
+                                       {})
+
+DECLARE_EIGEN_UNARY_ELEMENTWISE_KERNEL(Tanh, { EIGEN_Y = EIGEN_X.tanh(); }, {})
+
+DECLARE_EIGEN_UNARY_ELEMENTWISE_KERNEL(ThresholdedRelu,
+                                       {
+                                         EIGEN_X_VAR(xm);
+                                         EIGEN_Y = (xm >= Attr("alpha")).select(xm, 0);
+                                       },
+                                       {"alpha"})
+
+}  // namespace Lotus
+
+#endif  // !CORE_PROVIDERS_CPU_ACTIVATION_RELU_H

lotus/core/providers/cpu/activation/relu.cc

@@ -30,6 +30,7 @@ REGISTER_KERNEL(KernelDef("Clip")
                     .Domain(LotusIR::kOnnxDomain)
                     .SinceVersion(1, 2)
                     .Provider(LotusIR::kCpuExecutionProvider)
+                    .MayInplace(0, 0)
                     .TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
                 Clip<float>);
 }  // namespace Lotus