add LeakyReLU activation

docs/src/lib/Architecture.md

@@ -56,6 +56,7 @@ Id
 ReLU
 Sigmoid
 Tanh
+LeakyReLU
 ```
 
 The following strings can be parsed as activation functions:

src/Architecture/ActivationFunction.jl

@@ -57,6 +57,27 @@ struct Tanh <: ActivationFunction end
 
 (::Tanh)(x) = tanh.(x)
 
+"""
+    LeakyReLU{N<:Number}
+
+Leaky ReLU activation.
+
+```math
+    fₐ(x) = x > 0 ? x : a x
+```
+where ``a`` is the parameter.
+
+### Fields
+
+- `slope` -- parameter for negative inputs
+"""
+struct LeakyReLU{N<:Number} <: ActivationFunction
+    slope::N
+end
+
+(lr::LeakyReLU)(x::Number) = x >= zero(x) ? x : lr.slope * x
+(lr::LeakyReLU)(x::AbstractVector) = lr.(x)
+
 # constant instances of each activation function
 const _id = Id()
 const _relu = ReLU()

src/Architecture/Architecture.jl

@@ -10,7 +10,7 @@ using Requires
 export AbstractNeuralNetwork, AbstractLayerOp,
        FeedforwardNetwork, DenseLayerOp,
        dim_in, dim_out, dim,
-       ActivationFunction, Id, ReLU, Sigmoid, Tanh
+       ActivationFunction, Id, ReLU, Sigmoid, Tanh, LeakyReLU
 
 include("AbstractNeuralNetwork.jl")
 include("AbstractLayerOp.jl")

test/Architecture/DenseLayerOp.jl

@@ -47,11 +47,20 @@ function test_layer(L::DenseLayerOp{Tanh})
     @test L(x) ≈ [0.986, 0, -0.998] atol = 1e-3
 end
 
+function test_layer(L::DenseLayerOp{<:LeakyReLU})
+    @test L(x) == [2.5, 0, -0.035]
+end
+
 function test_layer(L::DenseLayerOp)
     return error("untested activation function: ", typeof(L.activation))
 end
 
 # run test with all activations
 for act in subtypes(ActivationFunction)
-    test_layer(DenseLayerOp(W, b, act()))
+    if act == LeakyReLU
+        act_inst = LeakyReLU(0.01)
+    else
+        act_inst = act()
+    end
+    test_layer(DenseLayerOp(W, b, act_inst))
 end