Merge pull request #3 from FluxML/dg/adam

add ADAM

src/rules.jl

@@ -22,3 +22,26 @@ end
 function (o::Descent)(m, m̄, st)
   update(o, m, m̄, st)
 end
+
+struct ADAM{T,K}
+  eta::T
+  beta::Tuple{K,K}
+end
+
+const ϵ = 1f-8
+
+function (o::ADAM)(m, m̄, state)
+  update(o, m, m̄, state)
+end
+
+init(o::ADAM, x::AbstractArray) = (zero(x), zero(x), o.beta)
+init(o::ADAM, x) = nothing
+
+function apply(o::ADAM, x, Δ, st)
+  η, β = o.eta, o.beta
+  mt, vt, βp = st
+  mt = β[1] .* mt .+ (1f0 .- β[1]) .* Δ
+  vt = β[2] .* vt .+ (1f0 .- β[2]) .* Δ .^ 2
+  Δ =  mt ./ (1 .- βp[1]) ./ (.√(vt ./ (1f0 .- βp[2])) .+ ϵ) .* η
+  return Δ, (mt, vt, βp .* β)
+end