From 30bd7077e332784f3caab9e466109269d2da6f72 Mon Sep 17 00:00:00 2001
From: opfromthestart <opfromthestart@gmail.com>
Date: Wed, 27 Sep 2023 23:46:45 -0400
Subject: [PATCH] Make gradient on edges smaller

---
 src/tensor_ops/sigmoidr/cpu_kernel.rs | 2 +-
 src/tensor_ops/sigmoidr/mod.rs        | 2 +-
 src/tensor_ops/sigmoidr/sigmoidr.cu   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/tensor_ops/sigmoidr/cpu_kernel.rs b/src/tensor_ops/sigmoidr/cpu_kernel.rs
index b73d163f..a1de352a 100644
--- a/src/tensor_ops/sigmoidr/cpu_kernel.rs
+++ b/src/tensor_ops/sigmoidr/cpu_kernel.rs
@@ -10,6 +10,6 @@ impl<F: num_traits::Float> UnaryDerivative<F> for super::SigmoidrKernelOp {
     #[inline(always)]
     fn df(&self, &fx: &F) -> F {
         let d = fx * (F::one() - fx);
-        F::max(d, F::from(0.0001).unwrap())
+        F::max(d, F::from(0.0000001).unwrap())
     }
 }
diff --git a/src/tensor_ops/sigmoidr/mod.rs b/src/tensor_ops/sigmoidr/mod.rs
index 4a807f7b..146335e6 100644
--- a/src/tensor_ops/sigmoidr/mod.rs
+++ b/src/tensor_ops/sigmoidr/mod.rs
@@ -53,7 +53,7 @@ mod tests {
         let g = r.mean().backward();
         assert_close_to_literal!(
             g.get(&x),
-            [0.020998716, 0.039322387, 0.05, 0.039322387, 0.00002]
+            [0.020998716, 0.039322387, 0.05, 0.039322387, 0.00000002]
         );
     }
 }
diff --git a/src/tensor_ops/sigmoidr/sigmoidr.cu b/src/tensor_ops/sigmoidr/sigmoidr.cu
index 1adc4b66..abc8798d 100644
--- a/src/tensor_ops/sigmoidr/sigmoidr.cu
+++ b/src/tensor_ops/sigmoidr/sigmoidr.cu
@@ -12,7 +12,7 @@ template<typename T>
 __device__ __forceinline__ T sigmoidr_bwd(T y) {
     T one = 1.0;
   T d = y * (one - y);
-    return max(d, 0.0001);
+    return max(d, 0.0000001);
 }
 
 UNARY_OP(__half, sigmoidr_fwd_f16, sigmoidr_bwd_f16, SigmoidrKernelOp,