Correcting calculation of Glorot uniform for Tensors (#576)

* Correcting calculation of Glorot uniform for Tensors.

* Restored the GRU test values to what they were before they were adjusted for the new uniform.

Author: BradLarson

Sources/TensorFlow/Initializers.swift

@@ -495,7 +495,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint {
     ///   - shape: The dimensions of the tensor.
     init(glorotUniform shape: TensorShape, seed: TensorFlowSeed = Context.local.randomSeed) {
         let (fanIn, fanOut) = shape.fans()
-        let limit = Tensor<Scalar>(6 / Scalar(fanIn + fanOut))
+        let limit = Tensor<Scalar>(Scalar.sqrt(6 / Scalar(fanIn + fanOut)))
         self.init(randomUniform: shape, lowerBound: -limit, upperBound: limit, seed: seed)
     }
 

Tests/TensorFlowTests/InitializerTests.swift

@@ -97,14 +97,14 @@ final class InitializerTests: XCTestCase {
     // Constants for testing distribution based initializers.
     private let fcShape = TensorShape([200, 100])
     private let convShape = TensorShape([25, 25, 20, 20])
-    private let tolerance = Float(3e-2)
 
     func testDistribution(
         _ t: Tensor<Float>,
         expectedMean: Float? = nil,
         expectedStandardDeviation: Float? = nil,
         expectedMin: Float? = nil,
-        expectedMax: Float? = nil
+        expectedMax: Float? = nil,
+        tolerance: Float = 3e-2
     ) {
         if let expectedMean = expectedMean {
             let mean = t.mean().scalarized()
@@ -159,7 +159,7 @@ final class InitializerTests: XCTestCase {
         let spatialSize = convShape[0..<2].contiguousSize
         let (fanIn, fanOut) = (convShape[2] * spatialSize, convShape[3] * spatialSize)
         let stdDev = sqrt(Float(2.0) / Float(fanIn + fanOut))
-        testDistribution(t, expectedMean: 0, expectedStandardDeviation: stdDev)
+        testDistribution(t, expectedMean: 0, expectedStandardDeviation: stdDev, tolerance: 1e-4)
     }
 
     func testGlorotNormal() {

Tests/TensorFlowTests/LayerTests.swift

@@ -1147,10 +1147,10 @@ final class LayerTests: XCTestCase {
             }
             assertEqual(
                 outputs.map { $0.value.squeezingShape(at: 0) }[0],
-                [[ 0.14798240, 0.14295710, -0.09766942, -0.131820890],
-                 [ 0.15757358, 0.19475500, -0.12810913, -0.112212844],
-                 [ 0.16168950, 0.20306197, -0.13058113, -0.123917180],
-                 [ 0.16325668, 0.20822097, -0.13273866, -0.121018395]],
+                [[ 0.20775771,  0.20080023, -0.13768704, -0.18534681],
+                 [ 0.22666009,  0.30019346, -0.19720285, -0.14683801],
+                 [ 0.23758979,  0.32101023, -0.20359215,  -0.1787096],
+                 [ 0.24337786,   0.3389194, -0.21143384,  -0.1675081]],
                 accuracy: 1e-6)
         }
         // TODO: Figure out why the following is numerically unstable.
@@ -1178,17 +1178,17 @@ final class LayerTests: XCTestCase {
                 }
                 assertEqual(
                     outputs.map { $0.cell.squeezingShape(at: 0) }[0],
-                    [[ 0.047114454, 0.013959665, -0.030737250, -0.038524970],
-                     [ 0.069171116, 0.020617897, -0.044740470, -0.058878290],
-                     [ 0.079530790, 0.023841830, -0.051080680, -0.069567055],
-                     [ 0.084416830, 0.025424266, -0.053918116, -0.075140170]],
+                    [[ 0.08981595, 0.027691621, -0.059235442, -0.075101905],
+                     [ 0.12952757, 0.040402323, -0.084273980, -0.116252676],
+                     [ 0.14727503, 0.046511370, -0.094689950, -0.138459030],
+                     [ 0.15532997, 0.049573865, -0.098824400, -0.150242210]],
                     accuracy: 1e-6)
                 assertEqual(
                     outputs.map { $0.hidden.squeezingShape(at: 0) }[0],
-                    [[ 0.024117637, 0.0066833394, -0.015753632, -0.019533360],
-                     [ 0.035230752, 0.0098582430, -0.022934474, -0.029750597],
-                     [ 0.040405065, 0.0113919870, -0.026185552, -0.035087958],
-                     [ 0.042834233, 0.0121438510, -0.027640648, -0.037863784]],
+                    [[ 0.046985064, 0.012670102, -0.031083463, -0.038572006],
+                     [ 0.066482050, 0.018388016, -0.044252350, -0.058907583],
+                     [ 0.074910110, 0.021107012, -0.049724963, -0.069670826],
+                     [ 0.078670055, 0.022462710, -0.051899005, -0.075331904]],
                     accuracy: 1e-6)
             }
         }
@@ -1207,11 +1207,13 @@ final class LayerTests: XCTestCase {
             let (outputs, _) = valueWithPullback(at: rnn, inputs) { rnn, inputs in
                 return rnn(inputs)
             }
-            XCTAssertEqual(outputs.map { $0.hidden },
-                           [[[0.12806869, 0.12806869, 0.12806869, 0.12806869]],
-                             [[0.2007559, 0.2007559, 0.2007559, 0.2007559]],
-                             [[0.23432666, 0.23432666, 0.23432666, 0.23432666]],
-                             [[0.24788898, 0.24788898, 0.24788898, 0.24788898]]])
+            assertEqual(
+                outputs.map { $0.hidden }[0],
+                [[0.1193780, 0.1193780, 0.1193780, 0.1193780],
+                 [0.1887644, 0.1887644, 0.1887644, 0.1887644],
+                 [0.2230835, 0.2230835, 0.2230835, 0.2230835],
+                 [0.2383619, 0.2383619, 0.2383619, 0.2383619]],
+                accuracy: 1e-5)
         }
     }
 

Tests/TensorFlowTests/SequentialTests.swift

@@ -63,7 +63,7 @@ final class SequentialTests: XCTestCase {
             }
         }
         XCTAssertEqual(model.inferring(from: [[0, 0], [0, 1], [1, 0], [1, 1]]),
-                       [[0.5567076], [0.5567076], [0.5567076], [0.5567076]])
+                       [[0.50378805], [0.50378805], [0.50378805], [0.50378805]])
     }
 
     static var allTests = [