From c9d7b63e2d4866ffa8fbf25edc8cc37b01863c32 Mon Sep 17 00:00:00 2001 From: Brad Larson Date: Fri, 6 Dec 2019 17:41:38 -0600 Subject: [PATCH 1/2] Correcting calculation of Glorot uniform for Tensors. --- Sources/TensorFlow/Initializers.swift | 2 +- Tests/TensorFlowTests/InitializerTests.swift | 6 ++--- Tests/TensorFlowTests/LayerTests.swift | 24 ++++++++++---------- Tests/TensorFlowTests/SequentialTests.swift | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Sources/TensorFlow/Initializers.swift b/Sources/TensorFlow/Initializers.swift index f25546241..d6b3960ae 100644 --- a/Sources/TensorFlow/Initializers.swift +++ b/Sources/TensorFlow/Initializers.swift @@ -495,7 +495,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { /// - shape: The dimensions of the tensor. init(glorotUniform shape: TensorShape, seed: TensorFlowSeed = Context.local.randomSeed) { let (fanIn, fanOut) = shape.fans() - let limit = Tensor(6 / Scalar(fanIn + fanOut)) + let limit = Tensor(Scalar.sqrt(6 / Scalar(fanIn + fanOut))) self.init(randomUniform: shape, lowerBound: -limit, upperBound: limit, seed: seed) } diff --git a/Tests/TensorFlowTests/InitializerTests.swift b/Tests/TensorFlowTests/InitializerTests.swift index 12a74cb17..9113be368 100644 --- a/Tests/TensorFlowTests/InitializerTests.swift +++ b/Tests/TensorFlowTests/InitializerTests.swift @@ -97,14 +97,14 @@ final class InitializerTests: XCTestCase { // Constants for testing distribution based initializers. private let fcShape = TensorShape([200, 100]) private let convShape = TensorShape([25, 25, 20, 20]) - private let tolerance = Float(3e-2) func testDistribution( _ t: Tensor, expectedMean: Float? = nil, expectedStandardDeviation: Float? = nil, expectedMin: Float? = nil, - expectedMax: Float? = nil + expectedMax: Float? = nil, + tolerance: Float = 3e-2 ) { if let expectedMean = expectedMean { let mean = t.mean().scalarized() @@ -159,7 +159,7 @@ final class InitializerTests: XCTestCase { let spatialSize = convShape[0..<2].contiguousSize let (fanIn, fanOut) = (convShape[2] * spatialSize, convShape[3] * spatialSize) let stdDev = sqrt(Float(2.0) / Float(fanIn + fanOut)) - testDistribution(t, expectedMean: 0, expectedStandardDeviation: stdDev) + testDistribution(t, expectedMean: 0, expectedStandardDeviation: stdDev, tolerance: 1e-4) } func testGlorotNormal() { diff --git a/Tests/TensorFlowTests/LayerTests.swift b/Tests/TensorFlowTests/LayerTests.swift index 5c759e565..78fdce42c 100644 --- a/Tests/TensorFlowTests/LayerTests.swift +++ b/Tests/TensorFlowTests/LayerTests.swift @@ -1147,10 +1147,10 @@ final class LayerTests: XCTestCase { } assertEqual( outputs.map { $0.value.squeezingShape(at: 0) }[0], - [[ 0.14798240, 0.14295710, -0.09766942, -0.131820890], - [ 0.15757358, 0.19475500, -0.12810913, -0.112212844], - [ 0.16168950, 0.20306197, -0.13058113, -0.123917180], - [ 0.16325668, 0.20822097, -0.13273866, -0.121018395]], + [[ 0.20775771, 0.20080023, -0.13768704, -0.18534681], + [ 0.22666009, 0.30019346, -0.19720285, -0.14683801], + [ 0.23758979, 0.32101023, -0.20359215, -0.1787096], + [ 0.24337786, 0.3389194, -0.21143384, -0.1675081]], accuracy: 1e-6) } // TODO: Figure out why the following is numerically unstable. @@ -1178,17 +1178,17 @@ final class LayerTests: XCTestCase { } assertEqual( outputs.map { $0.cell.squeezingShape(at: 0) }[0], - [[ 0.047114454, 0.013959665, -0.030737250, -0.038524970], - [ 0.069171116, 0.020617897, -0.044740470, -0.058878290], - [ 0.079530790, 0.023841830, -0.051080680, -0.069567055], - [ 0.084416830, 0.025424266, -0.053918116, -0.075140170]], + [[ 0.08981595, 0.027691621, -0.059235442, -0.075101905], + [ 0.12952757, 0.040402323, -0.084273980, -0.116252676], + [ 0.14727503, 0.046511370, -0.094689950, -0.138459030], + [ 0.15532997, 0.049573865, -0.098824400, -0.150242210]], accuracy: 1e-6) assertEqual( outputs.map { $0.hidden.squeezingShape(at: 0) }[0], - [[ 0.024117637, 0.0066833394, -0.015753632, -0.019533360], - [ 0.035230752, 0.0098582430, -0.022934474, -0.029750597], - [ 0.040405065, 0.0113919870, -0.026185552, -0.035087958], - [ 0.042834233, 0.0121438510, -0.027640648, -0.037863784]], + [[ 0.046985064, 0.012670102, -0.031083463, -0.038572006], + [ 0.066482050, 0.018388016, -0.044252350, -0.058907583], + [ 0.074910110, 0.021107012, -0.049724963, -0.069670826], + [ 0.078670055, 0.022462710, -0.051899005, -0.075331904]], accuracy: 1e-6) } } diff --git a/Tests/TensorFlowTests/SequentialTests.swift b/Tests/TensorFlowTests/SequentialTests.swift index 15d437179..587e4ed4c 100644 --- a/Tests/TensorFlowTests/SequentialTests.swift +++ b/Tests/TensorFlowTests/SequentialTests.swift @@ -63,7 +63,7 @@ final class SequentialTests: XCTestCase { } } XCTAssertEqual(model.inferring(from: [[0, 0], [0, 1], [1, 0], [1, 1]]), - [[0.5567076], [0.5567076], [0.5567076], [0.5567076]]) + [[0.50182396], [0.50182396], [0.50182396], [0.50182396]]) } static var allTests = [ From d40644c04f1563eee87ac2fa0e7ada00a5b879b9 Mon Sep 17 00:00:00 2001 From: Brad Larson Date: Sat, 7 Dec 2019 12:25:12 -0600 Subject: [PATCH 2/2] Restored the GRU test values to what they were before they were adjusted for the new uniform. --- Tests/TensorFlowTests/LayerTests.swift | 12 +++++++----- Tests/TensorFlowTests/SequentialTests.swift | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Tests/TensorFlowTests/LayerTests.swift b/Tests/TensorFlowTests/LayerTests.swift index 78fdce42c..cf2fd0f94 100644 --- a/Tests/TensorFlowTests/LayerTests.swift +++ b/Tests/TensorFlowTests/LayerTests.swift @@ -1207,11 +1207,13 @@ final class LayerTests: XCTestCase { let (outputs, _) = rnn.valueWithPullback(at: inputs) { rnn, inputs in return rnn(inputs) } - XCTAssertEqual(outputs.map { $0.hidden }, - [[[0.12806869, 0.12806869, 0.12806869, 0.12806869]], - [[0.2007559, 0.2007559, 0.2007559, 0.2007559]], - [[0.23432666, 0.23432666, 0.23432666, 0.23432666]], - [[0.24788898, 0.24788898, 0.24788898, 0.24788898]]]) + assertEqual( + outputs.map { $0.hidden }[0], + [[0.1193780, 0.1193780, 0.1193780, 0.1193780], + [0.1887644, 0.1887644, 0.1887644, 0.1887644], + [0.2230835, 0.2230835, 0.2230835, 0.2230835], + [0.2383619, 0.2383619, 0.2383619, 0.2383619]], + accuracy: 1e-5) } } diff --git a/Tests/TensorFlowTests/SequentialTests.swift b/Tests/TensorFlowTests/SequentialTests.swift index 587e4ed4c..bf2dd6547 100644 --- a/Tests/TensorFlowTests/SequentialTests.swift +++ b/Tests/TensorFlowTests/SequentialTests.swift @@ -63,7 +63,7 @@ final class SequentialTests: XCTestCase { } } XCTAssertEqual(model.inferring(from: [[0, 0], [0, 1], [1, 0], [1, 1]]), - [[0.50182396], [0.50182396], [0.50182396], [0.50182396]]) + [[0.50378805], [0.50378805], [0.50378805], [0.50378805]]) } static var allTests = [