diff --git a/newton-4.00/applications/ndSandbox/demos/ndQuadrupedTest_1.cpp b/newton-4.00/applications/ndSandbox/demos/ndQuadrupedTest_1.cpp index 35ba80f503..0e10e24b21 100644 --- a/newton-4.00/applications/ndSandbox/demos/ndQuadrupedTest_1.cpp +++ b/newton-4.00/applications/ndSandbox/demos/ndQuadrupedTest_1.cpp @@ -22,7 +22,7 @@ namespace ndQuadruped_1 { - //#define ND_TRAIN_MODEL + #define ND_TRAIN_MODEL #define CONTROLLER_NAME "ndQuadruped_1VPG.dnn" diff --git a/newton-4.00/applications/ndSandbox/toolbox/ndTestDeepBrain.cpp b/newton-4.00/applications/ndSandbox/toolbox/ndTestDeepBrain.cpp index 22e0bd4089..ef6c1dc830 100644 --- a/newton-4.00/applications/ndSandbox/toolbox/ndTestDeepBrain.cpp +++ b/newton-4.00/applications/ndSandbox/toolbox/ndTestDeepBrain.cpp @@ -12,12 +12,15 @@ #include "ndSandboxStdafx.h" #include "ndTestDeepBrain.h" +#define D_USE_CONVOLUTIONAL_LAYERS + static void ThreeLayersTwoInputsTwoOutputs() { ndBrain brain; ndInt32 hiddenNeurons = 16; ndFixSizeArray layers; + layers.PushBack(new ndBrainLayerLinear(2, hiddenNeurons)); layers.PushBack(new ndBrainLayerTanhActivation(hiddenNeurons)); @@ -52,14 +55,15 @@ static void ThreeLayersTwoInputsTwoOutputs() ndArray trainers; ndBrainThreadPool threads; - threads.SetThreadCount(4); + //threads.SetThreadCount(4); + threads.SetThreadCount(1); for (ndInt32 i = 0; i < bashSize; ++i) { trainers.PushBack(new ndBrainTrainer(&brain)); } - //ndBrainOptimizerSgd optimizer; ndBrainOptimizerAdam optimizer; + //ndBrainOptimizerSgd optimizer; ndInt32 randomeSelection[bashSize]; auto UpdateTrainer = ndMakeObject::ndFunction([&trainers, &randomeSelection, &inputBatch, &groundTruth, bashSize](ndInt32 threadIndex, ndInt32 threadCount) @@ -84,7 +88,7 @@ static void ThreeLayersTwoInputsTwoOutputs() randomeSelection[j] = ndInt32 (ndRandInt() % samples); } threads.ParallelExecute(UpdateTrainer); - optimizer.Update(&threads, trainers, ndReal(1.0e-3f)); + optimizer.Update(&threads, trainers, ndBrainFloat(1.0e-3f)); } ndBrainVector truth; @@ -276,7 +280,7 @@ static void MnistTrainingSet() ,m_bashBufferSize(64) { ndInt32 threadCount = ndMin(ndBrainThreadPool::GetMaxThreads(), ndMin(m_bashBufferSize, 16)); - //threadCount = 1; + threadCount = 1; SetThreadCount(threadCount); for (ndInt32 i = 0; i < m_bashBufferSize; ++i) { @@ -471,7 +475,6 @@ static void MnistTrainingSet() bool traningTest = fails < minTrainingFail; - //minTrainingFail = fails; minTrainingFail = ndMax(fails, 5); ndInt32 actualTraining = fails; ndBrainThreadPool::ParallelExecute(CrossValidateTest); @@ -517,18 +520,29 @@ static void MnistTrainingSet() ndInt32 neuronsPerLayers = 64; ndFixSizeArray layers; - layers.PushBack(new ndBrainLayerLinear(trainingDigits->GetColumns(), neuronsPerLayers)); - layers.PushBack(new ndBrainLayerApproximateTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize())); - - layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), neuronsPerLayers)); - layers.PushBack(new ndBrainLayerApproximateTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize())); +#ifdef D_USE_CONVOLUTIONAL_LAYERS + //layers.PushBack(new ndBrainLayerLinear(trainingDigits->GetColumns(), neuronsPerLayers)); + ndInt32 width = trainingDigits->GetColumns() / 28; + ndInt32 height = 28; + ndAssert((height * width) == trainingDigits->GetColumns()); + layers.PushBack(new ndBrainLayerConvolutional(28, width, height, 1, 5, 16)); + //layers.PushBack(new ndBrainLayerConvolutional(28, 3, 3, 3, 2, 2)); - layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), neuronsPerLayers)); + //layers.PushBack(new ndBrainLayerApproximateTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize())); +#else + layers.PushBack(new ndBrainLayerLinear(trainingDigits->GetColumns(), neuronsPerLayers)); layers.PushBack(new ndBrainLayerApproximateTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize())); - - layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), trainingLabels->GetColumns())); - //layers.PushBack(new ndBrainLayerSoftmaxActivation(layers[layers.GetCount() - 1]->GetOutputSize())); - layers.PushBack(new ndBrainLayerCategoricalSoftmaxActivation(layers[layers.GetCount() - 1]->GetOutputSize())); +#endif + + //layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), neuronsPerLayers)); + //layers.PushBack(new ndBrainLayerApproximateTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize())); + // + //layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), neuronsPerLayers)); + //layers.PushBack(new ndBrainLayerApproximateTanhActivation(layers[layers.GetCount() - 1]->GetOutputSize())); + // + //layers.PushBack(new ndBrainLayerLinear(layers[layers.GetCount() - 1]->GetOutputSize(), trainingLabels->GetColumns())); + ////layers.PushBack(new ndBrainLayerSoftmaxActivation(layers[layers.GetCount() - 1]->GetOutputSize())); + //layers.PushBack(new ndBrainLayerCategoricalSoftmaxActivation(layers[layers.GetCount() - 1]->GetOutputSize())); for (ndInt32 i = 0; i < layers.GetCount(); ++i) { diff --git a/newton-4.00/sdk/dBrain/ndBrain.cpp b/newton-4.00/sdk/dBrain/ndBrain.cpp index 3f0cde65b2..db1a421165 100644 --- a/newton-4.00/sdk/dBrain/ndBrain.cpp +++ b/newton-4.00/sdk/dBrain/ndBrain.cpp @@ -66,7 +66,7 @@ void ndBrain::CopyFrom(const ndBrain& src) const ndArray& srcLayers = src; for (ndInt32 i = 0; i < layers.GetCount(); ++i) { - layers[i]->CopyFrom(*srcLayers[i]); + layers[i]->Set(*srcLayers[i]); } } diff --git a/newton-4.00/sdk/dBrain/ndBrainLayer.cpp b/newton-4.00/sdk/dBrain/ndBrainLayer.cpp index 96b92aa962..ec4f4aa8ac 100644 --- a/newton-4.00/sdk/dBrain/ndBrainLayer.cpp +++ b/newton-4.00/sdk/dBrain/ndBrainLayer.cpp @@ -61,11 +61,6 @@ ndInt32 ndBrainLayer::GetOutputSize() const return 0; } -void ndBrainLayer::CopyFrom(const ndBrainLayer&) -{ - ndAssert(0); -} - bool ndBrainLayer::HasParameters() const { ndAssert(0); @@ -82,16 +77,51 @@ void ndBrainLayer::InitWeightsXavierMethod() ndAssert(0); } -ndBrainVector* ndBrainLayer::GetBias() +//ndBrainVector* ndBrainLayer::GetBias() +//{ +// ndAssert(0); +// return nullptr; +//} +// +//ndBrainMatrix* ndBrainLayer::GetWeights() +//{ +// ndAssert(0); +// return nullptr; +//} + +void ndBrainLayer::Clear() { ndAssert(0); - return nullptr; } -ndBrainMatrix* ndBrainLayer::GetWeights() +void ndBrainLayer::FlushToZero() +{ + ndAssert(0); +} + +void ndBrainLayer::Scale(ndBrainFloat) +{ + ndAssert(0); +} + +void ndBrainLayer::Set(const ndBrainLayer&) +{ + ndAssert(0); +} + +void ndBrainLayer::Add(const ndBrainLayer&) +{ + ndAssert(0); +} + +void ndBrainLayer::Mul(const ndBrainLayer&) +{ + ndAssert(0); +} + +void ndBrainLayer::ScaleAdd(const ndBrainLayer&, ndBrainFloat) { ndAssert(0); - return nullptr; } void ndBrainLayer::Blend(const ndBrainLayer&, ndBrainFloat) @@ -99,6 +129,11 @@ void ndBrainLayer::Blend(const ndBrainLayer&, ndBrainFloat) ndAssert(0); } +void ndBrainLayer::AdamUpdate(const ndBrainLayer&, const ndBrainLayer&, ndBrainFloat) +{ + ndAssert(0); +} + void ndBrainLayer::Save(const ndBrainSave* const) const { ndAssert(0); @@ -118,3 +153,9 @@ void ndBrainLayer::CalculateParamGradients(const ndBrainVector&, const ndBrainVe { ndAssert(0); } + +void ndBrainLayer::CalculateParamGradients(const ndBrainVector&, const ndBrainVector&, const ndBrainVector&, ndBrainVector&, ndBrainLayer* const) const +{ + ndAssert(0); +} + diff --git a/newton-4.00/sdk/dBrain/ndBrainLayer.h b/newton-4.00/sdk/dBrain/ndBrainLayer.h index 416ab7b7c7..a8fc35abb9 100644 --- a/newton-4.00/sdk/dBrain/ndBrainLayer.h +++ b/newton-4.00/sdk/dBrain/ndBrainLayer.h @@ -43,21 +43,36 @@ class ndBrainLayer : public ndClassAlloc virtual ndInt32 GetInputSize() const; virtual ndInt32 GetOutputSize() const; - virtual void CopyFrom(const ndBrainLayer& src); + virtual void Blend(const ndBrainLayer& src, ndBrainFloat blend); - virtual ndBrainVector* GetBias(); - virtual ndBrainMatrix* GetWeights(); + //virtual ndBrainVector* GetBias(); + //virtual ndBrainMatrix* GetWeights(); + virtual void Clear(); + virtual void FlushToZero(); + virtual void Scale(ndBrainFloat scale); + virtual void Set(const ndBrainLayer& src); + virtual void Add(const ndBrainLayer& src); + virtual void Mul(const ndBrainLayer& src); + virtual void ScaleAdd(const ndBrainLayer& src, ndBrainFloat scale); + virtual void InitWeightsXavierMethod(); virtual void InitWeights(ndBrainFloat weighVariance, ndBrainFloat biasVariance); virtual void MakePrediction(const ndBrainVector& input, ndBrainVector& output) const; virtual void InputDerivative(const ndBrainVector& output, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const; + virtual void CalculateParamGradients ( const ndBrainVector& input, const ndBrainVector& output, const ndBrainVector& outputDerivative, ndBrainVector& inputGradient, ndBrainVector& biasGradient, ndBrainMatrix& weightGradient); + virtual void CalculateParamGradients( + const ndBrainVector& input, const ndBrainVector& output, + const ndBrainVector& outputDerivative, ndBrainVector& inputGradient, ndBrainLayer* const gradientOut) const; + + virtual void AdamUpdate(const ndBrainLayer& u, const ndBrainLayer& v, ndBrainFloat epsilon); + virtual void Save(const ndBrainSave* const loadSave) const; }; diff --git a/newton-4.00/sdk/dBrain/ndBrainLayerActivation.cpp b/newton-4.00/sdk/dBrain/ndBrainLayerActivation.cpp index b582d8bdd9..df4cd32a25 100644 --- a/newton-4.00/sdk/dBrain/ndBrainLayerActivation.cpp +++ b/newton-4.00/sdk/dBrain/ndBrainLayerActivation.cpp @@ -72,15 +72,48 @@ ndInt32 ndBrainLayerActivation::GetInputSize() const return m_neurons; } -void ndBrainLayerActivation::InitWeightsXavierMethod() +void ndBrainLayerActivation::Clear() { + ndAssert(0); } -void ndBrainLayerActivation::InitWeights(ndBrainFloat, ndBrainFloat) +void ndBrainLayerActivation::FlushToZero() +{ +} + +void ndBrainLayerActivation::Set(const ndBrainLayer&) +{ +} + +void ndBrainLayerActivation::Scale(ndBrainFloat) +{ +} + +void ndBrainLayerActivation::Add(const ndBrainLayer&) +{ + ndAssert(0); +} + +void ndBrainLayerActivation::Mul(const ndBrainLayer&) +{ + ndAssert(0); +} + +void ndBrainLayerActivation::ScaleAdd(const ndBrainLayer&, ndBrainFloat) { + ndAssert(0); } -void ndBrainLayerActivation::CopyFrom(const ndBrainLayer&) +void ndBrainLayerActivation::AdamUpdate(const ndBrainLayer&, const ndBrainLayer&, ndBrainFloat) +{ + ndAssert(0); +} + +void ndBrainLayerActivation::InitWeightsXavierMethod() +{ +} + +void ndBrainLayerActivation::InitWeights(ndBrainFloat, ndBrainFloat) { } @@ -104,3 +137,8 @@ void ndBrainLayerActivation::CalculateParamGradients(const ndBrainVector&, const { InputDerivative(output, outputDerivative, inputGradient); } + +void ndBrainLayerActivation::CalculateParamGradients(const ndBrainVector&, const ndBrainVector& output, const ndBrainVector& outputDerivative, ndBrainVector& inputGradient, ndBrainLayer* const) const +{ + InputDerivative(output, outputDerivative, inputGradient); +} diff --git a/newton-4.00/sdk/dBrain/ndBrainLayerActivation.h b/newton-4.00/sdk/dBrain/ndBrainLayerActivation.h index 45d479bc71..d11c7f7ad4 100644 --- a/newton-4.00/sdk/dBrain/ndBrainLayerActivation.h +++ b/newton-4.00/sdk/dBrain/ndBrainLayerActivation.h @@ -39,7 +39,6 @@ class ndBrainLayerActivation : public ndBrainLayer virtual ndInt32 GetOutputSize() const; virtual ndInt32 GetInputSize() const; virtual const char* GetLabelId() const; - virtual void CopyFrom(const ndBrainLayer& src); virtual void Blend(const ndBrainLayer& src, ndBrainFloat blend); virtual void InitWeightsXavierMethod(); @@ -47,12 +46,26 @@ class ndBrainLayerActivation : public ndBrainLayer virtual void MakePrediction(const ndBrainVector& input, ndBrainVector& output) const; virtual void InputDerivative(const ndBrainVector& output, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const; + virtual void CalculateParamGradients( const ndBrainVector& input, const ndBrainVector& output, const ndBrainVector& outputDerivative, ndBrainVector& inputGradient, ndBrainVector& biasGradient, ndBrainMatrix& weightGradient); + virtual void CalculateParamGradients( + const ndBrainVector& input, const ndBrainVector& output, + const ndBrainVector& outputDerivative, ndBrainVector& inputGradient, ndBrainLayer* const gradientOut) const; + virtual void Save(const ndBrainSave* const loadSave) const; + void Clear(); + void FlushToZero(); + void Scale(ndBrainFloat scale); + void Set(const ndBrainLayer& src); + void Add(const ndBrainLayer& src); + void Mul(const ndBrainLayer& src); + void ScaleAdd(const ndBrainLayer& src, ndBrainFloat scale); + + void AdamUpdate(const ndBrainLayer& u, const ndBrainLayer& v, ndBrainFloat epsilon); protected: ndInt32 m_neurons; }; diff --git a/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.cpp b/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.cpp index 9fa1875f57..6485a23e41 100644 --- a/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.cpp +++ b/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.cpp @@ -23,23 +23,77 @@ #include "ndBrainSaveLoad.h" #include "ndBrainLayerConvolutional.h" -ndBrainLayerConvolutional::ndBrainLayerConvolutional(ndInt32 inputs, ndInt32 outputs) +ndBrainLayerConvolutional::ndBrainLayerConvolutional(ndInt32 inputStride, ndInt32 inputWidth, ndInt32 inputHeight, ndInt32 inputDepth, ndInt32 kernelSize, ndInt32 numberOfKernels) :ndBrainLayer() ,m_bias() - ,m_weights(outputs, inputs) + ,m_weights() + ,m_inputWidth(inputWidth) + ,m_inputStride(inputStride) + ,m_inputHeight(inputHeight) + ,m_inputDepth(inputDepth) + ,m_kernelSize(kernelSize) + ,m_numberOfKernels(numberOfKernels) { - m_bias.SetCount(outputs); + m_outputWidth = m_inputWidth - m_kernelSize + 1; + m_outputHeight = m_inputHeight - m_kernelSize + 1; + + m_bias.SetCount(m_numberOfKernels); + for (ndInt32 i = 0; i < m_numberOfKernels; ++i) + { + m_bias[i] = new ndBrainMatrix(m_outputWidth, m_outputHeight); + } + + ndAssert(0); + //m_weights.SetCount(m_numberOfKernels * m_inputDepth); + //m_weights.SetCount(m_numberOfKernels * (m_inputDepth * ); + for (ndInt32 i = 0; i < m_numberOfKernels * m_inputDepth; ++i) + { + m_weights[i] = new ndBrainMatrix(m_kernelSize, m_kernelSize); + } } ndBrainLayerConvolutional::ndBrainLayerConvolutional(const ndBrainLayerConvolutional& src) :ndBrainLayer(src) - ,m_bias(src.m_bias) - ,m_weights(src.m_weights) + ,m_bias() + ,m_weights() + ,m_inputWidth(src.m_inputWidth) + ,m_inputStride(src.m_inputStride) + ,m_inputHeight(src.m_inputHeight) + ,m_inputDepth(src.m_inputDepth) + ,m_kernelSize(src.m_kernelSize) + ,m_numberOfKernels(src.m_numberOfKernels) + ,m_outputWidth(src.m_outputWidth) + ,m_outputHeight(src.m_outputHeight) { + m_bias.SetCount(m_numberOfKernels); + for (ndInt32 i = 0; i < m_numberOfKernels; ++i) + { + //m_bias[i] = new ndBrainMatrix(width, height); + m_bias[i] = new ndBrainMatrix(m_kernelSize, m_kernelSize); + m_bias[i]->Set(*src.m_bias[i]); + } + + m_weights.SetCount(m_numberOfKernels * m_inputDepth); + for (ndInt32 i = 0; i < m_numberOfKernels * m_inputDepth; ++i) + { + //m_weights[i] = new ndBrainMatrix(width, height); + m_weights[i] = new ndBrainMatrix(m_kernelSize, m_kernelSize); + m_weights[i]->Set(*src.m_weights[i]); + } } ndBrainLayerConvolutional::~ndBrainLayerConvolutional() { + ndAssert(0); + for (ndInt32 i = 0; i < m_bias.GetCount(); ++i) + { + delete (m_bias[i]); + } + + for (ndInt32 i = 0; i < m_weights.GetCount(); ++i) + { + delete (m_weights[i]); + } } const char* ndBrainLayerConvolutional::GetLabelId() const @@ -54,24 +108,31 @@ ndBrainLayer* ndBrainLayerConvolutional::Clone() const ndInt32 ndBrainLayerConvolutional::GetOutputSize() const { - ndAssert(m_bias.GetCount() == m_weights.GetRows()); - return m_bias.GetCount(); + //ndAssert(m_bias.GetCount() == m_weights.GetRows()); + //return m_bias.GetCount(); + //return m_bias.GetCount() * m_bias[0]->GetRows() * m_bias[0]->GetColumns(); + return m_numberOfKernels * m_outputWidth * m_outputHeight; } ndInt32 ndBrainLayerConvolutional::GetInputSize() const { - return m_weights.GetColumns(); + //return m_weights.GetColumns(); + //return m_weights.GetCount() * m_inputWidth * m_inputHeight; + return m_inputDepth * m_inputWidth * m_inputHeight; } -ndBrainVector* ndBrainLayerConvolutional::GetBias() -{ - return &m_bias; -} - -ndBrainMatrix* ndBrainLayerConvolutional::GetWeights() -{ - return &m_weights; -} +//ndBrainVector* ndBrainLayerConvolutional::GetBias() +//{ +// ndAssert(0); +// return nullptr; +//} +// +//ndBrainMatrix* ndBrainLayerConvolutional::GetWeights() +//{ +// //return &m_weights; +// ndAssert(0); +// return nullptr; +//} bool ndBrainLayerConvolutional::HasParameters() const { @@ -80,20 +141,32 @@ bool ndBrainLayerConvolutional::HasParameters() const void ndBrainLayerConvolutional::InitWeightsXavierMethod() { - ndBrainFloat weighVariance = ndBrainFloat(ndSqrt(ndFloat32(6.0f) / ndFloat32(GetInputSize() + GetOutputSize()))); - InitWeights(weighVariance, ndBrainFloat(0.0f)); + //ndBrainFloat weighVariance = ndBrainFloat(ndSqrt(ndFloat32(6.0f) / ndFloat32(GetInputSize() + GetOutputSize()))); + //InitWeights(weighVariance, ndBrainFloat(0.0f)); + ndAssert(0); } void ndBrainLayerConvolutional::InitGaussianBias(ndBrainFloat variance) { - m_bias.InitGaussianWeights(variance); + for (ndInt32 i = m_bias.GetCount() - 1; i >= 0; --i) + { + ndBrainMatrix& biasMatrix = *m_bias[i]; + for (ndInt32 j = biasMatrix.GetRows() - 1; j >= 0 ; --j) + { + biasMatrix[j].InitGaussianWeights(variance); + } + } } void ndBrainLayerConvolutional::InitGaussianWeights(ndBrainFloat variance) { for (ndInt32 i = m_weights.GetCount() - 1; i >= 0; --i) { - m_weights[i].InitGaussianWeights(variance); + ndBrainMatrix& m_weightsMatrix = *m_weights[i]; + for (ndInt32 j = m_weightsMatrix.GetRows() - 1; j >= 0; --j) + { + m_weightsMatrix[j].InitGaussianWeights(variance); + } } } @@ -105,109 +178,169 @@ void ndBrainLayerConvolutional::InitWeights(ndBrainFloat weighVariance, ndBrainF InitGaussianWeights(weighVariance); } -void ndBrainLayerConvolutional::CopyFrom(const ndBrainLayer& src) +//void ndBrainLayerConvolutional::CopyFrom(const ndBrainLayer& src) +void ndBrainLayerConvolutional::Set(const ndBrainLayer& src) { - const ndBrainLayerConvolutional& linearSrc = (ndBrainLayerConvolutional&)src; - m_bias.Set(linearSrc.m_bias); - m_weights.Set(linearSrc.m_weights); -} + const ndBrainLayerConvolutional& convSrc = (ndBrainLayerConvolutional&)src; + //m_bias.Set(linearSrc.m_bias); + //m_weights.Set(linearSrc.m_weights); -void ndBrainLayerConvolutional::Blend(const ndBrainLayer& src, ndBrainFloat blend) -{ - const ndBrainLayerConvolutional& linearSrc = (ndBrainLayerConvolutional&)src; - m_bias.Blend(linearSrc.m_bias, blend); - m_weights.Blend(linearSrc.m_weights, blend); + for (ndInt32 i = 0; i < m_numberOfKernels; ++i) + { + m_bias[i]->Set(*convSrc.m_bias[i]); + } + + for (ndInt32 i = 0; i < m_numberOfKernels * m_inputDepth; ++i) + { + m_weights[i]->Set(*convSrc.m_weights[i]); + } } -void ndBrainLayerConvolutional::MakePrediction(const ndBrainVector& input, ndBrainVector& output) const +void ndBrainLayerConvolutional::Blend(const ndBrainLayer& src, ndBrainFloat blend) { - m_weights.Mul(input, output); - output.Add(m_bias); + //const ndBrainLayerConvolutional& linearSrc = (ndBrainLayerConvolutional&)src; + //m_bias.Blend(linearSrc.m_bias, blend); + //m_weights.Blend(linearSrc.m_weights, blend); + ndAssert(0); } void ndBrainLayerConvolutional::InputDerivative(const ndBrainVector&, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const { - m_weights.TransposeMul(outputDerivative, inputDerivative); + //m_weights.TransposeMul(outputDerivative, inputDerivative); + ndAssert(0); } void ndBrainLayerConvolutional::CalculateParamGradients(const ndBrainVector& input, const ndBrainVector& output, const ndBrainVector& outputDerivative, ndBrainVector& inputGradient, ndBrainVector& biasGradient, ndBrainMatrix& weightGradient) { - ndAssert(biasGradient.GetCount() == outputDerivative.GetCount()); - biasGradient.Set(outputDerivative); - for (ndInt32 i = outputDerivative.GetCount() - 1; i >= 0 ; --i) - { - ndBrainFloat value = outputDerivative[i]; - weightGradient[i].ScaleSet(input, value); - } - InputDerivative(output, outputDerivative, inputGradient); + //ndAssert(biasGradient.GetCount() == outputDerivative.GetCount()); + //biasGradient.Set(outputDerivative); + //for (ndInt32 i = outputDerivative.GetCount() - 1; i >= 0 ; --i) + //{ + // ndBrainFloat value = outputDerivative[i]; + // weightGradient[i].ScaleSet(input, value); + //} + //InputDerivative(output, outputDerivative, inputGradient); + ndAssert(0); } void ndBrainLayerConvolutional::Save(const ndBrainSave* const loadSave) const { - char buffer[1024]; - auto Save = [this, &buffer, &loadSave](const char* const fmt, ...) - { - va_list v_args; - buffer[0] = 0; - va_start(v_args, fmt); - vsprintf(buffer, fmt, v_args); - va_end(v_args); - loadSave->WriteData(buffer); - }; - - Save("\tinputs %d\n", m_weights.GetColumns()); - Save("\touputs %d\n", m_weights.GetCount()); - - Save("\tbias "); - for (ndInt32 i = 0; i < m_bias.GetCount(); ++i) - { - Save("%g ", m_bias[i]); - } - Save("\n"); + //char buffer[1024]; + //auto Save = [this, &buffer, &loadSave](const char* const fmt, ...) + //{ + // va_list v_args; + // buffer[0] = 0; + // va_start(v_args, fmt); + // vsprintf(buffer, fmt, v_args); + // va_end(v_args); + // loadSave->WriteData(buffer); + //}; + // + //Save("\tinputs %d\n", m_weights.GetColumns()); + //Save("\touputs %d\n", m_weights.GetCount()); + // + //Save("\tbias "); + //for (ndInt32 i = 0; i < m_bias.GetCount(); ++i) + //{ + // Save("%g ", m_bias[i]); + //} + //Save("\n"); + // + //Save("\tweights\n"); + //for (ndInt32 i = 0; i < m_weights.GetCount(); ++i) + //{ + // Save("\t\trow_%d ", i); + // const ndBrainVector& row = m_weights[i]; + // for (ndInt32 j = 0; j < GetInputSize(); ++j) + // { + // Save("%g ", row[j]); + // } + // Save("\n"); + //} - Save("\tweights\n"); - for (ndInt32 i = 0; i < m_weights.GetCount(); ++i) - { - Save("\t\trow_%d ", i); - const ndBrainVector& row = m_weights[i]; - for (ndInt32 j = 0; j < GetInputSize(); ++j) - { - Save("%g ", row[j]); - } - Save("\n"); - } + ndAssert(0); } ndBrainLayer* ndBrainLayerConvolutional::Load(const ndBrainLoad* const loadSave) { - char buffer[1024]; - loadSave->ReadString(buffer); + //char buffer[1024]; + //loadSave->ReadString(buffer); + // + //loadSave->ReadString(buffer); + //ndInt32 inputs = loadSave->ReadInt(); + //loadSave->ReadString(buffer); + //ndInt32 outputs = loadSave->ReadInt(); + //ndBrainLayerConvolutional* const layer = new ndBrainLayerConvolutional(inputs, outputs); + // + //loadSave->ReadString(buffer); + //for (ndInt32 i = 0; i < outputs; ++i) + //{ + // ndBrainFloat val = ndBrainFloat(loadSave->ReadFloat()); + // layer->m_bias[i] = val; + //} + // + //loadSave->ReadString(buffer); + //for (ndInt32 i = 0; i < outputs; ++i) + //{ + // loadSave->ReadString(buffer); + // for (ndInt32 j = 0; j < inputs; ++j) + // { + // ndBrainFloat val = ndBrainFloat(loadSave->ReadFloat()); + // layer->m_weights[i][j] = val; + // } + //} + // + //loadSave->ReadString(buffer); + //return layer; - loadSave->ReadString(buffer); - ndInt32 inputs = loadSave->ReadInt(); - loadSave->ReadString(buffer); - ndInt32 outputs = loadSave->ReadInt(); - ndBrainLayerConvolutional* const layer = new ndBrainLayerConvolutional(inputs, outputs); + ndAssert(0); + return nullptr; +} - loadSave->ReadString(buffer); - for (ndInt32 i = 0; i < outputs; ++i) - { - ndBrainFloat val = ndBrainFloat(loadSave->ReadFloat()); - layer->m_bias[i] = val; - } +void ndBrainLayerConvolutional::MakePrediction(const ndBrainVector& input, ndBrainVector& output) const +{ + //m_weights.Mul(input, output); + //output.Add(m_bias); - loadSave->ReadString(buffer); - for (ndInt32 i = 0; i < outputs; ++i) + output.Set(ndBrainFloat(ndFloat32(0.0f))); + //ndInt32 inputStart = 0; + ndInt32 outputStart = 0; +float xxxxx = 0; + for (ndInt32 i = 0; i < m_numberOfKernels; ++i) { - loadSave->ReadString(buffer); - for (ndInt32 j = 0; j < inputs; ++j) + const ndBrainMatrix& bias = *m_bias[i]; + for (ndInt32 j = 0; j < m_outputHeight; ++j) { - ndBrainFloat val = ndBrainFloat(loadSave->ReadFloat()); - layer->m_weights[i][j] = val; + ndBrainMemVector out(&output[outputStart], m_outputWidth); + out.Set(bias[j]); + + for (ndInt32 k = 0; k < m_outputWidth; ++k) + { + out[k] = xxxxx; + xxxxx += 1.0f; + } + outputStart += m_outputWidth; } - } - loadSave->ReadString(buffer); - return layer; -} + //for (ndInt32 j = 0; j < m_inputDepth; ++j) + //{ + // const ndBrainMatrix& weight = *m_weights[j]; + // + // for (ndInt32 k = 0; k < m_outputWidth; ++k) + // { + // for (ndInt32 n = 0; n < m_outputHeight; ++n) + // { + // + // } + // } + // + // //for (ndInt32 k = 0; k < m_kernelSize; ++k) + // //{ + // // const ndBrainVector& kernelRow = weight[j]; + // // const ndBrainMemVector in(&input[inputStart], m_inputWidth); + // // //inputStart += m_inputStride; + // //} + //} + } +} \ No newline at end of file diff --git a/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.h b/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.h index b6c1b51106..651ff953c3 100644 --- a/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.h +++ b/newton-4.00/sdk/dBrain/ndBrainLayerConvolutional.h @@ -30,7 +30,7 @@ class ndBrainLayerConvolutional : public ndBrainLayer { public: - ndBrainLayerConvolutional(ndInt32 inputs, ndInt32 outputs); + ndBrainLayerConvolutional(ndInt32 inputStride, ndInt32 inputWidth, ndInt32 inputHeight, ndInt32 inputDepth, ndInt32 kernelSize, ndInt32 numberOfKernels); ndBrainLayerConvolutional(const ndBrainLayerConvolutional& src); virtual ~ndBrainLayerConvolutional(); virtual ndBrainLayer* Clone() const; @@ -39,11 +39,7 @@ class ndBrainLayerConvolutional : public ndBrainLayer virtual ndInt32 GetOutputSize() const; virtual ndInt32 GetInputSize() const; virtual const char* GetLabelId() const; - virtual void CopyFrom(const ndBrainLayer& src); virtual void Blend(const ndBrainLayer& src, ndBrainFloat blend); - - virtual ndBrainVector* GetBias(); - virtual ndBrainMatrix* GetWeights(); virtual void InitWeightsXavierMethod(); virtual void InitWeights(ndBrainFloat weighVariance, ndBrainFloat biasVariance); @@ -56,13 +52,26 @@ class ndBrainLayerConvolutional : public ndBrainLayer virtual void Save(const ndBrainSave* const loadSave) const; static ndBrainLayer* Load(const ndBrainLoad* const loadSave); + + + void Set(const ndBrainLayer& src); private: void InitGaussianBias(ndBrainFloat variance); void InitGaussianWeights(ndBrainFloat variance); - ndBrainVector m_bias; - ndBrainMatrix m_weights; + ndArray m_bias; + ndArray m_weights; + + ndInt32 m_inputWidth; + ndInt32 m_inputStride; + ndInt32 m_inputHeight; + ndInt32 m_inputDepth; + ndInt32 m_kernelSize; + ndInt32 m_numberOfKernels; + + ndInt32 m_outputWidth; + ndInt32 m_outputHeight; }; diff --git a/newton-4.00/sdk/dBrain/ndBrainLayerLinear.cpp b/newton-4.00/sdk/dBrain/ndBrainLayerLinear.cpp index 3547fa9084..576673b276 100644 --- a/newton-4.00/sdk/dBrain/ndBrainLayerLinear.cpp +++ b/newton-4.00/sdk/dBrain/ndBrainLayerLinear.cpp @@ -106,13 +106,53 @@ void ndBrainLayerLinear::InitWeights(ndBrainFloat weighVariance, ndBrainFloat bi InitGaussianWeights(weighVariance); } -void ndBrainLayerLinear::CopyFrom(const ndBrainLayer& src) +void ndBrainLayerLinear::Clear() +{ + m_bias.Set(ndBrainFloat(0.0f)); + m_weights.Set(ndBrainFloat(0.0f)); +} + +void ndBrainLayerLinear::FlushToZero() +{ + m_bias.FlushToZero(); + m_weights.FlushToZero(); +} + +void ndBrainLayerLinear::Scale(ndBrainFloat scale) +{ + //const ndBrainLayerLinear& linearSrc = (ndBrainLayerLinear&)src; + m_bias.Scale(scale); + m_weights.Scale(scale); +} + +void ndBrainLayerLinear::Set(const ndBrainLayer& src) { const ndBrainLayerLinear& linearSrc = (ndBrainLayerLinear&)src; m_bias.Set(linearSrc.m_bias); m_weights.Set(linearSrc.m_weights); } +void ndBrainLayerLinear::Add(const ndBrainLayer& src) +{ + const ndBrainLayerLinear& linearSrc = (ndBrainLayerLinear&)src; + m_bias.Add(linearSrc.m_bias); + m_weights.Add(linearSrc.m_weights); +} + +void ndBrainLayerLinear::Mul(const ndBrainLayer& src) +{ + const ndBrainLayerLinear& linearSrc = (ndBrainLayerLinear&)src; + m_bias.Mul(linearSrc.m_bias); + m_weights.Mul(linearSrc.m_weights); +} + +void ndBrainLayerLinear::ScaleAdd(const ndBrainLayer& src, ndBrainFloat scale) +{ + const ndBrainLayerLinear& linearSrc = (ndBrainLayerLinear&)src; + m_bias.ScaleAdd(linearSrc.m_bias, scale); + m_weights.ScaleAdd(linearSrc.m_weights, scale); +} + void ndBrainLayerLinear::Blend(const ndBrainLayer& src, ndBrainFloat blend) { const ndBrainLayerLinear& linearSrc = (ndBrainLayerLinear&)src; @@ -120,6 +160,33 @@ void ndBrainLayerLinear::Blend(const ndBrainLayer& src, ndBrainFloat blend) m_weights.Blend(linearSrc.m_weights, blend); } +void ndBrainLayerLinear::AdamUpdate(const ndBrainLayer& u, const ndBrainLayer& v, ndBrainFloat epsilon) +{ + const ndBrainLayerLinear& linear_U = (ndBrainLayerLinear&)u; + const ndBrainLayerLinear& linear_V = (ndBrainLayerLinear&)v; + + const ndBrainVector& bias_U = linear_U.m_bias; + const ndBrainVector& bias_V = linear_V.m_bias; + for (ndInt32 i = m_bias.GetCount() - 1; i >= 0; --i) + { + ndBrainFloat bias_den = ndBrainFloat(1.0f) / (ndBrainFloat(ndSqrt(bias_V[i])) + epsilon); + //biasGradients[j] = uHat[j] * bias_den; + m_bias[i] = bias_U[i] * bias_den; + } + + const ndBrainMatrix& weight_U = linear_U.m_weights; + const ndBrainMatrix& weight_V = linear_V.m_weights; + for (ndInt32 i = m_weights.GetRows() - 1; i >= 0; --i) + { + for (ndInt32 j = m_weights[i].GetCount() - 1; j >= 0; --j) + { + ndBrainFloat weight_den = ndBrainFloat(1.0f) / (ndBrainFloat(ndSqrt(weight_V[i][j])) + epsilon); + //weightGradients[j][k] = uHat[j][k] * bias_den; + m_weights[i][j] = weight_U[i][j] * weight_den; + } + } +} + void ndBrainLayerLinear::MakePrediction(const ndBrainVector& input, ndBrainVector& output) const { m_weights.Mul(input, output); @@ -144,6 +211,22 @@ void ndBrainLayerLinear::CalculateParamGradients(const ndBrainVector& input, con InputDerivative(output, outputDerivative, inputGradient); } +void ndBrainLayerLinear::CalculateParamGradients( + const ndBrainVector& input, const ndBrainVector& output, + const ndBrainVector& outputDerivative, ndBrainVector& inputGradient, ndBrainLayer* const gradientOut) const +{ + ndAssert(!strcmp(GetLabelId(), gradientOut->GetLabelId())); + ndBrainLayerLinear* const gradients = (ndBrainLayerLinear*)gradientOut; + ndAssert(gradients->m_bias.GetCount() == outputDerivative.GetCount()); + gradients->m_bias.Set(outputDerivative); + for (ndInt32 i = outputDerivative.GetCount() - 1; i >= 0; --i) + { + ndBrainFloat value = outputDerivative[i]; + gradients->m_weights[i].ScaleSet(input, value); + } + InputDerivative(output, outputDerivative, inputGradient); +} + void ndBrainLayerLinear::Save(const ndBrainSave* const loadSave) const { char buffer[1024]; diff --git a/newton-4.00/sdk/dBrain/ndBrainLayerLinear.h b/newton-4.00/sdk/dBrain/ndBrainLayerLinear.h index 9f3d50c231..2ec551b271 100644 --- a/newton-4.00/sdk/dBrain/ndBrainLayerLinear.h +++ b/newton-4.00/sdk/dBrain/ndBrainLayerLinear.h @@ -39,7 +39,6 @@ class ndBrainLayerLinear : public ndBrainLayer virtual ndInt32 GetOutputSize() const; virtual ndInt32 GetInputSize() const; virtual const char* GetLabelId() const; - virtual void CopyFrom(const ndBrainLayer& src); virtual void Blend(const ndBrainLayer& src, ndBrainFloat blend); virtual ndBrainVector* GetBias(); @@ -50,16 +49,30 @@ class ndBrainLayerLinear : public ndBrainLayer virtual void MakePrediction(const ndBrainVector& input, ndBrainVector& output) const; virtual void InputDerivative(const ndBrainVector& output, const ndBrainVector& outputDerivative, ndBrainVector& inputDerivative) const; + virtual void CalculateParamGradients( const ndBrainVector& input, const ndBrainVector& output, const ndBrainVector& outputDerivative, ndBrainVector& inputGradient, ndBrainVector& biasGradient, ndBrainMatrix& weightGradient); + virtual void CalculateParamGradients( + const ndBrainVector& input, const ndBrainVector& output, + const ndBrainVector& outputDerivative, ndBrainVector& inputGradient, ndBrainLayer* const gradientOut) const; + virtual void Save(const ndBrainSave* const loadSave) const; static ndBrainLayer* Load(const ndBrainLoad* const loadSave); + void Clear(); + void FlushToZero(); + void Scale(ndBrainFloat scale); + void Set(const ndBrainLayer& src); + void Add(const ndBrainLayer& src); + void Mul(const ndBrainLayer& src); + void ScaleAdd(const ndBrainLayer& src, ndBrainFloat scale); + private: void InitGaussianBias(ndBrainFloat variance); void InitGaussianWeights(ndBrainFloat variance); + void AdamUpdate(const ndBrainLayer& u, const ndBrainLayer& v, ndBrainFloat epsilon); ndBrainVector m_bias; ndBrainMatrix m_weights; diff --git a/newton-4.00/sdk/dBrain/ndBrainOptimizerAdam.cpp b/newton-4.00/sdk/dBrain/ndBrainOptimizerAdam.cpp index dd295e22c9..51ad991ac6 100644 --- a/newton-4.00/sdk/dBrain/ndBrainOptimizerAdam.cpp +++ b/newton-4.00/sdk/dBrain/ndBrainOptimizerAdam.cpp @@ -31,31 +31,36 @@ class ndBrainOptimizerAdam::ndBrainOptimizerAdam::ndAdamData : public ndClassAll public: ndAdamData(ndBrainLayer* const layer) :ndClassAlloc() + ,m_u(nullptr) + ,m_v(nullptr) + ,m_v2(nullptr) { if (layer->HasParameters()) { - m_biasGradient_u.SetCount(layer->GetOutputSize()); - m_biasGradient_v.SetCount(layer->GetOutputSize()); - m_biasGradient_v2.SetCount(layer->GetOutputSize()); - m_weightGradient_u.Init(layer->GetOutputSize(), layer->GetInputSize()); - m_weightGradient_v.Init(layer->GetOutputSize(), layer->GetInputSize()); - m_weightGradient_v2.Init(layer->GetOutputSize(), layer->GetInputSize()); - - m_biasGradient_u.Set(ndBrainFloat(0.0f)); - m_biasGradient_v.Set(ndBrainFloat(0.0f)); - m_biasGradient_v2.Set(ndBrainFloat(0.0f)); - m_weightGradient_u.Set(ndBrainFloat(0.0f)); - m_weightGradient_v.Set(ndBrainFloat(0.0f)); - m_weightGradient_v2.Set(ndBrainFloat(0.0f)); + m_u = layer->Clone(); + m_v = layer->Clone(); + m_v2 = layer->Clone(); + + m_u->Clear(); + m_v->Clear(); + m_v2->Clear(); } } - ndBrainVector m_biasGradient_u; - ndBrainVector m_biasGradient_v; - ndBrainVector m_biasGradient_v2; - ndBrainMatrix m_weightGradient_u; - ndBrainMatrix m_weightGradient_v; - ndBrainMatrix m_weightGradient_v2; + ~ndAdamData() + { + ndAssert(0); + if (m_u) + { + delete m_u; + delete m_v; + delete m_v2; + } + } + + ndBrainLayer* m_u; + ndBrainLayer* m_v; + ndBrainLayer* m_v2; }; ndBrainOptimizerAdam::ndBrainOptimizerAdam() @@ -116,98 +121,45 @@ void ndBrainOptimizerAdam::Update(ndBrainThreadPool* const threadPool, ndArrayGetBrain())[index]->HasParameters()); - ndBrainVector& bias = *trainer->GetBias(index); - ndBrainMatrix& weight = *trainer->GetWeight(index); - ndBrainVector& biasGradients = *trainer->GetBiasGradients(index); - ndBrainMatrix& weightGradients = *trainer->GetWeightGradients(index); - + ndBrainLayer& weights = *trainer->GetWeightsLayer(index); + ndBrainLayer& gradients = *trainer->GetGradientLayer(index); + for (ndInt32 j = 1; j < partialGradients.GetCount(); ++j) { ndBrainTrainer* const src = partialGradients[j]; trainer->AcculumateGradients(*src, index); } - - ndAdamData& data = *m_data[index]; - biasGradients.Scale(den); - data.m_biasGradient_v.Scale(m_beta); - data.m_biasGradient_u.Scale(m_alpha); - data.m_biasGradient_v2.Set(biasGradients); - data.m_biasGradient_v2.Mul(biasGradients); - data.m_biasGradient_u.ScaleAdd(biasGradients, ndBrainFloat(1.0f) - m_alpha); - data.m_biasGradient_v.ScaleAdd(data.m_biasGradient_v2, ndBrainFloat (1.0f) - m_beta); + ndAdamData& data = *m_data[index]; + gradients.Scale(den); + data.m_v->Scale(m_beta); + data.m_u->Scale(m_alpha); + data.m_v2->Set(gradients); + data.m_v2->Mul(gradients); + + data.m_u->ScaleAdd(gradients, ndBrainFloat(1.0f) - m_alpha); + data.m_v->ScaleAdd(*data.m_v2, ndBrainFloat(1.0f) - m_beta); if (m_betaAcc > ndBrainFloat(0.0f)) { - ndBrainVector& vHat = biasGradients; - ndBrainVector& uHat = data.m_biasGradient_v2; - - uHat.Set(data.m_biasGradient_u); - vHat.Set(data.m_biasGradient_v); - vHat.Scale(betaWeight); - uHat.Scale(alphaWeight); - for (ndInt32 j = biasGradients.GetCount() - 1; j >= 0; --j) - { - ndBrainFloat bias_den = ndBrainFloat(1.0f) / (ndBrainFloat(ndSqrt(vHat[j])) + m_epsilon); - biasGradients[j] = uHat[j] * bias_den; - } - } - else - { - for (ndInt32 j = biasGradients.GetCount() - 1; j >= 0; --j) - { - ndBrainFloat bias_den = ndBrainFloat(1.0f) / (ndBrainFloat(ndSqrt(data.m_biasGradient_v[j])) + m_epsilon); - biasGradients[j] = data.m_biasGradient_u[j] * bias_den; - } - } - biasGradients.Scale(descendRate); - biasGradients.ScaleAdd(bias, regularizer); - bias.Add(biasGradients); - bias.FlushToZero(); - - - weightGradients.Scale(den); - data.m_weightGradient_v.Scale(m_beta); - data.m_weightGradient_u.Scale(m_alpha); - data.m_weightGradient_v2.Set(weightGradients); - data.m_weightGradient_v2.Mul(weightGradients); - data.m_weightGradient_v.ScaleAdd(data.m_weightGradient_v2, ndBrainFloat(1.0f) - m_beta); - data.m_weightGradient_u.ScaleAdd(weightGradients, ndBrainFloat(1.0f) - m_alpha); - - if (m_betaAcc > ndBrainFloat(0.0f)) - { - ndBrainMatrix& vHat = weightGradients; - ndBrainMatrix& uHat = data.m_weightGradient_v2; - - uHat.Set(data.m_weightGradient_u); - vHat.Set(data.m_weightGradient_v); + ndBrainLayer& vHat = gradients; + ndBrainLayer& uHat = *data.m_v2; + + uHat.Set(*data.m_u); + vHat.Set(*data.m_v); vHat.Scale(betaWeight); uHat.Scale(alphaWeight); - - for (ndInt32 j = weightGradients.GetRows() - 1; j >= 0; --j) - { - for (ndInt32 k = weightGradients[j].GetCount() - 1; k >= 0; --k) - { - ndBrainFloat bias_den = ndBrainFloat(1.0f) / (ndBrainFloat(ndSqrt(vHat[j][k])) + m_epsilon); - weightGradients[j][k] = uHat[j][k] * bias_den; - } - } + gradients.AdamUpdate(uHat, vHat, m_epsilon); } else { - for (ndInt32 j = weightGradients.GetRows() - 1; j >= 0; --j) - { - for (ndInt32 k = weightGradients[j].GetCount() - 1; k >= 0; --k) - { - ndBrainFloat bias_den = ndBrainFloat(1.0f) / (ndBrainFloat(ndSqrt(data.m_weightGradient_v[j][k])) + m_epsilon); - weightGradients[j][k] = data.m_weightGradient_u[j][k] * bias_den; - } - } + gradients.AdamUpdate(*data.m_u, *data.m_v, m_epsilon); } - weightGradients.Scale(descendRate); - weightGradients.ScaleAdd(weight, regularizer); - weight.Add(weightGradients); - weight.FlushToZero(); + + gradients.Scale(descendRate); + gradients.ScaleAdd(weights, regularizer); + weights.Add(gradients); + weights.FlushToZero(); } }); diff --git a/newton-4.00/sdk/dBrain/ndBrainOptimizerSgd.cpp b/newton-4.00/sdk/dBrain/ndBrainOptimizerSgd.cpp index 0c40420800..528f65544a 100644 --- a/newton-4.00/sdk/dBrain/ndBrainOptimizerSgd.cpp +++ b/newton-4.00/sdk/dBrain/ndBrainOptimizerSgd.cpp @@ -60,26 +60,20 @@ void ndBrainOptimizerSgd::Update(ndBrainThreadPool* const threadPool, ndArrayGetBrain())[index]->HasParameters()); - ndBrainVector& bias = *trainer->GetBias(index); - ndBrainMatrix& weight = *trainer->GetWeight(index); - ndBrainVector& biasGradients = *trainer->GetBiasGradients(index); - ndBrainMatrix& weightGradients = *trainer->GetWeightGradients(index); - + + ndBrainLayer& weights = *trainer->GetWeightsLayer(index); + ndBrainLayer& gradients = *trainer->GetGradientLayer(index); + for (ndInt32 j = 1; j < partialGradients.GetCount(); ++j) { ndBrainTrainer* const src = partialGradients[j]; trainer->AcculumateGradients(*src, index); } - - biasGradients.Scale(denScale); - biasGradients.ScaleAdd(bias, regularizer); - bias.Add(biasGradients); - bias.FlushToZero(); - - weightGradients.Scale(denScale); - weightGradients.ScaleAdd(weight, regularizer); - weight.Add(weightGradients); - weight.FlushToZero(); + + gradients.Scale(denScale); + gradients.ScaleAdd(weights, regularizer); + weights.Add(gradients); + weights.FlushToZero(); } }); diff --git a/newton-4.00/sdk/dBrain/ndBrainTrainer.cpp b/newton-4.00/sdk/dBrain/ndBrainTrainer.cpp index 79184089aa..403dae8019 100644 --- a/newton-4.00/sdk/dBrain/ndBrainTrainer.cpp +++ b/newton-4.00/sdk/dBrain/ndBrainTrainer.cpp @@ -34,11 +34,21 @@ class ndBrainTrainer::ndLayerData : public ndClassAlloc ndLayerData(ndBrainLayer* const layer) :ndClassAlloc() ,m_layer(layer) + ,m_gradient(nullptr) { if (layer->HasParameters()) { - m_gradBias.SetCount(layer->GetOutputSize()); - m_gradWeight.Init(layer->GetOutputSize(), layer->GetInputSize()); + //m_gradBias.SetCount(layer->GetOutputSize()); + //m_gradWeight.Init(layer->GetOutputSize(), layer->GetInputSize()); + m_gradient = layer->Clone(); + } + } + + ~ndLayerData() + { + if (m_gradient) + { + delete m_gradient; } } @@ -46,32 +56,37 @@ class ndBrainTrainer::ndLayerData : public ndClassAlloc { if (m_layer->HasParameters()) { - m_gradBias.Set(ndBrainFloat(0.0f)); - m_gradWeight.Set(ndBrainFloat(0.0f)); + ndAssert(0); + //m_gradBias.Set(ndBrainFloat(0.0f)); + //m_gradWeight.Set(ndBrainFloat(0.0f)); } } - void ScaleGradients(ndBrainFloat s) + //void AddGradients(const ndLayerData& src) + void Add(const ndLayerData& src) { if (m_layer->HasParameters()) { - m_gradBias.Scale(s); - m_gradWeight.Scale(s); + //m_gradBias.Add(src.m_gradBias); + //m_gradWeight.Add(src.m_gradWeight); + m_gradient->Add(*src.m_gradient); } } - void AddGradients(const ndLayerData& src) + void Scale(ndBrainFloat s) { if (m_layer->HasParameters()) { - m_gradBias.Add(src.m_gradBias); - m_gradWeight.Add(src.m_gradWeight); + //m_gradBias.Scale(s); + //m_gradWeight.Scale(s); + m_gradient->Scale(s); } } - ndBrainVector m_gradBias; - ndBrainMatrix m_gradWeight; + //ndBrainVector m_gradBias; + //ndBrainMatrix m_gradWeight; ndBrainLayer* m_layer; + ndBrainLayer* m_gradient; }; ndBrainTrainer::ndBrainTrainer(ndBrain* const brain) @@ -110,24 +125,14 @@ ndBrain* ndBrainTrainer::GetBrain() const return m_brain; } -ndBrainVector* ndBrainTrainer::GetBias(ndInt32 index) const -{ - return m_data[index]->m_layer->GetBias(); -} - -ndBrainMatrix* ndBrainTrainer::GetWeight(ndInt32 index) const -{ - return m_data[index]->m_layer->GetWeights(); -} - -ndBrainVector* ndBrainTrainer::GetBiasGradients(ndInt32 index) const +ndBrainLayer* ndBrainTrainer::GetWeightsLayer(ndInt32 index) const { - return &m_data[index]->m_gradBias; + return m_data[index]->m_layer; } -ndBrainMatrix* ndBrainTrainer::GetWeightGradients(ndInt32 index) const +ndBrainLayer* ndBrainTrainer::GetGradientLayer(ndInt32 index) const { - return &m_data[index]->m_gradWeight; + return m_data[index]->m_gradient; } void ndBrainTrainer::AcculumateGradients(const ndBrainTrainer& src, ndInt32 index) @@ -135,8 +140,9 @@ void ndBrainTrainer::AcculumateGradients(const ndBrainTrainer& src, ndInt32 inde ndLayerData* const dstData = m_data[index]; ndAssert(dstData->m_layer->HasParameters()); const ndLayerData* const srcData = src.m_data[index]; - dstData->m_gradBias.Add(srcData->m_gradBias); - dstData->m_gradWeight.Add(srcData->m_gradWeight); + //dstData->m_gradBias.Add(srcData->m_gradBias); + //dstData->m_gradWeight.Add(srcData->m_gradWeight); + dstData->Add(*srcData); } void ndBrainTrainer::ClearGradients() @@ -151,7 +157,7 @@ void ndBrainTrainer::AddGradients(const ndBrainTrainer* const src) { for (ndInt32 i = m_data.GetCount() - 1; i >= 0; --i) { - m_data[i]->AddGradients(*src->m_data[i]); + m_data[i]->Add(*src->m_data[i]); } } @@ -159,7 +165,7 @@ void ndBrainTrainer::ScaleWeights(const ndBrainFloat s) { for (ndInt32 i = m_data.GetCount() - 1; i >= 0; --i) { - m_data[i]->ScaleGradients(s); + m_data[i]->Scale(s); } } @@ -202,11 +208,12 @@ void ndBrainTrainer::BackPropagate(const ndBrainVector& input, ndBrainLoss& loss for (ndInt32 i = m_data.GetCount() - 1; i >= 0; --i) { - ndBrainLayer* const layer = m_data[i]->m_layer; + const ndBrainLayer* const layer = m_data[i]->m_layer; gradientIn.SetSize(layer->GetInputSize()); const ndBrainMemVector in(memBuffer + prefixScan[i + 0], layer->GetInputSize()); const ndBrainMemVector out(memBuffer + prefixScan[i + 1], layer->GetOutputSize()); - layer->CalculateParamGradients(in, out, gradientOut, gradientIn, m_data[i]->m_gradBias, m_data[i]->m_gradWeight); + //layer->CalculateParamGradients(in, out, gradientOut, gradientIn, m_data[i]->m_gradBias, m_data[i]->m_gradWeight); + layer->CalculateParamGradients(in, out, gradientOut, gradientIn, m_data[i]->m_gradient); gradientIn.Swap(gradientOut); } } diff --git a/newton-4.00/sdk/dBrain/ndBrainTrainer.h b/newton-4.00/sdk/dBrain/ndBrainTrainer.h index 80aa228e1d..b32d1c1156 100644 --- a/newton-4.00/sdk/dBrain/ndBrainTrainer.h +++ b/newton-4.00/sdk/dBrain/ndBrainTrainer.h @@ -41,10 +41,8 @@ class ndBrainTrainer: public ndClassAlloc void BackPropagate(const ndBrainVector& input, ndBrainLoss& loss); void AcculumateGradients(const ndBrainTrainer& src, ndInt32 index); - ndBrainVector* GetBias(ndInt32 index) const; - ndBrainMatrix* GetWeight(ndInt32 index) const; - ndBrainVector* GetBiasGradients(ndInt32 index) const; - ndBrainMatrix* GetWeightGradients(ndInt32 index) const; + ndBrainLayer* GetWeightsLayer(ndInt32 index) const; + ndBrainLayer* GetGradientLayer(ndInt32 index) const; void ClearGradients(); void ScaleWeights(const ndBrainFloat s);