diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..412eeda78 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,22 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# Custom for Visual Studio +*.cs diff=csharp +*.sln merge=union +*.csproj merge=union +*.vbproj merge=union +*.fsproj merge=union +*.dbproj merge=union + +# Standard to msysgit +*.doc diff=astextplain +*.DOC diff=astextplain +*.docx diff=astextplain +*.DOCX diff=astextplain +*.dot diff=astextplain +*.DOT diff=astextplain +*.pdf diff=astextplain +*.PDF diff=astextplain +*.rtf diff=astextplain +*.RTF diff=astextplain diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..5ebd21a16 --- /dev/null +++ b/.gitignore @@ -0,0 +1,163 @@ +################# +## Eclipse +################# + +*.pydevproject +.project +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.classpath +.settings/ +.loadpath + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# CDT-specific +.cproject + +# PDT-specific +.buildpath + + +################# +## Visual Studio +################# + +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# User-specific files +*.suo +*.user +*.sln.docstates + +# Build results +[Dd]ebug/ +[Rr]elease/ +*_i.c +*_p.c +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.vspscc +.builds +*.dotCover + +## TODO: If you have NuGet Package Restore enabled, uncomment this +#packages/ + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opensdf +*.sdf + +# Visual Studio profiler +*.psess +*.vsp + +# ReSharper is a .NET coding add-in +_ReSharper* + +# Installshield output folder +[Ee]xpress + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish + +# Others +[Bb]in +[Oo]bj +sql +TestResults +*.Cache +ClientBin +stylecop.* +~$* +*.dbmdl +Generated_Code #added for RIA/Silverlight projects + +# Backup & report files from converting an old project file to a newer +# Visual Studio version. Backup files are not needed, because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML + + + +############ +## Windows +############ + +# Windows image file caches +Thumbs.db + +# Folder config file +Desktop.ini + + +############# +## Python +############# + +*.py[co] + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox + +#Translations +*.mo + +#Mr Developer +.mr.developer.cfg + +# Mac crap +.DS_Store diff --git a/include/activation.h b/include/activation.h new file mode 100644 index 000000000..0f2c4877c --- /dev/null +++ b/include/activation.h @@ -0,0 +1,19 @@ +#pragma once + +namespace nn { + +struct sigmoid_activation { + static double f(double x) { return 1.0 / (1.0 + std::exp(-x)); } + static double df(double f_x) { return f_x * (1.0 - f_x); } +}; + +struct tanh_activation { + static double f(double x) { + const double ep = std::exp(x); + const double em = std::exp(-x); + return (ep - em) / (ep + em); + } + static double df(double f_x) { return f_x * (1.0 - f_x); } +}; + +} diff --git a/include/cnn.h b/include/cnn.h new file mode 100644 index 000000000..3e67cce67 --- /dev/null +++ b/include/cnn.h @@ -0,0 +1,121 @@ +#pragma once +#include +#include +#include + +#include "util.h" +#include "activation.h" +#include "learner.h" +#include "layer.h" + +namespace nn { + +class cnn { +public: + cnn(double alpha, double lambda) : lambda_(lambda) { + learner_ = new gradient_descent(alpha); + } + + cnn(learner *l, double lambda) : lambda_(lambda), learner_(l) {} + + ~cnn(){ + delete learner_; + } + + void add(layer *layer) { + layers_.add(layer); + //layer->unroll(¶ms_, &diffs_); + } + + int in_dim() const { return layers_.head()->in_dim(); } + int out_dim() const { return layers_.tail()->out_dim(); } + + void predict(const vec_t& in, vec_t *out) { + *out = *layers_.head()->forward_propagation(in); + } + + void train(const std::vector& in, const std::vector& training) { + calc_diff(in, training); + // update by delta and learning algorithm + learner_->update(layers_.all_param(), layers_.all_diff()); + } + + bool check(const std::vector& in, const std::vector& training) { + const int dim = layers_.all_param().size(); + vec_t diff1(dim), diff2(dim); + + calc_diff(in, training); + for (int i = 0; i < dim; i++) + diff1[i] = *layers_.all_diff()[i]; + + calc_diff_numeric(in, training); + for (int i = 0; i < dim; i++) + diff2[i] = *layers_.all_diff()[i]; + + float_t diff = sum_square(diff1 - diff2) / dim; + return diff < 1E-5; + } + + float_t loss_function(const std::vector& in, const std::vector& training) { + const int m = in.size(); + float_t loss_score = 0.0; + float_t norm_score = 0.0; + + for (int i = 0; i < m; i++) { + layers_.head()->forward_propagation(in[i]); + loss_score += sum_square(layers_.tail()->output() - training[i]); + } + loss_score /= (2 * m); + + norm_score = lambda_ * sum_square(layers_.weight()) / 2.0; // biasは含めない + return loss_score + norm_score; + } + +private: + void calc_diff(const std::vector& in, const std::vector& training) { + const int m = in.size(); + layers_.reset_diff(); + + for (int i = 0; i < m; i++) { + layers_.head()->forward_propagation(in[i]); + layers_.tail()->back_propagation(in[i], training[i]); + } + + pvec_t& w = layers_.weight(); + pvec_t& dw = layers_.weight_diff(); + for (size_t i = 0; i < w.size(); i++) + *dw[i] = *dw[i] / m + lambda_ * *w[i]; + + pvec_t& b = layers_.bias(); + pvec_t& db = layers_.bias_diff(); + for (size_t i = 0; i < b.size(); i++) + *db[i] = *db[i] / m; + } + + void calc_diff_numeric(const std::vector& in, const std::vector& training) { + static const float_t EPSILON = 1e-4; + const int m = in.size(); + layers_.reset_diff(); + + const int dim = layers_.all_param().size(); + + for (int i = 0; i < dim; i++) { + const float_t v = *layers_.all_param()[i]; + + *layers_.all_param()[i] = v + EPSILON; + const float_t Jp = loss_function(in, training); + + *layers_.all_param()[i] = v - EPSILON; + const float_t Jm = loss_function(in, training); + + const float_t diff = (Jp - Jm) / (2.0 * EPSILON); + *layers_.all_diff()[i] = diff; + } + } + + const double lambda_; // weight decay + layers layers_; + learner *learner_; +}; + +} diff --git a/include/convolutional_layer.h b/include/convolutional_layer.h new file mode 100644 index 000000000..345400e33 --- /dev/null +++ b/include/convolutional_layer.h @@ -0,0 +1,3 @@ +#pragma once +#include "util.h" +#include "cnn.h" \ No newline at end of file diff --git a/include/fully_connected_layer.h b/include/fully_connected_layer.h new file mode 100644 index 000000000..e99e46c0d --- /dev/null +++ b/include/fully_connected_layer.h @@ -0,0 +1,76 @@ +#pragma once +#include "cnn.h" + +namespace nn { + +// normal +template +class neural_net : public layer { +public: + neural_net() : layer(In, Out, In*Out+Out){ + W_.resize(In*Out); + b_.resize(Out); + dW_.resize(In*Out); + dB_.resize(Out); + } + + int in_dim() const { return In; } + int out_dim() const { return Out; } + int param_dim() const { return In*Out+Out; } + + const vec_t* forward_propagation(const vec_t& in) { + for (int r = 0; r < Out; r++) { + float_t z = 0.0; + for (int c = 0; c < In; c++) + z += W_[r*In+c] * in[c]; + z += b_[r]; + output_[r] = Activation::f(z); + } + + return next_ ? next_->forward_propagation(output_) : &output_; + } + + const vec_t* back_propagation(const vec_t& in, const vec_t& train_signal) { + if (!next_) { + for (int i = 0; i < Out; i++) + delta_[i] = (output_[i] - train_signal[i]) * Activation::df(output_[i]); + } + + const vec_t& prev_out = prev_ ? prev_->output() : in; + for (int c = 0; c < In; c++) + for (int r = 0; r < Out; r++) + dW_[r*In+c] += delta_[r] * prev_out[c]; + for (int r = 0; r < Out; r++) + dB_[r] += delta_[r]; + + if (!prev_) return &delta_; + + for (int c = 0; c < In; c++) { + prev_->delta()[c] = 0.0; + for (int r = 0; r < Out; r++) + prev_->delta()[c] += delta_[r] * W_[r*In+c]; + prev_->delta()[c] *= Activation::df(prev_->output()[c]); + } + + return prev_->back_propagation(in, train_signal); + } + + void unroll(pvec_t *w, pvec_t *dw, pvec_t *b, pvec_t *db) { + for (size_t i = 0; i < W_.size(); i++) + w->push_back(&W_[i]); + for (size_t i = 0; i < b_.size(); i++) + b->push_back(&b_[i]); + for (size_t i = 0; i < dW_.size(); i++) + dw->push_back(&dW_[i]); + for (size_t i = 0; i < dB_.size(); i++) + db->push_back(&dB_[i]); + } + +private: + vec_t dW_; + vec_t dB_; + vec_t W_; + vec_t b_; +}; + +} diff --git a/include/layer.h b/include/layer.h new file mode 100644 index 000000000..e7825155b --- /dev/null +++ b/include/layer.h @@ -0,0 +1,87 @@ +#pragma once +#include "util.h" + +namespace nn { + +class layers; + +// base class of all kind of NN layers +class layer { +public: + layer(int in, int out, int param) : next_(0), prev_(0) { + output_.resize(out); + delta_.resize(out); + } + void connect(layer* tail) { + if (tail->in_dim() != this->out_dim()) + throw std::domain_error("dimension mismatch"); + next_ = tail; + tail->prev_ = this; + } + vec_t& output() { return output_; } + vec_t& delta() { return delta_; } + + virtual int in_dim() const = 0; // 入力次元 + virtual int out_dim() const = 0; // 出力次元 + virtual int param_dim() const = 0; // パラメータ次元 + + // in: 前層の出力。in.size() == in_dim() + // ret: 出力層の出力ベクトル。 + virtual const vec_t* forward_propagation(const vec_t& in) = 0; + + // in: 出力層の出力ベクトル。 train_signal: 教師ベクトル + // ret: 入力層のδ + virtual const vec_t* back_propagation(const vec_t& in, const vec_t& train_signal) = 0; + + virtual void unroll(pvec_t *w, pvec_t *dw, pvec_t *b, pvec_t *db) = 0; + +protected: + friend class layers; + layer* next_; + layer* prev_; + vec_t output_; + vec_t delta_; +}; + +class layers { +public: + layers() : head_(0), tail_(0) {} + void add(layer * new_tail) { + if (!head_) head_ = new_tail; + if (tail_) tail_->connect(new_tail); + tail_ = new_tail; + + new_tail->unroll(&w_, &dw_, &b_, &db_); + unroll(); + } + bool empty() const { return head_ == 0; } + layer* head() const { return head_; } + layer* tail() const { return tail_; } + pvec_t& all_param() { return wb_; } + pvec_t& all_diff() { return dwb_; } + pvec_t& weight() { return w_; } + pvec_t& bias() { return b_; } + pvec_t& weight_diff() { return dw_; } + pvec_t& bias_diff() { return db_; } + void reset_diff() { + for (auto d : dwb_) + *d = 0.0; + } +private: + void unroll() { + wb_ = w_; + std::copy(b_.begin(), b_.end(), std::back_inserter(wb_)); + dwb_ = dw_; + std::copy(db_.begin(), db_.end(), std::back_inserter(dwb_)); + } + layer *head_; + layer *tail_; + pvec_t w_; + pvec_t b_; + pvec_t wb_; + pvec_t dw_; + pvec_t db_; + pvec_t dwb_; +}; + +} diff --git a/include/learner.h b/include/learner.h new file mode 100644 index 000000000..75a78dc4c --- /dev/null +++ b/include/learner.h @@ -0,0 +1,23 @@ +#pragma once + +namespace nn { + +class learner { +public: + virtual void update(pvec_t& target, const pvec_t& diff) = 0; +}; + +class gradient_descent : public learner { +public: + gradient_descent(double alpha) : alpha_(alpha){} + + void update(pvec_t& target, const pvec_t& diff) { + for (size_t i = 0; i < diff.size(); i++) + *target[i] = *target[i] - alpha_ * *diff[i]; + } + +private: + double alpha_; // learning rate +}; + +} diff --git a/include/util.h b/include/util.h new file mode 100644 index 000000000..dadf8b6e7 --- /dev/null +++ b/include/util.h @@ -0,0 +1,51 @@ +#pragma once +#include + +namespace nn { + +typedef double float_t; +typedef std::vector vec_t; +typedef std::vector pvec_t; +typedef std::vector mat_t; + +template +T& operator << (T& os, const std::vector& vec) { + + for (size_t i = 0; i < vec.size(); i++) + os << vec[i] << (i == vec.size() - 1 ? ',' : ' '); + return os; +} + +vec_t operator - (const vec_t& v1, const vec_t& v2) { + const int dim = v1.size(); + vec_t v(dim); + + for (int i = 0; i < dim; i++) + v[i] = v1[i] - v2[i]; + return v; +} + +vec_t operator + (const vec_t& v1, const vec_t& v2) { + const int dim = v1.size(); + vec_t v(dim); + + for (int i = 0; i < dim; i++) + v[i] = v1[i] + v2[i]; + return v; +} + +float_t sum_square(const vec_t& vec) { + float_t sum = 0.0; + for (auto v : vec) + sum += v * v; + return sum; +} + +float_t sum_square(const pvec_t& vec) { + float_t sum = 0.0; + for (auto v : vec) + sum += *v * *v; + return sum; +} + +} diff --git a/src/main.cpp b/src/main.cpp new file mode 100644 index 000000000..3b601887f --- /dev/null +++ b/src/main.cpp @@ -0,0 +1,30 @@ +#include +#include "cnn.h" +#include "fully_connected_layer.h" + +using namespace nn; + +int main(void) { + cnn nn(0.3, 0.3); + neural_net<3, 2> layer; + nn.add(&layer); + + + vec_t a(3, 0.0), t(2, 0.0); + + std::vector data, train; + + a[0] = 3.0; a[2] = -1.0; + t[0] = 0.3; t[1] = 0.7; + + data.push_back(a); + train.push_back(t); + nn.check(data, train); + for (int i = 0; i < 100; i++) + nn.train(data, train); + nn.check(data, train); + vec_t out; + nn.predict(a, &out); + + +} \ No newline at end of file diff --git a/src/wscript b/src/wscript new file mode 100644 index 000000000..e7fcadbfb --- /dev/null +++ b/src/wscript @@ -0,0 +1,6 @@ +def build(bld): + bld(features = 'cxx cprogram', + source = 'main.cpp', + target = 'main', + includes = '. ../include', + includes = ['.']) diff --git a/vc/cnn.sln b/vc/cnn.sln new file mode 100644 index 000000000..ef85e9015 --- /dev/null +++ b/vc/cnn.sln @@ -0,0 +1,20 @@ +ｻｿ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2012 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cnn", "cnn.vcxproj", "{C7B38F22-F235-45A5-834A-DE2167849C4C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {C7B38F22-F235-45A5-834A-DE2167849C4C}.Debug|Win32.ActiveCfg = Debug|Win32 + {C7B38F22-F235-45A5-834A-DE2167849C4C}.Debug|Win32.Build.0 = Debug|Win32 + {C7B38F22-F235-45A5-834A-DE2167849C4C}.Release|Win32.ActiveCfg = Release|Win32 + {C7B38F22-F235-45A5-834A-DE2167849C4C}.Release|Win32.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/vc/cnn.vcxproj b/vc/cnn.vcxproj new file mode 100644 index 000000000..182f97c03 --- /dev/null +++ b/vc/cnn.vcxproj @@ -0,0 +1,83 @@ +ｻｿ + + + + Debug + Win32 + + + Release + Win32 + + + + + + + + + + + + + + + + $(VCTargetsPath11) + + + {C7B38F22-F235-45A5-834A-DE2167849C4C} + cnn + + + + Application + true + v110 + MultiByte + + + Application + false + v110 + true + MultiByte + + + + + + + + + + + + + + + Level3 + Disabled + ..\include;C:\Program Files\boost\boost_1_51_0;C:\Program Files\eigen-eigen-5097c01bcdc4 + + + true + + + + + Level3 + MaxSpeed + true + true + + + true + true + true + + + + + + \ No newline at end of file