add example: MLP on mnist

src/CMakeLists.txt

@@ -46,3 +46,5 @@ cc_library(tape_function SRCS function.cc DEPS ${GLOB_OP_LIB} tape_variable tape
 cc_test(test_tape
         SRCS test_tape.cc
         DEPS tape tape_variable tape_function)
+
+add_subdirectory(example)

src/example/CMakeLists.txt

@@ -0,0 +1,16 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_subdirectory(mnist)

src/example/mnist/CMakeLists.txt

@@ -0,0 +1,18 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+cc_test(test_mnist
+        SRCS test_mnist.cc
+        DEPS tape tape_variable tape_function)

src/data/create_recordio.py → src/example/mnist/create_mnist_recordio.py

@@ -30,7 +30,7 @@ def create_mnist_recordio_files():
             ],
             place=fluid.CPUPlace())
         fluid.recordio_writer.convert_reader_to_recordio_file(
-            '/tape/src/data/mnist.recordio', reader, feeder)
+            '/tmp/mnist.recordio', reader, feeder)
 
 
 if __name__ == "__main__":

src/example/mnist/test_mnist.cc

@@ -0,0 +1,87 @@
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fstream>
+
+#include "gtest/gtest.h"
+#include "src/function.h"
+
+using paddle::tape::Linear;
+using paddle::tape::SGD;
+using paddle::tape::mean;
+using paddle::tape::softmax;
+using paddle::tape::cross_entropy;
+using paddle::tape::reset_global_tape;
+using paddle::tape::get_global_tape;
+
+using paddle::tape::CreateRecordioFileReader;
+using paddle::tape::ReadNext;
+
+bool is_file_exist(const std::string& fileName) {
+  std::ifstream infile(fileName);
+  return infile.good();
+}
+
+TEST(Mnist, TestCPU) {
+  std::string filename = "/tmp/mnist.recordio";
+  PADDLE_ENFORCE(is_file_exist(filename),
+                 "file doesn't exist; have you run create_mnist_recordio.py");
+  auto reader = CreateRecordioFileReader(
+      filename, {32, 1, 28, 28, 32, 1}, {4, 2}, {0, 0});
+
+  Linear linear1(784, 200, "relu");
+  Linear linear2(200, 200, "relu");
+  Linear linear3(200, 10, "relu");
+  SGD sgd(0.001);
+
+  int print_step = 100;
+  float avg_loss = 0.0;
+
+  for (int i = 0; i < 1000; ++i) {
+    reset_global_tape();
+    auto data_label = ReadNext(reader);
+    auto data = data_label[0];
+    auto label = data_label[1];
+
+    auto predict = softmax(linear3(linear2(linear1(data))));
+    auto loss = mean(cross_entropy(predict, label));
+    if (i % print_step == 0) {
+      avg_loss +=
+          loss->Value().Get<paddle::framework::LoDTensor>().data<float>()[0];
+      LOG(INFO) << avg_loss;
+      avg_loss = 0;
+    }
+
+    get_global_tape().Backward(loss);
+
+    for (auto w : linear1.Params()) {
+      sgd.Update(w);
+    }
+    for (auto w : linear2.Params()) {
+      sgd.Update(w);
+    }
+    for (auto w : linear3.Params()) {
+      sgd.Update(w);
+    }
+  }
+}
+
+int main(int argc, char** argv) {
+  std::vector<paddle::platform::Place> places;
+  places.emplace_back(paddle::platform::CPUPlace());
+  paddle::platform::DeviceContextPool::Init(places);
+
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

src/function.h

@@ -41,12 +41,9 @@ class Fill {
 
   void operator()(VariableHandle var) {
     if (initializer_ == "fill_constant") {
-      // fill_constant is not OperatorWithKernel, so we can't add it to the tape
-      framework::OpDesc op_desc =
-          CreateOpDesc(initializer_, {}, {{"Out", {var}}}, attrs_);
-      ScopeWrapper scope({}, {{"Out", {var}}});
-      framework::OpRegistry::CreateOp(op_desc)->Run(scope,
-                                                    platform::CPUPlace());
+      PADDLE_THROW(
+          "fill_constant is not supported, since it is not of type "
+          "OperatorWithKernel");
     } else {
       get_global_tape().AddOp(initializer_, {}, {{"Out", {var}}}, attrs_);
     }
@@ -73,16 +70,11 @@ void init_params(VariableHandle v,
   }
 }
 
-// TODO(tonyyang-svail): change this to a function
-// https://github.com/PaddlePaddle/tape/issues/23
-class Mean {
- public:
-  VariableHandle operator()(VariableHandle var) {
-    VariableHandle out(new Variable("mean"));
-    get_global_tape().AddOp("mean", {{"X", {var}}}, {{"Out", {out}}}, {});
-    return out;
-  }
-};
+VariableHandle mean(VariableHandle x) {
+  VariableHandle out(new Variable("mean"));
+  get_global_tape().AddOp("mean", {{"X", {x}}}, {{"Out", {out}}}, {});
+  return out;
+}
 
 VariableHandle relu(VariableHandle x) {
   VariableHandle out(new Variable("relu"));
@@ -253,11 +245,27 @@ VariableHandle CreateRecordioFileReader(std::string filename,
   return reader;
 }
 
-void ReadNext(VariableHandle reader, VariableHandle data_holder) {
+std::vector<VariableHandle> ReadNext(VariableHandle reader) {
   PADDLE_ENFORCE(reader->Var().IsType<framework::ReaderHolder>());
 
-  reader->GetMutable<paddle::framework::ReaderHolder>()->ReadNext(
-      data_holder->GetMutable<paddle::framework::LoDTensorArray>());
+  paddle::framework::LoDTensorArray data_holder;
+  reader->GetMutable<paddle::framework::ReaderHolder>()->ReadNext(&data_holder);
+  if (data_holder.empty()) {
+    reader->GetMutable<paddle::framework::ReaderHolder>()->ReInit();
+    reader->GetMutable<paddle::framework::ReaderHolder>()->ReadNext(
+        &data_holder);
+  }
+  PADDLE_ENFORCE(!data_holder.empty(), "Error reading file.");
+
+  std::vector<VariableHandle> rval;
+  for (size_t i = 0; i < data_holder.size(); ++i) {
+    rval.emplace_back(new Variable("data" + std::to_string(i)));
+    auto *lod_tensor = rval.back()->GetMutable<framework::LoDTensor>();
+    lod_tensor->ShareDataWith(data_holder[i]);
+    lod_tensor->set_lod(data_holder[i].lod());
+  }
+
+  return rval;
 }
 
 }  // namespace tape

src/tape.cc

@@ -123,7 +123,7 @@ void Tape::AddOp(const std::string &type,
 }
 
 void Tape::Forward() {
-  LOG(INFO) << "Starting forward -------------------------";
+  VLOG(3) << "Starting forward -------------------------";
   PADDLE_ENFORCE(!has_been_backwarded_);
   while (current_position_ < tape_.size()) {
     OpHandle &op = tape_[current_position_];
@@ -134,7 +134,7 @@ void Tape::Forward() {
     current_position_++;
   }
 
-  LOG(INFO) << "Finishing forward -------------------------";
+  VLOG(3) << "Finishing forward -------------------------";
 }
 
 void Tape::Backward(VariableHandle target) {

src/test_tape.cc

@@ -19,53 +19,29 @@ using paddle::tape::VariableHandle;
 using paddle::tape::Variable;
 using paddle::tape::Linear;
 using paddle::tape::Convolution2D;
-using paddle::tape::Mean;
 using paddle::tape::SGD;
 using paddle::tape::Fill;
+using paddle::tape::mean;
 using paddle::tape::softmax;
 using paddle::tape::cross_entropy;
 using paddle::tape::reset_global_tape;
 using paddle::tape::get_global_tape;
 using paddle::tape::CreateRecordioFileReader;
 using paddle::tape::ReadNext;
 
-TEST(Tape, TestReader) {
-  VariableHandle data_label(new paddle::tape::Variable("data_label"));
-  VariableHandle reader = CreateRecordioFileReader(
-      "/tape/src/data/mnist.recordio", {32, 1, 28, 28, 32, 1}, {4, 2}, {0, 0});
-  ReadNext(reader, data_label);
-  LOG(INFO) << *data_label;
-}
-
-TEST(Tape, TestRelu) {
-  std::string initializer = "uniform_random";
-  paddle::framework::AttributeMap attrs;
-  attrs["min"] = -1.0f;
-  attrs["max"] = 1.0f;
-  attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32;
-  attrs["shape"] = std::vector<int>{10};
-  attrs["seed"] = 123;
-  Fill filler(initializer, attrs);
-
-  VariableHandle input(new Variable("input"));
-  filler(input);
-  auto loss = relu(input);
-  LOG(INFO) << input->Value();
-  LOG(INFO) << loss->Value();
-}
-
 TEST(Tape, TestConv) {
   Convolution2D conv1(3, 16, 3, "relu");
   Convolution2D conv2(16, 1, 3, "relu");
-  Mean mean;
 
   SGD sgd(0.001);
 
-  std::string initializer = "fill_constant";
+  std::string initializer = "uniform_random";
   paddle::framework::AttributeMap attrs;
+  attrs["min"] = -1.0f;
+  attrs["max"] = 1.0f;
   attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32;
+  attrs["seed"] = 123;
   attrs["shape"] = std::vector<int>{32, 3, 8, 8};
-  attrs["value"] = 1.0f;
   Fill filler(initializer, attrs);
 
   for (int i = 0; i < 2; ++i) {
@@ -90,15 +66,16 @@ TEST(Tape, TestConv) {
 TEST(Tape, TestMLP) {
   Linear linear1(3, 3, "relu");
   Linear linear2(3, 3, "relu");
-  Mean mean;
 
   SGD sgd(0.001);
 
-  std::string initializer = "fill_constant";
+  std::string initializer = "uniform_random";
   paddle::framework::AttributeMap attrs;
+  attrs["min"] = -1.0f;
+  attrs["max"] = 1.0f;
   attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32;
+  attrs["seed"] = 123;
   attrs["shape"] = std::vector<int>{3, 3};
-  attrs["value"] = 1.0f;
   Fill filler(initializer, attrs);
 
   for (int i = 0; i < 2; ++i) {
@@ -121,7 +98,7 @@ TEST(Tape, TestMLP) {
   }
 }
 
-int main(int argc, char **argv) {
+int main(int argc, char** argv) {
   std::vector<paddle::platform::Place> places;
   places.emplace_back(paddle::platform::CPUPlace());
   paddle::platform::DeviceContextPool::Init(places);

src/variable.h

@@ -42,7 +42,7 @@ class Variable {
       : name_(pre_fix + (is_grad ? framework::kGradVarSuffix
                                  : std::to_string(count()))) {}
 
-  ~Variable() { LOG(INFO) << "Deleting " << Name(); }
+  ~Variable() { VLOG(10) << "Deleting " << Name(); }
 
   VariableHandle Grad() {
     if (grad_.expired()) {