Make Lint happy and improve small things, i.e. mdb_mapsize = 1 for re…

…ading

include/caffe/datum_DB.hpp

@@ -3,6 +3,7 @@
 
 #include <algorithm>
 #include <iterator>
+#include <map>
 #include <string>
 #include <utility>
 #include <vector>
@@ -70,7 +71,7 @@ class DatumLevelDB : public DatumDB {
   virtual void Put(const string& key, const Datum& datum);
   virtual void Commit();
 
-protected:
+ protected:
   virtual void Open();
   virtual void Close();
 
@@ -96,7 +97,7 @@ class DatumLMDB : public DatumDB {
   virtual void Put(const string& key, const Datum& datum);
   virtual void Commit();
 
-protected:
+ protected:
   virtual void Open();
   virtual void Close();
 
@@ -124,15 +125,15 @@ class DatumImagesDB : public DatumDB {
   virtual void Put(const string& key, const Datum& datum);
   virtual void Commit();
 
-protected:
+ protected:
   virtual void Open();
   virtual void Close();
 
   std::fstream file_;
   string root_images_;
   bool cache_images_;
   bool encode_images_;
-  
+
   vector<string> keys_;
   vector<string>::iterator read_it_;
   map<string, Datum> datum_database_;
@@ -141,4 +142,4 @@ class DatumImagesDB : public DatumDB {
 
 }  // namespace caffe
 
-#endif  // CAFFE_DATUMDB_HPP
+#endif  // CAFFE_DATUMDB_HPP

src/caffe/datum_DB.cpp

@@ -1,3 +1,5 @@
+#include <string>
+
 #include "caffe/datum_DB.hpp"
 
 namespace caffe {
@@ -36,4 +38,4 @@ DatumDBParameter_Backend DatumDB::GetBackend(const string& backend) {
   return enum_backend;
 }
 
-}  // namespace caffe
+}  // namespace caffe

src/caffe/datum_imagesdb.cpp

@@ -1,3 +1,4 @@
+#include <map>
 #include <string>
 #include <vector>
 
@@ -88,7 +89,7 @@ bool DatumImagesDB::Next() {
           LOG(INFO) << "Shuffling the keys";
           shuffle(keys_.begin(), keys_.end());
         }
-        LOG(INFO) << "Reached the end and looping.";
+        DLOG(INFO) << "Reached the end and looping.";
         SeekToFirst();
       } else {
         LOG(ERROR) << "Reached the end and not looping.";
@@ -107,7 +108,7 @@ bool DatumImagesDB::Prev() {
           LOG(INFO) << "Shuffling the keys";
           shuffle(keys_.begin(), keys_.end());
         }
-        LOG(INFO) << "Reached the beginning and looping.";
+        DLOG(INFO) << "Reached the beginning and looping.";
         SeekToLast();
       } else {
         LOG(ERROR) << "Reached the beginning and not looping.";
@@ -122,7 +123,7 @@ bool DatumImagesDB::Prev() {
 
 bool DatumImagesDB::SeekToFirst() {
   if (!keys_.empty()) {
-    read_it_ = keys_.begin();  
+    read_it_ = keys_.begin();
   }
   return Valid();
 }

src/caffe/datum_leveldb.cpp

@@ -58,7 +58,7 @@ bool DatumLevelDB::Next() {
     if (!iter_->Valid()) {
       if (param_.loop()) {
         // We have reached the end. Restart from the first.
-        LOG(INFO) << "Reached the end and looping.";
+        DLOG(INFO) << "Reached the end and looping.";
         iter_->SeekToFirst();
       } else {
         LOG(ERROR) << "Reached the end and not looping.";
@@ -74,13 +74,13 @@ bool DatumLevelDB::Prev() {
     if (!iter_->Valid()) {
       if (param_.loop()) {
         // We have reached the end. Restart from the first.
-        LOG(INFO) << "Reached the beginning and looping.";
+        DLOG(INFO) << "Reached the beginning and looping.";
         iter_->SeekToLast();
       } else {
         LOG(ERROR) << "Reached the beginning and not looping.";
       }
     }
-  }   
+  }
   return Valid();
 }
 

src/caffe/datum_lmdb.cpp

@@ -10,15 +10,13 @@ void DatumLMDB::Open() {
   LOG(INFO) << "Opening lmdb " << param_.source();
   mdb_status_ = mdb_env_create(&mdb_env_);
   CHECK_EQ(mdb_status_, MDB_SUCCESS) << mdb_strerror(mdb_status_);
-  mdb_status_ = mdb_env_set_mapsize(mdb_env_, 1099511627776); // 1TB
-  CHECK_EQ(mdb_status_, MDB_SUCCESS) << mdb_strerror(mdb_status_);
-  mdb_status_ = mdb_env_set_maxreaders(mdb_env_, 126); // 126 Readers
-  CHECK_EQ(mdb_status_, MDB_SUCCESS) << mdb_strerror(mdb_status_);
   switch (param_.mode()) {
   case DatumDBParameter_Mode_NEW:
     CHECK_EQ(mkdir(param_.source().c_str(), 0744), 0)
       << "mkdir " << param_.source() << "failed";
   case DatumDBParameter_Mode_WRITE:
+    mdb_status_ = mdb_env_set_mapsize(mdb_env_, param_.mdb_env_mapsize());
+    CHECK_EQ(mdb_status_, MDB_SUCCESS) << mdb_strerror(mdb_status_);
     mdb_status_ = mdb_env_open(mdb_env_, param_.source().c_str(), 0, 0664);
     CHECK_EQ(mdb_status_, MDB_SUCCESS) << mdb_strerror(mdb_status_);
     mdb_status_ = mdb_txn_begin(mdb_env_, NULL, 0, &mdb_txn_);
@@ -27,6 +25,10 @@ void DatumLMDB::Open() {
     CHECK_EQ(mdb_status_, MDB_SUCCESS) << mdb_strerror(mdb_status_);
     break;
   case DatumDBParameter_Mode_READ:
+    mdb_status_ = mdb_env_set_mapsize(mdb_env_, 1);
+    CHECK_EQ(mdb_status_, MDB_SUCCESS) << mdb_strerror(mdb_status_);
+    mdb_status_ = mdb_env_set_maxreaders(mdb_env_, param_.mdb_env_maxreaders());
+    CHECK_EQ(mdb_status_, MDB_SUCCESS) << mdb_strerror(mdb_status_);
     mdb_status_ = mdb_env_open(mdb_env_, param_.source().c_str(),
                   MDB_RDONLY|MDB_NOTLS, 0664);
     CHECK_EQ(mdb_status_, MDB_SUCCESS) << mdb_strerror(mdb_status_);
@@ -59,7 +61,7 @@ bool DatumLMDB::Next() {
   if (mdb_status_ != MDB_SUCCESS) {
     if (param_.loop()) {
       SeekToFirst();
-      LOG(INFO) << "Reached the end and looping.";
+      DLOG(INFO) << "Reached the end and looping.";
     } else {
       LOG(ERROR) << "Reached the end and not looping.";
     }
@@ -73,7 +75,7 @@ bool DatumLMDB::Prev() {
   if (mdb_status_ != MDB_SUCCESS) {
     if (param_.loop()) {
       SeekToLast();
-      LOG(INFO) << "Reached the beginning and looping.";
+      DLOG(INFO) << "Reached the beginning and looping.";
     } else {
       LOG(ERROR) << "Reached the beginning and not looping.";
     }
@@ -123,7 +125,7 @@ bool DatumLMDB::Get(const string& key, Datum* datum) {
     return true;
   } else {
     LOG(ERROR) << "key " << key << " not found";
-    return false;  
+    return false;
   }
 }
 

src/caffe/proto/caffe.proto

@@ -462,16 +462,28 @@ message DatumDBParameter {
     WRITE = 1;
     READ = 2;
   }
-  // Specify the data source.
+  // Specify the datumDB Backend.
   optional Backend backend = 1 [default = LEVELDB];
+  // Specify the datumDB mode.
   optional Mode mode = 2 [default = READ];
+  // Specify the datumDB source.
   optional string source = 3;
+  // If true it will loop around the datumDB when needed
   optional bool loop = 4 [default = true];
+  // Only used by LMDB
+  // Max size of LMDB, by default 1TB
+  optional uint64 mdb_env_mapsize = 5 [default = 1099511627776];
+  // Max number of readers, by default 126
+  optional uint32 mdb_env_maxreaders = 6 [default = 126];
   // Only used by IMAGESDB
-  optional string root_images = 5 [default = ""];
-  optional bool cache_images = 6 [default = false];
-  optional bool encode_images = 7 [default = false];
-  optional bool shuffle_images = 8 [default = false];
+  // Prefix used to locate the images
+  optional string root_images = 7 [default = ""];
+  // If true it will store the images in memory, be careful with memory
+  optional bool cache_images = 8 [default = false];
+  // If true it will use encoded images, just copy the file for later decoding
+  optional bool encode_images = 9 [default = false];
+  // If true it will shuffle the list of images
+  optional bool shuffle_images = 10 [default = false];
 }
 
 // Message that stores parameters used by DropoutLayer

src/caffe/test/test_datum_DB.cpp

@@ -134,7 +134,7 @@ TYPED_TEST(DatumDBTest, TestSeekToFirst) {
   string key;
   Datum datum;
   EXPECT_TRUE(datumdb->Current(&key, &datum));
-  EXPECT_TRUE(key == "cat.jpg");
+  EXPECT_EQ(key, "cat.jpg");
   EXPECT_EQ(datum.channels(), 3);
   EXPECT_EQ(datum.height(), 360);
   EXPECT_EQ(datum.width(), 480);
@@ -152,7 +152,7 @@ TYPED_TEST(DatumDBTest, TestSeekToLast) {
   string key;
   Datum datum;
   EXPECT_TRUE(datumdb->Current(&key, &datum));
-  EXPECT_TRUE(key == "fish-bike.jpg");
+  EXPECT_EQ(key, "fish-bike.jpg");
   EXPECT_EQ(datum.channels(), 3);
   EXPECT_EQ(datum.height(), 323);
   EXPECT_EQ(datum.width(), 481);
@@ -168,24 +168,24 @@ TYPED_TEST(DatumDBTest, TestCurrent) {
   string key;
   Datum datum;
   EXPECT_TRUE(datumdb->Current(&key, &datum));
-  EXPECT_TRUE(key == "cat.jpg");
+  EXPECT_EQ(key, "cat.jpg");
   EXPECT_EQ(datum.channels(), 3);
   EXPECT_EQ(datum.height(), 360);
   EXPECT_EQ(datum.width(), 480);
   EXPECT_TRUE(datumdb->Current(&key, &datum));
-  EXPECT_TRUE(key == "cat.jpg");
+  EXPECT_EQ(key, "cat.jpg");
   EXPECT_EQ(datum.channels(), 3);
   EXPECT_EQ(datum.height(), 360);
   EXPECT_EQ(datum.width(), 480);
   EXPECT_TRUE(datumdb->Next());
   EXPECT_TRUE(datumdb->Current(&key, &datum));
-  EXPECT_TRUE(key == "fish-bike.jpg");
+  EXPECT_EQ(key, "fish-bike.jpg");
   EXPECT_EQ(datum.channels(), 3);
   EXPECT_EQ(datum.height(), 323);
   EXPECT_EQ(datum.width(), 481);
   EXPECT_TRUE(datumdb->Prev());
   EXPECT_TRUE(datumdb->Current(&key, &datum));
-  EXPECT_TRUE(key == "cat.jpg");
+  EXPECT_EQ(key, "cat.jpg");
   EXPECT_EQ(datum.channels(), 3);
   EXPECT_EQ(datum.height(), 360);
   EXPECT_EQ(datum.width(), 480);
@@ -217,10 +217,10 @@ TYPED_TEST(DatumDBTest, TestCurrentKey) {
   shared_ptr<DatumDB> datumdb = DatumDB::GetDatumDB(param);
   string key;
   key = datumdb->CurrentKey();
-  EXPECT_TRUE(key == "cat.jpg");
+  EXPECT_EQ(key, "cat.jpg");
   EXPECT_TRUE(datumdb->Next());
   key = datumdb->CurrentKey();
-  EXPECT_TRUE(key == "fish-bike.jpg");
+  EXPECT_EQ(key, "fish-bike.jpg");
 }
 
 TYPED_TEST(DatumDBTest, TestCurrentDatum) {