Skip to content

Commit

Permalink
Merge pull request #2523 from BonsaiAI/separate-io-dependencies
Browse files Browse the repository at this point in the history
Separate dependencies for configurable installation
jeffdonahue committed Sep 17, 2015

Unverified

This user has not yet uploaded their public signing key.
2 parents 71e0587 + 68c9e2b commit 5ff3734
Showing 40 changed files with 266 additions and 61 deletions.
11 changes: 6 additions & 5 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -2,11 +2,12 @@
# one using CMake, and one using make.
env:
matrix:
- WITH_CUDA=false WITH_CMAKE=false
- WITH_CUDA=false WITH_CMAKE=true
- WITH_CUDA=true WITH_CMAKE=false
- WITH_CUDA=true WITH_CMAKE=true
- WITH_CUDA=false WITH_CMAKE=true PYTHON_VERSION=3
- WITH_CUDA=false WITH_CMAKE=false WITH_IO=true
- WITH_CUDA=false WITH_CMAKE=true WITH_IO=true PYTHON_VERSION=3
- WITH_CUDA=true WITH_CMAKE=false WITH_IO=true
- WITH_CUDA=true WITH_CMAKE=true WITH_IO=true
- WITH_CUDA=false WITH_CMAKE=false WITH_IO=false
- WITH_CUDA=false WITH_CMAKE=true WITH_IO=false PYTHON_VERSION=3

language: cpp

9 changes: 6 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -16,13 +16,16 @@ include(cmake/ConfigGen.cmake)

# ---[ Options
caffe_option(CPU_ONLY "Build Caffe without CUDA support" OFF) # TODO: rename to USE_CUDA
caffe_option(USE_CUDNN "Build Caffe with cuDNN libary support" ON IF NOT CPU_ONLY)
caffe_option(USE_CUDNN "Build Caffe with cuDNN library support" ON IF NOT CPU_ONLY)
caffe_option(BUILD_SHARED_LIBS "Build shared libraries" ON)
caffe_option(BUILD_python "Build Python wrapper" ON)
set(python_version "2" CACHE STRING "Specify which python version to use")
set(python_version "2" CACHE STRING "Specify which Python version to use")
caffe_option(BUILD_matlab "Build Matlab wrapper" OFF IF UNIX OR APPLE)
caffe_option(BUILD_docs "Build documentation" ON IF UNIX OR APPLE)
caffe_option(BUILD_python_layer "Build the Caffe python layer" ON)
caffe_option(BUILD_python_layer "Build the Caffe Python layer" ON)
caffe_option(USE_LMDB "Build with lmdb" ON)
caffe_option(USE_LEVELDB "Build with levelDB" ON)
caffe_option(USE_OPENCV "Build with OpenCV support" ON)

# ---[ Dependencies
include(cmake/Dependencies.cmake)
28 changes: 24 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
@@ -169,9 +169,18 @@ ifneq ($(CPU_ONLY), 1)
LIBRARY_DIRS += $(CUDA_LIB_DIR)
LIBRARIES := cudart cublas curand
endif
LIBRARIES += glog gflags protobuf leveldb snappy \
lmdb boost_system hdf5_hl hdf5 m \
opencv_core opencv_highgui opencv_imgproc

LIBRARIES += glog gflags protobuf boost_system m hdf5_hl hdf5

ifeq ($(USE_LEVELDB), 1)
LIBRARIES += leveldb snappy
endif
ifeq ($(USE_LMDB), 1)
LIBRARIES += lmdb
endif
ifeq ($(USE_OPENCV), 1)
LIBRARIES += opencv_core opencv_highgui opencv_imgproc
endif
PYTHON_LIBRARIES := boost_python python2.7
WARNINGS := -Wall -Wno-sign-compare

@@ -290,6 +299,17 @@ ifeq ($(USE_CUDNN), 1)
COMMON_FLAGS += -DUSE_CUDNN
endif

# i/o libraries configuration
ifeq ($(USE_OPENCV), 1)
COMMON_FLAGS += -DUSE_OPENCV
endif
ifeq ($(USE_LEVELDB), 1)
COMMON_FLAGS += -DUSE_LEVELDB
endif
ifeq ($(USE_LMDB), 1)
COMMON_FLAGS += -DUSE_LMDB
endif

# CPU-only configuration
ifeq ($(CPU_ONLY), 1)
OBJS := $(PROTO_OBJS) $(CXX_OBJS)
@@ -472,7 +492,7 @@ runtest: $(TEST_ALL_BIN)

pytest: py
cd python; python -m unittest discover -s caffe/test

mattest: mat
cd matlab; $(MATLAB_DIR)/bin/matlab -nodisplay -r 'caffe.run_tests(), exit()'

5 changes: 5 additions & 0 deletions Makefile.config.example
Original file line number Diff line number Diff line change
@@ -7,6 +7,11 @@
# CPU-only switch (uncomment to build without GPU support).
# CPU_ONLY := 1

# comment out to disable IO dependencies
USE_LEVELDB := 1
USE_LMDB := 1
USE_OPENCV := 1

# To customize your choice of compiler, uncomment and set the following.
# N.B. the default for Linux is g++ and the default for OSX is clang++
# CUSTOM_CXX := g++
12 changes: 12 additions & 0 deletions cmake/ConfigGen.cmake
Original file line number Diff line number Diff line change
@@ -56,6 +56,18 @@ function(caffe_generate_export_configs)
list(APPEND Caffe_DEFINITIONS -DCPU_ONLY)
endif()

if(USE_OPENCV)
list(APPEND Caffe_DEFINITIONS -DUSE_OPENCV)
endif()

if(USE_LMDB)
list(APPEND Caffe_DEFINITIONS -DUSE_LMDB)
endif()

if(USE_LEVELDB)
list(APPEND Caffe_DEFINITIONS -DUSE_LEVELDB)
endif()

if(NOT HAVE_CUDNN)
set(HAVE_CUDNN FALSE)
else()
41 changes: 26 additions & 15 deletions cmake/Dependencies.cmake
Original file line number Diff line number Diff line change
@@ -29,19 +29,27 @@ include_directories(SYSTEM ${HDF5_INCLUDE_DIRS} ${HDF5_HL_INCLUDE_DIR})
list(APPEND Caffe_LINKER_LIBS ${HDF5_LIBRARIES})

# ---[ LMDB
find_package(LMDB REQUIRED)
include_directories(SYSTEM ${LMDB_INCLUDE_DIR})
list(APPEND Caffe_LINKER_LIBS ${LMDB_LIBRARIES})
if(USE_LMDB)
find_package(LMDB REQUIRED)
include_directories(SYSTEM ${LMDB_INCLUDE_DIR})
list(APPEND Caffe_LINKER_LIBS ${LMDB_LIBRARIES})
add_definitions(-DUSE_LMDB)
endif()

# ---[ LevelDB
find_package(LevelDB REQUIRED)
include_directories(SYSTEM ${LevelDB_INCLUDE})
list(APPEND Caffe_LINKER_LIBS ${LevelDB_LIBRARIES})
if(USE_LEVELDB)
find_package(LevelDB REQUIRED)
include_directories(SYSTEM ${LevelDB_INCLUDE})
list(APPEND Caffe_LINKER_LIBS ${LevelDB_LIBRARIES})
add_definitions(-DUSE_LEVELDB)
endif()

# ---[ Snappy
find_package(Snappy REQUIRED)
include_directories(SYSTEM ${Snappy_INCLUDE_DIR})
list(APPEND Caffe_LINKER_LIBS ${Snappy_LIBRARIES})
if(USE_LEVELDB)
find_package(Snappy REQUIRED)
include_directories(SYSTEM ${Snappy_INCLUDE_DIR})
list(APPEND Caffe_LINKER_LIBS ${Snappy_LIBRARIES})
endif()

# ---[ CUDA
include(cmake/Cuda.cmake)
@@ -57,13 +65,16 @@ if(NOT HAVE_CUDA)
endif()

# ---[ OpenCV
find_package(OpenCV QUIET COMPONENTS core highgui imgproc imgcodecs)
if(NOT OpenCV_FOUND) # if not OpenCV 3.x, then imgcodecs are not found
find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc)
if(USE_OPENCV)
find_package(OpenCV QUIET COMPONENTS core highgui imgproc imgcodecs)
if(NOT OpenCV_FOUND) # if not OpenCV 3.x, then imgcodecs are not found
find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc)
endif()
include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})
list(APPEND Caffe_LINKER_LIBS ${OpenCV_LIBS})
message(STATUS "OpenCV found (${OpenCV_CONFIG_PATH})")
add_definitions(-DUSE_OPENCV)
endif()
include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})
list(APPEND Caffe_LINKER_LIBS ${OpenCV_LIBS})
message(STATUS "OpenCV found (${OpenCV_CONFIG_PATH})")

# ---[ BLAS
if(NOT APPLE)
18 changes: 13 additions & 5 deletions cmake/Summary.cmake
Original file line number Diff line number Diff line change
@@ -114,17 +114,26 @@ function(caffe_print_configuration_summary)
caffe_status(" BUILD_matlab : ${BUILD_matlab}")
caffe_status(" BUILD_docs : ${BUILD_docs}")
caffe_status(" CPU_ONLY : ${CPU_ONLY}")
caffe_status(" USE_LMDB : ${USE_LMDB}")
caffe_status(" USE_LEVELDB : ${USE_LEVELDB}")
caffe_status(" USE_OPENCV : ${USE_OPENCV}")
caffe_status("")
caffe_status("Dependencies:")
caffe_status(" BLAS : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})")
caffe_status(" Boost : Yes (ver. ${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION})")
caffe_status(" glog : Yes")
caffe_status(" gflags : Yes")
caffe_status(" protobuf : " PROTOBUF_FOUND THEN "Yes (ver. ${PROTOBUF_VERSION})" ELSE "No" )
caffe_status(" lmdb : " LMDB_FOUND THEN "Yes (ver. ${LMDB_VERSION})" ELSE "No")
caffe_status(" Snappy : " SNAPPY_FOUND THEN "Yes (ver. ${Snappy_VERSION})" ELSE "No" )
caffe_status(" LevelDB : " LEVELDB_FOUND THEN "Yes (ver. ${LEVELDB_VERSION})" ELSE "No")
caffe_status(" OpenCV : Yes (ver. ${OpenCV_VERSION})")
if(USE_LMDB)
caffe_status(" lmdb : " LMDB_FOUND THEN "Yes (ver. ${LMDB_VERSION})" ELSE "No")
endif()
if(USE_LEVELDB)
caffe_status(" LevelDB : " LEVELDB_FOUND THEN "Yes (ver. ${LEVELDB_VERSION})" ELSE "No")
caffe_status(" Snappy : " SNAPPY_FOUND THEN "Yes (ver. ${Snappy_VERSION})" ELSE "No" )
endif()
if(USE_OPENCV)
caffe_status(" OpenCV : Yes (ver. ${OpenCV_VERSION})")
endif()
caffe_status(" CUDA : " HAVE_CUDA THEN "Yes (ver. ${CUDA_VERSION})" ELSE "No" )
caffe_status("")
if(HAVE_CUDA)
@@ -165,4 +174,3 @@ function(caffe_print_configuration_summary)
caffe_status(" Install path : ${CMAKE_INSTALL_PREFIX}")
caffe_status("")
endfunction()

26 changes: 14 additions & 12 deletions cmake/Templates/CaffeConfig.cmake.in
Original file line number Diff line number Diff line change
@@ -17,22 +17,24 @@
# Caffe_HAVE_CUDNN - signals about cuDNN support


# OpenCV dependency
# OpenCV dependency (optional)

if(NOT OpenCV_FOUND)
set(Caffe_OpenCV_CONFIG_PATH "@OpenCV_CONFIG_PATH@")
if(Caffe_OpenCV_CONFIG_PATH)
get_filename_component(Caffe_OpenCV_CONFIG_PATH ${Caffe_OpenCV_CONFIG_PATH} ABSOLUTE)
if(@USE_OPENCV@)
if(NOT OpenCV_FOUND)
set(Caffe_OpenCV_CONFIG_PATH "@OpenCV_CONFIG_PATH@")
if(Caffe_OpenCV_CONFIG_PATH)
get_filename_component(Caffe_OpenCV_CONFIG_PATH ${Caffe_OpenCV_CONFIG_PATH} ABSOLUTE)

if(EXISTS ${Caffe_OpenCV_CONFIG_PATH} AND NOT TARGET opencv_core)
message(STATUS "Caffe: using OpenCV config from ${Caffe_OpenCV_CONFIG_PATH}")
include(${Caffe_OpenCV_CONFIG_PATH}/OpenCVModules.cmake)
endif()
if(EXISTS ${Caffe_OpenCV_CONFIG_PATH} AND NOT TARGET opencv_core)
message(STATUS "Caffe: using OpenCV config from ${Caffe_OpenCV_CONFIG_PATH}")
include(${Caffe_OpenCV_CONFIG_PATH}/OpenCVModules.cmake)
endif()

else()
find_package(OpenCV REQUIRED)
else()
find_package(OpenCV REQUIRED)
endif()
unset(Caffe_OpenCV_CONFIG_PATH)
endif()
unset(Caffe_OpenCV_CONFIG_PATH)
endif()

# Compute paths
5 changes: 5 additions & 0 deletions cmake/Templates/caffe_config.h.in
Original file line number Diff line number Diff line change
@@ -30,3 +30,8 @@

/* Matlab */
#cmakedefine HAVE_MATLAB

/* IO libraries */
#cmakedefine USE_OPENCV
#cmakedefine USE_LMDB
#cmakedefine USE_LEVELDB
9 changes: 6 additions & 3 deletions docs/installation.md
Original file line number Diff line number Diff line change
@@ -17,16 +17,19 @@ When updating Caffe, it's best to `make clean` before re-compiling.

## Prerequisites

Caffe has several dependencies.
Caffe has several dependencies:

* [CUDA](https://developer.nvidia.com/cuda-zone) is required for GPU mode.
* library version 7.0 and the latest driver version are recommended, but 6.* is fine too
* 5.5, and 5.0 are compatible but considered legacy
* [BLAS](http://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) via ATLAS, MKL, or OpenBLAS.
* [Boost](http://www.boost.org/) >= 1.55
* `protobuf`, `glog`, `gflags`, `hdf5`

Optional dependencies:

* [OpenCV](http://opencv.org/) >= 2.4 including 3.0
* `protobuf`, `glog`, `gflags`
* IO libraries `hdf5`, `leveldb`, `snappy`, `lmdb`
* IO libraries: `lmdb`, `leveldb` (note: leveldb requires `snappy`)

Pycaffe and Matcaffe interfaces have their own natural needs.

8 changes: 8 additions & 0 deletions examples/cpp_classification/classification.cpp
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
#include <caffe/caffe.hpp>
#ifdef USE_OPENCV
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#endif // USE_OPENCV
#include <algorithm>
#include <iosfwd>
#include <memory>
#include <string>
#include <utility>
#include <vector>

#ifdef USE_OPENCV
using namespace caffe; // NOLINT(build/namespaces)
using std::string;

@@ -255,3 +258,8 @@ int main(int argc, char** argv) {
<< p.first << "\"" << std::endl;
}
}
#else
int main(int argc, char** argv) {
LOG(FATAL) << "This example requires OpenCV; compile with USE_OPENCV.";
}
#endif // USE_OPENCV
12 changes: 12 additions & 0 deletions examples/mnist/convert_mnist_data.cpp
Original file line number Diff line number Diff line change
@@ -9,9 +9,13 @@
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <google/protobuf/text_format.h>

#if defined(USE_LEVELDB) && defined(USE_LMDB)
#include <leveldb/db.h>
#include <leveldb/write_batch.h>
#include <lmdb.h>
#endif

#include <stdint.h>
#include <sys/stat.h>

@@ -20,6 +24,8 @@

#include "caffe/proto/caffe.pb.h"

#if defined(USE_LEVELDB) && defined(USE_LMDB)

using namespace caffe; // NOLINT(build/namespaces)
using std::string;

@@ -196,3 +202,9 @@ int main(int argc, char** argv) {
}
return 0;
}
#else
int main(int argc, char** argv) {
LOG(FATAL) << "This example requires LevelDB and LMDB; " <<
"compile with USE_LEVELDB and USE_LMDB.";
}
#endif // USE_LEVELDB and USE_LMDB
9 changes: 8 additions & 1 deletion examples/siamese/convert_mnist_siamese_data.cpp
Original file line number Diff line number Diff line change
@@ -10,12 +10,14 @@

#include "glog/logging.h"
#include "google/protobuf/text_format.h"
#include "leveldb/db.h"
#include "stdint.h"

#include "caffe/proto/caffe.pb.h"
#include "caffe/util/math_functions.hpp"

#ifdef USE_LEVELDB
#include "leveldb/db.h"

uint32_t swap_endian(uint32_t val) {
val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF);
return (val << 16) | (val >> 16);
@@ -121,3 +123,8 @@ int main(int argc, char** argv) {
}
return 0;
}
#else
int main(int argc, char** argv) {
LOG(FATAL) << "This example requires LevelDB; compile with USE_LEVELDB.";
}
#endif // USE_LEVELDB
3 changes: 2 additions & 1 deletion include/caffe/data_layers.hpp
Original file line number Diff line number Diff line change
@@ -4,7 +4,6 @@
#include <string>
#include <utility>
#include <vector>

#include "hdf5.h"

#include "caffe/blob.hpp"
@@ -275,8 +274,10 @@ class MemoryDataLayer : public BaseDataLayer<Dtype> {
virtual inline int ExactNumTopBlobs() const { return 2; }

virtual void AddDatumVector(const vector<Datum>& datum_vector);
#ifdef USE_OPENCV
virtual void AddMatVector(const vector<cv::Mat>& mat_vector,
const vector<int>& labels);
#endif // USE_OPENCV

// Reset should accept const pointers, but can't, because the memory
// will be given to Blob, which is mutable
Loading

0 comments on commit 5ff3734

Please sign in to comment.