Travis scripts for python3 and pytest for cmake. Also fixes CUDA CMak…

…e build issue BVLC#2722.

.travis.yml

@@ -6,24 +6,31 @@ env:
  - WITH_CUDA=false WITH_CMAKE=true
  - WITH_CUDA=true WITH_CMAKE=false
  - WITH_CUDA=true WITH_CMAKE=true
+ - WITH_CUDA=false WITH_CMAKE=true PYTHON_VERSION=3
 
 language: cpp
 
 # Cache Ubuntu apt packages.
-cache: apt
+cache:
+ apt: true
+ directories:
+ - /home/travis/miniconda
+ - /home/travis/miniconda2
+ - /home/travis/miniconda3
 
 compiler: gcc
 
 before_install:
  - export NUM_THREADS=4
  - export SCRIPTS=./scripts/travis
+ - export CONDA_DIR="/home/travis/miniconda$PYTHON_VERSION"
 
 install:
  - sudo -E $SCRIPTS/travis_install.sh
 
 before_script:
- - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/cuda/lib64
- - export PATH=/home/travis/miniconda/bin:$PATH
+ - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/cuda/lib64:$CONDA_DIR/lib
+ - export PATH=$CONDA_DIR/bin:$PATH
  - if ! $WITH_CMAKE; then $SCRIPTS/travis_setup_makefile_config.sh; fi
 
 script: $SCRIPTS/travis_build_and_test.sh

CMakeLists.txt

@@ -62,6 +62,10 @@ add_subdirectory(docs)
 # ---[ Linter target
 add_custom_target(lint COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/lint.cmake)
 
+# ---[ pytest target
+add_custom_target(pytest COMMAND python${python_version} -m unittest discover -s caffe/test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python )
+add_dependencies(pytest pycaffe)
+
 # ---[ Configuration summary
 caffe_print_configuration_summary()
 

Makefile

@@ -228,7 +228,7 @@ ifeq ($(LINUX), 1)
  CXX ?= /usr/bin/g++
  GCCVERSION := $(shell $(CXX) -dumpversion | cut -f1,2 -d.)
  # older versions of gcc are too dumb to build boost with -Wuninitalized
- ifeq ($(shell echo $(GCCVERSION) \< 4.6 | bc), 1)
+ ifeq ($(shell echo | awk '{exit $(GCCVERSION) < 4.6;}'), 1)
  WARNINGS += -Wno-uninitialized
  endif
  # boost::thread is reasonably called boost_thread (compare OS X)
@@ -243,7 +243,7 @@ ifeq ($(OSX), 1)
  CXX := /usr/bin/clang++
  ifneq ($(CPU_ONLY), 1)
  CUDA_VERSION := $(shell $(CUDA_DIR)/bin/nvcc -V | grep -o 'release \d' | grep -o '\d')
- ifeq ($(shell echo $(CUDA_VERSION) \< 7.0 | bc), 1)
+ ifeq ($(shell echo | awk '{exit $(CUDA_VERSION) < 7.0;}'), 1)
  CXXFLAGS += -stdlib=libstdc++
  LINKFLAGS += -stdlib=libstdc++
  endif

cmake/Dependencies.cmake

@@ -106,14 +106,15 @@ if(BUILD_python)
 
  while(NOT "${version}" STREQUAL "" AND NOT Boost_PYTHON_FOUND)
  STRING( REGEX REPLACE "([0-9.]+).[0-9]+" "\\1" version ${version} )
- STRING( REGEX MATCHALL "([0-9.]+).[0-9]+" has_more_version ${version} )
- if("${has_more_version}" STREQUAL "")
- break()
- endif()
 
  STRING( REPLACE "." "" boost_py_version ${version} )
  find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}")
  set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND})
+
+ STRING( REGEX MATCHALL "([0-9.]+).[0-9]+" has_more_version ${version} )
+ if("${has_more_version}" STREQUAL "")
+ break()
+ endif()
  endwhile()
  if(NOT Boost_PYTHON_FOUND)
  find_package(Boost 1.46 COMPONENTS python)

python/caffe/test/test_net.py

@@ -2,6 +2,7 @@
 import tempfile
 import os
 import numpy as np
+import six
 
 import caffe
 
@@ -10,7 +11,7 @@ def simple_net_file(num_output):
  """Make a simple net prototxt, based on test_net.cpp, returning the name
  of the (temporary) file."""
 
- f = tempfile.NamedTemporaryFile(delete=False)
+ f = tempfile.NamedTemporaryFile(mode='w+', delete=False)
  f.write("""name: 'testnet' force_backward: true
  layer { type: 'DummyData' name: 'data' top: 'data' top: 'label'
  dummy_data_param { num: 5 channels: 2 height: 3 width: 4
@@ -47,7 +48,7 @@ def setUp(self):
  def test_memory(self):
  """Check that holding onto blob data beyond the life of a Net is OK"""
 
- params = sum(map(list, self.net.params.itervalues()), [])
+ params = sum(map(list, six.itervalues(self.net.params)), [])
  blobs = self.net.blobs.values()
  del self.net
 
@@ -67,7 +68,7 @@ def test_inputs_outputs(self):
  self.assertEqual(self.net.outputs, ['loss'])
 
  def test_save_and_read(self):
- f = tempfile.NamedTemporaryFile(delete=False)
+ f = tempfile.NamedTemporaryFile(mode='w+', delete=False)
  f.close()
  self.net.save(f.name)
  net_file = simple_net_file(self.num_output)

python/caffe/test/test_net_spec.py

@@ -43,7 +43,7 @@ def anon_lenet(batch_size):
 
 class TestNetSpec(unittest.TestCase):
  def load_net(self, net_proto):
- f = tempfile.NamedTemporaryFile(delete=False)
+ f = tempfile.NamedTemporaryFile(mode='w+', delete=False)
  f.write(str(net_proto))
  f.close()
  return caffe.Net(f.name, caffe.TEST)

python/caffe/test/test_python_layer.py

@@ -1,6 +1,7 @@
 import unittest
 import tempfile
 import os
+import six
 
 import caffe
 
@@ -22,7 +23,7 @@ def backward(self, top, propagate_down, bottom):
 
 
 def python_net_file():
- with tempfile.NamedTemporaryFile(delete=False) as f:
+ with tempfile.NamedTemporaryFile(mode='w+', delete=False) as f:
  f.write("""name: 'pythonnet' force_backward: true
  input: 'data' input_shape { dim: 10 dim: 9 dim: 8 }
  layer { type: 'Python' name: 'one' bottom: 'data' top: 'one'
@@ -58,6 +59,6 @@ def test_reshape(self):
  s = 4
  self.net.blobs['data'].reshape(s, s, s, s)
  self.net.forward()
- for blob in self.net.blobs.itervalues():
+ for blob in six.itervalues(self.net.blobs):
  for d in blob.data.shape:
  self.assertEqual(s, d)

python/caffe/test/test_solver.py

@@ -2,6 +2,7 @@
 import tempfile
 import os
 import numpy as np
+import six
 
 import caffe
 from test_net import simple_net_file
@@ -11,7 +12,7 @@ class TestSolver(unittest.TestCase):
  def setUp(self):
  self.num_output = 13
  net_f = simple_net_file(self.num_output)
- f = tempfile.NamedTemporaryFile(delete=False)
+ f = tempfile.NamedTemporaryFile(mode='w+', delete=False)
  f.write("""net: '""" + net_f + """'
  test_iter: 10 test_interval: 10 base_lr: 0.01 momentum: 0.9
  weight_decay: 0.0005 lr_policy: 'inv' gamma: 0.0001 power: 0.75
@@ -45,8 +46,8 @@ def test_net_memory(self):
 
  total = 0
  for net in nets:
- for ps in net.params.itervalues():
+ for ps in six.itervalues(net.params):
  for p in ps:
  total += p.data.sum() + p.diff.sum()
- for bl in net.blobs.itervalues():
+ for bl in six.itervalues(net.blobs):
  total += bl.data.sum() + bl.diff.sum()

scripts/travis/travis_build_and_test.sh

@@ -7,8 +7,17 @@ MAKE="make --jobs=$NUM_THREADS --keep-going"
 if $WITH_CMAKE; then
  mkdir build
  cd build
- cmake -DBUILD_python=ON -DCMAKE_BUILD_TYPE=Release -DCPU_ONLY=ON ..
+ CPU_ONLY=" -DCPU_ONLY=ON"
+ if ! $WITH_CUDA; then
+ CPU_ONLY=" -DCPU_ONLY=OFF"
+ fi
+ PYTHON_ARGS=""
+ if [ "$PYTHON_VERSION" = "3" ]; then
+ PYTHON_ARGS="$PYTHON_ARGS -Dpython_version=3 -DBOOST_LIBRARYDIR=$CONDA_DIR/lib/"
+ fi
+ cmake -DBUILD_python=ON -DCMAKE_BUILD_TYPE=Release $CPU_ONLY $PYTHON_ARGS -DCMAKE_INCLUDE_PATH="$CONDA_DIR/include/" -DCMAKE_LIBRARY_PATH="$CONDA_DIR/lib/" ..
  $MAKE
+ $MAKE pytest
  if ! $WITH_CUDA; then
  $MAKE runtest
  $MAKE lint

scripts/travis/travis_install.sh

@@ -4,17 +4,15 @@
 set -e
 
 MAKE="make --jobs=$NUM_THREADS"
-
 # Install apt packages where the Ubuntu 12.04 default and ppa works for Caffe
 
 # This ppa is for gflags and glog
 add-apt-repository -y ppa:tuleu/precise-backports
 apt-get -y update
 apt-get install \
  wget git curl \
- python-dev python-numpy \
+ python-dev python-numpy python3-dev\
  libleveldb-dev libsnappy-dev libopencv-dev \
- libboost-dev libboost-system-dev libboost-python-dev libboost-thread-dev \
  libprotobuf-dev protobuf-compiler \
  libatlas-dev libatlas-base-dev \
  libhdf5-serial-dev libgflags-dev libgoogle-glog-dev \
@@ -24,9 +22,10 @@ apt-get install \
 # if needed. By default, Aptitude in Ubuntu 12.04 installs CMake 2.8.7, but
 # Caffe requires a minimum CMake version of 2.8.8.
 if $WITH_CMAKE; then
- add-apt-repository -y ppa:ubuntu-sdk-team/ppa
- apt-get -y update
- apt-get -y install cmake
+ # cmake 3 will make sure that the python interpreter and libraries match
+ wget http://www.cmake.org/files/v3.2/cmake-3.2.3-Linux-x86_64.sh -O cmake3.sh
+ chmod +x cmake3.sh
+ ./cmake3.sh --prefix=/usr/ --skip-license --exclude-subdir
 fi
 
 # Install CUDA, if needed
@@ -60,10 +59,37 @@ rm -f $LMDB_FILE
 
 # Install the Python runtime dependencies via miniconda (this is much faster
 # than using pip for everything).
-wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
-chmod +x miniconda.sh
-./miniconda.sh -b
-export PATH=/home/travis/miniconda/bin:$PATH
-conda update --yes conda
-conda install --yes numpy scipy matplotlib scikit-image pip
-pip install protobuf
+export PATH=$CONDA_DIR/bin:$PATH
+if [ ! -d $CONDA_DIR ]; then
+ if [ "$PYTHON_VERSION" -eq "3" ]; then
+ wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+ else
+ wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
+ fi
+ chmod +x miniconda.sh
+ ./miniconda.sh -b -p $CONDA_DIR
+
+ conda update --yes conda
+ conda install --yes numpy scipy matplotlib scikit-image pip
+ # Let conda install boost (so that boost_python matches)
+ conda install --yes -c https://conda.binstar.org/menpo boost=1.56.0
+fi
+
+# install protobuf 3 (just use the miniconda3 directory to avoid having to setup the path again)
+if [ "$PYTHON_VERSION" -eq "3" ] && [ ! -e "$CONDA_DIR/bin/protoc" ]; then
+ pushd .
+ wget https://github.com/google/protobuf/archive/v3.0.0-alpha-3.1.tar.gz -O protobuf-3.tar.gz
+ tar -C /tmp -xzvf protobuf-3.tar.gz
+ cd /tmp/protobuf-3*/
+ ./autogen.sh
+ ./configure --prefix=$CONDA_DIR
+ $MAKE
+ $MAKE install
+ popd
+fi
+
+if [ "$PYTHON_VERSION" -eq "3" ]; then
+ pip install --pre protobuf
+else
+ pip install protobuf
+fi

scripts/travis/travis_setup_makefile_config.sh

@@ -12,7 +12,9 @@ if $WITH_CUDA; then
 fi
 
 cat << 'EOF' >> Makefile.config
-ANACONDA_HOME := $(HOME)/miniconda
+# Travis' nvcc doesn't like newer boost versions
+NVCCFLAGS := -Xcudafe --diag_suppress=cc_clobber_ignored -Xcudafe --diag_suppress=useless_using_declaration -Xcudafe --diag_suppress=set_but_not_used
+ANACONDA_HOME := $(CONDA_DIR)
 PYTHON_INCLUDE := $(ANACONDA_HOME)/include \
  $(ANACONDA_HOME)/include/python2.7 \
  $(ANACONDA_HOME)/lib/python2.7/site-packages/numpy/core/include

src/caffe/layer_factory.cpp

@@ -1,3 +1,8 @@
+// Make sure we include Python.h before any system header
+// to avoid _POSIX_C_SOURCE redefinition
+#ifdef WITH_PYTHON_LAYER
+#include <boost/python.hpp>
+#endif
 #include <string>
 
 #include "caffe/layer.hpp"