Add whitelist capability for smaller mobile binaries

Summary:
This helps adjusting mobile build sizes when necessary.
Closes facebookarchive#228

Differential Revision: D4795135

Pulled By: Yangqing

fbshipit-source-id: 70a0dc35b31d5c8038081aedeb464e47e4284217

.travis/build.sh

@@ -25,8 +25,7 @@ elif [[ $TRAVIS_OS_NAME == 'osx' ]]; then
   cmake .. \
       -DCMAKE_VERBOSE_MAKEFILE=ON \
       -DUSE_OPENCV=OFF \
-      -DUSE_NNPACK=ON \
-      && make
+  && make
 else
 #*************#
 # Linux build #
@@ -35,11 +34,11 @@ else
     cmake .. \
         -DCMAKE_VERBOSE_MAKEFILE=ON \
         -DBLAS=MKL \
-        -DUSE_NNPACK=ON \
-        && make
+        -DUSE_CUDA=OFF \
+    && make
   else
     cmake .. \
         -DCMAKE_VERBOSE_MAKEFILE=ON \
-        -DUSE_NNPACK=ON && make
+    && make
   fi
 fi

CMakeLists.txt

@@ -62,6 +62,9 @@ option(BUILD_TEST "Build C++ test binaries (need gtest and gbenchmark)" ON)
 
 # options that takes in strings instead of on and off statemens.
 option(CAFFE2_CPU_FLAGS "Flags to specify CPU features." "")
+option(CAFFE2_WHITELIST
+       "A whitelist file of files that one should build."
+       "")
 
 # Set default build type
 if(NOT CMAKE_BUILD_TYPE)
@@ -75,6 +78,9 @@ include(cmake/Dependencies.cmake)
 # ---[ Misc checks to cope with various compiler modes
 include(cmake/MiscCheck.cmake)
 
+# ---[ Whitelist file if whitelist is specified
+include(cmake/Whitelist.cmake)
+
 # ---[ Set link flag, handle additional deps for gcc 4.8 and above
 if(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.8.0 AND NOT ANDROID)
   message(STATUS "GCC ${CMAKE_CXX_COMPILER_VERSION}: Adding gcc and gcc_s libs to link line")
@@ -85,6 +91,7 @@ endif()
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "binaries")
 
 # ---[ Build flags
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11")
 if(NOT MSVC)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing")

caffe2/CMakeLists.txt

@@ -34,6 +34,13 @@ add_subdirectory(sgd)
 # add_subdirectory(test) # todo: use caffe2_gtest_main instead of gtest_main because we will need to call GlobalInit
 add_subdirectory(utils)
 
+# Advanced: if we have white list specified, we will do intersections for all
+# main lib srcs.
+if (CAFFE2_WHITELISTED_FILES)
+  caffe2_do_whitelist(Caffe2_CPU_SRCS CAFFE2_WHITELISTED_FILES)
+  caffe2_do_whitelist(Caffe2_GPU_SRCS CAFFE2_WHITELISTED_FILES)
+endif()
+
 # Debug messages - if you want to get a list of source files, enable the
 # following.
 if (FALSE)

cmake/Whitelist.cmake

@@ -0,0 +1,32 @@
+
+if (__caffe2_whitelist_included)
+  return()
+endif()
+
+set (__caffe2_whitelist_included TRUE)
+
+set(CAFFE2_WHITELISTED_FILES)
+if (NOT CAFFE2_WHITELIST)
+  return()
+endif()
+
+# First read the whitelist file and break it by line.
+file(READ "${CAFFE2_WHITELIST}" whitelist_content)
+# Convert file contents into a CMake list
+string(REGEX REPLACE "\n" ";" whitelist_content ${whitelist_content})
+
+foreach(item ${whitelist_content})
+  file(GLOB_RECURSE tmp ${item})
+  set(CAFFE2_WHITELISTED_FILES ${CAFFE2_WHITELISTED_FILES} ${tmp})
+endforeach()
+
+macro(caffe2_do_whitelist output whitelist)
+  set(_tmp)
+  foreach(item ${${output}})
+    list(FIND ${whitelist} ${item} _index)
+    if (${_index} GREATER -1)
+      set(_tmp ${_tmp} ${item})
+    endif()
+  endforeach()
+  set(${output} ${_tmp})
+endmacro()

scripts/diagnose_protobuf.py

@@ -0,0 +1,92 @@
+"""Diagnoses the current protobuf situation.
+
+Protocol buffer needs to be properly installed for Caffe2 to work, and
+sometimes it is rather tricky. Specifically, we will need to have a
+consistent version between C++ and python simultaneously. This is a
+convenience script for one to quickly check if this is so on one's local
+machine.
+
+Usage:
+    [set your environmental variables like PATH and PYTHONPATH]
+    python scripts/diagnose_protobuf.py
+"""
+
+import os
+import re
+from subprocess import Popen, PIPE
+
+# Get python protobuf version.
+try:
+    import google.protobuf
+    python_version = google.protobuf.__version__
+    python_protobuf_installed = True
+except ImportError: 
+    print("DEBUG: cannot find python protobuf install.")
+    python_protobuf_installed = False
+
+if os.name == 'nt':
+    protoc_name = 'protoc.exe'
+else:
+    protoc_name = 'protoc'
+
+try:
+    p = Popen([protoc_name, '--version'], stdout=PIPE, stderr=PIPE)
+    out, err = p.communicate()
+except:
+    print('DEBUG: did not find protoc binary.')
+    print('DEBUG: out: ' + out)
+    print('DEBUG: err: ' + err)
+    native_protobuf_installed = False
+else:
+    if p.returncode:
+        print('DEBUG: protoc returned a non-zero return code.')
+        print('DEBUG: out: ' + out)
+        print('DEBUG: err: ' + err)
+        native_protobuf_installed = False
+    else:
+        tmp = re.search('\d\.\d\.\d', out)
+        if tmp:
+            native_version = tmp.group(0)
+            native_protobuf_installed = True
+        else:
+            print('DEBUG: cannot parse protoc version string.')
+            print('DEBUG: out: ' + out)
+            native_protobuf_installed = False
+
+PYTHON_PROTOBUF_NOT_INSTALLED = """
+You have not installed python protobuf. Protobuf is needed to run caffe2. You
+can install protobuf via pip or conda (if you are using anaconda python).
+"""
+
+NATIVE_PROTOBUF_NOT_INSTALLED = """
+You have not installed the protoc binary. Protoc is needed to compile Caffe2
+protobuf source files. Depending on the platform you are on, you can install
+protobuf via:
+    (1) Mac: using homebrew and do brew install protobuf.
+    (2) Linux: use apt and do apt-get install libprotobuf-dev
+    (3) Windows: install from source, or from the releases here:
+        https://github.com/google/protobuf/releases/
+"""
+
+VERSION_MISMATCH = """
+Your python protobuf is of version {py_ver} but your native protoc version is of
+version {native_ver}. This will cause the installation to produce incompatible
+protobuf files. This is bad in general - consider installing the same version.
+""".format(py_ver=python_version, native_ver=native_version)
+
+# Now, give actual recommendations
+if not python_protobuf_installed:
+    print(PYTHON_PROTOBUF_NOT_INSTALLED)
+
+if not native_protobuf_installed:
+    print(NATIVE_PROTOBUF_NOT_INSTALLED)
+
+if python_protobuf_installed and native_protobuf_installed:
+    if python_version != native_version:
+        print(VERSION_MISMATCH)
+    else:
+        print('All looks good.')
+
+
+
+