Merge stable 1.3.1 release of MXNet

https://github.com/apache/incubator-mxnet/releases/tag/1.3.1

This updates to MXNet 1.3.1. We are skipping 1.2.x because they
indirectly reference missing commits. The update to 1.3.0 is skipped
because there is already the patched version 1.3.1 available.

The conflicts in `.gitmodules` are manually resolved to keep our
beaver and makd dependencies. For all other conflicts their version was
picked all the time.

To see our changes with respect to upstream MXNet diff against the MXNet
1.3.1 tag, i.e., `git diff 1.3.1..HEAD`.

* 3rdparty/dmlc-core e9446f5(e9446f5)...0a0e8ad(0a0e8ad) (41 commits)
  > Add OMPException class and use it for Text Parser (apache#445)
  > Fix build problem on windows (apache#450)
  > switch to safe_load for kubernetes config load (apache#449)
  > Add S3_IS_AWS env and fixed non-AWS behavior (apache#444)
  > add error message for s3 list (apache#439)
  (...)

* 3rdparty/mkldnn 0e7ca738(0e7ca738)...0e7ca738(0e7ca738) (99 commits)
  > build: bumped version to v0.14 in readme
  > build: bumped version to v0.14
  > cpu: reorder: start using jit uni for 8x8 transposition
  > cpu: reorder: jit uni: add 8x8 kernel
  > cpu: reorder: enable jit uni reorder
  (...)

* 3rdparty/mshadow a8c650c(a8c650c)...8a9e337(8a9e337) (9 commits)
  > Merge pull request apache#358 from eric-haibin-lin/revert
  > Merge pull request apache#357 from azai91/revert/d68d3
  > Merge pull request apache#356 from szha/omp
  > Add half_t support for batch_dot.  (apache#353)
  > Allow large array operation in MXNet (apache#348)
  (...)

* 3rdparty/onnx-tensorrt ()...3d8ee04(3d8ee04) (1 commits)
  > Refactor onnxGetBackendInfo (apache#39)

* 3rdparty/ps-lite v1+144(a6dda54)...v1+146(8a76389) (1 commits)
  > Merge pull request apache#133 from CodingCat/turn_up_down

* 3rdparty/tvm v0.3+434(90db723)d...v0.3+434(90db723)d (1 commits)
  > [FRONTEND] A Python hybrid frontend (apache#1251)

.codecov.yml

@@ -0,0 +1,28 @@
+# Codecov.io configuration file
+codecov:
+ notify:
+ require_ci_to_pass: yes
+
+coverage:
+ precision: 2
+ round: down
+ range: "70...100"
+
+ status:
+ project: yes
+ patch: yes
+ changes: no
+
+parsers:
+ gcov:
+ branch_detection:
+ conditional: yes
+ loop: yes
+ method: no
+ macro: no
+
+# Disable comments for now to gather data in the background
+comment: false
+# layout: "header, diff"
+# behavior: default
+# require_changes: no

.gitignore

@@ -166,3 +166,7 @@ python/.eggs
 *DartConfiguration.tcl
 tests/Makefile
 tests/mxnet_unit_tests
+
+# generated wrappers for ccache
+cc
+cxx

.gitmodules

@@ -7,9 +7,6 @@
 [submodule "3rdparty/ps-lite"]
  path = 3rdparty/ps-lite
  url = https://github.com/dmlc/ps-lite
-[submodule "3rdparty/nnvm"]
- path = 3rdparty/nnvm
- url = https://github.com/dmlc/nnvm
 [submodule "3rdparty/dlpack"]
  path = 3rdparty/dlpack
  url = https://github.com/dmlc/dlpack
@@ -26,6 +23,12 @@
 [submodule "3rdparty/cub"]
  path = 3rdparty/cub
  url = https://github.com/dmlc/cub
+[submodule "3rdparty/tvm"]
+ path = 3rdparty/tvm
+ url = https://github.com/dmlc/tvm
+[submodule "3rdparty/onnx-tensorrt"]
+ path = 3rdparty/onnx-tensorrt
+ url = https://github.com/onnx/onnx-tensorrt.git
 [submodule "beaver"]
  path = beaver
  url = https://github.com/sociomantic-tsunami/beaver.git

CMakeLists.txt

@@ -15,12 +15,12 @@ mxnet_option(USE_NCCL "Use NVidia NCCL with CUDA" OFF)
 mxnet_option(USE_OPENCV "Build with OpenCV support" ON)
 mxnet_option(USE_OPENMP "Build with Openmp support" ON)
 mxnet_option(USE_CUDNN "Build with cudnn support" ON) # one could set CUDNN_ROOT for search path
-mxnet_option(USE_SSE "Build with x86 SSE instruction support" ON)
+mxnet_option(USE_SSE "Build with x86 SSE instruction support" ON IF NOT ARM)
 mxnet_option(USE_F16C "Build with x86 F16C instruction support" ON) # autodetects support if ON
-mxnet_option(USE_LAPACK "Build with lapack support" ON IF NOT MSVC)
+mxnet_option(USE_LAPACK "Build with lapack support" ON)
 mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON)
-mxnet_option(USE_MKLML_MKL "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND UNIX AND (NOT APPLE))
-mxnet_option(USE_MKLDNN "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND UNIX AND (NOT APPLE))
+mxnet_option(USE_MKLML_MKL "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE))
+mxnet_option(USE_MKLDNN "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE))
 mxnet_option(USE_OPERATOR_TUNING "Enable auto-tuning of operators" ON IF NOT MSVC)
 mxnet_option(USE_GPERFTOOLS "Build with GPerfTools support (if found)" ON)
 mxnet_option(USE_JEMALLOC "Build with Jemalloc support" ON)
@@ -37,7 +37,9 @@ mxnet_option(ENABLE_CUDA_RTC "Build with CUDA runtime compilation support"
 mxnet_option(BUILD_CPP_EXAMPLES "Build cpp examples" ON)
 mxnet_option(INSTALL_EXAMPLES "Install the example source files." OFF)
 mxnet_option(USE_SIGNAL_HANDLER "Print stack traces on segfaults." OFF)
+mxnet_option(USE_TENSORRT "Enable infeference optimization with TensorRT." OFF)
 
+message(STATUS "CMAKE_SYSTEM_NAME ${CMAKE_SYSTEM_NAME}")
 if(USE_CUDA AND NOT USE_OLDCMAKECUDA)
  message(STATUS "CMake version '${CMAKE_VERSION}' using generator '${CMAKE_GENERATOR}'")
  if(
@@ -87,7 +89,6 @@ if(MSVC)
  add_definitions(-DNNVM_EXPORTS)
  add_definitions(-DDMLC_STRICT_CXX11)
  add_definitions(-DNOMINMAX)
- set(SUPPORT_F16C FALSE)
  if(USE_F16C)
  message("F16C instruction set is not yet supported for MSVC")
  endif()
@@ -130,7 +131,8 @@ else(MSVC)
  else()
  set(SUPPORT_F16C FALSE)
  endif()
- set(CMAKE_C_FLAGS "-Wall -Wno-unknown-pragmas -fPIC -Wno-sign-compare")
+ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+ set(CMAKE_C_FLAGS "-Wall -Wno-unknown-pragmas -Wno-sign-compare")
  if ("${CMAKE_CXX_COMPILER_ID}" MATCHES ".*Clang$")
  set(CMAKE_C_FLAGS "-Wno-braced-scalar-init")
  endif()
@@ -185,34 +187,51 @@ if(USE_VTUNE)
  list(APPEND mxnet_LINKER_LIBS dl)
 endif()
 
-if(USE_MKL_IF_AVAILABLE)
- if(USE_MKLDNN)
- add_subdirectory(3rdparty/mkldnn)
- include_directories(3rdparty/mkldnn/include)
- list(APPEND mxnet_LINKER_LIBS mkldnn)
+if(USE_TENSORRT)
+ message(STATUS "Using TensorRT")
+ set(ONNX_PATH 3rdparty/onnx-tensorrt/third_party/onnx/build/)
+ set(ONNX_TRT_PATH 3rdparty/onnx-tensorrt/build/)
+
+ include_directories(${ONNX_PATH})
+ include_directories(3rdparty/onnx-tensorrt/)
+ include_directories(3rdparty/)
+ add_definitions(-DMXNET_USE_TENSORRT=1)
+ add_definitions(-DONNX_NAMESPACE=onnx)
+
+ find_package(Protobuf REQUIRED)
+
+ find_library(ONNX_LIBRARY NAMES libonnx.so REQUIRED
+ PATHS ${ONNX_PATH}
+ DOC "Path to onnx library.")
+ find_library(ONNX_PROTO_LIBRARY NAMES libonnx_proto.so REQUIRED
+ PATHS ${ONNX_PATH}
+ DOC "Path to onnx_proto library.")
+ find_library(ONNX_TRT_RUNTIME_LIBRARY NAMES libnvonnxparser_runtime.so REQUIRED
+ PATHS ${ONNX_TRT_PATH}
+ DOC "Path to onnx_proto library.")
+ find_library(ONNX_TRT_PARSER_LIBRARY NAMES libnvonnxparser.so REQUIRED
+ PATHS ${ONNX_TRT_PATH}
+ DOC "Path to onnx_proto library.")
+
+ list(APPEND mxnet_LINKER_LIBS libnvinfer.so ${ONNX_TRT_PARSER_LIBRARY} ${ONNX_TRT_RUNTIME_LIBRARY}
+ ${ONNX_PROTO_LIBRARY} ${ONNX_LIBRARY} ${PROTOBUF_LIBRARY})
+endif()
+
+if(USE_MKLDNN)
+ include(cmake/MklDnn.cmake)
+ # CPU architecture (e.g., C5) can't run on another architecture (e.g., g3).
+ if(NOT MSVC)
+ set(ARCH_OPT_FLAGS "-mtune=generic")
  endif()
- find_package(MKL)
+ set(WITH_TEST OFF)
+ set(WITH_EXAMPLE OFF)
+ add_subdirectory(3rdparty/mkldnn)
 
- if(MKL_FOUND)
- include_directories(${MKL_INCLUDE_DIR})
- include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src/operator/mkl)
-
- if(USE_MKLDNN)
- add_definitions(-DMXNET_USE_MKLDNN=1)
- endif()
-
- add_definitions(-DUSE_MKL=1)
- add_definitions(-DCUB_MKL=1)
- list(APPEND mxnet_LINKER_LIBS ${MKL_LIBRARIES})
-
- if(NOT MSVC)
- list(APPEND mxnet_LINKER_LIBS dl)
- endif()
- # If using MKL, use the Intel OMP libraries
- list(APPEND mxnet_LINKER_LIBS iomp5)
- else()
- message(STATUS " MKL not found")
- endif()
+ include_directories(3rdparty/mkldnn/include)
+ add_definitions(-DUSE_MKL=1)
+ add_definitions(-DCUB_MKL=1)
+ add_definitions(-DMXNET_USE_MKLDNN=1)
+ list(APPEND mxnet_LINKER_LIBS mkldnn)
 endif()
 
 # Allow Cuda compiles outside of src tree to find things in 'src' and 'include'
@@ -223,7 +242,7 @@ if(USE_CUDA)
  find_package(CUDA REQUIRED)
  add_definitions(-DMSHADOW_USE_CUDA=1)
  if(FIRST_CUDA AND (NOT USE_OLDCMAKECUDA))
- if(CUDA_TOOLSET STREQUAL "")
+ if(NOT CUDA_TOOLSET)
  set(CUDA_TOOLSET "${CUDA_VERSION_STRING}")
  endif()
  set(CMAKE_GENERATOR_TOOLSET "cuda=${CUDA_TOOLSET},host=x64")
@@ -275,8 +294,8 @@ endforeach()
 include_directories("include")
 include_directories("3rdparty/mshadow")
 include_directories("3rdparty/cub")
-include_directories("3rdparty/nnvm/include")
-include_directories("3rdparty/nnvm/tvm/include")
+include_directories("3rdparty/tvm/nnvm/include")
+include_directories("3rdparty/tvm/include")
 include_directories("3rdparty/dmlc-core/include")
 include_directories("3rdparty/dlpack/include")
 
@@ -334,14 +353,15 @@ endif()
 
 # ---[ OpenCV
 if(USE_OPENCV)
- find_package(OpenCV QUIET COMPONENTS core highgui imgproc imgcodecs)
+ find_package(OpenCV COMPONENTS core highgui imgproc imgcodecs)
  if(NOT OpenCV_FOUND) # if not OpenCV 3.x, then imgcodecs are not found
+ message(STATUS "OpenCV imgcodecs missing")
  find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc)
  endif()
  include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})
  list(APPEND mxnet_LINKER_LIBS ${OpenCV_LIBS})
  message(STATUS " OpenCV_LIBS=${OpenCV_LIBS}")
- message(STATUS "OpenCV found (${OpenCV_CONFIG_PATH})")
+ message(STATUS "OpenCV ${OpenCV_VERSION} found (${OpenCV_CONFIG_PATH})")
  add_definitions(-DMXNET_USE_OPENCV=1)
 else(USE_OPENCV)
  message(STATUS "OpenCV Disabled")
@@ -353,7 +373,11 @@ if(USE_OPENMP)
  find_package(OpenMP REQUIRED)
  # This should build on Windows, but there's some problem and I don't have a Windows box, so
  # could a Windows user please fix?
- if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp/CMakeLists.txt AND SYSTEM_ARCHITECTURE STREQUAL "x86_64" AND NOT MSVC)
+ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp/CMakeLists.txt
+ AND SYSTEM_ARCHITECTURE STREQUAL "x86_64"
+ AND NOT MSVC
+ AND NOT CMAKE_CROSSCOMPILING)
+
  # Intel/llvm OpenMP: https://github.com/llvm-mirror/openmp
  set(OPENMP_STANDALONE_BUILD TRUE)
  set(LIBOMP_ENABLE_SHARED TRUE)
@@ -373,21 +397,21 @@ if(USE_OPENMP)
  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
  endif()
  endif()
-elseif(UNIX)
+elseif(UNIX AND NOT ANDROID)
  list(APPEND mxnet_LINKER_LIBS pthread)
 endif()
 
+
 # ---[ LAPack
-if(USE_LAPACK AND NOT MSVC)
+if(USE_LAPACK)
+ message("USE_LAPACK is ON")
  add_definitions(-DMXNET_USE_LAPACK=1)
- list(APPEND mxnet_LINKER_LIBS lapack)
-else(USE_LAPACK)
- # Workaround for Windows until using new Jenkinsfile.
- if(BLAS STREQUAL "Open" OR BLAS STREQUAL "open" OR USE_BLAS STREQUAL "Open" OR USE_BLAS STREQUAL "open")
- add_definitions(-DMXNET_USE_LAPACK=1)
+ if (NOT MSVC)
+ list(APPEND mxnet_LINKER_LIBS lapack)
  endif()
 endif()
 
+
 # ---[ jemalloc
 if(USE_JEMALLOC)
  find_package(JeMalloc)
@@ -432,13 +456,13 @@ FILE(GLOB_RECURSE CUDA "src/*.cu" "src/*.cuh")
 
 # add nnvm to source
 FILE(GLOB_RECURSE NNVMSOURCE
- 3rdparty/nnvm/src/c_api/*.cc
- 3rdparty/nnvm/src/core/*.cc
- 3rdparty/nnvm/src/pass/*.cc
- 3rdparty/nnvm/src/c_api/*.h
- 3rdparty/nnvm/src/core/*.h
- 3rdparty/nnvm/src/pass/*.h
- 3rdparty/nnvm/include/*.h)
+ 3rdparty/tvm/nnvm/src/c_api/*.cc
+ 3rdparty/tvm/nnvm/src/core/*.cc
+ 3rdparty/tvm/nnvm/src/pass/*.cc
+ 3rdparty/tvm/nnvm/src/c_api/*.h
+ 3rdparty/tvm/nnvm/src/core/*.h
+ 3rdparty/tvm/nnvm/src/pass/*.h
+ 3rdparty/tvm/nnvm/include/*.h)
 list(APPEND SOURCE ${NNVMSOURCE})
 
 # add mshadow file
@@ -448,8 +472,8 @@ list(APPEND SOURCE ${MSHADOWSOURCE})
 list(APPEND CUDA ${MSHADOW_CUDASOURCE})
 
 # add source group
-FILE(GLOB_RECURSE GROUP_SOURCE "src/*.cc" "3rdparty/nnvm/*.cc" "plugin/*.cc")
-FILE(GLOB_RECURSE GROUP_Include "src/*.h" "3rdparty/nnvm/*.h" "3rdparty/mshadow/mshadow/*.h" "plugin/*.h")
+FILE(GLOB_RECURSE GROUP_SOURCE "src/*.cc" "3rdparty/tvm/nnvm/*.cc" "plugin/*.cc")
+FILE(GLOB_RECURSE GROUP_Include "src/*.h" "3rdparty/tvm/nnvm/*.h" "3rdparty/mshadow/mshadow/*.h" "plugin/*.h")
 FILE(GLOB_RECURSE GROUP_CUDA "src/*.cu" "src/*.cuh" "3rdparty/mshadow/mshadow/*.cuh" "plugin/*.cu"
  "plugin/*.cuh" "3rdparty/cub/cub/*.cuh")
 assign_source_group("Source" ${GROUP_SOURCE})
@@ -588,11 +612,6 @@ if(USE_PLUGIN_CAFFE)
  endif()
 endif()
 
-if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/nnvm/CMakeLists.txt")
- set(nnvm_LINKER_LIBS nnvm)
- list(APPEND mxnet_LINKER_LIBS ${nnvm_LINKER_LIBS})
-endif()
-
 if(NOT MSVC)
  # Only add c++11 flags and definitions after cuda compiling
  add_definitions(-DDMLC_USE_CXX11)
@@ -607,7 +626,7 @@ else()
  set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /OPT:REF /OPT:ICF")
  set(CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL "${CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL} /OPT:REF /OPT:ICF")
  set(CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO} /OPT:REF /OPT:ICF")
- 
+
 endif()
 
 set(MXNET_INSTALL_TARGETS mxnet)
@@ -661,7 +680,7 @@ if(USE_PLUGINS_WARPCTC)
 endif()
 
 
-if(USE_OPENCV)
+if(USE_OPENCV AND OpenCV_VERSION_MAJOR GREATER 2)
  add_executable(im2rec "tools/im2rec.cc")
  if(MSVC)
  target_link_libraries(im2rec mxnet)
@@ -672,9 +691,11 @@ if(USE_OPENCV)
  ${mxnet_LINKER_LIBS}
  ${OpenCV_LIBS}
  dmlc
- ${nnvm_LINKER_LIBS}
  ${pslite_LINKER_LIBS}
  )
+else()
+ message(WARNING "OpenCV_VERSION_MAJOR: ${OpenCV_VERSION_MAJOR}, version 3 with imgcodecs \
+ is required for im2rec, im2rec will not be available")
 endif()
 
 target_link_libraries(mxnet PUBLIC dmlc)

CODEOWNERS

@@ -12,13 +12,28 @@
 * @apache/mxnet-committers
 
 # Language bindings
-/R-package/ @thirdwing
-/scala-package/ @yzhliu
-/perl-package/ @sergeykolychev
-/python/ @szha
+/R-package/ @thirdwing
+/scala-package/ @yzhliu @nswamy
+/perl-package/ @sergeykolychev
+/python/ @szha
+/contrib/clojure-package/ @gigasquid
 
 # C++ base
-/src/kvstore/ @rahul003
+/src/kvstore/ @rahul003 @anirudh2290
+/include/ @anirudh2290
+/src/c_api/ @anirudh2290
+/src/common/ @anirudh2290
+/src/engine/ @anirudh2290
+/src/executor/ @anirudh2290
+/src/imperative/ @anirudh2290
+/src/io/ @anirudh2290
+/src/ndarray/ @anirudh2290
+/src/nnvm/ @anirudh2290
+/src/operator/ @anirudh2290
+/src/profiler/ @anirudh2290
+/src/storage/ @anirudh2290
+/tests/cpp/ @anirudh2290
+/cpp-package/ @nswamy
 
 # CMake
 CMakeLists.txt @szha @rahul003