cmake: import gflags and glog using FetchContent

.github/workflows/test.yml

@@ -534,16 +534,6 @@ jobs:
         if: ${{ !fromJson(matrix.cache-hit) && matrix.test-type == 'misc' && matrix.device == 'cuda' }}
         run: |
           docker exec ${{ env.TEST_CONTAINER_NAME }} bash ci/test/doctest.sh
-      - name: Custom op test
-        timeout-minutes: 45
-        if: ${{ !fromJson(matrix.cache-hit) && matrix.test-type == 'misc' && matrix.device == 'cpu' }}
-        run: |
-          docker exec ${{ env.TEST_CONTAINER_NAME }} bash ci/test/1node_custom_op_test.sh
-      - name: Single client dry run test (run without runtime)
-        timeout-minutes: 45
-        if: ${{ !fromJson(matrix.cache-hit) && matrix.test-type == 'misc' && matrix.device == 'cuda' }}
-        run: |
-          docker exec ${{ env.TEST_CONTAINER_NAME }} bash ci/test/dry_run_test.sh
       - name: Checkout Oneflow-Inc/models
         if: ${{ !fromJson(matrix.cache-hit) && matrix.test-type == 'speed-test' && matrix.device == 'cuda' }}
         uses: actions/checkout@v2

cmake/oneflow.cmake

@@ -210,6 +210,7 @@ include(functional)
 GENERATE_FUNCTIONAL_API_AND_PYBIND11_CPP(
     FUNCTIONAL_GENERATED_SRCS FUNCTIONAL_GENERATED_HRCS FUNCTIONAL_PYBIND11_SRCS ${PROJECT_SOURCE_DIR})
 oneflow_add_library(of_functional_obj STATIC ${FUNCTIONAL_GENERATED_SRCS} ${FUNCTIONAL_GENERATED_HRCS})
+target_link_libraries(of_functional_obj glog::glog)
 add_dependencies(of_functional_obj of_cfgobj)
 add_dependencies(of_functional_obj prepare_oneflow_third_party)
 
@@ -226,6 +227,7 @@ if(BUILD_PYTHON)
   oneflow_add_library(of_functional_tensor_obj STATIC
       ${FUNCTIONAL_TENSOR_GENERATED_SRCS} ${FUNCTIONAL_TENSOR_GENERATED_HRCS}
       ${FUNCTIONAL_OPS_GENERATED_SRCS} ${FUNCTIONAL_OPS_GENERATED_HRCS})
+  target_link_libraries(of_functional_tensor_obj glog::glog)
   add_dependencies(of_functional_tensor_obj of_cfgobj)
   add_dependencies(of_functional_tensor_obj prepare_oneflow_third_party)
   target_include_directories(of_functional_tensor_obj PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS})
@@ -274,10 +276,10 @@ include(op_schema)
 
 if(APPLE)
   set(of_libs -Wl,-force_load oneflow of_protoobj of_cfgobj of_functional_obj of_op_schema)
-  target_link_libraries(oneflow of_protoobj of_cfgobj of_functional_obj glog_imported gflags_imported ${oneflow_third_party_libs})
+  target_link_libraries(oneflow of_protoobj of_cfgobj of_functional_obj ${oneflow_third_party_libs})
 elseif(UNIX)
   set(of_libs -Wl,--whole-archive oneflow of_protoobj of_cfgobj of_functional_obj of_op_schema -Wl,--no-whole-archive -ldl -lrt)
-  target_link_libraries(oneflow of_protoobj of_cfgobj of_functional_obj glog_imported gflags_imported ${oneflow_third_party_libs} -Wl,--no-whole-archive -ldl -lrt)
+  target_link_libraries(oneflow of_protoobj of_cfgobj of_functional_obj ${oneflow_third_party_libs} -Wl,--no-whole-archive -ldl -lrt)
   if(BUILD_CUDA)
     target_link_libraries(oneflow CUDA::cudart_static)
   endif()
@@ -321,7 +323,7 @@ if(BUILD_PYTHON)
                         of_api_common
                         ${oneflow_third_party_libs}
                         of_pyext_obj
-                        ${oneflow_exe_third_party_libs})
+                        glog::glog)
   target_include_directories(oneflow_internal PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS})
 
   target_compile_definitions(oneflow_internal PRIVATE ONEFLOW_CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE})
@@ -384,7 +386,7 @@ endfunction()
 if(BUILD_TESTING)
   if (of_all_test_cc)
     oneflow_add_test(oneflow_testexe SRCS ${of_all_test_cc} TEST_NAME oneflow_test)
-    target_link_libraries(oneflow_testexe ${of_libs} ${oneflow_third_party_libs} ${oneflow_exe_third_party_libs} ${oneflow_test_libs})
+    target_link_libraries(oneflow_testexe ${of_libs} ${oneflow_third_party_libs} glog::glog ${oneflow_test_libs})
   endif()
 
   if (BUILD_CPP_API)

cmake/op_schema.cmake

@@ -86,5 +86,6 @@ set_source_files_properties(
 )
 
 oneflow_add_library(of_op_schema OBJECT ${GENERATED_OP_SCHEMA_H} ${GENERATED_OP_SCHEMA_CPP})
+target_link_libraries(of_op_schema glog::glog)
 add_dependencies(of_op_schema of_cfgobj)
 add_dependencies(of_op_schema prepare_oneflow_third_party)

cmake/pybind11.cmake

@@ -1,8 +1,9 @@
 include(FetchContent)
 
-set(PYBIND11_URL https://github.com/pybind/pybind11/archive/v2.7.0.zip)
-use_mirror(VARIABLE PYBIND11_URL URL ${PYBIND11_URL})
-set(PYBIND11_URL_HASH 267807f790ef598ef912a79aceefdc10)
+set_mirror_url_with_hash(PYBIND11_URL 
+    https://github.com/pybind/pybind11/archive/v2.7.0.zip
+    267807f790ef598ef912a79aceefdc10
+)
 
 FetchContent_Declare(
     pybind11

cmake/third_party.cmake

@@ -8,7 +8,6 @@ if (WITH_ZLIB)
 endif()
 include(protobuf)
 include(googletest)
-include(gflags)
 include(glog)
 include(libjpeg-turbo)
 include(opencv)
@@ -130,12 +129,6 @@ else()
 endif()
 message(STATUS "Found Blas Lib: " ${BLAS_LIBRARIES})
 
-# libraries only a top level .so or exe should be linked to
-set(oneflow_exe_third_party_libs
-    glog_imported
-    gflags_imported
-)
-
 set(oneflow_test_libs
     ${GOOGLETEST_STATIC_LIBRARIES}
     ${GOOGLEMOCK_STATIC_LIBRARIES}
@@ -179,8 +172,6 @@ endif()
 
 set(oneflow_third_party_dependencies
   protobuf
-  gflags
-  glog
   googletest
   opencv_copy_headers_to_destination
   libpng_copy_headers_to_destination
@@ -210,8 +201,6 @@ endif()
 
 list(APPEND ONEFLOW_THIRD_PARTY_INCLUDE_DIRS
     ${ZLIB_INCLUDE_DIR}
-    ${GFLAGS_INCLUDE_DIR}
-    ${GLOG_INCLUDE_DIR}
     ${GOOGLETEST_INCLUDE_DIR}
     ${GOOGLEMOCK_INCLUDE_DIR}
     ${PROTOBUF_INCLUDE_DIR}

cmake/third_party/glog.cmake

@@ -1,68 +1,22 @@
 include (ExternalProject)
 
-set(GLOG_INSTALL_DIR ${THIRD_PARTY_DIR}/glog/install)
-set(GLOG_INCLUDE_DIR ${GLOG_INSTALL_DIR}/include)
-set(GLOG_LIBRARY_DIR ${GLOG_INSTALL_DIR}/lib)
-
-set(glog_URL https://github.com/Oneflow-Inc/glog/archive/4f3e18bf2.tar.gz)
-use_mirror(VARIABLE glog_URL URL ${glog_URL})
-
-if(WIN32)
-    set(GLOG_LIBRARY_NAMES glog.lib)
-else()
-    if(BUILD_SHARED_LIBS)
-      if("${CMAKE_SHARED_LIBRARY_SUFFIX}" STREQUAL ".dylib")
-        set(GLOG_LIBRARY_NAMES libglog.dylib)
-      elseif("${CMAKE_SHARED_LIBRARY_SUFFIX}" STREQUAL ".so")
-        set(GLOG_LIBRARY_NAMES libglog.so)
-      else()
-        message(FATAL_ERROR "${CMAKE_SHARED_LIBRARY_SUFFIX} not support for glog")
-      endif()
-    else()
-      set(GLOG_LIBRARY_NAMES libglog.a)
-    endif()
-endif()
-
-foreach(LIBRARY_NAME ${GLOG_LIBRARY_NAMES})
-    list(APPEND GLOG_STATIC_LIBRARIES ${GLOG_LIBRARY_DIR}/${LIBRARY_NAME})
-endforeach()
-
-set (GLOG_PUBLIC_H
-  ${CMAKE_CURRENT_BINARY_DIR}/glog/src/glog/config.h
-  ${CMAKE_CURRENT_BINARY_DIR}/glog/src/glog/glog/logging.h
-  ${CMAKE_CURRENT_BINARY_DIR}/glog/src/glog/glog/raw_logging.h
-  ${CMAKE_CURRENT_BINARY_DIR}/glog/src/glog/glog/stl_logging.h
-  ${CMAKE_CURRENT_BINARY_DIR}/glog/src/glog/glog/vlog_is_on.h
-  ${CMAKE_CURRENT_BINARY_DIR}/glog/src/glog/src/glog/log_severity.h
+set_mirror_url_with_hash(glog_URL 
+  https://github.com/google/glog/archive/refs/tags/v0.5.0.tar.gz
+  2368e3e0a95cce8b5b35a133271b480f
 )
 
-if(THIRD_PARTY)
+include(FetchContent)
 
-ExternalProject_Add(glog
-    DEPENDS gflags
-    PREFIX glog
-    URL ${glog_URL}
-    URL_MD5 3ca928ef755c0a890680e023e3d4b9a6
-    UPDATE_COMMAND ""
-    BUILD_IN_SOURCE 1
-    BUILD_BYPRODUCTS ${GLOG_STATIC_LIBRARIES}
-    CMAKE_CACHE_ARGS
-        -DCMAKE_C_COMPILER_LAUNCHER:STRING=${CMAKE_C_COMPILER_LAUNCHER}
-        -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=${CMAKE_CXX_COMPILER_LAUNCHER}
-        -DCMAKE_POLICY_DEFAULT_CMP0074:STRING=NEW
-        -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
-        -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG}
-        -DCMAKE_CXX_FLAGS_RELEASE:STRING=${CMAKE_CXX_FLAGS_RELEASE}
-        -DCMAKE_C_FLAGS_DEBUG:STRING=${CMAKE_C_FLAGS_DEBUG}
-        -DCMAKE_C_FLAGS_RELEASE:STRING=${CMAKE_C_FLAGS_RELEASE}
-        -DBUILD_SHARED_LIBS:BOOL=${PROTOBUF_BUILD_SHARED_LIBS}
-        -DBUILD_TESTING:BOOL=OFF
-        -DWITH_GFLAGS:BOOL=ON
-        -Dgflags_ROOT:STRING=${GFLAGS_INSTALL_DIR}
-        -DCMAKE_INSTALL_PREFIX:STRING=${GLOG_INSTALL_DIR}
-        -DCMAKE_INSTALL_MESSAGE:STRING=${CMAKE_INSTALL_MESSAGE}
+FetchContent_Declare(
+  glog
+  URL ${glog_URL}
+  URL_HASH MD5=${glog_URL_HASH}
 )
 
-endif(THIRD_PARTY)
-add_library(glog_imported UNKNOWN IMPORTED)
-set_property(TARGET glog_imported PROPERTY IMPORTED_LOCATION "${GLOG_STATIC_LIBRARIES}")
+set(WITH_GFLAGS OFF CACHE BOOL "")
+set(BUILD_SHARED_LIBS OFF CACHE BOOL "")
+FetchContent_MakeAvailable(glog)
+
+# just for tensorflow, DO NOT USE IN OTHER PLACE
+FetchContent_GetProperties(glog)
+set(GLOG_INCLUDE_DIR ${glog_BINARY_DIR})

oneflow/core/common/util.h

@@ -18,7 +18,6 @@ limitations under the License.
 
 #include "oneflow/core/common/preprocessor.h"
 
-#include <gflags/gflags.h>
 #include <glog/logging.h>
 #include <gtest/gtest.h>
 #include <algorithm>
@@ -42,8 +41,6 @@ limitations under the License.
 #include "oneflow/core/common/global.h"
 #include "oneflow/core/common/cpp_attribute.h"
 
-DECLARE_string(log_dir);
-
 #define CHECK_ISNULL(e) CHECK((e) == nullptr)
 
 namespace oneflow {

oneflow/ir/CMakeLists.txt

@@ -50,7 +50,7 @@ endif()
 
 set_property(GLOBAL PROPERTY ALL_ONEFLOW_LIBS
   -Wl,--no-as-needed oneflow -Wl,--as-needed
-  -Wl,--no-as-needed ${oneflow_exe_third_party_libs} -Wl,--as-needed
+  -Wl,--no-as-needed glog::glog -Wl,--as-needed
   -Wl,--no-as-needed ${oneflow_third_party_libs} -Wl,--as-needed
 )
 

oneflow/xrt/api.cpp

@@ -29,29 +29,33 @@ limitations under the License.
 #include "oneflow/xrt/tensorrt/trt_int8_calibrator.h"
 #endif  // WITH_TENSORRT
 
-DEFINE_int32(clustering_minimum_nodes, EnvToInt(FLAGS_clustering_minimum_nodes, 1),
-             "Minium nodes of a cluster after clustering.");
-DEFINE_int32(clustering_maximum_nodes, EnvToInt(FLAGS_clustering_maximum_nodes, 1000),
-             "Maxium nodes of a cluster after clustering.");
-DEFINE_bool(strict_clustering, EnvToBool(FLAGS_strict_clustering, true),
-            "Option to clustering with strict dependencies analysis.");
+// Minium nodes of a cluster after clustering.
+int32_t FLAGS_clustering_minimum_nodes = EnvToInt(FLAGS_clustering_minimum_nodes, 1);
+// Maxium nodes of a cluster after clustering.
+int32_t FLAGS_clustering_maximum_nodes = EnvToInt(FLAGS_clustering_maximum_nodes, 1000);
+// Option to clustering with strict dependencies analysis.
+bool FLAGS_strict_clustering = EnvToBool(FLAGS_strict_clustering, true);
 
 // DEFINE_string(engine, EnvToString(FLAGS_engine, "XLA"),
 //               "Which third party engine to be used. XLA and TENSORRT are "
 //               "valid, Default means using no engine.");
-DEFINE_bool(use_xla_jit, EnvToBool(FLAGS_use_xla_jit, false), "It's optional to use xla jit.");
-DEFINE_bool(use_tensorrt, EnvToBool(FLAGS_use_tensorrt, false), "It's optional to use tensorrt.");
-DEFINE_bool(use_openvino, EnvToBool(FLAGS_use_openvino, false), "It's optional to use openvino.");
-
-DEFINE_bool(tensorrt_fp16, EnvToBool(FLAGS_tensorrt_fp16, false),
-            "Enable fp16 precision for TENSORRT engine.");
-DEFINE_bool(tensorrt_int8, EnvToBool(FLAGS_tensorrt_int8, false),
-            "Enable int8 precision for TENSORRT engine.");
-
-DEFINE_string(int8_calibration, EnvToString(FLAGS_int8_calibration, ""),
-              "TensorRT int8 calibration table directory. "
-              "Default is empty, and this means the calibration table will be "
-              "implictly generated if tensorrt_int8 flag is true.");
+
+// It's optional to use xla jit.
+bool FLAGS_use_xla_jit = EnvToBool(FLAGS_use_xla_jit, false);
+// It's optional to use tensorrt.
+bool FLAGS_use_tensorrt = EnvToBool(FLAGS_use_tensorrt, false);
+// It's optional to use openvino.
+bool FLAGS_use_openvino = EnvToBool(FLAGS_use_openvino, false);
+
+// Enable fp16 precision for TENSORRT engine.
+bool FLAGS_tensorrt_fp16 = EnvToBool(FLAGS_tensorrt_fp16, false);
+// Enable int8 precision for TENSORRT engine.
+bool FLAGS_tensorrt_int8 = EnvToBool(FLAGS_tensorrt_int8, false);
+
+// TensorRT int8 calibration table directory.
+// Default is empty, and this means the calibration table will be implictly generated if
+// tensorrt_int8 flag is true.
+std::string FLAGS_int8_calibration = EnvToString(FLAGS_int8_calibration, "");
 
 namespace oneflow {
 namespace xrt {

oneflow/xrt/launch_kernel.cpp

@@ -23,15 +23,16 @@ limitations under the License.
 #include "oneflow/core/ep/cuda/cuda_stream.h"
 
 // General executable setup.
-DEFINE_int64(max_workspace_bytes, EnvToInt64(FLAGS_max_workspace_bytes, -1),
-             "Maximum temporary workspace bytes.");
+// Maximum temporary workspace bytes.
+int64_t FLAGS_max_workspace_bytes = EnvToInt64(FLAGS_max_workspace_bytes, -1);
+
 // TENSORRT executable setup.
-DEFINE_int32(max_batch_size, EnvToInt(FLAGS_max_batch_size, 1),
-             "Maximum batch size for builder of TENSORRT engine.");
+// Maximum batch size for builder of TENSORRT engine.
+int32_t FLAGS_max_batch_size = EnvToInt(FLAGS_max_batch_size, 1);
 
-DECLARE_bool(tensorrt_fp16);
-DECLARE_bool(tensorrt_int8);
-DECLARE_string(int8_calibration);
+extern bool FLAGS_tensorrt_fp16;
+extern bool FLAGS_tensorrt_int8;
+extern std::string FLAGS_int8_calibration;
 
 namespace oneflow {
 namespace xrt {

tools/oneflow-tblgen/CMakeLists.txt

@@ -37,7 +37,10 @@ target_link_libraries(${OF_TBLGEN_TARGET} PRIVATE
   pantor::inja
 )
 
-install(TARGETS oneflow_tblgen LLVMTableGen LLVMDemangle LLVMSupport COMPONENT OneFlowTableGen)
+install(TARGETS oneflow_tblgen LLVMTableGen LLVMDemangle LLVMSupport
+  COMPONENT OneFlowTableGen
+  LIBRARY DESTINATION lib
+)
 add_custom_target(install-oneflow-tblgen
   DEPENDS oneflow_tblgen
   COMMAND