Implement all User Ops with Op Schema

.github/workflows/test.yml

@@ -806,7 +806,7 @@ jobs:
             -DBUILD_TESTING=ON \
             -DCMAKE_C_COMPILER_LAUNCHER=ccache \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-          cmake --build . -j$(nproc) --target oneflow_deps of_cfgobj of_protoobj of_functional_obj of_functional_tensor_obj
+          cmake --build . -j$(nproc) --target oneflow_deps of_cfgobj of_protoobj of_functional_obj of_functional_tensor_obj of_op_schema
       - name: Fetch upstream
         if: ${{ !fromJSON(steps.save-cache.outputs.cache-hit) && github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name }}
         run: |

CMakeLists.txt

@@ -35,6 +35,9 @@ option(WITH_OPENVINO "Option to build with OpenVINO" OFF)
 option(WITH_MLIR "" OFF)
 option(WITH_MLIR_CUDA_CODEGEN "" OFF)
 set(LLVM_PROVIDER "in-tree" CACHE STRING "in-tree, install")
+if (NOT WITH_MLIR)
+  set(LLVM_PROVIDER "install" CACHE STRING "in-tree will build LLVM's ALL, not what we want when not building MLIR" FORCE)
+endif(NOT WITH_MLIR)
 option(WITH_COCOAPI "Option to build with COCO API" ON)
 option(WITH_ZLIB "" ON)
 option(BUILD_GIT_VERSION "" ON)
@@ -223,6 +226,12 @@ if(BUILD_CPP_API)
   endif(BUILD_SHARED_LIBS)
 endif(BUILD_CPP_API)
 
+set(INJA_URL https://github.com/pantor/inja/archive/refs/tags/v3.3.0.zip CACHE STRING "")
+use_mirror(VARIABLE INJA_URL URL ${INJA_URL})
+set(INJA_MD5 611e6b7206d0fb89728a3879f78b4775 CACHE STRING "")
+set(JSON_URL https://github.com/nlohmann/json/releases/download/v3.7.3/include.zip CACHE STRING "")
+use_mirror(VARIABLE JSON_URL URL ${JSON_URL})
+set(JSON_MD5 fb96f95cdf609143e998db401ca4f324 CACHE STRING "")
 include(third_party)
 
 if (BUILD_CUDA)
@@ -275,3 +284,4 @@ add_custom_target(oneflow_deps ALL DEPENDS prepare_oneflow_third_party)
 if (ONEFLOW)
   include(oneflow)
 endif()
+add_subdirectory(ci)

ci/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory(test)

ci/test/CMakeLists.txt

@@ -0,0 +1,25 @@
+set(PYTHON_EXECUTABLE python3 CACHE STRING "python3 exe to run test, usually is the python3 installation oneflow is linked to")
+set(ONEFLOW_SRC_DIR ${CMAKE_SOURCE_DIR} CACHE STRING "source dir of oneflow")
+set(IS_DEV ON CACHE BOOL "")
+set(CTEST_RESOURCE_SPEC_FILE "${CMAKE_CURRENT_SOURCE_DIR}/resource-spec/2x-rtx-2080.json" CACHE STRING "")
+
+# CTEST_OUTPUT_ON_FAILURE=1 CTEST_PARALLEL_LEVEL=20 ninja test
+
+file(GLOB_RECURSE PYTHON_TEST_FILES LIST_DIRECTORIES false RELATIVE ${ONEFLOW_SRC_DIR} "${ONEFLOW_SRC_DIR}/python/oneflow/test_*.py")
+foreach(PYTHON_TEST_FILE ${PYTHON_TEST_FILES})
+  set(TEST_NAME ${PYTHON_TEST_FILE})
+  add_test(NAME ${TEST_NAME}
+    COMMAND ${PYTHON_EXECUTABLE} ${ONEFLOW_SRC_DIR}/${PYTHON_TEST_FILE} --failfast --verbose
+  )
+  set_tests_properties(${TEST_NAME}
+    PROPERTIES
+      ENVIRONMENT "$<$<NOT:$<BOOL:${BUILD_CUDA}>>:ONEFLOW_TEST_CPU_ONLY=1>;$<$<BOOL:${IS_DEV}>:PYTHONPATH=${ONEFLOW_SRC_DIR}/python:$ENV{PYTHONPATH}>"
+      RESOURCE_GROUPS
+        "vram:2000"
+  )
+endforeach()
+set_tests_properties(python/oneflow/test/modules/test_rnn.py
+  PROPERTIES
+    RESOURCE_GROUPS
+      "vram:4000"
+)

ci/test/resource-spec/1x-gtx-1080.json

@@ -0,0 +1,16 @@
+{
+  "version": {
+    "major": 1,
+    "minor": 0
+  },
+  "local": [
+    {
+      "vram": [
+        {
+          "id": "0",
+          "slots": 8117
+        }
+      ]
+    }
+  ]
+}

ci/test/resource-spec/2x-rtx-2080.json

@@ -0,0 +1,20 @@
+{
+  "version": {
+    "major": 1,
+    "minor": 0
+  },
+  "local": [
+    {
+      "vram": [
+        {
+          "id": "0",
+          "slots": 7982
+        },
+        {
+          "id": "1",
+          "slots": 7982
+        }
+      ]
+    }
+  ]
+}

ci/test/resource-spec/4x-rtx-2080ti.json

@@ -0,0 +1,28 @@
+{
+  "version": {
+    "major": 1,
+    "minor": 0
+  },
+  "local": [
+    {
+      "vram": [
+        {
+          "id": "0",
+          "slots": 11019
+        },
+        {
+          "id": "1",
+          "slots": 11019
+        },
+        {
+          "id": "2",
+          "slots": 11019
+        },
+        {
+          "id": "3",
+          "slots": 11019
+        }
+      ]
+    }
+  ]
+}

cmake/oneflow.cmake

@@ -150,6 +150,7 @@ add_custom_target(of_format
   COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ci/check/run_license_format.py -i ${ONEFLOW_PYTHON_DIR} --fix --exclude="oneflow/include" --exclude="oneflow/core"
   COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ci/check/run_clang_format.py --source_dir ${CMAKE_CURRENT_SOURCE_DIR}/oneflow --fix --quiet
   COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ci/check/run_py_format.py --source_dir ${CMAKE_CURRENT_SOURCE_DIR} --fix
+  COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/ci/check/run_clang_format.py --source_dir ${CMAKE_CURRENT_SOURCE_DIR}/tools/oneflow-tblgen --fix --quiet
   )
 # clang tidy
 add_custom_target(of_tidy
@@ -244,6 +245,7 @@ oneflow_add_library(oneflow ${of_all_obj_cc})
 add_dependencies(oneflow of_protoobj)
 add_dependencies(oneflow of_cfgobj)
 add_dependencies(oneflow of_functional_obj)
+add_dependencies(oneflow of_op_schema)
 add_dependencies(oneflow of_git_version)
 
 if (USE_CLANG_FORMAT)
@@ -255,38 +257,33 @@ endif()
 
 target_compile_definitions(oneflow PRIVATE GOOGLE_LOGGING)
 
-set(ONEFLOW_TOOLS_DIR "${PROJECT_BINARY_DIR}/tools")
-oneflow_add_executable(oneflow-gen-ods EXCLUDE_FROM_ALL ${PROJECT_SOURCE_DIR}/oneflow/ir/oneflow-gen-ods/oneflow-gen-ods.cpp)
-set_target_properties(oneflow-gen-ods PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${ONEFLOW_TOOLS_DIR}")
+set(ONEFLOW_TOOLS_DIR "${PROJECT_BINARY_DIR}/tools" CACHE STRING "dir to put binary for debugging and development")
 
 set(LLVM_MONO_REPO_URL "https://github.com/llvm/llvm-project/archive/649d95371680cbf7f740c990c0357372c2bd4058.zip" CACHE STRING "")
 use_mirror(VARIABLE LLVM_MONO_REPO_URL URL ${LLVM_MONO_REPO_URL})
 set(LLVM_MONO_REPO_MD5 "9bda804e5cc61899085fb0f0dce1089f" CACHE STRING "")
 set(ONEFLOW_BUILD_ROOT_DIR "${PROJECT_BINARY_DIR}")
+add_subdirectory(${PROJECT_SOURCE_DIR}/oneflow/ir)
 if (WITH_MLIR)
-  add_subdirectory(${PROJECT_SOURCE_DIR}/oneflow/ir)
   set(ONEFLOW_MLIR_LIBS -Wl,--no-as-needed MLIROneFlowExtension -Wl,--as-needed)
 endif()
 
+include(op_schema)
+
 if(APPLE)
-  set(of_libs -Wl,-force_load oneflow of_protoobj of_cfgobj of_functional_obj)
+  set(of_libs -Wl,-force_load oneflow of_protoobj of_cfgobj of_functional_obj of_op_schema)
   target_link_libraries(oneflow of_protoobj of_cfgobj of_functional_obj glog_imported gflags_imported ${oneflow_third_party_libs})
 elseif(UNIX)
-  set(of_libs -Wl,--whole-archive oneflow of_protoobj of_cfgobj of_functional_obj -Wl,--no-whole-archive -ldl -lrt)
+  set(of_libs -Wl,--whole-archive oneflow of_protoobj of_cfgobj of_functional_obj of_op_schema -Wl,--no-whole-archive -ldl -lrt)
   target_link_libraries(oneflow of_protoobj of_cfgobj of_functional_obj glog_imported gflags_imported ${oneflow_third_party_libs} -Wl,--no-whole-archive -ldl -lrt)
   if(BUILD_CUDA)
     target_link_libraries(oneflow CUDA::cudart_static)
   endif()
 elseif(WIN32)
-  set(of_libs oneflow of_protoobj of_cfgobj of_functional_obj)
+  set(of_libs oneflow of_protoobj of_cfgobj of_functional_obj of_op_schema)
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /WHOLEARCHIVE:oneflow")
 endif()
 
-target_link_libraries(oneflow-gen-ods ${of_libs} ${oneflow_third_party_libs} ${oneflow_exe_third_party_libs})
-if (BUILD_CUDA)
-  target_link_libraries(oneflow-gen-ods CUDA::cudart_static)
-endif()
-
 if(BUILD_PYTHON)
 
   # py ext lib
@@ -301,7 +298,7 @@ if(BUILD_PYTHON)
   pybind11_add_module(oneflow_internal ${PYBIND11_SRCS} ${of_pybind_obj_cc} ${PYBIND_REGISTRY_CC})
   set_compile_options_to_oneflow_target(oneflow_internal)
   set_property(TARGET oneflow_internal PROPERTY CXX_VISIBILITY_PRESET "default")
-  add_dependencies(oneflow_internal of_cfgobj of_functional_obj of_functional_tensor_obj)
+  add_dependencies(oneflow_internal of_cfgobj of_functional_obj of_functional_tensor_obj of_op_schema)
   set_target_properties(oneflow_internal PROPERTIES PREFIX "_")
   set_target_properties(oneflow_internal PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${ONEFLOW_PYTHON_DIR}/oneflow")
   target_link_libraries(oneflow_internal PRIVATE
@@ -340,7 +337,7 @@ if(BUILD_PYTHON)
 endif(BUILD_PYTHON)
 
 if (BUILD_CPP_API)
-  file(GLOB_RECURSE of_cpp_api_files 
+  file(GLOB_RECURSE of_cpp_api_files
     ${PROJECT_SOURCE_DIR}/oneflow/api/cpp/*.cpp
     ${PROJECT_SOURCE_DIR}/oneflow/api/cpp/*.h)
   if(BUILD_MONOLITHIC_LIBONEFLOW_CPP_SO)
@@ -362,6 +359,11 @@ function(oneflow_add_test target_name)
   endif()
   set_target_properties(${target_name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/bin")
   add_test(NAME ${arg_TEST_NAME} COMMAND ${target_name})
+  set_tests_properties(
+    ${arg_TEST_NAME}
+  PROPERTIES
+    ENVIRONMENT "HTTP_PROXY='';HTTPS_PROXY='';http_proxy='';https_proxy='';"
+  )
 endfunction()
 
 # build test

cmake/op_schema.cmake

@@ -0,0 +1,90 @@
+get_property(LLVM_INSTALL_DIR GLOBAL PROPERTY LLVM_INSTALL_DIR)
+set(LLVM_INSTALL_DIR ${THIRD_PARTY_DIR}/llvm)
+set(LLVM_DIR ${LLVM_INSTALL_DIR}/lib/cmake/llvm)
+set(ONEFLOW_OP_GROUPS
+    "ASSIGN"
+    "BINARY"
+    "BROADCAST"
+    "CONV"
+    "CROSS_ENTROPY"
+    "CUDA"
+    "DATASET"
+    "DETECTION"
+    "EAGER"
+    "FUSED"
+    "IDEMPOTENT"
+    "IDENTITY"
+    "IMAGE"
+    "INDICES"
+    "INVOLUTION"
+    "LOSS"
+    "MATH"
+    "MATMUL"
+    "MISC"
+    "NCCL"
+    "NORMALIZATION"
+    "OPTIMIZER"
+    "PADDING"
+    "PARALLEL_CAST"
+    "POOL"
+    "QUANTIZATION"
+    "REDUCE"
+    "RESHAPE"
+    "SCALAR"
+    "SOFTMAX"
+    "SUMMARY"
+    "TENSOR_BUFFER"
+    "TEST"
+    "TRIGONOMETRIC"
+    "UNARY"
+    "UPSAMPLE"
+)
+foreach (OP_GROUP_NAME IN LISTS ONEFLOW_OP_GROUPS)
+    list(APPEND ONEFLOW_SCHEMA_TABLEGEN_FLAGS "-DGET_ONEFLOW_${OP_GROUP_NAME}_OP_DEFINITIONS")
+endforeach()
+list(APPEND ONEFLOW_SCHEMA_TABLEGEN_FLAGS "-DREMOVE_ONEFLOW_MLIR_ONLY_OP_DEFINITIONS")
+
+set(GENERATED_OP_SCHEMA_DIR oneflow/core/framework)
+set(GENERATED_IR_INCLUDE_DIR oneflow/ir/include)
+set(SOURCE_IR_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/oneflow/ir/include)
+set(ONEFLOW_ODS ${SOURCE_IR_INCLUDE_DIR}/OneFlow/OneFlowOps.td)
+
+list(APPEND ONEFLOW_SCHEMA_TABLEGEN_FLAGS "-I${GENERATED_IR_INCLUDE_DIR}")
+list(APPEND ONEFLOW_SCHEMA_TABLEGEN_FLAGS "-I${SOURCE_IR_INCLUDE_DIR}")
+list(APPEND ONEFLOW_SCHEMA_TABLEGEN_FLAGS "-I${LLVM_INSTALL_DIR}/include")
+
+set(GENERATED_OP_SCHEMA_H "${GENERATED_OP_SCHEMA_DIR}/op_generated.h")
+set(GENERATED_OP_SCHEMA_CPP "${GENERATED_OP_SCHEMA_DIR}/op_generated.cpp")
+
+
+set(ONEFLOW_TABLE_GEN_EXE ${LLVM_INSTALL_DIR}/bin/oneflow_tblgen)
+if(LLVM_PROVIDER STREQUAL "in-tree")
+    set(ONEFLOW_TABLE_GEN_TARGET oneflow_tblgen install-oneflow-tblgen install-mlir-headers)
+elseif(LLVM_PROVIDER STREQUAL "install")
+    set(ONEFLOW_TABLE_GEN_TARGET ${ONEFLOW_TABLE_GEN_EXE})
+endif()
+
+file(GLOB_RECURSE ODS_FILES LIST_DIRECTORIES false "${SOURCE_IR_INCLUDE_DIR}/*.td")
+if(NOT ODS_FILES)
+    message(FATAL_ERROR "ODS_FILES not found: ${ODS_FILES}")
+endif()
+add_custom_command(
+    OUTPUT ${GENERATED_OP_SCHEMA_H} ${GENERATED_OP_SCHEMA_CPP}
+    COMMAND ${CMAKE_COMMAND}
+    ARGS -E make_directory ${GENERATED_OP_SCHEMA_DIR}
+    COMMAND ${ONEFLOW_TABLE_GEN_EXE}
+    ARGS --gen-op-schema-h ${ONEFLOW_ODS} ${ONEFLOW_SCHEMA_TABLEGEN_FLAGS} -o ${GENERATED_OP_SCHEMA_H}
+    COMMAND ${ONEFLOW_TABLE_GEN_EXE}
+    ARGS --gen-op-schema-cpp ${ONEFLOW_ODS} ${ONEFLOW_SCHEMA_TABLEGEN_FLAGS}
+           --op-include ${GENERATED_OP_SCHEMA_H} -o ${GENERATED_OP_SCHEMA_CPP}
+    DEPENDS ${ONEFLOW_TABLE_GEN_TARGET}
+            ${ODS_FILES}
+    VERBATIM
+)
+set_source_files_properties(
+    ${GENERATED_OP_SCHEMA_H} ${GENERATED_OP_SCHEMA_CPP} PROPERTIES GENERATED TRUE
+)
+
+oneflow_add_library(of_op_schema OBJECT ${GENERATED_OP_SCHEMA_H} ${GENERATED_OP_SCHEMA_CPP})
+add_dependencies(of_op_schema of_cfgobj)
+add_dependencies(of_op_schema prepare_oneflow_third_party)

cmake/third_party.cmake

@@ -219,6 +219,7 @@ list(APPEND ONEFLOW_THIRD_PARTY_INCLUDE_DIRS
     ${COCOAPI_INCLUDE_DIR}
     ${HALF_INCLUDE_DIR}
     ${JSON_INCLUDE_DIR}
+    ${INJA_INCLUDE_DIR}
     ${ABSL_INCLUDE_DIR}
     ${OPENSSL_INCLUDE_DIR}
     ${FLATBUFFERS_INCLUDE_DIR}

cmake/third_party/json.cmake

@@ -23,6 +23,7 @@ if(THIRD_PARTY)
     )
     add_custom_target(json_create_header_dir
         COMMAND ${CMAKE_COMMAND} -E make_directory ${JSON_INCLUDE_DIR}
+        COMMAND ${CMAKE_COMMAND} -E make_directory ${JSON_INCLUDE_DIR}/nlohmann
         DEPENDS json
     )
     add_custom_target(json_copy_headers_to_destination
@@ -31,6 +32,7 @@ if(THIRD_PARTY)
     foreach(header_file ${JSON_HEADERS})
         add_custom_command(TARGET json_copy_headers_to_destination PRE_BUILD
             COMMAND ${CMAKE_COMMAND} -E copy_if_different ${header_file} ${JSON_INCLUDE_DIR}
+            COMMAND ${CMAKE_COMMAND} -E copy_if_different ${header_file} ${JSON_INCLUDE_DIR}/nlohmann
         )
     endforeach()
 endif(THIRD_PARTY)