Experimental serializer

.github/workflows/presubmit.yml

@@ -23,24 +23,24 @@ jobs:
  source: '.'
  extensions: 'h,c,cpp'
  clangFormatVersion: 16
+ # As of Aug 2023, the macOS runners have more RAM (14GB vs 7GB) and CPU (3 cores vs 2)
+ # than the Linux and Windows runners, so let's use those instead, since clang-tidy is
+ # a bit of a sluggard
  check_clang_tidy:
  name: Check clang-tidy
- runs-on: ubuntu-20.04
+ runs-on: macos-13
  steps:
  - uses: actions/checkout@v3
  - name: Install clang-tidy
  run: |
- # from apt.llvm.org
- # wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
- sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 15CF4D18AF4F7421
- sudo apt-add-repository "deb https://apt.llvm.org/$(lsb_release -sc)/ llvm-toolchain-$(lsb_release -sc)-16 main"
- sudo apt-get update
- sudo apt-get install llvm-16 clang-16 liblld-16-dev libclang-16-dev clang-tidy-16 ninja-build
+ brew update
+ brew install llvm@16 ninja coreutils
  - name: Run clang-tidy
  run: |
- export CC=clang-16
- export CXX=clang++-16
- export CLANG_TIDY_LLVM_INSTALL_DIR=/usr/lib/llvm-16
+ export CC=/usr/local/opt/llvm/bin/clang
+ export CXX=/usr/local/opt/llvm/bin/clang++
+ export CLANG_TIDY_LLVM_INSTALL_DIR=/usr/local/opt/llvm
+ export CMAKE_GENERATOR=Ninja
  ./run-clang-tidy.sh
  check_cmake_file_lists:
  name: Check CMake file lists

README.md

@@ -109,7 +109,7 @@ issue.
 
 ### TL;DR
 
-Have llvm-15.0 (or greater) installed and run `make` in the root directory of
+Have llvm-16.0 (or greater) installed and run `make` in the root directory of
 the repository (where this README is).
 
 ### Acquiring LLVM

packaging/CMakeLists.txt

@@ -39,7 +39,7 @@ if (TARGET Halide_Adams2019)
 endif ()
 
 # Halide_LLVM
-foreach (dep IN ITEMS Halide_LLVM Halide_wabt)
+foreach (dep IN ITEMS Halide_LLVM Halide_wabt Halide_flatbuffers)
  if (TARGET ${dep})
  install(TARGETS ${dep} EXPORT Halide_Targets)
  endif ()

run-clang-tidy.sh

@@ -4,9 +4,31 @@ set -e
 
 ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-[[ "$1" != "" && "$1" != "-fix" ]] && echo "The only supported argument is -fix" && exit
-
-FIX=$1
+usage() { echo "Usage: $0 [-j MAX_PROCESS_COUNT] [-f]" 1>&2; exit 1; }
+
+J=$(nproc)
+FIX=
+
+while getopts ":j:f" o; do
+ case "${o}" in
+ j)
+ J="${OPTARG}"
+ [[ "${J}" =~ ^[0-9]+$ ]] || ( echo "-j requires an integer argument"; usage )
+ ;;
+ f)
+ FIX="-fix"
+ ;;
+ *)
+ usage
+ ;;
+ esac
+done
+shift $((OPTIND-1))
+
+echo "Using ${J} processes."
+if [ -n "${FIX}" ]; then
+ echo "Operating in -fix mode!"
+fi
 
 # We are currently standardized on using LLVM/Clang16 for this script.
 # Note that this is totally independent of the version of LLVM that you
@@ -47,7 +69,11 @@ cmake -DCMAKE_BUILD_TYPE=Debug \
 [ -a ${CLANG_TIDY_BUILD_DIR}/compile_commands.json ]
 
 # We must populate the includes directory to check things outside of src/
-cmake --build ${CLANG_TIDY_BUILD_DIR} --target HalideIncludes
+echo Building HalideIncludes...
+cmake --build ${CLANG_TIDY_BUILD_DIR} -j $(nproc) --target HalideIncludes
+
+echo Building flatbuffer stuff...
+cmake --build ${CLANG_TIDY_BUILD_DIR} -j $(nproc) --target generate_fb_header
 
 RUN_CLANG_TIDY=${CLANG_TIDY_LLVM_INSTALL_DIR}/bin/run-clang-tidy
 
@@ -57,6 +83,7 @@ RUN_CLANG_TIDY=${CLANG_TIDY_LLVM_INSTALL_DIR}/bin/run-clang-tidy
 # Skip DefaultCostModel.cpp as it relies on cost_model.h.
 # Skip GenGen.cpp and RunGenMain.cpp as they bring clang-tidy to its knees,
 # for reasons that aren't entirely clear yet.
+echo Finding targets...
 CLANG_TIDY_TARGETS=$(find \
  "${ROOT_DIR}/src" \
  "${ROOT_DIR}/python_bindings" \
@@ -71,9 +98,10 @@ CLANG_TIDY_TARGETS=$(find \
 # so we will instead build an include filter
 CLANG_TIDY_HEADER_FILTER=".*/src/.*|.*/python_bindings/.*|.*/tools/.*|.*/util/.*"
 
+echo Running clang-tidy...
 ${RUN_CLANG_TIDY} \
  ${FIX} \
- -j $(nproc) \
+ -j ${J} \
  -header-filter="${CLANG_TIDY_HEADER_FILTER}" \
  -quiet \
  -p ${CLANG_TIDY_BUILD_DIR} \

src/CMakeLists.txt

@@ -52,6 +52,7 @@ set(HEADER_FILES
  Deinterleave.h
  Derivative.h
  DerivativeUtils.h
+ Deserialization.h
  DeviceAPI.h
  DeviceArgument.h
  DeviceInterface.h
@@ -140,6 +141,7 @@ set(HEADER_FILES
  ScheduleFunctions.h
  Scope.h
  SelectGPUAPI.h
+ Serialization.h
  Simplify.h
  SimplifyCorrelatedDifferences.h
  SimplifySpecializations.h
@@ -221,6 +223,7 @@ set(SOURCE_FILES
  Deinterleave.cpp
  Derivative.cpp
  DerivativeUtils.cpp
+ Deserialization.cpp
  DeviceArgument.cpp
  DeviceInterface.cpp
  Dimension.cpp
@@ -299,6 +302,7 @@ set(SOURCE_FILES
  Schedule.cpp
  ScheduleFunctions.cpp
  SelectGPUAPI.cpp
+ Serialization.cpp
  Simplify.cpp
  Simplify_Add.cpp
  Simplify_And.cpp
@@ -394,6 +398,7 @@ foreach (f IN LISTS HTML_TEMPLATE_FILES)
  target_sources(Halide_c_templates PRIVATE ${DST})
 endforeach ()
 
+
 ##
 # Build the Halide mono-header.
 ##
@@ -418,6 +423,91 @@ add_library(Halide
  # Including these as sources works around the need to "install" Halide_initmod
  $<TARGET_OBJECTS:Halide_initmod>
  $<TARGET_OBJECTS:Halide_c_templates>)
+
+##
+# Flatbuffers and Serialization dependencies.
+##
+
+# Build serialization, enabled by default
+option(WITH_SERIALIZATION "Include experimental Serialization/Deserialization code" ON)
+
+# flatbuffers is small and compiles quickly, but if you want/need to use
+# a local version (via find_package), configure with FLATBUFFERS_USE_FETCHCONTENT=OFF
+option(FLATBUFFERS_USE_FETCHCONTENT "Enable to download the Flatbuffers library via FetchContent" ON)
+set(FLATBUFFERS_VER 23.5.26 CACHE STRING "The Flatbuffers version to use (or download)")
+
+if (WITH_SERIALIZATION)
+ if (FLATBUFFERS_USE_FETCHCONTENT)
+ include(FetchContent)
+ message(STATUS "Fetching flatbuffers ${FLATBUFFERS_VER}...")
+ FetchContent_Declare(
+ flatbuffers
+ GIT_REPOSITORY https://github.com/google/flatbuffers.git
+ GIT_TAG v${FLATBUFFERS_VER}
+ GIT_SHALLOW TRUE
+ )
+ # configuration for flatbuffers
+ set(FLATBUFFERS_BUILD_TESTS OFF)
+ set(FLATBUFFERS_INSTALL OFF)
+ FetchContent_MakeAvailable(flatbuffers)
+ set_target_properties(flatbuffers PROPERTIES POSITION_INDEPENDENT_CODE ON)
+
+ add_library(Halide_flatbuffers INTERFACE)
+ target_sources(Halide_flatbuffers INTERFACE $<BUILD_INTERFACE:$<TARGET_OBJECTS:flatbuffers>>)
+ target_include_directories(Halide_flatbuffers
+ SYSTEM # Use -isystem instead of -I; this is a trick so that clang-tidy won't analyze these includes
+ INTERFACE
+ $<BUILD_INTERFACE:${flatbuffers_SOURCE_DIR}>/include
+ $<BUILD_INTERFACE:${flatbuffers_BINARY_DIR}>/include)
+ set_target_properties(Halide_flatbuffers PROPERTIES EXPORT_NAME flatbuffers)
+
+ add_executable(flatbuffers::flatc ALIAS flatc)
+ message(STATUS "Using fetched-and-built flatbuffers, version ${FLATBUFFERS_VER}")
+ else ()
+ # Sadly, there seem to be at least three variations of the Flatbuffer package
+ # in terms of the case of the relevant CMake files; if we guess wrong, we
+ # fail on case-sensitive file systems. We'll try this as a hack workaround:
+ # just try all three. (Note that the internal CMake library name appears to be
+ # `flatbuffers` in all cases.)
+ set(FB_NAME "")
+ foreach (N IN ITEMS flatbuffers Flatbuffers FlatBuffers)
+ # TODO: should we check the version here?
+ find_package(${N} QUIET)
+ if (${N}_FOUND)
+ set(FB_NAME ${N})
+ message(STATUS "Using installed flatbuffers, version ${${N}_VERSION}")
+ break()
+ endif ()
+ endforeach ()
+
+ if (NOT FB_NAME)
+ message(FATAL_ERROR "WITH_SERIALIZATION is ON and FLATBUFFERS_USE_FETCHCONTENT is OFF, "
+ "but could not find flatbuffers installed locally. "
+ "Either install flatbuffers or build with WITH_SERIALIZATION=OFF.")
+ endif ()
+
+ add_library(Halide_flatbuffers ALIAS flatbuffers::flatbuffers)
+ endif ()
+
+ set(fb_dir "${Halide_BINARY_DIR}/flatc/include")
+
+ set(fb_def "${CMAKE_CURRENT_SOURCE_DIR}/halide_ir.fbs")
+ set(fb_header "${fb_dir}/halide_ir_generated.h")
+ add_custom_command(
+ OUTPUT "${fb_header}"
+ COMMAND flatbuffers::flatc --cpp -o "${fb_dir}" "${fb_def}"
+ DEPENDS "${fb_def}"
+ VERBATIM
+ )
+ add_custom_target(generate_fb_header DEPENDS "${fb_header}")
+ set_source_files_properties("${fb_header}" PROPERTIES GENERATED TRUE)
+
+ add_dependencies(Halide generate_fb_header)
+ target_include_directories(Halide PRIVATE "$<BUILD_INTERFACE:${fb_dir}>")
+ target_link_libraries(Halide PRIVATE Halide_flatbuffers)
+ target_compile_definitions(Halide PRIVATE WITH_SERIALIZATION)
+endif ()
+
 add_library(Halide::Halide ALIAS Halide)
 
 target_link_libraries(Halide PRIVATE Halide::LLVM)

src/Definition.cpp

@@ -108,6 +108,18 @@ Definition::Definition(const std::vector<Expr> &args, const std::vector<Expr> &v
  }
 }
 
+Definition::Definition(bool is_init, const Expr &predicate, const std::vector<Expr> &args, const std::vector<Expr> &values,
+ const StageSchedule &schedule, const std::vector<Specialization> &specializations, const std::string &source_location)
+ : contents(new DefinitionContents) {
+ contents->is_init = is_init;
+ contents->values = values;
+ contents->args = args;
+ contents->predicate = predicate;
+ contents->stage_schedule = schedule;
+ contents->specializations = specializations;
+ contents->source_location = source_location;
+}
+
 Definition Definition::get_copy() const {
  internal_assert(contents.defined()) << "Cannot copy undefined Definition\n";
 

src/Definition.h

@@ -47,6 +47,10 @@ class Definition {
  Definition(const std::vector<Expr> &args, const std::vector<Expr> &values,
  const ReductionDomain &rdom, bool is_init);
 
+ /** Construct a Definition with deserialized data. */
+ Definition(bool is_init, const Expr &predicate, const std::vector<Expr> &args, const std::vector<Expr> &values,
+ const StageSchedule &schedule, const std::vector<Specialization> &specializations, const std::string &source_location);
+
  /** Construct an undefined Definition object. */
  Definition();