Experimental serializer

README.md

@@ -109,7 +109,7 @@ issue.
 
 ### TL;DR
 
-Have llvm-15.0 (or greater) installed and run `make` in the root directory of
+Have llvm-16.0 (or greater) installed and run `make` in the root directory of
 the repository (where this README is).
 
 ### Acquiring LLVM

packaging/CMakeLists.txt

@@ -39,7 +39,7 @@ if (TARGET Halide_Adams2019)
 endif ()
 
 # Halide_LLVM
-foreach (dep IN ITEMS Halide_LLVM Halide_wabt)
+foreach (dep IN ITEMS Halide_LLVM Halide_wabt Halide_flatbuffers)
     if (TARGET ${dep})
         install(TARGETS ${dep} EXPORT Halide_Targets)
     endif ()

run-clang-tidy.sh

@@ -72,6 +72,9 @@ cmake -DCMAKE_BUILD_TYPE=Debug \
 echo Building HalideIncludes...
 cmake --build ${CLANG_TIDY_BUILD_DIR} -j $(nproc) --target HalideIncludes
 
+echo Building flatbuffer stuff...
+cmake --build ${CLANG_TIDY_BUILD_DIR} -j $(nproc) --target generate_fb_header
+
 RUN_CLANG_TIDY=${CLANG_TIDY_LLVM_INSTALL_DIR}/bin/run-clang-tidy
 
 # We deliberately skip apps/ and test/ for now, as the compile commands won't include

src/CMakeLists.txt

@@ -52,6 +52,7 @@ set(HEADER_FILES
     Deinterleave.h
     Derivative.h
     DerivativeUtils.h
+    Deserialization.h
     DeviceAPI.h
     DeviceArgument.h
     DeviceInterface.h
@@ -140,6 +141,7 @@ set(HEADER_FILES
     ScheduleFunctions.h
     Scope.h
     SelectGPUAPI.h
+    Serialization.h
     Simplify.h
     SimplifyCorrelatedDifferences.h
     SimplifySpecializations.h
@@ -221,6 +223,7 @@ set(SOURCE_FILES
     Deinterleave.cpp
     Derivative.cpp
     DerivativeUtils.cpp
+    Deserialization.cpp
     DeviceArgument.cpp
     DeviceInterface.cpp
     Dimension.cpp
@@ -299,6 +302,7 @@ set(SOURCE_FILES
     Schedule.cpp
     ScheduleFunctions.cpp
     SelectGPUAPI.cpp
+    Serialization.cpp
     Simplify.cpp
     Simplify_Add.cpp
     Simplify_And.cpp
@@ -394,6 +398,7 @@ foreach (f IN LISTS HTML_TEMPLATE_FILES)
     target_sources(Halide_c_templates PRIVATE ${DST})
 endforeach ()
 
+
 ##
 # Build the Halide mono-header.
 ##
@@ -418,6 +423,91 @@ add_library(Halide
             # Including these as sources works around the need to "install" Halide_initmod
             $<TARGET_OBJECTS:Halide_initmod>
             $<TARGET_OBJECTS:Halide_c_templates>)
+
+##
+# Flatbuffers and Serialization dependencies.
+##
+
+# Build serialization, enabled by default
+option(WITH_SERIALIZATION "Include experimental Serialization/Deserialization code" ON)
+
+# flatbuffers is small and compiles quickly, but if you want/need to use
+# a local version (via find_package), configure with FLATBUFFERS_USE_FETCHCONTENT=OFF
+option(FLATBUFFERS_USE_FETCHCONTENT "Enable to download the Flatbuffers library via FetchContent" ON)
+set(FLATBUFFERS_VER 23.5.26 CACHE STRING "The Flatbuffers version to use (or download)")
+
+if (WITH_SERIALIZATION)
+    if (FLATBUFFERS_USE_FETCHCONTENT)
+        include(FetchContent)
+        message(STATUS "Fetching flatbuffers ${FLATBUFFERS_VER}...")
+        FetchContent_Declare(
+            flatbuffers
+            GIT_REPOSITORY https://github.com/google/flatbuffers.git
+            GIT_TAG v${FLATBUFFERS_VER}
+            GIT_SHALLOW TRUE
+        )
+        # configuration for flatbuffers
+        set(FLATBUFFERS_BUILD_TESTS OFF)
+        set(FLATBUFFERS_INSTALL OFF)
+        FetchContent_MakeAvailable(flatbuffers)
+        set_target_properties(flatbuffers PROPERTIES POSITION_INDEPENDENT_CODE ON)
+
+        add_library(Halide_flatbuffers INTERFACE)
+        target_sources(Halide_flatbuffers INTERFACE $<BUILD_INTERFACE:$<TARGET_OBJECTS:flatbuffers>>)
+        target_include_directories(Halide_flatbuffers
+                                   SYSTEM # Use -isystem instead of -I; this is a trick so that clang-tidy won't analyze these includes
+                                   INTERFACE
+                                   $<BUILD_INTERFACE:${flatbuffers_SOURCE_DIR}>/include
+                                   $<BUILD_INTERFACE:${flatbuffers_BINARY_DIR}>/include)
+        set_target_properties(Halide_flatbuffers PROPERTIES EXPORT_NAME flatbuffers)
+
+        add_executable(flatbuffers::flatc ALIAS flatc)
+        message(STATUS "Using fetched-and-built flatbuffers, version ${FLATBUFFERS_VER}")
+    else ()
+        # Sadly, there seem to be at least three variations of the Flatbuffer package
+        # in terms of the case of the relevant CMake files; if we guess wrong, we
+        # fail on case-sensitive file systems. We'll try this as a hack workaround:
+        # just try all three. (Note that the internal CMake library name appears to be
+        # `flatbuffers` in all cases.)
+        set(FB_NAME "")
+        foreach (N IN ITEMS flatbuffers Flatbuffers FlatBuffers)
+            # TODO: should we check the version here?
+            find_package(${N} QUIET)
+            if (${N}_FOUND)
+                set(FB_NAME ${N})
+                message(STATUS "Using installed flatbuffers, version ${${N}_VERSION}")
+                break()
+            endif ()
+        endforeach ()
+
+        if (NOT FB_NAME)
+            message(FATAL_ERROR "WITH_SERIALIZATION is ON and FLATBUFFERS_USE_FETCHCONTENT is OFF, "
+                                "but could not find flatbuffers installed locally. "
+                                "Either install flatbuffers or build with WITH_SERIALIZATION=OFF.")
+        endif ()
+
+        add_library(Halide_flatbuffers ALIAS flatbuffers::flatbuffers)
+    endif ()
+
+    set(fb_dir "${Halide_BINARY_DIR}/flatc/include")
+
+    set(fb_def "${CMAKE_CURRENT_SOURCE_DIR}/halide_ir.fbs")
+    set(fb_header "${fb_dir}/halide_ir_generated.h")
+    add_custom_command(
+        OUTPUT "${fb_header}"
+        COMMAND flatbuffers::flatc --cpp -o "${fb_dir}" "${fb_def}"
+        DEPENDS "${fb_def}"
+        VERBATIM
+    )
+    add_custom_target(generate_fb_header DEPENDS "${fb_header}")
+    set_source_files_properties("${fb_header}" PROPERTIES GENERATED TRUE)
+
+    add_dependencies(Halide generate_fb_header)
+    target_include_directories(Halide PRIVATE "$<BUILD_INTERFACE:${fb_dir}>")
+    target_link_libraries(Halide PRIVATE Halide_flatbuffers)
+    target_compile_definitions(Halide PRIVATE WITH_SERIALIZATION)
+endif ()
+
 add_library(Halide::Halide ALIAS Halide)
 
 target_link_libraries(Halide PRIVATE Halide::LLVM)

src/Definition.cpp

@@ -108,6 +108,18 @@ Definition::Definition(const std::vector<Expr> &args, const std::vector<Expr> &v
     }
 }
 
+Definition::Definition(bool is_init, const Expr &predicate, const std::vector<Expr> &args, const std::vector<Expr> &values,
+                       const StageSchedule &schedule, const std::vector<Specialization> &specializations, const std::string &source_location)
+    : contents(new DefinitionContents) {
+    contents->is_init = is_init;
+    contents->values = values;
+    contents->args = args;
+    contents->predicate = predicate;
+    contents->stage_schedule = schedule;
+    contents->specializations = specializations;
+    contents->source_location = source_location;
+}
+
 Definition Definition::get_copy() const {
     internal_assert(contents.defined()) << "Cannot copy undefined Definition\n";
 

src/Definition.h

@@ -47,6 +47,10 @@ class Definition {
     Definition(const std::vector<Expr> &args, const std::vector<Expr> &values,
                const ReductionDomain &rdom, bool is_init);
 
+    /** Construct a Definition with deserialized data. */
+    Definition(bool is_init, const Expr &predicate, const std::vector<Expr> &args, const std::vector<Expr> &values,
+               const StageSchedule &schedule, const std::vector<Specialization> &specializations, const std::string &source_location);
+
     /** Construct an undefined Definition object. */
     Definition();