Support using vcpkg to build dependencies on all platforms (halide#8387)

This PR adds support for using vcpkg to acquire and build Halide's
dependencies on all platforms.

It adds a top-level `vcpkg.json` file that explains the relationship
between Halide's features and its dependencies. These features include
the various LLVM `target-`s (which merely imply a dependency on the
corresponding LLVM backend), `serialization` (flatbuffers),
the `python-bindings` (pybind11), the `wasm-executor` (wabt), and a
few meta-features:

* `jit`: enables LLVM targets corresponding to the host system
* `target-all`: enables all LLVM targets
* `tests`: depends on everything needed for the tests and apps
* `developer`: includes all other features

All of these are optional (since x86 and WebAssembly are forced),
but `jit` and `serialization` are on by default.

vcpkg is intended to be an eventual replacement for FetchContent, at
least on the buildbots. It will accelerate builds beyond ccache by
directly restoring binary caches for our dependencies. Unlike
FetchContent, it does not pollute our build with third-party CMake
code. Indeed, our build has no idea at all when vcpkg is in use.

The primary drawback is that vcpkg installation happens during (or
ahead of) configuration time, so there is some initial wait.

## Try it!

I have provided many CMake presets to ease adoption. As long as you
have `VCPKG_ROOT` set to a fresh clone of `vcpkg`, they should work.
They come in two flavors:

* `vcpkg`: this acquires dependencies from the main vcpkg registry, but
  applies our own overlay, which disables building Python 3 (really!)
  and LLVM. The system is searched for these as usual.
* `vcpkg-full`: this disables the Halide overlay and attempts to build
  ALL dependencies.

All these presets enable the `developer` feature
in `VCPKG_MANIFEST_FEATURES`, which can be overridden in the usual way.

Here are the commands you should use to try it locally:

* On Linux or Windows: `cmake --preset release-vcpkg`
* On macOS: `cmake --preset macOS-vcpkg`
* To use Visual Studio: `cmake --preset win32`. Here, `vcpkg` is
  implied and `-vcpkg-full` can be added to build LLVM.

.gitignore

@@ -202,6 +202,9 @@ install_manifest.txt
 # Ninja files
 *.ninja*
 
+# Package managers
+vcpkg_installed/
+
 ################################################################################
 ## IDE directories and metadata
 

CMakeLists.txt

@@ -5,6 +5,16 @@ if (Halide_USE_FETCHCONTENT)
     list(APPEND CMAKE_PROJECT_TOP_LEVEL_INCLUDES "${CMAKE_CURRENT_LIST_DIR}/cmake/dependencies.cmake")
 endif ()
 
+# TODO: remove this after updating build bots.
+if (NOT DEFINED VCPKG_OVERLAY_PORTS)
+    set(VCPKG_OVERLAY_PORTS "${CMAKE_CURRENT_LIST_DIR}/cmake/vcpkg")
+endif ()
+
+# TODO: remove this after updating build bots.
+if (NOT DEFINED VCPKG_MANIFEST_FEATURES)
+    set(VCPKG_MANIFEST_FEATURES developer)
+endif ()
+
 project(Halide
         VERSION 19.0.0
         DESCRIPTION "Halide compiler and libraries"

CMakePresets.json

@@ -1,8 +1,8 @@
 {
-  "version": 3,
+  "version": 6,
   "cmakeMinimumRequired": {
     "major": 3,
-    "minor": 22,
+    "minor": 28,
     "patch": 0
   },
   "configurePresets": [
@@ -18,30 +18,42 @@
       "inherits": "base",
       "toolchainFile": "${sourceDir}/cmake/toolchain.${presetName}.cmake",
       "cacheVariables": {
-        "CMAKE_BUILD_TYPE": "RelWithDebInfo"
+        "CMAKE_BUILD_TYPE": "RelWithDebInfo",
+        "Halide_LLVM_SHARED_LIBS": false
       }
     },
     {
-      "name": "windows-only",
-      "hidden": true,
-      "condition": {
-        "type": "equals",
-        "lhs": "${hostSystemName}",
-        "rhs": "Windows"
+      "name": "vcpkg",
+      "inherits": "base",
+      "displayName": "vcpkg deps",
+      "description": "Build dependencies (with Halide exclusions) with vcpkg",
+      "toolchainFile": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
+      "cacheVariables": {
+        "VCPKG_MANIFEST_FEATURES": "developer",
+        "VCPKG_OVERLAY_PORTS": "${sourceDir}/cmake/vcpkg",
+        "Halide_USE_FETCHCONTENT": false
       }
     },
     {
-      "name": "vcpkg",
-      "hidden": true,
-      "toolchainFile": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake"
+      "name": "vcpkg-full",
+      "inherits": "vcpkg",
+      "displayName": "vcpkg deps (all dependencies)",
+      "description": "Build ALL dependencies with vcpkg",
+      "cacheVariables": {
+        "VCPKG_OVERLAY_PORTS": ""
+      }
     },
     {
       "name": "vs2022",
       "hidden": true,
       "inherits": [
-        "vcpkg",
-        "windows-only"
+        "vcpkg"
       ],
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Windows"
+      },
       "generator": "Visual Studio 17 2022",
       "toolset": "host=x64"
     },
@@ -64,52 +76,112 @@
       }
     },
     {
-      "name": "debian-debug",
-      "inherits": "debug",
-      "displayName": "Debian (Debug)",
-      "description": "Debug build assuming Debian-provided dependencies",
-      "cacheVariables": {
-        "Halide_LLVM_SHARED_LIBS": "ON"
-      }
+      "name": "debug-vcpkg",
+      "inherits": [
+        "debug",
+        "vcpkg"
+      ],
+      "displayName": "Debug (vcpkg)",
+      "description": "Debug build for a single-config generator, vcpkg dependencies"
     },
     {
-      "name": "debian-release",
-      "inherits": "debian-debug",
-      "displayName": "Debian (Release)",
-      "description": "Release build assuming Debian-provided dependencies",
-      "cacheVariables": {
-        "CMAKE_BUILD_TYPE": "Release"
-      }
+      "name": "release-vcpkg",
+      "inherits": [
+        "release",
+        "vcpkg"
+      ],
+      "displayName": "Release (vcpkg)",
+      "description": "Release build for a single-config generator, vcpkg dependencies"
     },
     {
-      "name": "win32",
+      "name": "debug-vcpkg-full",
+      "inherits": [
+        "debug",
+        "vcpkg-full"
+      ],
+      "displayName": "Debug (vcpkg-full)",
+      "description": "Debug build for a single-config generator, vcpkg-full dependencies"
+    },
+    {
+      "name": "release-vcpkg-full",
       "inherits": [
-        "vs2022",
-        "base"
+        "release",
+        "vcpkg-full"
       ],
+      "displayName": "Release (vcpkg-full)",
+      "description": "Release build for a single-config generator, vcpkg-full dependencies"
+    },
+    {
+      "name": "win32",
+      "inherits": "vs2022",
       "displayName": "Win32 (Visual Studio)",
       "description": "Visual Studio-based Win32 build with vcpkg dependencies.",
       "architecture": "Win32"
     },
     {
       "name": "win64",
-      "inherits": [
-        "vs2022",
-        "base"
-      ],
+      "inherits": "vs2022",
       "displayName": "Win64 (Visual Studio)",
       "description": "Visual Studio-based x64 build with vcpkg dependencies.",
       "architecture": "x64"
     },
+    {
+      "name": "win32-vcpkg-full",
+      "inherits": [
+        "vcpkg-full",
+        "vs2022"
+      ],
+      "displayName": "Win32 (Visual Studio/vcpkg-full)",
+      "description": "Visual Studio-based Win32 build with vcpkg-full dependencies.",
+      "architecture": "Win32"
+    },
+    {
+      "name": "win64-vcpkg-full",
+      "inherits": [
+        "vcpkg-full",
+        "vs2022"
+      ],
+      "displayName": "Win64 (Visual Studio/vcpkg-full)",
+      "description": "Visual Studio-based x64 build with vcpkg-full dependencies.",
+      "architecture": "x64"
+    },
     {
       "name": "macOS",
       "displayName": "macOS (Apple Clang)",
       "description": "macOS build using Apple Clang and Homebrew LLVM",
       "generator": "Ninja",
       "inherits": "release",
+      "condition": {
+        "type": "equals",
+        "lhs": "${hostSystemName}",
+        "rhs": "Darwin"
+      },
+      "cacheVariables": {
+        "CMAKE_PREFIX_PATH": "/opt/homebrew;/opt/homebrew/opt/llvm;/opt/homebrew/opt/jpeg"
+      }
+    },
+    {
+      "name": "macOS-vcpkg",
+      "inherits": [
+        "macOS",
+        "vcpkg"
+      ],
+      "displayName": "macOS (vcpkg)",
+      "description": "macOS build with vcpkg dependencies",
+      "cacheVariables": {
+        "CMAKE_PREFIX_PATH": "/opt/homebrew/opt/llvm"
+      }
+    },
+    {
+      "name": "macOS-vcpkg-full",
+      "inherits": [
+        "macOS",
+        "vcpkg-full"
+      ],
+      "displayName": "macOS (vcpkg-full)",
+      "description": "macOS build with vcpkg-full dependencies",
       "cacheVariables": {
-        "CMAKE_PREFIX_PATH": "/opt/homebrew;/opt/homebrew/opt/llvm;/opt/homebrew/opt/jpeg",
-        "Halide_LLVM_SHARED_LIBS": "YES"
+        "CMAKE_PREFIX_PATH": ""
       }
     },
     {
@@ -182,7 +254,6 @@
       "description": "Build everything with fuzzing enabled",
       "cacheVariables": {
         "LLVM_ROOT": "$penv{LLVM_ROOT}",
-        "TARGET_WEBASSEMBLY": "NO",
         "WITH_TUTORIALS": "NO",
         "WITH_UTILS": "NO",
         "WITH_PYTHON_BINDINGS": "NO",

cmake/vcpkg/llvm/portfile.cmake

@@ -0,0 +1,3 @@
+# This instructs vcpkg to do nothing, which causes find_package
+# to search the system for LLVM, rather than the vcpkg trees.
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)