Externalize templates to speed up build

.github/workflows/getdeps_linux.yml

@@ -0,0 +1,183 @@
+# This file was @generated by getdeps.py
+
+name: linux
+
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+
+permissions:
+  contents: read  #  to fetch code (actions/checkout)
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v4
+    - name: Show disk space at start
+      run: df -h
+    - name: Free up disk space
+      run: sudo rm -rf /usr/local/lib/android
+    - name: Show disk space after freeing up
+      run: df -h
+    - name: Update system package info
+      run: sudo apt-get update
+    - name: Install system deps
+      run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive mcrouter
+    - name: Install packaging system deps
+      run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf
+    - name: Fetch ninja
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja
+    - name: Fetch cmake
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake
+    - name: Fetch fmt
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt
+    - name: Fetch googletest
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest
+    - name: Fetch xxhash
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xxhash
+    - name: Fetch zstd
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd
+    - name: Fetch zlib
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zlib
+    - name: Fetch boost
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests boost
+    - name: Fetch double-conversion
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests double-conversion
+    - name: Fetch fast_float
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fast_float
+    - name: Fetch gflags
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags
+    - name: Fetch glog
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog
+    - name: Fetch libdwarf
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libdwarf
+    - name: Fetch libevent
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libevent
+    - name: Fetch lz4
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests lz4
+    - name: Fetch snappy
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy
+    - name: Fetch bz2
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests bz2
+    - name: Fetch openssl
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests openssl
+    - name: Fetch liboqs
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests liboqs
+    - name: Fetch autoconf
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests autoconf
+    - name: Fetch automake
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests automake
+    - name: Fetch libtool
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libtool
+    - name: Fetch ragel
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ragel
+    - name: Fetch libiberty
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libiberty
+    - name: Fetch libsodium
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libsodium
+    - name: Fetch libunwind
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libunwind
+    - name: Fetch xz
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xz
+    - name: Fetch folly
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests folly
+    - name: Fetch fizz
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fizz
+    - name: Fetch wangle
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests wangle
+    - name: Fetch mvfst
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests mvfst
+    - name: Fetch libffi
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libffi
+    - name: Fetch ncurses
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ncurses
+    - name: Fetch python
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests python
+    - name: Fetch fbthrift
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fbthrift
+    - name: Build ninja
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests ninja
+    - name: Build cmake
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests cmake
+    - name: Build fmt
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fmt
+    - name: Build googletest
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests googletest
+    - name: Build xxhash
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xxhash
+    - name: Build zstd
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zstd
+    - name: Build zlib
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zlib
+    - name: Build boost
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests boost
+    - name: Build double-conversion
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests double-conversion
+    - name: Build fast_float
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fast_float
+    - name: Build gflags
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests gflags
+    - name: Build glog
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests glog
+    - name: Build libdwarf
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libdwarf
+    - name: Build libevent
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libevent
+    - name: Build lz4
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests lz4
+    - name: Build snappy
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests snappy
+    - name: Build bz2
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests bz2
+    - name: Build openssl
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests openssl
+    - name: Build liboqs
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests liboqs
+    - name: Build autoconf
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests autoconf
+    - name: Build automake
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests automake
+    - name: Build libtool
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libtool
+    - name: Build ragel
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests ragel
+    - name: Build libiberty
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libiberty
+    - name: Build libsodium
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libsodium
+    - name: Build libunwind
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libunwind
+    - name: Build xz
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xz
+    - name: Build folly
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests folly
+    - name: Build fizz
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fizz
+    - name: Build wangle
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests wangle
+    - name: Build mvfst
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests mvfst
+    - name: Build libffi
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests libffi
+    - name: Build ncurses
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests ncurses
+    - name: Build python
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests python
+    - name: Build fbthrift
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests fbthrift
+    - name: Build mcrouter
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. mcrouter  --project-install-prefix mcrouter:/usr/local
+    - name: Copy artifacts
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --strip --src-dir=. mcrouter _artifacts/linux  --project-install-prefix mcrouter:/usr/local --final-install-prefix /usr/local
+    - uses: actions/upload-artifact@v4
+      with:
+        name: mcrouter
+        path: _artifacts
+    - name: Test mcrouter
+      run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. mcrouter  --project-install-prefix mcrouter:/usr/local
+    - name: Show disk space at end
+      if: always()
+      run: df -h

.gitignore

@@ -0,0 +1 @@
+compile_commands.json

CMakeLists.txt

@@ -0,0 +1,109 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the LICENSE file
+# in the root directory of this source tree.
+
+cmake_minimum_required(VERSION 3.19)
+
+project(mcrouter VERSION 1.0.0)
+
+set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/build/fbcode_builder/CMake"
+                      ${CMAKE_MODULE_PATH})
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_OPTIMIZE_DEPENDENCIES ON)
+
+option(BUILD_TESTS "If enabled, compile the tests." OFF)
+option(USE_CCACHE "Use ccache for compiler caching, if available" OFF)
+option(PROFILE_BUILD "Profile build times (requires clang)" OFF)
+
+# Opt out of using the upstream BoostConfig module with CMake >= 3.30,
+# since GitHub Actions provides Boost < 1.70 and therefore is missing the upstream module.
+if(POLICY CMP0167)
+  cmake_policy(SET CMP0167 OLD)
+endif()
+
+# Disable Meta-specific functionality for the OSS build.
+add_compile_definitions(LIBMC_FBTRACE_DISABLE DISABLE_COMPRESSION
+                        MCROUTER_OSS_BUILD)
+
+# Allow using a compiler cache locally on an opt-in basis.
+if(USE_CCACHE)
+  find_program(CCACHE_EXECUTABLE ccache)
+  if(CCACHE_EXECUTABLE)
+    set(ccacheEnv CCACHE_SLOPPINESS=pch_defines,time_macros)
+
+    foreach(lang IN ITEMS C CXX)
+      set(CMAKE_${lang}_COMPILER_LAUNCHER ${CMAKE_COMMAND} -E env ${ccacheEnv}
+                                          ${CCACHE_EXECUTABLE})
+    endforeach()
+  endif()
+endif()
+
+# Allow profiling build performance with clang.
+if(PROFILE_BUILD)
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+    add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:-ftime-trace>)
+    message("Enabling build time profiling")
+  else()
+    message(
+      "${CMAKE_CXX_COMPILER_ID} compiler detected, cannot enable build time profiling"
+    )
+  endif()
+endif()
+
+if(WIN32)
+  include(FBCompilerSettingsMSVC)
+else()
+  include(FBCompilerSettingsUnix)
+
+  # Create symlink to compile_commands.json for IDE to pick it up
+  execute_process(
+    COMMAND
+      ${CMAKE_COMMAND} -E create_symlink
+      ${CMAKE_BINARY_DIR}/compile_commands.json
+      ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json)
+endif()
+
+# Set up options for generated Thrift client code
+include(FBThriftLibrary)
+set(THRIFT_OPTIONS stack_arguments sync_methods_return_try
+                   deprecated_terse_writes)
+
+find_package(Boost 1.65.1 REQUIRED COMPONENTS system thread filesystem regex
+                                              context program_options)
+
+find_package(fmt REQUIRED)
+find_package(folly REQUIRED)
+find_package(Fizz REQUIRED)
+find_package(Glog REQUIRED)
+find_package(gflags REQUIRED)
+find_package(wangle REQUIRED)
+find_package(FBThrift REQUIRED)
+
+include_directories(.)
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+
+if(BUILD_TESTS)
+  enable_testing()
+  include(CTest)
+
+  find_package(GTest MODULE REQUIRED)
+
+  include(GoogleTest)
+endif()
+
+add_subdirectory(mcrouter)
+
+install(TARGETS mcrouter mcpiper)
+
+# Provide a basic CPack integration to allow easily creating packages for Linux
+# distributions.
+set(CPACK_PACKAGE_DESCRIPTION_SUMMARY
+    "mcrouter - a memcached protocol router for scaling memcached deployments.")
+set(CPACK_PACKAGE_VENDOR "Meta Platforms, Inc.")
+set(CPACK_VERBATIM_VARIABLES ON)
+include(CPack)

README.md

@@ -1,4 +1,4 @@
-# Mcrouter [![Build Status](https://github.com/facebook/mcrouter/workflows/build/badge.svg)](https://github.com/facebook/mcrouter/actions?workflow=build)
+# Mcrouter [![Build Status](https://github.com/facebook/mcrouter/actions/workflows/getdeps_linux.yml/badge.svg)](https://github.com/facebook/mcrouter/actions/workflows/getdeps_linux.yml)
 [![Support Ukraine](https://img.shields.io/badge/Support-Ukraine-FFD500?style=flat&labelColor=005BBB)](https://opensource.fb.com/support-ukraine) [![License](https://img.shields.io/badge/license-MIT-blue)](https://github.com/facebook/mcrouter/blob/master/LICENSE)
 
 Mcrouter (pronounced mc router) is a memcached protocol router for scaling [memcached](https://memcached.org/)
@@ -12,42 +12,33 @@ See https://github.com/facebook/mcrouter/wiki to get started.
 
 ## Quick start guide
 
-### New! Ubuntu package available
+### Building
 
-Currently, we support Ubuntu Bionic (18.04) amd64.
-Here is how to install it:
+Mcrouter depends on [folly](https://github.com/facebook/folly), [wangle](https://github.com/facebook/wangle), [fizz](https://github.com/facebookincubator/fizz), [fbthrift](https://github.com/facebook/fbthrift)
+and [mvfst](https://github.com/facebook/mvfst).
 
-Add the repo key:
+The `getdeps.py` tool, shared between many open source C++ projects at Meta, may be used to build mcrouter and its dependencies:
 
-    $ wget -O - https://facebook.github.io/mcrouter/debrepo/bionic/PUBLIC.KEY | sudo apt-key add
+    # Clone the repo
+    git clone https://github.com/facebook/mcrouter.git
+    cd mcrouter
 
-Add the following line to apt sources file /etc/apt/sources.list
+    # Build, using system dependencies if available, including tests
+    ./build/fbcode_builder/getdeps.py --allow-system-packages build mcrouter
 
-    deb https://facebook.github.io/mcrouter/debrepo/bionic bionic contrib
+    # Build, using system dependencies if available, without tests
+    ./build/fbcode_builder/getdeps.py --allow-system-packages --no-tests build mcrouter
 
-Update the local repo cache:
+Once built, you may use `getdeps.py` to run the tests as well:
 
-    $ sudo apt-get update
+    # Run the tests
+    ./build/fbcode_builder/getdeps.py --allow-system-packages test
 
-Install mcrouter:
+### Packaging
+mcrouter ships with a basic `CPack` configuration that may be used to package it for your Linux distribution.
+After building the project, change to its `getdeps.py` scratch dir, and run `cpack` with the appropriate generator.
 
-    $ sudo apt-get install mcrouter
-
-
-### Installing From Source
-
-See https://github.com/facebook/mcrouter/wiki/mcrouter-installation for more
-detailed installation instructions.
-
-Mcrouter depends on [folly](https://github.com/facebook/folly), [wangle](https://github.com/facebook/wangle), [fizz](https://github.com/facebookincubator/fizz), and [fbthrift](https://github.com/facebook/fbthrift).
-
-The installation is a standard autotools flow:
-
-    $ autoreconf --install
-    $ ./configure
-    $ make
-    $ sudo make install
-    $ mcrouter --help
+### Running mcrouter
 
 Assuming you have a memcached instance on the local host running on port 5001,
 the simplest mcrouter setup is:

build/fbcode_builder/.gitignore

@@ -0,0 +1,5 @@
+# Facebook-internal CI builds don't have write permission outside of the
+# source tree, so we install all projects into this directory.
+/facebook_ci
+__pycache__/
+*.pyc