getdeps: add xxhash ubuntu and homebrew packages, fix actions generat…

…ion and regenerate (#966)

Summary:
X-link: facebookincubator/zstrong#1010

EdenFS build on github CI was failing with missing xxhash include: https://github.com/facebook/sapling/actions/runs/11311478649/job/31457761727#step:108:3838

Add package mappings to the xxhash manifest

When I went to regenerate the github actions found I'd previously introduced a bug in the actions generation with a mis-merged run_tests check where if tests not run it wouldn't generate system dependency usage.  Fix it and regenerate


Test Plan:
Build locally on ubuntu-22.04 toolbox to repro issue, its broken,
```
./build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. eden
```

Install the packages:
```
./build/fbcode_builder/getdeps.py install-system-deps --recursive eden
```

Run again, it builds:
```
./build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. eden
```

Regenerate the github workflow:
```
 ./build/fbcode_builder/getdeps.py --allow-system-packages generate-github-actions --no-tests --free-up-disk --os-type=linux --src-dir=. --output-dir=.github/workflows --job-name "EdenFS " --job-file-prefix=edenfs_ eden --num-jobs=8  --project-install-prefix sapling:/usr/local
```

Differential Revision: D64302134

Pulled By: ahornby

.github/workflows/edenfs_linux.yml

@@ -27,9 +27,9 @@ jobs:
  - name: Update system package info
  run: sudo apt-get update
  - name: Install system deps
- run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive eden
+ run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --no-tests --recursive eden
  - name: Install packaging system deps
- run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf
+ run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --no-tests --recursive patchelf
  - name: Install Rust Stable
  uses: dtolnay/rust-toolchain@stable
  - name: Fetch lmdb
@@ -50,6 +50,8 @@ jobs:
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog
  - name: Fetch googletest
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest
+ - name: Fetch xxhash
+ run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xxhash
  - name: Fetch zstd
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd
  - name: Fetch boost
@@ -148,6 +150,8 @@ jobs:
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests glog
  - name: Build googletest
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests googletest
+ - name: Build xxhash
+ run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xxhash
  - name: Build zstd
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zstd
  - name: Build boost
@@ -237,4 +241,5 @@ jobs:
  name: eden
  path: _artifacts
  - name: Show disk space at end
+ if: always()
  run: df -h

.github/workflows/mononoke-integration_linux.yml

@@ -52,6 +52,10 @@ jobs:
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt
  - name: Fetch googletest
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest
+ - name: Fetch xxhash
+ run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xxhash
+ - name: Fetch python-setuptools
+ run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests python-setuptools
  - name: Fetch autoconf
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests autoconf
  - name: Fetch automake
@@ -136,6 +140,10 @@ jobs:
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt
  - name: Build googletest
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest
+ - name: Build xxhash
+ run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests xxhash
+ - name: Build python-setuptools
+ run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests python-setuptools
  - name: Build autoconf
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests autoconf
  - name: Build automake

.github/workflows/mononoke_linux.yml

@@ -32,6 +32,8 @@ jobs:
  run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf
  - name: Install Rust Stable
  uses: dtolnay/rust-toolchain@stable
+ - name: Fetch xxhash
+ run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xxhash
  - name: Fetch ninja
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja
  - name: Fetch cmake
@@ -102,6 +104,8 @@ jobs:
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fb303
  - name: Fetch rust-shed
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests rust-shed
+ - name: Build xxhash
+ run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xxhash
  - name: Build ninja
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests ninja
  - name: Build cmake
@@ -183,4 +187,5 @@ jobs:
  - name: Test mononoke
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. mononoke --project-install-prefix mononoke:/
  - name: Show disk space at end
+ if: always()
  run: df -h

.github/workflows/mononoke_mac.yml

@@ -24,6 +24,8 @@ jobs:
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive mononoke
  - name: Install Rust Stable
  uses: dtolnay/rust-toolchain@stable
+ - name: Fetch xxhash
+ run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xxhash
  - name: Fetch openssl
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests openssl
  - name: Fetch ninja
@@ -82,6 +84,8 @@ jobs:
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fb303
  - name: Fetch rust-shed
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests rust-shed
+ - name: Build xxhash
+ run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xxhash
  - name: Build openssl
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests openssl
  - name: Build ninja
@@ -151,4 +155,5 @@ jobs:
  - name: Test mononoke
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. mononoke --project-install-prefix mononoke:/usr/local
  - name: Show disk space at end
+ if: always()
  run: df -h

.github/workflows/sapling-cli-getdeps_linux.yml

@@ -38,6 +38,8 @@ jobs:
  uses: dtolnay/rust-toolchain@stable
  - name: Fetch hexdump
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests hexdump
+ - name: Fetch xxhash
+ run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xxhash
  - name: Fetch bz2
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests bz2
  - name: Fetch ninja
@@ -112,6 +114,8 @@ jobs:
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests rust-shed
  - name: Build hexdump
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests hexdump
+ - name: Build xxhash
+ run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xxhash
  - name: Build bz2
  run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests bz2
  - name: Build ninja

build/fbcode_builder/getdeps.py

@@ -959,6 +959,10 @@ def get_run_on(self, args):
  def write_job_for_platform(self, platform, args): # noqa: C901
  build_opts = setup_build_options(args, platform)
  ctx_gen = build_opts.get_context_generator()
+ if args.enable_tests:
+ ctx_gen.set_value_for_project(args.project, "test", "on")
+ else:
+ ctx_gen.set_value_for_project(args.project, "test", "off")
  loader = ManifestLoader(build_opts, ctx_gen)
  self.process_project_dir_arguments(args, loader)
  manifest = loader.load_manifest(args.project)
@@ -1081,7 +1085,10 @@ def write_job_for_platform(self, platform, args): # noqa: C901
  free_up_disk = ""
 
  allow_sys_arg = ""
- if run_tests:
+ if (
+ build_opts.allow_system_packages
+ and build_opts.host_type.get_package_manager()
+ ):
  sudo_arg = "sudo "
  allow_sys_arg = " --allow-system-packages"
  if build_opts.host_type.get_package_manager() == "deb":
@@ -1092,13 +1099,16 @@ def write_job_for_platform(self, platform, args): # noqa: C901
  if build_opts.is_darwin():
  # brew is installed as regular user
  sudo_arg = ""
+ tests_arg = "--no-tests "
+ if run_tests:
+ tests_arg = ""
  out.write(
- f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n"
+ f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps {tests_arg}--recursive {manifest.name}\n"
  )
  if build_opts.is_linux() or build_opts.is_freebsd():
  out.write(" - name: Install packaging system deps\n")
  out.write(
- f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf\n"
+ f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps {tests_arg}--recursive patchelf\n"
  )
  required_locales = manifest.get(
  "github.actions", "required_locales", ctx=manifest_ctx
@@ -1176,7 +1186,7 @@ def write_job_for_platform(self, platform, args): # noqa: C901
  no_deps_arg = "--no-deps "
 
  no_tests_arg = ""
- if not args.enable_tests:
+ if not run_tests:
  no_tests_arg = "--no-tests "
 
  out.write(

build/fbcode_builder/manifests/xxhash

@@ -8,6 +8,13 @@ sha256 = baee0c6afd4f03165de7a4e67988d16f0f2b257b51d0e3cb91909302a26a79c4
 [rpms]
 xxhash-devel
 
+[debs]
+libxxhash-dev
+xxhash
+
+[homebrew]
+xxhash
+
 [build.not(os=windows)]
 builder = make
 subdir = xxHash-0.8.2