From b59064f7617063e0e2335221d753bcf2f30fbfe6 Mon Sep 17 00:00:00 2001 From: Alex Hornby Date: Sun, 13 Oct 2024 10:33:26 -0700 Subject: [PATCH] getdeps: add xxhash ubuntu and homebrew packages, fix actions generation and regenerate (#966) Summary: X-link: https://github.com/facebookincubator/zstrong/pull/1010 EdenFS build on github CI was failing with missing xxhash include: https://github.com/facebook/sapling/actions/runs/11311478649/job/31457761727#step:108:3838 Add package mappings to the xxhash manifest When I went to regenerate the github actions found I'd previously introduced a bug in the actions generation with a mis-merged run_tests check where if tests not run it wouldn't generate system dependency usage. Fix it and regenerate Test Plan: Build locally on ubuntu-22.04 toolbox to repro issue, its broken, ``` ./build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. eden ``` Install the packages: ``` ./build/fbcode_builder/getdeps.py install-system-deps --recursive eden ``` Run again, it builds: ``` ./build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. eden ``` Regenerate the github workflow: ``` ./build/fbcode_builder/getdeps.py --allow-system-packages generate-github-actions --no-tests --free-up-disk --os-type=linux --src-dir=. --output-dir=.github/workflows --job-name "EdenFS " --job-file-prefix=edenfs_ eden --num-jobs=8 --project-install-prefix sapling:/usr/local ``` Differential Revision: D64302134 Pulled By: ahornby --- .github/workflows/edenfs_linux.yml | 9 +++++++-- .../workflows/mononoke-integration_linux.yml | 8 ++++++++ .github/workflows/mononoke_linux.yml | 5 +++++ .github/workflows/mononoke_mac.yml | 5 +++++ .../workflows/sapling-cli-getdeps_linux.yml | 4 ++++ build/fbcode_builder/getdeps.py | 18 ++++++++++++++---- build/fbcode_builder/manifests/xxhash | 7 +++++++ 7 files changed, 50 insertions(+), 6 deletions(-) diff --git a/.github/workflows/edenfs_linux.yml b/.github/workflows/edenfs_linux.yml index acb7fa9df6644..b45a638a1cc75 100644 --- a/.github/workflows/edenfs_linux.yml +++ b/.github/workflows/edenfs_linux.yml @@ -27,9 +27,9 @@ jobs: - name: Update system package info run: sudo apt-get update - name: Install system deps - run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive eden + run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --no-tests --recursive eden - name: Install packaging system deps - run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf + run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --no-tests --recursive patchelf - name: Install Rust Stable uses: dtolnay/rust-toolchain@stable - name: Fetch lmdb @@ -50,6 +50,8 @@ jobs: run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog - name: Fetch googletest run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest + - name: Fetch xxhash + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xxhash - name: Fetch zstd run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd - name: Fetch boost @@ -148,6 +150,8 @@ jobs: run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests glog - name: Build googletest run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests googletest + - name: Build xxhash + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xxhash - name: Build zstd run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests zstd - name: Build boost @@ -237,4 +241,5 @@ jobs: name: eden path: _artifacts - name: Show disk space at end + if: always() run: df -h diff --git a/.github/workflows/mononoke-integration_linux.yml b/.github/workflows/mononoke-integration_linux.yml index 73a847f7c8bf0..4d695cbe570e3 100644 --- a/.github/workflows/mononoke-integration_linux.yml +++ b/.github/workflows/mononoke-integration_linux.yml @@ -52,6 +52,10 @@ jobs: run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt - name: Fetch googletest run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest + - name: Fetch xxhash + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xxhash + - name: Fetch python-setuptools + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests python-setuptools - name: Fetch autoconf run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests autoconf - name: Fetch automake @@ -136,6 +140,10 @@ jobs: run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt - name: Build googletest run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest + - name: Build xxhash + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests xxhash + - name: Build python-setuptools + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests python-setuptools - name: Build autoconf run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests autoconf - name: Build automake diff --git a/.github/workflows/mononoke_linux.yml b/.github/workflows/mononoke_linux.yml index a080401e1abb7..e96a34a7bc626 100644 --- a/.github/workflows/mononoke_linux.yml +++ b/.github/workflows/mononoke_linux.yml @@ -32,6 +32,8 @@ jobs: run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf - name: Install Rust Stable uses: dtolnay/rust-toolchain@stable + - name: Fetch xxhash + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xxhash - name: Fetch ninja run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja - name: Fetch cmake @@ -102,6 +104,8 @@ jobs: run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fb303 - name: Fetch rust-shed run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests rust-shed + - name: Build xxhash + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xxhash - name: Build ninja run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests ninja - name: Build cmake @@ -183,4 +187,5 @@ jobs: - name: Test mononoke run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. mononoke --project-install-prefix mononoke:/ - name: Show disk space at end + if: always() run: df -h diff --git a/.github/workflows/mononoke_mac.yml b/.github/workflows/mononoke_mac.yml index 7e96aa90735fd..22450a27cc0be 100644 --- a/.github/workflows/mononoke_mac.yml +++ b/.github/workflows/mononoke_mac.yml @@ -24,6 +24,8 @@ jobs: run: python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive mononoke - name: Install Rust Stable uses: dtolnay/rust-toolchain@stable + - name: Fetch xxhash + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xxhash - name: Fetch openssl run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests openssl - name: Fetch ninja @@ -82,6 +84,8 @@ jobs: run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fb303 - name: Fetch rust-shed run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests rust-shed + - name: Build xxhash + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xxhash - name: Build openssl run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests openssl - name: Build ninja @@ -151,4 +155,5 @@ jobs: - name: Test mononoke run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. mononoke --project-install-prefix mononoke:/usr/local - name: Show disk space at end + if: always() run: df -h diff --git a/.github/workflows/sapling-cli-getdeps_linux.yml b/.github/workflows/sapling-cli-getdeps_linux.yml index 365cd76b14f63..f69f5f189f9d4 100644 --- a/.github/workflows/sapling-cli-getdeps_linux.yml +++ b/.github/workflows/sapling-cli-getdeps_linux.yml @@ -38,6 +38,8 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: Fetch hexdump run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests hexdump + - name: Fetch xxhash + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xxhash - name: Fetch bz2 run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests bz2 - name: Fetch ninja @@ -112,6 +114,8 @@ jobs: run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests rust-shed - name: Build hexdump run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests hexdump + - name: Build xxhash + run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests xxhash - name: Build bz2 run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --free-up-disk --no-tests bz2 - name: Build ninja diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 8a42fcd43c7c9..ef81d3c3c887b 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -959,6 +959,10 @@ def get_run_on(self, args): def write_job_for_platform(self, platform, args): # noqa: C901 build_opts = setup_build_options(args, platform) ctx_gen = build_opts.get_context_generator() + if args.enable_tests: + ctx_gen.set_value_for_project(args.project, "test", "on") + else: + ctx_gen.set_value_for_project(args.project, "test", "off") loader = ManifestLoader(build_opts, ctx_gen) self.process_project_dir_arguments(args, loader) manifest = loader.load_manifest(args.project) @@ -1081,7 +1085,10 @@ def write_job_for_platform(self, platform, args): # noqa: C901 free_up_disk = "" allow_sys_arg = "" - if run_tests: + if ( + build_opts.allow_system_packages + and build_opts.host_type.get_package_manager() + ): sudo_arg = "sudo " allow_sys_arg = " --allow-system-packages" if build_opts.host_type.get_package_manager() == "deb": @@ -1092,13 +1099,16 @@ def write_job_for_platform(self, platform, args): # noqa: C901 if build_opts.is_darwin(): # brew is installed as regular user sudo_arg = "" + tests_arg = "--no-tests " + if run_tests: + tests_arg = "" out.write( - f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n" + f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps {tests_arg}--recursive {manifest.name}\n" ) if build_opts.is_linux() or build_opts.is_freebsd(): out.write(" - name: Install packaging system deps\n") out.write( - f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf\n" + f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps {tests_arg}--recursive patchelf\n" ) required_locales = manifest.get( "github.actions", "required_locales", ctx=manifest_ctx @@ -1176,7 +1186,7 @@ def write_job_for_platform(self, platform, args): # noqa: C901 no_deps_arg = "--no-deps " no_tests_arg = "" - if not args.enable_tests: + if not run_tests: no_tests_arg = "--no-tests " out.write( diff --git a/build/fbcode_builder/manifests/xxhash b/build/fbcode_builder/manifests/xxhash index 8e490011e5800..557ded0ddd5e9 100644 --- a/build/fbcode_builder/manifests/xxhash +++ b/build/fbcode_builder/manifests/xxhash @@ -8,6 +8,13 @@ sha256 = baee0c6afd4f03165de7a4e67988d16f0f2b257b51d0e3cb91909302a26a79c4 [rpms] xxhash-devel +[debs] +libxxhash-dev +xxhash + +[homebrew] +xxhash + [build.not(os=windows)] builder = make subdir = xxHash-0.8.2