diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index c8b03802495be..037a695d26140 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -1193,9 +1193,13 @@ def write_job_for_platform(self, platform, args): # noqa: C901 and manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off" ): + num_jobs_arg = "" + if args.num_jobs: + num_jobs_arg = f"--num-jobs {args.num_jobs} " + out.write(" - name: Test %s\n" % manifest.name) out.write( - f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n" + f" run: {getdepscmd}{allow_sys_arg} test {num_jobs_arg}--src-dir=. {manifest.name} {project_prefix}\n" ) if build_opts.free_up_disk and not build_opts.is_windows(): out.write(" - name: Show disk space at end\n") diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 5a5ea2303b575..6ad9f2859a98d 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -315,9 +315,22 @@ def run_tests(self, schedule_type, owner, test_filter, retry, no_testpilot) -> N return env = self._compute_env() + if test_filter: + env["GETDEPS_TEST_FILTER"] = test_filter + else: + env["GETDEPS_TEST_FILTER"] = "" - cmd = [self._make_binary] + self.test_args + self._get_prefix() - self._run_cmd(cmd, env=env) + if retry: + env["GETDEPS_TEST_RETRY"] = retry + else: + env["GETDEPS_TEST_RETRY"] = 0 + + cmd = ( + [self._make_binary, "-j%s" % self.num_jobs] + + self.test_args + + self._get_prefix() + ) + self._run_cmd(cmd, allow_fail=False, env=env) class CMakeBootStrapBuilder(MakeBuilder): diff --git a/build/fbcode_builder/manifests/git-lfs b/build/fbcode_builder/manifests/git-lfs new file mode 100644 index 0000000000000..19b24e2470804 --- /dev/null +++ b/build/fbcode_builder/manifests/git-lfs @@ -0,0 +1,15 @@ +[manifest] +name = git-lfs + +[rpms] +git-lfs + +[debs] +git-lfs + +[homebrew] +git-lfs + +# only used from system packages currently +[build] +builder = nop diff --git a/build/fbcode_builder/manifests/jq b/build/fbcode_builder/manifests/jq new file mode 100644 index 0000000000000..354854b2ce87a --- /dev/null +++ b/build/fbcode_builder/manifests/jq @@ -0,0 +1,25 @@ +[manifest] +name = jq + +[rpms.distro=fedora] +jq + +[homebrew] +jq + +[download.not(os=windows)] +# we use jq-1.7+ to get fix for number truncation https://github.com/jqlang/jq/pull/1752 +url = https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-1.7.1.tar.gz +sha256 = 478c9ca129fd2e3443fe27314b455e211e0d8c60bc8ff7df703873deeee580c2 + +[build.not(os=windows)] +builder = autoconf +subdir = jq-1.7.1 + +[build.os=windows] +builder = nop + +[autoconf.args] +# This argument turns off some developers tool and it is recommended in jq's +# README +--disable-maintainer-mode diff --git a/build/fbcode_builder/manifests/mononoke_integration b/build/fbcode_builder/manifests/mononoke_integration new file mode 100644 index 0000000000000..e1f171403522a --- /dev/null +++ b/build/fbcode_builder/manifests/mononoke_integration @@ -0,0 +1,53 @@ +[manifest] +name = mononoke_integration +fbsource_path = fbcode/eden +shipit_project = eden +shipit_fbcode_builder = true + +[git] +repo_url = https://github.com/facebook/sapling.git + +[build.not(os=windows)] +builder = make +subdir = eden/mononoke/tests/integration + +[build.os=windows] +# building Mononoke on windows is not supported +builder = nop + +[make.build_args] +build-getdeps + +[make.install_args] +install-getdeps + +[make.test_args] +test-getdeps + +[shipit.pathmap] +fbcode/eden/mononoke/tests/integration = eden/mononoke/tests/integration + +[shipit.strip] +^.*/facebook/.*$ +^.*/fb/.*$ + +[dependencies] +git-lfs +jq +mononoke +nmap +python +python-click +ripgrep +sapling +tree +zstd + +[dependencies.os=linux] +sqlite3 + +[dependencies.os=darwin] +gnu-bash +gnu-coreutils +gnu-grep +gnu-sed diff --git a/build/fbcode_builder/manifests/nmap b/build/fbcode_builder/manifests/nmap new file mode 100644 index 0000000000000..3e935177b3593 --- /dev/null +++ b/build/fbcode_builder/manifests/nmap @@ -0,0 +1,30 @@ +[manifest] +name = nmap + +[rpms] +nmap +nmap-ncat + +[debs] +nmap + +# 18.04 combines ncat into the nmap package, newer need the separate one +[debs.not(all(distro=ubuntu,distro_vers="18.04"))] +ncat + +[download.not(os=windows)] +url = https://api.github.com/repos/nmap/nmap/tarball/ef8213a36c2e89233c806753a57b5cd473605408 +sha256 = eda39e5a8ef4964fac7db16abf91cc11ff568eac0fa2d680b0bfa33b0ed71f4a + +[build.not(os=windows)] +builder = autoconf +subdir = nmap-nmap-ef8213a +build_in_src_dir = true + +[build.os=windows] +builder = nop + +[autoconf.args] +# Without this option the build was filing to find some third party libraries +# that we don't need +enable_rdma=no diff --git a/build/fbcode_builder/manifests/python-click b/build/fbcode_builder/manifests/python-click new file mode 100644 index 0000000000000..ea9a9d2d3dc38 --- /dev/null +++ b/build/fbcode_builder/manifests/python-click @@ -0,0 +1,9 @@ +[manifest] +name = python-click + +[download] +url = https://files.pythonhosted.org/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl +sha256 = dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc + +[build] +builder = python-wheel diff --git a/build/fbcode_builder/manifests/ripgrep b/build/fbcode_builder/manifests/ripgrep new file mode 100644 index 0000000000000..140a4e8aff36f --- /dev/null +++ b/build/fbcode_builder/manifests/ripgrep @@ -0,0 +1,15 @@ +[manifest] +name = ripgrep + +[rpms] +ripgrep + +[debs] +ripgrep + +[homebrew] +ripgrep + +# only used from system packages currently +[build] +builder = nop diff --git a/build/fbcode_builder/manifests/sqlite3 b/build/fbcode_builder/manifests/sqlite3 index 1966f0fab10c4..5a983f8aa2e63 100644 --- a/build/fbcode_builder/manifests/sqlite3 +++ b/build/fbcode_builder/manifests/sqlite3 @@ -3,6 +3,7 @@ name = sqlite3 [debs] libsqlite3-dev +sqlite3 [homebrew] sqlite @@ -10,6 +11,7 @@ sqlite [rpms] sqlite-devel sqlite-libs +sqlite [pps] sqlite3 diff --git a/build/fbcode_builder/manifests/tree b/build/fbcode_builder/manifests/tree new file mode 100644 index 0000000000000..ccd0180a74c0d --- /dev/null +++ b/build/fbcode_builder/manifests/tree @@ -0,0 +1,37 @@ +[manifest] +name = tree + +[debs] +tree + +[homebrew] +tree + +[rpms] +tree + +[download.os=linux] +url = https://salsa.debian.org/debian/tree-packaging/-/archive/debian/1.8.0-1/tree-packaging-debian-1.8.0-1.tar.gz +sha256 = a841eee1d52bfd64a48f54caab9937b9bd92935055c48885c4ab1ae4dab7fae5 + +[download.os=darwin] +# The official package of tree source requires users of non-Linux platform to +# comment/uncomment certain lines in the Makefile to build for their platform. +# Besauce getdeps.py doesn't have that functionality we just use this custom +# fork of tree which has proper lines uncommented for a OSX build +url = https://github.com/lukaspiatkowski/tree-command/archive/debian/1.8.0-1-macos.tar.gz +sha256 = 9cbe889553d95cf5a2791dd0743795d46a3c092c5bba691769c0e5c52e11229e + +[build.os=linux] +builder = make +subdir = tree-packaging-debian-1.8.0-1 + +[build.os=darwin] +builder = make +subdir = tree-command-debian-1.8.0-1-macos + +[build.os=windows] +builder = nop + +[make.install_args] +install diff --git a/build/fbcode_builder/manifests/zstd b/build/fbcode_builder/manifests/zstd index aac189fb88fb5..1bd0875388f3f 100644 --- a/build/fbcode_builder/manifests/zstd +++ b/build/fbcode_builder/manifests/zstd @@ -7,6 +7,7 @@ zstd # 18.04 zstd is too old [debs.not(all(distro=ubuntu,distro_vers="18.04"))] libzstd-dev +zstd [rpms] libzstd-devel