restore mononoke getdeps integration tests

Summary:
X-link: facebookincubator/zstrong#995

Bring back mononoke getdeps integration tests.  This is a refesh of the previously working export-D34186407 branch from 2022

Main changes since:
 * depend on restored sapling manifest from previous commit
 * bring back selected manifests needed by mononoke that were deleted in D51869247. I added the sqlite binary packages from this to the sqlite3 manifest rather than bring back the sqllite3-bin manifest
 * add manifests for new tools used: ripgrep, git-lfs, and zstd cli (we already had the zstd libs)
 * fix a few test expecations that where too closely tied to git cli or TLS version etc (ubuntu 22.04 is on older version)
 * getdeps MakefileBuilder.run_tests() improvements
   * fix error status reporting, was not failing if tests failed
   * pass of --num-jobs to tests to stop it OOMing my machine
   * pass of --filter to tests so can iterate on one test more easily

Can to iterate local execution for one test with:
```
python3 ./build/fbcode_builder/getdeps.py test --num-jobs 4 --allow-system-packages --no-facebook-internal --src-dir=. mononoke_integration --retry 0 --filter server/test-gettreepack.t
```

X-link: facebook/sapling#951

Reviewed By: quark-zju

Differential Revision: D62978526

Pulled By: ahornby

fbshipit-source-id: 0070a67d798bb23ee9e78e1a5149ba5364d548c9

build/fbcode_builder/getdeps.py

@@ -1193,9 +1193,13 @@ def write_job_for_platform(self, platform, args): # noqa: C901
  and manifest.get("github.actions", "run_tests", ctx=manifest_ctx)
  != "off"
  ):
+ num_jobs_arg = ""
+ if args.num_jobs:
+ num_jobs_arg = f"--num-jobs {args.num_jobs} "
+
  out.write(" - name: Test %s\n" % manifest.name)
  out.write(
- f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n"
+ f" run: {getdepscmd}{allow_sys_arg} test {num_jobs_arg}--src-dir=. {manifest.name} {project_prefix}\n"
  )
  if build_opts.free_up_disk and not build_opts.is_windows():
  out.write(" - name: Show disk space at end\n")

build/fbcode_builder/getdeps/builder.py

@@ -315,9 +315,22 @@ def run_tests(self, schedule_type, owner, test_filter, retry, no_testpilot) -> N
  return
 
  env = self._compute_env()
+ if test_filter:
+ env["GETDEPS_TEST_FILTER"] = test_filter
+ else:
+ env["GETDEPS_TEST_FILTER"] = ""
 
- cmd = [self._make_binary] + self.test_args + self._get_prefix()
- self._run_cmd(cmd, env=env)
+ if retry:
+ env["GETDEPS_TEST_RETRY"] = retry
+ else:
+ env["GETDEPS_TEST_RETRY"] = 0
+
+ cmd = (
+ [self._make_binary, "-j%s" % self.num_jobs]
+ + self.test_args
+ + self._get_prefix()
+ )
+ self._run_cmd(cmd, allow_fail=False, env=env)
 
 
 class CMakeBootStrapBuilder(MakeBuilder):

build/fbcode_builder/manifests/git-lfs

@@ -0,0 +1,15 @@
+[manifest]
+name = git-lfs
+
+[rpms]
+git-lfs
+
+[debs]
+git-lfs
+
+[homebrew]
+git-lfs
+
+# only used from system packages currently
+[build]
+builder = nop

build/fbcode_builder/manifests/jq

@@ -0,0 +1,25 @@
+[manifest]
+name = jq
+
+[rpms.distro=fedora]
+jq
+
+[homebrew]
+jq
+
+[download.not(os=windows)]
+# we use jq-1.7+ to get fix for number truncation https://github.com/jqlang/jq/pull/1752
+url = https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-1.7.1.tar.gz
+sha256 = 478c9ca129fd2e3443fe27314b455e211e0d8c60bc8ff7df703873deeee580c2
+
+[build.not(os=windows)]
+builder = autoconf
+subdir = jq-1.7.1
+
+[build.os=windows]
+builder = nop
+
+[autoconf.args]
+# This argument turns off some developers tool and it is recommended in jq's
+# README
+--disable-maintainer-mode

build/fbcode_builder/manifests/mononoke_integration

@@ -0,0 +1,53 @@
+[manifest]
+name = mononoke_integration
+fbsource_path = fbcode/eden
+shipit_project = eden
+shipit_fbcode_builder = true
+
+[git]
+repo_url = https://github.com/facebook/sapling.git
+
+[build.not(os=windows)]
+builder = make
+subdir = eden/mononoke/tests/integration
+
+[build.os=windows]
+# building Mononoke on windows is not supported
+builder = nop
+
+[make.build_args]
+build-getdeps
+
+[make.install_args]
+install-getdeps
+
+[make.test_args]
+test-getdeps
+
+[shipit.pathmap]
+fbcode/eden/mononoke/tests/integration = eden/mononoke/tests/integration
+
+[shipit.strip]
+^.*/facebook/.*$
+^.*/fb/.*$
+
+[dependencies]
+git-lfs
+jq
+mononoke
+nmap
+python
+python-click
+ripgrep
+sapling
+tree
+zstd
+
+[dependencies.os=linux]
+sqlite3
+
+[dependencies.os=darwin]
+gnu-bash
+gnu-coreutils
+gnu-grep
+gnu-sed

build/fbcode_builder/manifests/nmap

@@ -0,0 +1,30 @@
+[manifest]
+name = nmap
+
+[rpms]
+nmap
+nmap-ncat
+
+[debs]
+nmap
+
+# 18.04 combines ncat into the nmap package, newer need the separate one
+[debs.not(all(distro=ubuntu,distro_vers="18.04"))]
+ncat
+
+[download.not(os=windows)]
+url = https://api.github.com/repos/nmap/nmap/tarball/ef8213a36c2e89233c806753a57b5cd473605408
+sha256 = eda39e5a8ef4964fac7db16abf91cc11ff568eac0fa2d680b0bfa33b0ed71f4a
+
+[build.not(os=windows)]
+builder = autoconf
+subdir = nmap-nmap-ef8213a
+build_in_src_dir = true
+
+[build.os=windows]
+builder = nop
+
+[autoconf.args]
+# Without this option the build was filing to find some third party libraries
+# that we don't need
+enable_rdma=no

build/fbcode_builder/manifests/python-click

@@ -0,0 +1,9 @@
+[manifest]
+name = python-click
+
+[download]
+url = https://files.pythonhosted.org/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl
+sha256 = dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc
+
+[build]
+builder = python-wheel

build/fbcode_builder/manifests/ripgrep

@@ -0,0 +1,15 @@
+[manifest]
+name = ripgrep
+
+[rpms]
+ripgrep
+
+[debs]
+ripgrep
+
+[homebrew]
+ripgrep
+
+# only used from system packages currently
+[build]
+builder = nop

build/fbcode_builder/manifests/sqlite3

@@ -3,13 +3,15 @@ name = sqlite3
 
 [debs]
 libsqlite3-dev
+sqlite3
 
 [homebrew]
 sqlite
 
 [rpms]
 sqlite-devel
 sqlite-libs
+sqlite
 
 [pps]
 sqlite3

build/fbcode_builder/manifests/tree

@@ -0,0 +1,37 @@
+[manifest]
+name = tree
+
+[debs]
+tree
+
+[homebrew]
+tree
+
+[rpms]
+tree
+
+[download.os=linux]
+url = https://salsa.debian.org/debian/tree-packaging/-/archive/debian/1.8.0-1/tree-packaging-debian-1.8.0-1.tar.gz
+sha256 = a841eee1d52bfd64a48f54caab9937b9bd92935055c48885c4ab1ae4dab7fae5
+
+[download.os=darwin]
+# The official package of tree source requires users of non-Linux platform to
+# comment/uncomment certain lines in the Makefile to build for their platform.
+# Besauce getdeps.py doesn't have that functionality we just use this custom
+# fork of tree which has proper lines uncommented for a OSX build
+url = https://github.com/lukaspiatkowski/tree-command/archive/debian/1.8.0-1-macos.tar.gz
+sha256 = 9cbe889553d95cf5a2791dd0743795d46a3c092c5bba691769c0e5c52e11229e
+
+[build.os=linux]
+builder = make
+subdir = tree-packaging-debian-1.8.0-1
+
+[build.os=darwin]
+builder = make
+subdir = tree-command-debian-1.8.0-1-macos
+
+[build.os=windows]
+builder = nop
+
+[make.install_args]
+install

build/fbcode_builder/manifests/zstd

@@ -7,6 +7,7 @@ zstd
 # 18.04 zstd is too old
 [debs.not(all(distro=ubuntu,distro_vers="18.04"))]
 libzstd-dev
+zstd
 
 [rpms]
 libzstd-devel