mononoke/integration: create a Makefile to run tests as part of getde…

…ps.py build (#67)

Summary:
Pull Request resolved: #67

With this change it will be possible to build dependencies of and run integration tests using getdeps.py.

Differential Revision: D24253268

fbshipit-source-id: be0ec4614959aa1e4682a1bedd891ec8245dcef9

.github/workflows/mononoke-integration_linux.yml

@@ -33,21 +33,20 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '2.7'
-    - name: Install Python 2 dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install "dulwich==0.18.6"
-    - name: Install Apt-get dependencies
-      run: |
-        sudo apt-get install nmap tree
     - name: Install system deps
-      run: sudo python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive eden_scm
+      run: >-
+        sudo python3 build/fbcode_builder/getdeps.py
+        --allow-system-packages
+        install-system-deps
+        --recursive
+        mononoke_integration
     - name: Build eden_scm dependencies
       run: >-
         python3 build/fbcode_builder/getdeps.py build
         --allow-system-packages
         --scratch-path /tmp/build
         --only-deps
+        --no-tests
         --src-dir=.
         eden_scm
     - name: Build eden_scm
@@ -56,6 +55,7 @@ jobs:
         --allow-system-packages
         --scratch-path /tmp/build
         --no-deps
+        --no-tests
         --src-dir=.
         eden_scm
     - name: Check space before cleanup
@@ -69,7 +69,7 @@ jobs:
         python3 build/fbcode_builder/getdeps.py build
         --allow-system-packages
         --scratch-path /tmp/build
-        --no-deps
+        --no-tests
         --src-dir=.
         eden_scm_lib_edenapi_tools
     - name: Check space before cleanup
@@ -84,6 +84,7 @@ jobs:
         --allow-system-packages
         --scratch-path /tmp/build
         --only-deps
+        --no-tests
         --src-dir=.
         mononoke
     - name: Build mononoke
@@ -92,6 +93,7 @@ jobs:
         --allow-system-packages
         --scratch-path /tmp/build
         --no-deps
+        --no-tests
         --src-dir=.
         mononoke
     - name: Check space before cleanup
@@ -104,27 +106,48 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '3.7'
-    - name: Install Python 3 dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install click
     - name: Check space before running tests
       run: df -h
-    - name: Run Monononke integration tests
+    - name: Build mononoke_integration dependencies
+      # This is a way of getting all the dependencies of mononoke_integration
+      # which were not covered by the dependencies of the previous 3 projects.
+      # We have to do this unfortunately to avoid rebuilding the previous
+      # project as they have been built with "--src-dir=." option.
+      run: >-
+        export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
+        for x in $(
+        sort
+        <(python3 build/fbcode_builder/getdeps.py list-deps mononoke_integration)
+        <(python3 build/fbcode_builder/getdeps.py list-deps mononoke)
+        <(python3 build/fbcode_builder/getdeps.py list-deps mononoke)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm_lib_edenapi_tools)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm_lib_edenapi_tools)
+        <(echo mononoke_integration)
+        | uniq -u
+        )
+        ;do
+        python3 build/fbcode_builder/getdeps.py build
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --no-tests
+        $x
+        ;done
+    - name: Build mononoke_integration
       run: >-
-        PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages"
-        python3
-        eden/mononoke/tests/integration/run_tests_getdeps.py
-        /tmp/build/installed
-        /tmp/build/build/mononoke_integration_test
-      continue-on-error: true
+        python3 build/fbcode_builder/getdeps.py build
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --no-deps
+        --src-dir=.
+        mononoke_integration
+    - name: Test mononoke_integration
+      run: >-
+        python3 build/fbcode_builder/getdeps.py test
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --src-dir=.
+        mononoke_integration
     - name: Check space after running tests
       run: df -h
-    - name: Rerun failed Monononke integration tests (reduce flakiness)
-      run: >-
-        cat eden/mononoke/tests/integration/.test* || true;
-        PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages"
-        python3
-        eden/mononoke/tests/integration/run_tests_getdeps.py
-        /tmp/build/installed /tmp/build/build/mononoke_integration_test
-        --rerun-failed

.github/workflows/mononoke-integration_mac.yml

@@ -25,28 +25,25 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '2.7'
-    - name: Install Python 2 dependencies
+    - name: Install curl-openssl
       run: |
-        python -m pip install --upgrade pip
-        pip install "dulwich==0.18.6"
-    - name: Install Brew dependencies
-      run: |
-        brew install bash coreutils curl-openssl gnu-sed grep jq nmap tree
+        brew install curl-openssl
     - name: Install system deps
       run: >-
         export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
         sudo python3 build/fbcode_builder/getdeps.py
         --allow-system-packages
         install-system-deps
         --recursive
-        eden_scm
+        mononoke_integration
     - name: Build eden_scm dependencies
       run: >-
         export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
         python3 build/fbcode_builder/getdeps.py build
         --allow-system-packages
         --scratch-path /tmp/build
         --only-deps
+        --no-tests
         --src-dir=.
         eden_scm
     - name: Build eden_scm
@@ -56,6 +53,7 @@ jobs:
         --allow-system-packages
         --scratch-path /tmp/build
         --no-deps
+        --no-tests
         --src-dir=.
         eden_scm
     - name: Build eden_scm_lib_edenapi_tools
@@ -64,7 +62,7 @@ jobs:
         python3 build/fbcode_builder/getdeps.py build
         --allow-system-packages
         --scratch-path /tmp/build
-        --no-deps
+        --no-tests
         --src-dir=.
         eden_scm_lib_edenapi_tools
     - name: Build mononoke dependencies
@@ -74,6 +72,7 @@ jobs:
         --allow-system-packages
         --scratch-path /tmp/build
         --only-deps
+        --no-tests
         --src-dir=.
         mononoke
     - name: Build mononoke
@@ -83,33 +82,57 @@ jobs:
         --allow-system-packages
         --scratch-path /tmp/build
         --no-deps
+        --no-tests
         --src-dir=.
         mononoke
     - name: Install Python 3.7
       uses: actions/setup-python@v2
       with:
         python-version: '3.7'
-    - name: Install Python 3 dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install click
     - name: Check space
       run: df -h
-    - name: Run Monononke integration tests
+    - name: Build mononoke_integration dependencies
+      # This is a way of getting all the dependencies of mononoke_integration
+      # which were not covered by the dependencies of the previous 3 projects.
+      # We have to do this unfortunately to avoid rebuilding the previous
+      # project as they have been built with "--src-dir=." option.
+      run: >-
+        export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
+        for x in $(
+        sort
+        <(python3 build/fbcode_builder/getdeps.py list-deps mononoke_integration)
+        <(python3 build/fbcode_builder/getdeps.py list-deps mononoke)
+        <(python3 build/fbcode_builder/getdeps.py list-deps mononoke)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm_lib_edenapi_tools)
+        <(python3 build/fbcode_builder/getdeps.py list-deps eden_scm_lib_edenapi_tools)
+        <(echo mononoke_integration)
+        | uniq -u
+        )
+        ;do
+        python3 build/fbcode_builder/getdeps.py build
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --no-tests
+        $x
+        ;done
+    - name: Build mononoke_integration
       run: >-
         export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
-        PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages"
-        python3
-        eden/mononoke/tests/integration/run_tests_getdeps.py
-        /tmp/build/installed
-        /tmp/build/build/mononoke_integration_test
-      continue-on-error: true
-    - name: Rerun failed Monononke integration tests (reduce flakiness)
+        python3 build/fbcode_builder/getdeps.py build
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --no-deps
+        --src-dir=.
+        mononoke_integration
+    - name: Test mononoke_integration
       run: >-
-        cat eden/mononoke/tests/integration/.test* || true;
         export PATH="/usr/local/opt/curl-openssl/bin:$PATH";
-        PYTHONPATH="$PYTHONPATH:/opt/hostedtoolcache/Python/2.7.18/x64/lib/python2.7/site-packages"
-        python3
-        eden/mononoke/tests/integration/run_tests_getdeps.py
-        /tmp/build/installed /tmp/build/build/mononoke_integration_test
-        --rerun-failed
+        python3 build/fbcode_builder/getdeps.py test
+        --allow-system-packages
+        --scratch-path /tmp/build
+        --src-dir=.
+        mononoke_integration
+    - name: Check space after running tests
+      run: df -h

build/fbcode_builder/getdeps/builder.py

@@ -145,12 +145,17 @@ def __init__(
         inst_dir,
         build_args,
         install_args,
+        test_args,
     ):
         super(MakeBuilder, self).__init__(
             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
         )
         self.build_args = build_args or []
         self.install_args = install_args or []
+        self.test_args = test_args
+
+    def _get_prefix(self):
+        return ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
 
     def _build(self, install_dirs, reconfigure):
         env = self._compute_env(install_dirs)
@@ -161,17 +166,24 @@ def _build(self, install_dirs, reconfigure):
         cmd = (
             ["make", "-j%s" % self.build_opts.num_jobs]
             + self.build_args
-            + ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
+            + self._get_prefix()
         )
         self._run_cmd(cmd, env=env)
 
-        install_cmd = (
-            ["make"]
-            + self.install_args
-            + ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
-        )
+        install_cmd = ["make"] + self.install_args + self._get_prefix()
         self._run_cmd(install_cmd, env=env)
 
+    def run_tests(
+        self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot
+    ):
+        if not self.test_args:
+            return
+
+        env = self._compute_env(install_dirs)
+
+        cmd = ["make"] + self.test_args + self._get_prefix()
+        self._run_cmd(cmd, env=env)
+
 
 class AutoconfBuilder(BuilderBase):
     def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, args):

build/fbcode_builder/getdeps/manifest.py

@@ -88,6 +88,7 @@
     "b2.args": {"optional_section": True},
     "make.build_args": {"optional_section": True},
     "make.install_args": {"optional_section": True},
+    "make.test_args": {"optional_section": True},
     "header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}},
     "shipit.pathmap": {"optional_section": True},
     "shipit.strip": {"optional_section": True},
@@ -437,6 +438,7 @@ def create_builder(  # noqa:C901
         if builder == "make":
             build_args = self.get_section_as_args("make.build_args", ctx)
             install_args = self.get_section_as_args("make.install_args", ctx)
+            test_args = self.get_section_as_args("make.test_args", ctx)
             return MakeBuilder(
                 build_options,
                 ctx,
@@ -446,6 +448,7 @@ def create_builder(  # noqa:C901
                 inst_dir,
                 build_args,
                 install_args,
+                test_args,
             )
 
         if builder == "autoconf":

build/fbcode_builder/manifests/mononoke

@@ -34,6 +34,7 @@ tools/rust/ossconfigs = .
 ^fbcode/eden/mononoke/Cargo\.toml$
 ^fbcode/eden/mononoke/(?!public_autocargo).+/Cargo\.toml$
 ^fbcode/configerator/structs/scm/mononoke/(?!public_autocargo).+/Cargo\.toml$
+^.*/facebook/.*$
 
 [dependencies]
 fbthrift-source

build/fbcode_builder/manifests/mononoke_integration

@@ -18,14 +18,20 @@ build-getdeps
 [make.install_args]
 install-getdeps
 
+[make.test_args]
+test-getdeps
+
 [shipit.pathmap]
 fbcode/eden/mononoke/tests/integration = eden/mononoke/tests/integration
 
 [shipit.strip]
 ^.*/facebook/.*$
 
 [dependencies]
+eden_scm
+eden_scm_lib_edenapi_tools
 jq
+mononoke
 nmap
 python-click
 python-dulwich