From 815cd44436a7e760bde76286f6207b59d06473e8 Mon Sep 17 00:00:00 2001
From: Brian Kroth <bpkroth@microsoft.com>
Date: Wed, 2 Dec 2020 13:57:24 -0600
Subject: [PATCH] Tweaks to Python Test CI pipelines

- Increase the timeouts on the pytest runs.
  There have been some timeouts recently, due in part to the addition of code
  coverage tracking (#187).

- Separate the pylint checks (short) from the pytest runs (long, flaky) so that
  we can have the docker image publish task proceed in nightly runs even if the
  pytest run was flaky.
---
 .github/workflows/main.yml | 76 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 72 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 09667476b8..e52829518e 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -451,7 +451,7 @@ jobs:
   windows-python-checks:
     name: Run Python checks on Windows
     runs-on: windows-2019
-    timeout-minutes: 30
+    timeout-minutes: 45
     strategy:
       matrix:
         # FIXME: MLOS currently fails pylint checks for 3.8
@@ -473,14 +473,15 @@ jobs:
       timeout-minutes: 2
       run: scripts\run-python-checks.cmd
     - name: Run Python unit tests (Windows)
-      timeout-minutes: 30
+      timeout-minutes: 40
       run: scripts\run-python-tests.cmd
 
+  # Run pylint only in this test and make the container image publish task depend on this.
   linux-python-checks:
     name: Run Python checks on Ubuntu ${{ matrix.UbuntuVersion }}
     runs-on: ubuntu-latest
     needs: [prep-vars, docker-image-cached-build]
-    timeout-minutes: 30
+    timeout-minutes: 15
     strategy:
       matrix:
         #UbuntuVersion: ${{ fromJson(needs.prep-vars.outputs.UbuntuVersionMatrix) }}
@@ -539,8 +540,75 @@ jobs:
     - name: Run pylint checks (Ubuntu ${{ matrix.UbuntuVersion }})
       timeout-minutes: 2
       run: docker exec mlos-build-python-${{ matrix.UbuntuVersion }} make python-checks
+    - name: Cleanup docker instance for Ubuntu ${{ matrix.UbuntuVersion }}
+      shell: bash
+      run: |
+        docker stop mlos-build-python-${{ matrix.UbuntuVersion }}
+        docker rm mlos-build-python-${{ matrix.UbuntuVersion }}
+
+  # Run the python unit tests separately from the pylint checks so that flakiness here doesn't block docker image publish step.
+  linux-python-tests:
+    name: Run Python unit tests on Ubuntu ${{ matrix.UbuntuVersion }}
+    runs-on: ubuntu-latest
+    needs: [prep-vars, docker-image-cached-build]
+    timeout-minutes: 45
+    strategy:
+      matrix:
+        #UbuntuVersion: ${{ fromJson(needs.prep-vars.outputs.UbuntuVersionMatrix) }}
+        UbuntuVersion: [20.04]
+        # TODO: add support for checking different versions of Python
+        # Right now, we install Python 3.7 in the container and default to
+        # using it if available, else fallback to the system python3.
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    # "Fetch" the docker image precached by the "docker-image-cached-build" job that
+    # we depend on by re-executing (roughly) its same steps.
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+    - name: Cache Docker layers
+      id: load-docker-cache
+      uses: actions/cache@v2
+      with:
+        path: /tmp/.buildx-cache
+        # Load the output cache from the docker-image-cached-build job.
+        key: mlos-buildx-${{ matrix.UbuntuVersion }}-${{ needs.prep-vars.outputs.utc-date }}-${{ github.sha }}
+        #restore-keys: (no fallback)
+    # To prevent race issues, abort if that fails.
+    - name: Check for docker cache hit
+      if: steps.load-docker-cache.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        echo "Failed to load docker image cache from previous job.  Perhaps another workflow forced it out.  Please try again."
+        exit 1
+    - name: Import cached image
+      timeout-minutes: 10
+      uses: docker/build-push-action@v2
+      with:
+        build-args: UbuntuVersion=${{ matrix.UbuntuVersion }}
+        # For this task we can get a slight speed up by skipping the dotnet,
+        # clang, cmake layers.
+        target: mlos-build-base-with-python
+        tags: mlos-build-python-${{ matrix.UbuntuVersion }}:${{ github.sha }}
+        cache-from: |
+          type=local,src=/tmp/.buildx-cache
+        #cache-to: type=local,dest=/tmp/.buildx-cache
+        load: true
+    # Now, start the image as a new container instance.
+    - name: Start docker instance for Ubuntu ${{ matrix.UbuntuVersion }}
+      # These images are expected to have installed all the pip dependencies already.
+      shell: bash
+      run: |
+        docker run -it -d -v $PWD:/src/MLOS -u $UID \
+          --name mlos-build-python-${{ matrix.UbuntuVersion }} \
+          mlos-build-python-${{ matrix.UbuntuVersion }}:${{ github.sha }}
+    - name: Setup local user in docker Container
+      shell: bash
+      run: |
+        docker exec -u root mlos-build-python-${{ matrix.UbuntuVersion }} \
+          /src/MLOS/scripts/setup-container-user.sh github-action-runner $(id -u) $(id -g)
     - name: Run Python unit tests (Ubuntu ${{ matrix.UbuntuVersion }})
-      timeout-minutes: 30
+      timeout-minutes: 45
       run: docker exec mlos-build-python-${{ matrix.UbuntuVersion }} make python-test
     - name: Upload coverage to codecov
       if: ${{ github.repository == 'microsoft/mlos' }}