Tweaks to Python Test CI pipelines (#198)

* Tweaks to Python Test CI pipelines

- Increase the timeouts on the pytest runs.
  There have been some timeouts recently, due in part to the addition of code
  coverage tracking (#187).

- Separate the pylint checks (short) from the pytest runs (long, flaky) so that
  we can have the docker image publish task proceed in nightly runs even if the
  pytest run was flaky.

* Avoid duplicate ctest runs during Github CI runs

Fixes #196

* pass the GITHUB_WORKFLOW env var through the docker process

* fixup argument order

.github/workflows/main.yml

@@ -321,7 +321,8 @@ jobs:
  timeout-minutes: 10
  shell: bash
  run: |
- docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \
+ docker exec --env GITHUB_WORKFLOW=$GITHUB_WORKFLOW \
+ mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \
  ./build.linux.sh --Configuration=${{ matrix.configuration }}
  - name: Run ${{ matrix.configuration }} external cmake integration build/test
  timeout-minutes: 5
@@ -451,7 +452,7 @@ jobs:
  windows-python-checks:
  name: Run Python checks on Windows
  runs-on: windows-2019
- timeout-minutes: 30
+ timeout-minutes: 45
  strategy:
  matrix:
  # FIXME: MLOS currently fails pylint checks for 3.8
@@ -473,14 +474,15 @@ jobs:
  timeout-minutes: 2
  run: scripts\run-python-checks.cmd
  - name: Run Python unit tests (Windows)
- timeout-minutes: 30
+ timeout-minutes: 40
  run: scripts\run-python-tests.cmd
 
+ # Run pylint only in this test and make the container image publish task depend on this.
  linux-python-checks:
  name: Run Python checks on Ubuntu ${{ matrix.UbuntuVersion }}
  runs-on: ubuntu-latest
  needs: [prep-vars, docker-image-cached-build]
- timeout-minutes: 30
+ timeout-minutes: 15
  strategy:
  matrix:
  #UbuntuVersion: ${{ fromJson(needs.prep-vars.outputs.UbuntuVersionMatrix) }}
@@ -539,8 +541,75 @@ jobs:
  - name: Run pylint checks (Ubuntu ${{ matrix.UbuntuVersion }})
  timeout-minutes: 2
  run: docker exec mlos-build-python-${{ matrix.UbuntuVersion }} make python-checks
+ - name: Cleanup docker instance for Ubuntu ${{ matrix.UbuntuVersion }}
+ shell: bash
+ run: |
+ docker stop mlos-build-python-${{ matrix.UbuntuVersion }}
+ docker rm mlos-build-python-${{ matrix.UbuntuVersion }}
+
+ # Run the python unit tests separately from the pylint checks so that flakiness here doesn't block docker image publish step.
+ linux-python-tests:
+ name: Run Python unit tests on Ubuntu ${{ matrix.UbuntuVersion }}
+ runs-on: ubuntu-latest
+ needs: [prep-vars, docker-image-cached-build]
+ timeout-minutes: 45
+ strategy:
+ matrix:
+ #UbuntuVersion: ${{ fromJson(needs.prep-vars.outputs.UbuntuVersionMatrix) }}
+ UbuntuVersion: [20.04]
+ # TODO: add support for checking different versions of Python
+ # Right now, we install Python 3.7 in the container and default to
+ # using it if available, else fallback to the system python3.
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v2
+ # "Fetch" the docker image precached by the "docker-image-cached-build" job that
+ # we depend on by re-executing (roughly) its same steps.
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+ - name: Cache Docker layers
+ id: load-docker-cache
+ uses: actions/cache@v2
+ with:
+ path: /tmp/.buildx-cache
+ # Load the output cache from the docker-image-cached-build job.
+ key: mlos-buildx-${{ matrix.UbuntuVersion }}-${{ needs.prep-vars.outputs.utc-date }}-${{ github.sha }}
+ #restore-keys: (no fallback)
+ # To prevent race issues, abort if that fails.
+ - name: Check for docker cache hit
+ if: steps.load-docker-cache.outputs.cache-hit != 'true'
+ shell: bash
+ run: |
+ echo "Failed to load docker image cache from previous job. Perhaps another workflow forced it out. Please try again."
+ exit 1
+ - name: Import cached image
+ timeout-minutes: 10
+ uses: docker/build-push-action@v2
+ with:
+ build-args: UbuntuVersion=${{ matrix.UbuntuVersion }}
+ # For this task we can get a slight speed up by skipping the dotnet,
+ # clang, cmake layers.
+ target: mlos-build-base-with-python
+ tags: mlos-build-python-${{ matrix.UbuntuVersion }}:${{ github.sha }}
+ cache-from: |
+ type=local,src=/tmp/.buildx-cache
+ #cache-to: type=local,dest=/tmp/.buildx-cache
+ load: true
+ # Now, start the image as a new container instance.
+ - name: Start docker instance for Ubuntu ${{ matrix.UbuntuVersion }}
+ # These images are expected to have installed all the pip dependencies already.
+ shell: bash
+ run: |
+ docker run -it -d -v $PWD:/src/MLOS -u $UID \
+ --name mlos-build-python-${{ matrix.UbuntuVersion }} \
+ mlos-build-python-${{ matrix.UbuntuVersion }}:${{ github.sha }}
+ - name: Setup local user in docker Container
+ shell: bash
+ run: |
+ docker exec -u root mlos-build-python-${{ matrix.UbuntuVersion }} \
+ /src/MLOS/scripts/setup-container-user.sh github-action-runner $(id -u) $(id -g)
  - name: Run Python unit tests (Ubuntu ${{ matrix.UbuntuVersion }})
- timeout-minutes: 30
+ timeout-minutes: 45
  run: docker exec mlos-build-python-${{ matrix.UbuntuVersion }} make python-test
  - name: Upload coverage to codecov
  if: ${{ github.repository == 'microsoft/mlos' }}

build.cake

@@ -29,6 +29,11 @@ string ObjectDirectory
  }
 }
 
+bool IsRunningInGithub()
+{
+ return !string.IsNullOrEmpty(EnvironmentVariable<string>("GITHUB_WORKFLOW", ""));
+}
+
 var MsBuildSettings = new DotNetCoreMSBuildSettings { MaxCpuCount = 0 };
 
 //
@@ -303,6 +308,7 @@ Task("Binplace-CMake")
 
 Task("Test-CMake")
  .WithCriteria(() => IsRunningOnUnix())
+ .WithCriteria(() => !IsRunningInGithub()) // github pipelines already test this via `make cmake-test`
  .IsDependentOn("Binplace-CMake")
  .Does(() =>
  {
@@ -327,6 +333,8 @@ Task("Test-CMake")
  });
 
 
+// This does *almost* the same thing as Test-CMake but using the Mlos.TestRun.proj to invoke the tests instead of ctest.
+//
 Task("Run-CMake-UnitTests")
  .IsDependentOn("Binplace-CMake")
  .WithCriteria(() => IsRunningOnUnix())