.github/workflows: disable performance tests (#10535)

solo-io · Dec 27, 2024 · 599a44e · 599a44e
1 parent 4b31f4a
commit 599a44e
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 7 deletions.
diff --git a/.github/workflows/nightly-tests.yaml b/.github/workflows/nightly-tests.yaml
@@ -2,6 +2,11 @@ name: Nightly
 
 env:
   GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} # necessary to pass upgrade tests
+  # https://github.com/solo-io/gloo/issues/10534
+  # Performance tests have been disabled. See the above issue for context around it
+  # To re-enable those tests (even temporarily) search for the usages of the env variable,
+  # and it will direct you to the code that needs to be changed
+  ENABLE_PERFORMANCE_TESTS: false
 
 on:
   # https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07
@@ -377,7 +382,6 @@ jobs:
           ref: v1.18.x
       - uses: ./.github/workflows/composite-actions/regression-tests
 
-
   regression_tests_17:
     name: v1.17.x regression tests
     if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'v1.17.x') || github.event.schedule == '0 7 * * 1' }}
@@ -415,8 +419,11 @@ jobs:
 
   performance_tests_on_demand:
     name: on demand performance tests
+    # This job does not respect the environment value of ENABLE_PERFORMANCE_TESTS
+    # This way, if a developer wanted to run the job on-demand, they could
     if: ${{ github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'workflow_initiating_branch' }}
-    runs-on: ubuntu-22.04
+    # We use a larger runner to ensure the performance tests can complete, without running out of disk space
+    runs-on: ubuntu-20.04-8core
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
@@ -427,7 +434,9 @@ jobs:
 
   performance_tests_main:
     name: main performance tests
-    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'main') || github.event.schedule == '0 5 * * *' }}
+    # Instead of false, we would want to define: env.ENABLE_PERFORMANCE_TESTS == 'true'
+    # Due to https://github.com/actions/runner/issues/1189#issuecomment-880110759, we cannot use env variables in job.if
+    if: ${{ false && ((github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'main') || github.event.schedule == '0 5 * * *') }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
     steps:
@@ -439,7 +448,9 @@ jobs:
 
   performance_tests_18:
     name: v1.18.x performance tests
-    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.18.x') || github.event.schedule == '0 6 * * 1' }}
+    # Instead of false, we would want to define: env.ENABLE_PERFORMANCE_TESTS == 'true'
+    # Due to https://github.com/actions/runner/issues/1189#issuecomment-880110759, we cannot use env variables in job.if
+    if: ${{ false && ((github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.18.x') || github.event.schedule == '0 6 * * 1') }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
     steps:
@@ -451,7 +462,9 @@ jobs:
 
   performance_tests_17:
     name: v1.17.x performance tests
-    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.17.x') || github.event.schedule == '0 7 * * 1' }}
+    # Instead of false, we would want to define: env.ENABLE_PERFORMANCE_TESTS == 'true'
+    # Due to https://github.com/actions/runner/issues/1189#issuecomment-880110759, we cannot use env variables in job.if
+    if: ${{ false && ((github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.17.x') || github.event.schedule == '0 7 * * 1') }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
     steps:
@@ -463,7 +476,9 @@ jobs:
 
   performance_tests_16:
     name: v1.16.x performance tests
-    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.16.x') || github.event.schedule == '0 8 * * 1' }}
+    # Instead of false, we would want to define: env.ENABLE_PERFORMANCE_TESTS == 'true'
+    # Due to https://github.com/actions/runner/issues/1189#issuecomment-880110759, we cannot use env variables in job.if
+    if: ${{ false && ((github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.16.x') || github.event.schedule == '0 8 * * 1') }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
     steps:

diff --git a/changelog/v1.19.0-beta3/disable-performance-tests.yaml b/changelog/v1.19.0-beta3/disable-performance-tests.yaml
@@ -0,0 +1,7 @@
+changelog:
+  - type: NON_USER_FACING
+    description: >-
+      Disable performance tests from running on nightly tests.
+      
+      skipCI-kube-tests:true
+      skipCI-docs-build:true
diff --git a/devel/testing/performance-tests.md b/devel/testing/performance-tests.md
@@ -1,5 +1,17 @@
 # Performance Tests
 
 ## Label
-Performance tests are labeled with `performance`. These tests are skipped on CI for PRs, but are run on a schedule as part of our [nightly tests](nightly-tests.md).
+Performance tests are labeled with `performance`.
 
+## When are these tests run?
+- These tests are skipped on CI for PRs.
+- They were run on a schedule as part of our [nightly tests](nightly-tests.md), but have since been disabled (https://github.com/solo-io/gloo/issues/10534)
+
+
+## How can you run these tests?
+You can run the `performance_test.go` locally with the make target:
+```bash
+make install-test-tools run-performance-tests
+```
+
+Or through GitHub actions via the `performance-test` workflow.