From 538b063d48df3ccbb06a09a7e2f88b2f66fecc23 Mon Sep 17 00:00:00 2001
From: Diego Hurtado <ocelotl@users.noreply.github.com>
Date: Thu, 28 Sep 2023 10:42:34 +0200
Subject: [PATCH 1/4] Add benchmarks workflow

Fixes #3449
---
 .github/workflows/benchmarks.yml | 59 ++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 .github/workflows/benchmarks.yml

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
new file mode 100644
index 00000000000..0192930ea74
--- /dev/null
+++ b/.github/workflows/benchmarks.yml
@@ -0,0 +1,59 @@
+name: SDK Benchmark Tests
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  sdk-benchmarks:
+    env:
+      py37: 3.7
+      py38: 3.8
+      py39: 3.9
+      py310: "3.10"
+      py311: "3.11"
+      pypy3: pypy-3.7
+      RUN_MATRIX_COMBINATION: ${{ matrix.python-version }}-sdk-${{ matrix.os }}
+    runs-on: self-hosted
+    strategy:
+      # Ensures the entire test matrix is run, even if one permutation fails
+      fail-fast: false
+      matrix:
+        python-version: [py37, py38, py39, py310, py311, pypy3]
+        os: [ubuntu-20.04, windows-2019]
+    steps:
+    - name: Checkout Core Repo @ SHA - ${{ github.sha }}
+      uses: actions/checkout@v2
+    - name: Set up Python ${{ env[matrix.python-version] }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ env[matrix.python-version] }}
+        architecture: 'x64'
+    - name: Install tox
+      run: pip install tox==3.27.1 -U tox-factor
+    - name: Cache tox environment
+      # Preserves .tox directory between runs for faster installs
+      uses: actions/cache@v2
+      with:
+        path: |
+          .tox
+          ~/.cache/pip
+        key: v3-tox-cache-${{ env.RUN_MATRIX_COMBINATION }}-${{ hashFiles('tox.ini',
+          'dev-requirements.txt') }}-core
+    - name: Run tox
+      run: tox -f ${{ matrix.python-version }}-sdk -- -k opentelemetry-sdk/tests/performance/benchmarks --benchmark-json=output.json
+    - name: Report on SDK benchmark results
+      uses: benchmark-action/github-action-benchmark@v1
+      with:
+        name: OpenTelemetry Python SDK Benchmarks - Python ${{ env[matrix.python-version ]}} - SDK
+        tool: pytest
+        output-file-path: opentelemetry-sdk/tests/output.json
+        gh-pages-branch: benchmarks
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        # Make a commit on `gh-pages` with benchmarks from previous step
+        benchmark-data-dir-path: "docs/benchmarks/loadtests"
+        auto-push: true
+        max-items-in-chart: 100
+        # Alert with a commit comment on possible performance regression
+        alert-threshold: 200%
+        fail-on-alert: true

From 4df920f128ed0864a3a087b8ed050a8d75b7cc1d Mon Sep 17 00:00:00 2001
From: Diego Hurtado <ocelotl@users.noreply.github.com>
Date: Sun, 1 Oct 2023 11:50:34 +0200
Subject: [PATCH 2/4] Refactor to align better with JS

---
 .github/workflows/benchmarks.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 0192930ea74..52e8ebe833f 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -1,8 +1,8 @@
 name: SDK Benchmark Tests
 
 on:
-  release:
-    types: [published]
+  push:
+    branches: [ main ]
 
 jobs:
   sdk-benchmarks:
@@ -51,7 +51,7 @@ jobs:
         gh-pages-branch: benchmarks
         github-token: ${{ secrets.GITHUB_TOKEN }}
         # Make a commit on `gh-pages` with benchmarks from previous step
-        benchmark-data-dir-path: "docs/benchmarks/loadtests"
+        benchmark-data-dir-path: "benchmarks"
         auto-push: true
         max-items-in-chart: 100
         # Alert with a commit comment on possible performance regression

From fe11e598cb28ab1856523a5b62f83450e3aec3d7 Mon Sep 17 00:00:00 2001
From: Diego Hurtado <ocelotl@users.noreply.github.com>
Date: Wed, 11 Oct 2023 14:30:18 -0600
Subject: [PATCH 3/4] Run tests in 3.11 only

---
 .github/workflows/benchmarks.yml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 52e8ebe833f..7e2365c0818 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -7,19 +7,14 @@ on:
 jobs:
   sdk-benchmarks:
     env:
-      py37: 3.7
-      py38: 3.8
-      py39: 3.9
-      py310: "3.10"
       py311: "3.11"
-      pypy3: pypy-3.7
       RUN_MATRIX_COMBINATION: ${{ matrix.python-version }}-sdk-${{ matrix.os }}
     runs-on: self-hosted
     strategy:
       # Ensures the entire test matrix is run, even if one permutation fails
       fail-fast: false
       matrix:
-        python-version: [py37, py38, py39, py310, py311, pypy3]
+        python-version: [py311]
         os: [ubuntu-20.04, windows-2019]
     steps:
     - name: Checkout Core Repo @ SHA - ${{ github.sha }}

From 56d7da1c0e3d976efd09f5cefa5c76ace5cdd333 Mon Sep 17 00:00:00 2001
From: Diego Hurtado <ocelotl@users.noreply.github.com>
Date: Fri, 13 Oct 2023 16:23:43 -0600
Subject: [PATCH 4/4] Fix alert commit config

---
 .github/workflows/benchmarks.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 7e2365c0818..2d7a6efd733 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -50,5 +50,5 @@ jobs:
         auto-push: true
         max-items-in-chart: 100
         # Alert with a commit comment on possible performance regression
-        alert-threshold: 200%
-        fail-on-alert: true
+        alert-threshold: '200%'
+        comment-on-alert: true