From 8f0257236c394684a8274843c4daa9c81be8c84a Mon Sep 17 00:00:00 2001
From: Hector Castejon Diaz <hector.castejon@databricks.com>
Date: Fri, 19 Jul 2024 11:13:21 +0200
Subject: [PATCH 1/4] [WIP] Run Integration Tests on PR

---
 .github/workflows/integration-tests.yml | 54 ++++++++++++++++++++++++
 .github/workflows/test-check.yml        | 55 +++++++++++++++++++++++++
 2 files changed, 109 insertions(+)
 create mode 100644 .github/workflows/integration-tests.yml
 create mode 100644 .github/workflows/test-check.yml

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
new file mode 100644
index 00000000..1adf468f
--- /dev/null
+++ b/.github/workflows/integration-tests.yml
@@ -0,0 +1,54 @@
+name: Integration Tests
+on:
+  repository_dispatch:
+    types: [integration-check]
+jobs:
+  myEvent:
+    runs-on: ubuntu-latest
+    steps:
+      ####################################################
+      # Update the status to show that the queued message
+      # was received and is being processed
+      ####################################################
+      - name: Acknowledge Request
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh api -X PATCH -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -f 'status=in_progress' \
+            -f 'output[title]=Integration Tests' \
+            -f 'output[summary]=Running Integration Tests for Py SDK' \
+            /repos/${{ github.repository }}/check-runs/${{ github.event.client_payload.checkRunId }}          
+
+      ####################################################
+      # Actually, we'll just sleep to simulate some work
+      ####################################################
+      - name: Processing
+        run: sleep 10
+
+      #####################################################
+      # Send a final message to complete the run and
+      # provide any final updates. Doing this one in JSON
+      # to make it more readable. This approach can also
+      # be used to get total control over the serialized
+      # data types (for example, integers).
+      #####################################################
+      - name: Complete Check
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh api -X PATCH -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/check-runs/${{ github.event.client_payload.checkRunId }} \
+            --input - <<- EOF
+            {
+              "conclusion": "success",
+              "details_url": "TODO",
+              "output": {
+                "title": "Integration Tests",
+                "summary": "**Summary**: The run completed.",
+                "text": "Everything worked as expected."
+               }
+            }
+          EOF
\ No newline at end of file
diff --git a/.github/workflows/test-check.yml b/.github/workflows/test-check.yml
new file mode 100644
index 00000000..c25cdec6
--- /dev/null
+++ b/.github/workflows/test-check.yml
@@ -0,0 +1,55 @@
+name: Integration Tests Validation
+
+# To trigger the check
+on:
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  start-check:
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write   # Permission to create a Check Run
+      contents: write # Permission to write a repository_dispatch requests
+    steps:
+      - name: Integration Tests Check
+        id: testcheck                    # An ID to allow the step to be referenced
+        env:
+          GH_TOKEN: ${{ github.token }} # Expose the token for GH CLI
+        run: |
+          ##########################################################
+          # Create a Check Run and indicate that it is being queued
+          # Use --jq to return the ID
+          ##########################################################
+
+          CHECKID=$(gh api -X POST -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -f name='Integration Tests Check' \
+            -f head_sha='${{ github.event.pull_request.head.sha }}' \
+            -f status='queued' \
+            -f 'output[title]=Integration Tests' \
+            -f 'output[summary]=Pending Integration Tests for Py SDK' \
+            --jq '.id' \
+            /repos/${{ github.repository }}/check-runs)
+          
+          ####################################################
+          # Put the ID into a step variable
+          ####################################################
+
+          echo "checkId=$CHECKID" >> $GITHUB_OUTPUT          
+
+      - name: Trigger Integration Tests
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          ##########################################################
+          # Create a repository_dispatch event of type my-check
+          # Send the SHA and the Check Run ID in the client_payload
+          ##########################################################
+
+          gh api -X POST -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -f 'event_type=integration-check' \
+            -f 'client_payload[checkRunId]=${{ steps.testcheck.outputs.checkId }}' \
+            -f 'client_payload[sha]=${{ github.sha }}' \
+            /repos/${{ github.repository }}/dispatches
\ No newline at end of file

From a5df9b4c055fc2510a2416853b3a3054c05c4e73 Mon Sep 17 00:00:00 2001
From: Hector Castejon Diaz <hector.castejon@databricks.com>
Date: Fri, 19 Jul 2024 11:23:03 +0200
Subject: [PATCH 2/4] test

---
 .github/workflows/integration-pr.yml    | 44 ++++++++++++++++++++
 .github/workflows/integration-tests.yml | 53 ++++++++++++++----------
 .github/workflows/test-check.yml        | 55 -------------------------
 3 files changed, 76 insertions(+), 76 deletions(-)
 create mode 100644 .github/workflows/integration-pr.yml
 delete mode 100644 .github/workflows/test-check.yml

diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml
new file mode 100644
index 00000000..a968fc4e
--- /dev/null
+++ b/.github/workflows/integration-pr.yml
@@ -0,0 +1,44 @@
+name: Integration Tests Check
+
+on:
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  start-check:
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write   # Permission to create a Check Run
+      actions: write  # Permission to run another workflow
+    steps:
+      - name: Create Check
+        id: checkrun
+        env:
+          GH_TOKEN: ${{ github.token }} # Expose the token for GH CLI
+        run: |
+          # Due to GitHub limitations, the check is assigned randomly to a check suit
+          # https://github.com/orgs/community/discussions/24616
+          CHECKID=$(gh api -X POST -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -f name='Integration Tests' \
+            -f head_sha='${{ github.event.pull_request.head.sha }}' \
+            -f status='queued' \
+            -f 'output[title]=Run Integration Tests' \
+            -f 'output[summary]=Running Integration Tests for Java SDK' \
+            --jq '.id' \
+            /repos/${{ github.repository }}/check-runs)
+          
+          # Put the ID into a step variable
+          echo "checkId=$CHECKID" >> $GITHUB_OUTPUT          
+
+      - name: Trigger Workflow
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # By default, this will run the workflow in the main branch. 
+          # To test changes the ref to the current branch.
+          gh api -X POST -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            -f 'inputs[checkRunId]=${{ steps.checkrun.outputs.checkId }}' \
+            -f "ref=run-tests" \
+            /repos/${{ github.repository }}/actions/workflows/integration-tests.yml/dispatches
\ No newline at end of file
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 1adf468f..606304d4 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -1,15 +1,15 @@
 name: Integration Tests
 on:
-  repository_dispatch:
-    types: [integration-check]
+  workflow_dispatch:
+    inputs:
+      checkRunId:
+        description: "ID for the Check Run in a PR"
+        type: string
+
 jobs:
   myEvent:
     runs-on: ubuntu-latest
     steps:
-      ####################################################
-      # Update the status to show that the queued message
-      # was received and is being processed
-      ####################################################
       - name: Acknowledge Request
         env:
           GH_TOKEN: ${{ github.token }}
@@ -17,9 +17,9 @@ jobs:
           gh api -X PATCH -H "Accept: application/vnd.github+json" \
             -H "X-GitHub-Api-Version: 2022-11-28" \
             -f 'status=in_progress' \
-            -f 'output[title]=Integration Tests' \
-            -f 'output[summary]=Running Integration Tests for Py SDK' \
-            /repos/${{ github.repository }}/check-runs/${{ github.event.client_payload.checkRunId }}          
+            -f 'output[title]=Run Integration Tests' \
+            -f 'output[summary]=Running Integration Tests for Java SDK' \
+            /repos/${{ github.repository }}/check-runs/${{ github.event.inputs.checkRunId }}          
 
       ####################################################
       # Actually, we'll just sleep to simulate some work
@@ -27,28 +27,39 @@ jobs:
       - name: Processing
         run: sleep 10
 
-      #####################################################
-      # Send a final message to complete the run and
-      # provide any final updates. Doing this one in JSON
-      # to make it more readable. This approach can also
-      # be used to get total control over the serialized
-      # data types (for example, integers).
-      #####################################################
       - name: Complete Check
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           gh api -X PATCH -H "Accept: application/vnd.github+json" \
             -H "X-GitHub-Api-Version: 2022-11-28" \
-            /repos/${{ github.repository }}/check-runs/${{ github.event.client_payload.checkRunId }} \
+            /repos/${{ github.repository }}/check-runs/${{ github.event.inputs.checkRunId }} \
             --input - <<- EOF
             {
               "conclusion": "success",
-              "details_url": "TODO",
               "output": {
-                "title": "Integration Tests",
-                "summary": "**Summary**: The run completed.",
-                "text": "Everything worked as expected."
+                "title": "Run Integration Tests",
+                "summary": "[Execution Details](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})",
+                "text": "Successful execution of Integration Tests for Java SDK"
+               }
+            }
+          EOF
+
+      - name: Fail Check
+        if: failure()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh api -X PATCH -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/check-runs/${{ github.event.inputs.checkRunId }} \
+            --input - <<- EOF
+            {
+              "conclusion": "failure",
+              "output": {
+                "title": "Run Integration Tests",
+                "summary": "[Execution Details](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})",
+                "text": "Failed execution of Integration Tests for Java SDK"
                }
             }
           EOF
\ No newline at end of file
diff --git a/.github/workflows/test-check.yml b/.github/workflows/test-check.yml
deleted file mode 100644
index c25cdec6..00000000
--- a/.github/workflows/test-check.yml
+++ /dev/null
@@ -1,55 +0,0 @@
-name: Integration Tests Validation
-
-# To trigger the check
-on:
-  pull_request:
-    branches: [ "main" ]
-
-jobs:
-  start-check:
-    runs-on: ubuntu-latest
-    permissions:
-      checks: write   # Permission to create a Check Run
-      contents: write # Permission to write a repository_dispatch requests
-    steps:
-      - name: Integration Tests Check
-        id: testcheck                    # An ID to allow the step to be referenced
-        env:
-          GH_TOKEN: ${{ github.token }} # Expose the token for GH CLI
-        run: |
-          ##########################################################
-          # Create a Check Run and indicate that it is being queued
-          # Use --jq to return the ID
-          ##########################################################
-
-          CHECKID=$(gh api -X POST -H "Accept: application/vnd.github+json" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            -f name='Integration Tests Check' \
-            -f head_sha='${{ github.event.pull_request.head.sha }}' \
-            -f status='queued' \
-            -f 'output[title]=Integration Tests' \
-            -f 'output[summary]=Pending Integration Tests for Py SDK' \
-            --jq '.id' \
-            /repos/${{ github.repository }}/check-runs)
-          
-          ####################################################
-          # Put the ID into a step variable
-          ####################################################
-
-          echo "checkId=$CHECKID" >> $GITHUB_OUTPUT          
-
-      - name: Trigger Integration Tests
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          ##########################################################
-          # Create a repository_dispatch event of type my-check
-          # Send the SHA and the Check Run ID in the client_payload
-          ##########################################################
-
-          gh api -X POST -H "Accept: application/vnd.github+json" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            -f 'event_type=integration-check' \
-            -f 'client_payload[checkRunId]=${{ steps.testcheck.outputs.checkId }}' \
-            -f 'client_payload[sha]=${{ github.sha }}' \
-            /repos/${{ github.repository }}/dispatches
\ No newline at end of file

From 090e369c81fa4d67c3278e3a155e32cb667897dc Mon Sep 17 00:00:00 2001
From: Hector Castejon Diaz <hector.castejon@databricks.com>
Date: Fri, 19 Jul 2024 14:56:33 +0200
Subject: [PATCH 3/4] Add PR number

---
 .github/workflows/integration-pr.yml                 | 12 +++++++++---
 .../{integration-tests.yml => sdk-nightly.yml}       |  7 ++++++-
 2 files changed, 15 insertions(+), 4 deletions(-)
 rename .github/workflows/{integration-tests.yml => sdk-nightly.yml} (92%)

diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml
index a968fc4e..dde9c81d 100644
--- a/.github/workflows/integration-pr.yml
+++ b/.github/workflows/integration-pr.yml
@@ -11,6 +11,11 @@ jobs:
       checks: write   # Permission to create a Check Run
       actions: write  # Permission to run another workflow
     steps:
+      - name: Extract branch name
+        shell: bash
+        run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+        id: extract_branch
+
       - name: Create Check
         id: checkrun
         env:
@@ -39,6 +44,7 @@ jobs:
           # To test changes the ref to the current branch.
           gh api -X POST -H "Accept: application/vnd.github+json" \
             -H "X-GitHub-Api-Version: 2022-11-28" \
-            -f 'inputs[checkRunId]=${{ steps.checkrun.outputs.checkId }}' \
-            -f "ref=run-tests" \
-            /repos/${{ github.repository }}/actions/workflows/integration-tests.yml/dispatches
\ No newline at end of file
+            -f 'inputs[check_run_id]=${{ steps.checkrun.outputs.checkId }}' \
+            -f 'inputs[pull_request_number]=${{ github.events.pull_request.number }}' \
+            -f "ref=${{ steps.extract_branch.outputs.branch }}" \
+            /repos/${{ github.repository }}/actions/workflows/sdk-nightly.yml/dispatches
\ No newline at end of file
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/sdk-nightly.yml
similarity index 92%
rename from .github/workflows/integration-tests.yml
rename to .github/workflows/sdk-nightly.yml
index 606304d4..a3ff7f72 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/sdk-nightly.yml
@@ -1,10 +1,15 @@
 name: Integration Tests
 on:
+  pull_request:
   workflow_dispatch:
     inputs:
-      checkRunId:
+      check_run_id:
         description: "ID for the Check Run in a PR"
         type: string
+        required: false
+      pull_request_number:
+        description: "Pull request number to test (if empty, tests run against main)"
+        required: false
 
 jobs:
   myEvent:

From 9fd04b74f6f0df8d65a67c9c976d0847a00004ea Mon Sep 17 00:00:00 2001
From: Hector Castejon Diaz <hector.castejon@databricks.com>
Date: Fri, 19 Jul 2024 15:13:09 +0200
Subject: [PATCH 4/4] More

---
 .github/workflows/integration-pr.yml |  2 --
 .github/workflows/sdk-nightly.yml    | 43 +++++++++++++++++++++-------
 2 files changed, 33 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/integration-pr.yml b/.github/workflows/integration-pr.yml
index dde9c81d..45a24fe9 100644
--- a/.github/workflows/integration-pr.yml
+++ b/.github/workflows/integration-pr.yml
@@ -40,8 +40,6 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          # By default, this will run the workflow in the main branch. 
-          # To test changes the ref to the current branch.
           gh api -X POST -H "Accept: application/vnd.github+json" \
             -H "X-GitHub-Api-Version: 2022-11-28" \
             -f 'inputs[check_run_id]=${{ steps.checkrun.outputs.checkId }}' \
diff --git a/.github/workflows/sdk-nightly.yml b/.github/workflows/sdk-nightly.yml
index a3ff7f72..2d319613 100644
--- a/.github/workflows/sdk-nightly.yml
+++ b/.github/workflows/sdk-nightly.yml
@@ -1,6 +1,5 @@
 name: Integration Tests
 on:
-  pull_request:
   workflow_dispatch:
     inputs:
       check_run_id:
@@ -11,9 +10,11 @@ on:
         description: "Pull request number to test (if empty, tests run against main)"
         required: false
 
+
 jobs:
-  myEvent:
+  acknowledge-request:
     runs-on: ubuntu-latest
+    if: ${{ github.event.inputs.check_run_id }} != ""
     steps:
       - name: Acknowledge Request
         env:
@@ -24,21 +25,39 @@ jobs:
             -f 'status=in_progress' \
             -f 'output[title]=Run Integration Tests' \
             -f 'output[summary]=Running Integration Tests for Java SDK' \
-            /repos/${{ github.repository }}/check-runs/${{ github.event.inputs.checkRunId }}          
+            /repos/${{ github.repository }}/check-runs/${{ github.event.inputs.check_run_id }}          
 
       ####################################################
       # Actually, we'll just sleep to simulate some work
       ####################################################
-      - name: Processing
-        run: sleep 10
+  integration-tests-prod:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.inputs.check_run_id }} != ""
+    steps:
+     - name: Processing
+       run: sleep 10
 
-      - name: Complete Check
+#  publish-lkg:
+#    if: ${{ github.ref == 'refs/heads/main' &&  inputs.pull_request_number == '' }}
+#    needs: [integration-tests-prod]
+#    uses: ./.github/workflows/publish-lkg.yml
+#    secrets: inherit
+#    with:
+#      lkg_sha: ${{ needs.integration-tests-prod.outputs.lkg_sha}}
+#      repository_name: "databricks-sdk-go"
+#      repository_org: "databricks"
+
+  complete-notification:
+    runs-on: ubuntu-latest
+    needs: [ integration-tests-prod ]
+    steps:
+      - name: Send Complete Notification
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           gh api -X PATCH -H "Accept: application/vnd.github+json" \
             -H "X-GitHub-Api-Version: 2022-11-28" \
-            /repos/${{ github.repository }}/check-runs/${{ github.event.inputs.checkRunId }} \
+            /repos/${{ github.repository }}/check-runs/${{ github.event.inputs.check_run_id }} \
             --input - <<- EOF
             {
               "conclusion": "success",
@@ -50,14 +69,18 @@ jobs:
             }
           EOF
 
-      - name: Fail Check
-        if: failure()
+  failure-notification:
+      runs-on: ubuntu-latest
+      if: failure()
+      needs: [integration-tests-prod]
+      steps:
+      - name: Send Failure Notification
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           gh api -X PATCH -H "Accept: application/vnd.github+json" \
             -H "X-GitHub-Api-Version: 2022-11-28" \
-            /repos/${{ github.repository }}/check-runs/${{ github.event.inputs.checkRunId }} \
+            /repos/${{ github.repository }}/check-runs/${{ github.event.inputs.check_run_id }} \
             --input - <<- EOF
             {
               "conclusion": "failure",