new cross-language task+configuration with input expansion service an…

…d python collect marker
apache · Mar 1, 2023 · 95f8f87 · 95f8f87
1 parent 797a526
commit 95f8f87
Show file tree

Hide file tree

Showing 318 changed files with 12,637 additions and 2,308 deletions.
diff --git a/.github/actions/setup-self-hosted-action/action.yml b/.github/actions/setup-self-hosted-action/action.yml
@@ -34,9 +34,9 @@ inputs:
     required: false
     description: 'Set as false if does not require java-8 setup'
     default: 'true'
-  requires-go-18:
+  requires-go-19:
     required: false
-    description: 'Set as false if does not require go-18 setup'
+    description: 'Set as false if does not require go-19 setup'
     default: 'true'
 
 runs:
@@ -64,7 +64,7 @@ runs:
         distribution: 'temurin'
         java-version: 8
     - name: Set Go Version
-      if: ${{ inputs.requires-go-18 == 'true'  }}
+      if: ${{ inputs.requires-go-19 == 'true'  }}
       uses: actions/setup-go@v3
       with:
-        go-version: '1.18.0'
+        go-version: '1.19.0'
diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml
@@ -34,7 +34,7 @@ jobs:
     name: Build Playground Frontend App
     runs-on: [self-hosted, ubuntu-20.04]
     env:
-      GO_VERSION: 1.18.0
+      GO_VERSION: 1.19.6
       BEAM_VERSION: 2.40.0
       TERRAFORM_VERSION: 1.0.9
       FLUTTER_VERSION: 3.3.2

diff --git a/.github/workflows/git_tag_released_version.yml b/.github/workflows/git_tag_released_version.yml
@@ -32,7 +32,7 @@ on:
 
 jobs:
   generate_tags:
-    runs-on: [self-hosted, ubuntu-20.04]
+    runs-on: ubuntu-latest
     env:
       VERSION_PATH: ${{ github.event.inputs.VERSION_TAG }}
     steps:

diff --git a/.github/workflows/go_tests.yml b/.github/workflows/go_tests.yml
@@ -35,7 +35,7 @@ concurrency:
   cancel-in-progress: true
 jobs:
   build:
-    runs-on: [self-hosted, ubuntu-20.04]
+    runs-on: ubuntu-latest
     name: Go Build
     steps:
       - name: Check out code
@@ -44,7 +44,7 @@ jobs:
           fetch-depth: 2
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18'
+          go-version: '1.19'
       - name: Delete old coverage
         run: "cd sdks/go/pkg && rm -rf .coverage || :"
       - name: Run coverage

diff --git a/.github/workflows/issue-tagger.yml b/.github/workflows/issue-tagger.yml
@@ -20,7 +20,7 @@ on:
 
 jobs:
   label:
-    runs-on: [self-hosted, ubuntu-20.04]
+    runs-on: ubuntu-latest
     permissions:
       issues: write
     steps:

diff --git a/.github/workflows/java_tests.yml b/.github/workflows/java_tests.yml
@@ -43,7 +43,7 @@ jobs:
   check_gcp_variables:
     timeout-minutes: 5
     name: "Check GCP variables set"
-    runs-on: [self-hosted, ubuntu-20.04]
+    runs-on: ubuntu-latest
     outputs:
       gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }}
     steps:
@@ -66,7 +66,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [[self-hosted, ubuntu-20.04], macos-latest, [self-hosted, windows-server-2019]]
+        os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
       - name: Check out code
         uses: actions/checkout@v3
@@ -125,7 +125,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [[self-hosted, ubuntu-20.04], macos-latest, [self-hosted, windows-server-2019]]
+        os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
       - name: Check out code
         uses: actions/checkout@v3
@@ -164,7 +164,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [[self-hosted, ubuntu-20.04],[self-hosted, windows-server-2019]]
+        os: [ubuntu-latest,windows-latest]
     if: |
       needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && (
         (github.event_name == 'push' || github.event_name == 'schedule') ||

diff --git a/.github/workflows/label_prs.yml b/.github/workflows/label_prs.yml
@@ -21,7 +21,7 @@ on: [pull_request_target]
 permissions: read-all
 jobs:
   label:
-    runs-on: [self-hosted, ubuntu-20.04]
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       pull-requests: write

diff --git a/.github/workflows/playground_deploy_backend.yml b/.github/workflows/playground_deploy_backend.yml
@@ -34,7 +34,7 @@ jobs:
     name: Build Playground Backend App
     runs-on: ubuntu-latest
     env:
-      GO_VERSION: 1.18.0
+      GO_VERSION: 1.19.6
       BEAM_VERSION: 2.40.0
       TERRAFORM_VERSION: 1.0.9
       STAND_SUFFIX: ''

diff --git a/.github/workflows/pr-bot-new-prs.yml b/.github/workflows/pr-bot-new-prs.yml
@@ -31,7 +31,7 @@ jobs:
       statuses: read
     # Don't run on forks
     if: github.repository == 'apache/beam'
-    runs-on: [self-hosted, ubuntu-20.04]
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: Setup Node

diff --git a/.github/workflows/pr-bot-pr-updates.yml b/.github/workflows/pr-bot-pr-updates.yml
@@ -31,7 +31,7 @@ jobs:
 
     # Don't run on forks
     if: github.repository == 'apache/beam'
-    runs-on: [self-hosted, ubuntu-20.04]
+    runs-on: ubuntu-latest
 
     steps:
       # Pin to master so users can't do anything malicious on their own branch and run it here.

diff --git a/.github/workflows/pr-bot-prs-needing-attention.yml b/.github/workflows/pr-bot-prs-needing-attention.yml
@@ -31,7 +31,7 @@ jobs:
       statuses: read
     # Don't run on forks
     if: github.repository == 'apache/beam'
-    runs-on: [self-hosted, ubuntu-20.04]
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: Setup Node

diff --git a/.github/workflows/publish_github_release_notes.yml b/.github/workflows/publish_github_release_notes.yml
@@ -31,7 +31,7 @@ permissions: read-all
 
 jobs:
   set-properties:
-    runs-on: [self-hosted, ubuntu-20.04]
+    runs-on: ubuntu-latest
     outputs:
       properties: ${{ steps.test-properties.outputs.properties }}
     steps:
@@ -41,7 +41,7 @@ jobs:
         uses: ./.github/actions/setup-default-test-properties
 
   publish_github_release_notes:
-    runs-on: [self-hosted, ubuntu-20.04]
+    runs-on: ubuntu-latest
     needs: set-properties
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml
@@ -179,7 +179,7 @@ jobs:
       - name: Install go
         uses: actions/setup-go@v3
         with:
-          go-version: '1.18'
+          go-version: '1.19'
       - name: Download source from artifacts
         uses: actions/download-artifact@v3
         with:

diff --git a/.github/workflows/reportGenerator.yml b/.github/workflows/reportGenerator.yml
@@ -24,7 +24,7 @@ on:
 jobs:
   assign:
     name: Generate issue report
-    runs-on: [self-hosted, ubuntu-20.04]
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
     - name: Setup Node

diff --git a/.github/workflows/self-assign.yml b/.github/workflows/self-assign.yml
@@ -23,7 +23,7 @@ jobs:
       issues: write
     name: Take or close an issue
     if: ${{ !github.event.issue.pull_request }}
-    runs-on: [self-hosted, ubuntu-20.04]
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/github-script@v6
       with:

diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
@@ -21,7 +21,7 @@ on:
 permissions: read-all
 jobs:
   stale:
-    runs-on: [self-hosted, ubuntu-20.04]
+    runs-on: ubuntu-latest
     permissions:
       issues: write
       pull-requests: write

diff --git a/.github/workflows/triaged-on-assign.yml b/.github/workflows/triaged-on-assign.yml
@@ -22,7 +22,7 @@ jobs:
     permissions:
       issues: write
     name: Mark issue as triaged when assigned
-    runs-on: [self-hosted, ubuntu-20.04]
+    runs-on: ubuntu-latest
     steps:
     - run: |
         ISSUE_NUMBER="$(jq '.issue.number' $GITHUB_EVENT_PATH)"

diff --git a/.test-infra/dataproc/flink_cluster.sh b/.test-infra/dataproc/flink_cluster.sh
@@ -46,8 +46,7 @@ set -Eeuxo pipefail
 
 # GCloud properties
 GCLOUD_ZONE="${GCLOUD_ZONE:=us-central1-a}"
-# TODO: replace preview once dataproc 2.1 released
-DATAPROC_VERSION="${DATAPROC_VERSION:=preview-debian11}"
+DATAPROC_VERSION="${DATAPROC_VERSION:=2.1-debian}"
 GCLOUD_REGION=`echo $GCLOUD_ZONE | sed -E "s/(-[a-z])?$//"`
 
 MASTER_NAME="$CLUSTER_NAME-m"

diff --git a/.test-infra/jenkins/CommonJobProperties.groovy b/.test-infra/jenkins/CommonJobProperties.groovy
@@ -69,6 +69,10 @@ class CommonJobProperties {
         extensions {
           wipeOutWorkspace()
           relativeTargetDirectory(checkoutDir)
+          cloneOptions {
+            shallow()
+            noTags()
+          }
           if (!allowRemotePoll) {
             disableRemotePoll()
           }

diff --git a/.test-infra/jenkins/PrecommitJobBuilder.groovy b/.test-infra/jenkins/PrecommitJobBuilder.groovy
@@ -47,13 +47,27 @@ class PrecommitJobBuilder {
   /** Whether to trigger on new PR commits. Useful to set to false when testing new jobs. */
   boolean commitTriggering = true
 
+  /**
+   * Whether to trigger on cron run. Useful to set jobs that runs tasks covered by
+   * other test suites but are deemed to triggered on pull request only.
+   */
+  boolean cronTriggering = true
+
+  /**
+   * Whether to configure defaultPathTriggers.
+   * Set to false for PreCommit only runs on certain code path change.
+   */
+  boolean defaultPathTriggering = true
+
   /**
    * Define a set of pre-commit jobs.
    *
    * @param additionalCustomization Job DSL closure with additional customization to apply to the job.
    */
   void build(Closure additionalCustomization = {}) {
-    defineCronJob additionalCustomization
+    if (cronTriggering) {
+      defineCronJob additionalCustomization
+    }
     if (commitTriggering) {
       defineCommitJob additionalCustomization
     }
@@ -82,7 +96,7 @@ class PrecommitJobBuilder {
       '^gradle.bat$',
       '^settings.gradle.kts$'
     ]
-    if (triggerPathPatterns) {
+    if (defaultPathTriggering && triggerPathPatterns) {
       triggerPathPatterns.addAll defaultPathTriggers
     }
     job.with {