Merge remote-tracking branch 'upstream/master' into feature/dead-lett…

…er-queue-core

.github/actions/gradle-command-self-hosted-action/action.yml

@@ -35,9 +35,9 @@ runs:
     - name: Run Gradle Command
       shell: bash
       run: |
-        # Removing settings.xml is a workaround to avoid a decryption issue
-        # of Beam's gradle-command-action plugin and github's provided
-        # maven settings.xml file
+        # This step is a workaround to avoid a decryption issue of Beam's
+        # net.linguica.gradle.maven.settings plugin and github's provided maven
+        # settings.xml file
         if [ -f ~/.m2/settings.xml ]; then
           rm ~/.m2/settings.xml
         fi

.github/workflows/java_tests.yml

@@ -82,16 +82,12 @@ jobs:
         with:
           java-version: 8
           go-version: 1.21
-      - name: Remove default github maven configuration
-        # This step is a workaround to avoid a decryption issue of Beam's
-        # net.linguica.gradle.maven.settings plugin and github's provided maven
-        # settings.xml file
-        run: rm ~/.m2/settings.xml
         # :sdks:java:core:test
       - name: Run :sdks:java:core:test
-        uses: ./.github/actions/gradle-command-action
+        uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          arguments: -p sdks/java/core/ test
+          gradle-command: test
+          arguments: -p sdks/java/core/
       - name: Upload test logs for :sdks:java:core:test
         uses: actions/upload-artifact@v3
         if: always()
@@ -100,9 +96,10 @@ jobs:
           path: sdks/java/core/build/reports/tests/test
       # :sdks:java:harness:test
       - name: Run :sdks:java:harness:test
-        uses: ./.github/actions/gradle-command-action
+        uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          arguments: -p sdks/java/harness/ test
+          gradle-command: test
+          arguments: -p sdks/java/harness/
         if: always()
       - name: Upload test logs for :sdks:java:harness:test
         uses: actions/upload-artifact@v3
@@ -112,9 +109,10 @@ jobs:
           path: sdks/java/harness/build/reports/tests/test
       # :runners:core-java:test
       - name: Run :runners:core-java:test
-        uses: ./.github/actions/gradle-command-action
+        uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          arguments: -p runners/core-java/ test
+          gradle-command: test
+          arguments: -p runners/core-java/
         if: always()
       - name: Upload test logs for :runners:core-java:test
         uses: actions/upload-artifact@v3
@@ -141,17 +139,13 @@ jobs:
         with:
           java-version: 8
           go-version: 1.21
-      - name: Remove default github maven configuration
-        # This step is a workaround to avoid a decryption issue of Beam's
-        # net.linguica.gradle.maven.settings plugin and github's provided maven
-        # settings.xml file
-        run: rm ~/.m2/settings.xml
       - name: Run WordCount Unix
-        uses: ./.github/actions/gradle-command-action
+        uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          arguments: -p examples/ integrationTest --tests org.apache.beam.examples.WordCountIT
+          gradle-command: integrationTest
+          arguments: -p examples/ --tests org.apache.beam.examples.WordCountIT
             -DintegrationTestRunner=direct
-            -DintegrationTestPipelineOptions=["--runner=DirectRunner","--tempRoot=./tmp"]
+            -DintegrationTestPipelineOptions=[\"--runner=DirectRunner\",\"--tempRoot=./tmp\"]
       - name: Upload test logs
         uses: actions/upload-artifact@v3
         if: always()
@@ -191,16 +185,12 @@ jobs:
           service_account_key: ${{ secrets.GCP_SA_KEY }}
           project_id: ${{ secrets.GCP_PROJECT_ID }}
           export_default_credentials: true
-      - name: Remove default github maven configuration
-        # This step is a workaround to avoid a decryption issue of Beam's
-        # gradle-command-action plugin and github's provided maven
-        # settings.xml file
-        run: rm ~/.m2/settings.xml
       - name: Run WordCount
-        uses: ./.github/actions/gradle-command-action
+        uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          arguments: -p examples/ integrationTest --tests org.apache.beam.examples.WordCountIT
-            -DintegrationTestPipelineOptions=["--runner=DataflowRunner","--project=${{ secrets.GCP_PROJECT_ID }}","--tempRoot=gs://${{ secrets.GCP_TESTING_BUCKET }}/tmp/"]
+          gradle-command: integrationTest
+          arguments: -p examples/ --tests org.apache.beam.examples.WordCountIT
+            -DintegrationTestPipelineOptions=[\"--runner=DataflowRunner\",\"--project=${{ secrets.GCP_PROJECT_ID }}\",\"--tempRoot=gs://${{ secrets.GCP_TESTING_BUCKET }}/tmp/\"]
             -DintegrationTestRunner=dataflow
       - name: Upload test logs
         uses: actions/upload-artifact@v3

.gitmodules

@@ -7,6 +7,3 @@
 [submodule ".github/actions/github-push-action"]
 	path = .github/actions/github-push-action
 	url = https://github.com/ad-m/github-push-action
-[submodule ".github/actions/gradle-command-action"]
-	path = .github/actions/gradle-command-action
-	url = https://github.com/eskatos/gradle-command-action

CHANGES.md

@@ -87,6 +87,8 @@
 ## Bugfixes
 
 * Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
+* Fixed MLTransform when duplicated elements are dropped in the output PCollection.([#29600](https://github.com/apache/beam/issues/29600))
+
 
 ## Security Fixes
 * Fixed (CVE-YYYY-NNNN)[https://www.cve.org/CVERecord?id=CVE-YYYY-NNNN] (Java/Python/Go) ([#X](https://github.com/apache/beam/issues/X)).
@@ -135,6 +137,10 @@ as a workaround, a copy of "old" `CountingSource` class should be placed into a
 * Fixed [CVE-2023-39325](https://www.cve.org/CVERecord?id=CVE-2023-39325) (Java/Python/Go) ([#29118](https://github.com/apache/beam/issues/29118)).
 * Mitigated [CVE-2023-47248](https://nvd.nist.gov/vuln/detail/CVE-2023-47248)  (Python) [#29392](https://github.com/apache/beam/issues/29392).
 
+## Known issues
+
+* MLTransform drops the identical elements in the output PCollection. For any duplicate elements, a single element will be emitted downstream. ([#29600](https://github.com/apache/beam/issues/29600)).
+
 # [2.51.0] - 2023-10-03
 
 ## New Features / Improvements
@@ -169,6 +175,8 @@ as a workaround, a copy of "old" `CountingSource` class should be placed into a
 * Long-running Python pipelines might experience a memory leak: [#28246](https://github.com/apache/beam/issues/28246).
 * Python pipelines using BigQuery Storage Read API might need to pin `fastavro`
   dependency to 1.8.3 or earlier on some runners that don't use Beam Docker containers: [#28811](https://github.com/apache/beam/issues/28811)
+* MLTransform drops the identical elements in the output PCollection. For any duplicate elements, a single element will be emitted downstream. ([#29600](https://github.com/apache/beam/issues/29600)).
+
 
 # [2.50.0] - 2023-08-30
 
@@ -230,6 +238,7 @@ as a workaround, a copy of "old" `CountingSource` class should be placed into a
 * Beam Python containers rely on a version of Debian/aom that has several security vulnerabilities: [CVE-2021-30474](https://nvd.nist.gov/vuln/detail/CVE-2021-30474), [CVE-2021-30475](https://nvd.nist.gov/vuln/detail/CVE-2021-30475), [CVE-2021-30473](https://nvd.nist.gov/vuln/detail/CVE-2021-30473), [CVE-2020-36133](https://nvd.nist.gov/vuln/detail/CVE-2020-36133), [CVE-2020-36131](https://nvd.nist.gov/vuln/detail/CVE-2020-36131), [CVE-2020-36130](https://nvd.nist.gov/vuln/detail/CVE-2020-36130), and [CVE-2020-36135](https://nvd.nist.gov/vuln/detail/CVE-2020-36135)
 * Python SDK's cross-language Bigtable sink mishandles records that don't have an explicit timestamp set: [#28632](https://github.com/apache/beam/issues/28632). To avoid this issue, set explicit timestamps for all records before writing to Bigtable.
 * Python SDK worker start-up logs, particularly PIP dependency installations, that are not logged at warning or higher are suppressed. This suppression is reverted in 2.51.0.
+* MLTransform drops the identical elements in the output PCollection. For any duplicate elements, a single element will be emitted downstream. ([#29600](https://github.com/apache/beam/issues/29600)).
 
 # [2.49.0] - 2023-07-17
 

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -731,7 +731,7 @@ class BeamModulePlugin implements Plugin<Project> {
         // Keep version consistent with the version in google_cloud_resourcemanager, managed by google_cloud_platform_libraries_bom
         google_api_services_cloudresourcemanager    : "com.google.apis:google-api-services-cloudresourcemanager:v1-rev20230806-$google_clients_version",
         google_api_services_dataflow                : "com.google.apis:google-api-services-dataflow:v1b3-rev20220920-$google_clients_version",
-        google_api_services_healthcare              : "com.google.apis:google-api-services-healthcare:v1-rev20231003-$google_clients_version",
+        google_api_services_healthcare              : "com.google.apis:google-api-services-healthcare:v1-rev20231101-$google_clients_version",
         google_api_services_pubsub                  : "com.google.apis:google-api-services-pubsub:v1-rev20220904-$google_clients_version",
         // Keep version consistent with the version in google_cloud_nio, managed by google_cloud_platform_libraries_bom
         google_api_services_storage                 : "com.google.apis:google-api-services-storage:v1-rev20230907-$google_clients_version",

contributor-docs/rc-testing-guide.md

@@ -0,0 +1,60 @@
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Release Candidate (RC) Testing Guide
+
+This guide is meant for anybody who is interested in testing Beam Release Candidates against downstream projects. Note
+that one need not have any status on the Apache Beam project (eg. PMC Member, Committer) to vote; all are welcome.
+Please subscribe to the [dev list](https://lists.apache.org/list.html?dev@beam.apache.org), and vote on the RC Vote email thread.
+
+
+## RC Testing Objectives
+
+The RC testing process aims to:
+
+ - Test new release candidates against existing code bases utilizing Apache Beam, to ensure there are no unexpected behaviors downstream.
+ - Incorporate a breadth of perspectives (including validation on multiple SDKs and multiple runners), before releasing a new version.
+ - Allow Beam Contributors to dogfood their changes and verify that they work as intended.
+
+
+## Beam Release process overview
+- For a comprehensive overview on the Beam release process, please take a look at our [release guide](https://github.com/apache/beam/blob/master/contributor-docs/release-guide.md).
+- Note that release candidate votes will be open for 72 hours after the voting email is sent.
+
+
+## Ideas for Python SDK Validators
+
+_Note: Do the following in a dev-like environment._
+- If you are a Python SDK user that utilizes notebooks (eg. Jupyter Notebooks, or Colab Notebooks), change `pip install`
+to point to the new RC (e.g. `pip install apache_beam[gcp]==2.52.0rc1`). Re-execute the workflow to ensure everything
+works as intended.
+- If your workflow utilizes [Dataflow Templates](https://github.com/GoogleCloudPlatform/DataflowTemplates), or another way of launching your job, modify your `requirements.txt` file, `setup.py` file, or `DockerFile` to point to the new Beam RC.
+- _Tip_: Run your pipeline both against Direct Runner, and another runner of your choice by modifying your job's `PipelineOptions`.
+
+
+## Ideas for Java SDK Validators
+_Note: Do the following in a dev-like environment._
+- If you are a Java SDK user that utilizes Maven or Gradle in your workflow, in your `pom.xml` file, modify the `beam.version` to reflect the newest RC, and modify the `<url>` tag under `<repository>`, to point to the repository given in the vote email. [Example Changes](https://github.com/GoogleCloudPlatform/DataflowTemplates/pull/1090/files).
+
+
+## Ideas for Go SDK Validators
+_Note: Do the following in a dev-like environment._
+- If you utilize the Go SDK, use `go-get` to use the desired RC, for example, `go get -d github.com/apache/beam/sdks/v2@v2.xx.0-RC1`
+- Utilize the `--environment_config`, to point to the new release, for example, `--environment_config=apache/beam_go_sdk:2.xx.0rc1`
+
+
+## After validation
+
+- Reply to [dev list](https://lists.apache.org/list.html?dev@beam.apache.org) vote thread with your [vote](https://www.apache.org/foundation/voting.html) and an explanation of the use case you tested.
+- [Optional]: If your use case can be well represented by a test, consider contributing a test to Beam! However, note that the of the value of validation is manual testing outside of the Beam CI workflow, so it is still recommended (and highly encouraged!) to validate these test cases in future releases.

contributor-docs/release-guide.md

@@ -817,7 +817,7 @@ template; please adjust as you see fit.
     Reviewers are encouraged to test their own use cases with the release candidate, and vote +1 if
     no issues are found. Only PMC member votes will count towards the final vote, but votes from all
     community members is encouraged and helpful for finding regressions; you can either test your own
-    use cases or use cases from the validation sheet [10].
+    use cases [13] or use cases from the validation sheet [10].
 
     The complete staging area is available for your review, which includes:
     * GitHub Release notes [1],
@@ -833,7 +833,7 @@ template; please adjust as you see fit.
 
     The vote will be open for at least 72 hours. It is adopted by majority approval, with at least 3 PMC affirmative votes.
 
-    For guidelines on how to try the release in your projects, check out our blog post at https://beam.apache.org/blog/validate-beam-release/.
+    For guidelines on how to try the release in your projects, check out our RC testing guide [13].
 
     Thanks,
     Release Manager
@@ -850,6 +850,7 @@ template; please adjust as you see fit.
     [10] https://docs.google.com/spreadsheets/d/1qk-N5vjXvbcEk68GjbkSZTR8AGqyNUM-oLFo_ZXBpJw/edit#gid=...
     [11] https://hub.docker.com/search?q=apache%2Fbeam&type=image
     [12] https://github.com/apache/beam/pull/...
+    [13] https://github.com/apache/beam/blob/master/contributor-docs/rc-testing-guide.md
 
 If there are any issues found in the release candidate, reply on the vote
 thread to cancel the vote.  There’s no need to wait 72 hours. Go back to
@@ -860,7 +861,8 @@ pull request, just correct it on the spot and the vote can continue as-is.
 ### Run validation tests
 
 The community is responsible for performing validation, but as release manager
-you are expected to contribute as well.
+you are expected to contribute as well. Please see the [RC Testing Guide](https://github.com/apache/beam/blob/master/contributor-docs/rc-testing-guide.md)
+for ideas on helping validate testing on downstream projects.
 
 Before accepting an RC, as a community we try to exercise most (if not all) of
 the tests listed in this

sdks/go.mod

@@ -28,9 +28,9 @@ require (
 	cloud.google.com/go/datastore v1.15.0
 	cloud.google.com/go/profiler v0.4.0
 	cloud.google.com/go/pubsub v1.33.0
-	cloud.google.com/go/spanner v1.51.0
+	cloud.google.com/go/spanner v1.53.1
 	cloud.google.com/go/storage v1.35.1
-	github.com/aws/aws-sdk-go-v2 v1.23.4
+	github.com/aws/aws-sdk-go-v2 v1.23.5
 	github.com/aws/aws-sdk-go-v2/config v1.25.8
 	github.com/aws/aws-sdk-go-v2/credentials v1.16.8
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.8
@@ -54,13 +54,13 @@ require (
 	github.com/xitongsys/parquet-go v1.6.2
 	github.com/xitongsys/parquet-go-source v0.0.0-20220315005136-aec0fe3e777c
 	go.mongodb.org/mongo-driver v1.12.1
-	golang.org/x/net v0.18.0
-	golang.org/x/oauth2 v0.13.0
+	golang.org/x/net v0.19.0
+	golang.org/x/oauth2 v0.14.0
 	golang.org/x/sync v0.5.0
-	golang.org/x/sys v0.14.0
+	golang.org/x/sys v0.15.0
 	golang.org/x/text v0.14.0
-	google.golang.org/api v0.151.0
-	google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b
+	google.golang.org/api v0.152.0
+	google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17
 	google.golang.org/grpc v1.59.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/retry.v1 v1.0.3
@@ -89,15 +89,15 @@ require (
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
-	golang.org/x/time v0.4.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
 )
 
 require (
-	cloud.google.com/go v0.110.8 // indirect
-	cloud.google.com/go/compute v1.23.1 // indirect
+	cloud.google.com/go v0.110.10 // indirect
+	cloud.google.com/go/compute v1.23.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v1.1.3 // indirect
-	cloud.google.com/go/longrunning v0.5.2 // indirect
+	cloud.google.com/go/iam v1.1.5 // indirect
+	cloud.google.com/go/longrunning v0.5.4 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
@@ -173,11 +173,11 @@ require (
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/crypto v0.15.0 // indirect
+	golang.org/x/crypto v0.16.0 // indirect
 	golang.org/x/mod v0.11.0 // indirect
 	golang.org/x/tools v0.10.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
 )