diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml index a84485c717fb..c03e2435a83b 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml @@ -80,24 +80,13 @@ jobs: java-version: | ${{ matrix.java_version }} 11 - - name: run jar Java${{ matrix.java_version }} script - run: | - ./gradlew runners:google-cloud-dataflow-java:testJar :runners:google-cloud-dataflow-java:worker:shadowJar \ - -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 - name: run validatesRunner Java${{ matrix.java_version }} script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:google-cloud-dataflow-java:validatesRunner arguments: | - -x shadowJar \ - -x shadowTestJar \ - -x compileJava \ - -x compileTestJava \ - -x jar \ - -x testJar \ - -x classes \ - -x testClasses \ - -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PtestJavaVersion=${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ max-workers: 12 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml index 11b18548249e..365b50e9e350 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml @@ -80,20 +80,13 @@ jobs: java-version: | ${{ matrix.java_version }} 11 - - name: run jar Java${{ matrix.java_version }} script - run: | - ./gradlew :runners:direct-java:shadowJar :runners:direct-java:shadowTestJar \ - -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 - name: run validatesRunner Java${{ matrix.java_version }} script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:direct-java:validatesRunner arguments: | - -x shadowJar \ - -x shadowTestJar \ - -x compileJava \ - -x compileTestJava \ - -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ + -PtestJavaVersion=${{ matrix.java_version }} \ + -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml index 7f7b9c9270d2..9b061028cbce 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml @@ -78,23 +78,13 @@ jobs: java-version: | 8 11 - - name: run jar Java8 script - run: | - ./gradlew :runners:flink:1.19:jar :runners:flink:1.19:testJar - name: run validatesRunner Java8 script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:flink:1.19:validatesRunner arguments: | - -x shadowJar \ - -x shadowTestJar \ - -x compileJava \ - -x compileTestJava \ - -x jar \ - -x testJar \ - -x classes \ - -x testClasses \ - -Dorg.gradle.java.home=$JAVA_HOME_8_X64 \ + -PtestJavaVersion=8 \ + -Pjava8Home=$JAVA_HOME_8_X64 \ max-workers: 12 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml index 0852fcdf4afa..dae408e4346f 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml @@ -78,23 +78,13 @@ jobs: java-version: | 8 11 - - name: run jar Java8 script - run: | - ./gradlew :runners:spark:3:jar :runners:spark:3:testJar - name: run validatesRunner Java8 script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:spark:3:validatesRunner arguments: | - -x shadowJar \ - -x shadowTestJar \ - -x compileJava \ - -x compileTestJava \ - -x jar \ - -x testJar \ - -x classes \ - -x testClasses \ - -Dorg.gradle.java.home=$JAVA_HOME_8_X64 \ + -PtestJavaVersion=8 \ + -Pjava8Home=$JAVA_HOME_8_X64 \ max-workers: 12 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index d053713cab91..9cb318c7f27f 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -92,9 +92,5 @@ gradlePlugin { id = "org.apache.beam.vendor-java" implementationClass = "org.apache.beam.gradle.VendorJavaPlugin" } - create("beamJenkins") { - id = "org.apache.beam.jenkins" - implementationClass = "org.apache.beam.gradle.BeamJenkinsPlugin" - } } } diff --git a/sdks/go/test/build.gradle b/sdks/go/test/build.gradle index 74b6a10cad4d..5576c40c0aab 100644 --- a/sdks/go/test/build.gradle +++ b/sdks/go/test/build.gradle @@ -41,7 +41,7 @@ task dataflowValidatesRunner() { ] exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" @@ -66,7 +66,7 @@ task dataflowValidatesRunnerARM64() { ] exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" @@ -95,7 +95,7 @@ task flinkValidatesRunner { ] exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" @@ -123,7 +123,7 @@ task samzaValidatesRunner { ] exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" @@ -151,7 +151,7 @@ task sparkValidatesRunner { ] exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" @@ -190,7 +190,7 @@ tasks.register("ulrValidatesRunner") { } exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", ". ${envdir}/bin/activate && ./run_validatesrunner_tests.sh ${options.join(' ')}" @@ -217,7 +217,7 @@ task prismValidatesRunner { ] exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" @@ -262,7 +262,7 @@ ext.goIoValidatesRunnerTask = { proj, name, scriptOpts, pipelineOpts -> logger.info("Running the command: sh -c ./run_validatesrunner_tests.sh ${options.join(' ')}") exec { if (fork_java_home) { - environment "JAVA_HOME", fork_java_home + environment "JAVA_HOME_JOB_SERVER", fork_java_home } executable "sh" args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}" diff --git a/sdks/go/test/run_validatesrunner_tests.sh b/sdks/go/test/run_validatesrunner_tests.sh index 4278c817d0a4..2e1b5fa3f396 100755 --- a/sdks/go/test/run_validatesrunner_tests.sh +++ b/sdks/go/test/run_validatesrunner_tests.sh @@ -259,8 +259,8 @@ s.close() TMPDIR=$(mktemp -d) -if [[ -n "$JAVA_HOME" ]]; then - JAVA_CMD="$JAVA_HOME/bin/java" +if [[ -n "$JAVA_HOME_JOB_SERVER" ]]; then + JAVA_CMD="$JAVA_HOME_JOB_SERVER/bin/java" else JAVA_CMD="java" fi @@ -374,18 +374,7 @@ if [[ "$RUNNER" == "dataflow" ]]; then CONTAINER=us.gcr.io/$PROJECT/$USER/beam_go_sdk echo "Using container $CONTAINER" - # TODO(https://github.com/apache/beam/issues/27674): remove this branch once the jenkins VM can build multiarch, or jenkins is deprecated. - if [[ "$USER" == "jenkins" ]]; then - ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG - - # Verify it exists - docker images | grep $TAG - - # Push the container - gcloud docker -- push $CONTAINER:$TAG - else - ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Pcontainer-architecture-list=arm64,amd64 -Ppush-containers - fi + ./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Pcontainer-architecture-list=arm64,amd64 -Ppush-containers if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then ARGS="$ARGS --experiments=use_portable_job_submission" @@ -395,7 +384,7 @@ if [[ "$RUNNER" == "dataflow" ]]; then JAVA_TAG=$(date +%Y%m%d-%H%M%S) JAVA_CONTAINER=us.gcr.io/$PROJECT/$USER/beam_java11_sdk echo "Using container $JAVA_CONTAINER for cross-language java transforms" - ./gradlew :sdks:java:container:java11:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$JAVA_TAG -Pjava11Home=$JAVA11_HOME + ./gradlew :sdks:java:container:java11:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$JAVA_TAG # Verify it exists docker images | grep $JAVA_TAG @@ -457,9 +446,6 @@ if [[ "$RUNNER" == "dataflow" ]]; then # Note: we don't delete the multi-arch containers here because this command only deletes the manifest list with the tag, # the associated container images can't be deleted because they are not tagged. However, multi-arch containers that are # older than 6 weeks old are deleted by stale_dataflow_prebuilt_image_cleaner.sh that runs daily. - if [[ "$USER" == "jenkins" ]]; then - gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container" - fi if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then # Delete the java cross-language container locally and remotely docker rmi $JAVA_CONTAINER:$JAVA_TAG || echo "Failed to remove container"