diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 8a85d26c0eca..4b4e34ecc014 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -80,12 +80,14 @@ jobs: pyspark=`./dev/is-changed.py -m $pyspark_modules` if [[ "${{ github.repository }}" != 'apache/spark' ]]; then pandas=$pyspark + yarn=`./dev/is-changed.py -m yarn` kubernetes=`./dev/is-changed.py -m kubernetes` sparkr=`./dev/is-changed.py -m sparkr` buf=true ui=true else pandas=false + yarn=false kubernetes=false sparkr=false buf=false @@ -102,6 +104,7 @@ jobs: \"tpcds-1g\": \"false\", \"docker-integration-tests\": \"false\", \"lint\" : \"true\", + \"yarn\" : \"$yarn\", \"k8s-integration-tests\" : \"$kubernetes\", \"buf\" : \"$buf\", \"ui\" : \"$ui\", @@ -155,8 +158,8 @@ jobs: - >- streaming, sql-kafka-0-10, streaming-kafka-0-10, streaming-kinesis-asl, kubernetes, hadoop-cloud, spark-ganglia-lgpl, protobuf - - >- - yarn, connect + - yarn + - connect # Here, we split Hive and SQL tests into some of slow ones and the rest of them. included-tags: [""] excluded-tags: [""] @@ -194,6 +197,11 @@ jobs: hive: hive2.3 excluded-tags: org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowSQLTest comment: "- other tests" + exclude: + # Always run if yarn == 'true', even infra-image is skip (such as non-master job) + # In practice, the build will run in individual PR, but not against the individual commit + # in Apache Spark repository. + - modules: ${{ fromJson(needs.precondition.outputs.required).yarn != 'true' && 'yarn' }} env: MODULES_TO_TEST: ${{ matrix.modules }} EXCLUDED_TAGS: ${{ matrix.excluded-tags }} diff --git a/.github/workflows/build_java21.yml b/.github/workflows/build_java21.yml index a2fb0e6e2c1d..871e1a9c07ef 100644 --- a/.github/workflows/build_java21.yml +++ b/.github/workflows/build_java21.yml @@ -47,6 +47,7 @@ jobs: "sparkr": "true", "tpcds-1g": "true", "docker-integration-tests": "true", + "yarn": "true", "k8s-integration-tests": "true", "buf": "true", "ui": "true" diff --git a/.github/workflows/build_non_ansi.yml b/.github/workflows/build_non_ansi.yml index ff3fda4625cc..902627690320 100644 --- a/.github/workflows/build_non_ansi.yml +++ b/.github/workflows/build_non_ansi.yml @@ -44,5 +44,6 @@ jobs: "pyspark": "true", "sparkr": "true", "tpcds-1g": "true", - "docker-integration-tests": "true" + "docker-integration-tests": "true", + "yarn": "true" } diff --git a/.github/workflows/build_rockdb_as_ui_backend.yml b/.github/workflows/build_rockdb_as_ui_backend.yml index a1cc34f7b54f..96009c41dbbf 100644 --- a/.github/workflows/build_rockdb_as_ui_backend.yml +++ b/.github/workflows/build_rockdb_as_ui_backend.yml @@ -42,5 +42,6 @@ jobs: { "build": "true", "pyspark": "true", - "sparkr": "true" + "sparkr": "true", + "yarn": "true" }