[WIP][SPARK-42240][INFRA][CONNECT][TESTS] Move ClientE2ETestSuite into a separate module and add new GA task to test shaded jvm client with maven

.github/workflows/build_and_test.yml

@@ -101,6 +101,7 @@ jobs:
               \"java-11-17\": \"true\",
               \"lint\" : \"true\",
               \"k8s-integration-tests\" : \"true\",
+              \"connect-jvm-e2e-tests\" : \"true\",
             }"
           echo $precondition # For debugging
           # Remove `\n` to avoid "Invalid format" error
@@ -689,6 +690,71 @@ jobs:
         ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=${JAVA_VERSION/-ea} install
         rm -rf ~/.m2/repository/org/apache/spark
 
+  connect-jvm-e2e-tests:
+    needs: precondition
+    if: fromJson(needs.precondition.outputs.required).connect-jvm-e2e-tests == 'true' &&
+      (inputs.branch != 'branch-3.2' && inputs.branch != 'branch-3.3')
+    name: Connect JVM Client E2E Tests with Maven
+    strategy:
+      fail-fast: false
+      matrix:
+        java:
+          - ${{ inputs.java }}
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout Spark repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          repository: apache/spark
+          ref: ${{ inputs.branch }}
+      - name: Sync the current branch with the latest in Apache Spark
+        if: github.repository != 'apache/spark'
+        run: |
+          git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/}
+          git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
+          git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty
+      - name: Cache Scala, SBT and Maven
+        uses: actions/cache@v3
+        with:
+          path: |
+            build/apache-maven-*
+            build/scala-*
+            build/*.jar
+            ~/.sbt
+          key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+          restore-keys: |
+            build-
+      - name: Cache Maven local repository
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2/repository
+          key: java${{ matrix.java }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            java${{ matrix.java }}-maven-
+      - name: Install Java ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.java }}
+      - name: Build and Test with Maven
+        run: |
+          export SCALA_VERSION=2.12
+          if [[ "SCALA_PROFILE" == "scala2.13" ]] ; then 
+             export SCALA_VERSION=2.13; 
+             ./dev/change-scala-version.sh 2.13
+          fi
+          export MAVEN_OPTS="-Xss64m -Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
+          export MAVEN_CLI_OPTS="--no-transfer-progress"
+          export JAVA_VERSION=${{ matrix.java }}
+          # It uses Maven's 'install' intentionally, see https://github.com/apache/spark/pull/26414.
+          ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl repl -am -Pscala-$SCALA_VERSION
+          ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/common -Pscala-$SCALA_VERSION
+          ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/server -Pscala-$SCALA_VERSION
+          ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/client/jvm -Pscala-$SCALA_VERSION
+          ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} clean test -pl connector/connect/client/jvm-e2e-tests -Pscala-$SCALA_VERSION
+          rm -rf ~/.m2/repository/org/apache/spark
+
   scala-213:
     needs: precondition
     if: fromJson(needs.precondition.outputs.required).scala-213 == 'true'

connector/connect/client/jvm-e2e-tests/pom.xml

@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.spark</groupId>
+        <artifactId>spark-parent_2.12</artifactId>
+        <version>3.5.0-SNAPSHOT</version>
+        <relativePath>../../../../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>spark-connect-client-jvm-e2e-tests_2.12</artifactId>
+    <packaging>jar</packaging>
+    <name>Spark Project Connect Client E2E Tests</name>
+    <url>https://spark.apache.org/</url>
+    <properties>
+        <sbt.project.name>connect-client-jvm-e2e-tests</sbt.project.name>
+        <guava.version>31.0.1-jre</guava.version>
+        <mima.version>1.1.0</mima.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-connect-client-jvm_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-connect-common_${scala.binary.version}</artifactId>
+            <scope>test</scope>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-compiler</artifactId>
+            <version>${scala.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <!-- Use mima to perform the compatibility check -->
+        <dependency>
+            <groupId>com.typesafe</groupId>
+            <artifactId>mima-core_${scala.binary.version}</artifactId>
+            <version>${mima.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+    <build>
+        <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+    </build>
+</project>

connector/connect/client/jvm-e2e-tests/src/test/resources/log4j2.properties

@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file target/unit-tests.log
+rootLogger.level = info
+rootLogger.appenderRef.file.ref = ${sys:test.appender:-File}
+
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex
+
+# Tests that launch java subprocesses can set the "test.appender" system property to
+# "console" to avoid having the child process's logs overwrite the unit test's
+# log file.
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %t: %m%n%ex
+
+# Ignore messages below warning level from Jetty, because it's a bit verbose
+logger.jetty.name = org.sparkproject.jetty
+logger.jetty.level = warn

connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala → connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala

@@ -49,7 +49,7 @@ class ClientE2ETestSuite extends RemoteSparkSession {
     assert(array(2).getLong(0) == 2)
   }
 
-  test("simple udf test") {
+  ignore("simple udf test") {
 
     def dummyUdf(x: Int): Int = x + 5
     val myUdf = udf(dummyUdf _)

connector/connect/client/jvm/pom.xml

@@ -33,7 +33,6 @@
   <properties>
     <sbt.project.name>connect-client-jvm</sbt.project.name>
     <guava.version>31.0.1-jre</guava.version>
-    <mima.version>1.1.0</mima.version>
   </properties>
 
   <dependencies>
@@ -93,13 +92,6 @@
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
-    <!-- Use mima to perform the compatibility check -->
-    <dependency>
-      <groupId>com.typesafe</groupId>
-      <artifactId>mima-core_${scala.binary.version}</artifactId>
-      <version>${mima.version}</version>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
   <build>
     <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>

dev/sparktestsupport/modules.py

@@ -280,6 +280,7 @@ def __hash__(self):
     sbt_test_goals=[
         "connect/test",
         "connect-client-jvm/test",
+        "connect-client-jvm-e2e-tests/test",
     ],
 )
 

pom.xml

@@ -103,6 +103,7 @@
     <module>connector/connect/server</module>
     <module>connector/connect/common</module>
     <module>connector/connect/client/jvm</module>
+    <module>connector/connect/client/jvm-e2e-tests</module>
     <module>connector/protobuf</module>
     <!-- See additional modules enabled by profiles below -->
   </modules>

project/SparkBuild.scala

@@ -55,13 +55,14 @@ object BuildCommons {
   val connectCommon = ProjectRef(buildLocation, "connect-common")
   val connect = ProjectRef(buildLocation, "connect")
   val connectClient = ProjectRef(buildLocation, "connect-client-jvm")
+  val connectClientE2ETests = ProjectRef(buildLocation, "connect-client-jvm-e2e-tests")
 
   val allProjects@Seq(
     core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle, launcher, unsafe, tags, sketch, kvstore, _*
   ) = Seq(
     "core", "graphx", "mllib", "mllib-local", "repl", "network-common", "network-shuffle", "launcher", "unsafe",
     "tags", "sketch", "kvstore"
-  ).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects ++ Seq(connectCommon, connect, connectClient)
+  ).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects ++ Seq(connectCommon, connect, connectClient, connectClientE2ETests)
 
   val optionallyEnabledProjects@Seq(kubernetes, mesos, yarn,
     sparkGangliaLgpl, streamingKinesisAsl,
@@ -403,7 +404,8 @@ object SparkBuild extends PomBuild {
   val mimaProjects = allProjects.filterNot { x =>
     Seq(
       spark, hive, hiveThriftServer, repl, networkCommon, networkShuffle, networkYarn,
-      unsafe, tags, tokenProviderKafka010, sqlKafka010, connectCommon, connect, connectClient, protobuf
+      unsafe, tags, tokenProviderKafka010, sqlKafka010, connectCommon, connect, connectClient,
+      connectClientE2ETests, protobuf
     ).contains(x)
   }
 
@@ -447,6 +449,7 @@ object SparkBuild extends PomBuild {
   enable(SparkConnectCommon.settings)(connectCommon)
   enable(SparkConnect.settings)(connect)
   enable(SparkConnectClient.settings)(connectClient)
+  enable(SparkConnectClientE2ETests.settings)(connectClientE2ETests)
 
   /* Protobuf settings */
   enable(SparkProtobuf.settings)(protobuf)
@@ -851,9 +854,6 @@ object SparkConnectClient {
       )
     },
 
-    // Make sure the connect server assembly jar is available for testing.
-    test := ((Test / test) dependsOn (LocalProject("connect") / assembly)).value,
-
     (assembly / test) := { },
 
     (assembly / logLevel) := Level.Info,
@@ -895,6 +895,32 @@ object SparkConnectClient {
   }
 }
 
+object SparkConnectClientE2ETests {
+  val buildTestDeps = TaskKey[Unit]("buildTestDeps", "Build needed dependencies for test.")
+  lazy val settings = Seq(
+    buildTestDeps := {
+      (LocalProject("sql") / Compile / Keys.`package`).value
+      (LocalProject("connect") / assembly).value
+      (LocalProject("connect-client-jvm") / assembly).value
+    },
+    test := ((Test / test) dependsOn (buildTestDeps)).value,
+    // Make sure the connect server assembly jar is available for testing.
+    test := ((Test / test) dependsOn (LocalProject("connect") / assembly)).value,
+    libraryDependencies ++= {
+      val guavaVersion =
+        SbtPomKeys.effectivePom.value.getProperties.get("guava.version").asInstanceOf[String]
+      Seq("com.google.guava" % "guava" % guavaVersion)
+    },
+    dependencyOverrides ++= {
+      val guavaVersion =
+        SbtPomKeys.effectivePom.value.getProperties.get("guava.version").asInstanceOf[String]
+      Seq(
+        "com.google.guava" % "guava" % guavaVersion
+      )
+    }
+  )
+}
+
 object SparkProtobuf {
   import BuildCommons.protoVersion