From d1712fc7191b88c92075883af4a14dcba9917d54 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 29 Jan 2023 12:32:04 +0800 Subject: [PATCH 01/31] first commit --- .../connect/client/integration-tests/pom.xml | 107 ++++++++++++++++++ .../src/test/resources/log4j2.properties | 39 +++++++ .../apache/spark/sql/ClientE2ETestSuite.scala | 0 .../client/util/RemoteSparkSession.scala | 2 +- connector/connect/client/jvm/pom.xml | 27 +---- pom.xml | 1 + project/SparkBuild.scala | 32 +++++- 7 files changed, 183 insertions(+), 25 deletions(-) create mode 100644 connector/connect/client/integration-tests/pom.xml create mode 100644 connector/connect/client/integration-tests/src/test/resources/log4j2.properties rename connector/connect/client/{jvm => integration-tests}/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala (100%) rename connector/connect/client/{jvm => integration-tests}/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala (99%) diff --git a/connector/connect/client/integration-tests/pom.xml b/connector/connect/client/integration-tests/pom.xml new file mode 100644 index 0000000000000..44f1c093ead6b --- /dev/null +++ b/connector/connect/client/integration-tests/pom.xml @@ -0,0 +1,107 @@ + + + + + 4.0.0 + + org.apache.spark + spark-parent_2.12 + 3.5.0-SNAPSHOT + ../../../../pom.xml + + + spark-connect-client-integration-tests_2.12 + jar + Spark Project Connect Client Integration Tests + https://spark.apache.org/ + + connect-client-integration-tests + 31.0.1-jre + 3.21.12 + + + + + + + org.apache.spark + spark-connect-client-jvm_${scala.binary.version} + ${project.version} + test + + + org.apache.spark + spark-connect-client-jvm_${scala.binary.version} + ${project.version} + test-jar + test + + + org.apache.spark + spark-catalyst_${scala.binary.version} + test + ${project.version} + + + org.apache.spark + spark-repl_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-sql_${scala.binary.version} + + + + + org.apache.spark + spark-connect-common_${scala.binary.version} + test + ${project.version} + + + org.apache.spark + spark-connect_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-sql_${scala.binary.version} + + + + + + target/scala-${scala.binary.version}/test-classes + + \ No newline at end of file diff --git a/connector/connect/client/integration-tests/src/test/resources/log4j2.properties b/connector/connect/client/integration-tests/src/test/resources/log4j2.properties new file mode 100644 index 0000000000000..ab02104c69697 --- /dev/null +++ b/connector/connect/client/integration-tests/src/test/resources/log4j2.properties @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.file.ref = ${sys:test.appender:-File} + +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex + +# Tests that launch java subprocesses can set the "test.appender" system property to +# "console" to avoid having the child process's logs overwrite the unit test's +# log file. +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %t: %m%n%ex + +# Ignore messages below warning level from Jetty, because it's a bit verbose +logger.jetty.name = org.sparkproject.jetty +logger.jetty.level = warn diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala b/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala similarity index 100% rename from connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala rename to connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala b/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala similarity index 99% rename from connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala rename to connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala index 79a7a30f1428f..e02556a60174e 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala +++ b/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala @@ -54,7 +54,7 @@ object SparkConnectServerUtils { } sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME")) } - private val isDebug = System.getProperty(DEBUG_SC_JVM_CLIENT, "false").toBoolean + private val isDebug = System.getProperty(DEBUG_SC_JVM_CLIENT, "true").toBoolean // Log server start stop debug info into console // scalastyle:off println diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index 44c653462fa28..b27eec5003820 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -65,28 +65,6 @@ ${guava.version} compile - - - org.apache.spark - spark-repl_${scala.binary.version} - ${project.version} - test - - - - org.apache.spark - spark-sql_${scala.binary.version} - - - - - org.scalacheck - scalacheck_${scala.binary.version} - test - org.mockito mockito-core @@ -105,11 +83,16 @@ com.google.guava:* + io.grpc:*: com.google.protobuf:* org.apache.spark:spark-connect-common_${scala.binary.version} + + io.grpc + ${spark.shade.packageName}.connect.grpc + com.google.protobuf ${spark.shade.packageName}.connect.protobuf diff --git a/pom.xml b/pom.xml index 5428aed8ad3f8..494a812b5b67d 100644 --- a/pom.xml +++ b/pom.xml @@ -103,6 +103,7 @@ connector/connect/server connector/connect/common connector/connect/client/jvm + connector/connect/client/integration-tests connector/protobuf diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 9f478d649bafe..750ce49b79098 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -55,13 +55,14 @@ object BuildCommons { val connectCommon = ProjectRef(buildLocation, "connect-common") val connect = ProjectRef(buildLocation, "connect") val connectClient = ProjectRef(buildLocation, "connect-client-jvm") + val connectClientTests = ProjectRef(buildLocation, "connect-client-integration-tests") val allProjects@Seq( core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle, launcher, unsafe, tags, sketch, kvstore, _* ) = Seq( "core", "graphx", "mllib", "mllib-local", "repl", "network-common", "network-shuffle", "launcher", "unsafe", "tags", "sketch", "kvstore" - ).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects ++ Seq(connectCommon, connect, connectClient) + ).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects ++ Seq(connectCommon, connect, connectClient, connectClientTests) val optionallyEnabledProjects@Seq(kubernetes, mesos, yarn, sparkGangliaLgpl, streamingKinesisAsl, @@ -403,7 +404,8 @@ object SparkBuild extends PomBuild { val mimaProjects = allProjects.filterNot { x => Seq( spark, hive, hiveThriftServer, repl, networkCommon, networkShuffle, networkYarn, - unsafe, tags, tokenProviderKafka010, sqlKafka010, connectCommon, connect, connectClient, protobuf + unsafe, tags, tokenProviderKafka010, sqlKafka010, connectCommon, connect, connectClient, + connectClientTests, protobuf ).contains(x) } @@ -447,6 +449,7 @@ object SparkBuild extends PomBuild { enable(SparkConnectCommon.settings)(connectCommon) enable(SparkConnect.settings)(connect) enable(SparkConnectClient.settings)(connectClient) + enable(SparkConnectClientTests.settings)(connectClientTests) /* Protobuf settings */ enable(SparkProtobuf.settings)(protobuf) @@ -870,6 +873,7 @@ object SparkConnectClient { }, (assembly / assemblyShadeRules) := Seq( + ShadeRule.rename("io.grpc.**" -> "org.sparkproject.connect.grpc.@0").inAll, ShadeRule.rename("com.google.protobuf.**" -> "org.sparkproject.connect.protobuf.@1").inAll, ShadeRule.rename("com.google.common.**" -> "org.sparkproject.connect.client.guava.@1").inAll, ShadeRule.rename("com.google.thirdparty.**" -> "org.sparkproject.connect.client.guava.@1").inAll, @@ -891,6 +895,30 @@ object SparkConnectClient { } } +object SparkConnectClientTests { + lazy val settings = Seq( + excludeDependencies ++= { + val scalaBinaryVersion = + SbtPomKeys.effectivePom.value.getProperties.get("scala.binary.version").asInstanceOf[String] + Seq( + ExclusionRule("org.apache.spark", s"spark-sql_${scalaBinaryVersion}") + ) + }, + libraryDependencies ++= { + val guavaVersion = + SbtPomKeys.effectivePom.value.getProperties.get("guava.version").asInstanceOf[String] + Seq("com.google.guava" % "guava" % guavaVersion) + }, + dependencyOverrides ++= { + val guavaVersion = + SbtPomKeys.effectivePom.value.getProperties.get("guava.version").asInstanceOf[String] + Seq( + "com.google.guava" % "guava" % guavaVersion + ) + } + ) +} + object SparkProtobuf { import BuildCommons.protoVersion From 22d6c582c3716d3506f6084accf29097515c84fe Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 29 Jan 2023 12:34:59 +0800 Subject: [PATCH 02/31] add connect-client-integration-tests to modules.py --- dev/sparktestsupport/modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index f95432c658d68..bec23fc58edea 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -280,6 +280,7 @@ def __hash__(self): sbt_test_goals=[ "connect/test", "connect-client-jvm/test", + "connect-client-integration-tests/test", ], ) From 309347ba075f2b81c23ea43af3b3b480337da330 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Sun, 29 Jan 2023 12:39:14 +0800 Subject: [PATCH 03/31] remove assembly --- .../spark/sql/connect/client/util/RemoteSparkSession.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala b/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala index e02556a60174e..f291b2d529bad 100644 --- a/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala +++ b/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala @@ -121,7 +121,7 @@ object SparkConnectServerUtils { val jars = recursiveListFiles(parentDir).filter { f => // SBT jar (f.getParentFile.getName.startsWith("scala-") && - f.getName.startsWith("spark-connect-assembly") && f.getName.endsWith(".jar")) || + f.getName.startsWith("spark-connect") && f.getName.endsWith(".jar")) || // Maven Jar (f.getParent.endsWith("target") && f.getName.startsWith("spark-connect") && f.getName.endsWith(".jar")) From 839c4d31efbd7427e3d62869a287f3a6ff266e0a Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 30 Jan 2023 13:01:45 +0800 Subject: [PATCH 04/31] more test --- connector/connect/client/integration-tests/pom.xml | 2 ++ .../spark/sql/connect/client/util/RemoteSparkSession.scala | 2 +- connector/connect/client/jvm/pom.xml | 5 +++++ project/SparkBuild.scala | 7 ------- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/connector/connect/client/integration-tests/pom.xml b/connector/connect/client/integration-tests/pom.xml index 44f1c093ead6b..5d7ecc3713e2e 100644 --- a/connector/connect/client/integration-tests/pom.xml +++ b/connector/connect/client/integration-tests/pom.xml @@ -89,6 +89,7 @@ test ${project.version} + target/scala-${scala.binary.version}/test-classes diff --git a/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala b/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala index f291b2d529bad..e02556a60174e 100644 --- a/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala +++ b/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala @@ -121,7 +121,7 @@ object SparkConnectServerUtils { val jars = recursiveListFiles(parentDir).filter { f => // SBT jar (f.getParentFile.getName.startsWith("scala-") && - f.getName.startsWith("spark-connect") && f.getName.endsWith(".jar")) || + f.getName.startsWith("spark-connect-assembly") && f.getName.endsWith(".jar")) || // Maven Jar (f.getParent.endsWith("target") && f.getName.startsWith("spark-connect") && f.getName.endsWith(".jar")) diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index b27eec5003820..725f78c04cf94 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -65,6 +65,11 @@ ${guava.version} compile + + org.scalacheck + scalacheck_${scala.binary.version} + test + org.mockito mockito-core diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 750ce49b79098..8997f3a6a7f00 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -897,13 +897,6 @@ object SparkConnectClient { object SparkConnectClientTests { lazy val settings = Seq( - excludeDependencies ++= { - val scalaBinaryVersion = - SbtPomKeys.effectivePom.value.getProperties.get("scala.binary.version").asInstanceOf[String] - Seq( - ExclusionRule("org.apache.spark", s"spark-sql_${scalaBinaryVersion}") - ) - }, libraryDependencies ++= { val guavaVersion = SbtPomKeys.effectivePom.value.getProperties.get("guava.version").asInstanceOf[String] From 762dbe0258e2f787c564d5e28993a89678890991 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 30 Jan 2023 13:11:52 +0800 Subject: [PATCH 05/31] add test --- connector/connect/client/integration-tests/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/connector/connect/client/integration-tests/pom.xml b/connector/connect/client/integration-tests/pom.xml index 5d7ecc3713e2e..a8bd4cdd777d9 100644 --- a/connector/connect/client/integration-tests/pom.xml +++ b/connector/connect/client/integration-tests/pom.xml @@ -76,6 +76,7 @@ org.apache.spark spark-repl_${scala.binary.version} ${project.version} + test-jar org.apache.spark From aea7bb1b59d107590d904aab846f01eb9bdb7356 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 30 Jan 2023 14:19:43 +0800 Subject: [PATCH 06/31] exlcue repl --- connector/connect/client/integration-tests/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/connector/connect/client/integration-tests/pom.xml b/connector/connect/client/integration-tests/pom.xml index a8bd4cdd777d9..1705a625a5cd4 100644 --- a/connector/connect/client/integration-tests/pom.xml +++ b/connector/connect/client/integration-tests/pom.xml @@ -72,6 +72,7 @@ test ${project.version} + org.apache.spark spark-connect-common_${scala.binary.version} From 615f0af251c6852a79066736626930ff72260676 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 30 Jan 2023 16:44:27 +0800 Subject: [PATCH 07/31] remove deps --- .../connect/client/integration-tests/pom.xml | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/connector/connect/client/integration-tests/pom.xml b/connector/connect/client/integration-tests/pom.xml index 1705a625a5cd4..63f8ec0070737 100644 --- a/connector/connect/client/integration-tests/pom.xml +++ b/connector/connect/client/integration-tests/pom.xml @@ -33,37 +33,13 @@ connect-client-integration-tests 31.0.1-jre - 3.21.12 - - - - org.apache.spark - spark-connect-client-jvm_${scala.binary.version} - ${project.version} - test - org.apache.spark spark-connect-client-jvm_${scala.binary.version} ${project.version} - test-jar test From abc4b85144d8c21c293424c10b4263c8c55a1304 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Mon, 30 Jan 2023 23:18:32 +0800 Subject: [PATCH 08/31] with profile --- .github/workflows/build_and_test.yml | 61 +++++++++++++++++++ .../pom.xml | 35 ++--------- .../src/test/resources/log4j2.properties | 0 .../apache/spark/sql/ClientE2ETestSuite.scala | 0 .../client/util/RemoteSparkSession.scala | 0 dev/sparktestsupport/modules.py | 1 - pom.xml | 8 ++- project/SparkBuild.scala | 15 ++--- 8 files changed, 81 insertions(+), 39 deletions(-) rename connector/connect/client/{integration-tests => jvm-e2e-tests}/pom.xml (67%) rename connector/connect/client/{integration-tests => jvm-e2e-tests}/src/test/resources/log4j2.properties (100%) rename connector/connect/client/{integration-tests => jvm-e2e-tests}/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala (100%) rename connector/connect/client/{integration-tests => jvm-e2e-tests}/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala (100%) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 021566a5b8e71..09a1d10f350a1 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -101,6 +101,7 @@ jobs: \"java-11-17\": \"true\", \"lint\" : \"true\", \"k8s-integration-tests\" : \"true\", + \"connect-jvm-e2e-tests\" : \"true\", }" echo $precondition # For debugging # Remove `\n` to avoid "Invalid format" error @@ -681,6 +682,66 @@ jobs: ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=${JAVA_VERSION/-ea} install rm -rf ~/.m2/repository/org/apache/spark + connect-jvm-e2e-tests: + needs: precondition + if: fromJson(needs.precondition.outputs.required).connect-jvm-e2e-tests == 'true' && + (inputs.branch != 'branch-3.2' && inputs.branch != 'branch-3.3') + name: Connect JVM Client E2E Tests with Maven + strategy: + fail-fast: false + matrix: + java: + - ${{ inputs.java }} + runs-on: ubuntu-22.04 + steps: + - name: Checkout Spark repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + repository: apache/spark + ref: ${{ inputs.branch }} + - name: Sync the current branch with the latest in Apache Spark + if: github.repository != 'apache/spark' + run: | + git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty + - name: Cache Scala, SBT and Maven + uses: actions/cache@v3 + with: + path: | + build/apache-maven-* + build/scala-* + build/*.jar + ~/.sbt + key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} + restore-keys: | + build- + - name: Cache Maven local repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: java${{ matrix.java }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + java${{ matrix.java }}-maven- + - name: Install Java ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: ${{ matrix.java }} + - name: Build with Maven + run: | + export MAVEN_OPTS="-Xss64m -Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" + export MAVEN_CLI_OPTS="--no-transfer-progress" + export JAVA_VERSION=${{ matrix.java }} + # It uses Maven's 'install' intentionally, see https://github.com/apache/spark/pull/26414. + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl repl -am + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/common + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/server + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/client/jvm + ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} clean test -pl connector/connect/client/jvm-e2e-tests -Pconnect-client-tests + rm -rf ~/.m2/repository/org/apache/spark + scala-213: needs: precondition if: fromJson(needs.precondition.outputs.required).scala-213 == 'true' diff --git a/connector/connect/client/integration-tests/pom.xml b/connector/connect/client/jvm-e2e-tests/pom.xml similarity index 67% rename from connector/connect/client/integration-tests/pom.xml rename to connector/connect/client/jvm-e2e-tests/pom.xml index 63f8ec0070737..149ee9be335d7 100644 --- a/connector/connect/client/integration-tests/pom.xml +++ b/connector/connect/client/jvm-e2e-tests/pom.xml @@ -26,12 +26,12 @@ ../../../../pom.xml - spark-connect-client-integration-tests_2.12 + spark-connect-client-jvm-e2e-tests_2.12 jar - Spark Project Connect Client Integration Tests + Spark Project Connect Client E2E Tests https://spark.apache.org/ - connect-client-integration-tests + connect-client-jvm-e2e-tests 31.0.1-jre @@ -42,26 +42,6 @@ ${project.version} test - - org.apache.spark - spark-catalyst_${scala.binary.version} - test - ${project.version} - - org.apache.spark spark-connect-common_${scala.binary.version} @@ -71,14 +51,9 @@ diff --git a/connector/connect/client/integration-tests/src/test/resources/log4j2.properties b/connector/connect/client/jvm-e2e-tests/src/test/resources/log4j2.properties similarity index 100% rename from connector/connect/client/integration-tests/src/test/resources/log4j2.properties rename to connector/connect/client/jvm-e2e-tests/src/test/resources/log4j2.properties diff --git a/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala b/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala similarity index 100% rename from connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala rename to connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala diff --git a/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala b/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala similarity index 100% rename from connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala rename to connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index bec23fc58edea..f95432c658d68 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -280,7 +280,6 @@ def __hash__(self): sbt_test_goals=[ "connect/test", "connect-client-jvm/test", - "connect-client-integration-tests/test", ], ) diff --git a/pom.xml b/pom.xml index 494a812b5b67d..41ab351318bfc 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,6 @@ connector/connect/server connector/connect/common connector/connect/client/jvm - connector/connect/client/integration-tests connector/protobuf @@ -3496,6 +3495,13 @@ + + connect-client-tests + + connector/connect/client/jvm-e2e-tests + + + target/scala-${scala.binary.version}/test-classes From 878f057fd9287cd81d711f1200e9f0a34640e520 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 31 Jan 2023 00:35:54 +0800 Subject: [PATCH 17/31] fix sbt compile --- project/SparkBuild.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index f932eaac34bd7..1aaa35310237b 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -55,7 +55,6 @@ object BuildCommons { val connectCommon = ProjectRef(buildLocation, "connect-common") val connect = ProjectRef(buildLocation, "connect") val connectClient = ProjectRef(buildLocation, "connect-client-jvm") - val connectClientE2ETests = ProjectRef(buildLocation, "connect-client-jvm-e2e-tests") val allProjects@Seq( core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle, launcher, unsafe, tags, sketch, kvstore, _* @@ -66,10 +65,10 @@ object BuildCommons { val optionallyEnabledProjects@Seq(kubernetes, mesos, yarn, sparkGangliaLgpl, streamingKinesisAsl, - dockerIntegrationTests, hadoopCloud, kubernetesIntegrationTests) = + dockerIntegrationTests, hadoopCloud, kubernetesIntegrationTests, connectClientE2ETests) = Seq("kubernetes", "mesos", "yarn", "ganglia-lgpl", "streaming-kinesis-asl", - "docker-integration-tests", "hadoop-cloud", "kubernetes-integration-tests").map(ProjectRef(buildLocation, _)) ++ Seq(connectClientE2ETests) + "docker-integration-tests", "hadoop-cloud", "kubernetes-integration-tests", "connect-client-jvm-e2e-tests").map(ProjectRef(buildLocation, _)) val assemblyProjects@Seq(networkYarn, streamingKafka010Assembly, streamingKinesisAslAssembly) = Seq("network-yarn", "streaming-kafka-0-10-assembly", "streaming-kinesis-asl-assembly") From 0c9fee3d22a13c897ba94489400a417a34e82d5f Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 31 Jan 2023 01:17:15 +0800 Subject: [PATCH 18/31] test scala -2.13 --- .github/workflows/build_and_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 398b357502695..41b89235e221a 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -738,11 +738,12 @@ jobs: - name: Build and Test with Maven run: | export SCALA_VERSION=2.12 + export SCALA_PROFILE=scala2.13 if [[ "SCALA_PROFILE" == "scala2.13" ]] ; then export SCALA_VERSION=2.13; ./dev/change-scala-version.sh 2.13 fi - export MAVEN_OPTS="-Xss64m -Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" + # export MAVEN_OPTS="-Xss64m -Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" export MAVEN_CLI_OPTS="--no-transfer-progress" export JAVA_VERSION=${{ matrix.java }} # It uses Maven's 'install' intentionally, see https://github.com/apache/spark/pull/26414. From 70b1e21e0ce13d0ab7d8684dcd9c4dc43919e300 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 31 Jan 2023 01:33:22 +0800 Subject: [PATCH 19/31] fix maven found jar --- .github/workflows/build_and_test.yml | 8 ++++---- .../sql/connect/client/util/RemoteSparkSession.scala | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 41b89235e221a..baf7135831277 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -747,10 +747,10 @@ jobs: export MAVEN_CLI_OPTS="--no-transfer-progress" export JAVA_VERSION=${{ matrix.java }} # It uses Maven's 'install' intentionally, see https://github.com/apache/spark/pull/26414. - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl repl -am -Pscala-$SCALA_VERSION - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/common -Pscala-$SCALA_VERSION - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/server -Pscala-$SCALA_VERSION - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/client/jvm -Pscala-$SCALA_VERSION + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} package -pl repl -am -Pscala-$SCALA_VERSION + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} package -pl connector/connect/common -Pscala-$SCALA_VERSION + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} package -pl connector/connect/server -Pscala-$SCALA_VERSION + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} package -pl connector/connect/client/jvm -Pscala-$SCALA_VERSION ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} clean test -pl connector/connect/client/jvm-e2e-tests -Pscala-$SCALA_VERSION -Pconnect-client-tests rm -rf ~/.m2/repository/org/apache/spark diff --git a/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala b/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala index e02556a60174e..ffcb8400c683c 100644 --- a/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala +++ b/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala @@ -124,7 +124,9 @@ object SparkConnectServerUtils { f.getName.startsWith("spark-connect-assembly") && f.getName.endsWith(".jar")) || // Maven Jar (f.getParent.endsWith("target") && - f.getName.startsWith("spark-connect") && f.getName.endsWith(".jar")) + f.getName.startsWith("spark-connect") && f.getName.endsWith(".jar") && + !f.getName.contains("test") && !f.getName.contains("source") && + !f.getName.contains("javadoc")) } // It is possible we found more than one: one built by maven, and another by SBT assert( From a331c72b42f90109d22ccdf0bd32d624a674f827 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 31 Jan 2023 01:48:23 +0800 Subject: [PATCH 20/31] find jar --- .../spark/sql/connect/client/util/RemoteSparkSession.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala b/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala index ffcb8400c683c..0ebea7f4fa7de 100644 --- a/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala +++ b/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala @@ -124,9 +124,8 @@ object SparkConnectServerUtils { f.getName.startsWith("spark-connect-assembly") && f.getName.endsWith(".jar")) || // Maven Jar (f.getParent.endsWith("target") && - f.getName.startsWith("spark-connect") && f.getName.endsWith(".jar") && - !f.getName.contains("test") && !f.getName.contains("source") && - !f.getName.contains("javadoc")) + f.getName.startsWith("spark-connect") + && f.getName.endsWith(s"${org.apache.spark.SPARK_VERSION}.jar")) } // It is possible we found more than one: one built by maven, and another by SBT assert( From e0f9ad59c3b02ecaadbbdcfc6d925b3901861fd9 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 31 Jan 2023 03:00:34 +0800 Subject: [PATCH 21/31] revert to scala 2.12 --- .github/workflows/build_and_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index baf7135831277..d64e924998a1a 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -738,7 +738,6 @@ jobs: - name: Build and Test with Maven run: | export SCALA_VERSION=2.12 - export SCALA_PROFILE=scala2.13 if [[ "SCALA_PROFILE" == "scala2.13" ]] ; then export SCALA_VERSION=2.13; ./dev/change-scala-version.sh 2.13 From 0c512d5e0791a24db1e9474d693f9f078051ea6a Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 31 Jan 2023 03:20:46 +0800 Subject: [PATCH 22/31] use install --- .github/workflows/build_and_test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d64e924998a1a..398b357502695 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -742,14 +742,14 @@ jobs: export SCALA_VERSION=2.13; ./dev/change-scala-version.sh 2.13 fi - # export MAVEN_OPTS="-Xss64m -Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" + export MAVEN_OPTS="-Xss64m -Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" export MAVEN_CLI_OPTS="--no-transfer-progress" export JAVA_VERSION=${{ matrix.java }} # It uses Maven's 'install' intentionally, see https://github.com/apache/spark/pull/26414. - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} package -pl repl -am -Pscala-$SCALA_VERSION - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} package -pl connector/connect/common -Pscala-$SCALA_VERSION - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} package -pl connector/connect/server -Pscala-$SCALA_VERSION - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} package -pl connector/connect/client/jvm -Pscala-$SCALA_VERSION + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl repl -am -Pscala-$SCALA_VERSION + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/common -Pscala-$SCALA_VERSION + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/server -Pscala-$SCALA_VERSION + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/client/jvm -Pscala-$SCALA_VERSION ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} clean test -pl connector/connect/client/jvm-e2e-tests -Pscala-$SCALA_VERSION -Pconnect-client-tests rm -rf ~/.m2/repository/org/apache/spark From f485ce454e728c19eeacbca6ef10f86d749e6618 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 1 Feb 2023 14:19:29 +0800 Subject: [PATCH 23/31] fix location --- connector/connect/client/jvm/pom.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index 725f78c04cf94..eebea82243dbe 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -88,7 +88,7 @@ com.google.guava:* - io.grpc:*: + io.grpc:* com.google.protobuf:* org.apache.spark:spark-connect-common_${scala.binary.version} @@ -97,6 +97,9 @@ io.grpc ${spark.shade.packageName}.connect.grpc + + io.grpc.** + com.google.protobuf From a249f068d2085d537c61756568f397aaf0f1ac25 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 1 Feb 2023 14:51:03 +0800 Subject: [PATCH 24/31] add ServicesResourceTransformer for maven --- .../spark/sql/connect/client/util/RemoteSparkSession.scala | 2 +- connector/connect/client/jvm/pom.xml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala b/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala index 552799d52297c..cd996da4a36b0 100644 --- a/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala +++ b/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala @@ -54,7 +54,7 @@ object SparkConnectServerUtils { } sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME")) } - private val isDebug = System.getProperty(DEBUG_SC_JVM_CLIENT, "false").toBoolean + private val isDebug = System.getProperty(DEBUG_SC_JVM_CLIENT, "true").toBoolean // Log server start stop debug info into console // scalastyle:off println diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index eebea82243dbe..65f288e23b25d 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -123,6 +123,9 @@ + + + From 47cda99ccb765b2e4c6dec9b61d1a8ccd81704cb Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 1 Feb 2023 15:51:53 +0800 Subject: [PATCH 25/31] move to connect.client --- connector/connect/client/jvm/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index 65f288e23b25d..eb7f28f2fb760 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -96,7 +96,7 @@ io.grpc - ${spark.shade.packageName}.connect.grpc + ${spark.shade.packageName}.connect.client.grpc io.grpc.** From b1fc936b6670343be2e17a92ef060790de8c639f Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 2 Feb 2023 11:01:09 +0800 Subject: [PATCH 26/31] remove profile --- .github/workflows/build_and_test.yml | 2 +- dev/sparktestsupport/modules.py | 1 + pom.xml | 8 +------- project/SparkBuild.scala | 9 ++++++--- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9eb06942db121..3f28ff03a6e96 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -752,7 +752,7 @@ jobs: ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/common -Pscala-$SCALA_VERSION ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/server -Pscala-$SCALA_VERSION ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Djava.version=${JAVA_VERSION/-ea} install -pl connector/connect/client/jvm -Pscala-$SCALA_VERSION - ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} clean test -pl connector/connect/client/jvm-e2e-tests -Pscala-$SCALA_VERSION -Pconnect-client-tests + ./build/mvn $MAVEN_CLI_OPTS -Djava.version=${JAVA_VERSION/-ea} clean test -pl connector/connect/client/jvm-e2e-tests -Pscala-$SCALA_VERSION rm -rf ~/.m2/repository/org/apache/spark scala-213: diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index f95432c658d68..c661feec41a49 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -280,6 +280,7 @@ def __hash__(self): sbt_test_goals=[ "connect/test", "connect-client-jvm/test", + "connect-client-jvm-e2e-tests/test", ], ) diff --git a/pom.xml b/pom.xml index ce5b1fab1d2b4..7050582e34c23 100644 --- a/pom.xml +++ b/pom.xml @@ -103,6 +103,7 @@ connector/connect/server connector/connect/common connector/connect/client/jvm + connector/connect/client/jvm-e2e-tests connector/protobuf @@ -3495,13 +3496,6 @@ - - connect-client-tests - - connector/connect/client/jvm-e2e-tests - - - + + com.typesafe + mima-core_${scala.binary.version} + ${mima.version} + test + target/scala-${scala.binary.version}/test-classes diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CompatibilitySuite.scala b/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/CompatibilitySuite.scala similarity index 100% rename from connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CompatibilitySuite.scala rename to connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/connect/CompatibilitySuite.scala diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index 3178b5a1ca3ba..9c5ce2f5d6643 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -93,13 +93,6 @@ mockito-core test - - - com.typesafe - mima-core_${scala.binary.version} - ${mima.version} - test - target/scala-${scala.binary.version}/test-classes From 7143c4e9046344f523fe6032000ae479a6916e1c Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 3 Feb 2023 13:52:51 +0800 Subject: [PATCH 29/31] ignore simple udf test --- .../test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala b/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala index db2b8b269876f..5c3615775f0b2 100644 --- a/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala +++ b/connector/connect/client/jvm-e2e-tests/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala @@ -49,7 +49,7 @@ class ClientE2ETestSuite extends RemoteSparkSession { assert(array(2).getLong(0) == 2) } - test("simple udf test") { + ignore("simple udf test") { def dummyUdf(x: Int): Int = x + 5 val myUdf = udf(dummyUdf _) From 408c15aaa852ea09e027d21e5b503149fc2f0ff8 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 3 Feb 2023 13:54:17 +0800 Subject: [PATCH 30/31] mima version --- connector/connect/client/jvm-e2e-tests/pom.xml | 1 + connector/connect/client/jvm/pom.xml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/connector/connect/client/jvm-e2e-tests/pom.xml b/connector/connect/client/jvm-e2e-tests/pom.xml index 38bba33d63941..225e8570f5bb9 100644 --- a/connector/connect/client/jvm-e2e-tests/pom.xml +++ b/connector/connect/client/jvm-e2e-tests/pom.xml @@ -33,6 +33,7 @@ connect-client-jvm-e2e-tests 31.0.1-jre + 1.1.0 diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index 9c5ce2f5d6643..e1f5cf0b41f12 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -33,7 +33,6 @@ connect-client-jvm 31.0.1-jre - 1.1.0 From 790f31ff693c05faf90bc5a011ff6475fa914ef3 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 3 Feb 2023 17:48:46 +0800 Subject: [PATCH 31/31] udpate sparkbuild --- project/SparkBuild.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index c3a2a1a73f51a..5067b38c0cef1 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -854,9 +854,6 @@ object SparkConnectClient { ) }, - // Make sure the connect server assembly jar is available for testing. - test := ((Test / test) dependsOn (LocalProject("connect") / assembly)).value, - (assembly / test) := { }, (assembly / logLevel) := Level.Info, @@ -899,7 +896,14 @@ object SparkConnectClient { } object SparkConnectClientE2ETests { + val buildTestDeps = TaskKey[Unit]("buildTestDeps", "Build needed dependencies for test.") lazy val settings = Seq( + buildTestDeps := { + (LocalProject("sql") / Compile / Keys.`package`).value + (LocalProject("connect") / assembly).value + (LocalProject("connect-client-jvm") / assembly).value + }, + test := ((Test / test) dependsOn (buildTestDeps)).value, // Make sure the connect server assembly jar is available for testing. test := ((Test / test) dependsOn (LocalProject("connect") / assembly)).value, libraryDependencies ++= {