diff --git a/connector/connect/client/integration-tests/pom.xml b/connector/connect/client/integration-tests/pom.xml new file mode 100644 index 0000000000000..44f1c093ead6b --- /dev/null +++ b/connector/connect/client/integration-tests/pom.xml @@ -0,0 +1,107 @@ + + + + + 4.0.0 + + org.apache.spark + spark-parent_2.12 + 3.5.0-SNAPSHOT + ../../../../pom.xml + + + spark-connect-client-integration-tests_2.12 + jar + Spark Project Connect Client Integration Tests + https://spark.apache.org/ + + connect-client-integration-tests + 31.0.1-jre + 3.21.12 + + + + + + + org.apache.spark + spark-connect-client-jvm_${scala.binary.version} + ${project.version} + test + + + org.apache.spark + spark-connect-client-jvm_${scala.binary.version} + ${project.version} + test-jar + test + + + org.apache.spark + spark-catalyst_${scala.binary.version} + test + ${project.version} + + + org.apache.spark + spark-repl_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-sql_${scala.binary.version} + + + + + org.apache.spark + spark-connect-common_${scala.binary.version} + test + ${project.version} + + + org.apache.spark + spark-connect_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-sql_${scala.binary.version} + + + + + + target/scala-${scala.binary.version}/test-classes + + \ No newline at end of file diff --git a/connector/connect/client/integration-tests/src/test/resources/log4j2.properties b/connector/connect/client/integration-tests/src/test/resources/log4j2.properties new file mode 100644 index 0000000000000..ab02104c69697 --- /dev/null +++ b/connector/connect/client/integration-tests/src/test/resources/log4j2.properties @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.file.ref = ${sys:test.appender:-File} + +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex + +# Tests that launch java subprocesses can set the "test.appender" system property to +# "console" to avoid having the child process's logs overwrite the unit test's +# log file. +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %t: %m%n%ex + +# Ignore messages below warning level from Jetty, because it's a bit verbose +logger.jetty.name = org.sparkproject.jetty +logger.jetty.level = warn diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala b/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala similarity index 100% rename from connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala rename to connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala b/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala similarity index 98% rename from connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala rename to connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala index 79a7a30f1428f..f291b2d529bad 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala +++ b/connector/connect/client/integration-tests/src/test/scala/org/apache/spark/sql/connect/client/util/RemoteSparkSession.scala @@ -54,7 +54,7 @@ object SparkConnectServerUtils { } sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME")) } - private val isDebug = System.getProperty(DEBUG_SC_JVM_CLIENT, "false").toBoolean + private val isDebug = System.getProperty(DEBUG_SC_JVM_CLIENT, "true").toBoolean // Log server start stop debug info into console // scalastyle:off println @@ -121,7 +121,7 @@ object SparkConnectServerUtils { val jars = recursiveListFiles(parentDir).filter { f => // SBT jar (f.getParentFile.getName.startsWith("scala-") && - f.getName.startsWith("spark-connect-assembly") && f.getName.endsWith(".jar")) || + f.getName.startsWith("spark-connect") && f.getName.endsWith(".jar")) || // Maven Jar (f.getParent.endsWith("target") && f.getName.startsWith("spark-connect") && f.getName.endsWith(".jar")) diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index 44c653462fa28..b27eec5003820 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -65,28 +65,6 @@ ${guava.version} compile - - - org.apache.spark - spark-repl_${scala.binary.version} - ${project.version} - test - - - - org.apache.spark - spark-sql_${scala.binary.version} - - - - - org.scalacheck - scalacheck_${scala.binary.version} - test - org.mockito mockito-core @@ -105,11 +83,16 @@ com.google.guava:* + io.grpc:*: com.google.protobuf:* org.apache.spark:spark-connect-common_${scala.binary.version} + + io.grpc + ${spark.shade.packageName}.connect.grpc + com.google.protobuf ${spark.shade.packageName}.connect.protobuf diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index f95432c658d68..bec23fc58edea 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -280,6 +280,7 @@ def __hash__(self): sbt_test_goals=[ "connect/test", "connect-client-jvm/test", + "connect-client-integration-tests/test", ], ) diff --git a/pom.xml b/pom.xml index 5428aed8ad3f8..494a812b5b67d 100644 --- a/pom.xml +++ b/pom.xml @@ -103,6 +103,7 @@ connector/connect/server connector/connect/common connector/connect/client/jvm + connector/connect/client/integration-tests connector/protobuf diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 9f478d649bafe..750ce49b79098 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -55,13 +55,14 @@ object BuildCommons { val connectCommon = ProjectRef(buildLocation, "connect-common") val connect = ProjectRef(buildLocation, "connect") val connectClient = ProjectRef(buildLocation, "connect-client-jvm") + val connectClientTests = ProjectRef(buildLocation, "connect-client-integration-tests") val allProjects@Seq( core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle, launcher, unsafe, tags, sketch, kvstore, _* ) = Seq( "core", "graphx", "mllib", "mllib-local", "repl", "network-common", "network-shuffle", "launcher", "unsafe", "tags", "sketch", "kvstore" - ).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects ++ Seq(connectCommon, connect, connectClient) + ).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects ++ Seq(connectCommon, connect, connectClient, connectClientTests) val optionallyEnabledProjects@Seq(kubernetes, mesos, yarn, sparkGangliaLgpl, streamingKinesisAsl, @@ -403,7 +404,8 @@ object SparkBuild extends PomBuild { val mimaProjects = allProjects.filterNot { x => Seq( spark, hive, hiveThriftServer, repl, networkCommon, networkShuffle, networkYarn, - unsafe, tags, tokenProviderKafka010, sqlKafka010, connectCommon, connect, connectClient, protobuf + unsafe, tags, tokenProviderKafka010, sqlKafka010, connectCommon, connect, connectClient, + connectClientTests, protobuf ).contains(x) } @@ -447,6 +449,7 @@ object SparkBuild extends PomBuild { enable(SparkConnectCommon.settings)(connectCommon) enable(SparkConnect.settings)(connect) enable(SparkConnectClient.settings)(connectClient) + enable(SparkConnectClientTests.settings)(connectClientTests) /* Protobuf settings */ enable(SparkProtobuf.settings)(protobuf) @@ -870,6 +873,7 @@ object SparkConnectClient { }, (assembly / assemblyShadeRules) := Seq( + ShadeRule.rename("io.grpc.**" -> "org.sparkproject.connect.grpc.@0").inAll, ShadeRule.rename("com.google.protobuf.**" -> "org.sparkproject.connect.protobuf.@1").inAll, ShadeRule.rename("com.google.common.**" -> "org.sparkproject.connect.client.guava.@1").inAll, ShadeRule.rename("com.google.thirdparty.**" -> "org.sparkproject.connect.client.guava.@1").inAll, @@ -891,6 +895,30 @@ object SparkConnectClient { } } +object SparkConnectClientTests { + lazy val settings = Seq( + excludeDependencies ++= { + val scalaBinaryVersion = + SbtPomKeys.effectivePom.value.getProperties.get("scala.binary.version").asInstanceOf[String] + Seq( + ExclusionRule("org.apache.spark", s"spark-sql_${scalaBinaryVersion}") + ) + }, + libraryDependencies ++= { + val guavaVersion = + SbtPomKeys.effectivePom.value.getProperties.get("guava.version").asInstanceOf[String] + Seq("com.google.guava" % "guava" % guavaVersion) + }, + dependencyOverrides ++= { + val guavaVersion = + SbtPomKeys.effectivePom.value.getProperties.get("guava.version").asInstanceOf[String] + Seq( + "com.google.guava" % "guava" % guavaVersion + ) + } + ) +} + object SparkProtobuf { import BuildCommons.protoVersion