diff --git a/LICENSE-binary b/LICENSE-binary index d2d4293f3da1..0d3dcd78fcbd 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -369,6 +369,7 @@ org.eclipse.jetty:jetty-util org.eclipse.jetty:jetty-webapp org.eclipse.jetty:jetty-xml org.scala-lang.modules:scala-xml_2.12 +org.opencypher:okapi-shade core/src/main/java/org/apache/spark/util/collection/TimSort.java core/src/main/resources/org/apache/spark/ui/static/bootstrap* diff --git a/assembly/pom.xml b/assembly/pom.xml index 68ebfadb668a..ef916fb99a04 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -64,6 +64,11 @@ spark-graphx_${scala.binary.version} ${project.version} + + org.apache.spark + spark-graph_${scala.binary.version} + ${project.version} + org.apache.spark spark-sql_${scala.binary.version} diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index b4364072997b..2650ae59a34a 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -151,6 +151,7 @@ minlog-1.3.0.jar netty-3.9.9.Final.jar netty-all-4.1.30.Final.jar objenesis-2.5.1.jar +okapi-shade-0.4.2.jar okhttp-3.8.1.jar okio-1.13.0.jar opencsv-2.3.jar diff --git a/dev/deps/spark-deps-hadoop-3.2 b/dev/deps/spark-deps-hadoop-3.2 index 1fc5e7344eb8..356fdb9fabd9 100644 --- a/dev/deps/spark-deps-hadoop-3.2 +++ b/dev/deps/spark-deps-hadoop-3.2 @@ -170,6 +170,7 @@ netty-3.9.9.Final.jar netty-all-4.1.30.Final.jar nimbus-jose-jwt-4.41.1.jar objenesis-2.5.1.jar +okapi-shade-0.4.2.jar okhttp-2.7.5.jar okhttp-3.8.1.jar okio-1.13.0.jar diff --git a/graph/api/pom.xml b/graph/api/pom.xml new file mode 100644 index 000000000000..a18c5290195f --- /dev/null +++ b/graph/api/pom.xml @@ -0,0 +1,55 @@ + + + + + 4.0.0 + + org.apache.spark + spark-parent_2.12 + 3.0.0-SNAPSHOT + ../../pom.xml + + + spark-graph-api_2.12 + jar + Spark Project Graph API + + graph-api + + http://spark.apache.org/ + + + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-sql_${scala.binary.version} + ${project.version} + + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + diff --git a/graph/cypher/pom.xml b/graph/cypher/pom.xml new file mode 100644 index 000000000000..12b3832d9777 --- /dev/null +++ b/graph/cypher/pom.xml @@ -0,0 +1,100 @@ + + + + + 4.0.0 + + org.apache.spark + spark-parent_2.12 + 3.0.0-SNAPSHOT + ../../pom.xml + + + spark-cypher_2.12 + jar + Spark Project Cypher + + cypher + + http://spark.apache.org/ + + + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-sql_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-catalyst_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-graph-api_${scala.binary.version} + ${project.version} + + + org.opencypher + okapi-shade + ${okapi.version} + + + + + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + test-jar + test + + + org.apache.spark + spark-sql_${scala.binary.version} + ${project.version} + test-jar + test + + + org.apache.spark + spark-catalyst_${scala.binary.version} + ${project.version} + test-jar + test + + + org.opencypher + okapi-tck + ${okapi.version} + test + + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + diff --git a/graph/graph/pom.xml b/graph/graph/pom.xml new file mode 100644 index 000000000000..7438ee5e54fe --- /dev/null +++ b/graph/graph/pom.xml @@ -0,0 +1,50 @@ + + + + + 4.0.0 + + org.apache.spark + spark-parent_2.12 + 3.0.0-SNAPSHOT + ../../pom.xml + + + spark-graph_2.12 + jar + Spark Project Graph + + graph + + http://spark.apache.org/ + + + + org.apache.spark + spark-cypher_${scala.binary.version} + ${project.version} + + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + diff --git a/pom.xml b/pom.xml index 7bf72d7620be..929961aef15c 100644 --- a/pom.xml +++ b/pom.xml @@ -106,6 +106,9 @@ external/kafka-0-10-assembly external/kafka-0-10-sql external/avro + graph/api + graph/cypher + graph/graph @@ -124,6 +127,7 @@ ${hadoop.version} 3.4.6 2.7.1 + 0.4.2 org.spark-project.hive diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 83fe904cc1f0..0944764d3102 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -48,12 +48,15 @@ object BuildCommons { val streamingProjects@Seq(streaming, streamingKafka010) = Seq("streaming", "streaming-kafka-0-10").map(ProjectRef(buildLocation, _)) + val graphProjects@Seq(graph, graphApi, cypher) = + Seq("graph", "graph-api", "cypher").map(ProjectRef(buildLocation, _)) + val allProjects@Seq( core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle, launcher, unsafe, tags, sketch, kvstore, _* ) = Seq( "core", "graphx", "mllib", "mllib-local", "repl", "network-common", "network-shuffle", "launcher", "unsafe", "tags", "sketch", "kvstore" - ).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects + ).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects ++ graphProjects val optionallyEnabledProjects@Seq(kubernetes, mesos, yarn, sparkGangliaLgpl, streamingKinesisAsl, @@ -332,7 +335,7 @@ object SparkBuild extends PomBuild { val mimaProjects = allProjects.filterNot { x => Seq( spark, hive, hiveThriftServer, catalyst, repl, networkCommon, networkShuffle, networkYarn, - unsafe, tags, tokenProviderKafka010, sqlKafka010, kvstore, avro + unsafe, tags, tokenProviderKafka010, sqlKafka010, kvstore, avro, graph, graphApi, cypher ).contains(x) }