Skip to content
This repository has been archived by the owner on Sep 18, 2023. It is now read-only.

Commit

Permalink
[NSE-1135] Unify the shim layer for spark-3.2.1 & spark-3.2.2 (#1154)
Browse files Browse the repository at this point in the history
PHILO-HE authored Nov 3, 2022
1 parent 1ef11e2 commit 9cbcd7e
Showing 8 changed files with 3 additions and 167 deletions.
30 changes: 0 additions & 30 deletions gazelle-dist/pom.xml
Original file line number Diff line number Diff line change
@@ -55,36 +55,6 @@
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-3.2.1</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark321</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-3.2.2</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark322</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</profile>
</profiles>

<dependencies>
34 changes: 0 additions & 34 deletions native-sql-engine/core/pom.xml
Original file line number Diff line number Diff line change
@@ -78,40 +78,6 @@
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-3.2.1</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark321</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
</profile>
<profile>
<id>spark-3.2.2</id>
<dependencies>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark32</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.intel.oap</groupId>
<artifactId>spark-sql-columnar-shims-spark322</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
</profile>
</profiles>

<dependencies>
21 changes: 0 additions & 21 deletions pom.xml
Original file line number Diff line number Diff line change
@@ -58,26 +58,6 @@
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
<profile>
<id>spark-3.2.1</id>
<properties>
<spark.version>${spark321.version}</spark.version>
<scala.version>2.12.15</scala.version>
<!--Jackson may be directly used in future UT. Align with the version in spark 3.2.-->
<jackson.version>2.12.0</jackson.version>
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
<profile>
<id>spark-3.2.2</id>
<properties>
<spark.version>${spark322.version}</spark.version>
<scala.version>2.12.15</scala.version>
<!--Jackson may be directly used in future UT. Align with the version in spark 3.2.-->
<jackson.version>2.12.0</jackson.version>
<maven.test.skip>true</maven.test.skip>
</properties>
</profile>
<profile>
<id>hadoop-2.7.4</id>
<properties>
@@ -142,7 +122,6 @@
<spark.version>3.1.1</spark.version>
<spark311.version>3.1.1</spark311.version>
<spark321.version>3.2.1</spark321.version>
<spark322.version>3.2.2</spark322.version>
<!-- Scala 2.12.10 is the version for default spark 3.1 -->
<scala.version>2.12.10</scala.version>
<java.version>1.8</java.version>
20 changes: 0 additions & 20 deletions shims/pom.xml
Original file line number Diff line number Diff line change
@@ -80,26 +80,6 @@
<module>spark321</module>
</modules>
</profile>
<profile>
<id>spark-3.2.1</id>
<properties>
</properties>
<modules>
<module>common</module>
<module>spark32</module>
<module>spark321</module>
</modules>
</profile>
<profile>
<id>spark-3.2.2</id>
<properties>
</properties>
<modules>
<module>common</module>
<module>spark32</module>
<module>spark322</module>
</modules>
</profile>
</profiles>

<build>
Original file line number Diff line number Diff line change
@@ -20,7 +20,9 @@ import com.intel.oap.sql.shims.{SparkShims, SparkShimDescriptor}

object SparkShimProvider {
val DESCRIPTOR = SparkShimDescriptor(3, 2, 1)
val DESCRIPTOR_STRINGS = Seq(s"$DESCRIPTOR")
val DESCRIPTOR_SPARK321 = SparkShimDescriptor(3, 2, 1)
val DESCRIPTOR_SPARK322 = SparkShimDescriptor(3, 2, 2)
val DESCRIPTOR_STRINGS = Seq(s"$DESCRIPTOR_SPARK321", s"$DESCRIPTOR_SPARK322")
}

class SparkShimProvider extends com.intel.oap.sql.shims.SparkShimProvider {

This file was deleted.

This file was deleted.

This file was deleted.

0 comments on commit 9cbcd7e

Please sign in to comment.