Skip to content

[BUILD] Revise Kyuubi Spark engine shaded #3776

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pan3793 wants to merge 4 commits into from
Closed

[BUILD] Revise Kyuubi Spark engine shaded #3776

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
19382ef
[BUILD] Revisit Kyuubi Spark engine shaded
pan3793 Nov 7, 2022
a53b7c0
nit
pan3793 Nov 10, 2022
e247923
1
pan3793 Nov 10, 2022
6a6e2a6
nit
pan3793 Nov 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion LICENSE-binary
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,6 @@ io.netty:netty-handler
io.netty:netty-resolver-dns
io.netty:netty-resolver
io.netty:netty-transport-classes-epoll
io.netty:netty-transport-classes-kqueue
io.netty:netty-transport-native-epoll
io.netty:netty-transport-native-unix-common
io.netty:netty-transport
Expand Down
141 changes: 131 additions & 10 deletions externals/kyuubi-spark-sql-engine/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,29 +208,37 @@
<shadedArtifactAttached>false</shadedArtifactAttached>
<artifactSet>
<includes>
<include>com.google.android:annotations</include>
<include>com.google.api.grpc:proto-google-common-protos</include>
<include>com.google.code.gson:gson</include>
<include>com.google.errorprone:error_prone_annotations</include>
<include>com.google.guava:*</include>
<include>com.google.j2objc:j2objc-annotations</include>
<include>com.google.protobuf:*</include>
<include>io.etcd:*</include>
<include>io.grpc:*</include>
<include>io.netty:*</include>
<include>io.perfmark:perfmark-api</include>
<include>io.vertx:*</include>
<include>net.jodah:failsafe</include>
<include>org.apache.kyuubi:kyuubi-common_${scala.binary.version}</include>
<include>org.apache.kyuubi:kyuubi-events_${scala.binary.version}</include>
<include>org.apache.kyuubi:kyuubi-ha_${scala.binary.version}</include>
<include>org.apache.curator:curator-client</include>
<include>org.apache.curator:curator-framework</include>
<include>org.apache.curator:curator-recipes</include>
<include>org.apache.zookeeper:zookeeper</include>
<include>org.apache.hive:hive-service-rpc</include>
<include>org.apache.kyuubi:kyuubi-common_${scala.binary.version}</include>
<include>org.apache.kyuubi:kyuubi-events_${scala.binary.version}</include>
<include>org.apache.kyuubi:kyuubi-ha_${scala.binary.version}</include>
<include>org.apache.thrift:*</include>
<include>org.apache.zookeeper:zookeeper</include>
<include>org.checkerframework:checker-qual</include>
<include>org.codehaus.mojo:animal-sniffer-annotations</include>
</includes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>**/*.proto</exclude>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
Expand Down Expand Up @@ -289,19 +297,27 @@
</includes>
</relocation>
<relocation>
<pattern>com.google</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google</shadedPattern>
<pattern>io.etcd</pattern>
<shadedPattern>${kyuubi.shade.packageName}.io.etcd</shadedPattern>
<includes>
<include>com.google.**</include>
<include>io.etcd.**</include>
</includes>
</relocation>
<relocation>
<pattern>io.grpc</pattern>
<shadedPattern>${kyuubi.shade.packageName}.io.grpc</shadedPattern>
</relocation>
<relocation>
<pattern>io.netty</pattern>
<shadedPattern>${kyuubi.shade.packageName}.io.netty</shadedPattern>
<includes>
<include>io.netty.**</include>
</includes>
</relocation>
<relocation>
<pattern>io.perfmark</pattern>
<shadedPattern>${kyuubi.shade.packageName}.io.perfmark</shadedPattern>
</relocation>
<relocation>
<pattern>io.vertx</pattern>
<shadedPattern>${kyuubi.shade.packageName}.io.vertx</shadedPattern>
Expand All @@ -317,12 +333,83 @@
</includes>
</relocation>
<relocation>
<pattern>io.perfmark</pattern>
<shadedPattern>${kyuubi.shade.packageName}.io.perfmark</shadedPattern>
<pattern>android.annotation</pattern>
<shadedPattern>${kyuubi.shade.packageName}.android.annotation</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.common</shadedPattern>
<includes>
<include>com.google.common.**</include>
</includes>
</relocation>
<relocation>
<pattern>com.google.thirdparty</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.thirdparty</shadedPattern>
<includes>
<include>io.perfmark.**</include>
<include>com.google.thirdparty.**</include>
</includes>
</relocation>
<relocation>
<pattern>com.google.protobuf</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.protobuf</shadedPattern>
<includes>
<include>com.google.protobuf.**</include>
</includes>
</relocation>
<relocation>
<pattern>org.codehaus.mojo.animal_sniffer</pattern>
<shadedPattern>${kyuubi.shade.packageName}.org.codehaus.mojo.animal_sniffer</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.j2objc.annotations</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.j2objc.annotations</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.errorprone.annotations</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.errorprone.annotations</shadedPattern>
</relocation>
<relocation>
<pattern>org.checkerframework</pattern>
<shadedPattern>${kyuubi.shade.packageName}.org.checkerframework</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.gson</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.gson</shadedPattern>
</relocation>
<!--
For `com.google.api.grpc:proto-google-common-protos`, do not directly define pattern
as `common.google`, otherwise, otherwise, the relocation result may be uncertain due
to the change of rule order.
-->
<relocation>
<pattern>com.google.api</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.api</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.cloud</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.cloud</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.geo</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.geo</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.logging</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.logging</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.longrunning</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.longrunning</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.rpc</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.rpc</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.type</pattern>
<shadedPattern>${kyuubi.shade.packageName}.com.google.type</shadedPattern>
</relocation>
</relocations>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
Expand All @@ -338,6 +425,40 @@
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>rename-native-library</id>
<phase>package</phase>
<configuration>
<target>
<echo message="unpacking netty jar"/>
<unzip src="${project.build.directory}/${project.artifactId}-${project.version}.jar"
dest="${project.build.directory}/unpacked/"/>
<echo message="renaming netty native libraries"/>
<move includeemptydirs="false"
todir="${project.build.directory}/unpacked/META-INF/native">
<fileset dir="${project.build.directory}/unpacked/META-INF/native"/>
<mapper type="regexp"
from="libnetty_transport_native_epoll_(aarch_64.so|x86_64.so)"
to="liborg_apache_kyuubi_shade_netty_transport_native_epoll_\1"/>
</move>
<echo message="deleting META-INF/native-image folder"/>
<delete dir="${project.build.directory}/unpacked/META-INF/native-image"/>
<echo message="repackaging netty jar"/>
<jar destfile="${project.build.directory}/${project.artifactId}-${project.version}.jar"
basedir="${project.build.directory}/unpacked"/>
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
Expand Down
43 changes: 33 additions & 10 deletions externals/kyuubi-spark-sql-engine/src/main/resources/META-INF/LICENSE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,31 +210,54 @@ Apache License Version 2.0
org.apache.curator:curator-client
org.apache.curator:curator-framework
org.apache.curator:curator-recipes
net.jodah:failsafe
io.grpc:grpc-core
io.grpc:grpc-protobuf
org.apache.hive:hive-service-rpc
org.apache.thrift:libfb303
org.apache.thrift:libthrift
org.apache.zookeeper:zookeeper
com.google.api.grpc:proto-google-common-protos
com.google.android:annotations
com.google.code.gson:gson
com.google.errorprone:error_prone_annotations
com.google.guava:guava
com.google.guava:failureaccess
com.google.j2objc:j2objc-annotations
io.etcd:jetcd-api
io.etcd:jetcd-common
io.etcd:jetcd-core
io.etcd:jetcd-grpc
io.grpc:grpc-api
io.grpc:grpc-context
io.grpc:grpc-core
io.grpc:grpc-grpclb
io.grpc:grpc-netty
io.grpc:grpc-protobuf
io.grpc:grpc-protobuf-lite
io.grpc:grpc-stub
com.google.guava:guava
io.etcd:jetcd-api
io.etcd:jetcd-common
io.etcd:jetcd-core
io.etcd:jetcd-grpc
io.netty:netty-buffer
io.netty:netty-codec
io.netty:netty-codec-dns
io.netty:netty-codec-http
io.netty:netty-codec-http2
io.netty:netty-codec-socks
io.netty:netty-common
io.netty:netty-handler
io.netty:netty-handler-proxy
io.netty:netty-resolver
io.netty:netty-resolver-dns
io.netty:netty-transport
io.netty:netty-transport-classes-epoll
io.netty:netty-transport-native-epoll
io.netty:netty-transport-native-unix-common
io.perfmark:perfmark-api
io.vertx:vertx-core
io.vertx:vertx-grpc
io.vertx:vertx-core
net.jodah:failsafe

BSD License
--------------------------
com.google.protobuf:protobuf-java-util
com.google.protobuf:protobuf-java
com.google.protobuf:protobuf-java-util

MIT license
--------------------------
org.codehaus.mojo:animal-sniffer-annotations
Loading