Skip to content

Commit 4108ec4

Browse files
henribhenrib
authored
HIVE-28059: Iceberg REST Catalog (#5606)
* HIVE-28059 : major rebase and simpler code; - Update iceberg_stats.q.out - Increase test server startup wait time; - Fixing src packaging; - Fixing PR comments, improve tests, remove dead code, refactored cache, simplify servlet, javadocs - cleaner init & server; - one issue/FIXME (no Thrift conf does not work correctly); - Moving configuration variables to the common place; - Remove first set of warnings, remove unused variables/arguments; - Control authentication type values (jwt, simple); - Removed SecureServletCaller, HmsThriftHttpServlet; - Add servlet proxy to execute calls through ServletSecurity; - Simplify PropertyServlet/HMSCatalogServlet and wrap them in proxy at instantiation time, property servlet tests; - - catalog & property servlet ports shall not be confused; - - some property tests were not referring to the proper port; - Javadoc; - Created helper class to abstract creating embedded Jetty serving one servlet on one port; - Added embedded jetty configuration generic properties (threadpool); - Use ServletServerBuilder to create PropertyServlet and HMSCatalogServlet servers; - Create only one Jetty instance for HMS property & Iceberg Catalog servlets; - Tests servlet builder including port sharing or not; - Moving module to metastore-catalog; - Fixing thrift over http servlet init;
1 parent 56a18bb commit 4108ec4

32 files changed

+3305
-318
lines changed

‎packaging/pom.xml

+5
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,11 @@
424424
<artifactId>hive-webhcat-java-client</artifactId>
425425
<version>${project.version}</version>
426426
</dependency>
427+
<dependency>
428+
<groupId>org.apache.hive</groupId>
429+
<artifactId>hive-standalone-metastore-rest-catalog</artifactId>
430+
<version>${project.version}</version>
431+
</dependency>
427432
<dependency>
428433
<groupId>org.apache.hadoop</groupId>
429434
<artifactId>hadoop-hdfs-client</artifactId>

‎packaging/src/main/assembly/src.xml

+1
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@
105105
<include>standalone-metastore/metastore-common/**/*</include>
106106
<include>standalone-metastore/metastore-server/**/*</include>
107107
<include>standalone-metastore/metastore-tools/**/*</include>
108+
<include>standalone-metastore/metastore-rest-catalog/**/*</include>
108109
<include>standalone-metastore/src/assembly/src.xml</include>
109110
<include>standalone-metastore/pom.xml</include>
110111
<include>streaming/**/*</include>

‎standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -304,8 +304,7 @@ static ThriftHiveMetastore.Iface callEmbeddedMetastore(Configuration conf) throw
304304
try {
305305
Class<?> clazz = Class.forName(HIVE_METASTORE_CLASS);
306306
//noinspection JavaReflectionMemberAccess
307-
Method method = clazz.getDeclaredMethod(HIVE_METASTORE_CREATE_HANDLER_METHOD,
308-
Configuration.class);
307+
Method method = clazz.getDeclaredMethod(HIVE_METASTORE_CREATE_HANDLER_METHOD, Configuration.class);
309308
method.setAccessible(true);
310309
return (ThriftHiveMetastore.Iface) method.invoke(null, conf);
311310
} catch (InvocationTargetException e) {

‎standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java

+38
Original file line numberDiff line numberDiff line change
@@ -1823,8 +1823,46 @@ public enum ConfVars {
18231823
),
18241824
PROPERTIES_SERVLET_AUTH("hive.metastore.properties.servlet.auth",
18251825
"hive.metastore.properties.servlet.auth", "jwt",
1826+
new StringSetValidator("simple", "jwt"),
18261827
"Property-maps servlet authentication method (simple or jwt)."
18271828
),
1829+
ICEBERG_CATALOG_SERVLET_FACTORY("hive.metastore.catalog.servlet.factory",
1830+
"hive.metastore.catalog.servlet.factory",
1831+
"org.apache.iceberg.rest.HMSCatalogFactory",
1832+
"HMS Iceberg Catalog servlet factory class name."
1833+
+ "The factory needs to expose a method: "
1834+
+ "public static HttpServlet createServlet(Configuration configuration);"
1835+
),
1836+
ICEBERG_CATALOG_SERVLET_PATH("hive.metastore.catalog.servlet.path",
1837+
"hive.metastore.catalog.servlet.path", "iceberg",
1838+
"HMS Iceberg Catalog servlet path component of URL endpoint."
1839+
),
1840+
ICEBERG_CATALOG_SERVLET_PORT("hive.metastore.catalog.servlet.port",
1841+
"hive.metastore.catalog.servlet.port", -1,
1842+
"HMS Iceberg Catalog servlet server port. Negative value disables the servlet," +
1843+
" 0 will let the system determine the catalog server port," +
1844+
" positive value will be used as-is."
1845+
),
1846+
ICEBERG_CATALOG_SERVLET_AUTH("hive.metastore.catalog.servlet.auth",
1847+
"hive.metastore.catalog.servlet.auth", "jwt",
1848+
"HMS Iceberg Catalog servlet authentication method (simple or jwt)."
1849+
),
1850+
ICEBERG_CATALOG_CACHE_EXPIRY("hive.metastore.catalog.cache.expiry",
1851+
"hive.metastore.catalog.cache.expiry", 60_000L,
1852+
"HMS Iceberg Catalog cache expiry."
1853+
),
1854+
HTTPSERVER_THREADPOOL_MIN("hive.metastore.httpserver.threadpool.min",
1855+
"hive.metastore.httpserver.threadpool.min", 8,
1856+
"HMS embedded HTTP server minimum number of threads."
1857+
),
1858+
HTTPSERVER_THREADPOOL_MAX("hive.metastore.httpserver.threadpool.max",
1859+
"hive.metastore.httpserver.threadpool.max", 256,
1860+
"HMS embedded HTTP server maximum number of threads."
1861+
),
1862+
HTTPSERVER_THREADPOOL_IDLE("hive.metastore.httpserver.threadpool.idle",
1863+
"hive.metastore.httpserver.threadpool.idle", 60_000L,
1864+
"HMS embedded HTTP server thread idle time."
1865+
),
18281866

18291867
// Deprecated Hive values that we are keeping for backwards compatibility.
18301868
@Deprecated

‎standalone-metastore/metastore-rest-catalog/pom.xml

+258
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
Unless required by applicable law or agreed to in writing, software
8+
distributed under the License is distributed on an "AS IS" BASIS,
9+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
See the License for the specific language governing permissions and
11+
limitations under the License.
12+
-->
13+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
14+
<parent>
15+
<artifactId>hive-standalone-metastore</artifactId>
16+
<groupId>org.apache.hive</groupId>
17+
<version>4.1.0-SNAPSHOT</version>
18+
</parent>
19+
<modelVersion>4.0.0</modelVersion>
20+
<artifactId>hive-standalone-metastore-rest-catalog</artifactId>
21+
<name>Hive Metastore REST Catalog</name>
22+
<properties>
23+
<standalone.metastore.path.to.root>..</standalone.metastore.path.to.root>
24+
<maven.compiler.source>8</maven.compiler.source>
25+
<maven.compiler.target>8</maven.compiler.target>
26+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
27+
<log4j2.debug>false</log4j2.debug>
28+
<hive.version>${project.parent.version}</hive.version>
29+
<iceberg.version>1.6.1</iceberg.version>
30+
</properties>
31+
<dependencies>
32+
<dependency>
33+
<groupId>org.apache.hive</groupId>
34+
<artifactId>hive-standalone-metastore-server</artifactId>
35+
<version>${hive.version}</version>
36+
</dependency>
37+
<dependency>
38+
<groupId>org.apache.hive</groupId>
39+
<artifactId>hive-standalone-metastore-common</artifactId>
40+
<version>${hive.version}</version>
41+
</dependency>
42+
<dependency>
43+
<groupId>org.apache.hive</groupId>
44+
<artifactId>hive-iceberg-catalog</artifactId>
45+
<version>${hive.version}</version>
46+
</dependency>
47+
<!-- Test dependencies -->
48+
<dependency>
49+
<groupId>org.apache.hive</groupId>
50+
<artifactId>hive-standalone-metastore-common</artifactId>
51+
<version>${hive.version}</version>
52+
<classifier>tests</classifier>
53+
<scope>test</scope>
54+
</dependency>
55+
<dependency>
56+
<groupId>org.apache.hive</groupId>
57+
<artifactId>hive-standalone-metastore-server</artifactId>
58+
<version>${hive.version}</version>
59+
<classifier>tests</classifier>
60+
<scope>test</scope>
61+
</dependency>
62+
<dependency>
63+
<groupId>org.apache.httpcomponents.core5</groupId>
64+
<artifactId>httpcore5</artifactId>
65+
<version>5.2</version>
66+
<scope>test</scope>
67+
</dependency>
68+
<dependency>
69+
<groupId>junit</groupId>
70+
<artifactId>junit</artifactId>
71+
<scope>test</scope>
72+
</dependency>
73+
<dependency>
74+
<groupId>com.github.tomakehurst</groupId>
75+
<artifactId>wiremock-jre8-standalone</artifactId>
76+
<version>${wiremock.jre8.standalone.version}</version>
77+
<scope>test</scope>
78+
</dependency>
79+
<dependency>
80+
<groupId>org.junit.jupiter</groupId>
81+
<artifactId>junit-jupiter-api</artifactId>
82+
<version>${junit.jupiter.version}</version>
83+
<scope>test</scope>
84+
</dependency>
85+
<dependency>
86+
<groupId>org.apache.hadoop</groupId>
87+
<artifactId>hadoop-auth</artifactId>
88+
<version>${hadoop.version}</version>
89+
<exclusions>
90+
<exclusion>
91+
<groupId>org.slf4j</groupId>
92+
<artifactId>slf4j-log4j12</artifactId>
93+
</exclusion>
94+
<exclusion>
95+
<groupId>org.slf4j</groupId>
96+
<artifactId>slf4j-reload4j</artifactId>
97+
</exclusion>
98+
<exclusion>
99+
<groupId>ch.qos.reload4j</groupId>
100+
<artifactId>reload4j</artifactId>
101+
</exclusion>
102+
<exclusion>
103+
<groupId>commons-logging</groupId>
104+
<artifactId>commons-logging</artifactId>
105+
</exclusion>
106+
</exclusions>
107+
</dependency>
108+
<dependency>
109+
<groupId>org.apache.hadoop</groupId>
110+
<artifactId>hadoop-common</artifactId>
111+
<version>${hadoop.version}</version>
112+
<exclusions>
113+
<exclusion>
114+
<groupId>org.slf4j</groupId>
115+
<artifactId>slf4j-log4j12</artifactId>
116+
</exclusion>
117+
<exclusion>
118+
<groupId>org.slf4j</groupId>
119+
<artifactId>slf4j-reload4j</artifactId>
120+
</exclusion>
121+
<exclusion>
122+
<groupId>ch.qos.reload4j</groupId>
123+
<artifactId>reload4j</artifactId>
124+
</exclusion>
125+
<exclusion>
126+
<groupId>commons-beanutils</groupId>
127+
<artifactId>commons-beanutils</artifactId>
128+
</exclusion>
129+
<exclusion>
130+
<groupId>commons-logging</groupId>
131+
<artifactId>commons-logging</artifactId>
132+
</exclusion>
133+
</exclusions>
134+
</dependency>
135+
<dependency>
136+
<groupId>org.apache.hadoop</groupId>
137+
<artifactId>hadoop-hdfs-client</artifactId>
138+
<version>${hadoop.version}</version>
139+
<exclusions>
140+
<exclusion>
141+
<groupId>org.slf4j</groupId>
142+
<artifactId>slf4j-log4j12</artifactId>
143+
</exclusion>
144+
<exclusion>
145+
<groupId>org.slf4j</groupId>
146+
<artifactId>slf4j-reload4j</artifactId>
147+
</exclusion>
148+
<exclusion>
149+
<groupId>ch.qos.reload4j</groupId>
150+
<artifactId>reload4j</artifactId>
151+
</exclusion>
152+
<exclusion>
153+
<groupId>commons-logging</groupId>
154+
<artifactId>commons-logging</artifactId>
155+
</exclusion>
156+
</exclusions>
157+
</dependency>
158+
<dependency>
159+
<groupId>org.apache.hadoop</groupId>
160+
<artifactId>hadoop-hdfs</artifactId>
161+
<version>${hadoop.version}</version>
162+
<exclusions>
163+
<exclusion>
164+
<groupId>org.slf4j</groupId>
165+
<artifactId>slf4j-log4j12</artifactId>
166+
</exclusion>
167+
<exclusion>
168+
<groupId>org.slf4j</groupId>
169+
<artifactId>slf4j-reload4j</artifactId>
170+
</exclusion>
171+
<exclusion>
172+
<groupId>ch.qos.reload4j</groupId>
173+
<artifactId>reload4j</artifactId>
174+
</exclusion>
175+
<exclusion>
176+
<groupId>commons-logging</groupId>
177+
<artifactId>commons-logging</artifactId>
178+
</exclusion>
179+
</exclusions>
180+
</dependency>
181+
<dependency>
182+
<groupId>org.apache.hadoop</groupId>
183+
<artifactId>hadoop-mapreduce-client-core</artifactId>
184+
<version>${hadoop.version}</version>
185+
<exclusions>
186+
<exclusion>
187+
<groupId>org.slf4j</groupId>
188+
<artifactId>slf4j-log4j12</artifactId>
189+
</exclusion>
190+
<exclusion>
191+
<groupId>org.slf4j</groupId>
192+
<artifactId>slf4j-reload4j</artifactId>
193+
</exclusion>
194+
<exclusion>
195+
<groupId>ch.qos.reload4j</groupId>
196+
<artifactId>reload4j</artifactId>
197+
</exclusion>
198+
<exclusion>
199+
<groupId>commons-logging</groupId>
200+
<artifactId>commons-logging</artifactId>
201+
</exclusion>
202+
</exclusions>
203+
</dependency>
204+
</dependencies>
205+
<build>
206+
<plugins>
207+
<!-- Suppress source assembly -->
208+
<plugin>
209+
<groupId>org.apache.maven.plugins</groupId>
210+
<artifactId>maven-assembly-plugin</artifactId>
211+
<executions>
212+
<execution>
213+
<id>assemble</id>
214+
<phase>none</phase>
215+
<goals>
216+
<goal>single</goal>
217+
</goals>
218+
</execution>
219+
</executions>
220+
</plugin>
221+
<plugin>
222+
<groupId>org.apache.rat</groupId>
223+
<artifactId>apache-rat-plugin</artifactId>
224+
<executions>
225+
<execution>
226+
<phase>process-resources</phase>
227+
<goals>
228+
<goal>check</goal>
229+
</goals>
230+
</execution>
231+
</executions>
232+
</plugin>
233+
<plugin>
234+
<groupId>org.apache.maven.plugins</groupId>
235+
<artifactId>maven-surefire-plugin</artifactId>
236+
<version>${surefire.version}</version>
237+
</plugin>
238+
<plugin>
239+
<groupId>org.codehaus.mojo</groupId>
240+
<artifactId>exec-maven-plugin</artifactId>
241+
<version>3.1.0</version>
242+
<executions>
243+
<execution>
244+
<phase>test</phase>
245+
<configuration>
246+
<systemProperties>
247+
<systemProperty>
248+
<key>log4j2.debug</key>
249+
<value>false</value>
250+
</systemProperty>
251+
</systemProperties>
252+
</configuration>
253+
</execution>
254+
</executions>
255+
</plugin>
256+
</plugins>
257+
</build>
258+
</project>

0 commit comments

Comments
 (0)