> returned = new TreeMap<>();
+ selected.forEach((k, v) -> returned.put(k, v.export(project == null)));
+ reactions.add(returned);
+ }
@Override
protected void doPut(HttpServletRequest request,
HttpServletResponse response) throws ServletException, IOException {
- security.execute(request, response, PropertyServlet.this::runPut);
- }
- private void runPut(HttpServletRequest request,
- HttpServletResponse response) throws ServletException {
final String ns = getNamespace(request.getRequestURI());
final RawStore ms = getMS();
try {
@@ -294,11 +276,6 @@ private void runPut(HttpServletRequest request,
@Override
protected void doGet(HttpServletRequest request,
HttpServletResponse response) throws ServletException, IOException {
- security.execute(request, response, PropertyServlet.this::runGet);
- }
-
- private void runGet(HttpServletRequest request,
- HttpServletResponse response) throws ServletException {
final String ns = getNamespace(request.getRequestURI());
final RawStore ms = getMS();
try {
@@ -331,42 +308,30 @@ private void runGet(HttpServletRequest request,
}
}
+ public static ServletServerBuilder.Descriptor createServlet(Configuration configuration) {
+ try {
+ int port = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PORT);
+ String path = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
+ if (port >= 0 && path != null && !path.isEmpty()) {
+ ServletSecurity security = new ServletSecurity(configuration);
+ HttpServlet servlet = security.proxy(new PropertyServlet(configuration));
+ return new ServletServerBuilder.Descriptor(port, path, servlet);
+ }
+ } catch (Exception io) {
+ LOGGER.error("Failed to create servlet ", io);
+ }
+ return null;
+ }
+
/**
* Convenience method to start a http server that only serves this servlet.
+ *
* @param conf the configuration
* @return the server instance
* @throws Exception if servlet initialization fails
*/
public static Server startServer(Configuration conf) throws Exception {
- // no port, no server
- int port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PORT);
- if (port < 0) {
- return null;
- }
- String cli = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
- // HTTP Server
- Server server = new Server();
- server.setStopAtShutdown(true);
-
- // Optional SSL
- final SslContextFactory sslContextFactory = ServletSecurity.createSslContextFactory(conf);
- final ServerConnector connector = new ServerConnector(server, sslContextFactory);
- connector.setPort(port);
- connector.setReuseAddress(true);
- server.addConnector(connector);
-
- // Hook the servlet
- ServletHandler handler = new ServletHandler();
- server.setHandler(handler);
- ServletHolder holder = handler.newServletHolder(Source.EMBEDDED);
- holder.setServlet(new PropertyServlet(conf)); //
- handler.addServletWithMapping(holder, "/"+cli+"/*");
- server.start();
- if (!server.isStarted()) {
- LOGGER.error("unable to start property-maps servlet server, path {}, port {}", cli, port);
- } else {
- LOGGER.info("started property-maps servlet server on {}", server.getURI());
- }
- return server;
+ return ServletServerBuilder.startServer(LOGGER, conf, PropertyServlet::createServlet);
}
+
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
index 76181722ca85..d0d48b04df75 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
@@ -33,9 +33,11 @@
import org.slf4j.LoggerFactory;
import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
+import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Enumeration;
@@ -43,27 +45,67 @@
/**
* Secures servlet processing.
+ * This is to be used by servlets that require impersonation through {@link UserGroupInformation#doAs(PrivilegedAction)}
+ * method when providing service. The servlet request header provides user identification
+ * that Hadoop{@literal '}s security uses to perform actions, the
+ * {@link ServletSecurity#execute(HttpServletRequest, HttpServletResponse, ServletSecurity.MethodExecutor)}
+ * method invokes the executor through a {@link PrivilegedAction} in the expected {@link UserGroupInformation} context.
+ *
+ * A typical usage in a servlet is the following:
+ *
+ * ServletSecurity security; // ...
+ * {@literal @}Override protected void doPost(HttpServletRequest request, HttpServletResponse response)
+ * throws ServletException, IOException {
+ * security.execute(request, response, this::runPost);
+ * }
+ * private void runPost(HttpServletRequest request, HttpServletResponse response) throws ServletException {
+ * ...
+ * }
+ *
+ *
+ * As a convenience, instead of embedding the security instance, one can wrap an existing servlet in a proxy that
+ * will ensure all its service methods are called with the expected {@link UserGroupInformation} .
+ *
+ *
+ * HttpServlet myServlet = ...;
+ * ServletSecurity security = ...: ;
+ * Servlet ugiServlet = security.proxy(mySerlvet);
+ * }
+ *
+ *
+ * This implementation performs user extraction and eventual JWT validation to
+ * execute (servlet service) methods within the context of the retrieved UserGroupInformation.
+ *
*/
public class ServletSecurity {
private static final Logger LOG = LoggerFactory.getLogger(ServletSecurity.class);
static final String X_USER = MetaStoreUtils.USER_NAME_HTTP_HEADER;
private final boolean isSecurityEnabled;
private final boolean jwtAuthEnabled;
- private JWTValidator jwtValidator = null;
private final Configuration conf;
+ private JWTValidator jwtValidator = null;
- ServletSecurity(Configuration conf, boolean jwt) {
+ public ServletSecurity(Configuration conf) {
+ this(conf, isAuthJwt(conf));
+ }
+
+ public ServletSecurity(Configuration conf, boolean jwt) {
this.conf = conf;
this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled();
this.jwtAuthEnabled = jwt;
}
+ public static boolean isAuthJwt(Configuration configuration) {
+ String auth = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.PROPERTIES_SERVLET_AUTH);
+ return "jwt".equalsIgnoreCase(auth);
+ }
+
/**
* Should be called in Servlet.init()
* @throws ServletException if the jwt validator creation throws an exception
*/
public void init() throws ServletException {
- if (jwtAuthEnabled) {
+ if (jwtAuthEnabled && jwtValidator == null) {
try {
jwtValidator = new JWTValidator(this.conf);
} catch (Exception e) {
@@ -73,11 +115,73 @@ public void init() throws ServletException {
}
}
+ /**
+ * Proxy a servlet instance service through this security executor.
+ */
+ public class ProxyServlet extends HttpServlet {
+ private final HttpServlet delegate;
+
+ ProxyServlet(HttpServlet delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void init() throws ServletException {
+ ServletSecurity.this.init();
+ delegate.init();
+ }
+
+ @Override
+ public void service(HttpServletRequest request, HttpServletResponse response) throws IOException {
+ execute(request, response, delegate::service);
+ }
+
+ @Override
+ public String getServletName() {
+ try {
+ return delegate.getServletName();
+ } catch (IllegalStateException ill) {
+ return delegate.toString();
+ }
+ }
+
+ @Override
+ public String getServletInfo() {
+ return delegate.getServletInfo();
+ }
+ }
+
+ /**
+ * Creates a proxy servlet.
+ * @param servlet the servlet to serve within this security context
+ * @return a servlet instance or null if security initialization fails
+ */
+ public HttpServlet proxy(HttpServlet servlet) {
+ try {
+ init();
+ } catch (ServletException e) {
+ LOG.error("Unable to proxy security for servlet {}", servlet.toString(), e);
+ return null;
+ }
+ return new ProxyServlet(servlet);
+ }
+
/**
* Any http method executor.
+ * A method whose signature is similar to
+ * {@link HttpServlet#doPost(HttpServletRequest, HttpServletResponse)},
+ * {@link HttpServlet#doGet(HttpServletRequest, HttpServletResponse)},
+ * etc.
*/
@FunctionalInterface
- interface MethodExecutor {
+ public interface MethodExecutor {
+ /**
+ * The method to call to secure the execution of a (http) method.
+ * @param request the request
+ * @param response the response
+ * @throws ServletException if the method executor fails
+ * @throws IOException if the Json in/out fail
+ */
void execute(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException;
}
@@ -86,13 +190,12 @@ interface MethodExecutor {
* @param request the request
* @param response the response
* @param executor the method executor
- * @throws ServletException if the method executor fails
* @throws IOException if the Json in/out fail
*/
public void execute(HttpServletRequest request, HttpServletResponse response, MethodExecutor executor)
- throws ServletException, IOException {
+ throws IOException {
if (LOG.isDebugEnabled()) {
- LOG.debug("Logging headers in "+request.getMethod()+" request");
+ LOG.debug("Logging headers in {} request", request.getMethod());
Enumeration headerNames = request.getHeaderNames();
while (headerNames.hasMoreElements()) {
String headerName = headerNames.nextElement();
@@ -100,9 +203,9 @@ public void execute(HttpServletRequest request, HttpServletResponse response, Me
request.getHeader(headerName));
}
}
+ final UserGroupInformation clientUgi;
try {
String userFromHeader = extractUserName(request, response);
- UserGroupInformation clientUgi;
// Temporary, and useless for now. Here only to allow this to work on an otherwise kerberized
// server.
if (isSecurityEnabled || jwtAuthEnabled) {
@@ -112,25 +215,25 @@ public void execute(HttpServletRequest request, HttpServletResponse response, Me
LOG.info("Creating remote user for: {}", userFromHeader);
clientUgi = UserGroupInformation.createRemoteUser(userFromHeader);
}
- PrivilegedExceptionAction action = () -> {
- executor.execute(request, response);
- return null;
- };
- try {
- clientUgi.doAs(action);
- } catch (InterruptedException e) {
- LOG.error("Exception when executing http request as user: " + clientUgi.getUserName(), e);
- Thread.currentThread().interrupt();
- } catch (RuntimeException e) {
- LOG.error("Exception when executing http request as user: " + clientUgi.getUserName(),
- e);
- throw new ServletException(e);
- }
} catch (HttpAuthenticationException e) {
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
response.getWriter().println("Authentication error: " + e.getMessage());
// Also log the error message on server side
LOG.error("Authentication error: ", e);
+ // no need to go further
+ return;
+ }
+ final PrivilegedExceptionAction action = () -> {
+ executor.execute(request, response);
+ return null;
+ };
+ try {
+ clientUgi.doAs(action);
+ } catch (InterruptedException e) {
+ LOG.info("Interrupted when executing http request as user: {}", clientUgi.getUserName(), e);
+ Thread.currentThread().interrupt();
+ } catch (RuntimeException e) {
+ throw new IOException("Exception when executing http request as user: "+ clientUgi.getUserName(), e);
}
}
@@ -178,7 +281,7 @@ private String extractBearerToken(HttpServletRequest request,
* @param conf the configuration
* @throws IOException if getting the server principal fails
*/
- static void loginServerPincipal(Configuration conf) throws IOException {
+ static void loginServerPrincipal(Configuration conf) throws IOException {
// This check is likely pointless, especially with the current state of the http
// servlet which respects whatever comes in. Putting this in place for the moment
// only to enable testing on an otherwise secure cluster.
@@ -193,6 +296,7 @@ static void loginServerPincipal(Configuration conf) throws IOException {
LOG.info("Security is not enabled. Not logging in via keytab");
}
}
+
/**
* Creates an SSL context factory if configuration states so.
* @param conf the configuration
@@ -204,24 +308,28 @@ static SslContextFactory createSslContextFactory(Configuration conf) throws IOEx
if (!useSsl) {
return null;
}
- String keyStorePath = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PATH).trim();
+ final String keyStorePath = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PATH).trim();
if (keyStorePath.isEmpty()) {
- throw new IllegalArgumentException(MetastoreConf.ConfVars.SSL_KEYSTORE_PATH.toString()
+ throw new IllegalArgumentException(MetastoreConf.ConfVars.SSL_KEYSTORE_PATH
+ " Not configured for SSL connection");
}
- String keyStorePassword =
+ final String keyStorePassword =
MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PASSWORD);
- String keyStoreType =
+ final String keyStoreType =
MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_TYPE).trim();
- String keyStoreAlgorithm =
+ final String keyStoreAlgorithm =
MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_KEYMANAGERFACTORY_ALGORITHM).trim();
-
+ final String[] excludedProtocols =
+ MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_PROTOCOL_BLACKLIST).split(",");
+ if (LOG.isInfoEnabled()) {
+ LOG.info("HTTP Server SSL: adding excluded protocols: {}", Arrays.toString(excludedProtocols));
+ }
SslContextFactory factory = new SslContextFactory.Server();
- String[] excludedProtocols = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_PROTOCOL_BLACKLIST).split(",");
- LOG.info("HTTP Server SSL: adding excluded protocols: " + Arrays.toString(excludedProtocols));
factory.addExcludeProtocols(excludedProtocols);
- LOG.info("HTTP Server SSL: SslContextFactory.getExcludeProtocols = "
- + Arrays.toString(factory.getExcludeProtocols()));
+ if (LOG.isInfoEnabled()) {
+ LOG.info("HTTP Server SSL: SslContextFactory.getExcludeProtocols = {}",
+ Arrays.toString(factory.getExcludeProtocols()));
+ }
factory.setKeyStorePath(keyStorePath);
factory.setKeyStorePassword(keyStorePassword);
factory.setKeyStoreType(keyStoreType);
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
new file mode 100644
index 000000000000..7323845ae35f
--- /dev/null
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
@@ -0,0 +1,347 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.handler.ContextHandlerCollection;
+import org.eclipse.jetty.server.handler.HandlerCollection;
+import org.eclipse.jetty.server.handler.gzip.GzipHandler;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.slf4j.Logger;
+
+import javax.servlet.Servlet;
+import javax.servlet.http.HttpServlet;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Helper class to ease creation of embedded Jetty serving servlets on
+ * different ports.
+ */
+public class ServletServerBuilder {
+ /**
+ * The configuration instance.
+ */
+ private final Configuration configuration;
+ /**
+ * Keeping track of descriptors.
+ */
+ private final Map descriptorsMap = new IdentityHashMap<>();
+
+ /**
+ * Creates a builder instance.
+ *
+ * @param conf the configuration
+ */
+ public ServletServerBuilder(Configuration conf) {
+ this.configuration = conf;
+ }
+
+ /**
+ * Creates a builder.
+ *
+ * @param conf the configuration
+ * @param describe the functions to call that create servlet descriptors
+ * @return the builder or null if no descriptors
+ */
+ @SafeVarargs
+ public static ServletServerBuilder builder(Configuration conf,
+ Function... describe) {
+ List descriptors = new ArrayList<>();
+ Arrays.asList(describe).forEach(functor -> {
+ ServletServerBuilder.Descriptor descriptor = functor.apply(conf);
+ if (descriptor != null) {
+ descriptors.add(descriptor);
+ }
+ });
+ if (!descriptors.isEmpty()) {
+ ServletServerBuilder builder = new ServletServerBuilder(conf);
+ descriptors.forEach(builder::addServlet);
+ return builder;
+ }
+ return null;
+ }
+
+ /**
+ * Helper for generic use case.
+ *
+ * @param logger the logger
+ * @param conf the configuration
+ * @param describe the functions to create descriptors
+ * @return a server instance
+ */
+ @SafeVarargs
+ public static Server startServer(
+ Logger logger,
+ Configuration conf,
+ Function... describe) {
+ return Objects.requireNonNull(builder(conf, describe)).start(logger);
+ }
+
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ /**
+ * Adds a servlet instance.
+ * The servlet port can be shared between servlets; if 0, the system will provide
+ * a port. If the port is < 0, the system will provide a port dedicated (ie non-shared)
+ * to the servlet.
+ *
+ * @param port the servlet port
+ * @param path the servlet path
+ * @param servlet a servlet instance
+ * @return a descriptor
+ */
+ public Descriptor addServlet(int port, String path, HttpServlet servlet) {
+ Descriptor descriptor = new Descriptor(port, path, servlet);
+ return addServlet(descriptor);
+ }
+
+ /**
+ * Adds a servlet instance.
+ *
+ * @param descriptor a descriptor
+ * @return the descriptor
+ */
+ public Descriptor addServlet(Descriptor descriptor) {
+ if (descriptor != null) {
+ descriptorsMap.put(descriptor.getServlet(), descriptor);
+ }
+ return descriptor;
+ }
+
+ /**
+ * Creates a server instance.
+ * Default use configuration to determine thread-pool constants?
+ *
+ * @return the server instance
+ */
+ private Server createServer() {
+ final int maxThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.HTTPSERVER_THREADPOOL_MAX);
+ final int minThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.HTTPSERVER_THREADPOOL_MIN);
+ final int idleTimeout = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.HTTPSERVER_THREADPOOL_IDLE);
+ final QueuedThreadPool threadPool = new QueuedThreadPool(maxThreads, minThreads, idleTimeout);
+ Server server = new Server(threadPool);
+ server.setStopAtShutdown(true);
+ return server;
+ }
+
+ /**
+ * Creates a server instance and a connector on a given port.
+ *
+ * @param server the server instance
+ * @param sslContextFactory the ssl factory
+ * @param port the port
+ * @return the server connector listening to the port
+ */
+ private ServerConnector createConnector(Server server, SslContextFactory sslContextFactory, int port) {
+ final ServerConnector connector = new ServerConnector(server, sslContextFactory);
+ connector.setPort(port);
+ connector.setReuseAddress(true);
+ HttpConnectionFactory httpFactory = connector.getConnectionFactory(HttpConnectionFactory.class);
+ // do not leak information
+ if (httpFactory != null) {
+ HttpConfiguration httpConf = httpFactory.getHttpConfiguration();
+ httpConf.setSendServerVersion(false);
+ httpConf.setSendXPoweredBy(false);
+ }
+ return connector;
+ }
+
+ /**
+ * Adds a servlet to its intended servlet context context.
+ *
+ * @param handlersMap the map of port to handlers
+ * @param descriptor the servlet descriptor
+ */
+ private void addServlet(Map handlersMap, Descriptor descriptor) {
+ final int port = descriptor.getPort();
+ final String path = descriptor.getPath();
+ final HttpServlet servlet = descriptor.getServlet();
+ // if port is < 0, use one for this servlet only
+ int key = port < 0 ? -1 - handlersMap.size() : port;
+ ServletContextHandler handler = handlersMap.computeIfAbsent(key, p -> {
+ ServletContextHandler servletHandler = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
+ servletHandler.setContextPath("/");
+ servletHandler.setGzipHandler(new GzipHandler());
+ return servletHandler;
+ });
+ ServletHolder servletHolder = new ServletHolder(servlet);
+ servletHolder.setInitParameter("javax.ws.rs.Application", "ServiceListPublic");
+ handler.addServlet(servletHolder, "/" + path + "/*");
+ }
+
+ /**
+ * Convenience method to start a http server that serves all configured
+ * servlets.
+ *
+ * @return the server instance or null if no servlet was configured
+ * @throws Exception if servlet initialization fails
+ */
+ public Server startServer() throws Exception {
+ // add all servlets
+ Map handlersMap = new HashMap<>();
+ for (Descriptor descriptor : descriptorsMap.values()) {
+ addServlet(handlersMap, descriptor);
+ }
+ final int size = handlersMap.size();
+ if (size == 0) {
+ return null;
+ }
+ final Server server = createServer();
+ // create the connectors
+ final SslContextFactory sslFactory = ServletSecurity.createSslContextFactory(configuration);
+ final ServerConnector[] connectors = new ServerConnector[size];
+ final ServletContextHandler[] handlers = new ServletContextHandler[size];
+ Iterator> it = handlersMap.entrySet().iterator();
+ for (int c = 0; it.hasNext(); ++c) {
+ Map.Entry entry = it.next();
+ int key = entry.getKey();
+ int port = Math.max(key, 0);
+ ServerConnector connector = createConnector(server, sslFactory, port);
+ connectors[c] = connector;
+ ServletContextHandler handler = entry.getValue();
+ handlers[c] = handler;
+ // make each servlet context be served only by its dedicated connector
+ String host = "hms" + c;
+ connector.setName(host);
+ handler.setVirtualHosts(new String[]{"@" + host});
+ }
+ // hook the connectors and the handlers
+ server.setConnectors(connectors);
+ HandlerCollection portHandler = new ContextHandlerCollection();
+ portHandler.setHandlers(handlers);
+ server.setHandler(portHandler);
+ // start the server
+ server.start();
+ // collect automatically assigned connector ports
+ for (int i = 0; i < connectors.length; ++i) {
+ int port = connectors[i].getLocalPort();
+ ServletContextHandler handler = handlers[i];
+ ServletHolder[] holders = handler.getServletHandler().getServlets();
+ for (ServletHolder holder : holders) {
+ Servlet servlet = holder.getServletInstance();
+ if (servlet != null) {
+ Descriptor descriptor = descriptorsMap.get(servlet);
+ if (descriptor != null) {
+ descriptor.setPort(port);
+ }
+ }
+ }
+ }
+ return server;
+ }
+
+ /**
+ * Creates and starts the server.
+ *
+ * @param logger a logger to output info
+ * @return the server instance (or null if error)
+ */
+ public Server start(Logger logger) {
+ try {
+ Server server = startServer();
+ if (server != null) {
+ if (!server.isStarted()) {
+ logger.error("Unable to start servlet server on {}", server.getURI());
+ } else {
+ descriptorsMap.values().forEach(descriptor -> logger.info("Started {} servlet on {}:{}",
+ descriptor.toString(),
+ descriptor.getPort(),
+ descriptor.getPath()));
+ }
+ }
+ return server;
+ } catch (Throwable throwable) {
+ logger.error("Unable to start servlet server", throwable);
+ return null;
+ }
+ }
+
+ /**
+ * A descriptor of a servlet.
+ * After server is started, unspecified port will be updated to reflect
+ * what the system allocated.
+ */
+ public static class Descriptor {
+ private final String path;
+ private final HttpServlet servlet;
+ private int port;
+
+ /**
+ * Create a servlet descriptor.
+ *
+ * @param port the servlet port (or 0 if system allocated)
+ * @param path the servlet path
+ * @param servlet the servlet instance
+ */
+ public Descriptor(int port, String path, HttpServlet servlet) {
+ this.port = port;
+ this.path = path;
+ this.servlet = servlet;
+ }
+
+ @Override
+ public String toString() {
+ String name = null;
+ try {
+ name = servlet.getServletName() + ":" + port + "/" + path;
+ } catch (IllegalStateException ill) {
+ // ignore, it may happen if servlet config is not set (yet)
+ }
+ if (name == null) {
+ name = servlet.getClass().getSimpleName();
+ }
+ return name + ":" + port + "/" + path;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ void setPort(int port) {
+ this.port = port;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public HttpServlet getServlet() {
+ return servlet;
+ }
+ }
+}
+
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestServletServerBuilder.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestServletServerBuilder.java
new file mode 100644
index 000000000000..e752ef592e26
--- /dev/null
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestServletServerBuilder.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import com.google.gson.Gson;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.Map;
+import java.util.function.Function;
+import javax.servlet.Servlet;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
+import org.eclipse.jetty.server.Server;
+import org.junit.experimental.categories.Category;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.hive.metastore.ServletServerBuilder.Descriptor;
+
+@Category(MetastoreUnitTest.class)
+public class TestServletServerBuilder {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestServletServerBuilder.class);
+
+ private static Function describeServlet(final Map descriptors, int port, String greeting) {
+ return configuration -> {
+ String name = greeting.toLowerCase();
+ HttpServlet s1 = new HelloServlet(greeting) {
+ @Override
+ public String getServletName() {
+ return name + "()";
+ }
+ };
+ Descriptor descriptor = new Descriptor(port, name, s1);
+ descriptors.put(s1.getServletName(), descriptor);
+ return descriptor;
+ };
+ }
+
+ @Test
+ public void testOne() throws Exception {
+ Configuration conf = new Configuration();
+ // keeping track of what is built
+ final Map descriptors = new HashMap();
+ Function fd1 = describeServlet(descriptors, 0, "ONE");
+ Function fd2 = describeServlet(descriptors, 0, "TWO");
+ // the 'conventional' way of starting the server
+ Server server = ServletServerBuilder.startServer(LOG, conf, fd1, fd2);
+
+ Descriptor d1 = descriptors.get("one()");
+ Descriptor d2 = descriptors.get("two()");
+ // same port for both servlets
+ Assert.assertTrue(d1.getPort() > 0);
+ Assert.assertEquals(d1.getPort(), d2.getPort());
+ // check
+ URI uri = URI.create("http://localhost:" + d1.getPort());
+ Object one = clientCall(uri.resolve("/one").toURL());
+ Assert.assertEquals("ONE", one);
+ uri = URI.create("http://localhost:" + d2.getPort());
+ Object two = clientCall(uri.resolve("/two").toURL());
+ Assert.assertEquals("TWO", two);
+ server.stop();
+ }
+
+ @Test
+ public void testOnePort() throws Exception {
+ int port;
+ try (ServerSocket server0 = new ServerSocket(0)) {
+ port = server0.getLocalPort();
+ } catch (IOException xio) {
+ // cant run test if can not get free port
+ return;
+ }
+ onePort(port);
+ }
+
+ @Test
+ public void testOnePortAuto() throws Exception {
+ onePort(0);
+ }
+
+ void onePort(int port) throws Exception {
+ Configuration conf = new Configuration();
+ ServletServerBuilder ssb = new ServletServerBuilder(conf);
+ HttpServlet s1 = new HelloServlet("ONE");
+ HttpServlet s2 = new HelloServlet("TWO");
+ Descriptor d1 = ssb.addServlet(port, "one", s1);
+ Descriptor d2 = ssb.addServlet(port, "two", s2);
+ Server server = ssb.startServer();
+ // same port for both servlets
+ Assert.assertTrue(d1.getPort() > 0);
+ Assert.assertEquals(d1.getPort(), d2.getPort());
+ // check
+ URI uri = URI.create("http://localhost:" + d1.getPort());
+ Object one = clientCall(uri.resolve("/one").toURL());
+ Assert.assertEquals("ONE", one);
+ uri = URI.create("http://localhost:" + d2.getPort());
+ Object two = clientCall(uri.resolve("/two").toURL());
+ Assert.assertEquals("TWO", two);
+ server.stop();
+ }
+
+ @Test
+ public void testTwoPorts() throws Exception {
+ runTwoPorts(-1, -2);
+ }
+
+ @Test
+ public void testTwoPortsAuto() throws Exception {
+ int p0, p1;
+ try (ServerSocket server0 = new ServerSocket(0); ServerSocket server1 = new ServerSocket(0)) {
+ p0 = server0.getLocalPort();
+ p1 = server1.getLocalPort();
+ } catch (IOException xio) {
+ // cant do test if can not get port
+ return;
+ }
+ runTwoPorts(p0, p1);
+ }
+
+ void runTwoPorts(int p1, int p2) throws Exception {
+ Configuration conf = new Configuration();
+ ServletServerBuilder ssb = new ServletServerBuilder(conf);
+ HttpServlet s1 = new HelloServlet("ONE");
+ HttpServlet s2 = new HelloServlet("TWO");
+ Descriptor d1 = ssb.addServlet(p1, "one", s1);
+ Descriptor d2 = ssb.addServlet(p2, "two", s2);
+ Map mappings = new IdentityHashMap<>();
+ Server server = ssb.startServer();
+ // different port for both servlets
+ Assert.assertNotEquals(d1.getPort(), d2.getPort());
+
+ URI uri = URI.create("http://localhost:" + d1.getPort());
+ Object one = clientCall(uri.resolve("/one").toURL());
+ Assert.assertEquals("ONE", one);
+ // fail, not found
+ Object o404 = clientCall(uri.resolve("/two").toURL());
+ Assert.assertEquals(404, o404);
+ uri = URI.create("http://localhost:" + d2.getPort());
+ Object two = clientCall(uri.resolve("/two").toURL());
+ Assert.assertEquals("TWO", two);
+ // fail, not found
+ o404 = clientCall(uri.resolve("/one").toURL());
+ Assert.assertEquals(404, o404);
+ server.stop();
+ }
+
+ static int findFreePort() throws IOException {
+ try (ServerSocket server0 = new ServerSocket(0)) {
+ return server0.getLocalPort();
+ }
+ }
+
+ static int find2FreePort() throws IOException {
+ try (ServerSocket socket0 = new ServerSocket(0)) {
+ return socket0.getLocalPort();
+ }
+ }
+
+ /**
+ * Performs a Json client call.
+ *
+ * @param url the url
+ * @return the result the was returned through Json
+ * @throws IOException if marshalling the request/response fail
+ */
+ static Object clientCall(URL url) throws IOException {
+ HttpURLConnection con = (HttpURLConnection) url.openConnection();
+ con.setRequestMethod("GET");
+ con.setRequestProperty("Content-Type", "application/json");
+ con.setRequestProperty("Accept", "application/json");
+ con.setDoOutput(true);
+ int responseCode = con.getResponseCode();
+ if (responseCode == HttpServletResponse.SC_OK) {
+ try (Reader reader = new BufferedReader(
+ new InputStreamReader(con.getInputStream(), StandardCharsets.UTF_8))) {
+ return new Gson().fromJson(reader, Object.class);
+ }
+ }
+ return responseCode;
+ }
+
+}
+
+class HelloServlet extends HttpServlet {
+
+ final String greeting;
+
+ public HelloServlet() {
+ this("Hello");
+ }
+
+ public HelloServlet(String greeting) {
+ this.greeting = greeting;
+ }
+
+ @Override
+ protected void doGet(HttpServletRequest request,
+ HttpServletResponse response) throws ServletException, IOException {
+ response.setContentType("application/json");
+ response.setStatus(HttpServletResponse.SC_OK);
+ response.getWriter().println(greeting);
+ }
+}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSDirectTest.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSDirectTest.java
index 7c3c77451649..3e7c0cd2e600 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSDirectTest.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSDirectTest.java
@@ -17,32 +17,33 @@
*/
package org.apache.hadoop.hive.metastore.properties;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.HMSHandler;
import org.apache.hadoop.hive.metastore.ObjectStore;
import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.TreeMap;
-
import static org.apache.hadoop.hive.metastore.properties.PropertyType.DATETIME;
import static org.apache.hadoop.hive.metastore.properties.PropertyType.DOUBLE;
import static org.apache.hadoop.hive.metastore.properties.PropertyType.INTEGER;
import static org.apache.hadoop.hive.metastore.properties.PropertyType.STRING;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
/**
* In-process property manager test.
*/
+@Category(MetastoreUnitTest.class)
public class HMSDirectTest extends HMSTestBase {
protected ObjectStore objectStore = null;
static Random RND = new Random(20230424);
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java
index 75f670409225..f196d66c33b3 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java
@@ -18,8 +18,25 @@
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.PropertyServlet;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
@@ -34,56 +51,52 @@
import org.apache.http.message.BasicNameValuePair;
import org.eclipse.jetty.server.Server;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
-import javax.servlet.http.HttpServletResponse;
-import java.io.BufferedReader;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.net.URL;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
+@Category(MetastoreUnitTest.class)
public class HMSServletTest extends HMSTestBase {
- protected static final String CLI = "hmscli";
+ String path = null;
Server servletServer = null;
- int sport = -1;
-
+ int servletPort = -1;
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
+ }
- @Override protected int createServer(Configuration conf) throws Exception {
+ @Override
+ protected int createServer(Configuration conf) throws Exception {
if (servletServer == null) {
servletServer = PropertyServlet.startServer(conf);
if (servletServer == null || !servletServer.isStarted()) {
Assert.fail("http server did not start");
}
- sport = servletServer.getURI().getPort();
+ servletPort = servletServer.getURI().getPort();
}
- return sport;
+ return servletPort;
}
/**
* Stops the server.
* @param port the server port
*/
- @Override protected void stopServer(int port) throws Exception {
+ @Override
+ protected void stopServer(int port) throws Exception {
if (servletServer != null) {
servletServer.stop();
servletServer = null;
- sport = -1;
+ servletPort = -1;
}
}
+
@Override
protected PropertyClient createClient(Configuration conf, int sport) throws Exception {
- URL url = new URL("http://hive@localhost:" + sport + "/" + CLI + "/" + NS);
+ String path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
+ URL url = new URL("http://hive@localhost:" + sport + "/" + path + "/" + NS);
String jwt = generateJWT();
return new JSonClient(jwt, url);
}
@@ -143,7 +156,7 @@ public Map getProperties(List selection) {
@Test
public void testServletEchoA() throws Exception {
- URL url = new URL("http://hive@localhost:" + sport + "/" + CLI + "/" + NS);
+ URL url = new URL("http://hive@localhost:" + servletPort + "/" + path + "/" + NS);
Map json = Collections.singletonMap("method", "echo");
String jwt = generateJWT();
// succeed
@@ -175,8 +188,8 @@ public void testProperties0() throws Exception {
.setScheme("http")
.setUserInfo("hive")
.setHost("localhost")
- .setPort(sport)
- .setPath("/" + CLI + "/" + NS)
+ .setPort(servletPort)
+ .setPath("/" + path + "/" + NS)
.setParameters(nvp)
.build();
HttpGet get = new HttpGet(uri);
@@ -292,7 +305,7 @@ public static Object clientCall(String jwt, URL url, String method, Object arg)
* @throws Exception
*/
private HttpPost createPost(String jwt, String msgBody) {
- HttpPost method = new HttpPost("http://hive@localhost:" + sport + "/" + CLI + "/" + NS);
+ HttpPost method = new HttpPost("http://hive@localhost:" + servletPort + "/" + path + "/" + NS);
method.addHeader("Authorization", "Bearer " + jwt);
method.addHeader("Content-Type", "application/json");
method.addHeader("Accept", "application/json");
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1.java
index db15d52e12d4..b1c8b803dff9 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1.java
@@ -18,7 +18,19 @@
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
@@ -29,30 +41,23 @@
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.HttpClients;
+import org.junit.experimental.categories.Category;
-import javax.servlet.http.HttpServletResponse;
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.net.URL;
-import java.nio.charset.Charset;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
+@Category(MetastoreUnitTest.class)
public class HMSServletTest1 extends HMSServletTest {
@Override
public void tearDown() throws Exception {
if (client instanceof AutoCloseable) {
((AutoCloseable) client).close();
+ client = null;
}
super.tearDown();
}
@Override
protected PropertyClient createClient(Configuration conf, int sport) throws Exception {
- URL url = new URL("http://hive@localhost:" + sport + "/" + CLI + "/" + NS);
+ String path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
+ URL url = new URL("http://hive@localhost:" + sport + "/" + path + "/" + NS);
String jwt = generateJWT();
return new JSonHttpClient(jwt, url.toString());
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java
index 5ff45d90dd82..fd58d53e1f19 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java
@@ -16,22 +16,23 @@
*/
package org.apache.hadoop.hive.metastore.properties;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.ok;
+import java.nio.file.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
import org.junit.Assert;
-
-import java.nio.file.Files;
-
-import static com.github.tomakehurst.wiremock.client.WireMock.get;
-import static com.github.tomakehurst.wiremock.client.WireMock.ok;
+import org.junit.experimental.categories.Category;
/**
* Test using the servlet server created by the MetaStore and
* the client based on Apache HttpClient.
*/
+@Category(MetastoreUnitTest.class)
public class HMSServletTest1A extends HMSServletTest1 {
protected int thriftPort;
@@ -45,12 +46,12 @@ protected int createServer(Configuration conf) throws Exception {
.willReturn(ok()
.withBody(Files.readAllBytes(jwtVerificationJWKSFile.toPath()))));
thriftPort = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf);
- servletServer = HiveMetaStore.getPropertyServer();
+ servletServer = HiveMetaStore.getServletServer();
if (servletServer == null || !servletServer.isStarted()) {
Assert.fail("http server did not start");
}
- sport = servletServer.getURI().getPort();
- return sport;
+ servletPort = HiveMetaStore.getPropertyServletPort();
+ return servletPort;
}
@Override
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java
index 10a54457ab12..3a8fb16028f0 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java
@@ -16,20 +16,22 @@
*/
package org.apache.hadoop.hive.metastore.properties;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.ok;
+import java.nio.file.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
import org.junit.Assert;
-import java.nio.file.Files;
-
-import static com.github.tomakehurst.wiremock.client.WireMock.get;
-import static com.github.tomakehurst.wiremock.client.WireMock.ok;
+import org.junit.experimental.categories.Category;
/**
* Test using the servlet server created by the MetaStore.
*/
+@Category(MetastoreUnitTest.class)
public class HMSServletTestA extends HMSServletTest {
protected int thriftPort;
@@ -43,12 +45,12 @@ protected int createServer(Configuration conf) throws Exception {
.willReturn(ok()
.withBody(Files.readAllBytes(jwtVerificationJWKSFile.toPath()))));
thriftPort = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf);
- servletServer = HiveMetaStore.getPropertyServer();
+ servletServer = HiveMetaStore.getServletServer();
if (servletServer == null || !servletServer.isStarted()) {
Assert.fail("http server did not start");
}
- sport = servletServer.getURI().getPort();
- return sport;
+ servletPort = HiveMetaStore.getPropertyServletPort();
+ return servletPort;
}
@Override
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
index 30635e8bbeee..cded98bd07cc 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hive.metastore.properties;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.ok;
import com.github.tomakehurst.wiremock.junit.WireMockRule;
import com.nimbusds.jose.JWSAlgorithm;
import com.nimbusds.jose.JWSHeader;
@@ -25,25 +27,6 @@
import com.nimbusds.jose.jwk.RSAKey;
import com.nimbusds.jwt.JWTClaimsSet;
import com.nimbusds.jwt.SignedJWT;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.jexl3.JxltEngine;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
-import org.apache.hadoop.hive.metastore.ObjectStore;
-import org.apache.hadoop.hive.metastore.TestObjectStore;
-
-import static com.github.tomakehurst.wiremock.client.WireMock.get;
-import static com.github.tomakehurst.wiremock.client.WireMock.ok;
-import static org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MaintenanceOpStatus;
-import static org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MaintenanceOpType;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.ClassRule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.IOException;
import java.io.StringWriter;
@@ -58,16 +41,29 @@
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.JEXL;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.jexl3.JxltEngine;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.ObjectStore;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import static org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MAINTENANCE_OPERATION;
import static org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MAINTENANCE_STATUS;
+import org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MaintenanceOpStatus;
+import org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MaintenanceOpType;
+import static org.apache.hadoop.hive.metastore.properties.PropertyManager.JEXL;
import static org.apache.hadoop.hive.metastore.properties.PropertyType.BOOLEAN;
import static org.apache.hadoop.hive.metastore.properties.PropertyType.DATETIME;
import static org.apache.hadoop.hive.metastore.properties.PropertyType.DOUBLE;
import static org.apache.hadoop.hive.metastore.properties.PropertyType.INTEGER;
import static org.apache.hadoop.hive.metastore.properties.PropertyType.JSON;
import static org.apache.hadoop.hive.metastore.properties.PropertyType.STRING;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public abstract class HMSTestBase {
protected static final String baseDir = System.getProperty("basedir");
@@ -87,12 +83,12 @@ public abstract class HMSTestBase {
/**
* Abstract the property client access on a given namespace.
*/
- interface PropertyClient {
+ protected interface PropertyClient {
boolean setProperties(Map properties);
Map> getProperties(String mapPrefix, String mapPredicate, String... selection) throws IOException;
}
- interface HttpPropertyClient extends PropertyClient {
+ protected interface HttpPropertyClient extends PropertyClient {
default Map getProperties(List selection) throws IOException {
throw new UnsupportedOperationException("not implemented in " + this.getClass());
}
@@ -100,7 +96,7 @@ default Map getProperties(List selection) throws IOExcep
protected Configuration conf = null;
- protected static final Logger LOG = LoggerFactory.getLogger(TestObjectStore.class.getName());
+ protected static final Logger LOG = LoggerFactory.getLogger(HMSTestBase.class);
static Random RND = new Random(20230424);
protected String NS;// = "hms" + RND.nextInt(100);
protected PropertyClient client;
@@ -111,7 +107,6 @@ public void setUp() throws Exception {
NS = "hms" + RND.nextInt(100);
conf = MetastoreConf.newMetastoreConf();
MetaStoreTestUtils.setConfForStandloneMode(conf);
-
MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST, true);
// Events that get cleaned happen in batches of 1 to exercise batching code
MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.EVENT_CLEAN_MAX_EVENTS, 1L);
@@ -184,9 +179,9 @@ private static String generateJWT(String user, Path keyFile, long lifeTimeMillis
/**
* Creates and starts the server.
- * @param conf
+ * @param conf the configuration
* @return the server port
- * @throws Exception
+ * @throws Exception if creation fails
*/
protected int createServer(Configuration conf) throws Exception {
return 0;
@@ -195,7 +190,7 @@ protected int createServer(Configuration conf) throws Exception {
/**
* Stops the server.
* @param port the server port
- * @throws Exception
+ * @throws Exception if stopping the server fails
*/
protected void stopServer(int port) throws Exception {
// nothing
@@ -203,13 +198,15 @@ protected void stopServer(int port) throws Exception {
/**
* Creates a client.
+ * @param conf the configuration
+ * @param port the servlet port
* @return the client instance
- * @throws Exception
+ * @throws Exception if client creation fails
*/
protected abstract PropertyClient createClient(Configuration conf, int port) throws Exception;
- public void runOtherProperties0(PropertyClient client) throws Exception {
+ void runOtherProperties0(PropertyClient client) throws Exception {
Map ptyMap = createProperties0();
boolean commit = client.setProperties(ptyMap);
Assert.assertTrue(commit);
@@ -236,7 +233,7 @@ static Map createProperties0() {
try {
String json = IOUtils.toString(
HMSDirectTest.class.getResourceAsStream("payload.json"),
- "UTF-8"
+ StandardCharsets.UTF_8
);
JxltEngine JXLT = JEXL.createJxltEngine();
JxltEngine.Template jsonjexl = JXLT.createTemplate(json, "table", "delta", "g");
@@ -265,7 +262,7 @@ static Map createProperties0() {
}
}
- public void runOtherProperties1(PropertyClient client) throws Exception {
+ void runOtherProperties1(PropertyClient client) throws Exception {
Map ptyMap = createProperties1();
boolean commit = client.setProperties(ptyMap);
Assert.assertTrue(commit);
@@ -278,12 +275,11 @@ public void runOtherProperties1(PropertyClient client) throws Exception {
HttpPropertyClient httpClient = (HttpPropertyClient) client;
// get fillfactors using getProperties, create args array from previous result
List keys = new ArrayList<>(maps.keySet());
- for (int k = 0; k < keys.size(); ++k) {
- keys.set(k, keys.get(k) + ".fillFactor");
- }
+ keys.replaceAll(s -> s + ".fillFactor");
Object values = httpClient.getProperties(keys);
Assert.assertTrue(values instanceof Map);
- Map getm = (Map) values;
+ @SuppressWarnings("unchecked")
+ final Map getm = (Map) values;
for (Map.Entry> entry : maps.entrySet()) {
Map map0v = entry.getValue();
Assert.assertEquals(map0v.get("fillFactor"), getm.get(entry.getKey() + ".fillFactor"));
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSThriftTest.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSThriftTest.java
index 33354ad17b54..b7fa65d6d771 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSThriftTest.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSThriftTest.java
@@ -17,17 +17,19 @@
*/
package org.apache.hadoop.hive.metastore.properties;
+import java.io.IOException;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
import org.apache.thrift.TException;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
-import java.io.IOException;
-import java.util.Map;
-
+@Category(MetastoreUnitTest.class)
public class HMSThriftTest extends HMSTestBase {
/**
* A Thrift based property client.
@@ -67,16 +69,11 @@ public Map> getProperties(String mapPrefix, String m
MetaStoreTestUtils.close(port);
}
- /**
- * Creates a client.
- * @return the client instance
- * @throws Exception
- */
@Override protected PropertyClient createClient(Configuration conf, int port) throws Exception {
MetastoreConf.setVar(conf, MetastoreConf.ConfVars.THRIFT_URIS, "http://localhost:" + port);
MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.EXECUTE_SET_UGI, false);
- HiveMetaStoreClient client = new HiveMetaStoreClient(conf);
- return new ThriftPropertyClient(NS, client);
+ HiveMetaStoreClient hiveClient = new HiveMetaStoreClient(conf);
+ return new ThriftPropertyClient(NS, hiveClient);
}
@Test
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/PropertyStoreTest.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/PropertyStoreTest.java
index 50ab770aaabf..1ef1c2119194 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/PropertyStoreTest.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/PropertyStoreTest.java
@@ -18,12 +18,14 @@
package org.apache.hadoop.hive.metastore.properties;
import com.google.common.base.Supplier;
+import java.nio.charset.StandardCharsets;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.HMSHandler;
import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
import org.apache.hadoop.hive.metastore.ObjectStore;
import org.apache.hadoop.hive.metastore.TestObjectStore;
import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.model.MMetastoreDBProperties;
@@ -31,11 +33,11 @@
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.nio.charset.StandardCharsets;
-
+@Category(MetastoreUnitTest.class)
public class PropertyStoreTest {
private ObjectStore objectStore = null;
private Configuration conf;
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index 6b5cb3e0e542..ae8331642b7e 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -29,6 +29,7 @@
metastore-common
metastore-server
metastore-tools
+ metastore-rest-catalog
4.1.0-SNAPSHOT
@@ -120,6 +121,7 @@
${basedir}/src/gen/thrift
-I ${thrift.home} -strict --gen java:beans,generated_annotations=undated --gen cpp --gen php --gen py --gen rb
+ 2.32.0
2024-01-01T00:00:00Z