diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml index 60288f495191..98f7c4e1a06f 100644 --- a/hbase-http/pom.xml +++ b/hbase-http/pom.xml @@ -78,7 +78,11 @@ org.apache.hbase.thirdparty - hbase-shaded-jetty + hbase-shaded-jetty-12-plus-core + + + org.apache.hbase.thirdparty + hbase-shaded-jetty-12-plus-ee8 org.apache.hbase.thirdparty diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/AdminAuthorizedServlet.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/AdminAuthorizedServlet.java index 2ad09b5ae5c7..1ecd75532ce6 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/AdminAuthorizedServlet.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/AdminAuthorizedServlet.java @@ -24,7 +24,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.DefaultServlet; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.DefaultServlet; /** * General servlet which is admin-authorized. diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java index 6012b24ec543..fe2a9a48c210 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java @@ -72,6 +72,12 @@ import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.DefaultServlet; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.FilterHolder; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.FilterMapping; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletContextHandler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.webapp.WebAppContext; import org.apache.hbase.thirdparty.org.eclipse.jetty.http.HttpVersion; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Handler; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConfiguration; @@ -84,18 +90,10 @@ import org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.ErrorHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.HandlerCollection; -import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.RequestLogHandler; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.gzip.GzipHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.DefaultServlet; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.FilterHolder; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.FilterMapping; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletContextHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder; -import org.apache.hbase.thirdparty.org.eclipse.jetty.util.MultiException; +import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ExceptionUtil; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ssl.SslContextFactory; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool; -import org.apache.hbase.thirdparty.org.eclipse.jetty.webapp.WebAppContext; import org.apache.hbase.thirdparty.org.glassfish.jersey.server.ResourceConfig; import org.apache.hbase.thirdparty.org.glassfish.jersey.servlet.ServletContainer; @@ -654,23 +652,21 @@ private void initializeWebServer(String name, String hostName, Configuration con Preconditions.checkNotNull(webAppContext); - HandlerCollection handlerCollection = new HandlerCollection(); + Handler.Sequence handlers = new Handler.Sequence(); ContextHandlerCollection contexts = new ContextHandlerCollection(); RequestLog requestLog = HttpRequestLog.getRequestLog(name); if (requestLog != null) { - RequestLogHandler requestLogHandler = new RequestLogHandler(); - requestLogHandler.setRequestLog(requestLog); - handlerCollection.addHandler(requestLogHandler); + webServer.setRequestLog(requestLog); } final String appDir = getWebAppsPath(name); - handlerCollection.addHandler(contexts); - handlerCollection.addHandler(webAppContext); + handlers.addHandler(contexts); + handlers.addHandler(webAppContext); - webServer.setHandler(handlerCollection); + webServer.setHandler(handlers); webAppContext.setAttribute(ADMINS_ACL, adminsAcl); @@ -715,8 +711,9 @@ private void initializeWebServer(String name, String hostName, Configuration con // Check if disable stack trace property is configured if (!conf.getBoolean(HTTP_UI_SHOW_STACKTRACE_KEY, true)) { // Disable stack traces for server errors in UI - webServer.setErrorHandler(new ErrorHandler()); - webServer.getErrorHandler().setShowStacks(false); + ErrorHandler errorHandler = new ErrorHandler(); + errorHandler.setShowStacks(false); + webServer.setErrorHandler(errorHandler); // Disable stack traces for web app errors in UI webAppContext.getErrorHandler().setShowStacks(false); } @@ -841,7 +838,8 @@ private void configureAliasChecks(ServletContextHandler context, boolean shouldS if (context.getAliasChecks().stream().anyMatch(aliasCheckerClass::isInstance)) { LOG.debug("{} is already part of alias check list", aliasCheckerClass.getName()); } else { - context.addAliasCheck(new SymlinkAllowedResourceAliasChecker(context)); + context + .addAliasCheck(new SymlinkAllowedResourceAliasChecker(context.getCoreContextHandler())); LOG.debug("{} added to the alias check list", aliasCheckerClass.getName()); } LOG.info("Serving aliases allowed for /logs context"); @@ -1258,14 +1256,14 @@ public void start() throws IOException { } catch (IOException ex) { LOG.info("HttpServer.start() threw a non Bind IOException", ex); throw ex; - } catch (MultiException ex) { - LOG.info("HttpServer.start() threw a MultiException", ex); + } catch (Exception ex) { + LOG.info("HttpServer.start() threw a Exception", ex); throw ex; } // Make sure there is no handler failures. - Handler[] handlers = webServer.getHandlers(); - for (int i = 0; i < handlers.length; i++) { - if (handlers[i].isFailed()) { + List handlers = webServer.getHandlers(); + for (Handler handler : handlers) { + if (handler.isFailed()) { throw new IOException("Problem in starting http server. Server handlers failed"); } } @@ -1335,7 +1333,7 @@ void openListeners() throws Exception { * stop the server */ public void stop() throws Exception { - MultiException exception = null; + ExceptionUtil.MultiException exception = null; for (ListenerInfo li : listeners) { if (!li.isManaged) { continue; @@ -1372,9 +1370,10 @@ public void stop() throws Exception { } - private MultiException addMultiException(MultiException exception, Exception e) { + private ExceptionUtil.MultiException addMultiException(ExceptionUtil.MultiException exception, + Exception e) { if (exception == null) { - exception = new MultiException(); + exception = new ExceptionUtil.MultiException(); } exception.add(e); return exception; diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServerUtil.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServerUtil.java index ecfb32742fd1..0f1c9707d1c2 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServerUtil.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServerUtil.java @@ -22,11 +22,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hbase.thirdparty.org.eclipse.jetty.security.ConstraintMapping; -import org.apache.hbase.thirdparty.org.eclipse.jetty.security.ConstraintSecurityHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.FilterHolder; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletContextHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.util.security.Constraint; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.nested.ServletConstraint; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.security.ConstraintMapping; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.security.ConstraintSecurityHandler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.FilterHolder; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletContextHandler; /** * HttpServer utility. @@ -43,7 +43,7 @@ public final class HttpServerUtil { */ public static void constrainHttpMethods(ServletContextHandler ctxHandler, boolean allowOptionsMethod) { - Constraint c = new Constraint(); + ServletConstraint c = new ServletConstraint(); c.setAuthenticate(true); ConstraintMapping cmt = new ConstraintMapping(); diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java index ea73be808f07..5a09315ed774 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java @@ -29,7 +29,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.apache.hbase.thirdparty.com.google.common.net.HostAndPort; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder; /** * Create a Jetty embedded server to answer http requests. The primary goal is to serve up status diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileOutputServlet.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileOutputServlet.java index d92e7d009f68..12d0c4625044 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileOutputServlet.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/ProfileOutputServlet.java @@ -27,7 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.DefaultServlet; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.DefaultServlet; /** * Servlet to serve files generated by {@link ProfileServlet} diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java index a9e8fa7cbe16..915f7e299183 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java @@ -41,7 +41,6 @@ import org.apache.hadoop.security.authentication.client.AuthenticatedURL; import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; import org.apache.hadoop.security.ssl.SSLFactory; -import org.apache.hadoop.util.HttpExceptionUtils; import org.apache.hadoop.util.ServletUtil; import org.apache.hadoop.util.Tool; import org.apache.yetus.audience.InterfaceAudience; @@ -267,7 +266,11 @@ private void process(String urlString) throws Exception { HttpURLConnection connection = connect(url); - HttpExceptionUtils.validateResponse(connection, 200); + // We now use the validateResponse method of hbase to handle for HTML response, + // as with Jetty 12: getResponseMessage() returns "Precondition Failed" vs + // "Modification of logger protected.org.apache.hadoop.hbase.http.log.TestLogLevel is + // disallowed in configuration" in Jetty 9 + LogLevelExceptionUtils.validateResponse(connection, 200); // read from the servlet diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java new file mode 100644 index 000000000000..2776c9b4313b --- /dev/null +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevelExceptionUtils.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.http.log; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * HTTP utility class to help propagate server side exception in log level servlet to the client + * over HTTP (HTML payload) It parses HTTP client connections and recreates the exception. + */ +@InterfaceAudience.Private +public class LogLevelExceptionUtils { + + private static void throwEx(Throwable ex) { + LogLevelExceptionUtils. throwException(ex); + } + + @SuppressWarnings("unchecked") + private static void throwException(Throwable ex) throws E { + throw (E) ex; + } + + /** + * Validates the status of an HttpURLConnection against an expected HTTP status code. + * If the current status code is not the expected one it throws an exception with a detail message + * using Server side error messages if available. + *

+ * NOTE: This is an adapted version of the original method in HttpServerUtil.java of Hadoop, + * but we handle for HTML response. + * @param conn the HttpURLConnection. + * @param expectedStatus the expected HTTP status code. + * @throws IOException thrown if the current status code does not match the expected one. + */ + @SuppressWarnings("unchecked") + public static void validateResponse(HttpURLConnection conn, int expectedStatus) + throws IOException { + if (conn.getResponseCode() != expectedStatus) { + Exception toThrow = null; + + try (InputStream es = conn.getErrorStream()) { + if (es != null) { + try (InputStreamReader isr = new InputStreamReader(es, StandardCharsets.UTF_8); + BufferedReader reader = new BufferedReader(isr)) { + final String errorAsHtml = reader.lines().collect(Collectors.joining("\n")); + + final String status = extractValue(errorAsHtml, "STATUS:(\\d+)"); + final String message = extractValue(errorAsHtml, "MESSAGE:([^<]+)"); + final String uri = extractValue(errorAsHtml, "URI:([^<]+)"); + final String exception = extractValue(errorAsHtml, "([^<]+)"); + + toThrow = new IOException( + String.format("HTTP status [%s], message [%s], URL [%s], exception [%s]", status, + message, uri, exception)); + } + } + } catch (Exception ex) { + toThrow = + new IOException(String.format("HTTP status [%d], message [%s], URL [%s], exception [%s]", + conn.getResponseCode(), conn.getResponseMessage(), conn.getURL(), ex), ex); + } + if (toThrow != null) { + throwEx(toThrow); + } + } + } + + private static String extractValue(String html, String regex) { + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(html); + if (matcher.find()) { + return matcher.group(1); + } + return null; + } + +} diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java index ad9c9d3a0671..724663928271 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java @@ -507,7 +507,7 @@ public void testRequestQuoterWithNotNull() { @SuppressWarnings("unchecked") private static Map parse(String jsonString) { - return (Map) JSON.parse(jsonString); + return (Map) new JSON().fromJSON(jsonString); } @Test @@ -615,6 +615,9 @@ private HttpServer checkBindAddress(String host, int port, boolean findPort) thr ServerConnector listener = server.getServerConnectors().get(0); assertEquals(port, listener.getPort()); + // We are doing this as otherwise testBindAddress fails, not sure how we were even starting + // server in jetty 9 without this call + server.start(); // verify hostname is what was given server.openListeners(); assertEquals(host, server.getConnectorAddress(0).getHostName()); diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/conf/TestConfServlet.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/conf/TestConfServlet.java index fe202906e9c5..dc14be96d404 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/conf/TestConfServlet.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/conf/TestConfServlet.java @@ -73,7 +73,7 @@ public void testWriteJson() throws Exception { Set programSet = new HashSet<>(); programSet.add("programatically"); programSet.add("programmatically"); - Object parsed = JSON.parse(json); + Object parsed = new JSON().fromJSON(json); Object[] properties = ((Map) parsed).get("properties"); for (Object o : properties) { Map propertyInfo = (Map) o; diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java index bc45a6295551..04452f613bae 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java @@ -235,7 +235,10 @@ private boolean validateCommand(String[] args) { * @throws Exception if unable to create or start a Jetty server */ private HttpServer createServer(String protocol, boolean isSpnego) throws Exception { - HttpServer.Builder builder = new HttpServer.Builder().setName("..") + // Changed to "" as ".." moves it a steps back in path because the path is relative to the + // current working directory. throws "java.lang.IllegalArgumentException: Base Resource is not + // valid: hbase-http/target/test-classes/static" as it is not able to find the static folder. + HttpServer.Builder builder = new HttpServer.Builder().setName("") .addEndpoint(new URI(protocol + "://localhost:0")).setFindPort(true).setConf(serverConf); if (isSpnego) { // Set up server Kerberos credentials. diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/resource/JerseyResource.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/resource/JerseyResource.java index 89d71b403af7..4e97dcfae3fa 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/resource/JerseyResource.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/resource/JerseyResource.java @@ -54,7 +54,7 @@ public Response get(@PathParam(PATH) @DefaultValue("UNKNOWN_" + PATH) final Stri final Map m = new TreeMap<>(); m.put(PATH, path); m.put(OP, op); - final String js = JSON.toString(m); + final String js = new JSON().toJSON(m); return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java index 5817d071f02c..be50b3137256 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/MockHttpApiRule.java @@ -17,28 +17,31 @@ */ package org.apache.hadoop.hbase; -import java.io.IOException; -import java.io.PrintWriter; import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.BiConsumer; import java.util.regex.Pattern; -import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.junit.rules.ExternalResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.javax.ws.rs.core.HttpHeaders; import org.apache.hbase.thirdparty.javax.ws.rs.core.MediaType; +import org.apache.hbase.thirdparty.org.eclipse.jetty.server.CustomRequestLog; +import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Handler; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Request; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.RequestLog; +import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Response; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector; -import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Slf4jRequestLog; -import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.AbstractHandler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Slf4jRequestLogWriter; +import org.apache.hbase.thirdparty.org.eclipse.jetty.util.Callback; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.RegexSet; /** @@ -55,7 +58,7 @@ public class MockHttpApiRule extends ExternalResource { * Register a callback handler for the specified path target. */ public MockHttpApiRule addRegistration(final String pathRegex, - final BiConsumer responder) { + final BiConsumer responder) { handler.register(pathRegex, responder); return this; } @@ -65,16 +68,21 @@ public MockHttpApiRule addRegistration(final String pathRegex, */ public MockHttpApiRule registerOk(final String pathRegex, final String responseBody) { return addRegistration(pathRegex, (target, resp) -> { - try { - resp.setStatus(HttpServletResponse.SC_OK); - resp.setCharacterEncoding("UTF-8"); - resp.setContentType(MediaType.APPLICATION_JSON_TYPE.toString()); - final PrintWriter writer = resp.getWriter(); - writer.write(responseBody); - writer.flush(); - } catch (IOException e) { - throw new RuntimeException(e); - } + resp.setStatus(HttpServletResponse.SC_OK); + resp.getHeaders().put(HttpHeaders.CONTENT_ENCODING, "UTF-8"); + resp.getHeaders().put(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_TYPE.toString()); + ByteBuffer content = ByteBuffer.wrap(responseBody.getBytes(StandardCharsets.UTF_8)); + resp.write(true, content, new Callback() { + @Override + public void succeeded() { + // nothing to do + } + + @Override + public void failed(Throwable x) { + throw new RuntimeException(x); + } + }); }); } @@ -115,20 +123,18 @@ protected void after() { } private static RequestLog buildRequestLog() { - final Slf4jRequestLog requestLog = new Slf4jRequestLog(); - requestLog.setLoggerName(LOG.getName() + ".RequestLog"); - requestLog.setExtended(true); - return requestLog; + Slf4jRequestLogWriter writer = new Slf4jRequestLogWriter(); + writer.setLoggerName(LOG.getName() + ".RequestLog"); + return new CustomRequestLog(writer, CustomRequestLog.EXTENDED_NCSA_FORMAT); } - private static class MockHandler extends AbstractHandler { + private static class MockHandler extends Handler.Abstract { private final ReadWriteLock responseMappingLock = new ReentrantReadWriteLock(); - private final Map> responseMapping = - new HashMap<>(); + private final Map> responseMapping = new HashMap<>(); private final RegexSet regexSet = new RegexSet(); - void register(final String pathRegex, final BiConsumer responder) { + void register(final String pathRegex, final BiConsumer responder) { LOG.debug("Registering responder to '{}'", pathRegex); responseMappingLock.writeLock().lock(); try { @@ -151,20 +157,25 @@ void clearRegistrations() { } @Override - public void handle(final String target, final Request baseRequest, - final HttpServletRequest request, final HttpServletResponse response) { + public boolean handle(Request request, Response response, Callback callback) throws Exception { + String target = request.getHttpURI().getPath(); responseMappingLock.readLock().lock(); try { if (!regexSet.matches(target)) { response.setStatus(HttpServletResponse.SC_NOT_FOUND); - return; + callback.succeeded(); + return true; } responseMapping.entrySet().stream().filter(e -> Pattern.matches(e.getKey(), target)) .findAny().map(Map.Entry::getValue).orElseThrow(() -> noMatchFound(target)) .accept(target, response); + callback.succeeded(); + } catch (Exception e) { + callback.failed(e); } finally { responseMappingLock.readLock().unlock(); } + return true; } private static RuntimeException noMatchFound(final String target) { diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java index 760f2ca8b41c..666c994d6924 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RESTServer.java @@ -23,8 +23,10 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import javax.servlet.DispatcherType; import org.apache.commons.lang3.ArrayUtils; @@ -57,7 +59,11 @@ import org.apache.hbase.thirdparty.org.apache.commons.cli.Options; import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException; import org.apache.hbase.thirdparty.org.apache.commons.cli.PosixParser; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.FilterHolder; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletContextHandler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder; import org.apache.hbase.thirdparty.org.eclipse.jetty.http.HttpVersion; +import org.apache.hbase.thirdparty.org.eclipse.jetty.http.UriCompliance; import org.apache.hbase.thirdparty.org.eclipse.jetty.jmx.MBeanContainer; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConfiguration; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConnectionFactory; @@ -65,9 +71,6 @@ import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.SslConnectionFactory; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.FilterHolder; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletContextHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ssl.SslContextFactory; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool; import org.apache.hbase.thirdparty.org.glassfish.jersey.server.ResourceConfig; @@ -287,6 +290,13 @@ public synchronized void run() throws Exception { httpConfig.setSendServerVersion(false); httpConfig.setSendDateHeader(false); + // In Jetty 12, ambiguous path separators, suspicious path characters, and ambiguous empty + // segments are considered violations of the URI specification and hence are not allowed. + // Refer to https://github.com/jetty/jetty.project/issues/11890#issuecomment-2156449534 + // We must set a URI compliance to allow for this violation so that client requests are not + // automatically rejected. Our rest endpoints rely on this behavior to handle encoded uri paths. + setUriComplianceRules(httpConfig); + ServerConnector serverConnector; boolean isSecure = false; if (conf.getBoolean(REST_SSL_ENABLED, false)) { @@ -400,6 +410,14 @@ public synchronized void run() throws Exception { server.start(); } + private static void setUriComplianceRules(HttpConfiguration httpConfig) { + Set complianceViolationSet = new HashSet<>(); + complianceViolationSet.add(UriCompliance.Violation.AMBIGUOUS_PATH_SEPARATOR); + complianceViolationSet.add(UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS); + complianceViolationSet.add(UriCompliance.Violation.AMBIGUOUS_EMPTY_SEGMENT); + httpConfig.setUriCompliance(UriCompliance.from(complianceViolationSet)); + } + private static String getHostName(Configuration conf) throws UnknownHostException { return Strings.domainNamePointerToHostName(DNS.getDefaultHost( conf.get(REST_DNS_INTERFACE, "default"), conf.get(REST_DNS_NAMESERVER, "default"))); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java index b20baea9df8c..21e948e3cfc8 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java @@ -321,6 +321,11 @@ public void testLatestCellGetJSON() throws IOException { @Test public void testURLEncodedKey() throws IOException, JAXBException { + // Requires UriCompliance.Violation.AMBIGUOUS_PATH_SEPARATOR + // Otherwise fails with "400: Ambiguous URI path separator" + // In this test, request url resolves to "/TestRowResource/http%3A%2F%2Fexample.com%2Ffoo/a:1" + // and is considered ambiguous by Jetty 12. + // Basically we are having a URL encoded string as row key here! String urlKey = "http://example.com/foo"; StringBuilder path = new StringBuilder(); path.append('/'); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java index fcbe7c7ca02e..c464a51813e7 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/client/TestRemoteTable.java @@ -164,6 +164,12 @@ public void testGetTableDescriptor() throws IOException { @Test public void testGet() throws IOException { + // Requires UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS + // Otherwise fails with "400: Suspicious Path Character" + // In this test, the request path resolves to + // "/TestRemoteTable_-./testrow1%7C%22%5C%5E%7B%7D%01%02%03%04%05%06%07%08%09%0B%0C/" + // and is considered suspicious by the Jetty 12. + // Basically ROW_1 contains invalid URL characters here. Get get = new Get(ROW_1); Result result = remoteTable.get(get); byte[] value1 = result.getValue(COLUMN_1, QUALIFIER_1); @@ -264,6 +270,9 @@ public void testGet() throws IOException { @Test public void testMultiGet() throws Exception { + // In case of multi gets, the request path resolves to + // "/TestRemoteTable_-./multiget/?row=testrow1%7C%22%5C%5E&row=testrow2%7C%22%5C%5E%&v=3" + // and hence is not considered suspicious by the Jetty 12. ArrayList gets = new ArrayList<>(2); gets.add(new Get(ROW_1)); gets.add(new Get(ROW_2)); @@ -303,6 +312,8 @@ public void testMultiGet() throws Exception { @Test public void testPut() throws IOException { + // Requires UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS + // Otherwise fails with "400: Suspicious Path Character" Put put = new Put(ROW_3); put.addColumn(COLUMN_1, QUALIFIER_1, VALUE_1); remoteTable.put(put); @@ -348,6 +359,13 @@ public void testPut() throws IOException { @Test public void testDelete() throws IOException { + // Requires UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS for put, + // otherwise fails with "400: Suspicious Path Character" + // This example is considered suspicious by the Jetty 12 due to reasons same as shown in + // testGet() + + // Also, requires UriCompliance.Violation.AMBIGUOUS_EMPTY_SEGMENT + // Otherwise fails with "400: Ambiguous URI empty segment" Put put = new Put(ROW_3); put.addColumn(COLUMN_1, QUALIFIER_1, VALUE_1); put.addColumn(COLUMN_2, QUALIFIER_2, VALUE_2); @@ -387,6 +405,9 @@ public void testDelete() throws IOException { assertTrue(Bytes.equals(VALUE_1, value1)); assertNull(value2); + // This leads to path which resolves to + // "/TestRemoteTable_-./testrow3%7C%22%5C%5E%7B%7D%01%02%03%04%05%06%07%08%09%0B%0C//1" + // causing "400: Ambiguous URI empty segment" error with Jetty 12. delete = new Delete(ROW_3); delete.setTimestamp(1L); remoteTable.delete(delete); @@ -493,6 +514,10 @@ public void testScanner() throws IOException { @Test public void testCheckAndDelete() throws IOException { + // Requires UriCompliance.Violation.SUSPICIOUS_PATH_CHARACTERS + // Otherwise fails with "400: Suspicious Path Character" + // This example is considered suspicious by the Jetty 12 due to reasons same as shown in + // testGet() Get get = new Get(ROW_1); Result result = remoteTable.get(get); byte[] value1 = result.getValue(COLUMN_1, QUALIFIER_1); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 19f58ebe6ad0..317f571ba8ac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -287,10 +287,10 @@ import org.apache.hbase.thirdparty.com.google.gson.JsonParseException; import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors; import org.apache.hbase.thirdparty.com.google.protobuf.Service; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.webapp.WebAppContext; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder; -import org.apache.hbase.thirdparty.org.eclipse.jetty.webapp.WebAppContext; import org.apache.hbase.thirdparty.org.glassfish.jersey.server.ResourceConfig; import org.apache.hbase.thirdparty.org.glassfish.jersey.servlet.ServletContainer; diff --git a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index 1d1350712b12..8b63889f1950 100644 --- a/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -98,6 +98,8 @@ allowed_expr+="|^about.html$" allowed_expr+="|^jetty-dir.css$" # Coming from Guava, see https://github.com/google/guava/commit/2cc8c5eddb587db3ac12dacdd5563e79a4681ec4 allowed_expr+="|^org/jspecify/$|^org/jspecify/annotations/$|^org/jspecify/annotations/.*\.class$" +# Required by jetty 12 on ee8 +allowed_expr="(|^javax/$)" if [ -n "${allow_hadoop}" ]; then # * classes in packages that start with org.apache.hadoop, which by diff --git a/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml b/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml deleted file mode 100644 index 8f10b517eb55..000000000000 --- a/hbase-shaded/hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/eclipse/jetty/webapp/webdefault.xml +++ /dev/null @@ -1,550 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - Default web.xml file. - This file is applied to a Web application before it's own WEB_INF/web.xml file - - - - - - - - org.apache.hadoop.hbase.shaded.org.eclipse.jetty.servlet.listener.ELContextCleaner - - - - - - - - org.apache.hadoop.hbase.shaded.org.eclipse.jetty.servlet.listener.IntrospectorCleaner - - - - - - - - - - - - - - - - - default - org.apache.hadoop.hbase.shaded.org.eclipse.jetty.servlet.DefaultServlet - - aliases - false - - - acceptRanges - true - - - dirAllowed - true - - - welcomeServlets - false - - - redirectWelcome - false - - - maxCacheSize - 256000000 - - - maxCachedFileSize - 200000000 - - - maxCachedFiles - 2048 - - - gzip - false - - - etags - false - - - useFileMappedBuffer - true - - - - 0 - - - - default - / - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - jsp - org.apache.hadoop.hbase.shaded.org.eclipse.jetty.jsp.JettyJspServlet - - logVerbosityLevel - DEBUG - - - fork - false - - - xpoweredBy - false - - - compilerTargetVM - 1.7 - - - compilerSourceVM - 1.7 - - - 0 - - - - jsp - *.jsp - *.jspf - *.jspx - *.xsp - *.JSP - *.JSPF - *.JSPX - *.XSP - - - - - - - - 30 - - - - - - - - - - - - - - - index.html - index.htm - index.jsp - - - - - - - - ar - ISO-8859-6 - - - be - ISO-8859-5 - - - bg - ISO-8859-5 - - - ca - ISO-8859-1 - - - cs - ISO-8859-2 - - - da - ISO-8859-1 - - - de - ISO-8859-1 - - - el - ISO-8859-7 - - - en - ISO-8859-1 - - - es - ISO-8859-1 - - - et - ISO-8859-1 - - - fi - ISO-8859-1 - - - fr - ISO-8859-1 - - - hr - ISO-8859-2 - - - hu - ISO-8859-2 - - - is - ISO-8859-1 - - - it - ISO-8859-1 - - - iw - ISO-8859-8 - - - ja - Shift_JIS - - - ko - EUC-KR - - - lt - ISO-8859-2 - - - lv - ISO-8859-2 - - - mk - ISO-8859-5 - - - nl - ISO-8859-1 - - - no - ISO-8859-1 - - - pl - ISO-8859-2 - - - pt - ISO-8859-1 - - - ro - ISO-8859-2 - - - ru - ISO-8859-5 - - - sh - ISO-8859-5 - - - sk - ISO-8859-2 - - - sl - ISO-8859-2 - - - sq - ISO-8859-2 - - - sr - ISO-8859-5 - - - sv - ISO-8859-1 - - - tr - ISO-8859-9 - - - uk - ISO-8859-5 - - - zh - GB2312 - - - zh_TW - Big5 - - - - - - - - - Disable TRACE - / - TRACE - - - - - - Enable everything but TRACE - / - TRACE - - - - diff --git a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh index 1d1350712b12..8b63889f1950 100644 --- a/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh +++ b/hbase-shaded/hbase-shaded-with-hadoop-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh @@ -98,6 +98,8 @@ allowed_expr+="|^about.html$" allowed_expr+="|^jetty-dir.css$" # Coming from Guava, see https://github.com/google/guava/commit/2cc8c5eddb587db3ac12dacdd5563e79a4681ec4 allowed_expr+="|^org/jspecify/$|^org/jspecify/annotations/$|^org/jspecify/annotations/.*\.class$" +# Required by jetty 12 on ee8 +allowed_expr="(|^javax/$)" if [ -n "${allow_hadoop}" ]; then # * classes in packages that start with org.apache.hadoop, which by diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java index 7f2d37440297..a0370aab0fb1 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java @@ -144,6 +144,8 @@ import org.apache.hbase.thirdparty.org.apache.commons.cli.DefaultParser; import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter; import org.apache.hbase.thirdparty.org.apache.commons.cli.Options; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletContextHandler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.ee8.servlet.ServletHolder; import org.apache.hbase.thirdparty.org.eclipse.jetty.http.HttpVersion; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConfiguration; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.HttpConnectionFactory; @@ -151,8 +153,6 @@ import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.SslConnectionFactory; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletContextHandler; -import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.ServletHolder; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ssl.SslContextFactory; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.thread.QueuedThreadPool; diff --git a/pom.xml b/pom.xml index 30c63fca3819..bb24fe435cf3 100644 --- a/pom.xml +++ b/pom.xml @@ -893,9 +893,9 @@ 2.19.2 2.19.2 2.3.1 - 3.1.0 + 4.0.1 2.1.1 - 9.0.104 + 9.0.108 9.4.12.1 4.13.2 1.3 @@ -975,7 +975,7 @@ - 9.0.93 + ${tomcat.jasper.version} 0.8.8 @@ -1847,7 +1847,12 @@ org.apache.hbase.thirdparty - hbase-shaded-jetty + hbase-shaded-jetty-12-plus-core + ${hbase-thirdparty.version} + + + org.apache.hbase.thirdparty + hbase-shaded-jetty-12-plus-ee8 ${hbase-thirdparty.version}