From e748691da36334ca7bb4ec73e62ddda0a9be4cf6 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 9 Nov 2016 15:13:06 +1100 Subject: [PATCH 01/65] Issue #68 added request context scope --- jetty9-base/pom.xml | 2 +- jetty9-base/src/main/jetty-base/etc/gae-web.xml | 15 +++++---------- pom.xml | 6 +++--- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/jetty9-base/pom.xml b/jetty9-base/pom.xml index eb0e2347..7e7ef309 100644 --- a/jetty9-base/pom.xml +++ b/jetty9-base/pom.xml @@ -26,7 +26,7 @@ Google App Engine Image Jetty (jetty9-base) jetty9-base - pom + jar diff --git a/jetty9-base/src/main/jetty-base/etc/gae-web.xml b/jetty9-base/src/main/jetty-base/etc/gae-web.xml index 47215c61..a921d5ad 100644 --- a/jetty9-base/src/main/jetty-base/etc/gae-web.xml +++ b/jetty9-base/src/main/jetty-base/etc/gae-web.xml @@ -1,16 +1,11 @@ - - - - - - - /app.yaml - - - + + + + + diff --git a/pom.xml b/pom.xml index 91521af7..5bb1bd70 100644 --- a/pom.xml +++ b/pom.xml @@ -34,9 +34,9 @@ UTF-8 yyyy-MM-dd_HH_mm 1.9.40 - 3 - 8 - 9.${jetty9.minor.version}.${jetty9.dot.version}.v20160314 + 4 + 0 + 9.${jetty9.minor.version}.${jetty9.dot.version}.RC1 ${docker.tag.prefix}9.${jetty9.minor.version} ${docker.tag.prefix}9.${jetty9.minor.version}-${maven.build.timestamp} From f6d1488695dbdc00ce808502b845726a33c92367 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 9 Nov 2016 16:07:20 +1100 Subject: [PATCH 02/65] Issue #68 added request context scope --- jetty9-base/pom.xml | 17 ++++--- jetty9-base/src/main/assembly/assembly.xml | 7 --- .../runtimes/jetty9/RequestContextScope.java | 49 +++++++++++++++++++ 3 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java diff --git a/jetty9-base/pom.xml b/jetty9-base/pom.xml index 7e7ef309..60aac7c2 100644 --- a/jetty9-base/pom.xml +++ b/jetty9-base/pom.xml @@ -29,12 +29,6 @@ jar - - com.google.cloud.runtimes - logging - 0.1.0-SNAPSHOT - jar - org.eclipse.jetty jetty-server @@ -52,6 +46,14 @@ + + org.apache.maven.plugins + maven-jar-plugin + 2.3.1 + + ${project.build.directory}/jetty-base/lib/gae + + maven-resources-plugin @@ -135,7 +137,8 @@ -jar ../jetty-distribution-${jetty9.version}/start.jar - --add-to-startd=http,deploy,jsp,jstl,http-forwarded,resources,gae + --add-to-startd=http,deploy,jsp,jstl,http-forwarded,resources,gae,logging-jul + --approve-all-licenses diff --git a/jetty9-base/src/main/assembly/assembly.xml b/jetty9-base/src/main/assembly/assembly.xml index 56cd4409..e2578fe2 100644 --- a/jetty9-base/src/main/assembly/assembly.xml +++ b/jetty9-base/src/main/assembly/assembly.xml @@ -33,12 +33,5 @@ **/META-INF/** - - ${project.build.directory} - jetty-base/lib/gae - - ${artifactId}-${version}.jar - - diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java new file mode 100644 index 00000000..a05b5e48 --- /dev/null +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java @@ -0,0 +1,49 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.runtimes.jetty9; + +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.ContextHandler; +import org.eclipse.jetty.server.handler.ContextHandler.Context; + +import java.util.Stack; + +public class RequestContextScope implements ContextHandler.ContextScopeListener { + + private static ThreadLocal> _request = new ThreadLocal>() { + @Override + protected Stack initialValue() { + return new Stack<>(); + } + }; + + @Override + public void enterScope(Context context, Request request, Object reason) { + if (request != null) { + _request.get().push(request); + } + } + + @Override + public void exitScope(Context context, Request request) { + if (request != null) { + _request.get().pop(); + } + } + + public static Request getCurrentRequest() { + return _request.get().peek(); + } +} From 2ad99b0db8ca4cd6fef3b5e2b1877bc3fb73d3c8 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 9 Nov 2016 17:40:01 +1100 Subject: [PATCH 03/65] Issue #68 added skeleton Logging Handler --- .../cloud/runtimes/jetty9/LoggingHandler.java | 36 +++++++++++++++ .../runtimes/jetty9/RequestContextScope.java | 45 ++++++++++++++++--- .../etc/java-util-logging.properties | 4 ++ 3 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java create mode 100644 jetty9-base/src/main/jetty-base/etc/java-util-logging.properties diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java new file mode 100644 index 00000000..02e45ffa --- /dev/null +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java @@ -0,0 +1,36 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.runtimes.jetty9; + +import java.util.Date; +import java.util.logging.Handler; +import java.util.logging.LogRecord; + +public class LoggingHandler extends Handler { + + @Override + public void publish(LogRecord record) { + String traceid = RequestContextScope.getCurrentTraceid(); + System.err.printf("%s:%s:%s:%s:%s%n", new Date(record.getMillis()), traceid, record.getLevel(), + record.getLoggerName(), record.getMessage()); + } + + @Override + public void flush() {} + + @Override + public void close() throws SecurityException {} + +} diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java index a05b5e48..81dec8a8 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java @@ -19,31 +19,66 @@ import org.eclipse.jetty.server.handler.ContextHandler.Context; import java.util.Stack; +import java.util.logging.Level; +import java.util.logging.Logger; public class RequestContextScope implements ContextHandler.ContextScopeListener { + static final Logger logger = Logger.getLogger(RequestContextScope.class.getName()); - private static ThreadLocal> _request = new ThreadLocal>() { + private static final String X_CLOUD_TRACE = "x-cloud-trace-context"; + private static final ThreadLocal traceid = new ThreadLocal<>(); + private static final ThreadLocal> requestStack = + new ThreadLocal>() { @Override protected Stack initialValue() { return new Stack<>(); } }; - + @Override public void enterScope(Context context, Request request, Object reason) { if (request != null) { - _request.get().push(request); + Stack stack = requestStack.get(); + if (stack.isEmpty()) { + String id = (String) request.getAttribute(X_CLOUD_TRACE); + if (id == null) { + id = request.getHeader(X_CLOUD_TRACE); + if (id != null) { + int slash = id.indexOf('/'); + if (slash >= 0) { + id = id.substring(0, slash); + } + request.setAttribute(X_CLOUD_TRACE, id); + } + } + traceid.set(id); + } + stack.push(request); + } + if (logger.isLoggable(Level.FINE)) { + logger.fine("enterScope " + context); } } @Override public void exitScope(Context context, Request request) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("exitScope " + context); + } if (request != null) { - _request.get().pop(); + Stack stack = requestStack.get(); + stack.pop(); + if (stack.isEmpty()) { + traceid.set(null); + } } } public static Request getCurrentRequest() { - return _request.get().peek(); + return requestStack.get().peek(); + } + + public static String getCurrentTraceid() { + return traceid.get(); } } diff --git a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties new file mode 100644 index 00000000..2568cd13 --- /dev/null +++ b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties @@ -0,0 +1,4 @@ +.level = INFO + +handlers = com.google.cloud.runtimes.jetty9.LoggingHandler +com.google.cloud.runtimes.jetty9.LoggingHandler.level = INFO From 30b3e80d3d6c458261e525b4583bff55de7aa7fb Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 11 Nov 2016 12:55:23 +1100 Subject: [PATCH 04/65] Issue #68 work in progress --- jetty9-base/pom.xml | 14 +- .../runtimes/jetty9/AsyncLoggingHandler.java | 103 +++++ .../cloud/runtimes/jetty9/LoggingHandler.java | 410 +++++++++++++++++- .../runtimes/jetty9/RequestContextScope.java | 10 + .../runtimes/jetty9/StackDriverLogging.java | 33 ++ .../runtimes/jetty9/TracingLogHandler.java | 47 ++ .../etc/java-util-logging.properties | 7 +- pom.xml | 2 +- 8 files changed, 608 insertions(+), 18 deletions(-) create mode 100644 jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/AsyncLoggingHandler.java create mode 100644 jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/StackDriverLogging.java create mode 100644 jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java diff --git a/jetty9-base/pom.xml b/jetty9-base/pom.xml index 60aac7c2..307b1d73 100644 --- a/jetty9-base/pom.xml +++ b/jetty9-base/pom.xml @@ -42,6 +42,17 @@ ${project.version} pom + + com.google.cloud + google-cloud-logging + ${gcloud.api.version} + + + javax.servlet + servlet-api + + + @@ -137,7 +148,8 @@ -jar ../jetty-distribution-${jetty9.version}/start.jar - --add-to-startd=http,deploy,jsp,jstl,http-forwarded,resources,gae,logging-jul + --create-startd + --add-to-start=http,deploy,jsp,jstl,http-forwarded,resources,gae,gcloud,jcl-slf4j,logging-jul --approve-all-licenses diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/AsyncLoggingHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/AsyncLoggingHandler.java new file mode 100644 index 00000000..c786597b --- /dev/null +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/AsyncLoggingHandler.java @@ -0,0 +1,103 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.runtimes.jetty9; + +import com.google.cloud.MonitoredResource; +import com.google.cloud.logging.LogEntry; +import com.google.cloud.logging.Logging.WriteOption; +import com.google.cloud.logging.LoggingOptions; + +import java.util.List; +import java.util.logging.Filter; +import java.util.logging.Formatter; +import java.util.logging.Logger; +import java.util.logging.SimpleFormatter; + +/** + * A logging handler that asynchronously outputs logs generated with + * {@link java.util.logging.Logger} to Stackdriver Logging. + * + *

Java logging levels (see {@link java.util.logging.Level}) are mapped to the following Google + * Stackdriver Logging severities: + * + * + * + * + * + * + * + * + * + * + *
Java LevelStackdriver Logging Severity
SEVEREERROR
WARNINGWARNING
INFOINFO
CONFIGINFO
FINEDEBUG
FINERDEBUG
FINESTDEBUG
+ * + *

Original Java logging levels are added as labels (with {@code levelName} and + * {@code levelValue} keys, respectively) to the corresponding Stackdriver Logging {@link LogEntry}. + * You can read entry labels using {@link LogEntry#labels()}. To use logging levels that correspond + * to Stackdriver Logging severities you can use {@link LoggingLevel}. + * + *

Configuration: By default each {@code AsyncLoggingHandler} is initialized using the + * following {@code LogManager} configuration properties (that you can set in the + * {@code logging.properties} file. If properties are not defined (or have invalid values) then the + * specified default values are used. + *

    + *
  • {@code com.google.cloud.logging.AsyncLoggingHandler.log} the log name (defaults to + * {@code java.log}). + *
  • {@code com.google.cloud.logging.AsyncLoggingHandler.level} specifies the default level for + * the handler (defaults to {@code Level.INFO}). + *
  • {@code com.google.cloud.logging.AsyncLoggingHandler.filter} specifies the name of a + * {@link Filter} class to use (defaults to no filter). + *
  • {@code com.google.cloud.logging.AsyncLoggingHandler.formatter} specifies the name of a + * {@link Formatter} class to use (defaults to {@link SimpleFormatter}). + *
  • {@code com.google.cloud.logging.AsyncLoggingHandler.flushSize} specifies the maximum size of + * the log buffer. Once reached, logs are transmitted to the Stackdriver Logging service + * (defaults to 1). + *
  • {@code com.google.cloud.logging.AsyncLoggingHandler.flushLevel} specifies the flush log + * level. When a log with this level is published, logs are transmitted to the Stackdriver + * Logging service (defaults to {@link LoggingLevel#ERROR}). + *
+ * + *

To add a {@code LoggingHandler} to an existing {@link Logger} and be sure to avoid infinite + * recursion when logging, use the {@link #addHandler(Logger, LoggingHandler)} method. Alternatively + * you can add the handler via {@code logging.properties}. For example using the following line: + * 

+ * {@code com.example.mypackage.handlers=com.google.cloud.logging.AsyncLoggingHandler}
+ *
+ */ +public class AsyncLoggingHandler extends LoggingHandler { + + public AsyncLoggingHandler() { + super(); + } + + public AsyncLoggingHandler(String logName) { + super(logName); + } + + public AsyncLoggingHandler(String logName, LoggingOptions options) { + super(logName, options); + } + + public AsyncLoggingHandler(String logName, LoggingOptions options, MonitoredResource resource) { + super(logName, options, resource); + } + + @Override + void write(List entries, WriteOption... options) { + getLogging().writeAsync(entries, options); + } +} diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java index 02e45ffa..d20dfcfa 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java @@ -1,36 +1,418 @@ /* * Copyright 2016 Google Inc. All Rights Reserved. * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.google.cloud.runtimes.jetty9; -import java.util.Date; +import static com.google.common.base.MoreObjects.firstNonNull; + +import com.google.cloud.MonitoredResource; +import com.google.cloud.logging.LogEntry; +import com.google.cloud.logging.Logging; +import com.google.cloud.logging.Logging.WriteOption; +import com.google.cloud.logging.LoggingLevel; +import com.google.cloud.logging.LoggingOptions; +import com.google.cloud.logging.Payload; +import com.google.cloud.logging.Severity; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.ErrorManager; +import java.util.logging.Filter; +import java.util.logging.Formatter; import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogManager; import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.logging.SimpleFormatter; +/** + * A logging handler that synchronously outputs logs generated with {@link java.util.logging.Logger} + * to Stackdriver Logging. + * + *

Java logging levels (see {@link java.util.logging.Level}) are mapped to the following Google + * Stackdriver Logging severities: + * + * + * + * + * + * + * + * + * + * + *
Java LevelStackdriver Logging Severity
SEVEREERROR
WARNINGWARNING
INFOINFO
CONFIGINFO
FINEDEBUG
FINERDEBUG
FINESTDEBUG
+ * + *

Original Java logging levels are added as labels (with {@code levelName} and + * {@code levelValue} keys, respectively) to the corresponding Stackdriver Logging {@link LogEntry}. + * You can read entry labels using {@link LogEntry#labels()}. To use logging levels that correspond + * to Stackdriver Logging severities you can use {@link LoggingLevel}. + * + *

Configuration: By default each {@code LoggingHandler} is initialized using the + * following {@code LogManager} configuration properties (that you can set in the + * {@code logging.properties} file). If properties are not defined (or have invalid values) then the + * specified default values are used. + *

    + *
  • {@code com.google.cloud.logging.LoggingHandler.log} the log name (defaults to + * {@code java.log}). + *
  • {@code com.google.cloud.logging.LoggingHandler.level} specifies the default level for the + * handler (defaults to {@code Level.INFO}). + *
  • {@code com.google.cloud.logging.LoggingHandler.filter} specifies the name of a {@link Filter} + * class to use (defaults to no filter). + *
  • {@code com.google.cloud.logging.LoggingHandler.formatter} specifies the name of a + * {@link Formatter} class to use (defaults to {@link SimpleFormatter}). + *
  • {@code com.google.cloud.logging.LoggingHandler.flushSize} specifies the maximum size of the + * log buffer. Once reached, logs are transmitted to the Stackdriver Logging service (defaults + * to 1). + *
  • {@code com.google.cloud.logging.LoggingHandler.flushLevel} specifies the flush log level. + * When a log with this level is published, logs are transmitted to the Stackdriver Logging + * service (defaults to {@link LoggingLevel#ERROR}). + *
+ * + *

To add a {@code LoggingHandler} to an existing {@link Logger} and be sure to avoid infinite + * recursion when logging, use the {@link #addHandler(Logger, LoggingHandler)} method. Alternatively + * you can add the handler via {@code logging.properties}. For example using the following line: + * 

+ * {@code com.example.mypackage.handlers=com.google.cloud.logging.LoggingHandler}
+ *
+ */ public class LoggingHandler extends Handler { + private static final String HANDLERS_PROPERTY = "handlers"; + private static final String ROOT_LOGGER_NAME = ""; + private static final String[] NO_HANDLERS = new String[0]; + private static final Set EXCLUDED_LOGGERS = ImmutableSet.of("io.grpc", "io.netty", + "com.google.api.client.http", "sun.net.www.protocol.http"); + + private final LoggingOptions options; + private final List buffer = new LinkedList<>(); + private final WriteOption[] writeOptions; + private Logging logging; + private Level flushLevel; + private long flushSize; + + /** + * Creates an handler that publishes messages to Stackdriver Logging. + */ + public LoggingHandler() { + this(null, null, null); + } + + /** + * Creates a handler that publishes messages to Stackdriver Logging. + * + * @param log the name of the log to which log entries are written + */ + public LoggingHandler(String log) { + this(log, null, null); + } + + /** + * Creates a handler that publishes messages to Stackdriver Logging. + * + * @param log the name of the log to which log entries are written + * @param options options for the Stackdriver Logging service + */ + public LoggingHandler(String log, LoggingOptions options) { + this(log, options, null); + } + + /** + * Creates a handler that publishes messages to Stackdriver Logging. + * + * @param log the name of the log to which log entries are written + * @param options options for the Stackdriver Logging service + * @param monitoredResource the monitored resource to which log entries refer + */ + public LoggingHandler(String log, LoggingOptions options, MonitoredResource monitoredResource) { + LogConfigHelper helper = new LogConfigHelper(); + String className = getClass().getName(); + this.options = options != null ? options : LoggingOptions.getDefaultInstance(); + this.flushLevel = helper.getLevelProperty(className + ".flushLevel", LoggingLevel.ERROR); + this.flushSize = helper.getLongProperty(className + ".flushSize", 1L); + setLevel(helper.getLevelProperty(className + ".level", Level.INFO)); + setFilter(helper.getFilterProperty(className + ".filter", null)); + setFormatter(helper.getFormatterProperty(className + ".formatter", new SimpleFormatter())); + String logName = firstNonNull(log, helper.getProperty(className + ".log", "java.log")); + MonitoredResource resource = firstNonNull(monitoredResource, getDefaultResource()); + writeOptions = new WriteOption[]{WriteOption.logName(logName), WriteOption.resource(resource)}; + maskLoggers(); + } + + private static void maskLoggers() { + for (String loggerName : EXCLUDED_LOGGERS) { + Logger logger = Logger.getLogger(loggerName); + // We remove the Clould Logging handler if it has been registered for a logger that should be + // masked + List loggingHandlers = getLoggingHandlers(logger); + for (LoggingHandler loggingHandler : loggingHandlers) { + logger.removeHandler(loggingHandler); + } + // We mask ancestors if they have a Stackdriver Logging Handler registered + Logger currentLogger = logger; + Logger ancestor = currentLogger.getParent(); + boolean masked = false; + while (ancestor != null && !masked) { + if (hasLoggingHandler(ancestor)) { + currentLogger.setUseParentHandlers(false); + masked = true; + } + currentLogger = ancestor; + ancestor = ancestor.getParent(); + } + } + } + + private static List getLoggingHandlers(Logger logger) { + ImmutableList.Builder builder = ImmutableList.builder(); + for (Handler handler : logger.getHandlers()) { + if (handler instanceof LoggingHandler) { + builder.add((LoggingHandler) handler); + } + } + return builder.build(); + } + + private static boolean hasLoggingHandler(Logger logger) { + // look for Stackdriver Logging handler registered with addHandler() + for (Handler handler : logger.getHandlers()) { + if (handler instanceof LoggingHandler) { + return true; + } + } + // look for Stackdriver Logging handler registered via logging.properties + String loggerName = logger.getName(); + String propertyName = loggerName.equals(ROOT_LOGGER_NAME) + ? HANDLERS_PROPERTY : loggerName + "." + HANDLERS_PROPERTY; + String handlersProperty = LogManager.getLogManager().getProperty(propertyName); + String[] handlers = handlersProperty != null ? handlersProperty.split(",") : NO_HANDLERS; + for (String handlerName : handlers) { + if (handlerName.contains(LoggingHandler.class.getPackage().getName())) { + return true; + } + } + return false; + } + + private MonitoredResource getDefaultResource() { + return MonitoredResource.of("global", ImmutableMap.of("project_id", options.getProjectId())); + } + + private static class LogConfigHelper { + + private final LogManager manager = LogManager.getLogManager(); + + String getProperty(String name, String defaultValue) { + return firstNonNull(manager.getProperty(name), defaultValue); + } + + long getLongProperty(String name, long defaultValue) { + try { + return Long.parseLong(manager.getProperty(name)); + } catch (NumberFormatException ex) { + // If the level does not exist we fall back to default value + } + return defaultValue; + } + + Level getLevelProperty(String name, Level defaultValue) { + String stringLevel = manager.getProperty(name); + if (stringLevel == null) { + return defaultValue; + } + try { + return Level.parse(stringLevel); + } catch (IllegalArgumentException ex) { + // If the level does not exist we fall back to default value + } + return defaultValue; + } + + Filter getFilterProperty(String name, Filter defaultValue) { + String stringFilter = manager.getProperty(name); + try { + if (stringFilter != null) { + Class clz = ClassLoader.getSystemClassLoader().loadClass(stringFilter); + return (Filter) clz.newInstance(); + } + } catch (Exception ex) { + // If we cannot create the filter we fall back to default value + } + return defaultValue; + } + + Formatter getFormatterProperty(String name, Formatter defaultValue) { + String stringFilter = manager.getProperty(name); + try { + if (stringFilter != null) { + Class clz = ClassLoader.getSystemClassLoader().loadClass(stringFilter); + return (Formatter) clz.newInstance(); + } + } catch (Exception ex) { + // If we cannot create the filter we fall back to default value + } + return defaultValue; + } + } + + /** + * Returns an instance of the logging service. + */ + Logging getLogging() { + if (logging == null) { + logging = options.getService(); + } + return logging; + } + @Override - public void publish(LogRecord record) { - String traceid = RequestContextScope.getCurrentTraceid(); - System.err.printf("%s:%s:%s:%s:%s%n", new Date(record.getMillis()), traceid, record.getLevel(), - record.getLoggerName(), record.getMessage()); + public synchronized void publish(LogRecord record) { + // check that the log record should be logged + if (!isLoggable(record)) { + return; + } + LogEntry entry = entryFor(record); + if (entry != null) { + buffer.add(entry); + } + if (buffer.size() >= flushSize || record.getLevel().intValue() >= flushLevel.intValue()) { + flush(); + } + } + + private LogEntry entryFor(LogRecord record) { + String payload; + try { + payload = getFormatter().format(record); + } catch (Exception ex) { + // Formatting can fail but we should not throw an exception, we report the error instead + reportError(null, ex, ErrorManager.FORMAT_FAILURE); + return null; + } + Level level = record.getLevel(); + LogEntry.Builder builder = LogEntry.newBuilder(Payload.StringPayload.of(payload)) + .addLabel("levelName", level.getName()) + .addLabel("levelValue", String.valueOf(level.intValue())) + .setSeverity(severityFor(level)); + return buildEntryFor(record, builder); + } + + protected LogEntry buildEntryFor(LogRecord record, LogEntry.Builder builder) { + return builder.build(); + } + + private static Severity severityFor(Level level) { + if (level instanceof LoggingLevel) { + return ((LoggingLevel) level).getSeverity(); + } + switch (level.intValue()) { + // FINEST + case 300: + return Severity.DEBUG; + // FINER + case 400: + return Severity.DEBUG; + // FINE + case 500: + return Severity.DEBUG; + // CONFIG + case 700: + return Severity.INFO; + // INFO + case 800: + return Severity.INFO; + // WARNING + case 900: + return Severity.WARNING; + // SEVERE + case 1000: + return Severity.ERROR; + default: + return Severity.DEFAULT; + } + } + + /** + * Writes the provided list of log entries to Stackdriver Logging. Override this method to change + * how entries should be written. + */ + void write(List entries, WriteOption... options) { + getLogging().write(entries, options); } @Override - public void flush() {} + public synchronized void flush() { + try { + write(buffer, writeOptions); + } catch (Exception ex) { + // writing can fail but we should not throw an exception, we report the error instead + reportError(null, ex, ErrorManager.FLUSH_FAILURE); + } finally { + buffer.clear(); + } + } + /** + * Closes the handler and the associated {@link Logging} object. + */ @Override - public void close() throws SecurityException {} + public synchronized void close() throws SecurityException { + if (logging != null) { + try { + logging.close(); + } catch (Exception ex) { + // ignore + } + } + logging = null; + } + + /** + * Sets the flush log level. When a log with this level is published, the log buffer is + * transmitted to the Stackdriver Logging service, regardless of its size. If not set, + * {@link LoggingLevel#ERROR} is used. + */ + public synchronized Level setFlushLevel(Level flushLevel) { + this.flushLevel = flushLevel; + return flushLevel; + } + /** + * Sets the maximum size of the log buffer. Once the maximum size of the buffer is reached, logs + * are transmitted to the Stackdriver Logging service. If not set, a log is sent to the service as + * soon as published. + */ + public synchronized long setFlushSize(long flushSize) { + this.flushSize = flushSize; + return flushSize; + } + + /** + * Adds the provided {@code LoggingHandler} to {@code logger}. Use this method to register Cloud + * Logging handlers instead of {@link Logger#addHandler(Handler)} to avoid infinite recursion + * when logging. + */ + public static void addHandler(Logger logger, LoggingHandler handler) { + logger.addHandler(handler); + maskLoggers(); + } } diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java index 81dec8a8..0427cb81 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java @@ -74,6 +74,16 @@ public void exitScope(Context context, Request request) { } } + public static void runWith(String traceid, Runnable runnable) { + String original = RequestContextScope.traceid.get(); + RequestContextScope.traceid.set(traceid); + try { + runnable.run(); + } finally { + RequestContextScope.traceid.set(original); + } + } + public static Request getCurrentRequest() { return requestStack.get().peek(); } diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/StackDriverLogging.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/StackDriverLogging.java new file mode 100644 index 00000000..87fe0653 --- /dev/null +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/StackDriverLogging.java @@ -0,0 +1,33 @@ +package com.google.cloud.runtimes.jetty9; + +import java.util.Arrays; +import java.util.logging.Logger; + +public class StackDriverLogging { + + static void init() throws Exception { + Logger log = Logger.getLogger(StackDriverLogging.class.getName()); + + for (Logger l : new Logger[] {log, + Logger.getLogger("sun.net.www.protocol.http.HttpURLConnection")}) { + while (l != null) { + System.err.printf("Log '%s' upl=%b %s%n", l.getName(), l.getUseParentHandlers(), + Arrays.asList(l.getHandlers())); + l = l.getParent(); + } + } + System.err.println("httpurl log"); + Logger.getLogger("sun.net.www.protocol.http.HttpURLConnection").info("test"); + System.err.println("some.random.log log"); + Logger.getLogger("some.random.log").info("test"); + System.err.println("com.google.cloud.runtimes.jetty9.StackDriverLogging log info"); + log.info("test info"); + System.err.println("com.google.cloud.runtimes.jetty9.StackDriverLogging log fine"); + log.fine("test fine"); + System.err.println("done"); + } + + public static void main(String... args) throws Exception { + init(); + } +} diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java new file mode 100644 index 00000000..dd91d3a8 --- /dev/null +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java @@ -0,0 +1,47 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.runtimes.jetty9; + +import static com.google.common.base.MoreObjects.firstNonNull; + +import com.google.cloud.MonitoredResource; +import com.google.cloud.logging.LogEntry; +import com.google.cloud.logging.LogEntry.Builder; +import com.google.cloud.logging.LoggingOptions; + +import java.util.logging.LogManager; +import java.util.logging.LogRecord; + +public class TracingLogHandler extends AsyncLoggingHandler { + + public TracingLogHandler() { + this(firstNonNull( + LogManager.getLogManager().getProperty(TracingLogHandler.class.getName() + ".log"), + "jetty.log"), null, null); + } + + public TracingLogHandler(String logName, LoggingOptions options, MonitoredResource resource) { + super(logName, options, resource); + } + + @Override + protected LogEntry buildEntryFor(LogRecord record, Builder builder) { + String traceid = RequestContextScope.getCurrentTraceid(); + if (traceid != null) { + builder.addLabel("traceid", traceid); + } + return super.buildEntryFor(record, builder); + } +} diff --git a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties index 2568cd13..291c4c5f 100644 --- a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties +++ b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties @@ -1,4 +1,7 @@ .level = INFO -handlers = com.google.cloud.runtimes.jetty9.LoggingHandler -com.google.cloud.runtimes.jetty9.LoggingHandler.level = INFO +handlers = com.google.cloud.runtimes.jetty9.TracingLogHandler +com.google.cloud.runtimes.jetty9.TracingLogHandler.level = INFO + +# handlers = java.util.logging.ConsoleHandler +# java.util.logging.ConsoleHandler.level = INFO \ No newline at end of file diff --git a/pom.xml b/pom.xml index 5bb1bd70..ac494ccf 100644 --- a/pom.xml +++ b/pom.xml @@ -34,13 +34,13 @@ UTF-8 yyyy-MM-dd_HH_mm 1.9.40 + 0.5.1 4 0 9.${jetty9.minor.version}.${jetty9.dot.version}.RC1 ${docker.tag.prefix}9.${jetty9.minor.version} ${docker.tag.prefix}9.${jetty9.minor.version}-${maven.build.timestamp} - gcr.io/google_appengine/openjdk From 7ec8ff35b9319d00f889404393a0227d67ad40f9 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 16 Nov 2016 17:53:27 +1100 Subject: [PATCH 05/65] Issue #68 work in progress --- jetty9-base/pom.xml | 1 + .../runtimes/jetty9/RequestContextScope.java | 23 +++---- .../runtimes/jetty9/StackDriverLogging.java | 18 +++--- .../runtimes/jetty9/TracingLogHandler.java | 16 +++++ .../etc/java-util-logging.properties | 2 +- .../cloud/runtimes/jetty9/TestServlet.java | 64 +++++++++++++++++++ 6 files changed, 103 insertions(+), 21 deletions(-) create mode 100644 jetty9-base/src/test/java/com/google/cloud/runtimes/jetty9/TestServlet.java diff --git a/jetty9-base/pom.xml b/jetty9-base/pom.xml index 307b1d73..e23d6fe1 100644 --- a/jetty9-base/pom.xml +++ b/jetty9-base/pom.xml @@ -176,5 +176,6 @@ + diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java index 0427cb81..7bf3bf7a 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java @@ -18,7 +18,8 @@ import org.eclipse.jetty.server.handler.ContextHandler; import org.eclipse.jetty.server.handler.ContextHandler.Context; -import java.util.Stack; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.logging.Level; import java.util.logging.Logger; @@ -27,18 +28,18 @@ public class RequestContextScope implements ContextHandler.ContextScopeListener private static final String X_CLOUD_TRACE = "x-cloud-trace-context"; private static final ThreadLocal traceid = new ThreadLocal<>(); - private static final ThreadLocal> requestStack = - new ThreadLocal>() { + private static final ThreadLocal> requestStack = + new ThreadLocal>() { @Override - protected Stack initialValue() { - return new Stack<>(); + protected Deque initialValue() { + return new ArrayDeque<>(); } }; @Override public void enterScope(Context context, Request request, Object reason) { if (request != null) { - Stack stack = requestStack.get(); + Deque stack = requestStack.get(); if (stack.isEmpty()) { String id = (String) request.getAttribute(X_CLOUD_TRACE); if (id == null) { @@ -66,14 +67,14 @@ public void exitScope(Context context, Request request) { logger.fine("exitScope " + context); } if (request != null) { - Stack stack = requestStack.get(); - stack.pop(); - if (stack.isEmpty()) { - traceid.set(null); - } + requestStack.get().pop(); } } + /** Run a Runnable in the scope of a traceid. + * @param traceid The trace ID + * @param runnable the runnable + */ public static void runWith(String traceid, Runnable runnable) { String original = RequestContextScope.traceid.get(); RequestContextScope.traceid.set(traceid); diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/StackDriverLogging.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/StackDriverLogging.java index 87fe0653..f278824f 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/StackDriverLogging.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/StackDriverLogging.java @@ -6,6 +6,7 @@ public class StackDriverLogging { static void init() throws Exception { + Logger log = Logger.getLogger(StackDriverLogging.class.getName()); for (Logger l : new Logger[] {log, @@ -16,15 +17,14 @@ static void init() throws Exception { l = l.getParent(); } } - System.err.println("httpurl log"); - Logger.getLogger("sun.net.www.protocol.http.HttpURLConnection").info("test"); - System.err.println("some.random.log log"); - Logger.getLogger("some.random.log").info("test"); - System.err.println("com.google.cloud.runtimes.jetty9.StackDriverLogging log info"); - log.info("test info"); - System.err.println("com.google.cloud.runtimes.jetty9.StackDriverLogging log fine"); - log.fine("test fine"); - System.err.println("done"); + Logger.getLogger("").info("root info!"); + log.info("test info!"); + log.fine("test fine!"); + + ((TracingLogHandler) Logger.getLogger("").getHandlers()[0]).flush(); + System.err.println("flushed"); + + Thread.sleep(5000); } public static void main(String... args) throws Exception { diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java index dd91d3a8..2642e909 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java @@ -21,11 +21,19 @@ import com.google.cloud.logging.LogEntry.Builder; import com.google.cloud.logging.LoggingOptions; +import org.eclipse.jetty.server.Request; + import java.util.logging.LogManager; import java.util.logging.LogRecord; +/** + * A Google Cloud Logging Handler extended with a request traceid label. + */ public class TracingLogHandler extends AsyncLoggingHandler { + /** + * Construct a TracingLogHandler for "jetty.log" + */ public TracingLogHandler() { this(firstNonNull( LogManager.getLogManager().getProperty(TracingLogHandler.class.getName() + ".log"), @@ -41,6 +49,14 @@ protected LogEntry buildEntryFor(LogRecord record, Builder builder) { String traceid = RequestContextScope.getCurrentTraceid(); if (traceid != null) { builder.addLabel("traceid", traceid); + } else { + Request request = RequestContextScope.getCurrentRequest(); + if (request != null) { + builder.addLabel("http-scheme", request.getScheme()); + builder.addLabel("http-method", request.getMethod()); + builder.addLabel("http-uri", request.getOriginalURI()); + builder.addLabel("http-remote-addr", request.getRemoteAddr()); + } } return super.buildEntryFor(record, builder); } diff --git a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties index 291c4c5f..d47e5e45 100644 --- a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties +++ b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties @@ -3,5 +3,5 @@ handlers = com.google.cloud.runtimes.jetty9.TracingLogHandler com.google.cloud.runtimes.jetty9.TracingLogHandler.level = INFO -# handlers = java.util.logging.ConsoleHandler +# handlers = com.google.cloud.runtimes.jetty9.TracingLogHandler,java.util.logging.ConsoleHandler # java.util.logging.ConsoleHandler.level = INFO \ No newline at end of file diff --git a/jetty9-base/src/test/java/com/google/cloud/runtimes/jetty9/TestServlet.java b/jetty9-base/src/test/java/com/google/cloud/runtimes/jetty9/TestServlet.java new file mode 100644 index 00000000..c6937df7 --- /dev/null +++ b/jetty9-base/src/test/java/com/google/cloud/runtimes/jetty9/TestServlet.java @@ -0,0 +1,64 @@ +// +// ======================================================================== +// Copyright (c) 1995-2016 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package com.google.cloud.runtimes.jetty9; + +import java.io.IOException; +import java.util.logging.Logger; + +import javax.annotation.PostConstruct; +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + + +@SuppressWarnings("serial") +@WebServlet(urlPatterns = {"/", "/test/*"}, name = "TestServlet") +public class TestServlet extends HttpServlet { + static final Logger log = Logger.getLogger(TestServlet.class.getName()); + + @PostConstruct + private void myPostConstructMethod() { + log.info("preconstructed"); + } + + @Override + public void init() throws ServletException { + log.info("init info"); + getServletContext().log("init ServletContext.log"); + } + + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + log.info("doGet info"); + getServletContext().log("doGet ServletContext.log"); + + if (request.getParameter("ex") != null) { + try { + throw (Throwable) Class.forName(request.getParameter("ex")).newInstance(); + } catch (ServletException | IOException e) { + throw e; + } catch (Throwable e) { + throw new ServletException(e); + } + } + response.getWriter().println("Log Test"); + } +} From 1d3ca8a2d3164f67334fc8e525184e82583d0f83 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Thu, 17 Nov 2016 16:51:07 +1100 Subject: [PATCH 06/65] working implementation using a threadlocal to stop looping --- jetty9-base/pom.xml | 7 -- .../cloud/runtimes/jetty9/LoggingHandler.java | 79 ++----------------- .../etc/java-util-logging.properties | 4 +- pom.xml | 36 +-------- 4 files changed, 12 insertions(+), 114 deletions(-) diff --git a/jetty9-base/pom.xml b/jetty9-base/pom.xml index e23d6fe1..d6d81ea9 100644 --- a/jetty9-base/pom.xml +++ b/jetty9-base/pom.xml @@ -36,12 +36,6 @@ jar provided - - com.google.cloud.runtimes - openjdk8 - ${project.version} - pom - com.google.cloud google-cloud-logging @@ -60,7 +54,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.3.1 ${project.build.directory}/jetty-base/lib/gae diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java index d20dfcfa..93dbb465 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java @@ -26,14 +26,10 @@ import com.google.cloud.logging.LoggingOptions; import com.google.cloud.logging.Payload; import com.google.cloud.logging.Severity; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.logging.ErrorManager; import java.util.logging.Filter; import java.util.logging.Formatter; @@ -96,13 +92,7 @@ * */ public class LoggingHandler extends Handler { - - private static final String HANDLERS_PROPERTY = "handlers"; - private static final String ROOT_LOGGER_NAME = ""; - private static final String[] NO_HANDLERS = new String[0]; - private static final Set EXCLUDED_LOGGERS = ImmutableSet.of("io.grpc", "io.netty", - "com.google.api.client.http", "sun.net.www.protocol.http"); - + private final ThreadLocal flushing = new ThreadLocal<>(); private final LoggingOptions options; private final List buffer = new LinkedList<>(); private final WriteOption[] writeOptions; @@ -155,63 +145,8 @@ public LoggingHandler(String log, LoggingOptions options, MonitoredResource moni String logName = firstNonNull(log, helper.getProperty(className + ".log", "java.log")); MonitoredResource resource = firstNonNull(monitoredResource, getDefaultResource()); writeOptions = new WriteOption[]{WriteOption.logName(logName), WriteOption.resource(resource)}; - maskLoggers(); - } - - private static void maskLoggers() { - for (String loggerName : EXCLUDED_LOGGERS) { - Logger logger = Logger.getLogger(loggerName); - // We remove the Clould Logging handler if it has been registered for a logger that should be - // masked - List loggingHandlers = getLoggingHandlers(logger); - for (LoggingHandler loggingHandler : loggingHandlers) { - logger.removeHandler(loggingHandler); - } - // We mask ancestors if they have a Stackdriver Logging Handler registered - Logger currentLogger = logger; - Logger ancestor = currentLogger.getParent(); - boolean masked = false; - while (ancestor != null && !masked) { - if (hasLoggingHandler(ancestor)) { - currentLogger.setUseParentHandlers(false); - masked = true; - } - currentLogger = ancestor; - ancestor = ancestor.getParent(); - } - } - } - - private static List getLoggingHandlers(Logger logger) { - ImmutableList.Builder builder = ImmutableList.builder(); - for (Handler handler : logger.getHandlers()) { - if (handler instanceof LoggingHandler) { - builder.add((LoggingHandler) handler); - } - } - return builder.build(); } - private static boolean hasLoggingHandler(Logger logger) { - // look for Stackdriver Logging handler registered with addHandler() - for (Handler handler : logger.getHandlers()) { - if (handler instanceof LoggingHandler) { - return true; - } - } - // look for Stackdriver Logging handler registered via logging.properties - String loggerName = logger.getName(); - String propertyName = loggerName.equals(ROOT_LOGGER_NAME) - ? HANDLERS_PROPERTY : loggerName + "." + HANDLERS_PROPERTY; - String handlersProperty = LogManager.getLogManager().getProperty(propertyName); - String[] handlers = handlersProperty != null ? handlersProperty.split(",") : NO_HANDLERS; - for (String handlerName : handlers) { - if (handlerName.contains(LoggingHandler.class.getPackage().getName())) { - return true; - } - } - return false; - } private MonitoredResource getDefaultResource() { return MonitoredResource.of("global", ImmutableMap.of("project_id", options.getProjectId())); @@ -251,7 +186,7 @@ Filter getFilterProperty(String name, Filter defaultValue) { String stringFilter = manager.getProperty(name); try { if (stringFilter != null) { - Class clz = ClassLoader.getSystemClassLoader().loadClass(stringFilter); + Class clz = ClassLoader.getSystemClassLoader().loadClass(stringFilter); return (Filter) clz.newInstance(); } } catch (Exception ex) { @@ -264,7 +199,7 @@ Formatter getFormatterProperty(String name, Formatter defaultValue) { String stringFilter = manager.getProperty(name); try { if (stringFilter != null) { - Class clz = ClassLoader.getSystemClassLoader().loadClass(stringFilter); + Class clz = ClassLoader.getSystemClassLoader().loadClass(stringFilter); return (Formatter) clz.newInstance(); } } catch (Exception ex) { @@ -286,8 +221,8 @@ Logging getLogging() { @Override public synchronized void publish(LogRecord record) { - // check that the log record should be logged - if (!isLoggable(record)) { + // check that the log record should be logged and we are not already flushing logs + if (!isLoggable(record) || Boolean.TRUE.equals(flushing.get())) { return; } LogEntry entry = entryFor(record); @@ -362,11 +297,13 @@ void write(List entries, WriteOption... options) { @Override public synchronized void flush() { try { + flushing.set(Boolean.TRUE); write(buffer, writeOptions); } catch (Exception ex) { // writing can fail but we should not throw an exception, we report the error instead reportError(null, ex, ErrorManager.FLUSH_FAILURE); } finally { + flushing.set(Boolean.FALSE); buffer.clear(); } } @@ -411,8 +348,8 @@ public synchronized long setFlushSize(long flushSize) { * Logging handlers instead of {@link Logger#addHandler(Handler)} to avoid infinite recursion * when logging. */ + @Deprecated public static void addHandler(Logger logger, LoggingHandler handler) { logger.addHandler(handler); - maskLoggers(); } } diff --git a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties index d47e5e45..797d9932 100644 --- a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties +++ b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties @@ -1,7 +1,9 @@ .level = INFO +io.netty.level = INFO +io.gcr.level = INFO handlers = com.google.cloud.runtimes.jetty9.TracingLogHandler com.google.cloud.runtimes.jetty9.TracingLogHandler.level = INFO # handlers = com.google.cloud.runtimes.jetty9.TracingLogHandler,java.util.logging.ConsoleHandler -# java.util.logging.ConsoleHandler.level = INFO \ No newline at end of file +# java.util.logging.ConsoleHandler.level = INFO diff --git a/pom.xml b/pom.xml index ac494ccf..4ad78745 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ 0.5.1 4 0 - 9.${jetty9.minor.version}.${jetty9.dot.version}.RC1 + 9.${jetty9.minor.version}.${jetty9.dot.version}.RC2 ${docker.tag.prefix}9.${jetty9.minor.version} ${docker.tag.prefix}9.${jetty9.minor.version}-${maven.build.timestamp} @@ -89,12 +89,6 @@ - - com.google.code.gson - gson - 2.3.1 - - junit junit @@ -278,32 +272,4 @@ - - - - sonatype-oss-release - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.5 - - - sign-artifacts - verify - - sign - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - - - From aa81e139a45296333739b2d425f3578c311133e9 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 18 Nov 2016 10:58:14 +1100 Subject: [PATCH 07/65] Issue #68 update to latest google-cloud-java LoggingHandler --- .../cloud/runtimes/jetty9/LoggingHandler.java | 89 ++++++++++++++++--- .../runtimes/jetty9/StackDriverLogging.java | 33 ------- .../runtimes/jetty9/TracingLogHandler.java | 26 +++++- 3 files changed, 100 insertions(+), 48 deletions(-) delete mode 100644 jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/StackDriverLogging.java diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java index 93dbb465..7675d1ed 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java @@ -26,10 +26,13 @@ import com.google.cloud.logging.LoggingOptions; import com.google.cloud.logging.Payload; import com.google.cloud.logging.Severity; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import java.util.LinkedList; import java.util.List; +import java.util.Set; import java.util.logging.ErrorManager; import java.util.logging.Filter; import java.util.logging.Formatter; @@ -40,6 +43,8 @@ import java.util.logging.Logger; import java.util.logging.SimpleFormatter; + + /** * A logging handler that synchronously outputs logs generated with {@link java.util.logging.Logger} * to Stackdriver Logging. @@ -92,7 +97,13 @@ * */ public class LoggingHandler extends Handler { - private final ThreadLocal flushing = new ThreadLocal<>(); + + private static final String HANDLERS_PROPERTY = "handlers"; + private static final String ROOT_LOGGER_NAME = ""; + private static final String[] NO_HANDLERS = new String[0]; + private static final Set EXCLUDED_LOGGERS = ImmutableSet.of("io.grpc", "io.netty", + "com.google.api.client.http", "sun.net.www.protocol.http"); + private final LoggingOptions options; private final List buffer = new LinkedList<>(); private final WriteOption[] writeOptions; @@ -145,8 +156,63 @@ public LoggingHandler(String log, LoggingOptions options, MonitoredResource moni String logName = firstNonNull(log, helper.getProperty(className + ".log", "java.log")); MonitoredResource resource = firstNonNull(monitoredResource, getDefaultResource()); writeOptions = new WriteOption[]{WriteOption.logName(logName), WriteOption.resource(resource)}; + maskLoggers(); + } + + private static void maskLoggers() { + for (String loggerName : EXCLUDED_LOGGERS) { + Logger logger = Logger.getLogger(loggerName); + // We remove the Clould Logging handler if it has been registered for a logger that should be + // masked + List loggingHandlers = getLoggingHandlers(logger); + for (LoggingHandler loggingHandler : loggingHandlers) { + logger.removeHandler(loggingHandler); + } + // We mask ancestors if they have a Stackdriver Logging Handler registered + Logger currentLogger = logger; + Logger ancestor = currentLogger.getParent(); + boolean masked = false; + while (ancestor != null && !masked) { + if (hasLoggingHandler(ancestor)) { + currentLogger.setUseParentHandlers(false); + masked = true; + } + currentLogger = ancestor; + ancestor = ancestor.getParent(); + } + } + } + + private static List getLoggingHandlers(Logger logger) { + ImmutableList.Builder builder = ImmutableList.builder(); + for (Handler handler : logger.getHandlers()) { + if (handler instanceof LoggingHandler) { + builder.add((LoggingHandler) handler); + } + } + return builder.build(); } + private static boolean hasLoggingHandler(Logger logger) { + // look for Stackdriver Logging handler registered with addHandler() + for (Handler handler : logger.getHandlers()) { + if (handler instanceof LoggingHandler) { + return true; + } + } + // look for Stackdriver Logging handler registered via logging.properties + String loggerName = logger.getName(); + String propertyName = loggerName.equals(ROOT_LOGGER_NAME) + ? HANDLERS_PROPERTY : loggerName + "." + HANDLERS_PROPERTY; + String handlersProperty = LogManager.getLogManager().getProperty(propertyName); + String[] handlers = handlersProperty != null ? handlersProperty.split(",") : NO_HANDLERS; + for (String handlerName : handlers) { + if (handlerName.contains(LoggingHandler.class.getPackage().getName())) { + return true; + } + } + return false; + } private MonitoredResource getDefaultResource() { return MonitoredResource.of("global", ImmutableMap.of("project_id", options.getProjectId())); @@ -186,7 +252,7 @@ Filter getFilterProperty(String name, Filter defaultValue) { String stringFilter = manager.getProperty(name); try { if (stringFilter != null) { - Class clz = ClassLoader.getSystemClassLoader().loadClass(stringFilter); + Class clz = ClassLoader.getSystemClassLoader().loadClass(stringFilter); return (Filter) clz.newInstance(); } } catch (Exception ex) { @@ -199,7 +265,7 @@ Formatter getFormatterProperty(String name, Formatter defaultValue) { String stringFilter = manager.getProperty(name); try { if (stringFilter != null) { - Class clz = ClassLoader.getSystemClassLoader().loadClass(stringFilter); + Class clz = ClassLoader.getSystemClassLoader().loadClass(stringFilter); return (Formatter) clz.newInstance(); } } catch (Exception ex) { @@ -221,8 +287,8 @@ Logging getLogging() { @Override public synchronized void publish(LogRecord record) { - // check that the log record should be logged and we are not already flushing logs - if (!isLoggable(record) || Boolean.TRUE.equals(flushing.get())) { + // check that the log record should be logged + if (!isLoggable(record)) { return; } LogEntry entry = entryFor(record); @@ -248,13 +314,14 @@ private LogEntry entryFor(LogRecord record) { .addLabel("levelName", level.getName()) .addLabel("levelValue", String.valueOf(level.intValue())) .setSeverity(severityFor(level)); - return buildEntryFor(record, builder); + enhanceLogEntry(builder, record); + return builder.build(); } - protected LogEntry buildEntryFor(LogRecord record, LogEntry.Builder builder) { - return builder.build(); + protected void enhanceLogEntry(LogEntry.Builder builder, LogRecord record) { + // no-op in this class } - + private static Severity severityFor(Level level) { if (level instanceof LoggingLevel) { return ((LoggingLevel) level).getSeverity(); @@ -297,13 +364,11 @@ void write(List entries, WriteOption... options) { @Override public synchronized void flush() { try { - flushing.set(Boolean.TRUE); write(buffer, writeOptions); } catch (Exception ex) { // writing can fail but we should not throw an exception, we report the error instead reportError(null, ex, ErrorManager.FLUSH_FAILURE); } finally { - flushing.set(Boolean.FALSE); buffer.clear(); } } @@ -348,8 +413,8 @@ public synchronized long setFlushSize(long flushSize) { * Logging handlers instead of {@link Logger#addHandler(Handler)} to avoid infinite recursion * when logging. */ - @Deprecated public static void addHandler(Logger logger, LoggingHandler handler) { logger.addHandler(handler); + maskLoggers(); } } diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/StackDriverLogging.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/StackDriverLogging.java deleted file mode 100644 index f278824f..00000000 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/StackDriverLogging.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.google.cloud.runtimes.jetty9; - -import java.util.Arrays; -import java.util.logging.Logger; - -public class StackDriverLogging { - - static void init() throws Exception { - - Logger log = Logger.getLogger(StackDriverLogging.class.getName()); - - for (Logger l : new Logger[] {log, - Logger.getLogger("sun.net.www.protocol.http.HttpURLConnection")}) { - while (l != null) { - System.err.printf("Log '%s' upl=%b %s%n", l.getName(), l.getUseParentHandlers(), - Arrays.asList(l.getHandlers())); - l = l.getParent(); - } - } - Logger.getLogger("").info("root info!"); - log.info("test info!"); - log.fine("test fine!"); - - ((TracingLogHandler) Logger.getLogger("").getHandlers()[0]).flush(); - System.err.println("flushed"); - - Thread.sleep(5000); - } - - public static void main(String... args) throws Exception { - init(); - } -} diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java index 2642e909..8d374477 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java @@ -17,7 +17,6 @@ import static com.google.common.base.MoreObjects.firstNonNull; import com.google.cloud.MonitoredResource; -import com.google.cloud.logging.LogEntry; import com.google.cloud.logging.LogEntry.Builder; import com.google.cloud.logging.LoggingOptions; @@ -31,6 +30,8 @@ */ public class TracingLogHandler extends AsyncLoggingHandler { + private final ThreadLocal flushing = new ThreadLocal<>(); + /** * Construct a TracingLogHandler for "jetty.log" */ @@ -40,12 +41,22 @@ public TracingLogHandler() { "jetty.log"), null, null); } + @Override + public synchronized void publish(LogRecord record) { + // check we are not already flushing logs + if (Boolean.TRUE.equals(flushing.get())) { + return; + } + super.publish(record); + } + public TracingLogHandler(String logName, LoggingOptions options, MonitoredResource resource) { super(logName, options, resource); } @Override - protected LogEntry buildEntryFor(LogRecord record, Builder builder) { + protected void enhanceLogEntry(Builder builder, LogRecord record) { + super.enhanceLogEntry(builder, record); String traceid = RequestContextScope.getCurrentTraceid(); if (traceid != null) { builder.addLabel("traceid", traceid); @@ -58,6 +69,15 @@ protected LogEntry buildEntryFor(LogRecord record, Builder builder) { builder.addLabel("http-remote-addr", request.getRemoteAddr()); } } - return super.buildEntryFor(record, builder); + } + + @Override + public synchronized void flush() { + try { + flushing.set(Boolean.TRUE); + super.flush(); + } finally { + flushing.set(Boolean.FALSE); + } } } From 48df809c528f4bb3720b04e2f2df09385c6472c7 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Thu, 1 Dec 2016 10:24:49 +1100 Subject: [PATCH 08/65] Issue #68 Updated to latest gcloud API removed copied LoggingHandlers removed instanceid from monitored resource --- .../runtimes/jetty9/AsyncLoggingHandler.java | 103 ----- .../cloud/runtimes/jetty9/LoggingHandler.java | 420 ------------------ .../runtimes/jetty9/TracingLogHandler.java | 21 +- pom.xml | 2 +- 4 files changed, 18 insertions(+), 528 deletions(-) delete mode 100644 jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/AsyncLoggingHandler.java delete mode 100644 jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/AsyncLoggingHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/AsyncLoggingHandler.java deleted file mode 100644 index c786597b..00000000 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/AsyncLoggingHandler.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.runtimes.jetty9; - -import com.google.cloud.MonitoredResource; -import com.google.cloud.logging.LogEntry; -import com.google.cloud.logging.Logging.WriteOption; -import com.google.cloud.logging.LoggingOptions; - -import java.util.List; -import java.util.logging.Filter; -import java.util.logging.Formatter; -import java.util.logging.Logger; -import java.util.logging.SimpleFormatter; - -/** - * A logging handler that asynchronously outputs logs generated with - * {@link java.util.logging.Logger} to Stackdriver Logging. - * - *

Java logging levels (see {@link java.util.logging.Level}) are mapped to the following Google - * Stackdriver Logging severities: - * - * - * - * - * - * - * - * - * - * - *
Java LevelStackdriver Logging Severity
SEVEREERROR
WARNINGWARNING
INFOINFO
CONFIGINFO
FINEDEBUG
FINERDEBUG
FINESTDEBUG
- * - *

Original Java logging levels are added as labels (with {@code levelName} and - * {@code levelValue} keys, respectively) to the corresponding Stackdriver Logging {@link LogEntry}. - * You can read entry labels using {@link LogEntry#labels()}. To use logging levels that correspond - * to Stackdriver Logging severities you can use {@link LoggingLevel}. - * - *

Configuration: By default each {@code AsyncLoggingHandler} is initialized using the - * following {@code LogManager} configuration properties (that you can set in the - * {@code logging.properties} file. If properties are not defined (or have invalid values) then the - * specified default values are used. - *

    - *
  • {@code com.google.cloud.logging.AsyncLoggingHandler.log} the log name (defaults to - * {@code java.log}). - *
  • {@code com.google.cloud.logging.AsyncLoggingHandler.level} specifies the default level for - * the handler (defaults to {@code Level.INFO}). - *
  • {@code com.google.cloud.logging.AsyncLoggingHandler.filter} specifies the name of a - * {@link Filter} class to use (defaults to no filter). - *
  • {@code com.google.cloud.logging.AsyncLoggingHandler.formatter} specifies the name of a - * {@link Formatter} class to use (defaults to {@link SimpleFormatter}). - *
  • {@code com.google.cloud.logging.AsyncLoggingHandler.flushSize} specifies the maximum size of - * the log buffer. Once reached, logs are transmitted to the Stackdriver Logging service - * (defaults to 1). - *
  • {@code com.google.cloud.logging.AsyncLoggingHandler.flushLevel} specifies the flush log - * level. When a log with this level is published, logs are transmitted to the Stackdriver - * Logging service (defaults to {@link LoggingLevel#ERROR}). - *
- * - *

To add a {@code LoggingHandler} to an existing {@link Logger} and be sure to avoid infinite - * recursion when logging, use the {@link #addHandler(Logger, LoggingHandler)} method. Alternatively - * you can add the handler via {@code logging.properties}. For example using the following line: - * 

- * {@code com.example.mypackage.handlers=com.google.cloud.logging.AsyncLoggingHandler}
- *
- */ -public class AsyncLoggingHandler extends LoggingHandler { - - public AsyncLoggingHandler() { - super(); - } - - public AsyncLoggingHandler(String logName) { - super(logName); - } - - public AsyncLoggingHandler(String logName, LoggingOptions options) { - super(logName, options); - } - - public AsyncLoggingHandler(String logName, LoggingOptions options, MonitoredResource resource) { - super(logName, options, resource); - } - - @Override - void write(List entries, WriteOption... options) { - getLogging().writeAsync(entries, options); - } -} diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java deleted file mode 100644 index 7675d1ed..00000000 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/LoggingHandler.java +++ /dev/null @@ -1,420 +0,0 @@ -/* - * Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.runtimes.jetty9; - -import static com.google.common.base.MoreObjects.firstNonNull; - -import com.google.cloud.MonitoredResource; -import com.google.cloud.logging.LogEntry; -import com.google.cloud.logging.Logging; -import com.google.cloud.logging.Logging.WriteOption; -import com.google.cloud.logging.LoggingLevel; -import com.google.cloud.logging.LoggingOptions; -import com.google.cloud.logging.Payload; -import com.google.cloud.logging.Severity; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; - -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.logging.ErrorManager; -import java.util.logging.Filter; -import java.util.logging.Formatter; -import java.util.logging.Handler; -import java.util.logging.Level; -import java.util.logging.LogManager; -import java.util.logging.LogRecord; -import java.util.logging.Logger; -import java.util.logging.SimpleFormatter; - - - -/** - * A logging handler that synchronously outputs logs generated with {@link java.util.logging.Logger} - * to Stackdriver Logging. - * - *

Java logging levels (see {@link java.util.logging.Level}) are mapped to the following Google - * Stackdriver Logging severities: - * - * - * - * - * - * - * - * - * - * - *
Java LevelStackdriver Logging Severity
SEVEREERROR
WARNINGWARNING
INFOINFO
CONFIGINFO
FINEDEBUG
FINERDEBUG
FINESTDEBUG
- * - *

Original Java logging levels are added as labels (with {@code levelName} and - * {@code levelValue} keys, respectively) to the corresponding Stackdriver Logging {@link LogEntry}. - * You can read entry labels using {@link LogEntry#labels()}. To use logging levels that correspond - * to Stackdriver Logging severities you can use {@link LoggingLevel}. - * - *

Configuration: By default each {@code LoggingHandler} is initialized using the - * following {@code LogManager} configuration properties (that you can set in the - * {@code logging.properties} file). If properties are not defined (or have invalid values) then the - * specified default values are used. - *

    - *
  • {@code com.google.cloud.logging.LoggingHandler.log} the log name (defaults to - * {@code java.log}). - *
  • {@code com.google.cloud.logging.LoggingHandler.level} specifies the default level for the - * handler (defaults to {@code Level.INFO}). - *
  • {@code com.google.cloud.logging.LoggingHandler.filter} specifies the name of a {@link Filter} - * class to use (defaults to no filter). - *
  • {@code com.google.cloud.logging.LoggingHandler.formatter} specifies the name of a - * {@link Formatter} class to use (defaults to {@link SimpleFormatter}). - *
  • {@code com.google.cloud.logging.LoggingHandler.flushSize} specifies the maximum size of the - * log buffer. Once reached, logs are transmitted to the Stackdriver Logging service (defaults - * to 1). - *
  • {@code com.google.cloud.logging.LoggingHandler.flushLevel} specifies the flush log level. - * When a log with this level is published, logs are transmitted to the Stackdriver Logging - * service (defaults to {@link LoggingLevel#ERROR}). - *
- * - *

To add a {@code LoggingHandler} to an existing {@link Logger} and be sure to avoid infinite - * recursion when logging, use the {@link #addHandler(Logger, LoggingHandler)} method. Alternatively - * you can add the handler via {@code logging.properties}. For example using the following line: - * 

- * {@code com.example.mypackage.handlers=com.google.cloud.logging.LoggingHandler}
- *
- */ -public class LoggingHandler extends Handler { - - private static final String HANDLERS_PROPERTY = "handlers"; - private static final String ROOT_LOGGER_NAME = ""; - private static final String[] NO_HANDLERS = new String[0]; - private static final Set EXCLUDED_LOGGERS = ImmutableSet.of("io.grpc", "io.netty", - "com.google.api.client.http", "sun.net.www.protocol.http"); - - private final LoggingOptions options; - private final List buffer = new LinkedList<>(); - private final WriteOption[] writeOptions; - private Logging logging; - private Level flushLevel; - private long flushSize; - - /** - * Creates an handler that publishes messages to Stackdriver Logging. - */ - public LoggingHandler() { - this(null, null, null); - } - - /** - * Creates a handler that publishes messages to Stackdriver Logging. - * - * @param log the name of the log to which log entries are written - */ - public LoggingHandler(String log) { - this(log, null, null); - } - - /** - * Creates a handler that publishes messages to Stackdriver Logging. - * - * @param log the name of the log to which log entries are written - * @param options options for the Stackdriver Logging service - */ - public LoggingHandler(String log, LoggingOptions options) { - this(log, options, null); - } - - /** - * Creates a handler that publishes messages to Stackdriver Logging. - * - * @param log the name of the log to which log entries are written - * @param options options for the Stackdriver Logging service - * @param monitoredResource the monitored resource to which log entries refer - */ - public LoggingHandler(String log, LoggingOptions options, MonitoredResource monitoredResource) { - LogConfigHelper helper = new LogConfigHelper(); - String className = getClass().getName(); - this.options = options != null ? options : LoggingOptions.getDefaultInstance(); - this.flushLevel = helper.getLevelProperty(className + ".flushLevel", LoggingLevel.ERROR); - this.flushSize = helper.getLongProperty(className + ".flushSize", 1L); - setLevel(helper.getLevelProperty(className + ".level", Level.INFO)); - setFilter(helper.getFilterProperty(className + ".filter", null)); - setFormatter(helper.getFormatterProperty(className + ".formatter", new SimpleFormatter())); - String logName = firstNonNull(log, helper.getProperty(className + ".log", "java.log")); - MonitoredResource resource = firstNonNull(monitoredResource, getDefaultResource()); - writeOptions = new WriteOption[]{WriteOption.logName(logName), WriteOption.resource(resource)}; - maskLoggers(); - } - - private static void maskLoggers() { - for (String loggerName : EXCLUDED_LOGGERS) { - Logger logger = Logger.getLogger(loggerName); - // We remove the Clould Logging handler if it has been registered for a logger that should be - // masked - List loggingHandlers = getLoggingHandlers(logger); - for (LoggingHandler loggingHandler : loggingHandlers) { - logger.removeHandler(loggingHandler); - } - // We mask ancestors if they have a Stackdriver Logging Handler registered - Logger currentLogger = logger; - Logger ancestor = currentLogger.getParent(); - boolean masked = false; - while (ancestor != null && !masked) { - if (hasLoggingHandler(ancestor)) { - currentLogger.setUseParentHandlers(false); - masked = true; - } - currentLogger = ancestor; - ancestor = ancestor.getParent(); - } - } - } - - private static List getLoggingHandlers(Logger logger) { - ImmutableList.Builder builder = ImmutableList.builder(); - for (Handler handler : logger.getHandlers()) { - if (handler instanceof LoggingHandler) { - builder.add((LoggingHandler) handler); - } - } - return builder.build(); - } - - private static boolean hasLoggingHandler(Logger logger) { - // look for Stackdriver Logging handler registered with addHandler() - for (Handler handler : logger.getHandlers()) { - if (handler instanceof LoggingHandler) { - return true; - } - } - // look for Stackdriver Logging handler registered via logging.properties - String loggerName = logger.getName(); - String propertyName = loggerName.equals(ROOT_LOGGER_NAME) - ? HANDLERS_PROPERTY : loggerName + "." + HANDLERS_PROPERTY; - String handlersProperty = LogManager.getLogManager().getProperty(propertyName); - String[] handlers = handlersProperty != null ? handlersProperty.split(",") : NO_HANDLERS; - for (String handlerName : handlers) { - if (handlerName.contains(LoggingHandler.class.getPackage().getName())) { - return true; - } - } - return false; - } - - private MonitoredResource getDefaultResource() { - return MonitoredResource.of("global", ImmutableMap.of("project_id", options.getProjectId())); - } - - private static class LogConfigHelper { - - private final LogManager manager = LogManager.getLogManager(); - - String getProperty(String name, String defaultValue) { - return firstNonNull(manager.getProperty(name), defaultValue); - } - - long getLongProperty(String name, long defaultValue) { - try { - return Long.parseLong(manager.getProperty(name)); - } catch (NumberFormatException ex) { - // If the level does not exist we fall back to default value - } - return defaultValue; - } - - Level getLevelProperty(String name, Level defaultValue) { - String stringLevel = manager.getProperty(name); - if (stringLevel == null) { - return defaultValue; - } - try { - return Level.parse(stringLevel); - } catch (IllegalArgumentException ex) { - // If the level does not exist we fall back to default value - } - return defaultValue; - } - - Filter getFilterProperty(String name, Filter defaultValue) { - String stringFilter = manager.getProperty(name); - try { - if (stringFilter != null) { - Class clz = ClassLoader.getSystemClassLoader().loadClass(stringFilter); - return (Filter) clz.newInstance(); - } - } catch (Exception ex) { - // If we cannot create the filter we fall back to default value - } - return defaultValue; - } - - Formatter getFormatterProperty(String name, Formatter defaultValue) { - String stringFilter = manager.getProperty(name); - try { - if (stringFilter != null) { - Class clz = ClassLoader.getSystemClassLoader().loadClass(stringFilter); - return (Formatter) clz.newInstance(); - } - } catch (Exception ex) { - // If we cannot create the filter we fall back to default value - } - return defaultValue; - } - } - - /** - * Returns an instance of the logging service. - */ - Logging getLogging() { - if (logging == null) { - logging = options.getService(); - } - return logging; - } - - @Override - public synchronized void publish(LogRecord record) { - // check that the log record should be logged - if (!isLoggable(record)) { - return; - } - LogEntry entry = entryFor(record); - if (entry != null) { - buffer.add(entry); - } - if (buffer.size() >= flushSize || record.getLevel().intValue() >= flushLevel.intValue()) { - flush(); - } - } - - private LogEntry entryFor(LogRecord record) { - String payload; - try { - payload = getFormatter().format(record); - } catch (Exception ex) { - // Formatting can fail but we should not throw an exception, we report the error instead - reportError(null, ex, ErrorManager.FORMAT_FAILURE); - return null; - } - Level level = record.getLevel(); - LogEntry.Builder builder = LogEntry.newBuilder(Payload.StringPayload.of(payload)) - .addLabel("levelName", level.getName()) - .addLabel("levelValue", String.valueOf(level.intValue())) - .setSeverity(severityFor(level)); - enhanceLogEntry(builder, record); - return builder.build(); - } - - protected void enhanceLogEntry(LogEntry.Builder builder, LogRecord record) { - // no-op in this class - } - - private static Severity severityFor(Level level) { - if (level instanceof LoggingLevel) { - return ((LoggingLevel) level).getSeverity(); - } - switch (level.intValue()) { - // FINEST - case 300: - return Severity.DEBUG; - // FINER - case 400: - return Severity.DEBUG; - // FINE - case 500: - return Severity.DEBUG; - // CONFIG - case 700: - return Severity.INFO; - // INFO - case 800: - return Severity.INFO; - // WARNING - case 900: - return Severity.WARNING; - // SEVERE - case 1000: - return Severity.ERROR; - default: - return Severity.DEFAULT; - } - } - - /** - * Writes the provided list of log entries to Stackdriver Logging. Override this method to change - * how entries should be written. - */ - void write(List entries, WriteOption... options) { - getLogging().write(entries, options); - } - - @Override - public synchronized void flush() { - try { - write(buffer, writeOptions); - } catch (Exception ex) { - // writing can fail but we should not throw an exception, we report the error instead - reportError(null, ex, ErrorManager.FLUSH_FAILURE); - } finally { - buffer.clear(); - } - } - - /** - * Closes the handler and the associated {@link Logging} object. - */ - @Override - public synchronized void close() throws SecurityException { - if (logging != null) { - try { - logging.close(); - } catch (Exception ex) { - // ignore - } - } - logging = null; - } - - /** - * Sets the flush log level. When a log with this level is published, the log buffer is - * transmitted to the Stackdriver Logging service, regardless of its size. If not set, - * {@link LoggingLevel#ERROR} is used. - */ - public synchronized Level setFlushLevel(Level flushLevel) { - this.flushLevel = flushLevel; - return flushLevel; - } - - /** - * Sets the maximum size of the log buffer. Once the maximum size of the buffer is reached, logs - * are transmitted to the Stackdriver Logging service. If not set, a log is sent to the service as - * soon as published. - */ - public synchronized long setFlushSize(long flushSize) { - this.flushSize = flushSize; - return flushSize; - } - - /** - * Adds the provided {@code LoggingHandler} to {@code logger}. Use this method to register Cloud - * Logging handlers instead of {@link Logger#addHandler(Handler)} to avoid infinite recursion - * when logging. - */ - public static void addHandler(Logger logger, LoggingHandler handler) { - logger.addHandler(handler); - maskLoggers(); - } -} diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java index 8d374477..c50f5153 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java @@ -17,6 +17,7 @@ import static com.google.common.base.MoreObjects.firstNonNull; import com.google.cloud.MonitoredResource; +import com.google.cloud.logging.AsyncLoggingHandler; import com.google.cloud.logging.LogEntry.Builder; import com.google.cloud.logging.LoggingOptions; @@ -31,6 +32,8 @@ public class TracingLogHandler extends AsyncLoggingHandler { private final ThreadLocal flushing = new ThreadLocal<>(); + private final MonitoredResource monitored; + private final String instanceid; /** * Construct a TracingLogHandler for "jetty.log" @@ -41,6 +44,16 @@ public TracingLogHandler() { "jetty.log"), null, null); } + public TracingLogHandler(String logName, LoggingOptions options, MonitoredResource resource) { + super(logName, options, resource); + monitored = MonitoredResource.newBuilder("gae_app") + .addLabel("project_id", System.getenv("GCLOUD_PROJECT")) + .addLabel("module_id", System.getenv("GAE_SERVICE")) + .addLabel("version_id", System.getenv("GAE_VERSION")) + .build(); + instanceid = System.getenv("GAE_INSTANCE"); + } + @Override public synchronized void publish(LogRecord record) { // check we are not already flushing logs @@ -49,15 +62,12 @@ public synchronized void publish(LogRecord record) { } super.publish(record); } - - public TracingLogHandler(String logName, LoggingOptions options, MonitoredResource resource) { - super(logName, options, resource); - } @Override protected void enhanceLogEntry(Builder builder, LogRecord record) { super.enhanceLogEntry(builder, record); String traceid = RequestContextScope.getCurrentTraceid(); + builder.setResource(monitored); if (traceid != null) { builder.addLabel("traceid", traceid); } else { @@ -69,6 +79,9 @@ protected void enhanceLogEntry(Builder builder, LogRecord record) { builder.addLabel("http-remote-addr", request.getRemoteAddr()); } } + if (instanceid != null) { + builder.addLabel("instanceid", instanceid); + } } @Override diff --git a/pom.xml b/pom.xml index 4ad78745..00ec070c 100644 --- a/pom.xml +++ b/pom.xml @@ -34,7 +34,7 @@ UTF-8 yyyy-MM-dd_HH_mm 1.9.40 - 0.5.1 + 0.7.0 4 0 9.${jetty9.minor.version}.${jetty9.dot.version}.RC2 From 1e38c88c9a36e169ee7370c8949962074d4efea7 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 2 Dec 2016 08:19:49 +1100 Subject: [PATCH 09/65] Issue #68 Use same labels as nginx --- .../cloud/runtimes/jetty9/TracingLogHandler.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java index c50f5153..8605e511 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java @@ -44,6 +44,13 @@ public TracingLogHandler() { "jetty.log"), null, null); } + /** + * Construct a TracingLogHandler. + * + * @param logName Name of the log + * @param options LoggingOptions to access Google cloud API + * @param resource The resource to log against + */ public TracingLogHandler(String logName, LoggingOptions options, MonitoredResource resource) { super(logName, options, resource); monitored = MonitoredResource.newBuilder("gae_app") @@ -69,7 +76,7 @@ protected void enhanceLogEntry(Builder builder, LogRecord record) { String traceid = RequestContextScope.getCurrentTraceid(); builder.setResource(monitored); if (traceid != null) { - builder.addLabel("traceid", traceid); + builder.addLabel("appengine.googleapis.com/trace_id", traceid); } else { Request request = RequestContextScope.getCurrentRequest(); if (request != null) { @@ -80,7 +87,7 @@ protected void enhanceLogEntry(Builder builder, LogRecord record) { } } if (instanceid != null) { - builder.addLabel("instanceid", instanceid); + builder.addLabel("appengine.googleapis.com/instance_name", instanceid); } } From df430d6125c5c36e77ae4bc212cb16bdd1b8a6b7 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Tue, 6 Dec 2016 10:13:33 +1100 Subject: [PATCH 10/65] Issue #68 Code cleanups after review --- jetty9-base/pom.xml | 2 +- .../runtimes/jetty9/RequestContextScope.java | 28 ++++++++----------- .../runtimes/jetty9/TracingLogHandler.java | 25 +++++++++-------- 3 files changed, 25 insertions(+), 30 deletions(-) diff --git a/jetty9-base/pom.xml b/jetty9-base/pom.xml index 4f98a78f..b9cc9cdc 100644 --- a/jetty9-base/pom.xml +++ b/jetty9-base/pom.xml @@ -142,7 +142,7 @@ -jar ../jetty-distribution-${jetty9.version}/start.jar --create-startd - --add-to-start=http,deploy,jsp,jstl,http-forwarded,resources,gae,gcloud,jcl-slf4j,logging-jul + --add-to-start=server,http,deploy,jsp,jstl,http-forwarded,resources,gae,gcloud,jcl-slf4j,logging-jul --approve-all-licenses diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java index 7bf3bf7a..94ad0ef4 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java @@ -17,17 +17,25 @@ import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.ContextHandler; import org.eclipse.jetty.server.handler.ContextHandler.Context; +import org.eclipse.jetty.server.handler.ContextHandler.ContextScopeListener; import java.util.ArrayDeque; import java.util.Deque; import java.util.logging.Level; import java.util.logging.Logger; +/** + * A Jetty {@link ContextScopeListener} that is called whenever + * a container managed thread enters or exits the scope of a context and/or request. + * Used to maintain {@link ThreadLocal} references to the current request and + * Google traceID, primarily for logging. + * @see TracingLogHandler + */ public class RequestContextScope implements ContextHandler.ContextScopeListener { static final Logger logger = Logger.getLogger(RequestContextScope.class.getName()); private static final String X_CLOUD_TRACE = "x-cloud-trace-context"; - private static final ThreadLocal traceid = new ThreadLocal<>(); + private static final ThreadLocal traceId = new ThreadLocal<>(); private static final ThreadLocal> requestStack = new ThreadLocal>() { @Override @@ -52,7 +60,7 @@ public void enterScope(Context context, Request request, Object reason) { request.setAttribute(X_CLOUD_TRACE, id); } } - traceid.set(id); + traceId.set(id); } stack.push(request); } @@ -70,26 +78,12 @@ public void exitScope(Context context, Request request) { requestStack.get().pop(); } } - - /** Run a Runnable in the scope of a traceid. - * @param traceid The trace ID - * @param runnable the runnable - */ - public static void runWith(String traceid, Runnable runnable) { - String original = RequestContextScope.traceid.get(); - RequestContextScope.traceid.set(traceid); - try { - runnable.run(); - } finally { - RequestContextScope.traceid.set(original); - } - } public static Request getCurrentRequest() { return requestStack.get().peek(); } public static String getCurrentTraceid() { - return traceid.get(); + return traceId.get(); } } diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java index 8605e511..2feaeb6a 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java @@ -39,12 +39,18 @@ public class TracingLogHandler extends AsyncLoggingHandler { * Construct a TracingLogHandler for "jetty.log" */ public TracingLogHandler() { - this(firstNonNull( - LogManager.getLogManager().getProperty(TracingLogHandler.class.getName() + ".log"), - "jetty.log"), null, null); + this( + firstNonNull( + LogManager.getLogManager().getProperty(TracingLogHandler.class.getName() + ".log"), + "jetty.log"), + null, + MonitoredResource.newBuilder("gae_app") + .addLabel("project_id", System.getenv("GCLOUD_PROJECT")) + .addLabel("module_id", System.getenv("GAE_SERVICE")) + .addLabel("version_id", System.getenv("GAE_VERSION")).build()); } - /** + /** * Construct a TracingLogHandler. * * @param logName Name of the log @@ -53,21 +59,16 @@ public TracingLogHandler() { */ public TracingLogHandler(String logName, LoggingOptions options, MonitoredResource resource) { super(logName, options, resource); - monitored = MonitoredResource.newBuilder("gae_app") - .addLabel("project_id", System.getenv("GCLOUD_PROJECT")) - .addLabel("module_id", System.getenv("GAE_SERVICE")) - .addLabel("version_id", System.getenv("GAE_VERSION")) - .build(); + monitored = resource; instanceid = System.getenv("GAE_INSTANCE"); } @Override public synchronized void publish(LogRecord record) { // check we are not already flushing logs - if (Boolean.TRUE.equals(flushing.get())) { - return; + if (!Boolean.TRUE.equals(flushing.get())) { + super.publish(record); } - super.publish(record); } @Override From 9de4fc1796c4404f15a9de24a2425a38f9baf10e Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 23 Dec 2016 16:44:43 +1100 Subject: [PATCH 11/65] Issue #68 Removed stackdriver logging, so this just configures JUL to capture the traceid and log to stderr --- jetty9-base/pom.xml | 11 --- .../runtimes/jetty9/RequestContextScope.java | 18 ++-- .../runtimes/jetty9/TracingLogHandler.java | 96 ++++--------------- .../etc/java-util-logging.properties | 7 +- .../src/main/jetty-config/jetty.commands | 4 +- .../cloud/runtimes/jetty9/TestServlet.java | 32 +++---- 6 files changed, 45 insertions(+), 123 deletions(-) diff --git a/jetty9-base/pom.xml b/jetty9-base/pom.xml index e1392f30..ba2cf0a9 100644 --- a/jetty9-base/pom.xml +++ b/jetty9-base/pom.xml @@ -36,17 +36,6 @@ jar provided - - com.google.cloud - google-cloud-logging - ${gcloud.api.version} - - - javax.servlet - servlet-api - - - diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java index 94ad0ef4..dc26b83f 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java @@ -1,15 +1,17 @@ /* - * Copyright 2016 Google Inc. All Rights Reserved. + * Copyright (C) 2016 Google Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.google.cloud.runtimes.jetty9; diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java index 2feaeb6a..dded0055 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java @@ -1,104 +1,40 @@ /* - * Copyright 2016 Google Inc. All Rights Reserved. + * Copyright (C) 2016 Google Inc. * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package com.google.cloud.runtimes.jetty9; -import static com.google.common.base.MoreObjects.firstNonNull; - -import com.google.cloud.MonitoredResource; -import com.google.cloud.logging.AsyncLoggingHandler; -import com.google.cloud.logging.LogEntry.Builder; -import com.google.cloud.logging.LoggingOptions; - -import org.eclipse.jetty.server.Request; - -import java.util.logging.LogManager; +import java.util.logging.ConsoleHandler; import java.util.logging.LogRecord; /** - * A Google Cloud Logging Handler extended with a request traceid label. + * A Logging Handler extended with a request traceid label. */ -public class TracingLogHandler extends AsyncLoggingHandler { - - private final ThreadLocal flushing = new ThreadLocal<>(); - private final MonitoredResource monitored; - private final String instanceid; - - /** - * Construct a TracingLogHandler for "jetty.log" - */ - public TracingLogHandler() { - this( - firstNonNull( - LogManager.getLogManager().getProperty(TracingLogHandler.class.getName() + ".log"), - "jetty.log"), - null, - MonitoredResource.newBuilder("gae_app") - .addLabel("project_id", System.getenv("GCLOUD_PROJECT")) - .addLabel("module_id", System.getenv("GAE_SERVICE")) - .addLabel("version_id", System.getenv("GAE_VERSION")).build()); - } +public class TracingLogHandler extends ConsoleHandler { /** * Construct a TracingLogHandler. - * - * @param logName Name of the log - * @param options LoggingOptions to access Google cloud API - * @param resource The resource to log against */ - public TracingLogHandler(String logName, LoggingOptions options, MonitoredResource resource) { - super(logName, options, resource); - monitored = resource; - instanceid = System.getenv("GAE_INSTANCE"); - } + public TracingLogHandler() {} @Override public synchronized void publish(LogRecord record) { - // check we are not already flushing logs - if (!Boolean.TRUE.equals(flushing.get())) { - super.publish(record); - } - } - - @Override - protected void enhanceLogEntry(Builder builder, LogRecord record) { - super.enhanceLogEntry(builder, record); String traceid = RequestContextScope.getCurrentTraceid(); - builder.setResource(monitored); if (traceid != null) { - builder.addLabel("appengine.googleapis.com/trace_id", traceid); - } else { - Request request = RequestContextScope.getCurrentRequest(); - if (request != null) { - builder.addLabel("http-scheme", request.getScheme()); - builder.addLabel("http-method", request.getMethod()); - builder.addLabel("http-uri", request.getOriginalURI()); - builder.addLabel("http-remote-addr", request.getRemoteAddr()); - } - } - if (instanceid != null) { - builder.addLabel("appengine.googleapis.com/instance_name", instanceid); - } - } - - @Override - public synchronized void flush() { - try { - flushing.set(Boolean.TRUE); - super.flush(); - } finally { - flushing.set(Boolean.FALSE); + record.setMessage(record.getMessage() + " traceid=" + traceid); } + super.publish(record); } } diff --git a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties index 797d9932..ad1dc2d9 100644 --- a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties +++ b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties @@ -1,9 +1,6 @@ .level = INFO -io.netty.level = INFO -io.gcr.level = INFO handlers = com.google.cloud.runtimes.jetty9.TracingLogHandler com.google.cloud.runtimes.jetty9.TracingLogHandler.level = INFO - -# handlers = com.google.cloud.runtimes.jetty9.TracingLogHandler,java.util.logging.ConsoleHandler -# java.util.logging.ConsoleHandler.level = INFO +com.google.cloud.runtimes.jetty9.TracingLogHandler.formatter = java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=%1$tc:%4$s:%2$s: %5$s%n \ No newline at end of file diff --git a/jetty9-base/src/main/jetty-config/jetty.commands b/jetty9-base/src/main/jetty-config/jetty.commands index 3bf2c746..9a8c81e9 100644 --- a/jetty9-base/src/main/jetty-config/jetty.commands +++ b/jetty9-base/src/main/jetty-config/jetty.commands @@ -1,6 +1,6 @@ --create-startd - ---add-to-start=server,webapp,http,deploy,jsp,jstl,http-forwarded,resources,gae +--approve-all-licenses +--add-to-start=server,webapp,http,deploy,jsp,jstl,http-forwarded,resources,logging-jul,gae jetty.httpConfig.outputAggregationSize=32768 jetty.httpConfig.headerCacheSize=512 diff --git a/jetty9-base/src/test/java/com/google/cloud/runtimes/jetty9/TestServlet.java b/jetty9-base/src/test/java/com/google/cloud/runtimes/jetty9/TestServlet.java index c6937df7..33e86cfc 100644 --- a/jetty9-base/src/test/java/com/google/cloud/runtimes/jetty9/TestServlet.java +++ b/jetty9-base/src/test/java/com/google/cloud/runtimes/jetty9/TestServlet.java @@ -1,20 +1,18 @@ -// -// ======================================================================== -// Copyright (c) 1995-2016 Mort Bay Consulting Pty. Ltd. -// ------------------------------------------------------------------------ -// All rights reserved. This program and the accompanying materials -// are made available under the terms of the Eclipse Public License v1.0 -// and Apache License v2.0 which accompanies this distribution. -// -// The Eclipse Public License is available at -// http://www.eclipse.org/legal/epl-v10.html -// -// The Apache License v2.0 is available at -// http://www.opensource.org/licenses/apache2.0.php -// -// You may elect to redistribute this code under either of these licenses. -// ======================================================================== -// +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.cloud.runtimes.jetty9; From abc7b09bcac3e99d345cd9af3dfe06fcfb069dfc Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Thu, 29 Dec 2016 18:50:34 +1100 Subject: [PATCH 12/65] renamed traceid to traceId --- .../google/cloud/runtimes/jetty9/RequestContextScope.java | 2 +- .../google/cloud/runtimes/jetty9/TracingLogHandler.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java index dc26b83f..477d9b58 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java @@ -85,7 +85,7 @@ public static Request getCurrentRequest() { return requestStack.get().peek(); } - public static String getCurrentTraceid() { + public static String getCurrentTraceId() { return traceId.get(); } } diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java index dded0055..bf243ef1 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java @@ -20,7 +20,7 @@ import java.util.logging.LogRecord; /** - * A Logging Handler extended with a request traceid label. + * A Logging Handler extended with a request traceId label. */ public class TracingLogHandler extends ConsoleHandler { @@ -31,9 +31,9 @@ public TracingLogHandler() {} @Override public synchronized void publish(LogRecord record) { - String traceid = RequestContextScope.getCurrentTraceid(); - if (traceid != null) { - record.setMessage(record.getMessage() + " traceid=" + traceid); + String traceId = RequestContextScope.getCurrentTraceId(); + if (traceId != null) { + record.setMessage(record.getMessage() + " traceId=" + traceId); } super.publish(record); } From 9318aa1220fa3a81651bf74efb8a0e55ca4107cb Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Thu, 29 Dec 2016 18:57:55 +1100 Subject: [PATCH 13/65] gcloud api not currently used --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index 59681767..73af7c4e 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,6 @@ UTF-8 UTF-8 yyyy-MM-dd_HH_mm - 0.7.0 4 0 9.${jetty9.minor.version}.${jetty9.dot.version}.v20161208 From d50f5ae63ee6435a0ad205b4a5f5e3ae7db6e678 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 4 Jan 2017 16:43:24 +1100 Subject: [PATCH 14/65] Tunnel traceId in parameters #68 --- .../runtimes/jetty9/RequestContextScope.java | 60 ++++++------- .../runtimes/jetty9/TracingLogFormatter.java | 90 +++++++++++++++++++ .../runtimes/jetty9/TracingLogHandler.java | 80 ++++++++++++++++- .../etc/java-util-logging.properties | 3 +- .../cloud/runtimes/jetty9/TestServlet.java | 9 +- 5 files changed, 199 insertions(+), 43 deletions(-) create mode 100644 jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogFormatter.java diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java index 477d9b58..d2049b53 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java @@ -21,8 +21,6 @@ import org.eclipse.jetty.server.handler.ContextHandler.Context; import org.eclipse.jetty.server.handler.ContextHandler.ContextScopeListener; -import java.util.ArrayDeque; -import java.util.Deque; import java.util.logging.Level; import java.util.logging.Logger; @@ -37,37 +35,33 @@ public class RequestContextScope implements ContextHandler.ContextScopeListener static final Logger logger = Logger.getLogger(RequestContextScope.class.getName()); private static final String X_CLOUD_TRACE = "x-cloud-trace-context"; - private static final ThreadLocal traceId = new ThreadLocal<>(); - private static final ThreadLocal> requestStack = - new ThreadLocal>() { - @Override - protected Deque initialValue() { - return new ArrayDeque<>(); - } - }; + private static final ThreadLocal contextDepth = new ThreadLocal<>(); @Override public void enterScope(Context context, Request request, Object reason) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("enterScope " + context); + } if (request != null) { - Deque stack = requestStack.get(); - if (stack.isEmpty()) { - String id = (String) request.getAttribute(X_CLOUD_TRACE); - if (id == null) { - id = request.getHeader(X_CLOUD_TRACE); - if (id != null) { - int slash = id.indexOf('/'); + Integer depth = contextDepth.get(); + if (depth == null || depth.intValue() == 0) { + depth = 1; + String traceId = (String) request.getAttribute(X_CLOUD_TRACE); + if (traceId == null) { + traceId = request.getHeader(X_CLOUD_TRACE); + if (traceId != null) { + int slash = traceId.indexOf('/'); if (slash >= 0) { - id = id.substring(0, slash); + traceId = traceId.substring(0, slash); } - request.setAttribute(X_CLOUD_TRACE, id); + request.setAttribute(X_CLOUD_TRACE, traceId); + TracingLogHandler.setCurrentTraceId(traceId); } + } else { + depth = depth + 1; } - traceId.set(id); + contextDepth.set(depth); } - stack.push(request); - } - if (logger.isLoggable(Level.FINE)) { - logger.fine("enterScope " + context); } } @@ -76,16 +70,14 @@ public void exitScope(Context context, Request request) { if (logger.isLoggable(Level.FINE)) { logger.fine("exitScope " + context); } - if (request != null) { - requestStack.get().pop(); + Integer depth = contextDepth.get(); + if (depth != null) { + if (depth > 1) { + contextDepth.set(depth - 1); + } else { + contextDepth.remove(); + TracingLogHandler.setCurrentTraceId(null); + } } } - - public static Request getCurrentRequest() { - return requestStack.get().peek(); - } - - public static String getCurrentTraceId() { - return traceId.get(); - } } diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogFormatter.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogFormatter.java new file mode 100644 index 00000000..4063a51d --- /dev/null +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogFormatter.java @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.runtimes.jetty9; + +import com.google.cloud.runtimes.jetty9.TracingLogHandler.TraceId; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Date; +import java.util.logging.Formatter; +import java.util.logging.LogRecord; + +public class TracingLogFormatter extends Formatter { + + private final String format; + private final Date date = new Date(); + + /** + * Construct TracingLogFormater. + * Only called from TracingLogHandler + * @param format The format to log + */ + TracingLogFormatter(String format) { + this.format = format != null + ? format + : "%1$tc:%4$s:%2$s: %5$s traceId=%7$s %6$s%n"; + } + + @Override + public synchronized String format(LogRecord record) { + date.setTime(record.getMillis()); + String source; + if (record.getSourceClassName() != null) { + source = record.getSourceClassName(); + if (record.getSourceMethodName() != null) { + source += " " + record.getSourceMethodName(); + } + } else { + source = record.getLoggerName(); + } + + // Look for a TraceId in the message format parameters array. + String traceId = null; + Object[] params = record.getParameters(); + if (params != null && params.length > 0 && params[params.length - 1] instanceof TraceId) { + // remove the TraceId from the parameters array prior to formatting the message + traceId = params[params.length - 1].toString(); + record.setParameters(params.length == 1 ? null : Arrays.copyOf(params, params.length - 1)); + } + + // format the message + String message = formatMessage(record); + + // format any associated Throwable. + String throwable = ""; + if (record.getThrown() != null) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + pw.println(); + record.getThrown().printStackTrace(pw); + // TODO Should the thrown cause and/or suppressed exceptions also be logged? + pw.close(); + throwable = sw.toString(); + } + return String.format( + format, + /* %1 */ date, + /* %2 */ source, + /* %3 */ record.getLoggerName(), + /* %4 */ record.getLevel().toString(), + /* %5 */ message, + /* %6 */ throwable, + /* %7 */ traceId); + } +} diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java index bf243ef1..e4ba376a 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java @@ -16,25 +16,99 @@ package com.google.cloud.runtimes.jetty9; +import java.util.Arrays; import java.util.logging.ConsoleHandler; +import java.util.logging.LogManager; import java.util.logging.LogRecord; /** * A Logging Handler extended with a request traceId label. + * The traceId is passed in the LogRecord as an additional + * message format parameter. This must be stripped by the + * formatter before formatting the message, as is done by + * the {@link TracingLogFormatter}. + * @see TracingLogFormatter */ public class TracingLogHandler extends ConsoleHandler { + private static final ThreadLocal traceId = new ThreadLocal<>(); + + /** + * Set the Trace ID associated with any logging done by + * the current thread. + * @param id The traceID + */ + public static void setCurrentTraceId(String id) { + traceId.set(id); + } + + /** + * Get the Trace ID associated with any logging done by + * the current thread. + * @return id The traceID + */ + public static String getCurrentTraceId() { + return traceId.get(); + } + /** * Construct a TracingLogHandler. */ - public TracingLogHandler() {} + public TracingLogHandler() { + configure(); + } + + // Private method to configure a StreamHandler from LogManager + // properties and/or default values as specified in the class + // javadoc. + private void configure() { + LogManager manager = LogManager.getLogManager(); + String cname = getClass().getName(); + + String formatter = manager.getProperty(cname + ".formatter"); + if (formatter != null && formatter.equals(TracingLogFormatter.class.getName())) { + throw new IllegalStateException("Use String 'format' rather than Class 'fomatter' for " + + TracingLogFormatter.class.getName()); + } + + String format = manager.getProperty(cname + ".format"); + setFormatter(new TracingLogFormatter(format)); + } @Override public synchronized void publish(LogRecord record) { - String traceId = RequestContextScope.getCurrentTraceId(); + String traceId = getCurrentTraceId(); if (traceId != null) { - record.setMessage(record.getMessage() + " traceId=" + traceId); + // Encode the traceId as an additional message format + // parameter for the LogRecord. This should be removed by + // the formatter before formatting the message and used to + // format the LogRecord itself + Object[] params = record.getParameters(); + if (params == null) { + record.setParameters(new Object[] {new TraceId(traceId)}); + } else { + params = Arrays.copyOf(params,params.length + 1); + params[params.length - 1] = new TraceId(traceId); + record.setParameters(params); + } } super.publish(record); } + + /** + * Class to identify and hold the traceId in the + * LogRecord parameters array. + */ + public static class TraceId { + private final String traceId; + + public TraceId(String traceId) { + this.traceId = traceId; + } + + public String toString() { + return traceId; + } + } + } diff --git a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties index ad1dc2d9..752d84ce 100644 --- a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties +++ b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties @@ -2,5 +2,4 @@ handlers = com.google.cloud.runtimes.jetty9.TracingLogHandler com.google.cloud.runtimes.jetty9.TracingLogHandler.level = INFO -com.google.cloud.runtimes.jetty9.TracingLogHandler.formatter = java.util.logging.SimpleFormatter -java.util.logging.SimpleFormatter.format=%1$tc:%4$s:%2$s: %5$s%n \ No newline at end of file +com.google.cloud.runtimes.jetty9.TracingLogHandler.format = %1$tc:%4$s:%2$s: %5$s traceId=%7$s %6$s%n \ No newline at end of file diff --git a/jetty9-base/src/test/java/com/google/cloud/runtimes/jetty9/TestServlet.java b/jetty9-base/src/test/java/com/google/cloud/runtimes/jetty9/TestServlet.java index 33e86cfc..e6faf3e1 100644 --- a/jetty9-base/src/test/java/com/google/cloud/runtimes/jetty9/TestServlet.java +++ b/jetty9-base/src/test/java/com/google/cloud/runtimes/jetty9/TestServlet.java @@ -43,6 +43,7 @@ public void init() throws ServletException { getServletContext().log("init ServletContext.log"); } + @Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { log.info("doGet info"); @@ -51,10 +52,10 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) if (request.getParameter("ex") != null) { try { throw (Throwable) Class.forName(request.getParameter("ex")).newInstance(); - } catch (ServletException | IOException e) { - throw e; - } catch (Throwable e) { - throw new ServletException(e); + } catch (ServletException | IOException ex) { + throw ex; + } catch (Throwable th) { + throw new ServletException(th); } } response.getWriter().println("Log Test"); From 9e5e2dba3032b1d80ca9f8fc9ee4bf5221695761 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 4 Jan 2017 17:07:00 +1100 Subject: [PATCH 15/65] Simplified with traceId lookup in formatter #68 --- .../runtimes/jetty9/TracingLogFormatter.java | 14 +----- .../runtimes/jetty9/TracingLogHandler.java | 46 +------------------ 2 files changed, 3 insertions(+), 57 deletions(-) diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogFormatter.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogFormatter.java index 4063a51d..6ba8db80 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogFormatter.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogFormatter.java @@ -16,11 +16,8 @@ package com.google.cloud.runtimes.jetty9; -import com.google.cloud.runtimes.jetty9.TracingLogHandler.TraceId; - import java.io.PrintWriter; import java.io.StringWriter; -import java.util.Arrays; import java.util.Date; import java.util.logging.Formatter; import java.util.logging.LogRecord; @@ -54,15 +51,6 @@ public synchronized String format(LogRecord record) { source = record.getLoggerName(); } - // Look for a TraceId in the message format parameters array. - String traceId = null; - Object[] params = record.getParameters(); - if (params != null && params.length > 0 && params[params.length - 1] instanceof TraceId) { - // remove the TraceId from the parameters array prior to formatting the message - traceId = params[params.length - 1].toString(); - record.setParameters(params.length == 1 ? null : Arrays.copyOf(params, params.length - 1)); - } - // format the message String message = formatMessage(record); @@ -85,6 +73,6 @@ public synchronized String format(LogRecord record) { /* %4 */ record.getLevel().toString(), /* %5 */ message, /* %6 */ throwable, - /* %7 */ traceId); + /* %7 */ TracingLogHandler.getCurrentTraceId()); } } diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java index e4ba376a..047171bf 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java @@ -16,17 +16,12 @@ package com.google.cloud.runtimes.jetty9; -import java.util.Arrays; import java.util.logging.ConsoleHandler; import java.util.logging.LogManager; -import java.util.logging.LogRecord; /** - * A Logging Handler extended with a request traceId label. - * The traceId is passed in the LogRecord as an additional - * message format parameter. This must be stripped by the - * formatter before formatting the message, as is done by - * the {@link TracingLogFormatter}. + * A Logging Handler that uses the {@link TracingLogFormatter} + * to log the trace ID. * @see TracingLogFormatter */ public class TracingLogHandler extends ConsoleHandler { @@ -74,41 +69,4 @@ private void configure() { String format = manager.getProperty(cname + ".format"); setFormatter(new TracingLogFormatter(format)); } - - @Override - public synchronized void publish(LogRecord record) { - String traceId = getCurrentTraceId(); - if (traceId != null) { - // Encode the traceId as an additional message format - // parameter for the LogRecord. This should be removed by - // the formatter before formatting the message and used to - // format the LogRecord itself - Object[] params = record.getParameters(); - if (params == null) { - record.setParameters(new Object[] {new TraceId(traceId)}); - } else { - params = Arrays.copyOf(params,params.length + 1); - params[params.length - 1] = new TraceId(traceId); - record.setParameters(params); - } - } - super.publish(record); - } - - /** - * Class to identify and hold the traceId in the - * LogRecord parameters array. - */ - public static class TraceId { - private final String traceId; - - public TraceId(String traceId) { - this.traceId = traceId; - } - - public String toString() { - return traceId; - } - } - } From d5f56138f6c07c8f3fd60bb433d24119b61c9250 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Thu, 12 Jan 2017 18:27:25 +1100 Subject: [PATCH 16/65] Use 8.2 beta stackdriver --- jetty9-base/pom.xml | 11 +++ .../runtimes/jetty9/TracingLogFormatter.java | 78 ------------------ .../runtimes/jetty9/TracingLogHandler.java | 79 ++++++++++++++----- .../etc/java-util-logging.properties | 1 - pom.xml | 1 + 5 files changed, 71 insertions(+), 99 deletions(-) delete mode 100644 jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogFormatter.java diff --git a/jetty9-base/pom.xml b/jetty9-base/pom.xml index ba2cf0a9..e1392f30 100644 --- a/jetty9-base/pom.xml +++ b/jetty9-base/pom.xml @@ -36,6 +36,17 @@ jar provided + + com.google.cloud + google-cloud-logging + ${gcloud.api.version} + + + javax.servlet + servlet-api + + + diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogFormatter.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogFormatter.java deleted file mode 100644 index 6ba8db80..00000000 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogFormatter.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (C) 2016 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.runtimes.jetty9; - -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.Date; -import java.util.logging.Formatter; -import java.util.logging.LogRecord; - -public class TracingLogFormatter extends Formatter { - - private final String format; - private final Date date = new Date(); - - /** - * Construct TracingLogFormater. - * Only called from TracingLogHandler - * @param format The format to log - */ - TracingLogFormatter(String format) { - this.format = format != null - ? format - : "%1$tc:%4$s:%2$s: %5$s traceId=%7$s %6$s%n"; - } - - @Override - public synchronized String format(LogRecord record) { - date.setTime(record.getMillis()); - String source; - if (record.getSourceClassName() != null) { - source = record.getSourceClassName(); - if (record.getSourceMethodName() != null) { - source += " " + record.getSourceMethodName(); - } - } else { - source = record.getLoggerName(); - } - - // format the message - String message = formatMessage(record); - - // format any associated Throwable. - String throwable = ""; - if (record.getThrown() != null) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - pw.println(); - record.getThrown().printStackTrace(pw); - // TODO Should the thrown cause and/or suppressed exceptions also be logged? - pw.close(); - throwable = sw.toString(); - } - return String.format( - format, - /* %1 */ date, - /* %2 */ source, - /* %3 */ record.getLoggerName(), - /* %4 */ record.getLevel().toString(), - /* %5 */ message, - /* %6 */ throwable, - /* %7 */ TracingLogHandler.getCurrentTraceId()); - } -} diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java index 047171bf..b0935b59 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java @@ -16,15 +16,21 @@ package com.google.cloud.runtimes.jetty9; -import java.util.logging.ConsoleHandler; + +import static com.google.common.base.MoreObjects.firstNonNull; + +import com.google.cloud.MonitoredResource; +import com.google.cloud.logging.AsyncLoggingHandler; +import com.google.cloud.logging.LogEntry.Builder; +import com.google.cloud.logging.LoggingOptions; + import java.util.logging.LogManager; +import java.util.logging.LogRecord; /** - * A Logging Handler that uses the {@link TracingLogFormatter} - * to log the trace ID. - * @see TracingLogFormatter + * A Google Cloud Logging Handler extended with a request traceid label. */ -public class TracingLogHandler extends ConsoleHandler { +public class TracingLogHandler extends AsyncLoggingHandler { private static final ThreadLocal traceId = new ThreadLocal<>(); @@ -46,27 +52,60 @@ public static String getCurrentTraceId() { return traceId.get(); } + private final MonitoredResource monitored; + private final String instanceid; + /** - * Construct a TracingLogHandler. + * Construct a TracingLogHandler for "jetty.log" */ public TracingLogHandler() { - configure(); + this( + firstNonNull( + LogManager.getLogManager().getProperty(TracingLogHandler.class.getName() + ".log"), + "jetty.log"), + null, + addLabel(addLabel( + addLabel(MonitoredResource.newBuilder("gae_app"), + "project_id", "GCLOUD_PROJECT"), + "module_id", "GAE_SERVICE"), + "version_id", "GAE_VERSION").build()); } - // Private method to configure a StreamHandler from LogManager - // properties and/or default values as specified in the class - // javadoc. - private void configure() { - LogManager manager = LogManager.getLogManager(); - String cname = getClass().getName(); - - String formatter = manager.getProperty(cname + ".formatter"); - if (formatter != null && formatter.equals(TracingLogFormatter.class.getName())) { - throw new IllegalStateException("Use String 'format' rather than Class 'fomatter' for " - + TracingLogFormatter.class.getName()); + private static MonitoredResource.Builder addLabel( + MonitoredResource.Builder builder, + String label, + String env) { + String value = System.getenv(env); + if (value != null) { + builder.addLabel(label, value); } + return builder; + } + + /** + * Construct a TracingLogHandler. + * + * @param logName Name of the log + * @param options LoggingOptions to access Google cloud API + * @param resource The resource to log against + */ + public TracingLogHandler(String logName, LoggingOptions options, MonitoredResource resource) { + super(logName, options, resource); + monitored = resource; + instanceid = System.getenv("GAE_INSTANCE"); + } - String format = manager.getProperty(cname + ".format"); - setFormatter(new TracingLogFormatter(format)); + @Override + protected void enhanceLogEntry(Builder builder, LogRecord record) { + super.enhanceLogEntry(builder, record); + String traceId = getCurrentTraceId(); + builder.setResource(monitored); + if (traceId != null) { + builder.addLabel("appengine.googleapis.com/trace_id", traceId); + } + if (instanceid != null) { + builder.addLabel("appengine.googleapis.com/instance_name", instanceid); + } } } + diff --git a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties index 752d84ce..ca630033 100644 --- a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties +++ b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties @@ -2,4 +2,3 @@ handlers = com.google.cloud.runtimes.jetty9.TracingLogHandler com.google.cloud.runtimes.jetty9.TracingLogHandler.level = INFO -com.google.cloud.runtimes.jetty9.TracingLogHandler.format = %1$tc:%4$s:%2$s: %5$s traceId=%7$s %6$s%n \ No newline at end of file diff --git a/pom.xml b/pom.xml index 73af7c4e..ccd9082b 100644 --- a/pom.xml +++ b/pom.xml @@ -33,6 +33,7 @@ UTF-8 UTF-8 yyyy-MM-dd_HH_mm + 0.8.2-beta-SNAPSHOT 4 0 9.${jetty9.minor.version}.${jetty9.dot.version}.v20161208 From 9a22c3d5ace3fc1e259e0624c5e07410a6e75900 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 13 Jan 2017 09:30:37 +1100 Subject: [PATCH 17/65] Improved formatting --- .../com/google/cloud/runtimes/jetty9/TracingLogHandler.java | 1 + .../src/main/jetty-base/etc/java-util-logging.properties | 2 ++ 2 files changed, 3 insertions(+) diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java index b0935b59..04f1af54 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java @@ -106,6 +106,7 @@ protected void enhanceLogEntry(Builder builder, LogRecord record) { if (instanceid != null) { builder.addLabel("appengine.googleapis.com/instance_name", instanceid); } + builder.setTimestamp(record.getMillis()); } } diff --git a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties index ca630033..07876a68 100644 --- a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties +++ b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties @@ -2,3 +2,5 @@ handlers = com.google.cloud.runtimes.jetty9.TracingLogHandler com.google.cloud.runtimes.jetty9.TracingLogHandler.level = INFO +com.google.cloud.runtimes.jetty9.TracingLogHandler.formatter = java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=%2$s: %5$s%6$s \ No newline at end of file From 4f738b7915672789514faaa829bec628e4cd3f8e Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 13 Jan 2017 11:00:07 +1100 Subject: [PATCH 18/65] use logger name rather than source --- .../src/main/jetty-base/etc/java-util-logging.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties index 07876a68..f429a98d 100644 --- a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties +++ b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties @@ -3,4 +3,4 @@ handlers = com.google.cloud.runtimes.jetty9.TracingLogHandler com.google.cloud.runtimes.jetty9.TracingLogHandler.level = INFO com.google.cloud.runtimes.jetty9.TracingLogHandler.formatter = java.util.logging.SimpleFormatter -java.util.logging.SimpleFormatter.format=%2$s: %5$s%6$s \ No newline at end of file +java.util.logging.SimpleFormatter.format=%3$s: %5$s%6$s \ No newline at end of file From 34e91036b4f76aa4d7b9257291f20b62d9b10f3f Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 13 Jan 2017 15:51:17 +1100 Subject: [PATCH 19/65] run jetty from the webapp working directory so logging configuration may be relative --- jetty9-base/src/main/jetty-base/etc/gae-web.xml | 2 +- jetty9/src/main/docker/docker-entrypoint.bash | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/jetty9-base/src/main/jetty-base/etc/gae-web.xml b/jetty9-base/src/main/jetty-base/etc/gae-web.xml index 7a65a351..3dfd924e 100644 --- a/jetty9-base/src/main/jetty-base/etc/gae-web.xml +++ b/jetty9-base/src/main/jetty-base/etc/gae-web.xml @@ -20,6 +20,6 @@ - etc/gae-override-web.xml + /etc/gae-override-web.xml diff --git a/jetty9/src/main/docker/docker-entrypoint.bash b/jetty9/src/main/docker/docker-entrypoint.bash index 1c4be3e4..1ae8973f 100644 --- a/jetty9/src/main/docker/docker-entrypoint.bash +++ b/jetty9/src/main/docker/docker-entrypoint.bash @@ -5,8 +5,8 @@ DEFAULT_ARGS="-Djetty.base=$JETTY_BASE -jar $JETTY_HOME/start.jar" # Unpack a WAR app (if present) beforehand so that Stackdriver Debugger # can load it. This should be done before the JVM for Jetty starts up. -ROOT_WAR=$JETTY_BASE/webapps/root.war -ROOT_DIR=$JETTY_BASE/webapps/root +: ${ROOT_WAR:=$JETTY_BASE/webapps/root.war} +: ${ROOT_DIR:=$JETTY_BASE/webapps/root} if [ -e "$ROOT_WAR" ]; then # Unpack it only if $ROOT_DIR doesn't exist or the root is older than the war. if [ -e "$ROOT_WAR" -a \( \( ! -e "$ROOT_DIR" \) -o \( "$ROOT_DIR" -ot "$ROOT_WAR" \) \) ]; then @@ -20,6 +20,9 @@ if [ ! -e "$ROOT_DIR" -a -d /app ]; then ln -s /app "$ROOT_DIR" fi +# Set working directory to the root application directory +cd "$ROOT_DIR" + # If the passed arguments start with the java command if [ "java" = "$1" -o "$(which java)" = "$1" ] ; then # ignore the java command as it is the default From f255ee13549299da02ea79d635aa965f4e7db65d Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Thu, 19 Jan 2017 00:35:26 +1100 Subject: [PATCH 20/65] Testing logging Test https://github.com/GoogleCloudPlatform/google-cloud-java/pull/1535 and jetty-9.4.1-SNAPSHOT --- .../runtimes/jetty9/RequestContextScope.java | 6 ++-- .../runtimes/jetty9/TracingLogHandler.java | 29 ++----------------- .../etc/java-util-logging.properties | 13 +++++---- pom.xml | 4 +-- 4 files changed, 16 insertions(+), 36 deletions(-) diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java index d2049b53..4b694171 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java @@ -16,6 +16,8 @@ package com.google.cloud.runtimes.jetty9; +import com.google.cloud.logging.GaeFlexLoggingEnhancer; + import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.ContextHandler; import org.eclipse.jetty.server.handler.ContextHandler.Context; @@ -55,7 +57,7 @@ public void enterScope(Context context, Request request, Object reason) { traceId = traceId.substring(0, slash); } request.setAttribute(X_CLOUD_TRACE, traceId); - TracingLogHandler.setCurrentTraceId(traceId); + GaeFlexLoggingEnhancer.setCurrentTraceId(traceId); } } else { depth = depth + 1; @@ -76,7 +78,7 @@ public void exitScope(Context context, Request request) { contextDepth.set(depth - 1); } else { contextDepth.remove(); - TracingLogHandler.setCurrentTraceId(null); + GaeFlexLoggingEnhancer.setCurrentTraceId(null); } } } diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java index 04f1af54..e5e52295 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java @@ -51,10 +51,7 @@ public static void setCurrentTraceId(String id) { public static String getCurrentTraceId() { return traceId.get(); } - - private final MonitoredResource monitored; - private final String instanceid; - + /** * Construct a TracingLogHandler for "jetty.log" */ @@ -63,23 +60,7 @@ public TracingLogHandler() { firstNonNull( LogManager.getLogManager().getProperty(TracingLogHandler.class.getName() + ".log"), "jetty.log"), - null, - addLabel(addLabel( - addLabel(MonitoredResource.newBuilder("gae_app"), - "project_id", "GCLOUD_PROJECT"), - "module_id", "GAE_SERVICE"), - "version_id", "GAE_VERSION").build()); - } - - private static MonitoredResource.Builder addLabel( - MonitoredResource.Builder builder, - String label, - String env) { - String value = System.getenv(env); - if (value != null) { - builder.addLabel(label, value); - } - return builder; + null,null); } /** @@ -91,21 +72,15 @@ private static MonitoredResource.Builder addLabel( */ public TracingLogHandler(String logName, LoggingOptions options, MonitoredResource resource) { super(logName, options, resource); - monitored = resource; - instanceid = System.getenv("GAE_INSTANCE"); } @Override protected void enhanceLogEntry(Builder builder, LogRecord record) { super.enhanceLogEntry(builder, record); String traceId = getCurrentTraceId(); - builder.setResource(monitored); if (traceId != null) { builder.addLabel("appengine.googleapis.com/trace_id", traceId); } - if (instanceid != null) { - builder.addLabel("appengine.googleapis.com/instance_name", instanceid); - } builder.setTimestamp(record.getMillis()); } } diff --git a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties index f429a98d..5631f48e 100644 --- a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties +++ b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties @@ -1,6 +1,9 @@ -.level = INFO +.level=INFO -handlers = com.google.cloud.runtimes.jetty9.TracingLogHandler -com.google.cloud.runtimes.jetty9.TracingLogHandler.level = INFO -com.google.cloud.runtimes.jetty9.TracingLogHandler.formatter = java.util.logging.SimpleFormatter -java.util.logging.SimpleFormatter.format=%3$s: %5$s%6$s \ No newline at end of file +handlers=com.google.cloud.logging.LoggingHandler +com.google.cloud.logging.LoggingHandler.level=FINE +com.google.cloud.logging.LoggingHandler.log=gae_app.log +com.google.cloud.logging.LoggingHandler.resourceType=gae_app +com.google.cloud.logging.LoggingHandler.enhancers=com.google.cloud.logging.GaeFlexLoggingEnhancer +com.google.cloud.logging.LoggingHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=%3$s: %5$s%6$s diff --git a/pom.xml b/pom.xml index ccd9082b..94b6a5ca 100644 --- a/pom.xml +++ b/pom.xml @@ -35,8 +35,8 @@ yyyy-MM-dd_HH_mm 0.8.2-beta-SNAPSHOT 4 - 0 - 9.${jetty9.minor.version}.${jetty9.dot.version}.v20161208 + 1 + 9.${jetty9.minor.version}.${jetty9.dot.version}-SNAPSHOT ${docker.tag.prefix}9.${jetty9.minor.version} ${docker.tag.prefix}9.${jetty9.minor.version}-${maven.build.timestamp} From 24a90558b3b19d50f3040c6533611882b41d219d Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Tue, 24 Jan 2017 12:01:06 +1100 Subject: [PATCH 21/65] use released 9.4.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 94b6a5ca..4b531f30 100644 --- a/pom.xml +++ b/pom.xml @@ -36,7 +36,7 @@ 0.8.2-beta-SNAPSHOT 4 1 - 9.${jetty9.minor.version}.${jetty9.dot.version}-SNAPSHOT + 9.${jetty9.minor.version}.${jetty9.dot.version}.v20170120 ${docker.tag.prefix}9.${jetty9.minor.version} ${docker.tag.prefix}9.${jetty9.minor.version}-${maven.build.timestamp} From 9182d7ed5025d6e5811e600f6fd48fedbaface92 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 25 Jan 2017 10:48:30 +1100 Subject: [PATCH 22/65] Issue #68 minor clean ups --- .../runtimes/jetty9/TracingLogHandler.java | 87 ------------------- .../src/main/jetty-base/etc/gae-web.xml | 4 +- 2 files changed, 2 insertions(+), 89 deletions(-) delete mode 100644 jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java deleted file mode 100644 index e5e52295..00000000 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/TracingLogHandler.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (C) 2016 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.runtimes.jetty9; - - -import static com.google.common.base.MoreObjects.firstNonNull; - -import com.google.cloud.MonitoredResource; -import com.google.cloud.logging.AsyncLoggingHandler; -import com.google.cloud.logging.LogEntry.Builder; -import com.google.cloud.logging.LoggingOptions; - -import java.util.logging.LogManager; -import java.util.logging.LogRecord; - -/** - * A Google Cloud Logging Handler extended with a request traceid label. - */ -public class TracingLogHandler extends AsyncLoggingHandler { - - private static final ThreadLocal traceId = new ThreadLocal<>(); - - /** - * Set the Trace ID associated with any logging done by - * the current thread. - * @param id The traceID - */ - public static void setCurrentTraceId(String id) { - traceId.set(id); - } - - /** - * Get the Trace ID associated with any logging done by - * the current thread. - * @return id The traceID - */ - public static String getCurrentTraceId() { - return traceId.get(); - } - - /** - * Construct a TracingLogHandler for "jetty.log" - */ - public TracingLogHandler() { - this( - firstNonNull( - LogManager.getLogManager().getProperty(TracingLogHandler.class.getName() + ".log"), - "jetty.log"), - null,null); - } - - /** - * Construct a TracingLogHandler. - * - * @param logName Name of the log - * @param options LoggingOptions to access Google cloud API - * @param resource The resource to log against - */ - public TracingLogHandler(String logName, LoggingOptions options, MonitoredResource resource) { - super(logName, options, resource); - } - - @Override - protected void enhanceLogEntry(Builder builder, LogRecord record) { - super.enhanceLogEntry(builder, record); - String traceId = getCurrentTraceId(); - if (traceId != null) { - builder.addLabel("appengine.googleapis.com/trace_id", traceId); - } - builder.setTimestamp(record.getMillis()); - } -} - diff --git a/jetty9-base/src/main/jetty-base/etc/gae-web.xml b/jetty9-base/src/main/jetty-base/etc/gae-web.xml index 3dfd924e..9b098de2 100644 --- a/jetty9-base/src/main/jetty-base/etc/gae-web.xml +++ b/jetty9-base/src/main/jetty-base/etc/gae-web.xml @@ -1,7 +1,7 @@ - + true @@ -21,5 +21,5 @@ /etc/gae-override-web.xml - + From 688ed8e31dbbc43985126e91c45307e492dab7dc Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 25 Jan 2017 13:18:19 +1100 Subject: [PATCH 23/65] Do not CD to non existant directory --- jetty9/src/main/docker/docker-entrypoint.bash | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jetty9/src/main/docker/docker-entrypoint.bash b/jetty9/src/main/docker/docker-entrypoint.bash index 4d58843e..8427775a 100644 --- a/jetty9/src/main/docker/docker-entrypoint.bash +++ b/jetty9/src/main/docker/docker-entrypoint.bash @@ -24,7 +24,9 @@ if [ ! -e "$ROOT_DIR" -a -d /app ]; then fi # Set working directory to the root application directory -cd "$ROOT_DIR" +if [ -d "$ROOT_DIR" ]; then + cd "$ROOT_DIR" +fi # If the passed arguments start with the java command if [ "java" = "$1" -o "$(which java)" = "$1" ] ; then From 06b1a2e1e8f8a6627d9b40061651eedb8eafa7d0 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Tue, 31 Jan 2017 16:40:59 +1100 Subject: [PATCH 24/65] improved README --- README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/README.md b/README.md index 55f95b28..cebd70fd 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,42 @@ java $JAVA_OPTS \ -jar $JETTY_HOME/start.jar \ "$@" ``` +### Extending logging +The `java.util.logging` configuration may be changed at runtime by providing an alternate +properties file. This can be done either by extending the image and replacing the +default configuration at `$JETTY_BASE/etc/java-util-logging.properties`, or a new +configuration can be provided as part of the application (eg `WEB-INF/logging.properties`) +and setting the Java System Property `java.util.logging.config.file` to point to it. +System properties can be set by setting the unix environment variable `JAVA_USER_OPTS`. + +An example of running the image locally with a mounted application is: +```bash +docker run \ + -e JAVA_USER_OPTS=-Djava.util.logging.config.file=WEB-INF/logging.properties \ + -v /some-path/your-application:/app gcr.io/google_appengine/jetty +``` + +When deploying via the gcloud SDK and/or pluging, the `JAVA_USER_OPTS` may be set in +the `app.yaml` file: +```yaml +env_variables: + JAVA_USER_OPTS: -Djava.util.logging.config.file=WEB-INF/logging.properties +``` + +The default logging.properties file contains the following to configure `java.util.logging` +to use the gcloud stackdriver logging mechanism: +``` +.level=INFO + +handlers=com.google.cloud.logging.LoggingHandler +com.google.cloud.logging.LoggingHandler.level=FINE +com.google.cloud.logging.LoggingHandler.log=gae_app.log +com.google.cloud.logging.LoggingHandler.resourceType=gae_app +com.google.cloud.logging.LoggingHandler.enhancers=com.google.cloud.logging.GaeFlexLoggingEnhancer +com.google.cloud.logging.LoggingHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=%3$s: %5$s%6$s +``` + # Contributing changes From 9f64c08d43a6c091661ed5e8368b6f43fa5d4d9b Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 1 Feb 2017 08:04:23 +1100 Subject: [PATCH 25/65] fixed Cloud SDK --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index cebd70fd..6086e446 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ docker run \ -v /some-path/your-application:/app gcr.io/google_appengine/jetty ``` -When deploying via the gcloud SDK and/or pluging, the `JAVA_USER_OPTS` may be set in +When deploying via the Cloud SDK and/or plugin, the `JAVA_USER_OPTS` may be set in the `app.yaml` file: ```yaml env_variables: @@ -148,7 +148,6 @@ com.google.cloud.logging.LoggingHandler.formatter=java.util.logging.SimpleFormat java.util.logging.SimpleFormatter.format=%3$s: %5$s%6$s ``` - # Contributing changes * See [CONTRIBUTING.md](CONTRIBUTING.md) From 22e75189283a90504dfdb18713ed0ca2c265ad24 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 3 Feb 2017 09:08:37 +1100 Subject: [PATCH 26/65] Released beta of google-cloud-logging --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6c81320d..6c3d59d6 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ UTF-8 UTF-8 yyyy-MM-dd-HH-mm - 0.8.2-beta-SNAPSHOT + 0.8.2-beta 4 1 9.${jetty9.minor.version}.${jetty9.dot.version}.v20170120 From 4835abcb90aead04f3ad8862ad9d7e916fb4eecb Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Thu, 9 Feb 2017 10:32:23 +1100 Subject: [PATCH 27/65] updated google-cloud API to 0.8.3-beta --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c8fdfa92..0c5b7f9c 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ UTF-8 UTF-8 yyyy-MM-dd-HH-mm - 0.8.2-beta + 0.8.3-beta 4 1 9.${jetty9.minor.version}.${jetty9.dot.version}.v20170120 From f1e55dc9e8d6701ac00f9465236c6549cf55bde3 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 10 Feb 2017 15:37:40 +1100 Subject: [PATCH 28/65] added remote test to check logging --- tests/test-war-smoke/pom.xml | 11 +++ .../runtimes/jetty/test/smoke/LogServlet.java | 63 +++++++++++++++++ .../test/smoke/LoggingIntegrationTest.java | 68 +++++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java create mode 100644 tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java diff --git a/tests/test-war-smoke/pom.xml b/tests/test-war-smoke/pom.xml index 097923f7..1ac61399 100644 --- a/tests/test-war-smoke/pom.xml +++ b/tests/test-war-smoke/pom.xml @@ -34,6 +34,17 @@ 3.1.0 provided + + com.google.cloud + google-cloud-logging + ${gcloud.api.version} + + + javax.servlet + servlet-api + + + com.google.cloud.runtimes.tests gcloud-testing-core diff --git a/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java new file mode 100644 index 00000000..7b783db6 --- /dev/null +++ b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.runtimes.jetty.test.smoke; + +import com.google.cloud.Page; +import com.google.cloud.logging.LogEntry; +import com.google.cloud.logging.Logging; +import com.google.cloud.logging.Logging.EntryListOption; +import com.google.cloud.logging.LoggingOptions; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Iterator; + +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@WebServlet(urlPatterns = {"/log/*"}) +public class LogServlet extends HttpServlet { + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + resp.setContentType("text/plain"); + + LoggingOptions options = LoggingOptions.getDefaultInstance(); + + String filter = "resource.type=gae_app" + " AND resource.labels.module_id=smoke"; + String id = req.getPathInfo(); + if (id.length() > 1 && id.startsWith("/")) { + id = id.substring(1); + filter += " AND textPayload:" + id; + } + + PrintWriter out = resp.getWriter(); + out.println("Log Entries " + filter + ":"); + try (Logging logging = options.getService()) { + Page entries = logging.listLogEntries(EntryListOption.filter(filter)); + Iterator entryIterator = entries.iterateAll(); + while (entryIterator.hasNext()) { + out.println(entryIterator.next()); + } + } catch (Exception ex) { + throw new ServletException(ex); + } + } +} \ No newline at end of file diff --git a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java new file mode 100644 index 00000000..3f9f940d --- /dev/null +++ b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.runtimes.jetty.test.smoke; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +import com.google.cloud.runtime.jetty.test.AbstractIntegrationTest; +import com.google.cloud.runtime.jetty.test.annotation.RemoteOnly; +import com.google.cloud.runtime.jetty.util.HttpUrlUtil; + +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.StringReader; +import java.net.HttpURLConnection; +import java.net.URI; +import java.util.concurrent.TimeUnit; + +public class LoggingIntegrationTest extends AbstractIntegrationTest { + @Test + @RemoteOnly + public void testForwardedIgnored() throws Exception { + String id = Long.toHexString(System.nanoTime()); + URI target = getUri().resolve("/dump/info/" + id); + assertThat(target.getPath(), containsString("/dump/info/" + id)); + + HttpURLConnection http = HttpUrlUtil.openTo(target); + assertThat(http.getResponseCode(), is(200)); + String responseBody = HttpUrlUtil.getResponseBody(http); + + assertThat(responseBody, containsString(id)); + + TimeUnit.SECONDS.sleep(1); + + target = getUri().resolve("/log/" + id); + + http = HttpUrlUtil.openTo(target); + assertThat(http.getResponseCode(), is(200)); + responseBody = HttpUrlUtil.getResponseBody(http); + + BufferedReader in = new BufferedReader(new StringReader(responseBody)); + String line = in.readLine(); + assertThat(line,containsString("Log Entries ")); + assertThat(line,containsString("textPayload:" + id)); + + line = in.readLine(); + assertThat(line,containsString("JUL.info:/dump/info/" + id)); + + line = in.readLine(); + assertThat(line,containsString("ServletContext.log:/dump/info/" + id)); + } +} From 64a0a20d66fa94f1e5dcd5a33811e2f58bd265ab Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Mon, 13 Feb 2017 12:25:35 +1100 Subject: [PATCH 29/65] Stackdriver logging testing improved comments check for traceid --- jetty9-base/pom.xml | 1 + tests/test-war-smoke/pom.xml | 1 + .../runtimes/jetty/test/smoke/LogServlet.java | 5 ++++ .../test/smoke/LoggingIntegrationTest.java | 30 ++++++++++++++----- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/jetty9-base/pom.xml b/jetty9-base/pom.xml index e1392f30..b1a918c5 100644 --- a/jetty9-base/pom.xml +++ b/jetty9-base/pom.xml @@ -41,6 +41,7 @@ google-cloud-logging ${gcloud.api.version} + javax.servlet servlet-api diff --git a/tests/test-war-smoke/pom.xml b/tests/test-war-smoke/pom.xml index 1ac61399..78a9eda1 100644 --- a/tests/test-war-smoke/pom.xml +++ b/tests/test-war-smoke/pom.xml @@ -39,6 +39,7 @@ google-cloud-logging ${gcloud.api.version} + javax.servlet servlet-api diff --git a/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java index 7b783db6..9b693117 100644 --- a/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java +++ b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java @@ -32,6 +32,11 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +/** + * A servlet to query the Stackdriver logging for the gae_app smoke module. + * An ID string is passed as the pathInfo and is used to filter the textPayload + * and all log records found are included in the text response. + */ @WebServlet(urlPatterns = {"/log/*"}) public class LogServlet extends HttpServlet { @Override diff --git a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java index 3f9f940d..ac058123 100644 --- a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java +++ b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java @@ -30,24 +30,33 @@ import java.io.StringReader; import java.net.HttpURLConnection; import java.net.URI; -import java.util.concurrent.TimeUnit; +import java.util.List; +import java.util.stream.Collectors; public class LoggingIntegrationTest extends AbstractIntegrationTest { + + @Test @RemoteOnly - public void testForwardedIgnored() throws Exception { + public void testLogging() throws Exception { + + // Create unique ID to relate request with log entries String id = Long.toHexString(System.nanoTime()); - URI target = getUri().resolve("/dump/info/" + id); - assertThat(target.getPath(), containsString("/dump/info/" + id)); + // Hit the DumpServlet that will log the request path + URI target = getUri().resolve("/dump/info/" + id); HttpURLConnection http = HttpUrlUtil.openTo(target); assertThat(http.getResponseCode(), is(200)); String responseBody = HttpUrlUtil.getResponseBody(http); + + List lines = + new BufferedReader(new StringReader(responseBody)).lines().collect(Collectors.toList()); + assertThat(lines.stream().filter(s -> s.startsWith("requestURI=")).findFirst().get(), + containsString(id)); + String traceId = lines.stream().filter(s -> s.startsWith("X-Cloud-Trace-Context: ")) + .findFirst().get().split("[ /]")[1]; - assertThat(responseBody, containsString(id)); - - TimeUnit.SECONDS.sleep(1); - + // Hit the LogServlet to query the resulting log entries target = getUri().resolve("/log/" + id); http = HttpUrlUtil.openTo(target); @@ -61,8 +70,13 @@ public void testForwardedIgnored() throws Exception { line = in.readLine(); assertThat(line,containsString("JUL.info:/dump/info/" + id)); + assertThat(line,containsString("appengine.googleapis.com/trace_id=" + traceId)); + // TODO check zone once google-cloud-logging is updated to >= 0.8.4 line = in.readLine(); assertThat(line,containsString("ServletContext.log:/dump/info/" + id)); + assertThat(line,containsString("appengine.googleapis.com/trace_id=" + traceId)); + // TODO check zone once google-cloud-logging is updated to >= 0.8.4 + } } From cb852d3dfabebc7c5c9c894113ea9a483fd2b834 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 15 Feb 2017 10:22:39 +1100 Subject: [PATCH 30/65] upgrade to 0.9.2 --- pom.xml | 2 +- .../runtimes/jetty/test/smoke/LoggingIntegrationTest.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 0c5b7f9c..9275ed85 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ UTF-8 UTF-8 yyyy-MM-dd-HH-mm - 0.8.3-beta + 0.9.2-beta 4 1 9.${jetty9.minor.version}.${jetty9.dot.version}.v20170120 diff --git a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java index ac058123..a00044b2 100644 --- a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java +++ b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java @@ -48,7 +48,7 @@ public void testLogging() throws Exception { HttpURLConnection http = HttpUrlUtil.openTo(target); assertThat(http.getResponseCode(), is(200)); String responseBody = HttpUrlUtil.getResponseBody(http); - + List lines = new BufferedReader(new StringReader(responseBody)).lines().collect(Collectors.toList()); assertThat(lines.stream().filter(s -> s.startsWith("requestURI=")).findFirst().get(), @@ -62,7 +62,7 @@ public void testLogging() throws Exception { http = HttpUrlUtil.openTo(target); assertThat(http.getResponseCode(), is(200)); responseBody = HttpUrlUtil.getResponseBody(http); - + BufferedReader in = new BufferedReader(new StringReader(responseBody)); String line = in.readLine(); assertThat(line,containsString("Log Entries ")); @@ -71,12 +71,12 @@ public void testLogging() throws Exception { line = in.readLine(); assertThat(line,containsString("JUL.info:/dump/info/" + id)); assertThat(line,containsString("appengine.googleapis.com/trace_id=" + traceId)); - // TODO check zone once google-cloud-logging is updated to >= 0.8.4 + // TODO assertThat(line,containsString("zone=")); line = in.readLine(); assertThat(line,containsString("ServletContext.log:/dump/info/" + id)); assertThat(line,containsString("appengine.googleapis.com/trace_id=" + traceId)); - // TODO check zone once google-cloud-logging is updated to >= 0.8.4 + // TODO assertThat(line,containsString("zone=")); } } From f3ef3693a3635e23cf3518d18ce2231a8d77c93c Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 15 Feb 2017 13:57:03 +1100 Subject: [PATCH 31/65] Test for zone --- .../runtimes/jetty/test/smoke/LoggingIntegrationTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java index a00044b2..47e3912e 100644 --- a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java +++ b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java @@ -71,12 +71,11 @@ public void testLogging() throws Exception { line = in.readLine(); assertThat(line,containsString("JUL.info:/dump/info/" + id)); assertThat(line,containsString("appengine.googleapis.com/trace_id=" + traceId)); - // TODO assertThat(line,containsString("zone=")); + assertThat(line,containsString("zone=")); line = in.readLine(); assertThat(line,containsString("ServletContext.log:/dump/info/" + id)); assertThat(line,containsString("appengine.googleapis.com/trace_id=" + traceId)); - // TODO assertThat(line,containsString("zone=")); - + assertThat(line,containsString("zone=")); } } From 2c022d7512de958e84995faa9acf5e681c1ca81a Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 24 Mar 2017 10:26:28 +1100 Subject: [PATCH 32/65] upgrade to 10.0 gcloud API --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1d8f4b01..97791048 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ UTF-8 UTF-8 yyyy-MM-dd-HH-mm - 0.9.2-beta + 0.10.0-beta 4 1 9.${jetty9.minor.version}.${jetty9.dot.version}.v20170120 From cd8bdf8552bf128c8a6bfb4361c3bb624c068f84 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 24 Mar 2017 14:41:12 +1100 Subject: [PATCH 33/65] enable gae module only of GAE_INSTANCE environment variable is set --- jetty9-base/src/main/jetty-config/jetty.commands | 2 +- jetty9/src/main/docker/setup-env-ext.bash | 5 +++++ .../jetty/test/smoke/ConfigurationIntegrationTest.java | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/jetty9-base/src/main/jetty-config/jetty.commands b/jetty9-base/src/main/jetty-config/jetty.commands index ecb92413..5e062211 100644 --- a/jetty9-base/src/main/jetty-config/jetty.commands +++ b/jetty9-base/src/main/jetty-config/jetty.commands @@ -1,6 +1,6 @@ --create-startd ---add-to-start=server,webapp,http,deploy,jsp,jstl,http-forwarded,resources,gae +--add-to-start=server,webapp,http,deploy,jsp,jstl,http-forwarded,resources jetty.httpConfig.outputAggregationSize=32768 jetty.httpConfig.headerCacheSize=512 diff --git a/jetty9/src/main/docker/setup-env-ext.bash b/jetty9/src/main/docker/setup-env-ext.bash index 0855487e..a1c3b53f 100644 --- a/jetty9/src/main/docker/setup-env-ext.bash +++ b/jetty9/src/main/docker/setup-env-ext.bash @@ -34,4 +34,9 @@ if ! type "$1" &>/dev/null; then export JAVA_OPTS="$JAVA_OPTS $JETTY_ARGS" fi +# If we are deployed on a GAE platform, enable the gae module +if [ -n "$GAE_INSTANCE" -a ! -e "$JETTY_BASE/start.d/gae.ini" ]; then + echo "--module=gae" > $JETTY_BASE/start.d/gae.ini +fi + # End setup-env-ext.bash diff --git a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/ConfigurationIntegrationTest.java b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/ConfigurationIntegrationTest.java index c8941609..c3e72b9b 100644 --- a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/ConfigurationIntegrationTest.java +++ b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/ConfigurationIntegrationTest.java @@ -22,6 +22,7 @@ import static org.hamcrest.Matchers.not; import com.google.cloud.runtime.jetty.test.AbstractIntegrationTest; +import com.google.cloud.runtime.jetty.test.annotation.RemoteOnly; import com.google.cloud.runtime.jetty.util.HttpUrlUtil; import org.hamcrest.Matchers; @@ -39,6 +40,7 @@ public class ConfigurationIntegrationTest extends AbstractIntegrationTest { * @throws IOException test in error */ @Test + @RemoteOnly public void testNoDirectoryListing() throws IOException { URI target = getUri().resolve("/directory/"); From 6ad4bd87a2eb328f5d34fa506ba9e9974a15116c Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 24 Mar 2017 14:48:36 +1100 Subject: [PATCH 34/65] improved gae.mod documentation --- jetty9-base/src/main/jetty-base/modules/gae.mod | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/jetty9-base/src/main/jetty-base/modules/gae.mod b/jetty9-base/src/main/jetty-base/modules/gae.mod index 79e10960..86f047cf 100644 --- a/jetty9-base/src/main/jetty-base/modules/gae.mod +++ b/jetty9-base/src/main/jetty-base/modules/gae.mod @@ -1,5 +1,18 @@ # -# GAE Module for Jetty 9 Flex Image +# GAE Module for Jetty 9 Google Cloud Platform Image +# +# Enabling this module will: +# * apply the $JETTY_BASE/etc/gae-web.xml file to each deployed webapplication +# * add an com.google.cloud.runtimes.jetty9.DeploymentCheck instance to the server +# that checks for a deployed web application and listening connector. +# +# The $JETTY_BASE/etc/gae-web.xml when applied to a webapplication will: +# * set throwUnavailableOnStartupException to true +# * add /app.yaml to the protected targets +# * set $JETTY_BASE/etc/gae-override-web.xml as the override descriptor. +# +# The $JETTY_BASE/etc/gae-override-web.xml override descriptor will: +# * set the dirAllowed init-param to false # [depend] From 9080fc7d7c29687bdafb5600c62090d319c0a1cd Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 29 Mar 2017 12:32:19 +1100 Subject: [PATCH 35/65] GCP module Rename gae module and configuration to gcp Split the jetty.commands into jetty.commands and gcp.commands moved commands file to jetty-base/config-scripts updated setup-env-ext to run the gcp.commands when image is run with a GAE_INSTANCE set --- jetty9-base/pom.xml | 8 ++++---- jetty9-base/src/main/assembly/assembly.xml | 2 +- .../jetty-base/config-scripts/gcp.commands | 11 +++++++++++ .../jetty-base/config-scripts/jetty.commands | 11 +++++++++++ ...-override-web.xml => gcp-override-web.xml} | 0 .../etc/{gae-web.xml => gcp-web.xml} | 2 +- .../main/jetty-base/etc/{gae.xml => gcp.xml} | 2 +- .../jetty-base/modules/{gae.mod => gcp.mod} | 14 +++++++------- .../src/main/jetty-config/jetty.commands | 19 ------------------- jetty9/src/main/docker/setup-env-ext.bash | 10 +++++++--- 10 files changed, 43 insertions(+), 36 deletions(-) create mode 100644 jetty9-base/src/main/jetty-base/config-scripts/gcp.commands create mode 100644 jetty9-base/src/main/jetty-base/config-scripts/jetty.commands rename jetty9-base/src/main/jetty-base/etc/{gae-override-web.xml => gcp-override-web.xml} (100%) rename jetty9-base/src/main/jetty-base/etc/{gae-web.xml => gcp-web.xml} (95%) rename jetty9-base/src/main/jetty-base/etc/{gae.xml => gcp.xml} (98%) rename jetty9-base/src/main/jetty-base/modules/{gae.mod => gcp.mod} (56%) delete mode 100644 jetty9-base/src/main/jetty-config/jetty.commands diff --git a/jetty9-base/pom.xml b/jetty9-base/pom.xml index ba2cf0a9..d7de2fd8 100644 --- a/jetty9-base/pom.xml +++ b/jetty9-base/pom.xml @@ -44,7 +44,7 @@ org.apache.maven.plugins maven-jar-plugin - ${project.build.directory}/jetty-base/lib/gae + ${project.build.directory}/jetty-base/lib/gcp @@ -95,13 +95,13 @@ - gae-jars + gcp-jars pre-integration-test copy-dependencies - ${project.build.directory}/jetty-base/lib/gae + ${project.build.directory}/jetty-base/lib/gcp false true false @@ -130,7 +130,7 @@ -jar ../jetty-home-${jetty9.version}/start.jar - --commands=${basedir}/src/main/jetty-config/jetty.commands + --commands=${project.build.directory}/jetty-base/config-scripts/jetty.commands diff --git a/jetty9-base/src/main/assembly/assembly.xml b/jetty9-base/src/main/assembly/assembly.xml index 56cd4409..5acda39d 100644 --- a/jetty9-base/src/main/assembly/assembly.xml +++ b/jetty9-base/src/main/assembly/assembly.xml @@ -35,7 +35,7 @@ ${project.build.directory} - jetty-base/lib/gae + jetty-base/lib/gcp ${artifactId}-${version}.jar diff --git a/jetty9-base/src/main/jetty-base/config-scripts/gcp.commands b/jetty9-base/src/main/jetty-base/config-scripts/gcp.commands new file mode 100644 index 00000000..221bae72 --- /dev/null +++ b/jetty9-base/src/main/jetty-base/config-scripts/gcp.commands @@ -0,0 +1,11 @@ +--update-ini +--add-to-start=gcp,http-forwarded + +# Configure secure port redirection to 443 (default 8443 for requests recevied on 8080) +jetty.httpConfig.securePort=443 + +# Disable hot deploy of webapps +jetty.deploy.scanInterval=0 + +# Disable support for RFC7239 header +jetty.httpConfig.forwardedHeader= diff --git a/jetty9-base/src/main/jetty-base/config-scripts/jetty.commands b/jetty9-base/src/main/jetty-base/config-scripts/jetty.commands new file mode 100644 index 00000000..409ac395 --- /dev/null +++ b/jetty9-base/src/main/jetty-base/config-scripts/jetty.commands @@ -0,0 +1,11 @@ +--create-startd + +--add-to-start=server,webapp,http,deploy,jsp,jstl,resources + +jetty.httpConfig.outputAggregationSize=32768 +jetty.httpConfig.headerCacheSize=512 +jetty.httpConfig.sendServerVersion=true +jetty.httpConfig.sendDateHeader=false + +jetty.server.stopTimeout=30000 + diff --git a/jetty9-base/src/main/jetty-base/etc/gae-override-web.xml b/jetty9-base/src/main/jetty-base/etc/gcp-override-web.xml similarity index 100% rename from jetty9-base/src/main/jetty-base/etc/gae-override-web.xml rename to jetty9-base/src/main/jetty-base/etc/gcp-override-web.xml diff --git a/jetty9-base/src/main/jetty-base/etc/gae-web.xml b/jetty9-base/src/main/jetty-base/etc/gcp-web.xml similarity index 95% rename from jetty9-base/src/main/jetty-base/etc/gae-web.xml rename to jetty9-base/src/main/jetty-base/etc/gcp-web.xml index 01e8e2c5..4650f4a3 100644 --- a/jetty9-base/src/main/jetty-base/etc/gae-web.xml +++ b/jetty9-base/src/main/jetty-base/etc/gcp-web.xml @@ -15,7 +15,7 @@ - /etc/gae-override-web.xml + /etc/gcp-override-web.xml diff --git a/jetty9-base/src/main/jetty-base/etc/gae.xml b/jetty9-base/src/main/jetty-base/etc/gcp.xml similarity index 98% rename from jetty9-base/src/main/jetty-base/etc/gae.xml rename to jetty9-base/src/main/jetty-base/etc/gcp.xml index e89d5570..ffa94bed 100644 --- a/jetty9-base/src/main/jetty-base/etc/gae.xml +++ b/jetty9-base/src/main/jetty-base/etc/gcp.xml @@ -21,7 +21,7 @@ - /etc/gae-web.xml + /etc/gcp-web.xml diff --git a/jetty9-base/src/main/jetty-base/modules/gae.mod b/jetty9-base/src/main/jetty-base/modules/gcp.mod similarity index 56% rename from jetty9-base/src/main/jetty-base/modules/gae.mod rename to jetty9-base/src/main/jetty-base/modules/gcp.mod index 86f047cf..b48345bd 100644 --- a/jetty9-base/src/main/jetty-base/modules/gae.mod +++ b/jetty9-base/src/main/jetty-base/modules/gcp.mod @@ -1,17 +1,17 @@ # -# GAE Module for Jetty 9 Google Cloud Platform Image +# GCP Module for Jetty 9 Google Cloud Platform Image # # Enabling this module will: -# * apply the $JETTY_BASE/etc/gae-web.xml file to each deployed webapplication +# * apply the $JETTY_BASE/etc/gcp-web.xml file to each deployed webapplication # * add an com.google.cloud.runtimes.jetty9.DeploymentCheck instance to the server # that checks for a deployed web application and listening connector. # -# The $JETTY_BASE/etc/gae-web.xml when applied to a webapplication will: +# The $JETTY_BASE/etc/gcp-web.xml when applied to a webapplication will: # * set throwUnavailableOnStartupException to true # * add /app.yaml to the protected targets -# * set $JETTY_BASE/etc/gae-override-web.xml as the override descriptor. +# * set $JETTY_BASE/etc/gcp-override-web.xml as the override descriptor. # -# The $JETTY_BASE/etc/gae-override-web.xml override descriptor will: +# The $JETTY_BASE/etc/gcp-override-web.xml override descriptor will: # * set the dirAllowed init-param to false # @@ -23,7 +23,7 @@ resources deploy [xml] -etc/gae.xml +etc/gcp.xml [lib] -lib/gae/*.jar +lib/gcp/*.jar diff --git a/jetty9-base/src/main/jetty-config/jetty.commands b/jetty9-base/src/main/jetty-config/jetty.commands deleted file mode 100644 index 5e062211..00000000 --- a/jetty9-base/src/main/jetty-config/jetty.commands +++ /dev/null @@ -1,19 +0,0 @@ ---create-startd - ---add-to-start=server,webapp,http,deploy,jsp,jstl,http-forwarded,resources - -jetty.httpConfig.outputAggregationSize=32768 -jetty.httpConfig.headerCacheSize=512 -jetty.httpConfig.sendServerVersion=true -jetty.httpConfig.sendDateHeader=false - -jetty.server.stopTimeout=30000 - -# Configure secure port redirection to 443 (default 8443 for requests recevied on 8080) -jetty.httpConfig.securePort=443 - -# Disable hot deploy of webapps -jetty.deploy.scanInterval=0 - -# Disable support for RFC7239 header -jetty.httpConfig.forwardedHeader= diff --git a/jetty9/src/main/docker/setup-env-ext.bash b/jetty9/src/main/docker/setup-env-ext.bash index a1c3b53f..88b30a7e 100644 --- a/jetty9/src/main/docker/setup-env-ext.bash +++ b/jetty9/src/main/docker/setup-env-ext.bash @@ -34,9 +34,13 @@ if ! type "$1" &>/dev/null; then export JAVA_OPTS="$JAVA_OPTS $JETTY_ARGS" fi -# If we are deployed on a GAE platform, enable the gae module -if [ -n "$GAE_INSTANCE" -a ! -e "$JETTY_BASE/start.d/gae.ini" ]; then - echo "--module=gae" > $JETTY_BASE/start.d/gae.ini +# If we are deployed on a GCP platform, enable the gcp module +if [ -n "$GAE_INSTANCE" -a ! -e "$JETTY_BASE/start.d/gcp.ini" ]; then + java \ + -Djetty.base=$JETTY_BASE \ + -Djetty.home=$JETTY_HOME \ + -jar $JETTY_HOME/start.jar \ + --commands=$JETTY_BASE/config-scripts/gcp.commands fi # End setup-env-ext.bash From 63d986de4f964314e0907d2838bebc4dd38b5367 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 29 Mar 2017 14:27:55 +1100 Subject: [PATCH 36/65] fixed warnings --- .../runtimes/jetty/test/smoke/ConfigurationIntegrationTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/ConfigurationIntegrationTest.java b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/ConfigurationIntegrationTest.java index c3e72b9b..ce6c4a9f 100644 --- a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/ConfigurationIntegrationTest.java +++ b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/ConfigurationIntegrationTest.java @@ -19,13 +19,11 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.not; import com.google.cloud.runtime.jetty.test.AbstractIntegrationTest; import com.google.cloud.runtime.jetty.test.annotation.RemoteOnly; import com.google.cloud.runtime.jetty.util.HttpUrlUtil; -import org.hamcrest.Matchers; import org.junit.Test; import java.io.IOException; From cda775a2e4ec8a0359228400dfb91b6eb30b40dc Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 29 Mar 2017 20:35:11 +1100 Subject: [PATCH 37/65] turn off stackdriver logging by default. Added instructions to enable. --- README.md | 80 ++++++++++++------- .../etc/java-util-logging.properties | 19 +++-- .../src/main/appengine/app.yaml | 3 + .../main/webapp/WEB-INF/logging.properties | 9 +++ 4 files changed, 77 insertions(+), 34 deletions(-) create mode 100644 tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties diff --git a/README.md b/README.md index 7e8b4d67..d183560e 100644 --- a/README.md +++ b/README.md @@ -101,46 +101,72 @@ java $JAVA_OPTS \ -jar $JETTY_HOME/start.jar \ "$@" ``` -### Extending logging -The `java.util.logging` configuration may be changed at runtime by providing an alternate -properties file. This can be done either by extending the image and replacing the -default configuration at `$JETTY_BASE/etc/java-util-logging.properties`, or a new -configuration can be provided as part of the application (eg `WEB-INF/logging.properties`) -and setting the Java System Property `java.util.logging.config.file` to point to it. -System properties can be set by setting the unix environment variable `JAVA_USER_OPTS`. - -An example of running the image locally with a mounted application is: +## Logging +This image is configured to use [Java Util Logging](https://docs.oracle.com/javase/8/docs/api/java/util/logging/package-summary.html)(JUL) to capture all logging from +the container and it's dependencies. Applications that also use the JUL API will inherit the same logging configuration. + +By default JUL is configured to use a [ConsoleHandler](https://docs.oracle.com/javase/8/docs/api/java/util/logging/ConsoleHandler.html) to send logs to the `stderr` of the container process. When run on as a GCP deployment, all output to `stderr` is captured and is available via the Stackdriver logging console, however more detailed and integrated logs are available if the Stackdriver logging mechanism is used directly (see below). + +To alter logging configuration a new `logging.properties` file must be provided to the image that among other things can: alter log levels generated by Loggers; alter log levels accepted by handlers; add/remove/configure log handlers. + + +### Providing `logging.properties` via the web application +A new logging configuration file can be provided as part of the application (typically at `WEB-INF/logging.properties`) +and the Java System Property `java.util.logging.config.file` updated to reference it. + +When running in a GCP environment, the system property can be set in `app.yaml`: +```yaml +env_variables: + JAVA_USER_OPTS: -Djava.util.logging.config.file=WEB-INF/logging.properties +``` + +If the image is run directly, then a `-e` argument to the `docker run` command can be used to set the system property: + ```bash docker run \ -e JAVA_USER_OPTS=-Djava.util.logging.config.file=WEB-INF/logging.properties \ - -v /some-path/your-application:/app gcr.io/google_appengine/jetty + ... ``` -When deploying via the Cloud SDK and/or plugin, the `JAVA_USER_OPTS` may be set in -the `app.yaml` file: -```yaml -env_variables: - JAVA_USER_OPTS: -Djava.util.logging.config.file=WEB-INF/logging.properties +### Providing `logging.properties` via a custom image +If this image is being used as the base of a custom image, then the following `Dockerfile` commands can be used to add either replace the existing logging configuration file or to add a new `logging.properties` file. + +The default logging configuration file is located at `/var/lib/jetty/etc/java-util-logging.properties`, which can be replaced in a custom image is built. The default configuration can be replaced with a `Dockerfile` like: + +```Dockerfile +FROM gcr.io/google-appengine/jetty +ADD logging.properties /var/lib/jetty/etc/java-util-logging.properties +... +``` + +Alternately an entirely new location for the file can be provided and the environment amended in a `Dockerfile` like: + +```Dockerfile +FROM gcr.io/google-appengine/jetty +ADD logging.properties /etc/logging.properties +ENV JAVA_USER_OPTS -Djava.util.logging.config.file=/etc/logging.properties +... ``` -The default logging.properties file contains the following to configure `java.util.logging` -to use the gcloud stackdriver logging mechanism: +### Providing `logging.properties` via docker run +A `logging.properties` file may be added to an existing images using the `docker run` command if the deployment environment allows for the run arguments to be modified. The `-v` option can be used to bind a new `logging.properties` file to the running instance and the `-e` option can be used to set the system property to point to it: +```shell +docker run -it --rm \ +-v /mylocaldir/logging.properties:/etc/logging.properties \ +-e JAVA_USER_OPTS="-Djava.util.logging.config.file=/etc/logging.properties" \ +... ``` -.level=INFO -handlers=com.google.cloud.logging.LoggingHandler -com.google.cloud.logging.LoggingHandler.level=FINE -com.google.cloud.logging.LoggingHandler.log=gae_app.log +### Enhanced Stackdriver Logging +When running on the Google Cloud Platform Flex environment, the Stackdriver logging can be enhanced with additional information about the environment by adding the following lines to the `logging.properties`: +``` com.google.cloud.logging.LoggingHandler.resourceType=gae_app com.google.cloud.logging.LoggingHandler.enhancers=com.google.cloud.logging.GaeFlexLoggingEnhancer -com.google.cloud.logging.LoggingHandler.formatter=java.util.logging.SimpleFormatter -java.util.logging.SimpleFormatter.format=%3$s: %5$s%6$s ``` +This enables the [GaeFlexLoggingEnhancer](http://googlecloudplatform.github.io/google-cloud-java/0.10.0/apidocs/com/google/cloud/logging/GaeFlexLoggingEnhancer.html), which enhances the logs generated be linking them to the `nginx` request log in the logging console by `traceid` (The traceId for a request on a Google Cloud Platform is obtained from the `setCurrentTraceId` HTTP header as the first field of the `'/'` delimited value). + +When this image is deployed on a GCP environment, then the `gcp` module will automatically call the [setCurrentTraceId](http://googlecloudplatform.github.io/google-cloud-java/0.10.0/apidocs/com/google/cloud/logging/GaeFlexLoggingEnhancer.html#setCurrentTraceId-java.lang.String-) for any thread handling a request. -The configuration of the jetty container in this image can be viewed by running the image locally: -``` -docker run --rm -it launcher.gcr.io/google/jetty --list-config --list-modules -``` ## Extending the image The image produced by this project may be automatically used/extended by the Cloud SDK and/or App Engine maven plugin. diff --git a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties index 5631f48e..3f60ca3f 100644 --- a/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties +++ b/jetty9-base/src/main/jetty-base/etc/java-util-logging.properties @@ -1,9 +1,14 @@ .level=INFO -handlers=com.google.cloud.logging.LoggingHandler -com.google.cloud.logging.LoggingHandler.level=FINE -com.google.cloud.logging.LoggingHandler.log=gae_app.log -com.google.cloud.logging.LoggingHandler.resourceType=gae_app -com.google.cloud.logging.LoggingHandler.enhancers=com.google.cloud.logging.GaeFlexLoggingEnhancer -com.google.cloud.logging.LoggingHandler.formatter=java.util.logging.SimpleFormatter -java.util.logging.SimpleFormatter.format=%3$s: %5$s%6$s +handlers=java.util.logging.ConsoleHandler +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=%3$s: %5$s%6$s%n + +## To use Stackdriver Logging replace the configuration above with the configuration below: +# handlers=com.google.cloud.logging.LoggingHandler +# com.google.cloud.logging.LoggingHandler.level=FINE +# com.google.cloud.logging.LoggingHandler.log=gae_app.log +# com.google.cloud.logging.LoggingHandler.resourceType=gae_app +# com.google.cloud.logging.LoggingHandler.enhancers=com.google.cloud.logging.GaeFlexLoggingEnhancer +# com.google.cloud.logging.LoggingHandler.formatter=java.util.logging.SimpleFormatter +# java.util.logging.SimpleFormatter.format=%3$s: %5$s%6$s diff --git a/tests/test-war-smoke/src/main/appengine/app.yaml b/tests/test-war-smoke/src/main/appengine/app.yaml index fef2315c..53494429 100644 --- a/tests/test-war-smoke/src/main/appengine/app.yaml +++ b/tests/test-war-smoke/src/main/appengine/app.yaml @@ -12,3 +12,6 @@ handlers: - url: /.* script: ignored secure: optional + +env_variables: + JAVA_USER_OPTS: -Djava.util.logging.config.file=WEB-INF/logging.properties diff --git a/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties b/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties new file mode 100644 index 00000000..5631f48e --- /dev/null +++ b/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties @@ -0,0 +1,9 @@ +.level=INFO + +handlers=com.google.cloud.logging.LoggingHandler +com.google.cloud.logging.LoggingHandler.level=FINE +com.google.cloud.logging.LoggingHandler.log=gae_app.log +com.google.cloud.logging.LoggingHandler.resourceType=gae_app +com.google.cloud.logging.LoggingHandler.enhancers=com.google.cloud.logging.GaeFlexLoggingEnhancer +com.google.cloud.logging.LoggingHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=%3$s: %5$s%6$s From 3a2bfa9f5d605c00f1b6d4fa57a897e21ad33737 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Thu, 13 Apr 2017 10:31:07 +1000 Subject: [PATCH 38/65] Updates Update to latest openjdk-runtime with setup-env.d Update to latest jetty release --- .../src/main/docker/{setup-env-ext.bash => 50-jetty.bash} | 2 -- jetty9/src/main/docker/Dockerfile | 6 ++---- jetty9/src/test/resources/structure.yaml | 4 ++-- pom.xml | 6 +++--- 4 files changed, 7 insertions(+), 11 deletions(-) rename jetty9/src/main/docker/{setup-env-ext.bash => 50-jetty.bash} (91%) diff --git a/jetty9/src/main/docker/setup-env-ext.bash b/jetty9/src/main/docker/50-jetty.bash similarity index 91% rename from jetty9/src/main/docker/setup-env-ext.bash rename to jetty9/src/main/docker/50-jetty.bash index 0855487e..a42e68c0 100644 --- a/jetty9/src/main/docker/setup-env-ext.bash +++ b/jetty9/src/main/docker/50-jetty.bash @@ -1,4 +1,3 @@ -# The script below is from setup-env-ext.bash, which is appended to /setup-env.bash # default jetty arguments export JETTY_ARGS="-Djetty.base=$JETTY_BASE -jar $JETTY_HOME/start.jar" @@ -34,4 +33,3 @@ if ! type "$1" &>/dev/null; then export JAVA_OPTS="$JAVA_OPTS $JETTY_ARGS" fi -# End setup-env-ext.bash diff --git a/jetty9/src/main/docker/Dockerfile b/jetty9/src/main/docker/Dockerfile index 2cfa0da6..75bfd8de 100644 --- a/jetty9/src/main/docker/Dockerfile +++ b/jetty9/src/main/docker/Dockerfile @@ -38,10 +38,8 @@ RUN mkdir -p webapps $TMPDIR \ && java -jar $JETTY_HOME/start.jar --add-to-start=setuid \ && chown -R jetty:jetty $JETTY_BASE $TMPDIR -# Start Jetty directly -COPY setup-env-ext.bash /tmp/ -RUN cat /tmp/setup-env-ext.bash >> /setup-env.bash \ - && rm /tmp/setup-env-ext.bash +# Setup Jetty +ADD 50-jetty.bash /setup-env.d/ EXPOSE 8080 CMD ["java","-Djetty.base=${jetty.base}","-jar","${jetty.home}/start.jar"] diff --git a/jetty9/src/test/resources/structure.yaml b/jetty9/src/test/resources/structure.yaml index 221df028..7454fd98 100644 --- a/jetty9/src/test/resources/structure.yaml +++ b/jetty9/src/test/resources/structure.yaml @@ -32,8 +32,8 @@ fileExistenceTests: path: '${jetty.base}/webapps' isDirectory: true shouldExist: true -- name: 'setup-env.bash exists' - path: '/setup-env.bash' +- name: '50-jetty.bash exists' + path: '/setup-env.d/50-jetty.bash' isDirectory: false shouldExist: true - name: 'docker entrypoint exists' diff --git a/pom.xml b/pom.xml index 305d86af..d96594c8 100644 --- a/pom.xml +++ b/pom.xml @@ -30,12 +30,12 @@ UTF-8 yyyy-MM-dd-HH-mm 4 - 1 - 9.${jetty9.minor.version}.${jetty9.dot.version}.v20170120 + 3 + 9.${jetty9.minor.version}.${jetty9.dot.version}.v20170317 ${docker.tag.prefix}9.${jetty9.minor.version} ${docker.tag.prefix}9.${jetty9.minor.version}-${maven.build.timestamp} - gcr.io/google-appengine/openjdk:8-2017-03-22_16_09 + gcr.io/google-appengine/openjdk:8-2017-04-12_14_02 From c58bbaf552c7b9999394230c5ab7440148f74e5b Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Thu, 13 Apr 2017 11:00:47 +1000 Subject: [PATCH 39/65] Use the PLATFORM env var --- jetty9/src/main/docker/50-jetty.bash | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jetty9/src/main/docker/50-jetty.bash b/jetty9/src/main/docker/50-jetty.bash index fe9159a9..0763683c 100644 --- a/jetty9/src/main/docker/50-jetty.bash +++ b/jetty9/src/main/docker/50-jetty.bash @@ -20,7 +20,7 @@ if [ ! -e "$ROOT_DIR" -a -d /app ]; then fi # If the passed arguments start with the java command -if [ "java" = "$1" -o "$(which java)" = "$1" ] ; then +if [ "java" = "$1" -o "$(which java)" = "$1" ]; then # ignore the java command as it is the default shift # clear the JETTY args as the java command has been explicitly set @@ -34,7 +34,8 @@ if ! type "$1" &>/dev/null; then fi # If we are deployed on a GCP platform, enable the gcp module -if [ -n "$GAE_INSTANCE" -a ! -e "$JETTY_BASE/start.d/gcp.ini" ]; then +if [ "$PLATFORM" = "gae" ]; then + # TODO make this a modification to runtime args rather than an extra invocation of java java \ -Djetty.base=$JETTY_BASE \ -Djetty.home=$JETTY_HOME \ From c3d5f0d36f822a7a25d828e1dee1169f3463c0a9 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Thu, 13 Apr 2017 17:49:07 +1000 Subject: [PATCH 40/65] Improved jetty startup Added JETTY_PROPERTIES, JETTY_MODULES_ENABLE & JETTY_MODULES_DISABLE Removed duplicate code fron 50-jetty.bash --- README.md | 22 ++++++--- .../jetty-base/config-scripts/gcp.commands | 11 ----- .../src/main/jetty-base/modules/gcp.mod | 12 +++++ jetty9/src/main/docker/50-jetty.bash | 46 +++++++++++-------- .../jetty/test/smoke/DumpServlet.java | 14 ++++++ .../test/smoke/ForwardedIntegrationTest.java | 38 --------------- 6 files changed, 69 insertions(+), 74 deletions(-) delete mode 100644 jetty9-base/src/main/jetty-base/config-scripts/gcp.commands diff --git a/README.md b/README.md index 00728b71..c21d4819 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,20 @@ The layout of this image is intended to mostly mimic the official [docker-jetty] Arguments passed to the `docker run` command are passed to Jetty, so the configuration of the jetty server can be seen with a command like: ```console -docker run launcher.gcr.io/google/jetty --list-config +docker run gcr.io/google-appengine/jetty --list-config ``` Alternate commands can also be passed to the `docker run` command, so the image can be explored with ```console -docker run -it --rm launcher.gcr.io/google/jetty bash +docker run -it --rm gcr.io/google-appengine/jetty bash +``` + +Various environment variables (see below) can also be used to set jetty properties, enable modules and +disable modules. These variables may be set either in an `app.yaml` or passed in to a docker run +command eg. +```console +docker run -it --rm -e JETTY_PROPERTIES=jetty.http.idleTimeout=10000 gcr.io/google-appengine/jetty ``` To update the server configuration in a derived Docker image, the `Dockerfile` may @@ -70,12 +77,12 @@ The [/docker-entrypoint.bash](https://github.com/GoogleCloudPlatform/openjdk-run for the image is inherited from the openjdk-runtime and its capabilities are described in the associated [README](https://github.com/GoogleCloudPlatform/openjdk-runtime/blob/master/README.md) -This image updates the docker `CMD` and appends to the -[setup-env.bash](https://github.com/GoogleCloudPlatform/openjdk-runtime/blob/master/openjdk8/src/main/docker/setup-env.bash) +This image updates the docker `CMD` and adds the +[/setup-env.d/50-jetty.bash](https://github.com/GoogleCloudPlatform/openjdk-runtime/blob/master/openjdk8/src/main/docker/50-jetty.bash) script to include options and arguments to run the Jetty container, unless an executable argument is passed to the docker image. -Additional environment variables are set including: +Additional environment variables are used/set including: -|Env Var | Maven Prop | Value | +|Env Var | Maven Prop | Value/Comment | |------------------|-----------------|------------------------------------------------------| |`JETTY_VERSION` |`jetty9.version` | | |`GAE_IMAGE_NAME` | |`jetty` | @@ -83,6 +90,9 @@ Additional environment variables are set including: |`JETTY_HOME` |`jetty.home` |`/opt/jetty-home` | |`JETTY_BASE` |`jetty.base` |`/var/lib/jetty` | |`TMPDIR` | |`/tmp/jetty` | +|`JETTY_PROPERTIES`| |Comma separated list of `name=value` pairs appended to `$JETTY_ARGS` | +|`JETTY_MODULES_ENABLED`| |Comma separated list of modules to enable by appending to `$JETTY_ARGS` | +|`JETTY_MODULES_DISABLED`| |Comma separated list of modules to disable by removing from `$JETTY_BASE/start.d` | |`JETTY_ARGS` | |`-Djetty.base=$JETTY_BASE -jar $JETTY_HOME/start.jar` | |`ROOT_WAR` | |`$JETTY_BASE/webapps/root.war` | |`ROOT_DIR` | |`$JETTY_BASE/webapps/root` | diff --git a/jetty9-base/src/main/jetty-base/config-scripts/gcp.commands b/jetty9-base/src/main/jetty-base/config-scripts/gcp.commands deleted file mode 100644 index 221bae72..00000000 --- a/jetty9-base/src/main/jetty-base/config-scripts/gcp.commands +++ /dev/null @@ -1,11 +0,0 @@ ---update-ini ---add-to-start=gcp,http-forwarded - -# Configure secure port redirection to 443 (default 8443 for requests recevied on 8080) -jetty.httpConfig.securePort=443 - -# Disable hot deploy of webapps -jetty.deploy.scanInterval=0 - -# Disable support for RFC7239 header -jetty.httpConfig.forwardedHeader= diff --git a/jetty9-base/src/main/jetty-base/modules/gcp.mod b/jetty9-base/src/main/jetty-base/modules/gcp.mod index b48345bd..6278503b 100644 --- a/jetty9-base/src/main/jetty-base/modules/gcp.mod +++ b/jetty9-base/src/main/jetty-base/modules/gcp.mod @@ -17,6 +17,7 @@ [depend] server +http-forwarded [optional] resources @@ -27,3 +28,14 @@ etc/gcp.xml [lib] lib/gcp/*.jar + +[ini] +# Configure secure port redirection to 443 if not explicitly set +jetty.httpConfig.securePort?=443 + +# Disable hot deploy of webapps if not explicitly set +jetty.deploy.scanInterval?=0 + +# Disable support for RFC7239 header +jetty.httpConfig.forwardedHeader= + diff --git a/jetty9/src/main/docker/50-jetty.bash b/jetty9/src/main/docker/50-jetty.bash index 0763683c..8fed5438 100644 --- a/jetty9/src/main/docker/50-jetty.bash +++ b/jetty9/src/main/docker/50-jetty.bash @@ -1,6 +1,6 @@ +# Configure the environment and CMD for the Jetty Container -# default jetty arguments -export JETTY_ARGS="-Djetty.base=$JETTY_BASE -jar $JETTY_HOME/start.jar" +set -x # Unpack a WAR app (if present) beforehand so that Stackdriver Debugger # can load it. This should be done before the JVM for Jetty starts up. @@ -19,27 +19,35 @@ if [ ! -e "$ROOT_DIR" -a -d /app ]; then ln -s /app "$ROOT_DIR" fi -# If the passed arguments start with the java command -if [ "java" = "$1" -o "$(which java)" = "$1" ]; then - # ignore the java command as it is the default - shift - # clear the JETTY args as the java command has been explicitly set - JETTY_ARGS= +# move command line args to $JETTY_ARGS +export JETTY_ARGS="${@/$1/}" +set - "$1" + +# Add any Jetty properties to the JETTY_ARGS +if [ "$JETTY_PROPERTIES" ]; then + JETTY_ARGS="$JETTY_ARGS ${JETTY_PROPERTIES//,/ }" +fi + +# Enable jetty modules +if [ "$JETTY_MODULES_ENABLE" ]; then + for M in ${JETTY_MODULES_ENABLE//,/ }; do + JETTY_ARGS="$JETTY_ARGS --module=$M" + done fi -# If the first argument is not executable -if ! type "$1" &>/dev/null; then - # then jetty is being used so add the JETTY_ARGS to the JAVA_OPTS - export JAVA_OPTS="$JAVA_OPTS $JETTY_ARGS" +# Disable jetty modules +if [ "$JETTY_MODULES_DISABLE" ]; then + for M in ${JETTY_MODULES_DISABLE//,/ }; do + rm -f ${JETTY_BASE}/start.d/${M}.ini + done fi -# If we are deployed on a GCP platform, enable the gcp module +# If we are deployed on a GAE platform, enable the gcp module if [ "$PLATFORM" = "gae" ]; then - # TODO make this a modification to runtime args rather than an extra invocation of java - java \ - -Djetty.base=$JETTY_BASE \ - -Djetty.home=$JETTY_HOME \ - -jar $JETTY_HOME/start.jar \ - --commands=$JETTY_BASE/config-scripts/gcp.commands + JETTY_ARGS="$JETTY_ARGS --module=gcp" fi +# add the JETTY_ARGS to the JAVA_OPTS +export JAVA_OPTS="$JAVA_OPTS $JETTY_ARGS" + + diff --git a/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/DumpServlet.java b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/DumpServlet.java index 80aa7506..b2705fad 100644 --- a/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/DumpServlet.java +++ b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/DumpServlet.java @@ -106,5 +106,19 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) out.printf("encodedURL(\"/foo/bar\")=%s%n", response.encodeURL("/foo/bar")); out.printf("encodedRedirectURL(\"/foo/bar\")=%s%n", response.encodeRedirectURL("/foo/bar")); out.println(""); + + out.println("

Environment:

"); + out.println("
");
+    for (String n : System.getenv().keySet()) {
+      out.printf("%s=%s%n", n, System.getenv(n));
+    }
+    out.println("
"); + + out.println("

System Properties:

"); + out.println("
");
+    for (Object n : System.getProperties().keySet()) {
+      out.printf("%s=%s%n", n, System.getProperty(String.valueOf(n)));
+    }
+    out.println("
"); } } diff --git a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/ForwardedIntegrationTest.java b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/ForwardedIntegrationTest.java index 3c4df4db..cafceb7f 100644 --- a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/ForwardedIntegrationTest.java +++ b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/ForwardedIntegrationTest.java @@ -34,44 +34,6 @@ public class ForwardedIntegrationTest extends AbstractIntegrationTest { - /** - * Validate proxy proto header is handled. - * - * @throws IOException test in error - */ - @Test - @LocalOnly - public void testLocalProto() throws IOException { - URI target = getUri().resolve("/dump/info"); - assertThat(target.getPath(), containsString("/dump/info")); - - HttpURLConnection http = HttpUrlUtil.openTo(target); - http.setRequestProperty("X-Forwarded-Proto", "https"); - - assertThat(http.getResponseCode(), is(200)); - String responseBody = HttpUrlUtil.getResponseBody(http); - assertThat(responseBody, containsString("scheme=https(secure=true)")); - } - - /** - * Validate proxy for header is handled. - * - * @throws IOException test in error - */ - @Test - @LocalOnly - public void testLocalFor() throws IOException { - URI target = getUri().resolve("/dump/info"); - assertThat(target.getPath(), containsString("/dump/info")); - - HttpURLConnection http = HttpUrlUtil.openTo(target); - http.setRequestProperty("X-Forwarded-For", "1.2.3.4,5.6.7.8"); - - assertThat(http.getResponseCode(), is(200)); - String responseBody = HttpUrlUtil.getResponseBody(http); - assertThat(responseBody, containsString("remoteHost/Addr:port=1.2.3.4")); - } - /** * Validate proxy proto header is handled. * From 87cf4ae462991bb7a9af29fede38d89741da8041 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 14 Apr 2017 08:17:30 +1000 Subject: [PATCH 41/65] use launcher URL --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c21d4819..433a1f5d 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,14 @@ docker run gcr.io/google-appengine/jetty --list-config Alternate commands can also be passed to the `docker run` command, so the image can be explored with ```console -docker run -it --rm gcr.io/google-appengine/jetty bash +docker run -it --rm launcher.gcr.io/google/jetty bash ``` Various environment variables (see below) can also be used to set jetty properties, enable modules and disable modules. These variables may be set either in an `app.yaml` or passed in to a docker run command eg. ```console -docker run -it --rm -e JETTY_PROPERTIES=jetty.http.idleTimeout=10000 gcr.io/google-appengine/jetty +docker run -it --rm -e JETTY_PROPERTIES=jetty.http.idleTimeout=10000 launcher.gcr.io/google/jetty ``` To update the server configuration in a derived Docker image, the `Dockerfile` may @@ -45,13 +45,13 @@ When using App Engine Flexible, you can use the runtime without worrying about D runtime: java env: flex ``` -The runtime image `gcr.io/google-appengine/jetty` will be automatically selected if you are attempting to deploy a WAR (`*.war` file). +The runtime image `launcher.gcr.io/google/jetty` will be automatically selected if you are attempting to deploy a WAR (`*.war` file). If you want to use the image as a base for a custom runtime, you can specify `runtime: custom` in your `app.yaml` and then write the Dockerfile like this: ```dockerfile -FROM gcr.io/google-appengine/jetty +FROM launcher.gcr.io/google/jetty ADD your-application.war $JETTY_BASE/webapps/root.war ``` From 8ea12381b2f173611139aa4a39a9ae0786d832bf Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 14 Apr 2017 08:21:39 +1000 Subject: [PATCH 42/65] Trimmed GCP specific configuration --- .../src/main/jetty-base/config-scripts/jetty.commands | 7 ------- jetty9-base/src/main/jetty-base/modules/gcp.mod | 5 +++++ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/jetty9-base/src/main/jetty-base/config-scripts/jetty.commands b/jetty9-base/src/main/jetty-base/config-scripts/jetty.commands index 409ac395..6d3d9d1b 100644 --- a/jetty9-base/src/main/jetty-base/config-scripts/jetty.commands +++ b/jetty9-base/src/main/jetty-base/config-scripts/jetty.commands @@ -2,10 +2,3 @@ --add-to-start=server,webapp,http,deploy,jsp,jstl,resources -jetty.httpConfig.outputAggregationSize=32768 -jetty.httpConfig.headerCacheSize=512 -jetty.httpConfig.sendServerVersion=true -jetty.httpConfig.sendDateHeader=false - -jetty.server.stopTimeout=30000 - diff --git a/jetty9-base/src/main/jetty-base/modules/gcp.mod b/jetty9-base/src/main/jetty-base/modules/gcp.mod index 6278503b..c5b53247 100644 --- a/jetty9-base/src/main/jetty-base/modules/gcp.mod +++ b/jetty9-base/src/main/jetty-base/modules/gcp.mod @@ -39,3 +39,8 @@ jetty.deploy.scanInterval?=0 # Disable support for RFC7239 header jetty.httpConfig.forwardedHeader= +# Send Server version +jetty.httpConfig.sendServerVersion?=true + +# Don't send date header +jetty.httpConfig.sendDateHeader?=false From 4766ae5d15ba5e805f38bc7699ffa1f49b136878 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 14 Apr 2017 10:32:02 +1000 Subject: [PATCH 43/65] Added structure tests for jetty setup script Also fixed unpack bug found as a result --- jetty9/pom.xml | 2 ++ jetty9/src/main/docker/50-jetty.bash | 4 +-- jetty9/src/main/docker/Dockerfile | 7 ++-- jetty9/src/test/resources/structure.yaml | 20 +++++++++++ .../src/test/workspace/jetty-setup-app.bash | 15 ++++++++ .../workspace/jetty-setup-default-gcp.bash | 12 +++++++ .../test/workspace/jetty-setup-default.bash | 11 ++++++ .../test/workspace/jetty-setup-modules.bash | 22 ++++++++++++ .../test/workspace/jetty-setup-unpack.bash | 36 +++++++++++++++++++ 9 files changed, 123 insertions(+), 6 deletions(-) create mode 100755 jetty9/src/test/workspace/jetty-setup-app.bash create mode 100755 jetty9/src/test/workspace/jetty-setup-default-gcp.bash create mode 100755 jetty9/src/test/workspace/jetty-setup-default.bash create mode 100755 jetty9/src/test/workspace/jetty-setup-modules.bash create mode 100755 jetty9/src/test/workspace/jetty-setup-unpack.bash diff --git a/jetty9/pom.xml b/jetty9/pom.xml index 5a60f99f..6239d4f5 100644 --- a/jetty9/pom.xml +++ b/jetty9/pom.xml @@ -106,6 +106,8 @@ --image jetty:${docker.tag.long} + --workspace + ${project.basedir}/src/test/workspace --config ${project.build.testOutputDirectory}/structure.yaml diff --git a/jetty9/src/main/docker/50-jetty.bash b/jetty9/src/main/docker/50-jetty.bash index 8fed5438..5f7548d0 100644 --- a/jetty9/src/main/docker/50-jetty.bash +++ b/jetty9/src/main/docker/50-jetty.bash @@ -1,7 +1,6 @@ +#!/bin/bash # Configure the environment and CMD for the Jetty Container -set -x - # Unpack a WAR app (if present) beforehand so that Stackdriver Debugger # can load it. This should be done before the JVM for Jetty starts up. export ROOT_WAR=$JETTY_BASE/webapps/root.war @@ -9,6 +8,7 @@ export ROOT_DIR=$JETTY_BASE/webapps/root if [ -e "$ROOT_WAR" ]; then # Unpack it only if $ROOT_DIR doesn't exist or the root is older than the war. if [ -e "$ROOT_WAR" -a \( \( ! -e "$ROOT_DIR" \) -o \( "$ROOT_DIR" -ot "$ROOT_WAR" \) \) ]; then + rm -fr $ROOT_DIR unzip $ROOT_WAR -d $ROOT_DIR chown -R jetty:jetty $ROOT_DIR fi diff --git a/jetty9/src/main/docker/Dockerfile b/jetty9/src/main/docker/Dockerfile index 75bfd8de..8c2358aa 100644 --- a/jetty9/src/main/docker/Dockerfile +++ b/jetty9/src/main/docker/Dockerfile @@ -33,13 +33,12 @@ RUN chown -R jetty:jetty $JETTY_HOME ENV JETTY_BASE ${jetty.base} ENV TMPDIR /tmp/jetty ADD jetty-base $JETTY_BASE +ADD 50-jetty.bash /setup-env.d/ WORKDIR $JETTY_BASE RUN mkdir -p webapps $TMPDIR \ && java -jar $JETTY_HOME/start.jar --add-to-start=setuid \ - && chown -R jetty:jetty $JETTY_BASE $TMPDIR - -# Setup Jetty -ADD 50-jetty.bash /setup-env.d/ + && chown -R jetty:jetty $JETTY_BASE $TMPDIR \ + && chmod +x /setup-env.d/50-jetty.bash EXPOSE 8080 CMD ["java","-Djetty.base=${jetty.base}","-jar","${jetty.home}/start.jar"] diff --git a/jetty9/src/test/resources/structure.yaml b/jetty9/src/test/resources/structure.yaml index 7454fd98..5d7835e3 100644 --- a/jetty9/src/test/resources/structure.yaml +++ b/jetty9/src/test/resources/structure.yaml @@ -14,6 +14,26 @@ commandTests: - name: 'GAE_IMAGE_LABEL is set' command: ['env'] expectedOutput: ['GAE_IMAGE_LABEL=${docker.tag.long}'] +- name: 'check jetty setup default' + setup: [[ 'chmod', '+x', '/workspace/jetty-setup-default.bash' ]] + command: [ '/workspace/jetty-setup-default.bash' ] + expectedOutput: ['OK'] +- name: 'check jetty setup default GCP' + setup: [[ 'chmod', '+x', '/workspace/jetty-setup-default-gcp.bash' ]] + command: [ '/workspace/jetty-setup-default-gcp.bash' ] + expectedOutput: ['OK'] +- name: 'check jetty setup unpack' + setup: [[ 'chmod', '+x', '/workspace/jetty-setup-unpack.bash' ]] + command: [ '/workspace/jetty-setup-unpack.bash' ] + expectedOutput: ['OK'] +- name: 'check jetty setup app' + setup: [[ 'chmod', '+x', '/workspace/jetty-setup-app.bash' ]] + command: [ '/workspace/jetty-setup-app.bash' ] + expectedOutput: ['OK'] +- name: 'check jetty setup modules' + setup: [[ 'chmod', '+x', '/workspace/jetty-setup-modules.bash' ]] + command: [ '/workspace/jetty-setup-modules.bash' ] + expectedOutput: ['OK'] fileExistenceTests: - name: 'jetty start.jar exists' diff --git a/jetty9/src/test/workspace/jetty-setup-app.bash b/jetty9/src/test/workspace/jetty-setup-app.bash new file mode 100755 index 00000000..0df6291b --- /dev/null +++ b/jetty9/src/test/workspace/jetty-setup-app.bash @@ -0,0 +1,15 @@ +#!/bin/bash + +rm -fr $JETTY_BASE/webapps/root $JETTY_BASE/webapps/root.war +mkdir /app +echo original > /app/index.html + +source /setup-env.d/50-jetty.bash + +if [ "$(cat $JETTY_BASE/webapps/root/index.html)" != "original" ]; then + echo FAILED not linked to /app + exit 1 +fi + +echo OK + diff --git a/jetty9/src/test/workspace/jetty-setup-default-gcp.bash b/jetty9/src/test/workspace/jetty-setup-default-gcp.bash new file mode 100755 index 00000000..54fd07e7 --- /dev/null +++ b/jetty9/src/test/workspace/jetty-setup-default-gcp.bash @@ -0,0 +1,12 @@ +#!/bin/bash +set - zero one two three +JAVA_OPTS="-java -options" +PLATFORM=gae +source /setup-env.d/50-jetty.bash +if [ "$(echo ${JETTY_ARGS} | xargs)" != "one two three --module=gcp" ]; then + echo "JETTY_ARGS='${JETTY_ARGS}'" +elif [ "$(echo ${JAVA_OPTS} | xargs)" != "-java -options one two three --module=gcp" ]; then + echo "JAVA_OPTS='${JAVA_OPTS}'" +else + echo OK +fi diff --git a/jetty9/src/test/workspace/jetty-setup-default.bash b/jetty9/src/test/workspace/jetty-setup-default.bash new file mode 100755 index 00000000..052afee8 --- /dev/null +++ b/jetty9/src/test/workspace/jetty-setup-default.bash @@ -0,0 +1,11 @@ +#!/bin/bash +set - zero one two three +JAVA_OPTS="-java -options" +source /setup-env.d/50-jetty.bash +if [ "$(echo ${JETTY_ARGS} | xargs)" != "one two three" ]; then + echo "JETTY_ARGS='${JETTY_ARGS}'" +elif [ "$(echo ${JAVA_OPTS} | xargs)" != "-java -options one two three" ]; then + echo "JAVA_OPTS='${JAVA_OPTS}'" +else + echo OK +fi diff --git a/jetty9/src/test/workspace/jetty-setup-modules.bash b/jetty9/src/test/workspace/jetty-setup-modules.bash new file mode 100755 index 00000000..572384cc --- /dev/null +++ b/jetty9/src/test/workspace/jetty-setup-modules.bash @@ -0,0 +1,22 @@ +#!/bin/bash +set - zero arg +JAVA_OPTS="-X" +JETTY_PROPERTIES="prop0=value0,prop1=value1" +JETTY_MODULES_ENABLE="mod0,mod1" +JETTY_MODILES_DISABLE="gone0,gone1" + +touch $JETTY_BASE/start.d/gone0.ini +touch $JETTY_BASE/start.d/gone1.ini + +source /setup-env.d/50-jetty.bash +if [ "$(echo ${JETTY_ARGS} | xargs)" != "arg prop0=value0 prop1=value1 --module=mod0 --module=mod1" ]; then + echo "JETTY_ARGS='${JETTY_ARGS}'" +elif [ "$(echo ${JAVA_OPTS} | xargs)" != "-X arg prop0=value0 prop1=value1 --module=mod0 --module=mod1" ]; then + echo "JAVA_OPTS='${JAVA_OPTS}'" +elif [ -x JETTY_BASE/start.d/gone0.ini ]; then + echo gone0.ini not deleted +elif [ -x JETTY_BASE/start.d/gone1.ini ]; then + echo gone1.ini not deleted +else + echo OK +fi diff --git a/jetty9/src/test/workspace/jetty-setup-unpack.bash b/jetty9/src/test/workspace/jetty-setup-unpack.bash new file mode 100755 index 00000000..aa442228 --- /dev/null +++ b/jetty9/src/test/workspace/jetty-setup-unpack.bash @@ -0,0 +1,36 @@ +#!/bin/bash + +rm -fr $JETTY_BASE/webapps/root $JETTY_BASE/webapps/root.war +mkdir $JETTY_BASE/webapps/root +echo original > $JETTY_BASE/webapps/root/index.html +cd $JETTY_BASE/webapps/root +jar cf ../root.war * +cd .. + +rm -fr $JETTY_BASE/webapps/root +source /setup-env.d/50-jetty.bash + +if [ "$(cat $JETTY_BASE/webapps/root/index.html)" != "original" ]; then + echo FAILED not unpacked when no root + exit 1 +fi + +echo updated > $JETTY_BASE/webapps/root/index.html +source /setup-env.d/50-jetty.bash +if [ "$(cat $JETTY_BASE/webapps/root/index.html)" != "updated" ]; then + echo FAILED unpacked when war older + exit 1 +fi + +sleep 1 + +touch $JETTY_BASE/webapps/root.war +source /setup-env.d/50-jetty.bash +if [ "$(cat $JETTY_BASE/webapps/root/index.html)" != "original" ]; then + echo FAILED not unpacked when war newer + exit 1 +fi + + +echo OK + From 441a3fb3da98a490f5db660af510733bee592b71 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 14 Apr 2017 13:01:16 +1000 Subject: [PATCH 44/65] Support passing just args --- jetty9/src/main/docker/50-jetty.bash | 5 +++++ jetty9/src/test/workspace/jetty-setup-default-gcp.bash | 8 ++++---- jetty9/src/test/workspace/jetty-setup-default.bash | 6 +++--- jetty9/src/test/workspace/jetty-setup-modules.bash | 6 +++--- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/jetty9/src/main/docker/50-jetty.bash b/jetty9/src/main/docker/50-jetty.bash index 5f7548d0..20ef84b9 100644 --- a/jetty9/src/main/docker/50-jetty.bash +++ b/jetty9/src/main/docker/50-jetty.bash @@ -23,6 +23,11 @@ fi export JETTY_ARGS="${@/$1/}" set - "$1" +# Check for start.jar +if [ "$(echo $JETTY_ARGS | egrep start.jar | wc -l )" = "0" ]; then + JETTY_ARGS="-Djetty.base=${JETTY_BASE} -jar ${JETTY_HOME}/start.jar $JETTY_ARGS" +fi + # Add any Jetty properties to the JETTY_ARGS if [ "$JETTY_PROPERTIES" ]; then JETTY_ARGS="$JETTY_ARGS ${JETTY_PROPERTIES//,/ }" diff --git a/jetty9/src/test/workspace/jetty-setup-default-gcp.bash b/jetty9/src/test/workspace/jetty-setup-default-gcp.bash index 54fd07e7..52b31794 100755 --- a/jetty9/src/test/workspace/jetty-setup-default-gcp.bash +++ b/jetty9/src/test/workspace/jetty-setup-default-gcp.bash @@ -3,10 +3,10 @@ set - zero one two three JAVA_OPTS="-java -options" PLATFORM=gae source /setup-env.d/50-jetty.bash -if [ "$(echo ${JETTY_ARGS} | xargs)" != "one two three --module=gcp" ]; then - echo "JETTY_ARGS='${JETTY_ARGS}'" -elif [ "$(echo ${JAVA_OPTS} | xargs)" != "-java -options one two three --module=gcp" ]; then - echo "JAVA_OPTS='${JAVA_OPTS}'" +if [ "$(echo ${JETTY_ARGS} | xargs)" != "-Djetty.base=/var/lib/jetty -jar /opt/jetty-home/start.jar one two three --module=gcp" ]; then + echo "JETTY_ARGS='$(echo ${JETTY_ARGS} | xargs)'" +elif [ "$(echo ${JAVA_OPTS} | xargs)" != "-java -options -Djetty.base=/var/lib/jetty -jar /opt/jetty-home/start.jar one two three --module=gcp" ]; then + echo "JAVA_OPTS='$(echo ${JAVA_OPTS} | xargs)'" else echo OK fi diff --git a/jetty9/src/test/workspace/jetty-setup-default.bash b/jetty9/src/test/workspace/jetty-setup-default.bash index 052afee8..04a018b5 100755 --- a/jetty9/src/test/workspace/jetty-setup-default.bash +++ b/jetty9/src/test/workspace/jetty-setup-default.bash @@ -1,10 +1,10 @@ #!/bin/bash -set - zero one two three +set - java start.jar two three JAVA_OPTS="-java -options" source /setup-env.d/50-jetty.bash -if [ "$(echo ${JETTY_ARGS} | xargs)" != "one two three" ]; then +if [ "$(echo ${JETTY_ARGS} | xargs)" != "start.jar two three" ]; then echo "JETTY_ARGS='${JETTY_ARGS}'" -elif [ "$(echo ${JAVA_OPTS} | xargs)" != "-java -options one two three" ]; then +elif [ "$(echo ${JAVA_OPTS} | xargs)" != "-java -options start.jar two three" ]; then echo "JAVA_OPTS='${JAVA_OPTS}'" else echo OK diff --git a/jetty9/src/test/workspace/jetty-setup-modules.bash b/jetty9/src/test/workspace/jetty-setup-modules.bash index 572384cc..82549217 100755 --- a/jetty9/src/test/workspace/jetty-setup-modules.bash +++ b/jetty9/src/test/workspace/jetty-setup-modules.bash @@ -1,5 +1,5 @@ #!/bin/bash -set - zero arg +set - java start.jar arg JAVA_OPTS="-X" JETTY_PROPERTIES="prop0=value0,prop1=value1" JETTY_MODULES_ENABLE="mod0,mod1" @@ -9,9 +9,9 @@ touch $JETTY_BASE/start.d/gone0.ini touch $JETTY_BASE/start.d/gone1.ini source /setup-env.d/50-jetty.bash -if [ "$(echo ${JETTY_ARGS} | xargs)" != "arg prop0=value0 prop1=value1 --module=mod0 --module=mod1" ]; then +if [ "$(echo ${JETTY_ARGS} | xargs)" != "start.jar arg prop0=value0 prop1=value1 --module=mod0 --module=mod1" ]; then echo "JETTY_ARGS='${JETTY_ARGS}'" -elif [ "$(echo ${JAVA_OPTS} | xargs)" != "-X arg prop0=value0 prop1=value1 --module=mod0 --module=mod1" ]; then +elif [ "$(echo ${JAVA_OPTS} | xargs)" != "-X start.jar arg prop0=value0 prop1=value1 --module=mod0 --module=mod1" ]; then echo "JAVA_OPTS='${JAVA_OPTS}'" elif [ -x JETTY_BASE/start.d/gone0.ini ]; then echo gone0.ini not deleted From 81014f6d8a8c3c7f718f8a8e06de03ac11322fe8 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 14 Apr 2017 13:11:47 +1000 Subject: [PATCH 45/65] fix merge --- jetty9-base/pom.xml | 10 +++++++--- .../src/main/jetty-base/config-scripts/jetty.commands | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/jetty9-base/pom.xml b/jetty9-base/pom.xml index 71993271..8f05b015 100644 --- a/jetty9-base/pom.xml +++ b/jetty9-base/pom.xml @@ -41,6 +41,10 @@ google-cloud-logging ${gcloud.api.version} + + commons-logging + commons-logging + javax.servlet @@ -108,7 +112,7 @@ gcp-jars - pre-integration-test + prepare-package copy-dependencies @@ -130,7 +134,7 @@ add-jetty-modules - pre-integration-test + prepare-package exec @@ -151,7 +155,7 @@ maven-assembly-plugin - pre-integration-test + prepare-package single diff --git a/jetty9-base/src/main/jetty-base/config-scripts/jetty.commands b/jetty9-base/src/main/jetty-base/config-scripts/jetty.commands index e97e1da3..c48fa1df 100644 --- a/jetty9-base/src/main/jetty-base/config-scripts/jetty.commands +++ b/jetty9-base/src/main/jetty-base/config-scripts/jetty.commands @@ -1,4 +1,4 @@ --create-startd - ---add-to-start=server,webapp,http,deploy,jsp,jstl,resources,logging-jul +--approve-all-licenses +--add-to-start=server,webapp,http,deploy,jsp,jstl,resources,logging-jul,jcl-slf4j From bdad80aec05d2402a4930d7f4d2ddf765452f8a0 Mon Sep 17 00:00:00 2001 From: Alex Sloan Date: Fri, 14 Apr 2017 14:29:44 -0400 Subject: [PATCH 46/65] Fixed test workspace paths for cloud build --- jetty9/pom.xml | 4 ++-- jetty9/src/test/resources/structure.yaml | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/jetty9/pom.xml b/jetty9/pom.xml index 6239d4f5..7664f5c7 100644 --- a/jetty9/pom.xml +++ b/jetty9/pom.xml @@ -106,8 +106,8 @@ --image jetty:${docker.tag.long} - --workspace - ${project.basedir}/src/test/workspace + --workspace + ${project.basedir}/.. --config ${project.build.testOutputDirectory}/structure.yaml diff --git a/jetty9/src/test/resources/structure.yaml b/jetty9/src/test/resources/structure.yaml index 5d7835e3..80cb90bb 100644 --- a/jetty9/src/test/resources/structure.yaml +++ b/jetty9/src/test/resources/structure.yaml @@ -15,24 +15,24 @@ commandTests: command: ['env'] expectedOutput: ['GAE_IMAGE_LABEL=${docker.tag.long}'] - name: 'check jetty setup default' - setup: [[ 'chmod', '+x', '/workspace/jetty-setup-default.bash' ]] - command: [ '/workspace/jetty-setup-default.bash' ] + setup: [[ 'chmod', '+x', '/workspace/jetty9/src/test/workspace/jetty-setup-default.bash' ]] + command: [ '/workspace/jetty9/src/test/workspace/jetty-setup-default.bash' ] expectedOutput: ['OK'] - name: 'check jetty setup default GCP' - setup: [[ 'chmod', '+x', '/workspace/jetty-setup-default-gcp.bash' ]] - command: [ '/workspace/jetty-setup-default-gcp.bash' ] + setup: [[ 'chmod', '+x', '/workspace/jetty9/src/test/workspace/jetty-setup-default-gcp.bash' ]] + command: [ '/workspace/jetty9/src/test/workspace/jetty-setup-default-gcp.bash' ] expectedOutput: ['OK'] - name: 'check jetty setup unpack' - setup: [[ 'chmod', '+x', '/workspace/jetty-setup-unpack.bash' ]] - command: [ '/workspace/jetty-setup-unpack.bash' ] + setup: [[ 'chmod', '+x', '/workspace/jetty9/src/test/workspace/jetty-setup-unpack.bash' ]] + command: [ '/workspace/jetty9/src/test/workspace/jetty-setup-unpack.bash' ] expectedOutput: ['OK'] - name: 'check jetty setup app' - setup: [[ 'chmod', '+x', '/workspace/jetty-setup-app.bash' ]] - command: [ '/workspace/jetty-setup-app.bash' ] + setup: [[ 'chmod', '+x', '/workspace/jetty9/src/test/workspace/jetty-setup-app.bash' ]] + command: [ '/workspace/jetty9/src/test/workspace/jetty-setup-app.bash' ] expectedOutput: ['OK'] - name: 'check jetty setup modules' - setup: [[ 'chmod', '+x', '/workspace/jetty-setup-modules.bash' ]] - command: [ '/workspace/jetty-setup-modules.bash' ] + setup: [[ 'chmod', '+x', '/workspace/jetty9/src/test/workspace/jetty-setup-modules.bash' ]] + command: [ '/workspace/jetty9/src/test/workspace/jetty-setup-modules.bash' ] expectedOutput: ['OK'] fileExistenceTests: From 3240d37cc743e58608e67f6487f07d509540ab6b Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Tue, 18 Apr 2017 17:09:52 +1000 Subject: [PATCH 47/65] review feedback --- README.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d4ce2400..15e71a79 100644 --- a/README.md +++ b/README.md @@ -91,8 +91,8 @@ Additional environment variables are used/set including: |`JETTY_BASE` |`jetty.base` |`/var/lib/jetty` | |`TMPDIR` | |`/tmp/jetty` | |`JETTY_PROPERTIES`| |Comma separated list of `name=value` pairs appended to `$JETTY_ARGS` | -|`JETTY_MODULES_ENABLED`| |Comma separated list of modules to enable by appending to `$JETTY_ARGS` | -|`JETTY_MODULES_DISABLED`| |Comma separated list of modules to disable by removing from `$JETTY_BASE/start.d` | +|`JETTY_MODULES_ENABLE`| |Comma separated list of modules to enable by appending to `$JETTY_ARGS` | +|`JETTY_MODULES_DISABLE`| |Comma separated list of modules to disable by removing from `$JETTY_BASE/start.d` | |`JETTY_ARGS` | |`-Djetty.base=$JETTY_BASE -jar $JETTY_HOME/start.jar` | |`ROOT_WAR` | |`$JETTY_BASE/webapps/root.war` | |`ROOT_DIR` | |`$JETTY_BASE/webapps/root` | @@ -113,7 +113,7 @@ java $JAVA_OPTS \ ``` ## Logging This image is configured to use [Java Util Logging](https://docs.oracle.com/javase/8/docs/api/java/util/logging/package-summary.html)(JUL) to capture all logging from -the container and it's dependencies. Applications that also use the JUL API will inherit the same logging configuration. +the container and its dependencies. Applications that also use the JUL API will inherit the same logging configuration. By default JUL is configured to use a [ConsoleHandler](https://docs.oracle.com/javase/8/docs/api/java/util/logging/ConsoleHandler.html) to send logs to the `stderr` of the container process. When run on as a GCP deployment, all output to `stderr` is captured and is available via the Stackdriver logging console, however more detailed and integrated logs are available if the Stackdriver logging mechanism is used directly (see below). @@ -167,15 +167,22 @@ docker run -it --rm \ ... ``` -### Enhanced Stackdriver Logging -When running on the Google Cloud Platform Flex environment, the Stackdriver logging can be enhanced with additional information about the environment by adding the following lines to the `logging.properties`: +### Enhanced Stackdriver Logging (BETA!) +When running on the Google Cloud Platform Flex environment, the Java Util Logging can be configured to send logs to Google Stackdriver Logging by providing a `logging.properties` file that configures a [LoggingHandler](http://googlecloudplatform.github.io/google-cloud-java/0.10.0/apidocs/com/google/cloud/logging/LoggingHandler.html) as follows: ``` +handlers=com.google.cloud.logging.LoggingHandler +com.google.cloud.logging.LoggingHandler.level=FINE +com.google.cloud.logging.LoggingHandler.log=gae_app.log com.google.cloud.logging.LoggingHandler.resourceType=gae_app com.google.cloud.logging.LoggingHandler.enhancers=com.google.cloud.logging.GaeFlexLoggingEnhancer +com.google.cloud.logging.LoggingHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=%3$s: %5$s%6$s + ``` -This enables the [GaeFlexLoggingEnhancer](http://googlecloudplatform.github.io/google-cloud-java/0.10.0/apidocs/com/google/cloud/logging/GaeFlexLoggingEnhancer.html), which enhances the logs generated be linking them to the `nginx` request log in the logging console by `traceid` (The traceId for a request on a Google Cloud Platform is obtained from the `setCurrentTraceId` HTTP header as the first field of the `'/'` delimited value). -When this image is deployed on a GCP environment, then the `gcp` module will automatically call the [setCurrentTraceId](http://googlecloudplatform.github.io/google-cloud-java/0.10.0/apidocs/com/google/cloud/logging/GaeFlexLoggingEnhancer.html#setCurrentTraceId-java.lang.String-) for any thread handling a request. +This uses the [GaeFlexLoggingEnhancer](http://googlecloudplatform.github.io/google-cloud-java/0.10.0/apidocs/com/google/cloud/logging/GaeFlexLoggingEnhancer.html) to enhances the logs generated be linking them to the `nginx` request log in the logging console by `traceid` (The traceId for a request on a Google Cloud Platform is obtained from the `setCurrentTraceId` HTTP header as the first field of the `'/'` delimited value). + +When an image so configured is deployed on a GCP environment, then the `gcp` module will automatically call the [setCurrentTraceId](http://googlecloudplatform.github.io/google-cloud-java/0.10.0/apidocs/com/google/cloud/logging/GaeFlexLoggingEnhancer.html#setCurrentTraceId-java.lang.String-) for any thread handling a request. ## Extending the image From 0fc12151998961d2e8bb3a90e4765f6874ea2fc5 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Tue, 18 Apr 2017 18:04:40 +1000 Subject: [PATCH 48/65] working directory is the root webapp --- jetty9/src/main/docker/50-jetty.bash | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jetty9/src/main/docker/50-jetty.bash b/jetty9/src/main/docker/50-jetty.bash index 20ef84b9..991d73cd 100644 --- a/jetty9/src/main/docker/50-jetty.bash +++ b/jetty9/src/main/docker/50-jetty.bash @@ -19,6 +19,10 @@ if [ ! -e "$ROOT_DIR" -a -d /app ]; then ln -s /app "$ROOT_DIR" fi +if [ -d "$ROOT_DIR" ]; then + cd "$ROOT_DIR" +fi + # move command line args to $JETTY_ARGS export JETTY_ARGS="${@/$1/}" set - "$1" From 42ba89cafc1bec08366b5aab1bdc7b76535d3441 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 19 Apr 2017 00:12:05 +1000 Subject: [PATCH 49/65] Improve handling of various types of command line. Fixed problem with handling of command line like "ls /var" Requires duplication of test for java from openjdk-runtime docker-entrypoint.bash, which should be modified to avoid the duplication. --- jetty9/src/main/docker/50-jetty.bash | 18 +++++++++++++++--- jetty9/src/test/workspace/jetty-setup-app.bash | 1 + .../workspace/jetty-setup-default-gcp.bash | 6 +++--- .../src/test/workspace/jetty-setup-unpack.bash | 3 ++- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/jetty9/src/main/docker/50-jetty.bash b/jetty9/src/main/docker/50-jetty.bash index 991d73cd..a4d8e488 100644 --- a/jetty9/src/main/docker/50-jetty.bash +++ b/jetty9/src/main/docker/50-jetty.bash @@ -19,13 +19,25 @@ if [ ! -e "$ROOT_DIR" -a -d /app ]; then ln -s /app "$ROOT_DIR" fi +# If a webapp root directory exist, use it as the work directory if [ -d "$ROOT_DIR" ]; then cd "$ROOT_DIR" fi -# move command line args to $JETTY_ARGS -export JETTY_ARGS="${@/$1/}" -set - "$1" +# TODO update docker-entrypoint.bash to do this before calling setup scripts +# If the first argument is the java command +if [ "java" = "$1" -o "$(which java)" = "$1" ] ; then + # ignore it as java is the default command + shift +fi + +# If the first argument is not executable +if ! type "$1" &>/dev/null; then + # The arguments are additional arguments for the default command to run jetty + # move command line args to $JETTY_ARGS + export JETTY_ARGS="$@" + set - "$(which java)" +fi # Check for start.jar if [ "$(echo $JETTY_ARGS | egrep start.jar | wc -l )" = "0" ]; then diff --git a/jetty9/src/test/workspace/jetty-setup-app.bash b/jetty9/src/test/workspace/jetty-setup-app.bash index 0df6291b..5324af63 100755 --- a/jetty9/src/test/workspace/jetty-setup-app.bash +++ b/jetty9/src/test/workspace/jetty-setup-app.bash @@ -2,6 +2,7 @@ rm -fr $JETTY_BASE/webapps/root $JETTY_BASE/webapps/root.war mkdir /app +trap "rm -rf /app" EXIT echo original > /app/index.html source /setup-env.d/50-jetty.bash diff --git a/jetty9/src/test/workspace/jetty-setup-default-gcp.bash b/jetty9/src/test/workspace/jetty-setup-default-gcp.bash index 52b31794..2f85e5b5 100755 --- a/jetty9/src/test/workspace/jetty-setup-default-gcp.bash +++ b/jetty9/src/test/workspace/jetty-setup-default-gcp.bash @@ -1,11 +1,11 @@ #!/bin/bash -set - zero one two three +set - -zero one two three JAVA_OPTS="-java -options" PLATFORM=gae source /setup-env.d/50-jetty.bash -if [ "$(echo ${JETTY_ARGS} | xargs)" != "-Djetty.base=/var/lib/jetty -jar /opt/jetty-home/start.jar one two three --module=gcp" ]; then +if [ "$(echo ${JETTY_ARGS} | xargs)" != "-Djetty.base=/var/lib/jetty -jar /opt/jetty-home/start.jar -zero one two three --module=gcp" ]; then echo "JETTY_ARGS='$(echo ${JETTY_ARGS} | xargs)'" -elif [ "$(echo ${JAVA_OPTS} | xargs)" != "-java -options -Djetty.base=/var/lib/jetty -jar /opt/jetty-home/start.jar one two three --module=gcp" ]; then +elif [ "$(echo ${JAVA_OPTS} | xargs)" != "-java -options -Djetty.base=/var/lib/jetty -jar /opt/jetty-home/start.jar -zero one two three --module=gcp" ]; then echo "JAVA_OPTS='$(echo ${JAVA_OPTS} | xargs)'" else echo OK diff --git a/jetty9/src/test/workspace/jetty-setup-unpack.bash b/jetty9/src/test/workspace/jetty-setup-unpack.bash index aa442228..80f62c65 100755 --- a/jetty9/src/test/workspace/jetty-setup-unpack.bash +++ b/jetty9/src/test/workspace/jetty-setup-unpack.bash @@ -1,12 +1,14 @@ #!/bin/bash rm -fr $JETTY_BASE/webapps/root $JETTY_BASE/webapps/root.war +trap "rm -rf $JETTY_BASE/webapps/root $JETTY_BASE/webapps/root.war" EXIT mkdir $JETTY_BASE/webapps/root echo original > $JETTY_BASE/webapps/root/index.html cd $JETTY_BASE/webapps/root jar cf ../root.war * cd .. + rm -fr $JETTY_BASE/webapps/root source /setup-env.d/50-jetty.bash @@ -31,6 +33,5 @@ if [ "$(cat $JETTY_BASE/webapps/root/index.html)" != "original" ]; then exit 1 fi - echo OK From 1c73bb6acb2f259efae8e7ffbf24d0ca8d1f43a3 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 19 Apr 2017 09:25:11 +1000 Subject: [PATCH 50/65] upgrade cgloud API version to 0.13.0-beta --- pom.xml | 2 +- .../cloud/runtimes/jetty/test/smoke/LogServlet.java | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 8e31824f..651ca768 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ UTF-8 UTF-8 yyyy-MM-dd-HH-mm - 0.10.0-beta + 0.13.0-beta 4 3 9.${jetty9.minor.version}.${jetty9.dot.version}.v20170317 diff --git a/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java index 9b693117..a294dbe6 100644 --- a/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java +++ b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java @@ -16,7 +16,7 @@ package com.google.cloud.runtimes.jetty.test.smoke; -import com.google.cloud.Page; +import com.google.api.gax.core.Page; import com.google.cloud.logging.LogEntry; import com.google.cloud.logging.Logging; import com.google.cloud.logging.Logging.EntryListOption; @@ -24,7 +24,6 @@ import java.io.IOException; import java.io.PrintWriter; -import java.util.Iterator; import javax.servlet.ServletException; import javax.servlet.annotation.WebServlet; @@ -57,9 +56,8 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) out.println("Log Entries " + filter + ":"); try (Logging logging = options.getService()) { Page entries = logging.listLogEntries(EntryListOption.filter(filter)); - Iterator entryIterator = entries.iterateAll(); - while (entryIterator.hasNext()) { - out.println(entryIterator.next()); + for (LogEntry entry : entries.iterateAll()) { + out.println(entry); } } catch (Exception ex) { throw new ServletException(ex); From 21fe0cdf0b592e0976b07bbdc4a51f492494f0b6 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 19 Apr 2017 09:25:29 +1000 Subject: [PATCH 51/65] use package target --- jetty9-base/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jetty9-base/pom.xml b/jetty9-base/pom.xml index 8f05b015..bd018bf9 100644 --- a/jetty9-base/pom.xml +++ b/jetty9-base/pom.xml @@ -112,7 +112,7 @@ gcp-jars - prepare-package + package copy-dependencies @@ -134,7 +134,7 @@ add-jetty-modules - prepare-package + package exec @@ -155,7 +155,7 @@ maven-assembly-plugin - prepare-package + package single From 151d4968c5553d2f3e7169b575db50e78a8f54c3 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 19 Apr 2017 11:09:37 +1000 Subject: [PATCH 52/65] tested with FINE log level --- .../cloud/runtimes/jetty/test/smoke/LogServlet.java | 2 +- .../src/main/webapp/WEB-INF/logging.properties | 1 + .../runtimes/jetty/test/smoke/LoggingIntegrationTest.java | 8 ++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java index a294dbe6..a82e1f43 100644 --- a/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java +++ b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java @@ -45,7 +45,7 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) LoggingOptions options = LoggingOptions.getDefaultInstance(); - String filter = "resource.type=gae_app" + " AND resource.labels.module_id=smoke"; + String filter = "resource.type=gae_app AND resource.labels.module_id=smoke"; String id = req.getPathInfo(); if (id.length() > 1 && id.startsWith("/")) { id = id.substring(1); diff --git a/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties b/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties index 5631f48e..85f25771 100644 --- a/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties +++ b/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties @@ -1,4 +1,5 @@ .level=INFO +io.grpc.netty.level=INFO handlers=com.google.cloud.logging.LoggingHandler com.google.cloud.logging.LoggingHandler.level=FINE diff --git a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java index 47e3912e..200debc6 100644 --- a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java +++ b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java @@ -24,6 +24,7 @@ import com.google.cloud.runtime.jetty.test.annotation.RemoteOnly; import com.google.cloud.runtime.jetty.util.HttpUrlUtil; +import org.hamcrest.Matchers; import org.junit.Test; import java.io.BufferedReader; @@ -55,6 +56,7 @@ public void testLogging() throws Exception { containsString(id)); String traceId = lines.stream().filter(s -> s.startsWith("X-Cloud-Trace-Context: ")) .findFirst().get().split("[ /]")[1]; + assertThat(traceId,Matchers.notNullValue()); // Hit the LogServlet to query the resulting log entries target = getUri().resolve("/log/" + id); @@ -69,11 +71,17 @@ public void testLogging() throws Exception { assertThat(line,containsString("textPayload:" + id)); line = in.readLine(); + while (line != null && !line.contains("severity=INFO")) { + line = in.readLine(); + } assertThat(line,containsString("JUL.info:/dump/info/" + id)); assertThat(line,containsString("appengine.googleapis.com/trace_id=" + traceId)); assertThat(line,containsString("zone=")); line = in.readLine(); + while (line != null && !line.contains("severity=INFO")) { + line = in.readLine(); + } assertThat(line,containsString("ServletContext.log:/dump/info/" + id)); assertThat(line,containsString("appengine.googleapis.com/trace_id=" + traceId)); assertThat(line,containsString("zone=")); From a7e7f3731d9b349adfb770e66cd418b4d993a8ed Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 19 Apr 2017 15:22:53 +1000 Subject: [PATCH 53/65] Simplify entrypoint args processing The $@ arg array is kept complete between all scripts. --- jetty9/src/main/docker/50-jetty.bash | 42 +++++++++---------- jetty9/src/main/docker/Dockerfile | 3 +- jetty9/src/main/docker/docker-entrypoint.bash | 28 +++++++++++++ .../workspace/jetty-setup-default-gcp.bash | 8 ++-- .../test/workspace/jetty-setup-default.bash | 8 ++-- .../test/workspace/jetty-setup-modules.bash | 9 ++-- 6 files changed, 63 insertions(+), 35 deletions(-) create mode 100755 jetty9/src/main/docker/docker-entrypoint.bash diff --git a/jetty9/src/main/docker/50-jetty.bash b/jetty9/src/main/docker/50-jetty.bash index a4d8e488..2ef5b496 100644 --- a/jetty9/src/main/docker/50-jetty.bash +++ b/jetty9/src/main/docker/50-jetty.bash @@ -1,6 +1,10 @@ #!/bin/bash # Configure the environment and CMD for the Jetty Container +set -x +echo $1 +echo $@ + # Unpack a WAR app (if present) beforehand so that Stackdriver Debugger # can load it. This should be done before the JVM for Jetty starts up. export ROOT_WAR=$JETTY_BASE/webapps/root.war @@ -24,25 +28,8 @@ if [ -d "$ROOT_DIR" ]; then cd "$ROOT_DIR" fi -# TODO update docker-entrypoint.bash to do this before calling setup scripts -# If the first argument is the java command -if [ "java" = "$1" -o "$(which java)" = "$1" ] ; then - # ignore it as java is the default command - shift -fi - -# If the first argument is not executable -if ! type "$1" &>/dev/null; then - # The arguments are additional arguments for the default command to run jetty - # move command line args to $JETTY_ARGS - export JETTY_ARGS="$@" - set - "$(which java)" -fi - -# Check for start.jar -if [ "$(echo $JETTY_ARGS | egrep start.jar | wc -l )" = "0" ]; then - JETTY_ARGS="-Djetty.base=${JETTY_BASE} -jar ${JETTY_HOME}/start.jar $JETTY_ARGS" -fi +# Calculate the Jetty args +export JETTY_ARGS # Add any Jetty properties to the JETTY_ARGS if [ "$JETTY_PROPERTIES" ]; then @@ -68,7 +55,20 @@ if [ "$PLATFORM" = "gae" ]; then JETTY_ARGS="$JETTY_ARGS --module=gcp" fi -# add the JETTY_ARGS to the JAVA_OPTS -export JAVA_OPTS="$JAVA_OPTS $JETTY_ARGS" +# If command line is running java, then assume it is jetty and mix in JETTY_ARGS +if [ "$1" = "java" ]; then + # Check for jetty start.jar and prepend if missing + if [ "$(echo $@ | egrep start.jar | wc -l )" = "0" ]; then + shift + set -- java -Djetty.base=${JETTY_BASE} -jar ${JETTY_HOME}/start.jar $@ + fi + + # Append JETTY_ARGS + if [ -n "$JETTY_ARGS" ]; then + shift + set -- java $@ $JETTY_ARGS + fi +fi +echo $@ diff --git a/jetty9/src/main/docker/Dockerfile b/jetty9/src/main/docker/Dockerfile index 8c2358aa..62527e6f 100644 --- a/jetty9/src/main/docker/Dockerfile +++ b/jetty9/src/main/docker/Dockerfile @@ -33,12 +33,13 @@ RUN chown -R jetty:jetty $JETTY_HOME ENV JETTY_BASE ${jetty.base} ENV TMPDIR /tmp/jetty ADD jetty-base $JETTY_BASE +ADD docker-entrypoint.bash / ADD 50-jetty.bash /setup-env.d/ WORKDIR $JETTY_BASE RUN mkdir -p webapps $TMPDIR \ && java -jar $JETTY_HOME/start.jar --add-to-start=setuid \ && chown -R jetty:jetty $JETTY_BASE $TMPDIR \ - && chmod +x /setup-env.d/50-jetty.bash + && chmod +x /setup-env.d/50-jetty.bash /docker-entrypoint.bash EXPOSE 8080 CMD ["java","-Djetty.base=${jetty.base}","-jar","${jetty.home}/start.jar"] diff --git a/jetty9/src/main/docker/docker-entrypoint.bash b/jetty9/src/main/docker/docker-entrypoint.bash new file mode 100755 index 00000000..24910c16 --- /dev/null +++ b/jetty9/src/main/docker/docker-entrypoint.bash @@ -0,0 +1,28 @@ +#!/bin/bash + +# If the first argument is the if full java command +if [ "$(which java)" = "$1" ] ; then + #normalize it + shift + set -- java "$@" +# else if the first argument is not executable assume java +elif ! type "$1" &>/dev/null; then + set -- java "$@" +fi + +# scan the setup-env.d directory for scripts to source for additional setup +if [ -d "${SETUP_ENV:=/setup-env.d}" ]; then + for SCRIPT in $( ls "${SETUP_ENV}/"[0-9]*.bash | sort ) ; do + source ${SCRIPT} + done +fi + +# Do we have JAVA_OPTS for a java command? +if [ "$1" = "java" -a -n "$JAVA_OPTS" ]; then + shift + set -- java $JAVA_OPTS "$@" +fi + +# exec the entry point arguments as a command +exec "$@" + diff --git a/jetty9/src/test/workspace/jetty-setup-default-gcp.bash b/jetty9/src/test/workspace/jetty-setup-default-gcp.bash index 2f85e5b5..d6277802 100755 --- a/jetty9/src/test/workspace/jetty-setup-default-gcp.bash +++ b/jetty9/src/test/workspace/jetty-setup-default-gcp.bash @@ -1,12 +1,12 @@ #!/bin/bash -set - -zero one two three +set - java -zero one two three JAVA_OPTS="-java -options" PLATFORM=gae source /setup-env.d/50-jetty.bash -if [ "$(echo ${JETTY_ARGS} | xargs)" != "-Djetty.base=/var/lib/jetty -jar /opt/jetty-home/start.jar -zero one two three --module=gcp" ]; then +if [ "$(echo ${JETTY_ARGS} | xargs)" != "--module=gcp" ]; then echo "JETTY_ARGS='$(echo ${JETTY_ARGS} | xargs)'" -elif [ "$(echo ${JAVA_OPTS} | xargs)" != "-java -options -Djetty.base=/var/lib/jetty -jar /opt/jetty-home/start.jar -zero one two three --module=gcp" ]; then - echo "JAVA_OPTS='$(echo ${JAVA_OPTS} | xargs)'" +elif [ "$(echo $@ | xargs)" != "java -Djetty.base=/var/lib/jetty -jar /opt/jetty-home/start.jar -zero one two three --module=gcp" ]; then + echo "@='$(echo $@ | xargs)'" else echo OK fi diff --git a/jetty9/src/test/workspace/jetty-setup-default.bash b/jetty9/src/test/workspace/jetty-setup-default.bash index 04a018b5..2592b6f0 100755 --- a/jetty9/src/test/workspace/jetty-setup-default.bash +++ b/jetty9/src/test/workspace/jetty-setup-default.bash @@ -2,10 +2,10 @@ set - java start.jar two three JAVA_OPTS="-java -options" source /setup-env.d/50-jetty.bash -if [ "$(echo ${JETTY_ARGS} | xargs)" != "start.jar two three" ]; then - echo "JETTY_ARGS='${JETTY_ARGS}'" -elif [ "$(echo ${JAVA_OPTS} | xargs)" != "-java -options start.jar two three" ]; then - echo "JAVA_OPTS='${JAVA_OPTS}'" +if [ "$(echo ${JETTY_ARGS} | xargs)" != "" ]; then + echo "JETTY_ARGS='$(echo ${JETTY_ARGS} | xargs)'" +elif [ "$(echo $@ | xargs)" != "java start.jar two three" ]; then + echo "@='$(echo $@ | xargs)'" else echo OK fi diff --git a/jetty9/src/test/workspace/jetty-setup-modules.bash b/jetty9/src/test/workspace/jetty-setup-modules.bash index 82549217..050ed98d 100755 --- a/jetty9/src/test/workspace/jetty-setup-modules.bash +++ b/jetty9/src/test/workspace/jetty-setup-modules.bash @@ -1,6 +1,5 @@ #!/bin/bash set - java start.jar arg -JAVA_OPTS="-X" JETTY_PROPERTIES="prop0=value0,prop1=value1" JETTY_MODULES_ENABLE="mod0,mod1" JETTY_MODILES_DISABLE="gone0,gone1" @@ -9,10 +8,10 @@ touch $JETTY_BASE/start.d/gone0.ini touch $JETTY_BASE/start.d/gone1.ini source /setup-env.d/50-jetty.bash -if [ "$(echo ${JETTY_ARGS} | xargs)" != "start.jar arg prop0=value0 prop1=value1 --module=mod0 --module=mod1" ]; then - echo "JETTY_ARGS='${JETTY_ARGS}'" -elif [ "$(echo ${JAVA_OPTS} | xargs)" != "-X start.jar arg prop0=value0 prop1=value1 --module=mod0 --module=mod1" ]; then - echo "JAVA_OPTS='${JAVA_OPTS}'" +if [ "$(echo ${JETTY_ARGS} | xargs)" != "prop0=value0 prop1=value1 --module=mod0 --module=mod1" ]; then + echo "JETTY_ARGS='$(echo ${JETTY_ARGS} | xargs)'" +elif [ "$(echo $@ | xargs)" != "java start.jar arg prop0=value0 prop1=value1 --module=mod0 --module=mod1" ]; then + echo "@='$(echo $@ | xargs)'" elif [ -x JETTY_BASE/start.d/gone0.ini ]; then echo gone0.ini not deleted elif [ -x JETTY_BASE/start.d/gone1.ini ]; then From 5a57c6d811f0c6ff4c0d7a2b2aed72d9e00c5e16 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 19 Apr 2017 15:25:06 +1000 Subject: [PATCH 54/65] remove debug --- jetty9/src/main/docker/50-jetty.bash | 4 ---- 1 file changed, 4 deletions(-) diff --git a/jetty9/src/main/docker/50-jetty.bash b/jetty9/src/main/docker/50-jetty.bash index 2ef5b496..72134079 100644 --- a/jetty9/src/main/docker/50-jetty.bash +++ b/jetty9/src/main/docker/50-jetty.bash @@ -1,10 +1,6 @@ #!/bin/bash # Configure the environment and CMD for the Jetty Container -set -x -echo $1 -echo $@ - # Unpack a WAR app (if present) beforehand so that Stackdriver Debugger # can load it. This should be done before the JVM for Jetty starts up. export ROOT_WAR=$JETTY_BASE/webapps/root.war From d1d9a4759a4935b6dfb43f9c6cfa907d197a5e6a Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 19 Apr 2017 15:25:49 +1000 Subject: [PATCH 55/65] remove debug --- jetty9/src/main/docker/50-jetty.bash | 2 -- 1 file changed, 2 deletions(-) diff --git a/jetty9/src/main/docker/50-jetty.bash b/jetty9/src/main/docker/50-jetty.bash index 72134079..ed9b2a64 100644 --- a/jetty9/src/main/docker/50-jetty.bash +++ b/jetty9/src/main/docker/50-jetty.bash @@ -66,5 +66,3 @@ if [ "$1" = "java" ]; then fi fi -echo $@ - From 3f32e5471e40d7c0189aad8cdf1aaa09fca3e5aa Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 21 Apr 2017 08:05:31 +1000 Subject: [PATCH 56/65] Test that the logging dependencies are hidden from webapp classpath --- .../src/main/jetty-base/modules/gcp.mod | 3 + tests/test-war-smoke/pom.xml | 1 + .../jetty/test/smoke/ClassLoaderServlet.java | 62 ++++++++++++++ .../runtimes/jetty/test/smoke/LogServlet.java | 66 --------------- .../main/webapp/WEB-INF/logging.properties | 2 + .../test/smoke/LoggingIntegrationTest.java | 83 ++++++++++++------- 6 files changed, 120 insertions(+), 97 deletions(-) create mode 100644 tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/ClassLoaderServlet.java delete mode 100644 tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java diff --git a/jetty9-base/src/main/jetty-base/modules/gcp.mod b/jetty9-base/src/main/jetty-base/modules/gcp.mod index c5b53247..34da110e 100644 --- a/jetty9-base/src/main/jetty-base/modules/gcp.mod +++ b/jetty9-base/src/main/jetty-base/modules/gcp.mod @@ -44,3 +44,6 @@ jetty.httpConfig.sendServerVersion?=true # Don't send date header jetty.httpConfig.sendDateHeader?=false + +# Hide the gcloud libraries from deployed webapps +jetty.webapp.addServerClasses+=,${jetty.base.uri}/lib/gcp/ diff --git a/tests/test-war-smoke/pom.xml b/tests/test-war-smoke/pom.xml index 5a1cc7cf..9547bda0 100644 --- a/tests/test-war-smoke/pom.xml +++ b/tests/test-war-smoke/pom.xml @@ -45,6 +45,7 @@ servlet-api + test com.google.cloud.runtimes.tests diff --git a/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/ClassLoaderServlet.java b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/ClassLoaderServlet.java new file mode 100644 index 00000000..8a306727 --- /dev/null +++ b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/ClassLoaderServlet.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2016 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.runtimes.jetty.test.smoke; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Test that classes are correctly hidden from webapp. + */ +@WebServlet(urlPatterns = {"/classloader"}) +public class ClassLoaderServlet extends HttpServlet { + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + resp.setContentType("text/plain"); + + String[] hidden = { + "com.google.cloud.logging.Logging.class", + "org.eclipse.jetty.server.Server", + "com.google.cloud.BaseService", + "io.netty.util.Constant.class" + }; + + int notFound = 0; + int found = 0; + + ClassLoader loader = Thread.currentThread().getContextClassLoader(); + for (String clazz : hidden) { + try { + loader.loadClass(clazz); + resp.getWriter().printf("Context loaded %s%n", clazz); + found++; + } catch (ClassNotFoundException e) { + resp.getWriter().printf("Context Not Found %s%n", clazz); + notFound++; + } + } + + resp.getWriter().printf("Found classes = %s (0 expected)%n", found); + resp.getWriter().printf("Not found classes = %s (%d expected)%n", notFound, hidden.length); + } +} diff --git a/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java deleted file mode 100644 index a82e1f43..00000000 --- a/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/LogServlet.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2016 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.runtimes.jetty.test.smoke; - -import com.google.api.gax.core.Page; -import com.google.cloud.logging.LogEntry; -import com.google.cloud.logging.Logging; -import com.google.cloud.logging.Logging.EntryListOption; -import com.google.cloud.logging.LoggingOptions; - -import java.io.IOException; -import java.io.PrintWriter; - -import javax.servlet.ServletException; -import javax.servlet.annotation.WebServlet; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -/** - * A servlet to query the Stackdriver logging for the gae_app smoke module. - * An ID string is passed as the pathInfo and is used to filter the textPayload - * and all log records found are included in the text response. - */ -@WebServlet(urlPatterns = {"/log/*"}) -public class LogServlet extends HttpServlet { - @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) - throws ServletException, IOException { - resp.setContentType("text/plain"); - - LoggingOptions options = LoggingOptions.getDefaultInstance(); - - String filter = "resource.type=gae_app AND resource.labels.module_id=smoke"; - String id = req.getPathInfo(); - if (id.length() > 1 && id.startsWith("/")) { - id = id.substring(1); - filter += " AND textPayload:" + id; - } - - PrintWriter out = resp.getWriter(); - out.println("Log Entries " + filter + ":"); - try (Logging logging = options.getService()) { - Page entries = logging.listLogEntries(EntryListOption.filter(filter)); - for (LogEntry entry : entries.iterateAll()) { - out.println(entry); - } - } catch (Exception ex) { - throw new ServletException(ex); - } - } -} \ No newline at end of file diff --git a/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties b/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties index 85f25771..d4f98571 100644 --- a/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties +++ b/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties @@ -1,5 +1,7 @@ .level=INFO +org.eclipse.jetty.http.level=FINE io.grpc.netty.level=INFO +sun.net.level=INFO handlers=com.google.cloud.logging.LoggingHandler com.google.cloud.logging.LoggingHandler.level=FINE diff --git a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java index 200debc6..f7c0ad1b 100644 --- a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java +++ b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java @@ -20,11 +20,18 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import com.google.api.gax.core.Page; +import com.google.cloud.logging.LogEntry; +import com.google.cloud.logging.Logging; +import com.google.cloud.logging.Logging.EntryListOption; +import com.google.cloud.logging.LoggingOptions; +import com.google.cloud.logging.Severity; import com.google.cloud.runtime.jetty.test.AbstractIntegrationTest; import com.google.cloud.runtime.jetty.test.annotation.RemoteOnly; import com.google.cloud.runtime.jetty.util.HttpUrlUtil; import org.hamcrest.Matchers; +import org.junit.Assert; import org.junit.Test; import java.io.BufferedReader; @@ -40,7 +47,6 @@ public class LoggingIntegrationTest extends AbstractIntegrationTest { @Test @RemoteOnly public void testLogging() throws Exception { - // Create unique ID to relate request with log entries String id = Long.toHexString(System.nanoTime()); @@ -49,41 +55,56 @@ public void testLogging() throws Exception { HttpURLConnection http = HttpUrlUtil.openTo(target); assertThat(http.getResponseCode(), is(200)); String responseBody = HttpUrlUtil.getResponseBody(http); - + List lines = new BufferedReader(new StringReader(responseBody)).lines().collect(Collectors.toList()); assertThat(lines.stream().filter(s -> s.startsWith("requestURI=")).findFirst().get(), containsString(id)); - String traceId = lines.stream().filter(s -> s.startsWith("X-Cloud-Trace-Context: ")) - .findFirst().get().split("[ /]")[1]; - assertThat(traceId,Matchers.notNullValue()); - - // Hit the LogServlet to query the resulting log entries - target = getUri().resolve("/log/" + id); + String traceId = lines.stream().filter(s -> s.startsWith("X-Cloud-Trace-Context: ")).findFirst() + .get().split("[ /]")[1]; + assertThat(traceId, Matchers.notNullValue()); - http = HttpUrlUtil.openTo(target); - assertThat(http.getResponseCode(), is(200)); - responseBody = HttpUrlUtil.getResponseBody(http); - - BufferedReader in = new BufferedReader(new StringReader(responseBody)); - String line = in.readLine(); - assertThat(line,containsString("Log Entries ")); - assertThat(line,containsString("textPayload:" + id)); - - line = in.readLine(); - while (line != null && !line.contains("severity=INFO")) { - line = in.readLine(); - } - assertThat(line,containsString("JUL.info:/dump/info/" + id)); - assertThat(line,containsString("appengine.googleapis.com/trace_id=" + traceId)); - assertThat(line,containsString("zone=")); - - line = in.readLine(); - while (line != null && !line.contains("severity=INFO")) { - line = in.readLine(); + LoggingOptions options = LoggingOptions.getDefaultInstance(); + + String filter = + "resource.type=gae_app AND resource.labels.module_id=smoke" + " AND textPayload:" + id; + + int expected = 2; + try (Logging logging = options.getService()) { + Page entries = logging.listLogEntries(EntryListOption.filter(filter)); + for (LogEntry entry : entries.iterateAll()) { + if (entry.getSeverity() == Severity.INFO) { + assertThat(entry.getLogName(), is("gae_app.log")); + assertThat(entry.getResource().getType(), is("gae_app")); + assertThat(entry.getResource().getLabels().get("module_id"), is("smoke")); + assertThat(entry.getResource().getLabels().get("zone"), Matchers.notNullValue()); + assertThat(entry.getLabels().get("appengine.googleapis.com/trace_id"), is(traceId)); + assertThat(entry.getLabels().get("appengine.googleapis.com/instance_name"), + Matchers.notNullValue()); + + if (entry.getPayload().toString().contains("JUL.info:/dump/info/")) { + expected--; + assertThat(entry.getPayload().toString(), Matchers.containsString(id)); + } + if (entry.getPayload().toString().contains("ServletContext.log:/dump/info")) { + expected--; + assertThat(entry.getPayload().toString(), Matchers.containsString(id)); + } + } + } + assertThat(expected, is(0)); } - assertThat(line,containsString("ServletContext.log:/dump/info/" + id)); - assertThat(line,containsString("appengine.googleapis.com/trace_id=" + traceId)); - assertThat(line,containsString("zone=")); + } + + @Test + public void testClassPath() throws Exception { + + URI target = getUri().resolve("/classloader"); + HttpURLConnection http = HttpUrlUtil.openTo(target); + assertThat(http.getResponseCode(), is(200)); + String responseBody = HttpUrlUtil.getResponseBody(http); + + Assert.assertThat(responseBody, Matchers.containsString("Found classes = 0 (0 expected)")); + Assert.assertThat(responseBody, Matchers.containsString("Not found classes = 4 (4 expected)")); } } From 37f5bd4f5048cab93692534bb2de8e6070a86348 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Fri, 21 Apr 2017 08:14:13 +1000 Subject: [PATCH 57/65] update README.md with instructions to keep INFO level on io.grpc.netty.level=INFO --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 10b829a3..6ec145cf 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,10 @@ docker run -it --rm \ ### Enhanced Stackdriver Logging (BETA!) When running on the Google Cloud Platform Flex environment, the Java Util Logging can be configured to send logs to Google Stackdriver Logging by providing a `logging.properties` file that configures a [LoggingHandler](http://googlecloudplatform.github.io/google-cloud-java/0.10.0/apidocs/com/google/cloud/logging/LoggingHandler.html) as follows: ``` +.level=INFO +io.grpc.netty.level=INFO +sun.net.level=INFO + handlers=com.google.cloud.logging.LoggingHandler com.google.cloud.logging.LoggingHandler.level=FINE com.google.cloud.logging.LoggingHandler.log=gae_app.log @@ -180,15 +184,16 @@ com.google.cloud.logging.LoggingHandler.formatter=java.util.logging.SimpleFormat java.util.logging.SimpleFormatter.format=%3$s: %5$s%6$s ``` +This uses the [GaeFlexLoggingEnhancer](http://googlecloudplatform.github.io/google-cloud-java/0.10.0/apidocs/com/google/cloud/logging/GaeFlexLoggingEnhancer.html) to enhances the logs generated be linking them to the `nginx` request log in the logging console by `traceid` (The traceId for a request on a Google Cloud Platform is obtained from the `setCurrentTraceId` HTTP header as the first field of the `'/'` delimited value). -This uses the [GaeFlexLoggingEnhancer](http://googlecloudplatform.github.io/google-cloud-java/0.10.0/apidocs/com/google/cloud/logging/GaeFlexLoggingEnhancer.html) to enhances the logs generated be linking them to the `nginx` request log in the logging console by `traceid` (The traceId for a request on a Google Cloud Platform is obtained from the `setCurrentTraceId` HTTP header as the first field of the `'/'` delimited value). +When an image so configured is deployed on a GCP environment, then the `gcp` module will automatically call the [setCurrentTraceId](http://googlecloudplatform.github.io/google-cloud-java/0.10.0/apidocs/com/google/cloud/logging/GaeFlexLoggingEnhancer.html#setCurrentTraceId-java.lang.String-) for any thread handling a request. -When an image so configured is deployed on a GCP environment, then the `gcp` module will automatically call the [setCurrentTraceId](http://googlecloudplatform.github.io/google-cloud-java/0.10.0/apidocs/com/google/cloud/logging/GaeFlexLoggingEnhancer.html#setCurrentTraceId-java.lang.String-) for any thread handling a request. +When using Stackdriver logging, it is recommended that `io.grpc` and `sun.net` logging level is kept at INFO level, as both these packages are used by Stackdriver internals and can result in verbose and/or initialisation problems. ## Extending the image The image produced by this project may be automatically used/extended by the Cloud SDK and/or App Engine maven plugin. -Alternately it may be explicitly extended with a custom Dockerfile. +Alternately it may be explicitly extended with a custom Dockerfile. The latest released version of this image is available at `launcher.gcr.io/google/jetty`, alternately you may build and push your own version with the shell commands: From 4754bc47962b48bccdad9f5c82df8137cadb87e7 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Sat, 22 Apr 2017 11:05:49 +1000 Subject: [PATCH 58/65] Updated to lasted openjdk-runtime --- jetty9-base/pom.xml | 5 ---- jetty9/src/main/docker/Dockerfile | 3 +- jetty9/src/main/docker/docker-entrypoint.bash | 28 ------------------- pom.xml | 2 +- 4 files changed, 2 insertions(+), 36 deletions(-) delete mode 100755 jetty9/src/main/docker/docker-entrypoint.bash diff --git a/jetty9-base/pom.xml b/jetty9-base/pom.xml index bd018bf9..cdcc3a8b 100644 --- a/jetty9-base/pom.xml +++ b/jetty9-base/pom.xml @@ -45,11 +45,6 @@ commons-logging commons-logging - - - javax.servlet - servlet-api - diff --git a/jetty9/src/main/docker/Dockerfile b/jetty9/src/main/docker/Dockerfile index 62527e6f..8c2358aa 100644 --- a/jetty9/src/main/docker/Dockerfile +++ b/jetty9/src/main/docker/Dockerfile @@ -33,13 +33,12 @@ RUN chown -R jetty:jetty $JETTY_HOME ENV JETTY_BASE ${jetty.base} ENV TMPDIR /tmp/jetty ADD jetty-base $JETTY_BASE -ADD docker-entrypoint.bash / ADD 50-jetty.bash /setup-env.d/ WORKDIR $JETTY_BASE RUN mkdir -p webapps $TMPDIR \ && java -jar $JETTY_HOME/start.jar --add-to-start=setuid \ && chown -R jetty:jetty $JETTY_BASE $TMPDIR \ - && chmod +x /setup-env.d/50-jetty.bash /docker-entrypoint.bash + && chmod +x /setup-env.d/50-jetty.bash EXPOSE 8080 CMD ["java","-Djetty.base=${jetty.base}","-jar","${jetty.home}/start.jar"] diff --git a/jetty9/src/main/docker/docker-entrypoint.bash b/jetty9/src/main/docker/docker-entrypoint.bash deleted file mode 100755 index 24910c16..00000000 --- a/jetty9/src/main/docker/docker-entrypoint.bash +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# If the first argument is the if full java command -if [ "$(which java)" = "$1" ] ; then - #normalize it - shift - set -- java "$@" -# else if the first argument is not executable assume java -elif ! type "$1" &>/dev/null; then - set -- java "$@" -fi - -# scan the setup-env.d directory for scripts to source for additional setup -if [ -d "${SETUP_ENV:=/setup-env.d}" ]; then - for SCRIPT in $( ls "${SETUP_ENV}/"[0-9]*.bash | sort ) ; do - source ${SCRIPT} - done -fi - -# Do we have JAVA_OPTS for a java command? -if [ "$1" = "java" -a -n "$JAVA_OPTS" ]; then - shift - set -- java $JAVA_OPTS "$@" -fi - -# exec the entry point arguments as a command -exec "$@" - diff --git a/pom.xml b/pom.xml index 651ca768..eb9c90f7 100644 --- a/pom.xml +++ b/pom.xml @@ -36,7 +36,7 @@ ${docker.tag.prefix}9.${jetty9.minor.version} ${docker.tag.prefix}9.${jetty9.minor.version}-${maven.build.timestamp} - gcr.io/google-appengine/openjdk:8-2017-04-12_14_02 + gcr.io/google-appengine/openjdk:8-2017-04-21_15_02 From 86154106a77a3686e64da245c9641ffb661316f2 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 26 Apr 2017 14:30:59 +0200 Subject: [PATCH 59/65] fixed classloader test --- .../src/main/jetty-base/modules/gcp.mod | 2 +- .../src/main/jetty-base/modules/slf4j-api.mod | 44 +++++++++++++++++++ .../jetty/test/smoke/ClassLoaderServlet.java | 14 +++--- .../test/smoke/LoggingIntegrationTest.java | 2 +- 4 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 jetty9-base/src/main/jetty-base/modules/slf4j-api.mod diff --git a/jetty9-base/src/main/jetty-base/modules/gcp.mod b/jetty9-base/src/main/jetty-base/modules/gcp.mod index 34da110e..07a84b7a 100644 --- a/jetty9-base/src/main/jetty-base/modules/gcp.mod +++ b/jetty9-base/src/main/jetty-base/modules/gcp.mod @@ -46,4 +46,4 @@ jetty.httpConfig.sendServerVersion?=true jetty.httpConfig.sendDateHeader?=false # Hide the gcloud libraries from deployed webapps -jetty.webapp.addServerClasses+=,${jetty.base.uri}/lib/gcp/ +jetty.webapp.addServerClasses+=,file://${jetty.base}/lib/gcp/ diff --git a/jetty9-base/src/main/jetty-base/modules/slf4j-api.mod b/jetty9-base/src/main/jetty-base/modules/slf4j-api.mod new file mode 100644 index 00000000..411f60b1 --- /dev/null +++ b/jetty9-base/src/main/jetty-base/modules/slf4j-api.mod @@ -0,0 +1,44 @@ +[description] +Provides SLF4J API. Requires a slf4j implementation (eg slf4j-simple-impl) +otherwise a noop implementation is used. + +TODO Remove once jetty-9.5.4 released + +[tags] +logging +slf4j +internal + +[files] +maven://org.slf4j/slf4j-api/${slf4j.version}|lib/slf4j/slf4j-api-${slf4j.version}.jar + +[lib] +lib/slf4j/slf4j-api-${slf4j.version}.jar + +[ini] +slf4j.version=1.7.21 +jetty.webapp.addServerClasses+=,file://${jetty.base}/lib/slf4j/ + +[license] +SLF4J is distributed under the MIT License. +Copyright (c) 2004-2013 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/ClassLoaderServlet.java b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/ClassLoaderServlet.java index 8a306727..9562cb5c 100644 --- a/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/ClassLoaderServlet.java +++ b/tests/test-war-smoke/src/main/java/com/google/cloud/runtimes/jetty/test/smoke/ClassLoaderServlet.java @@ -35,23 +35,25 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) resp.setContentType("text/plain"); String[] hidden = { - "com.google.cloud.logging.Logging.class", + "com.google.cloud.logging.Logging", "org.eclipse.jetty.server.Server", "com.google.cloud.BaseService", - "io.netty.util.Constant.class" + "io.netty.channel.Channel", + "io.netty.util.Timer", + "org.slf4j.Logger" }; int notFound = 0; int found = 0; ClassLoader loader = Thread.currentThread().getContextClassLoader(); - for (String clazz : hidden) { + for (String name : hidden) { try { - loader.loadClass(clazz); - resp.getWriter().printf("Context loaded %s%n", clazz); + Class clazz = loader.loadClass(name); + resp.getWriter().printf("Context loaded %s from%s%n", name, clazz.getClassLoader()); found++; } catch (ClassNotFoundException e) { - resp.getWriter().printf("Context Not Found %s%n", clazz); + resp.getWriter().printf("Context Not Found %s%n", name); notFound++; } } diff --git a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java index f7c0ad1b..9d3a3a20 100644 --- a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java +++ b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java @@ -105,6 +105,6 @@ public void testClassPath() throws Exception { String responseBody = HttpUrlUtil.getResponseBody(http); Assert.assertThat(responseBody, Matchers.containsString("Found classes = 0 (0 expected)")); - Assert.assertThat(responseBody, Matchers.containsString("Not found classes = 4 (4 expected)")); + Assert.assertThat(responseBody, Matchers.containsString("Not found classes = 6 (6 expected)")); } } From 115759a56cfc1f2baacec69d7b048982186653a1 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 26 Apr 2017 19:34:10 +0200 Subject: [PATCH 60/65] fixed TODO --- jetty9-base/src/main/jetty-base/modules/slf4j-api.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jetty9-base/src/main/jetty-base/modules/slf4j-api.mod b/jetty9-base/src/main/jetty-base/modules/slf4j-api.mod index 411f60b1..a08faa43 100644 --- a/jetty9-base/src/main/jetty-base/modules/slf4j-api.mod +++ b/jetty9-base/src/main/jetty-base/modules/slf4j-api.mod @@ -2,7 +2,7 @@ Provides SLF4J API. Requires a slf4j implementation (eg slf4j-simple-impl) otherwise a noop implementation is used. -TODO Remove once jetty-9.5.4 released +TODO Remove this file once jetty-9.5.5 released [tags] logging From 91a62d8e319d47a4fc5d85a80c0da3359951e910 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 3 May 2017 15:43:31 +0200 Subject: [PATCH 61/65] Update to jetty 9.4.5 and gcloud 1.0.1-SNAPSHOT --- .../runtimes/jetty9/RequestContextScope.java | 6 +-- .../src/main/jetty-base/modules/slf4j-api.mod | 44 ------------------- pom.xml | 6 +-- tests/test-war-smoke/pom.xml | 7 --- .../main/webapp/WEB-INF/logging.properties | 2 - .../test/smoke/LoggingIntegrationTest.java | 2 +- 6 files changed, 7 insertions(+), 60 deletions(-) delete mode 100644 jetty9-base/src/main/jetty-base/modules/slf4j-api.mod diff --git a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java index 4b694171..cb7708b6 100644 --- a/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java +++ b/jetty9-base/src/main/java/com/google/cloud/runtimes/jetty9/RequestContextScope.java @@ -16,7 +16,7 @@ package com.google.cloud.runtimes.jetty9; -import com.google.cloud.logging.GaeFlexLoggingEnhancer; +import com.google.cloud.logging.TraceLoggingEnhancer; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.ContextHandler; @@ -57,7 +57,7 @@ public void enterScope(Context context, Request request, Object reason) { traceId = traceId.substring(0, slash); } request.setAttribute(X_CLOUD_TRACE, traceId); - GaeFlexLoggingEnhancer.setCurrentTraceId(traceId); + TraceLoggingEnhancer.setCurrentTraceId(traceId); } } else { depth = depth + 1; @@ -78,7 +78,7 @@ public void exitScope(Context context, Request request) { contextDepth.set(depth - 1); } else { contextDepth.remove(); - GaeFlexLoggingEnhancer.setCurrentTraceId(null); + TraceLoggingEnhancer.setCurrentTraceId(null); } } } diff --git a/jetty9-base/src/main/jetty-base/modules/slf4j-api.mod b/jetty9-base/src/main/jetty-base/modules/slf4j-api.mod deleted file mode 100644 index a08faa43..00000000 --- a/jetty9-base/src/main/jetty-base/modules/slf4j-api.mod +++ /dev/null @@ -1,44 +0,0 @@ -[description] -Provides SLF4J API. Requires a slf4j implementation (eg slf4j-simple-impl) -otherwise a noop implementation is used. - -TODO Remove this file once jetty-9.5.5 released - -[tags] -logging -slf4j -internal - -[files] -maven://org.slf4j/slf4j-api/${slf4j.version}|lib/slf4j/slf4j-api-${slf4j.version}.jar - -[lib] -lib/slf4j/slf4j-api-${slf4j.version}.jar - -[ini] -slf4j.version=1.7.21 -jetty.webapp.addServerClasses+=,file://${jetty.base}/lib/slf4j/ - -[license] -SLF4J is distributed under the MIT License. -Copyright (c) 2004-2013 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pom.xml b/pom.xml index eb9c90f7..224e1ed8 100644 --- a/pom.xml +++ b/pom.xml @@ -29,10 +29,10 @@ UTF-8 UTF-8 yyyy-MM-dd-HH-mm - 0.13.0-beta + 1.0.1-SNAPSHOT 4 - 3 - 9.${jetty9.minor.version}.${jetty9.dot.version}.v20170317 + 5 + 9.${jetty9.minor.version}.${jetty9.dot.version}.v20170502 ${docker.tag.prefix}9.${jetty9.minor.version} ${docker.tag.prefix}9.${jetty9.minor.version}-${maven.build.timestamp} diff --git a/tests/test-war-smoke/pom.xml b/tests/test-war-smoke/pom.xml index 9547bda0..9ec50693 100644 --- a/tests/test-war-smoke/pom.xml +++ b/tests/test-war-smoke/pom.xml @@ -38,13 +38,6 @@ com.google.cloud google-cloud-logging ${gcloud.api.version} - - - - javax.servlet - servlet-api - - test diff --git a/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties b/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties index d4f98571..7e278424 100644 --- a/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties +++ b/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties @@ -6,7 +6,5 @@ sun.net.level=INFO handlers=com.google.cloud.logging.LoggingHandler com.google.cloud.logging.LoggingHandler.level=FINE com.google.cloud.logging.LoggingHandler.log=gae_app.log -com.google.cloud.logging.LoggingHandler.resourceType=gae_app -com.google.cloud.logging.LoggingHandler.enhancers=com.google.cloud.logging.GaeFlexLoggingEnhancer com.google.cloud.logging.LoggingHandler.formatter=java.util.logging.SimpleFormatter java.util.logging.SimpleFormatter.format=%3$s: %5$s%6$s diff --git a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java index 9d3a3a20..dd82ad4e 100644 --- a/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java +++ b/tests/test-war-smoke/src/test/java/com/google/cloud/runtimes/jetty/test/smoke/LoggingIntegrationTest.java @@ -20,7 +20,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import com.google.api.gax.core.Page; +import com.google.api.gax.paging.Page; import com.google.cloud.logging.LogEntry; import com.google.cloud.logging.Logging; import com.google.cloud.logging.Logging.EntryListOption; From 4b80727a780e6d7e3ef17e506d1f65ac843e6aba Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Tue, 9 May 2017 11:17:46 +0200 Subject: [PATCH 62/65] upgraded to gcloud-logging 1.0.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4adc80fa..1efa9fc5 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ UTF-8 UTF-8 yyyy-MM-dd-HH-mm - 1.0.1-SNAPSHOT + 1.0.1 4 5 9.${jetty9.minor.version}.${jetty9.dot.version}.v20170502 From ae6b9ce1d512a5f9b31501d3020cb4e1ff4f9f93 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Tue, 9 May 2017 15:50:01 +0200 Subject: [PATCH 63/65] turn off debug --- tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties b/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties index 7e278424..267ac4d5 100644 --- a/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties +++ b/tests/test-war-smoke/src/main/webapp/WEB-INF/logging.properties @@ -1,5 +1,5 @@ .level=INFO -org.eclipse.jetty.http.level=FINE +org.eclipse.jetty.http.level=INFO io.grpc.netty.level=INFO sun.net.level=INFO From df47a079c3c596b830984db9dee57fd0da496189 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 10 May 2017 12:22:22 +0200 Subject: [PATCH 64/65] updated README for latest 1.0.1 gcloud-logging --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0b1c7c76..097c7681 100644 --- a/README.md +++ b/README.md @@ -178,15 +178,17 @@ sun.net.level=INFO handlers=com.google.cloud.logging.LoggingHandler com.google.cloud.logging.LoggingHandler.level=FINE com.google.cloud.logging.LoggingHandler.log=gae_app.log -com.google.cloud.logging.LoggingHandler.resourceType=gae_app -com.google.cloud.logging.LoggingHandler.enhancers=com.google.cloud.logging.GaeFlexLoggingEnhancer com.google.cloud.logging.LoggingHandler.formatter=java.util.logging.SimpleFormatter java.util.logging.SimpleFormatter.format=%3$s: %5$s%6$s ``` -This uses the [GaeFlexLoggingEnhancer](http://googlecloudplatform.github.io/google-cloud-java/0.10.0/apidocs/com/google/cloud/logging/GaeFlexLoggingEnhancer.html) to enhances the logs generated be linking them to the `nginx` request log in the logging console by `traceid` (The traceId for a request on a Google Cloud Platform is obtained from the `setCurrentTraceId` HTTP header as the first field of the `'/'` delimited value). +When deployed on the GCP Flex environment, an image so configured will automatically be configured with: +* a [LabelLoggingEnhancer](https://github.com/GoogleCloudPlatform/google-cloud-java/blob/v0.17.2/google-cloud-logging/src/main/java/com/google/cloud/logging/MonitoredResourceUtil.java#L224) instance, that will add labels from the monitored resource to each log entry. +* a [TraceLoggingEnhancer](https://github.com/GoogleCloudPlatform/google-cloud-java/blob/v0.17.2/google-cloud-logging/src/main/java/com/google/cloud/logging/TraceLoggingEnhancer.java) instance that will add any trace-id set to each log entry. +* the `gcp` module will be enabled that configures jetty so that the [setCurrentTraceId](https://github.com/GoogleCloudPlatform/google-cloud-java/blob/v0.17.2/google-cloud-logging/src/main/java/com/google/cloud/logging/TraceLoggingEnhancer.java#L40) method is called for any thread handling a request. -When an image so configured is deployed on a GCP environment, then the `gcp` module will automatically call the [setCurrentTraceId](http://googlecloudplatform.github.io/google-cloud-java/0.10.0/apidocs/com/google/cloud/logging/GaeFlexLoggingEnhancer.html#setCurrentTraceId-java.lang.String-) for any thread handling a request. +When deployed in other environments, logging enhancers can be manually configured by setting a comma separated list of class names as the +`com.google.cloud.logging.LoggingHandler.enhancers` property. When using Stackdriver logging, it is recommended that `io.grpc` and `sun.net` logging level is kept at INFO level, as both these packages are used by Stackdriver internals and can result in verbose and/or initialisation problems. From bfc9e4598c4cd091432f134ca278366bc594050f Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Thu, 11 May 2017 15:05:09 +0200 Subject: [PATCH 65/65] update classpath exclusion --- jetty9-base/src/main/jetty-base/modules/gcp.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jetty9-base/src/main/jetty-base/modules/gcp.mod b/jetty9-base/src/main/jetty-base/modules/gcp.mod index 07a84b7a..34da110e 100644 --- a/jetty9-base/src/main/jetty-base/modules/gcp.mod +++ b/jetty9-base/src/main/jetty-base/modules/gcp.mod @@ -46,4 +46,4 @@ jetty.httpConfig.sendServerVersion?=true jetty.httpConfig.sendDateHeader?=false # Hide the gcloud libraries from deployed webapps -jetty.webapp.addServerClasses+=,file://${jetty.base}/lib/gcp/ +jetty.webapp.addServerClasses+=,${jetty.base.uri}/lib/gcp/