Add support for JUL auto ECS-reformatting

CHANGELOG.asciidoc

@@ -20,8 +20,8 @@ endif::[]
 
 === Unreleased
 
-[[release-notes-1.30.2]]
-==== 1.30.2 - YYYY/MM/DD
+[[release-notes-1.31.0]]
+==== 1.31.0 - YYYY/MM/DD
 
 [float]
 ===== Potentially breaking changes
@@ -40,6 +40,7 @@ allows for spans representing asynchronous handling of requests for which the co
 [float]
 ===== Features
 * Set the service version when using the ECS reformatting of the application logs: {pull}2603[#2603]
+* Add ECS-reformatting support for `java.util.logging` - {pull}2591[#2591]
 
 [float]
 ===== Bug fixes

apm-agent-core/src/main/java/co/elastic/apm/agent/logging/LoggingConfiguration.java

@@ -155,7 +155,7 @@ static LogLevel mapLogLevel(LogLevel original) {
         .tags("added[1.22.0]", "experimental")
         .description("Specifying whether and how the agent should automatically reformat application logs \n" +
             "into {ecs-logging-ref}/intro.html[ECS-compatible JSON], suitable for ingestion into Elasticsearch for \n" +
-            "further Log analysis. This functionality is available for log4j1, log4j2 and Logback. \n" +
+            "further Log analysis. This functionality is available for log4j1, log4j2, Logback and `java.util.logging`. \n" +
             "The ECS log lines will include active trace/transaction/error IDs, if there are such. \n" +
             "\n" +
             "This option only applies to pattern layouts/formatters by default.\n" +
@@ -210,7 +210,9 @@ static LogLevel mapLogLevel(LogLevel original) {
         .buildWithDefault(Arrays.asList(
             WildcardMatcher.valueOf("*PatternLayout*"),
             WildcardMatcher.valueOf("org.apache.log4j.SimpleLayout"),
-            WildcardMatcher.valueOf("ch.qos.logback.core.encoder.EchoEncoder")
+            WildcardMatcher.valueOf("ch.qos.logback.core.encoder.EchoEncoder"),
+            WildcardMatcher.valueOf("java.util.logging.SimpleFormatter"),
+            WildcardMatcher.valueOf("org.springframework.boot.logging.java.SimpleFormatter")
         ));
 
     private final ConfigurationOption<String> logEcsFormattingDestinationDir = ConfigurationOption.stringOption()
@@ -384,6 +386,10 @@ public long getLogFileSize() {
         return logFileSize.get().getBytes();
     }
 
+    public long getDefaultLogFileSize() {
+        return logFileSize.getValueConverter().convert(logFileSize.getDefaultValueAsString()).getBytes();
+    }
+
     public boolean isShipAgentLogs() {
         return shipAgentLogs.get();
     }

apm-agent-plugins/apm-ecs-logging-plugin/pom.xml

@@ -19,7 +19,7 @@
         <dependency>
             <groupId>co.elastic.logging</groupId>
             <artifactId>log4j-ecs-layout</artifactId>
-            <version>1.2.0</version>
+            <version>1.4.0</version>
             <scope>provided</scope>
         </dependency>
         <dependency>

apm-agent-plugins/apm-logging-plugin/apm-jul-plugin/pom.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>apm-logging-plugin</artifactId>
+        <groupId>co.elastic.apm</groupId>
+        <version>1.30.2-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>apm-jul-plugin</artifactId>
+    <name>${project.groupId}:${project.artifactId}</name>
+
+    <properties>
+        <apm-agent-parent.base.dir>${project.basedir}/../../..</apm-agent-parent.base.dir>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>apm-logging-plugin-common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>co.elastic.logging</groupId>
+            <artifactId>jul-ecs-formatter</artifactId>
+            <version>${version.ecs.logging}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>apm-logging-plugin-common</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <!-- adds the ability to use reflection to inspect java.util.logging classes -->
+                    <argLine>--add-opens java.logging/java.util.logging=ALL-UNNAMED</argLine>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

apm-agent-plugins/apm-logging-plugin/apm-jul-plugin/src/main/java/co/elastic/apm/agent/jul/JulPluginClassLoaderRootPackageCustomizer.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.jul;
+
+import co.elastic.apm.agent.loginstr.LoggingPluginClassLoaderRootPackageCustomizer;
+
+public class JulPluginClassLoaderRootPackageCustomizer extends LoggingPluginClassLoaderRootPackageCustomizer {}

apm-agent-plugins/apm-logging-plugin/apm-jul-plugin/src/main/java/co/elastic/apm/agent/jul/error/package-info.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+@NonnullApi
+package co.elastic.apm.agent.jul.error;
+
+import co.elastic.apm.agent.sdk.NonnullApi;

apm-agent-plugins/apm-logging-plugin/apm-jul-plugin/src/main/java/co/elastic/apm/agent/jul/reformatting/JulConsoleHandlerPublishAdvice.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.jul.reformatting;
+
+import net.bytebuddy.asm.Advice;
+import net.bytebuddy.implementation.bytecode.assign.Assigner;
+
+import java.util.logging.ConsoleHandler;
+import java.util.logging.LogRecord;
+import java.util.logging.StreamHandler;
+
+public class JulConsoleHandlerPublishAdvice {
+
+    private static final JulEcsReformattingHelper helper = new JulEcsReformattingHelper();
+
+    @SuppressWarnings("unused")
+    @Advice.OnMethodEnter(suppress = Throwable.class, skipOn = Advice.OnNonDefaultValue.class, inline = false)
+    public static boolean initializeReformatting(@Advice.This(typing = Assigner.Typing.DYNAMIC) ConsoleHandler thisHandler) {
+        return helper.onAppendEnter(thisHandler);
+    }
+
+    @SuppressWarnings({"unused"})
+    @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class, inline = false)
+    public static void reformatLoggingEvent(@Advice.Argument(value = 0, typing = Assigner.Typing.DYNAMIC) final LogRecord logRecord,
+                                            @Advice.This(typing = Assigner.Typing.DYNAMIC) ConsoleHandler thisHandler) {
+
+        StreamHandler shadeAppender = helper.onAppendExit(thisHandler);
+        if (shadeAppender != null) {
+            shadeAppender.publish(logRecord);
+        }
+    }
+}

apm-agent-plugins/apm-logging-plugin/apm-jul-plugin/src/main/java/co/elastic/apm/agent/jul/reformatting/JulEcsReformattingHelper.java

@@ -0,0 +1,161 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.jul.reformatting;
+
+import co.elastic.apm.agent.loginstr.reformatting.AbstractEcsReformattingHelper;
+import co.elastic.apm.agent.loginstr.reformatting.Utils;
+import co.elastic.apm.agent.sdk.logging.Logger;
+import co.elastic.apm.agent.sdk.logging.LoggerFactory;
+import co.elastic.apm.agent.util.LoggerUtils;
+import co.elastic.logging.AdditionalField;
+import co.elastic.logging.jul.EcsFormatter;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.ConsoleHandler;
+import java.util.logging.FileHandler;
+import java.util.logging.Formatter;
+import java.util.logging.StreamHandler;
+
+class JulEcsReformattingHelper extends AbstractEcsReformattingHelper<StreamHandler, Formatter> {
+
+    private static final Logger logger = LoggerFactory.getLogger(JulEcsReformattingHelper.class);
+    private static final Logger oneTimeLogFileLimitWarningLogger = LoggerUtils.logOnce(logger);
+
+    private static final ThreadLocal<String> currentPattern = new ThreadLocal<>();
+    private static final ThreadLocal<Path> currentExampleLogFile = new ThreadLocal<>();
+
+    JulEcsReformattingHelper() {}
+
+    public boolean onAppendEnter(FileHandler fileHandler, String pattern, File exampleLogFile) {
+        try {
+            currentPattern.set(pattern);
+            currentExampleLogFile.set(exampleLogFile.toPath());
+            return super.onAppendEnter(fileHandler);
+        } finally {
+            currentPattern.remove();
+            currentExampleLogFile.remove();
+        }
+    }
+
+    @Nullable
+    @Override
+    protected Formatter getFormatterFrom(StreamHandler handler) {
+        return handler.getFormatter();
+    }
+
+    @Override
+    protected void setFormatter(StreamHandler handler, Formatter formatter) {
+        handler.setFormatter(formatter);
+    }
+
+    @Override
+    protected String getAppenderName(StreamHandler handler) {
+        if (handler instanceof FileHandler) {
+            return "FILE";
+        } else if (handler instanceof ConsoleHandler) {
+            return "CONSOLE";
+        } else {
+            return handler.getClass().getSimpleName();
+        }
+    }
+
+    @Override
+    protected Formatter createEcsFormatter(String eventDataset, @Nullable String serviceName, @Nullable String serviceVersion,
+                                           @Nullable String serviceNodeName, @Nullable Map<String, String> additionalFields,
+                                           Formatter originalFormatter) {
+        EcsFormatter ecsFormatter = new EcsFormatter();
+        ecsFormatter.setServiceName(serviceName);
+        ecsFormatter.setServiceVersion(serviceVersion);
+        ecsFormatter.setServiceNodeName(serviceNodeName);
+        ecsFormatter.setEventDataset(eventDataset);
+        if (additionalFields != null && !additionalFields.isEmpty()) {
+            List<AdditionalField> additionalFieldList = new ArrayList<>();
+            for (Map.Entry<String, String> keyValuePair : additionalFields.entrySet()) {
+                additionalFieldList.add(new AdditionalField(keyValuePair.getKey(), keyValuePair.getValue()));
+            }
+            ecsFormatter.setAdditionalFields(additionalFieldList);
+        }
+        ecsFormatter.setIncludeOrigin(false);
+        ecsFormatter.setStackTraceAsArray(false);
+        return ecsFormatter;
+    }
+
+    @Nullable
+    @Override
+    protected StreamHandler createAndStartEcsAppender(StreamHandler originalHandler, String ecsAppenderName, Formatter ecsFormatter) {
+        StreamHandler shadeHandler = null;
+        if (originalHandler instanceof FileHandler) {
+            try {
+                String pattern = computeEcsFileHandlerPattern(
+                    currentPattern.get(),
+                    currentExampleLogFile.get(),
+                    getConfiguredReformattingDir(),
+                    true
+                );
+                // In earlier versions, there is only constructor with log file limit given as int, whereas in later ones there are
+                // overloads for both either int or long. Typically, this should be enough, but not necessarily
+                int maxLogFileSize = (int) getMaxLogFileSize();
+                if ((long) maxLogFileSize != getMaxLogFileSize()) {
+                    maxLogFileSize = (int) getDefaultMaxLogFileSize();
+                    oneTimeLogFileLimitWarningLogger.warn("Configured log max size ({} bytes) is too big for JUL settings, which " +
+                        "use int to configure the file size limit. Consider reducing the log max size configuration to a value below " +
+                        "Integer#MAX_VALUE. Defaulting to {} bytes.", getMaxLogFileSize(), maxLogFileSize);
+                }
+                shadeHandler = new FileHandler(pattern, maxLogFileSize, 2, true);
+                shadeHandler.setFormatter(ecsFormatter);
+            } catch (Exception e) {
+                logger.error("Failed to create Log shading FileAppender. Auto ECS reformatting will not work.", e);
+            }
+        }
+        return shadeHandler;
+    }
+
+    static String computeEcsFileHandlerPattern(String pattern, Path originalFilePath, @Nullable String configuredReformattingDir,
+                                               boolean createDirs) throws IOException {
+        pattern = Utils.replaceFileExtensionToEcsJson(pattern);
+        // if the pattern does not contain rotation component, append one at the end
+        if (!pattern.contains("%g")) {
+            pattern = pattern + ".%g";
+        }
+        int lastPathSeparatorIndex = pattern.lastIndexOf('/');
+        if (lastPathSeparatorIndex > 0 && pattern.length() > lastPathSeparatorIndex) {
+            pattern = pattern.substring(lastPathSeparatorIndex + 1);
+        }
+        Path logReformattingDir = Utils.computeLogReformattingDir(originalFilePath, configuredReformattingDir);
+        if (logReformattingDir != null) {
+            if (createDirs && !Files.exists(logReformattingDir)) {
+                Files.createDirectories(logReformattingDir);
+            }
+            pattern = logReformattingDir.resolve(pattern).toString();
+        }
+        return pattern;
+    }
+
+    @Override
+    protected void closeShadeAppender(StreamHandler shadeHandler) {
+        shadeHandler.close();
+    }
+}