set service name & version in ecs-logging (#3064)

Author: SylvainJuge

CHANGELOG.asciidoc

@@ -23,6 +23,13 @@ endif::[]
 [[release-notes-1.37.0]]
 ==== 1.37.0 - YYYY/MM/DD
 
+[float]
+===== Potentially breaking changes
+* The list of instrumentation groups for log-related features have been simplified to `logging`, `log-reformatting`, `log-correlation` and `log-error`,
+as a consequence any agent configuration with `enable_instrumentations` or `disable_instrumentations` might require adjustments as the following instrumentation
+groups no longer exist: `jul-ecs`, `jul-error`, `log4j1-correlation`, `log4j1-ecs`, `log4j1-error`, `log4j2-correlation`, `log4j2-ecs`, `log4j2-error`, 
+`logback-correlation`, `logback-ecs` and `slf4j-error`.
+
 [float]
 ===== Features
 * Add the <<config-disable-outgoing-tracecontext-headers>> config option to disable injection of `tracecontext` on outgoing
@@ -32,6 +39,7 @@ communication - {pull}2996[#2996]
 * Added agent.activation_method telemetry - {pull}2926[#2926]
 * Allow creation of exit spans with `@CaptureSpan` and `@Traced` annotations - {pull}3046[#3046]
 * Add the <<config-long-field-max-length>> config to enable capturing larger values for specific fields - {pull}3027[#3027]
+* Provide fallback correlation when `ecs-logging-java` is used - {pull}3064[#3064]
 
 [float]
 ===== Bug fixes
@@ -49,6 +57,7 @@ communication - {pull}2996[#2996]
 [[release-notes-1.36.0]]
 ==== 1.36.0 - 2023/01/27
 
+[float]
 ===== Potentially breaking changes
 * Previously, agent instrumented all implementations of `javax.jms.MessageListener`, which could cause performance issues.
 From now on, the instrumentation is limited to implementations that fit at least one of the following criteria:

CONTRIBUTING.md

@@ -256,7 +256,7 @@ See [`apm-agent-plugins/README.md`](apm-agent-plugins/README.md)
 
 HTML Documentation is generated from text files stored in `docs` folder using [AsciiDoc](http://asciidoc.org/) format.
 The `configuration.asciidoc` file is generated from running `co.elastic.apm.agent.configuration.ConfigurationExporter`
-(e.g. via `./mvnw test -Dtest="ConfigurationExporterTest" -DfailIfNoTests=false`). All the other asciidoc text files
+(e.g. via `./mvnw -Dsurefire.failIfNoTests=false -Dtest=ConfigurationExporterTest -pl apm-agent -am clean test`). All the other asciidoc text files
 are written manually.
 
 A preview of the documentation is generated for each pull-request.

apm-agent-core/src/test/java/co/elastic/apm/agent/testutils/TestClassWithDependencyRunner.java

@@ -98,7 +98,9 @@ public void run() {
             System.out.println(failure);
             failure.getException().printStackTrace();
         }
-        assertThat(listener.getSummary().getTestsFailedCount()).isZero();
+        assertThat(listener.getSummary().getTestsFailedCount())
+            .describedAs("at least one test failure reported, see stack trace for investigation")
+            .isZero();
     }
 
 }

apm-agent-plugins/apm-ecs-logging-plugin/pom.xml

@@ -16,36 +16,82 @@
     </properties>
 
     <dependencies>
+
+        <!-- log4j 1.x -->
         <dependency>
             <groupId>co.elastic.logging</groupId>
             <artifactId>log4j-ecs-layout</artifactId>
             <version>${version.ecs.logging}</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <version>1.2.17</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+
+        <!-- log4j 2.x -->
         <dependency>
             <groupId>co.elastic.logging</groupId>
             <artifactId>log4j2-ecs-layout</artifactId>
             <version>${version.ecs.logging}</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+            <version>2.20.0</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+
+        <!-- JUL -->
         <dependency>
             <groupId>co.elastic.logging</groupId>
             <artifactId>jul-ecs-formatter</artifactId>
             <version>${version.ecs.logging}</version>
             <scope>provided</scope>
         </dependency>
+
+        <!-- Jboss logmanager -->
         <dependency>
-            <groupId>co.elastic.apm</groupId>
-            <artifactId>apm-logging-plugin-common</artifactId>
-            <version>${project.version}</version>
+            <groupId>co.elastic.logging</groupId>
+            <artifactId>jboss-logmanager-ecs-formatter</artifactId>
+            <version>${version.ecs.logging}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
-            <groupId>org.apache.logging.log4j</groupId>
-            <artifactId>log4j-core</artifactId>
-            <version>2.20.0</version>
+            <groupId>org.jboss.logmanager</groupId>
+            <artifactId>jboss-logmanager</artifactId>
+            <version>2.1.15.Final</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+
+        <!-- logback -->
+        <dependency>
+            <groupId>co.elastic.logging</groupId>
+            <artifactId>logback-ecs-encoder</artifactId>
+            <version>${version.ecs.logging}</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>1.2.8</version>
+            <scope>provided</scope>
+            <optional>true</optional>
+        </dependency>
+
+        <dependency>
+            <groupId>co.elastic.apm</groupId>
+            <artifactId>apm-logging-plugin-common</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.ivy</groupId>
             <artifactId>ivy</artifactId>

apm-agent-plugins/apm-ecs-logging-plugin/src/main/java/co/elastic/apm/agent/ecs_logging/AbstractLog4j2ServiceInstrumentation.java renamed to apm-agent-plugins/apm-ecs-logging-plugin/src/main/java/co/elastic/apm/agent/ecs_logging/EcsLoggingInstrumentation.java

@@ -18,37 +18,23 @@
  */
 package co.elastic.apm.agent.ecs_logging;
 
-import co.elastic.apm.agent.bci.TracerAwareInstrumentation;
 import co.elastic.apm.agent.bci.bytebuddy.CustomElementMatchers;
-import net.bytebuddy.description.method.MethodDescription;
-import net.bytebuddy.description.type.TypeDescription;
+import co.elastic.apm.agent.loginstr.AbstractLogIntegrationInstrumentation;
 import net.bytebuddy.matcher.ElementMatcher;
 
-import java.util.Arrays;
-import java.util.Collection;
-
-import static net.bytebuddy.matcher.ElementMatchers.named;
 import static net.bytebuddy.matcher.ElementMatchers.not;
 
-public class AbstractLog4j2ServiceInstrumentation extends TracerAwareInstrumentation {
-
-    @Override
-    public ElementMatcher.Junction<ClassLoader> getClassLoaderMatcher() {
-        return not(CustomElementMatchers.isAgentClassLoader());
-    }
+public abstract class EcsLoggingInstrumentation extends AbstractLogIntegrationInstrumentation {
 
     @Override
-    public ElementMatcher.Junction<? super TypeDescription> getTypeMatcher() {
-        return named("co.elastic.logging.log4j2.EcsLayout$Builder");
+    protected String getLoggingInstrumentationGroupName() {
+        // so far all ECS logging instrumentation is for log correlation, at trace and service levels.
+        return LOG_CORRELATION;
     }
 
     @Override
-    public ElementMatcher<? super MethodDescription> getMethodMatcher() {
-        return named("build");
-    }
-
-    @Override
-    public Collection<String> getInstrumentationGroupNames() {
-        return Arrays.asList("logging", "log4j2-ecs");
+    public ElementMatcher.Junction<ClassLoader> getClassLoaderMatcher() {
+        // ECS formatter that is loaded within the agent should not be instrumented
+        return not(CustomElementMatchers.isInternalPluginClassLoader());
     }
 }

apm-agent-plugins/apm-ecs-logging-plugin/src/main/java/co/elastic/apm/agent/ecs_logging/EcsLoggingUtils.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.ecs_logging;
+
+import co.elastic.apm.agent.configuration.CoreConfiguration;
+import co.elastic.apm.agent.configuration.ServiceInfo;
+import co.elastic.apm.agent.impl.ElasticApmTracer;
+import co.elastic.apm.agent.sdk.logging.Logger;
+import co.elastic.apm.agent.sdk.logging.LoggerFactory;
+import co.elastic.apm.agent.sdk.weakconcurrent.WeakConcurrent;
+import co.elastic.apm.agent.sdk.weakconcurrent.WeakSet;
+import co.elastic.apm.agent.tracer.GlobalTracer;
+
+import javax.annotation.Nullable;
+import java.util.Objects;
+
+public class EcsLoggingUtils {
+
+    private static final Logger log = LoggerFactory.getLogger(EcsLoggingUtils.class);
+
+    private static final WeakSet<Object> nameChecked = WeakConcurrent.buildSet();
+    private static final WeakSet<Object> versionChecked = WeakConcurrent.buildSet();
+
+    private static final ElasticApmTracer tracer = GlobalTracer.get().require(ElasticApmTracer.class);
+
+    @Nullable
+    public static String getServiceName() {
+        ServiceInfo serviceInfo = tracer.getServiceInfoForClassLoader(Thread.currentThread().getContextClassLoader());
+        String configuredServiceName = tracer.getConfig(CoreConfiguration.class).getServiceName();
+        return serviceInfo != null ? serviceInfo.getServiceName() : configuredServiceName;
+    }
+
+    @Nullable
+    public static String getServiceVersion() {
+        ServiceInfo serviceInfo = tracer.getServiceInfoForClassLoader(Thread.currentThread().getContextClassLoader());
+        String configuredServiceVersion = tracer.getConfig(CoreConfiguration.class).getServiceVersion();
+        return serviceInfo != null ? serviceInfo.getServiceVersion() : configuredServiceVersion;
+    }
+
+    private static void warnIfMisConfigured(String key, @Nullable String configuredValue, @Nullable String agentValue) {
+        if (!Objects.equals(agentValue, configuredValue)) {
+            log.warn("configuration values differ for '{}': ecs-logging='{}', agent='{}', traces and logs might not correlate properly", key, configuredValue, agentValue);
+        }
+    }
+
+    @Nullable
+    public static String getOrWarnServiceVersion(Object target, @Nullable String value) {
+        if (!versionChecked.add(target)) {
+            return value;
+        }
+        if (value == null) {
+            return getServiceVersion();
+        } else {
+            warnIfMisConfigured("service.version", value, getServiceVersion());
+            return value;
+        }
+    }
+
+    @Nullable
+    public static String getOrWarnServiceName(Object target, @Nullable String value) {
+        if (!nameChecked.add(target)) {
+            return value;
+        }
+        if (value == null) {
+            return getServiceName();
+        } else {
+            warnIfMisConfigured("service.name", value, getServiceName());
+            return value;
+        }
+    }
+}