Moved implementation of code.stacktrace into separate processor (#144)

Co-authored-by: SylvainJuge <763082+SylvainJuge@users.noreply.github.com>

buildSrc/src/main/kotlin/elastic-otel.agent-packaging-conventions.gradle.kts

@@ -26,7 +26,6 @@ dependencies {
   bootstrapLibs(project(":bootstrap"))
 
   javaagentLibs(project(":custom"))
-  javaagentLibs(project(":resources"))
 }
 
 fun relocatePackages( shadowJar : ShadowJar) {

common/src/main/java/co/elastic/otel/common/ChainingSpanProcessorsInstaller.java

@@ -36,7 +36,9 @@ public class ChainingSpanProcessorsInstaller implements AutoConfigurationCustomi
   @Override
   public void customize(AutoConfigurationCustomizer autoConfigurationCustomizer) {
     List<ChainingSpanProcessorAutoConfiguration> autoConfigs = new ArrayList<>();
-    ServiceLoader.load(ChainingSpanProcessorAutoConfiguration.class)
+    ServiceLoader.load(
+            ChainingSpanProcessorAutoConfiguration.class,
+            ChainingSpanProcessorsInstaller.class.getClassLoader())
         .iterator()
         .forEachRemaining(autoConfigs::add);
     if (!autoConfigs.isEmpty()) {

custom/build.gradle.kts

@@ -6,7 +6,7 @@ dependencies {
   implementation(project(":common"))
   implementation(project(":inferred-spans"))
   compileOnly(project(":bootstrap"))
-  compileOnly(project(":resources"))
+  implementation(project(":resources"))
   compileOnly("io.opentelemetry:opentelemetry-sdk")
   compileOnly("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi")
   compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-extension-api")
@@ -21,6 +21,7 @@ dependencies {
   compileOnly("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure")
 
   // test dependencies
+  testImplementation(project(":testing-common"))
   testImplementation("io.opentelemetry:opentelemetry-sdk")
   testImplementation("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi")
   testImplementation("io.opentelemetry.javaagent:opentelemetry-javaagent-tooling")

custom/src/main/java/co/elastic/otel/ElasticSpanProcessor.java

@@ -18,14 +18,11 @@
  */
 package co.elastic.otel;
 
-import co.elastic.otel.common.ElasticAttributes;
 import io.opentelemetry.context.Context;
 import io.opentelemetry.sdk.common.CompletableResultCode;
 import io.opentelemetry.sdk.trace.ReadWriteSpan;
 import io.opentelemetry.sdk.trace.ReadableSpan;
 import io.opentelemetry.sdk.trace.SpanProcessor;
-import java.io.PrintWriter;
-import java.io.StringWriter;
 
 public class ElasticSpanProcessor implements SpanProcessor {
 
@@ -49,8 +46,6 @@ public boolean isStartRequired() {
   @Override
   public void onEnd(ReadableSpan span) {
     breakdownMetrics.onSpanEnd(span);
-
-    captureStackTrace(span);
   }
 
   @Override
@@ -66,24 +61,4 @@ public CompletableResultCode shutdown() {
   public void registerSpanExporter(ElasticSpanExporter spanExporter) {
     this.spanExporter = spanExporter;
   }
-
-  private void captureStackTrace(ReadableSpan span) {
-    if (spanExporter == null) {
-      return;
-    }
-
-    // do not overwrite stacktrace if present
-    if (span.getAttribute(ElasticAttributes.SPAN_STACKTRACE) == null) {
-      Throwable exception = new Throwable();
-      StringWriter stringWriter = new StringWriter();
-      try (PrintWriter printWriter = new PrintWriter(stringWriter)) {
-        exception.printStackTrace(printWriter);
-      }
-
-      // TODO should we filter-out the calling code that is within the agent: at least onEnd +
-      // captureStackTrace will be included here
-      spanExporter.addAttribute(
-          span.getSpanContext(), ElasticAttributes.SPAN_STACKTRACE, stringWriter.toString());
-    }
-  }
 }

custom/src/main/java/co/elastic/otel/SpanStackTraceProcessor.java

@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.otel;
+
+import co.elastic.otel.common.AbstractSimpleChainingSpanProcessor;
+import co.elastic.otel.common.ElasticAttributes;
+import co.elastic.otel.common.MutableSpan;
+import io.opentelemetry.sdk.trace.ReadableSpan;
+import io.opentelemetry.sdk.trace.SpanProcessor;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class SpanStackTraceProcessor extends AbstractSimpleChainingSpanProcessor {
+
+  private static final Logger logger = Logger.getLogger(SpanStackTraceProcessor.class.getName());
+
+  private final long minSpanDurationNanos;
+
+  public SpanStackTraceProcessor(SpanProcessor next, long minSpanDurationNanos) {
+    super(next);
+    this.minSpanDurationNanos = minSpanDurationNanos;
+    logger.log(
+        Level.FINE,
+        "Stack traces will be added to spans with a minimum duration of {0} nanos",
+        minSpanDurationNanos);
+  }
+
+  @Override
+  protected boolean requiresStart() {
+    return false;
+  }
+
+  @Override
+  protected boolean requiresEnd() {
+    return true;
+  }
+
+  @Override
+  protected ReadableSpan doOnEnd(ReadableSpan span) {
+    if (span.getLatencyNanos() < minSpanDurationNanos) {
+      return span;
+    }
+    if (span.getAttribute(ElasticAttributes.SPAN_STACKTRACE) != null) {
+      // Span already has a stacktrace, do not override
+      return span;
+    }
+    Boolean isInferred = span.getAttribute(ElasticAttributes.IS_INFERRED);
+    if (isInferred != null && isInferred) {
+      // Do not add a stacktrace for inferred spans: If a good one was available
+      // it would have been added by the module creating this span
+      return span;
+    }
+    MutableSpan mutableSpan = MutableSpan.makeMutable(span);
+
+    String stacktrace = generateSpanEndStacktrace();
+    mutableSpan.setAttribute(ElasticAttributes.SPAN_STACKTRACE, stacktrace);
+    return mutableSpan;
+  }
+
+  private static String generateSpanEndStacktrace() {
+    Throwable exception = new Throwable();
+    StringWriter stringWriter = new StringWriter();
+    try (PrintWriter printWriter = new PrintWriter(stringWriter)) {
+      exception.printStackTrace(printWriter);
+    }
+    return removeInternalFrames(stringWriter.toString());
+  }
+
+  private static String removeInternalFrames(String stackTrace) {
+    String lastInternal = "at io.opentelemetry.sdk.trace.SdkSpan.end";
+
+    int idx = stackTrace.lastIndexOf(lastInternal);
+    if (idx == -1) {
+      // should usually not happen, this means that the span processor was called from somewhere
+      // else
+      return stackTrace;
+    }
+    int nextNewLine = stackTrace.indexOf('\n', idx);
+    if (nextNewLine == -1) {
+      nextNewLine = stackTrace.length() - 1;
+    }
+    return stackTrace.substring(nextNewLine + 1);
+  }
+}

custom/src/main/java/co/elastic/otel/SpanStackTraceProcessorAutoConfig.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.otel;
+
+import co.elastic.otel.common.ChainingSpanProcessorAutoConfiguration;
+import co.elastic.otel.common.ChainingSpanProcessorRegisterer;
+import com.google.auto.service.AutoService;
+import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
+import java.time.Duration;
+
+@AutoService(ChainingSpanProcessorAutoConfiguration.class)
+public class SpanStackTraceProcessorAutoConfig implements ChainingSpanProcessorAutoConfiguration {
+
+  static final String MIN_DURATION_CONFIG_OPTION = "elastic.otel.span.stack.trace.min.duration";
+
+  @Override
+  public void registerSpanProcessors(
+      ConfigProperties properties, ChainingSpanProcessorRegisterer registerer) {
+
+    Duration minDuration = properties.getDuration(MIN_DURATION_CONFIG_OPTION, Duration.ofMillis(5));
+    registerer.register(
+        next -> new SpanStackTraceProcessor(next, minDuration.toNanos()),
+        ChainingSpanProcessorRegisterer.ORDER_FIRST);
+  }
+}