Use sampling on timeline events (#4861)

integration-tests/profiler/profiler.spec.js

@@ -105,10 +105,9 @@ async function gatherNetworkTimelineEvents (cwd, scriptFilePath, eventType, args
   const proc = fork(path.join(cwd, scriptFilePath), args, {
     cwd,
     env: {
-      DD_PROFILING_PROFILERS: 'wall',
       DD_PROFILING_EXPORTERS: 'file',
       DD_PROFILING_ENABLED: 1,
-      DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED: 1
+      DD_INTERNAL_PROFILING_TIMELINE_SAMPLING_ENABLED: 0 // capture all events
     }
   })
 
@@ -205,12 +204,8 @@ describe('profiler', () => {
       const proc = fork(path.join(cwd, 'profiler/codehotspots.js'), {
         cwd,
         env: {
-          DD_PROFILING_PROFILERS: 'wall',
           DD_PROFILING_EXPORTERS: 'file',
-          DD_PROFILING_ENABLED: 1,
-          DD_PROFILING_CODEHOTSPOTS_ENABLED: 1,
-          DD_PROFILING_ENDPOINT_COLLECTION_ENABLED: 1,
-          DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED: 1
+          DD_PROFILING_ENABLED: 1
         }
       })
 

packages/dd-trace/src/profiling/config.js

@@ -21,6 +21,7 @@ class Config {
     const {
       DD_AGENT_HOST,
       DD_ENV,
+      DD_INTERNAL_PROFILING_TIMELINE_SAMPLING_ENABLED, // used for testing
       DD_PROFILING_CODEHOTSPOTS_ENABLED,
       DD_PROFILING_CPU_ENABLED,
       DD_PROFILING_DEBUG_SOURCE_MAPS,
@@ -175,6 +176,8 @@ class Config {
       DD_PROFILING_EXPERIMENTAL_TIMELINE_ENABLED, samplingContextsAvailable))
     logExperimentalVarDeprecation('TIMELINE_ENABLED')
     checkOptionWithSamplingContextAllowed(this.timelineEnabled, 'Timeline view')
+    this.timelineSamplingEnabled = isTrue(coalesce(options.timelineSamplingEnabled,
+      DD_INTERNAL_PROFILING_TIMELINE_SAMPLING_ENABLED, true))
 
     this.codeHotspotsEnabled = isTrue(coalesce(options.codeHotspotsEnabled,
       DD_PROFILING_CODEHOTSPOTS_ENABLED,

packages/dd-trace/src/profiling/profilers/event_plugins/event.js

@@ -5,9 +5,10 @@ const { performance } = require('perf_hooks')
 // We are leveraging the TracingPlugin class for its functionality to bind
 // start/error/finish methods to the appropriate diagnostic channels.
 class EventPlugin extends TracingPlugin {
-  constructor (eventHandler) {
+  constructor (eventHandler, eventFilter) {
     super()
     this.eventHandler = eventHandler
+    this.eventFilter = eventFilter
     this.store = storage('profiling')
     this.entryType = this.constructor.entryType
   }
@@ -30,17 +31,20 @@ class EventPlugin extends TracingPlugin {
     }
     const duration = performance.now() - startTime
 
-    const context = this.activeSpan?.context()
-    const _ddSpanId = context?.toSpanId()
-    const _ddRootSpanId = context?._trace.started[0]?.context().toSpanId() || _ddSpanId
-
     const event = {
       entryType: this.entryType,
       startTime,
-      duration,
-      _ddSpanId,
-      _ddRootSpanId
+      duration
+    }
+
+    if (!this.eventFilter(event)) {
+      return
     }
+
+    const context = this.activeSpan?.context()
+    event._ddSpanId = context?.toSpanId()
+    event._ddRootSpanId = context?._trace.started[0]?.context().toSpanId() || event._ddSpanId
+
     this.eventHandler(this.extendEvent(event, startEvent))
   }
 }

packages/dd-trace/src/profiling/profilers/events.js

@@ -254,10 +254,10 @@ class NodeApiEventSource {
 }
 
 class DatadogInstrumentationEventSource {
-  constructor (eventHandler) {
+  constructor (eventHandler, eventFilter) {
     this.plugins = ['dns_lookup', 'dns_lookupservice', 'dns_resolve', 'dns_reverse', 'net'].map(m => {
       const Plugin = require(`./event_plugins/${m}`)
-      return new Plugin(eventHandler)
+      return new Plugin(eventHandler, eventFilter)
     })
 
     this.started = false
@@ -292,29 +292,68 @@ class CompositeEventSource {
   }
 }
 
+function createPossionProcessSamplingFilter (samplingIntervalMillis) {
+  let nextSamplingInstant = performance.now()
+  let currentSamplingInstant = 0
+  setNextSamplingInstant()
+
+  return event => {
+    const endTime = event.startTime + event.duration
+    while (endTime >= nextSamplingInstant) {
+      setNextSamplingInstant()
+    }
+    // An event is sampled if it started before, and ended on or after a sampling instant. The above
+    // while loop will ensure that the ending invariant is always true for the current sampling
+    // instant so we don't have to test for it below. Across calls, the invariant also holds as long
+    // as the events arrive in endTime order. This is true for events coming from
+    // DatadogInstrumentationEventSource; they will be ordered by endTime by virtue of this method
+    // being invoked synchronously with the plugins' finish() handler which evaluates
+    // performance.now(). OTOH, events coming from NodeAPIEventSource (GC in typical setup) might be
+    // somewhat delayed as they are queued by Node, so they can arrive out of order with regard to
+    // events coming from the non-queued source. By omitting the endTime check, we will pass through
+    // some short events that started and ended before the current sampling instant. OTOH, if we
+    // were to check for this.currentSamplingInstant <= endTime, we would discard some long events
+    // that also ended before the current sampling instant. We'd rather err on the side of including
+    // some short events than excluding some long events.
+    return event.startTime < currentSamplingInstant
+  }
+
+  function setNextSamplingInstant () {
+    currentSamplingInstant = nextSamplingInstant
+    nextSamplingInstant -= Math.log(1 - Math.random()) * samplingIntervalMillis
+  }
+}
+
 /**
  * This class generates pprof files with timeline events. It combines an event
- * source with an event serializer.
+ * source with a sampling event filter and an event serializer.
  */
 class EventsProfiler {
   constructor (options = {}) {
     this.type = 'events'
     this.eventSerializer = new EventSerializer()
 
-    const eventHandler = event => {
-      this.eventSerializer.addEvent(event)
+    const eventHandler = event => this.eventSerializer.addEvent(event)
+    const eventFilter = options.timelineSamplingEnabled
+      // options.samplingInterval comes in microseconds, we need millis
+      ? createPossionProcessSamplingFilter((options.samplingInterval ?? 1e6 / 99) / 1000)
+      : _ => true
+    const filteringEventHandler = event => {
+      if (eventFilter(event)) {
+        eventHandler(event)
+      }
     }
 
     if (options.codeHotspotsEnabled) {
       // Use Datadog instrumentation to collect events with span IDs. Still use
       // Node API for GC events.
       this.eventSource = new CompositeEventSource([
-        new DatadogInstrumentationEventSource(eventHandler),
-        new NodeApiEventSource(eventHandler, ['gc'])
+        new DatadogInstrumentationEventSource(eventHandler, eventFilter),
+        new NodeApiEventSource(filteringEventHandler, ['gc'])
       ])
     } else {
       // Use Node API instrumentation to collect events without span IDs
-      this.eventSource = new NodeApiEventSource(eventHandler)
+      this.eventSource = new NodeApiEventSource(filteringEventHandler)
     }
   }