Merge pull request #3718 from DataDog/anmarchenko/telemetry_send_even…

…ts_in_background_thread

[SDTEST-409] Send telemetry events in background thread to remove blocking HTTP calls from critical path

Rakefile

@@ -67,13 +67,13 @@ TEST_METADATA = {
     'elasticsearch-8' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
   },
   'ethon' => {
-    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
+    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby'
   },
   'excon' => {
-    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
+    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby'
   },
   'faraday' => {
-    'http'        => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby',
+    'http'        => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby',
     'contrib-old' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ❌ 3.0 / ❌ 3.1 / ❌ 3.2 / ❌ 3.3 / ❌ 3.4 / ✅ jruby'
   },
   'grape' => {
@@ -95,13 +95,13 @@ TEST_METADATA = {
     'contrib' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby'
   },
   'http' => {
-    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
+    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby'
   },
   'httpclient' => {
-    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
+    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby'
   },
   'httprb' => {
-    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
+    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby'
   },
   'kafka' => {
     'activesupport' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
@@ -146,7 +146,7 @@ TEST_METADATA = {
     'resque2-redis4' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
   },
   'rest_client' => {
-    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
+    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby'
   },
   'roda' => {
     'contrib' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
@@ -167,7 +167,7 @@ TEST_METADATA = {
     'contrib' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
   },
   'stripe' => {
-    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'
+    'http' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby'
   },
   'sucker_punch' => {
     'contrib' => '✅ 2.5 / ✅ 2.6 / ✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby'

lib/datadog/core/configuration.rb

@@ -84,23 +84,16 @@ def configure
         configuration = self.configuration
         yield(configuration)
 
-        built_components = false
-
-        components = safely_synchronize do |write_components|
+        safely_synchronize do |write_components|
           write_components.call(
             if components?
               replace_components!(configuration, @components)
             else
-              components = build_components(configuration)
-              built_components = true
-              components
+              build_components(configuration)
             end
           )
         end
 
-        # Should only be called the first time components are built
-        components.telemetry.started! if built_components
-
         configuration
       end
 
@@ -200,20 +193,13 @@ def components(allow_initialization: true)
         current_components = COMPONENTS_READ_LOCK.synchronize { defined?(@components) && @components }
         return current_components if current_components || !allow_initialization
 
-        built_components = false
-
-        components = safely_synchronize do |write_components|
+        safely_synchronize do |write_components|
           if defined?(@components) && @components
             @components
           else
-            built_components = true
             write_components.call(build_components(configuration))
           end
         end
-
-        # Should only be called the first time components are built
-        components&.telemetry&.started! if built_components
-        components
       end
 
       private

lib/datadog/core/configuration/components.rb

@@ -6,7 +6,7 @@
 require_relative '../diagnostics/health'
 require_relative '../logger'
 require_relative '../runtime/metrics'
-require_relative '../telemetry/client'
+require_relative '../telemetry/component'
 require_relative '../workers/runtime_metrics'
 
 require_relative '../remote/component'
@@ -62,7 +62,7 @@ def build_telemetry(settings, agent_settings, logger)
               logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
             end
 
-            Telemetry::Client.new(
+            Telemetry::Component.new(
               enabled: enabled,
               heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
               dependency_collection: settings.telemetry.dependency_collection
@@ -169,8 +169,9 @@ def shutdown!(replacement = nil)
           unused_statsd = (old_statsd - (old_statsd & new_statsd))
           unused_statsd.each(&:close)
 
-          telemetry.stop!
+          # enqueue closing event before stopping telemetry so it will be send out on shutdown
           telemetry.emit_closing! unless replacement
+          telemetry.stop!
         end
       end
     end

lib/datadog/core/telemetry/component.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative 'emitter'
+require_relative 'event'
+require_relative 'worker'
+require_relative '../utils/forking'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
+      class Component
+        attr_reader :enabled
+
+        include Core::Utils::Forking
+
+        # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
+        # @param heartbeat_interval_seconds [Float] How frequently heartbeats will be reported, in seconds.
+        # @param [Boolean] dependency_collection Whether to send the `app-dependencies-loaded` event
+        def initialize(heartbeat_interval_seconds:, dependency_collection:, enabled: true)
+          @enabled = enabled
+          @stopped = false
+
+          @worker = Telemetry::Worker.new(
+            enabled: @enabled,
+            heartbeat_interval_seconds: heartbeat_interval_seconds,
+            emitter: Emitter.new,
+            dependency_collection: dependency_collection
+          )
+          @worker.start
+        end
+
+        def disable!
+          @enabled = false
+          @worker.enabled = false
+        end
+
+        def stop!
+          return if @stopped
+
+          @worker.stop(true)
+          @stopped = true
+        end
+
+        def emit_closing!
+          return if !@enabled || forked?
+
+          @worker.enqueue(Event::AppClosing.new)
+        end
+
+        def integrations_change!
+          return if !@enabled || forked?
+
+          @worker.enqueue(Event::AppIntegrationsChange.new)
+        end
+
+        # Report configuration changes caused by Remote Configuration.
+        def client_configuration_change!(changes)
+          return if !@enabled || forked?
+
+          @worker.enqueue(Event::AppClientConfigurationChange.new(changes, 'remote_config'))
+        end
+      end
+    end
+  end
+end

lib/datadog/core/telemetry/http/adapters/net.rb

@@ -15,7 +15,7 @@ class Net
               :timeout,
               :ssl
 
-            DEFAULT_TIMEOUT = 30
+            DEFAULT_TIMEOUT = 2
 
             def initialize(hostname:, port: nil, timeout: DEFAULT_TIMEOUT, ssl: true)
               @hostname = hostname