[exporter/datadog] Add config trace_buffer (#28582)

**Description:**
Add a new traces config `trace_buffer` that specifies the number of
Datadog Agent TracerPayloads to buffer before dropping.

**Link to tracking Issue:**
Fixes #28577

**Testing:** <Describe what testing was performed and which tests were
added.>

**Documentation:** <Describe the documentation added.>

.chloggen/datadog-trace-buffer.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datadogexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Add a new traces config `trace_buffer` that specifies the number of outgoing trace payloads to buffer before dropping."
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [28577]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: "If you start seeing log messages like `Payload in channel full. Dropped 1 payload.` in the datadog exporter, consider setting a higher `trace_buffer` to avoid traces being dropped."
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

exporter/datadogexporter/config.go

@@ -279,6 +279,10 @@ type TracesConfig struct {
 	// If the overhead remains high, it will be due to a high cardinality of `peer.service` values from the traces. You may need to check your instrumentation.
 	PeerServiceAggregation bool `mapstructure:"peer_service_aggregation"`
 
+	// TraceBuffer specifies the number of Datadog Agent TracerPayloads to buffer before dropping.
+	// The default value is 0, meaning the Datadog Agent TracerPayloads are unbuffered.
+	TraceBuffer int `mapstructure:"trace_buffer"`
+
 	// flushInterval defines the interval in seconds at which the writer flushes traces
 	// to the intake; used in tests.
 	flushInterval float64

exporter/datadogexporter/config_test.go

@@ -101,6 +101,13 @@ func TestValidate(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "With trace_buffer",
+			cfg: &Config{
+				API:    APIConfig{Key: "notnull"},
+				Traces: TracesConfig{TraceBuffer: 10},
+			},
+		},
 	}
 	for _, testInstance := range tests {
 		t.Run(testInstance.name, func(t *testing.T) {

exporter/datadogexporter/examples/collector.yaml

@@ -340,6 +340,12 @@ exporters:
       #
       # peer_service_aggregation: true
 
+      ## @param trace_buffer - specifies the number of Datadog Agent TracerPayloads to buffer before dropping.
+      ## If unset, the default value is 0, meaning the Datadog Agent TracerPayloads are unbuffered.
+      ## If you start seeing log messages like `Payload in channel full. Dropped 1 payload.` in the datadog exporter, consider
+      ## setting a higher `trace_buffer` to avoid traces being dropped. 
+      # trace_buffer: 10
+
     ## @param host_metadata - custom object - optional
     ## Host metadata specific configuration.
     ## Host metadata is the information used for populating the infrastructure list, the host map and providing host tags functionality within the Datadog app.

exporter/datadogexporter/factory_test.go

@@ -217,6 +217,7 @@ func TestLoadConfig(t *testing.T) {
 					},
 					SpanNameAsResourceName: true,
 					IgnoreResources:        []string{},
+					TraceBuffer:            10,
 				},
 				Logs: LogsConfig{
 					TCPAddr: confignet.TCPAddr{

exporter/datadogexporter/testdata/config.yaml

@@ -10,6 +10,7 @@ datadog/api:
       "old_name1": "new_name1"
       "old_name2": "new_name2"
     span_name_as_resource_name: true
+    trace_buffer: 10
 
 datadog/api2:
   hostname: customhostname

exporter/datadogexporter/traces_exporter.go

@@ -194,6 +194,9 @@ func newTraceAgent(ctx context.Context, params exporter.CreateSettings, cfg *Con
 	if v := cfg.Traces.flushInterval; v > 0 {
 		acfg.TraceWriter.FlushPeriodSeconds = v
 	}
+	if v := cfg.Traces.TraceBuffer; v > 0 {
+		acfg.TraceBuffer = v
+	}
 	if addr := cfg.Traces.Endpoint; addr != "" {
 		acfg.Endpoints[0].Host = addr
 	}