Include tracing information in the Kibana logs

.buildkite/scripts/common/env.sh

@@ -38,8 +38,10 @@ export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1
 if is_pr; then
   if [[ "${GITHUB_PR_LABELS:-}" == *"ci:collect-apm"* ]]; then
     export ELASTIC_APM_ACTIVE=true
+    export ELASTIC_APM_DISABLE_SEND=false
   else
-    export ELASTIC_APM_ACTIVE=false
+    export ELASTIC_APM_ACTIVE=true
+    export ELASTIC_APM_DISABLE_SEND=true
   fi
 
   export CHECKS_REPORTER_ACTIVE=true
@@ -57,6 +59,7 @@ if is_pr; then
   export PR_TARGET_BRANCH="$GITHUB_PR_TARGET_BRANCH"
 else
   export ELASTIC_APM_ACTIVE=true
+  export ELASTIC_APM_DISABLE_SEND=false
   export CHECKS_REPORTER_ACTIVE=false
 fi
 

packages/kbn-apm-config-loader/src/config.test.ts

@@ -21,7 +21,7 @@ describe('ApmConfiguration', () => {
   beforeEach(() => {
     // start with an empty env to avoid CI from spoiling snapshots, env is unique for each jest file
     process.env = {};
-
+    devConfigMock.raw = {};
     packageMock.raw = {
       version: '8.0.0',
       build: {
@@ -86,10 +86,11 @@ describe('ApmConfiguration', () => {
     let config = new ApmConfiguration(mockedRootDir, {}, false);
     expect(config.getConfig('serviceName')).toMatchInlineSnapshot(`
       Object {
-        "active": false,
+        "active": true,
         "breakdownMetrics": true,
         "captureSpanStackTraces": false,
         "centralConfig": false,
+        "disableSend": true,
         "environment": "development",
         "globalLabels": Object {},
         "logUncaughtExceptions": true,
@@ -105,12 +106,13 @@ describe('ApmConfiguration', () => {
     config = new ApmConfiguration(mockedRootDir, {}, true);
     expect(config.getConfig('serviceName')).toMatchInlineSnapshot(`
       Object {
-        "active": false,
+        "active": true,
         "breakdownMetrics": false,
         "captureBody": "off",
         "captureHeaders": false,
         "captureSpanStackTraces": false,
         "centralConfig": false,
+        "disableSend": true,
         "environment": "development",
         "globalLabels": Object {
           "git_rev": "sha",
@@ -162,13 +164,12 @@ describe('ApmConfiguration', () => {
 
   it('does not load the configuration from the dev config in distributable', () => {
     devConfigMock.raw = {
-      active: true,
-      serverUrl: 'https://dev-url.co',
+      active: false,
     };
     const config = new ApmConfiguration(mockedRootDir, {}, true);
     expect(config.getConfig('serviceName')).toEqual(
       expect.objectContaining({
-        active: false,
+        active: true,
       })
     );
   });
@@ -224,4 +225,107 @@ describe('ApmConfiguration', () => {
       })
     );
   });
+
+  describe('disableSend', () => {
+    it('sets "active: true" and "disableSend: true" by default', () => {
+      expect(new ApmConfiguration(mockedRootDir, {}, false).getConfig('serviceName')).toEqual(
+        expect.objectContaining({
+          active: true,
+          disableSend: true,
+        })
+      );
+
+      expect(new ApmConfiguration(mockedRootDir, {}, true).getConfig('serviceName')).toEqual(
+        expect.objectContaining({
+          active: true,
+          disableSend: true,
+        })
+      );
+    });
+
+    it('value from config overrides the default', () => {
+      const kibanaConfig = {
+        elastic: {
+          apm: {
+            active: false,
+            disableSend: false,
+          },
+        },
+      };
+
+      expect(
+        new ApmConfiguration(mockedRootDir, kibanaConfig, false).getConfig('serviceName')
+      ).toEqual(
+        expect.objectContaining({
+          active: false,
+          disableSend: false,
+        })
+      );
+
+      expect(
+        new ApmConfiguration(mockedRootDir, kibanaConfig, true).getConfig('serviceName')
+      ).toEqual(
+        expect.objectContaining({
+          active: false,
+          disableSend: false,
+        })
+      );
+    });
+
+    it('is "false" if "active:true" and "disableSend" is not specified', () => {
+      const kibanaConfig = {
+        elastic: {
+          apm: {
+            active: true,
+          },
+        },
+      };
+
+      expect(
+        new ApmConfiguration(mockedRootDir, kibanaConfig, false).getConfig('serviceName')
+      ).toEqual(
+        expect.objectContaining({
+          active: true,
+          disableSend: false,
+        })
+      );
+
+      expect(
+        new ApmConfiguration(mockedRootDir, kibanaConfig, true).getConfig('serviceName')
+      ).toEqual(
+        expect.objectContaining({
+          active: true,
+          disableSend: false,
+        })
+      );
+    });
+
+    it('is "true" if "active:false" and "disableSend" is not specified', () => {
+      const kibanaConfig = {
+        elastic: {
+          apm: {
+            active: false,
+          },
+        },
+      };
+
+      expect(
+        new ApmConfiguration(mockedRootDir, kibanaConfig, false).getConfig('serviceName')
+      ).toEqual(
+        expect.objectContaining({
+          active: false,
+          disableSend: true,
+        })
+      );
+
+      expect(
+        new ApmConfiguration(mockedRootDir, kibanaConfig, true).getConfig('serviceName')
+      ).toEqual(
+        expect.objectContaining({
+          active: false,
+          disableSend: true,
+        })
+      );
+    });
+  });
 });

packages/kbn-apm-config-loader/src/config.ts

@@ -16,7 +16,8 @@ import type { AgentConfigOptions } from 'elastic-apm-node';
 
 // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html
 const DEFAULT_CONFIG: AgentConfigOptions = {
-  active: false,
+  active: true,
+  disableSend: true,
   environment: 'development',
   logUncaughtExceptions: true,
   globalLabels: {},
@@ -112,6 +113,13 @@ export class ApmConfiguration {
 
     if (process.env.ELASTIC_APM_ACTIVE === 'true') {
       config.active = true;
+      config.disableSend = false;
+    }
+
+    if (process.env.ELASTIC_APM_DISABLE_SEND === 'true') {
+      config.disableSend = true;
+    } else if (process.env.ELASTIC_APM_DISABLE_SEND === 'false') {
+      config.disableSend = false;
     }
 
     if (process.env.ELASTIC_APM_ENVIRONMENT || process.env.NODE_ENV) {
@@ -143,7 +151,12 @@ export class ApmConfiguration {
    * default config.
    */
   private getConfigFromKibanaConfig(): AgentConfigOptions {
-    return this.rawKibanaConfig?.elastic?.apm ?? {};
+    const config: AgentConfigOptions = this.rawKibanaConfig?.elastic?.apm ?? {};
+
+    return {
+      ...config,
+      disableSend: config.disableSend ?? (config.active === true ? false : true),
+    };
   }
 
   /**

packages/kbn-logging/src/log_record.ts

@@ -20,4 +20,7 @@ export interface LogRecord {
   error?: Error;
   meta?: { [name: string]: any };
   pid: number;
+  spanId?: string;
+  traceId?: string;
+  transactionId?: string;
 }

src/core/server/logging/layouts/json_layout.test.ts

@@ -58,6 +58,16 @@ const records: LogRecord[] = [
     timestamp,
     pid: 5355,
   },
+  {
+    context: 'context-7',
+    level: LogLevel.Trace,
+    message: 'message-6',
+    timestamp,
+    pid: 5355,
+    spanId: 'spanId-1',
+    traceId: 'traceId-1',
+    transactionId: 'transactionId-1',
+  },
 ];
 
 test('`createConfigSchema()` creates correct schema.', () => {
@@ -310,3 +320,40 @@ test('format() meta can not override timestamp', () => {
     },
   });
 });
+
+test('format() meta can not override tracing properties', () => {
+  const layout = new JsonLayout();
+  expect(
+    JSON.parse(
+      layout.format({
+        message: 'foo',
+        timestamp,
+        level: LogLevel.Debug,
+        context: 'bar',
+        pid: 3,
+        meta: {
+          span: 'span_override',
+          trace: 'trace_override',
+          transaction: 'transaction_override',
+        },
+        spanId: 'spanId-1',
+        traceId: 'traceId-1',
+        transactionId: 'transactionId-1',
+      })
+    )
+  ).toStrictEqual({
+    ecs: { version: expect.any(String) },
+    '@timestamp': '2012-02-01T09:30:22.011-05:00',
+    message: 'foo',
+    log: {
+      level: 'DEBUG',
+      logger: 'bar',
+    },
+    process: {
+      pid: 3,
+    },
+    span: { id: 'spanId-1' },
+    trace: { id: 'traceId-1' },
+    transaction: { id: 'transactionId-1' },
+  });
+});

src/core/server/logging/layouts/json_layout.ts

@@ -54,6 +54,9 @@ export class JsonLayout implements Layout {
       process: {
         pid: record.pid,
       },
+      span: record.spanId ? { id: record.spanId } : undefined,
+      trace: record.traceId ? { id: record.traceId } : undefined,
+      transaction: record.transactionId ? { id: record.transactionId } : undefined,
     };
     const output = record.meta ? merge({ ...record.meta }, log) : log;
 

src/core/server/logging/logger.ts

@@ -5,7 +5,7 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-
+import apmAgent from 'elastic-apm-node';
 import { Appender, LogLevel, LogRecord, LoggerFactory, LogMeta, Logger } from '@kbn/logging';
 
 function isError(x: any): x is Error {
@@ -73,6 +73,7 @@ export class BaseLogger implements Logger {
         meta,
         timestamp: new Date(),
         pid: process.pid,
+        ...this.getTraceIds(),
       };
     }
 
@@ -83,6 +84,15 @@ export class BaseLogger implements Logger {
       meta,
       timestamp: new Date(),
       pid: process.pid,
+      ...this.getTraceIds(),
+    };
+  }
+
+  private getTraceIds() {
+    return {
+      spanId: apmAgent.currentTraceIds['span.id'],
+      traceId: apmAgent.currentTraceIds['trace.id'],
+      transactionId: apmAgent.currentTraceIds['transaction.id'],
     };
   }
 }

vars/kibanaPipeline.groovy

@@ -93,7 +93,7 @@ def withFunctionalTestEnv(List additionalEnvs = [], Closure closure) {
   def corsTestServerPort = "64${parallelId}3"
   // needed for https://github.com/elastic/kibana/issues/107246
   def proxyTestServerPort = "64${parallelId}4"
-  def apmActive = githubPr.isPr() ? "false" : "true"
+  def apmDisableSend = githubPr.isPr() ? "true" : "false"
 
   withEnv([
     "CI_GROUP=${parallelId}",
@@ -109,7 +109,8 @@ def withFunctionalTestEnv(List additionalEnvs = [], Closure closure) {
     "KBN_NP_PLUGINS_BUILT=true",
     "FLEET_PACKAGE_REGISTRY_PORT=${fleetPackageRegistryPort}",
     "ALERTING_PROXY_PORT=${alertingProxyPort}",
-    "ELASTIC_APM_ACTIVE=${apmActive}",
+    "ELASTIC_APM_ACTIVE=true",
+    "ELASTIC_APM_DISABLE_SEND=${apmDisableSend}",
     "ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1",
   ] + additionalEnvs) {
     closure()