[APM] Generate stack monitoring data (#118302)

* Generate stack monitoring data * Update file import * Fix imports after bad merge from upstream * A cluster stats generator * Wiring kibana docs to ES docs * Adding fields to get kibana cards rendering * [apm-synthtrace] Export types Fields, ApmException, ApmSynthtraceEsClient * [APM] Update integration tests with synthtrace changes * [APM] Update Cypress E2E tests with synthtrace changes * Fix lint errors Co-authored-by: Milton Hultgren <milton.hultgren@elastic.co> Co-authored-by: Mat Schaffer <mat@elastic.co>
elastic · Nov 29, 2021 · 39a29cc · 39a29cc
1 parent a0650c7
commit 39a29cc
Show file tree

Hide file tree

Showing 72 changed files with 1,022 additions and 679 deletions.
diff --git a/packages/elastic-apm-synthtrace/src/index.ts b/packages/elastic-apm-synthtrace/src/index.ts
@@ -6,17 +6,11 @@
  * Side Public License, v 1.
  */
 
-export type { Exception } from './lib/entity';
-export { service } from './lib/service';
-export { browser } from './lib/browser';
 export { timerange } from './lib/timerange';
-export { getTransactionMetrics } from './lib/utils/get_transaction_metrics';
-export { getSpanDestinationMetrics } from './lib/utils/get_span_destination_metrics';
-export { getObserverDefaults } from './lib/defaults/get_observer_defaults';
-export { getChromeUserAgentDefaults } from './lib/defaults/get_chrome_user_agent_defaults';
-export { toElasticsearchOutput } from './lib/output/to_elasticsearch_output';
-export { getBreakdownMetrics } from './lib/utils/get_breakdown_metrics';
+export { apm } from './lib/apm';
+export { stackMonitoring } from './lib/stack_monitoring';
 export { cleanWriteTargets } from './lib/utils/clean_write_targets';
-export { getWriteTargets } from './lib/utils/get_write_targets';
-export { SynthtraceEsClient } from './lib/client/synthtrace_es_client';
 export { createLogger, LogLevel } from './lib/utils/create_logger';
+
+export type { Fields } from './lib/entity';
+export type { ApmException, ApmSynthtraceEsClient } from './lib/apm';
diff --git a/...astic-apm-synthtrace/src/lib/apm_error.ts → ...c-apm-synthtrace/src/lib/apm/apm_error.ts b/...astic-apm-synthtrace/src/lib/apm_error.ts → ...c-apm-synthtrace/src/lib/apm/apm_error.ts
@@ -6,12 +6,12 @@
  * Side Public License, v 1.
  */
 
-import { Fields } from './entity';
-import { Serializable } from './serializable';
-import { generateLongId, generateShortId } from './utils/generate_id';
+import { ApmFields } from './apm_fields';
+import { Serializable } from '../serializable';
+import { generateLongId, generateShortId } from '../utils/generate_id';
 
-export class ApmError extends Serializable {
-  constructor(fields: Fields) {
+export class ApmError extends Serializable<ApmFields> {
+  constructor(fields: ApmFields) {
     super({
       ...fields,
       'processor.event': 'error',

diff --git a/packages/elastic-apm-synthtrace/src/lib/apm/apm_fields.ts b/packages/elastic-apm-synthtrace/src/lib/apm/apm_fields.ts
@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Fields } from '../entity';
+
+export type ApmApplicationMetricFields = Partial<{
+  'system.process.memory.size': number;
+  'system.memory.actual.free': number;
+  'system.memory.total': number;
+  'system.cpu.total.norm.pct': number;
+  'system.process.memory.rss.bytes': number;
+  'system.process.cpu.total.norm.pct': number;
+}>;
+
+export type ApmUserAgentFields = Partial<{
+  'user_agent.original': string;
+  'user_agent.os.name': string;
+  'user_agent.name': string;
+  'user_agent.device.name': string;
+  'user_agent.version': number;
+}>;
+
+export interface ApmException {
+  message: string;
+}
+
+export type ApmFields = Fields &
+  Partial<{
+    'agent.name': string;
+    'agent.version': string;
+    'container.id': string;
+    'ecs.version': string;
+    'event.outcome': string;
+    'event.ingested': number;
+    'error.id': string;
+    'error.exception': ApmException[];
+    'error.grouping_name': string;
+    'error.grouping_key': string;
+    'host.name': string;
+    'kubernetes.pod.uid': string;
+    'metricset.name': string;
+    'observer.version': string;
+    'observer.version_major': number;
+    'parent.id': string;
+    'processor.event': string;
+    'processor.name': string;
+    'trace.id': string;
+    'transaction.name': string;
+    'transaction.type': string;
+    'transaction.id': string;
+    'transaction.duration.us': number;
+    'transaction.duration.histogram': {
+      values: number[];
+      counts: number[];
+    };
+    'transaction.sampled': true;
+    'service.name': string;
+    'service.environment': string;
+    'service.node.name': string;
+    'span.id': string;
+    'span.name': string;
+    'span.type': string;
+    'span.subtype': string;
+    'span.duration.us': number;
+    'span.destination.service.name': string;
+    'span.destination.service.resource': string;
+    'span.destination.service.type': string;
+    'span.destination.service.response_time.sum.us': number;
+    'span.destination.service.response_time.count': number;
+    'span.self_time.count': number;
+    'span.self_time.sum.us': number;
+  }> &
+  ApmApplicationMetricFields;
diff --git a/...astic-apm-synthtrace/src/lib/base_span.ts → ...c-apm-synthtrace/src/lib/apm/base_span.ts b/...astic-apm-synthtrace/src/lib/base_span.ts → ...c-apm-synthtrace/src/lib/apm/base_span.ts
@@ -6,16 +6,16 @@
  * Side Public License, v 1.
  */
 
-import { Fields } from './entity';
-import { Serializable } from './serializable';
+import { Serializable } from '../serializable';
 import { Span } from './span';
 import { Transaction } from './transaction';
-import { generateLongId } from './utils/generate_id';
+import { generateLongId } from '../utils/generate_id';
+import { ApmFields } from './apm_fields';
 
-export class BaseSpan extends Serializable {
+export class BaseSpan extends Serializable<ApmFields> {
   private readonly _children: BaseSpan[] = [];
 
-  constructor(fields: Fields) {
+  constructor(fields: ApmFields) {
     super({
       ...fields,
       'event.outcome': 'unknown',
@@ -60,7 +60,7 @@ export class BaseSpan extends Serializable {
     return this;
   }
 
-  serialize(): Fields[] {
+  serialize(): ApmFields[] {
     return [this.fields, ...this._children.flatMap((child) => child.serialize())];
   }
 

diff --git a/...elastic-apm-synthtrace/src/lib/browser.ts → ...tic-apm-synthtrace/src/lib/apm/browser.ts b/...elastic-apm-synthtrace/src/lib/browser.ts → ...tic-apm-synthtrace/src/lib/apm/browser.ts
@@ -6,11 +6,12 @@
  * Side Public License, v 1.
  */
 
-import { Entity, UserAgentFields } from './entity';
+import { ApmFields, ApmUserAgentFields } from './apm_fields';
+import { Entity } from '../entity';
 import { RumSpan } from './rum_span';
 import { RumTransaction } from './rum_transaction';
 
-export class Browser extends Entity {
+export class Browser extends Entity<ApmFields> {
   transaction(transactionName: string, transactionType: string = 'page-load') {
     return new RumTransaction({
       ...this.fields,
@@ -29,7 +30,7 @@ export class Browser extends Entity {
   }
 }
 
-export function browser(serviceName: string, production: string, userAgent: UserAgentFields) {
+export function browser(serviceName: string, production: string, userAgent: ApmUserAgentFields) {
   return new Browser({
     'agent.name': 'rum-js',
     'service.name': serviceName,

diff --git a/...ce/src/lib/client/synthtrace_es_client.ts → ...ib/apm/client/apm_synthtrace_es_client.ts b/...ce/src/lib/client/synthtrace_es_client.ts → ...ib/apm/client/apm_synthtrace_es_client.ts
@@ -7,45 +7,52 @@
  */
 
 import { Client } from '@elastic/elasticsearch';
-import { uploadEvents } from '../../scripts/utils/upload_events';
-import { Fields } from '../entity';
-import { cleanWriteTargets } from '../utils/clean_write_targets';
+import { uploadEvents } from '../../../scripts/utils/upload_events';
+import { Fields } from '../../entity';
+import { cleanWriteTargets } from '../../utils/clean_write_targets';
 import { getBreakdownMetrics } from '../utils/get_breakdown_metrics';
 import { getSpanDestinationMetrics } from '../utils/get_span_destination_metrics';
 import { getTransactionMetrics } from '../utils/get_transaction_metrics';
-import { getWriteTargets } from '../utils/get_write_targets';
-import { Logger } from '../utils/logger';
+import { getApmWriteTargets } from '../utils/get_apm_write_targets';
+import { Logger } from '../../utils/create_logger';
+import { apmEventsToElasticsearchOutput } from '../utils/apm_events_to_elasticsearch_output';
 
-export class SynthtraceEsClient {
+export class ApmSynthtraceEsClient {
   constructor(private readonly client: Client, private readonly logger: Logger) {}
 
   private getWriteTargets() {
-    return getWriteTargets({ client: this.client });
+    return getApmWriteTargets({ client: this.client });
   }
 
   clean() {
     return this.getWriteTargets().then((writeTargets) =>
-      cleanWriteTargets({ client: this.client, writeTargets, logger: this.logger })
+      cleanWriteTargets({
+        client: this.client,
+        targets: Object.values(writeTargets),
+        logger: this.logger,
+      })
     );
   }
 
   async index(events: Fields[]) {
-    const eventsToIndex = [
-      ...events,
-      ...getTransactionMetrics(events),
-      ...getSpanDestinationMetrics(events),
-      ...getBreakdownMetrics(events),
-    ];
-
     const writeTargets = await this.getWriteTargets();
 
+    const eventsToIndex = apmEventsToElasticsearchOutput({
+      events: [
+        ...events,
+        ...getTransactionMetrics(events),
+        ...getSpanDestinationMetrics(events),
+        ...getBreakdownMetrics(events),
+      ],
+      writeTargets,
+    });
+
     await uploadEvents({
       batchSize: 1000,
       client: this.client,
       clientWorkers: 5,
       events: eventsToIndex,
       logger: this.logger,
-      writeTargets,
     });
 
     return this.client.indices.refresh({

diff --git a/...efaults/get_chrome_user_agent_defaults.ts → ...efaults/get_chrome_user_agent_defaults.ts b/...efaults/get_chrome_user_agent_defaults.ts → ...efaults/get_chrome_user_agent_defaults.ts
@@ -6,9 +6,9 @@
  * Side Public License, v 1.
  */
 
-import { UserAgentFields } from '../entity';
+import { ApmUserAgentFields } from '../../apm/apm_fields';
 
-export function getChromeUserAgentDefaults(): UserAgentFields {
+export function getChromeUserAgentDefaults(): ApmUserAgentFields {
   return {
     'user_agent.original':
       'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36',

diff --git a/...src/lib/defaults/get_observer_defaults.ts → ...lib/apm/defaults/get_observer_defaults.ts b/...src/lib/defaults/get_observer_defaults.ts → ...lib/apm/defaults/get_observer_defaults.ts
@@ -6,9 +6,9 @@
  * Side Public License, v 1.
  */
 
-import { Fields } from '../entity';
+import { ApmFields } from '../apm_fields';
 
-export function getObserverDefaults(): Fields {
+export function getObserverDefaults(): ApmFields {
   return {
     'observer.version': '7.16.0',
     'observer.version_major': 7,

diff --git a/packages/elastic-apm-synthtrace/src/lib/apm/index.ts b/packages/elastic-apm-synthtrace/src/lib/apm/index.ts
@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import { service } from './service';
+import { browser } from './browser';
+import { getTransactionMetrics } from './utils/get_transaction_metrics';
+import { getSpanDestinationMetrics } from './utils/get_span_destination_metrics';
+import { getObserverDefaults } from './defaults/get_observer_defaults';
+import { getChromeUserAgentDefaults } from './defaults/get_chrome_user_agent_defaults';
+import { apmEventsToElasticsearchOutput } from './utils/apm_events_to_elasticsearch_output';
+import { getBreakdownMetrics } from './utils/get_breakdown_metrics';
+import { getApmWriteTargets } from './utils/get_apm_write_targets';
+import { ApmSynthtraceEsClient } from './client/apm_synthtrace_es_client';
+
+import type { ApmException } from './apm_fields';
+
+export const apm = {
+  service,
+  browser,
+  getTransactionMetrics,
+  getSpanDestinationMetrics,
+  getObserverDefaults,
+  getChromeUserAgentDefaults,
+  apmEventsToElasticsearchOutput,
+  getBreakdownMetrics,
+  getApmWriteTargets,
+  ApmSynthtraceEsClient,
+};
+
+export type { ApmSynthtraceEsClient, ApmException };
diff --git a/...lastic-apm-synthtrace/src/lib/instance.ts → ...ic-apm-synthtrace/src/lib/apm/instance.ts b/...lastic-apm-synthtrace/src/lib/instance.ts → ...ic-apm-synthtrace/src/lib/apm/instance.ts
@@ -7,12 +7,13 @@
  */
 
 import { ApmError } from './apm_error';
-import { ApplicationMetricFields, Entity } from './entity';
+import { Entity } from '../entity';
 import { Metricset } from './metricset';
 import { Span } from './span';
 import { Transaction } from './transaction';
+import { ApmApplicationMetricFields, ApmFields } from './apm_fields';
 
-export class Instance extends Entity {
+export class Instance extends Entity<ApmFields> {
   transaction(transactionName: string, transactionType = 'request') {
     return new Transaction({
       ...this.fields,
@@ -43,7 +44,7 @@ export class Instance extends Entity {
     return this;
   }
 
-  appMetrics(metrics: ApplicationMetricFields) {
+  appMetrics(metrics: ApmApplicationMetricFields) {
     return new Metricset({
       ...this.fields,
       'metricset.name': 'app',

diff --git a/...astic-apm-synthtrace/src/lib/metricset.ts → ...c-apm-synthtrace/src/lib/apm/metricset.ts b/...astic-apm-synthtrace/src/lib/metricset.ts → ...c-apm-synthtrace/src/lib/apm/metricset.ts
@@ -6,11 +6,11 @@
  * Side Public License, v 1.
  */
 
-import { Fields } from './entity';
-import { Serializable } from './serializable';
+import { Serializable } from '../serializable';
+import { ApmFields } from './apm_fields';
 
-export class Metricset extends Serializable {
-  constructor(fields: Fields) {
+export class Metricset extends Serializable<ApmFields> {
+  constructor(fields: ApmFields) {
     super({
       'processor.event': 'metric',
       'processor.name': 'metric',

diff --git a/...lastic-apm-synthtrace/src/lib/rum_span.ts → ...ic-apm-synthtrace/src/lib/apm/rum_span.ts b/...lastic-apm-synthtrace/src/lib/rum_span.ts → ...ic-apm-synthtrace/src/lib/apm/rum_span.ts
diff --git a/...apm-synthtrace/src/lib/rum_transaction.ts → ...synthtrace/src/lib/apm/rum_transaction.ts b/...apm-synthtrace/src/lib/rum_transaction.ts → ...synthtrace/src/lib/apm/rum_transaction.ts
diff --git a/...elastic-apm-synthtrace/src/lib/service.ts → ...tic-apm-synthtrace/src/lib/apm/service.ts b/...elastic-apm-synthtrace/src/lib/service.ts → ...tic-apm-synthtrace/src/lib/apm/service.ts
@@ -6,10 +6,11 @@
  * Side Public License, v 1.
  */
 
-import { Entity } from './entity';
+import { Entity } from '../entity';
+import { ApmFields } from './apm_fields';
 import { Instance } from './instance';
 
-export class Service extends Entity {
+export class Service extends Entity<ApmFields> {
   instance(instanceName: string) {
     return new Instance({
       ...this.fields,

diff --git a/...es/elastic-apm-synthtrace/src/lib/span.ts → ...lastic-apm-synthtrace/src/lib/apm/span.ts b/...es/elastic-apm-synthtrace/src/lib/span.ts → ...lastic-apm-synthtrace/src/lib/apm/span.ts
@@ -7,11 +7,11 @@
  */
 
 import { BaseSpan } from './base_span';
-import { Fields } from './entity';
-import { generateShortId } from './utils/generate_id';
+import { generateShortId } from '../utils/generate_id';
+import { ApmFields } from './apm_fields';
 
 export class Span extends BaseSpan {
-  constructor(fields: Fields) {
+  constructor(fields: ApmFields) {
     super({
       ...fields,
       'processor.event': 'span',

diff --git a/...tic-apm-synthtrace/src/lib/transaction.ts → ...apm-synthtrace/src/lib/apm/transaction.ts b/...tic-apm-synthtrace/src/lib/transaction.ts → ...apm-synthtrace/src/lib/apm/transaction.ts
@@ -8,14 +8,14 @@
 
 import { ApmError } from './apm_error';
 import { BaseSpan } from './base_span';
-import { Fields } from './entity';
-import { generateShortId } from './utils/generate_id';
+import { generateShortId } from '../utils/generate_id';
+import { ApmFields } from './apm_fields';
 
 export class Transaction extends BaseSpan {
   private _sampled: boolean = true;
   private readonly _errors: ApmError[] = [];
 
-  constructor(fields: Fields) {
+  constructor(fields: ApmFields) {
     super({
       ...fields,
       'processor.event': 'transaction',