[APM] Implementing Journey for APM

.buildkite/ftr_configs.yml

@@ -412,5 +412,6 @@ enabled:
   - x-pack/performance/journeys/ecommerce_dashboard_tsvb_gauge_only.ts
   - x-pack/performance/journeys/dashboard_listing_page.ts
   - x-pack/performance/journeys/cloud_security_dashboard.ts
+  - x-pack/performance/journeys/apm_service_inventory.ts
   - x-pack/test/custom_branding/config.ts
   - x-pack/test/profiling_api_integration/cloud/config.ts

x-pack/performance/configs/apm_config.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { FtrConfigProviderContext } from '@kbn/test';
+import { CA_CERT_PATH } from '@kbn/dev-utils';
+
+// eslint-disable-next-line import/no-default-export
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const xpackFunctionalConfig = await readConfigFile(
+    // eslint-disable-next-line @kbn/imports/no_boundary_crossing
+    require.resolve('../../test/functional/config.base.js')
+  );
+
+  return {
+    ...xpackFunctionalConfig.getAll(),
+
+    esTestCluster: {
+      ...xpackFunctionalConfig.get('esTestCluster'),
+      serverArgs: [
+        ...xpackFunctionalConfig.get('esTestCluster.serverArgs'),
+        // define custom es server here
+        // API Keys is enabled at the top level
+        'xpack.security.enabled=true',
+      ],
+    },
+
+    kbnTestServer: {
+      ...xpackFunctionalConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'),
+        '--home.disableWelcomeScreen=true',
+        '--csp.strict=false',
+        '--csp.warnLegacyBrowsers=false',
+        // define custom kibana server args here
+        `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`,
+        '--http.eluMonitor.enabled=true',
+        '--http.eluMonitor.logging.enabled=true',
+        '--http.eluMonitor.logging.threshold.ela=250',
+        '--http.eluMonitor.logging.threshold.elu=0.15',
+      ],
+    },
+  };
+}

x-pack/performance/journeys/apm_service_inventory.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Journey } from '@kbn/journeys';
+import { SynthtraceClient } from '../services/synthtrace';
+import { generateData } from '../synthtrace_data_generators/apm_data';
+
+export const journey = new Journey({
+  beforeSteps: async ({ kbnUrl, log, auth, es }) => {
+    // Install APM Package
+    const synthClient = new SynthtraceClient({
+      kbnBaseUrl: kbnUrl.get(),
+      logger: log,
+      username: auth.getUsername(),
+      password: auth.getPassword(),
+      esClient: es,
+    });
+
+    await synthClient.installApmPackage();
+    // Setup Synthtrace Client
+    await synthClient.initialiseEsClient();
+    // Generate data using Synthtrace
+    const start = Date.now() - 1000;
+    const end = Date.now();
+    await synthClient.index(
+      generateData({
+        from: new Date(start).getTime(),
+        to: new Date(end).getTime(),
+      })
+    );
+  },
+})
+  .step('Navigate to Service Inventory Page', async ({ page, kbnUrl }) => {
+    await page.goto(kbnUrl.get(`app/apm/services`));
+    await page.waitForSelector(`[data-test-subj="serviceLink_nodejs"]`);
+  })
+  .step('Navigate to Service Overview Page', async ({ page, kbnUrl }) => {
+    await page.click(`[data-test-subj="serviceLink_nodejs"]`);
+    await page.waitForSelector(`[data-test-subj="apmMainTemplateHeaderServiceName"]`);
+  })
+  .step('Navigate to Transactions tabs', async ({ page, kbnUrl }) => {
+    await page.click(`[data-test-subj="transactionsTab"]`);
+    await page.waitForSelector(`[data-test-subj="apmTransactionDetailLinkLink"]`);
+  })
+  .step('Wait for Trace Waterfall on the page to load', async ({ page, kbnUrl }) => {
+    await page.click(`[data-test-subj="apmTransactionDetailLinkLink"]`);
+    await page.waitForSelector(`[data-test-subj="apmWaterfallButton"]`);
+  });

x-pack/performance/services/synthtrace/index.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ApmSynthtraceEsClient, ApmSynthtraceKibanaClient } from '@kbn/apm-synthtrace';
+import Url from 'url';
+import { Readable } from 'stream';
+import { ApmFields, SynthtraceGenerator } from '@kbn/apm-synthtrace-client';
+
+export interface SynthtraceClientParams {
+  kbnBaseUrl: string;
+  logger: any;
+  username: string;
+  password: string;
+  esClient: any;
+}
+
+export class SynthtraceClient {
+  private synthtraceEsClient: ApmSynthtraceEsClient | undefined;
+  private packageVersion: string = '';
+  private readonly kibanaUrlWithAuth: string;
+
+  constructor(private readonly baseParams: SynthtraceClientParams) {
+    const kibanaUrl = new URL(this.baseParams.kbnBaseUrl);
+    this.kibanaUrlWithAuth = Url.format({
+      protocol: kibanaUrl.protocol,
+      hostname: kibanaUrl.hostname,
+      port: kibanaUrl.port,
+      auth: `${this.baseParams.username}:${this.baseParams.password}`,
+    });
+  }
+
+  async installApmPackage() {
+    const kibanaClient = new ApmSynthtraceKibanaClient({
+      logger: this.baseParams.logger,
+      target: this.kibanaUrlWithAuth,
+    });
+    this.packageVersion = await kibanaClient.fetchLatestApmPackageVersion();
+
+    await kibanaClient.installApmPackage(this.packageVersion);
+  }
+
+  async initialiseEsClient() {
+    this.synthtraceEsClient = new ApmSynthtraceEsClient({
+      client: this.baseParams.esClient,
+      logger: this.baseParams.logger,
+      refreshAfterIndex: true,
+      version: this.packageVersion,
+    });
+
+    this.synthtraceEsClient.pipeline(this.synthtraceEsClient.getDefaultPipeline(false));
+  }
+
+  async index(events: SynthtraceGenerator<ApmFields>) {
+    if (this.synthtraceEsClient) {
+      await this.synthtraceEsClient.index(
+        Readable.from(Array.from(events).flatMap((event) => event.serialize()))
+      );
+    } else {
+      throw new Error('ES Client not initialised');
+    }
+  }
+}

x-pack/performance/synthtrace_data_generators/apm_data.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { apm, httpExitSpan, timerange } from '@kbn/apm-synthtrace-client';
+
+export function generateData({ from, to }: { from: number; to: number }) {
+  const range = timerange(from, to);
+  const transactionName = '240rpm/75% 1000ms';
+
+  const synthRum = apm
+    .service({ name: 'synth-rum', environment: 'production', agentName: 'rum-js' })
+    .instance('my-instance');
+  const synthNode = apm
+    .service({ name: 'synth-node', environment: 'production', agentName: 'nodejs' })
+    .instance('my-instance');
+  const synthGo = apm
+    .service({ name: 'synth-go', environment: 'production', agentName: 'go' })
+    .instance('my-instance');
+
+  return range.interval('1m').generator((timestamp) => {
+    return synthRum
+      .transaction({ transactionName })
+      .duration(400)
+      .timestamp(timestamp)
+      .children(
+        // synth-rum -> synth-node
+        synthRum
+          .span(
+            httpExitSpan({
+              spanName: 'GET /api/products/top',
+              destinationUrl: 'http://synth-node:3000',
+            })
+          )
+          .duration(300)
+          .timestamp(timestamp)
+
+          .children(
+            // synth-node
+            synthNode
+              .transaction({ transactionName: 'Initial transaction in synth-node' })
+              .duration(300)
+              .timestamp(timestamp)
+              .children(
+                synthNode
+                  // synth-node -> synth-go
+                  .span(
+                    httpExitSpan({
+                      spanName: 'GET synth-go:3000',
+                      destinationUrl: 'http://synth-go:3000',
+                    })
+                  )
+                  .timestamp(timestamp)
+                  .duration(400)
+
+                  .children(
+                    // synth-go
+                    synthGo
+
+                      .transaction({ transactionName: 'Initial transaction in synth-go' })
+                      .timestamp(timestamp)
+                      .duration(200)
+                      .children(
+                        synthGo
+                          .span({ spanName: 'custom_operation', spanType: 'custom' })
+                          .timestamp(timestamp)
+                          .duration(100)
+                          .success()
+                      )
+                  )
+              )
+          )
+      );
+  });
+}

x-pack/performance/tsconfig.json

@@ -14,5 +14,8 @@
     "@kbn/tooling-log",
     "@kbn/test",
     "@kbn/expect",
+    "@kbn/dev-utils",
+    "@kbn/apm-synthtrace",
+    "@kbn/apm-synthtrace-client",
   ]
 }

x-pack/plugins/security_solution/common/constants.ts

@@ -233,6 +233,9 @@ export const DETECTION_ENGINE_RULES_BULK_CREATE =
 export const DETECTION_ENGINE_RULES_BULK_UPDATE =
   `${DETECTION_ENGINE_RULES_URL}/_bulk_update` as const;
 
+/**
+ * Internal Risk Score routes
+ */
 export const INTERNAL_RISK_SCORE_URL = '/internal/risk_score' as const;
 export const DEV_TOOL_PREBUILT_CONTENT =
   `${INTERNAL_RISK_SCORE_URL}/prebuilt_content/dev_tool/{console_id}` as const;
@@ -244,16 +247,21 @@ export const prebuiltSavedObjectsBulkCreateUrl = (templateName: string) =>
 export const PREBUILT_SAVED_OBJECTS_BULK_DELETE = `${INTERNAL_RISK_SCORE_URL}/prebuilt_content/saved_objects/_bulk_delete/{template_name}`;
 export const prebuiltSavedObjectsBulkDeleteUrl = (templateName: string) =>
   `${INTERNAL_RISK_SCORE_URL}/prebuilt_content/saved_objects/_bulk_delete/${templateName}` as const;
-
-export const INTERNAL_DASHBOARDS_URL = `/internal/dashboards` as const;
-export const INTERNAL_TAGS_URL = `/internal/tags`;
-
 export const RISK_SCORE_CREATE_INDEX = `${INTERNAL_RISK_SCORE_URL}/indices/create`;
 export const RISK_SCORE_DELETE_INDICES = `${INTERNAL_RISK_SCORE_URL}/indices/delete`;
 export const RISK_SCORE_CREATE_STORED_SCRIPT = `${INTERNAL_RISK_SCORE_URL}/stored_scripts/create`;
 export const RISK_SCORE_DELETE_STORED_SCRIPT = `${INTERNAL_RISK_SCORE_URL}/stored_scripts/delete`;
 export const RISK_SCORE_PREVIEW_URL = `${INTERNAL_RISK_SCORE_URL}/preview`;
 
+/**
+ * Public Risk Score routes
+ */
+export const RISK_ENGINE_PUBLIC_PREFIX = '/api/risk_scores' as const;
+export const RISK_SCORE_CALCULATION_URL = `${RISK_ENGINE_PUBLIC_PREFIX}/calculation` as const;
+
+export const INTERNAL_DASHBOARDS_URL = `/internal/dashboards` as const;
+export const INTERNAL_TAGS_URL = `/internal/tags`;
+
 /**
  * Internal detection engine routes
  */

x-pack/plugins/security_solution/common/risk_engine/risk_score_calculation/request_schema.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as t from 'io-ts';
+import { DataViewId } from '../../api/detection_engine';
+import { afterKeysSchema } from '../after_keys';
+import { identifierTypeSchema } from '../identifier_types';
+import { riskWeightsSchema } from '../risk_weights/schema';
+
+export const riskScoreCalculationRequestSchema = t.exact(
+  t.intersection([
+    t.type({
+      data_view_id: DataViewId,
+      identifier_type: identifierTypeSchema,
+      range: t.type({
+        start: t.string,
+        end: t.string,
+      }),
+    }),
+    t.partial({
+      after_keys: afterKeysSchema,
+      debug: t.boolean,
+      filter: t.unknown,
+      page_size: t.number,
+      weights: riskWeightsSchema,
+    }),
+  ])
+);

x-pack/plugins/security_solution/common/risk_engine/risk_score_preview/request_schema.ts

@@ -12,18 +12,22 @@ import { identifierTypeSchema } from '../identifier_types';
 import { riskWeightsSchema } from '../risk_weights/schema';
 
 export const riskScorePreviewRequestSchema = t.exact(
-  t.partial({
-    after_keys: afterKeysSchema,
-    data_view_id: DataViewId,
-    debug: t.boolean,
-    filter: t.unknown,
-    page_size: t.number,
-    identifier_type: identifierTypeSchema,
-    range: t.type({
-      start: t.string,
-      end: t.string,
+  t.intersection([
+    t.type({
+      data_view_id: DataViewId,
     }),
-    weights: riskWeightsSchema,
-  })
+    t.partial({
+      after_keys: afterKeysSchema,
+      debug: t.boolean,
+      filter: t.unknown,
+      page_size: t.number,
+      identifier_type: identifierTypeSchema,
+      range: t.type({
+        start: t.string,
+        end: t.string,
+      }),
+      weights: riskWeightsSchema,
+    }),
+  ])
 );
 export type RiskScorePreviewRequestSchema = t.TypeOf<typeof riskScorePreviewRequestSchema>;