Merge branch 'main' into make-metrics-internals

.buildkite/ftr_configs.yml

@@ -413,5 +413,6 @@ enabled:
   - x-pack/performance/journeys/ecommerce_dashboard_tsvb_gauge_only.ts
   - x-pack/performance/journeys/dashboard_listing_page.ts
   - x-pack/performance/journeys/cloud_security_dashboard.ts
+  - x-pack/performance/journeys/apm_service_inventory.ts
   - x-pack/test/custom_branding/config.ts
   - x-pack/test/profiling_api_integration/cloud/config.ts

docs/settings/security-settings.asciidoc

@@ -210,7 +210,7 @@ Sets the interval at which {kib} tries to remove expired and invalid sessions fr
 +
 TIP: Use a string of `<count>[ms\|s\|m\|h\|d\|w\|M\|Y]` (e.g. '20m', '24h', '7d', '1w').
 
-xpack.security.session.сoncurrentSessions.maxSessions::
+xpack.security.session.concurrentSessions.maxSessions {ess-icon}::
 Set the maximum number of sessions each user is allowed to have active at any given time.
 By default, no limit is applied.
 If set, the value of this option should be an integer between `1` and `1000`.

x-pack/performance/configs/apm_config.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { FtrConfigProviderContext } from '@kbn/test';
+import { CA_CERT_PATH } from '@kbn/dev-utils';
+
+// eslint-disable-next-line import/no-default-export
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const xpackFunctionalConfig = await readConfigFile(
+    // eslint-disable-next-line @kbn/imports/no_boundary_crossing
+    require.resolve('../../test/functional/config.base.js')
+  );
+
+  return {
+    ...xpackFunctionalConfig.getAll(),
+
+    kbnTestServer: {
+      ...xpackFunctionalConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'),
+        '--home.disableWelcomeScreen=true',
+        '--csp.strict=false',
+        '--csp.warnLegacyBrowsers=false',
+        // define custom kibana server args here
+        `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`,
+        '--server.eluMonitor.enabled=true',
+        '--server.eluMonitor.logging.enabled=true',
+        '--server.eluMonitor.logging.threshold.ela=250',
+        '--server.eluMonitor.logging.threshold.elu=0.15',
+      ],
+    },
+  };
+}

x-pack/performance/journeys/apm_service_inventory.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Journey } from '@kbn/journeys';
+import { SynthtraceClient } from '../services/synthtrace';
+import { generateData } from '../synthtrace_data/apm_data';
+
+export const journey = new Journey({
+  beforeSteps: async ({ kbnUrl, log, auth, es }) => {
+    // Install APM Package
+    const synthClient = new SynthtraceClient({
+      kbnBaseUrl: kbnUrl.get(),
+      logger: log,
+      username: auth.getUsername(),
+      password: auth.getPassword(),
+      esClient: es,
+    });
+
+    await synthClient.installApmPackage();
+    // Setup Synthtrace Client
+    await synthClient.initialiseEsClient();
+    // Generate data using Synthtrace
+    const start = Date.now() - 1000;
+    const end = Date.now();
+    await synthClient.index(
+      generateData({
+        from: new Date(start).getTime(),
+        to: new Date(end).getTime(),
+      })
+    );
+  },
+  ftrConfigPath: 'x-pack/performance/configs/apm_config.ts',
+})
+  .step('Navigate to Service Inventory Page', async ({ page, kbnUrl }) => {
+    await page.goto(kbnUrl.get(`app/apm/services`));
+    await page.waitForSelector(`[data-test-subj="serviceLink_nodejs"]`);
+  })
+  .step('Navigate to Service Overview Page', async ({ page, kbnUrl }) => {
+    await page.click(`[data-test-subj="serviceLink_nodejs"]`);
+    await page.waitForSelector(`[data-test-subj="apmMainTemplateHeaderServiceName"]`);
+  })
+  .step('Navigate to Transactions tabs', async ({ page, kbnUrl }) => {
+    await page.click(`[data-test-subj="transactionsTab"]`);
+    await page.waitForSelector(`[data-test-subj="apmTransactionDetailLinkLink"]`);
+  })
+  .step('Wait for Trace Waterfall on the page to load', async ({ page, kbnUrl }) => {
+    await page.click(`[data-test-subj="apmTransactionDetailLinkLink"]`);
+    await page.waitForSelector(`[data-test-subj="apmWaterfallButton"]`);
+  });

x-pack/performance/services/synthtrace/index.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ApmSynthtraceEsClient, ApmSynthtraceKibanaClient } from '@kbn/apm-synthtrace';
+import Url from 'url';
+import { Readable } from 'stream';
+import { ApmFields, SynthtraceGenerator } from '@kbn/apm-synthtrace-client';
+
+export interface SynthtraceClientParams {
+  kbnBaseUrl: string;
+  logger: any;
+  username: string;
+  password: string;
+  esClient: any;
+}
+
+export class SynthtraceClient {
+  private synthtraceEsClient: ApmSynthtraceEsClient | undefined;
+  private packageVersion: string = '';
+  private readonly kibanaUrlWithAuth: string;
+
+  constructor(private readonly baseParams: SynthtraceClientParams) {
+    const kibanaUrl = new URL(this.baseParams.kbnBaseUrl);
+    this.kibanaUrlWithAuth = Url.format({
+      protocol: kibanaUrl.protocol,
+      hostname: kibanaUrl.hostname,
+      port: kibanaUrl.port,
+      auth: `${this.baseParams.username}:${this.baseParams.password}`,
+    });
+  }
+
+  async installApmPackage() {
+    const kibanaClient = new ApmSynthtraceKibanaClient({
+      logger: this.baseParams.logger,
+      target: this.kibanaUrlWithAuth,
+    });
+    this.packageVersion = await kibanaClient.fetchLatestApmPackageVersion();
+
+    await kibanaClient.installApmPackage(this.packageVersion);
+  }
+
+  async initialiseEsClient() {
+    this.synthtraceEsClient = new ApmSynthtraceEsClient({
+      client: this.baseParams.esClient,
+      logger: this.baseParams.logger,
+      refreshAfterIndex: true,
+      version: this.packageVersion,
+    });
+
+    this.synthtraceEsClient.pipeline(this.synthtraceEsClient.getDefaultPipeline(false));
+  }
+
+  async index(events: SynthtraceGenerator<ApmFields>) {
+    if (this.synthtraceEsClient) {
+      await this.synthtraceEsClient.index(
+        Readable.from(Array.from(events).flatMap((event) => event.serialize()))
+      );
+    } else {
+      throw new Error('ES Client not initialised');
+    }
+  }
+}

x-pack/performance/synthtrace_data/apm_data.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { apm, httpExitSpan, timerange } from '@kbn/apm-synthtrace-client';
+
+export function generateData({ from, to }: { from: number; to: number }) {
+  const range = timerange(from, to);
+  const transactionName = '240rpm/75% 1000ms';
+
+  const synthRum = apm
+    .service({ name: 'synth-rum', environment: 'production', agentName: 'rum-js' })
+    .instance('my-instance');
+  const synthNode = apm
+    .service({ name: 'synth-node', environment: 'production', agentName: 'nodejs' })
+    .instance('my-instance');
+  const synthGo = apm
+    .service({ name: 'synth-go', environment: 'production', agentName: 'go' })
+    .instance('my-instance');
+
+  return range.interval('1m').generator((timestamp) => {
+    return synthRum
+      .transaction({ transactionName })
+      .duration(400)
+      .timestamp(timestamp)
+      .children(
+        // synth-rum -> synth-node
+        synthRum
+          .span(
+            httpExitSpan({
+              spanName: 'GET /api/products/top',
+              destinationUrl: 'http://synth-node:3000',
+            })
+          )
+          .duration(300)
+          .timestamp(timestamp)
+
+          .children(
+            // synth-node
+            synthNode
+              .transaction({ transactionName: 'Initial transaction in synth-node' })
+              .duration(300)
+              .timestamp(timestamp)
+              .children(
+                synthNode
+                  // synth-node -> synth-go
+                  .span(
+                    httpExitSpan({
+                      spanName: 'GET synth-go:3000',
+                      destinationUrl: 'http://synth-go:3000',
+                    })
+                  )
+                  .timestamp(timestamp)
+                  .duration(400)
+
+                  .children(
+                    // synth-go
+                    synthGo
+
+                      .transaction({ transactionName: 'Initial transaction in synth-go' })
+                      .timestamp(timestamp)
+                      .duration(200)
+                      .children(
+                        synthGo
+                          .span({ spanName: 'custom_operation', spanType: 'custom' })
+                          .timestamp(timestamp)
+                          .duration(100)
+                          .success()
+                      )
+                  )
+              )
+          )
+      );
+  });
+}

x-pack/performance/tsconfig.json

@@ -14,5 +14,8 @@
     "@kbn/tooling-log",
     "@kbn/test",
     "@kbn/expect",
+    "@kbn/dev-utils",
+    "@kbn/apm-synthtrace",
+    "@kbn/apm-synthtrace-client",
   ]
 }

x-pack/plugins/actions/server/usage/actions_telemetry.ts

@@ -476,7 +476,7 @@ export async function getExecutionsPerDayCount(
   countFailedByType: Record<string, number>;
   avgExecutionTime: number;
   avgExecutionTimeByType: Record<string, number>;
-  countRunOutcomeByConnectorType: Record<string, number>;
+  countRunOutcomeByConnectorType: Record<string, Record<string, number>>;
 }> {
   const scriptedMetric = {
     scripted_metric: {

x-pack/plugins/actions/server/usage/task_state.ts

@@ -66,7 +66,7 @@ export const stateSchemaByVersion = {
       avg_execution_time_by_type_per_day: schema.recordOf(schema.string(), schema.number()),
       count_connector_types_by_action_run_outcome_per_day: schema.recordOf(
         schema.string(),
-        schema.number()
+        schema.recordOf(schema.string(), schema.number())
       ),
     }),
   },

x-pack/plugins/apm/dev_docs/overflow_bucket_setup.md

@@ -154,7 +154,11 @@ For more detailed instructions you can check [apm-server dev documentation](http
 
 - Copy paste the below script in a file called `tx_max_group.go`
   This file is responsible for generating 3 transactions per service.
-  ```
+
+<details>
+  <summary>tx_max_group.go</summary>
+
+  ```go
   package main
 
   import (
@@ -185,9 +189,14 @@ For more detailed instructions you can check [apm-server dev documentation](http
     span.End()
   }
   ```
-
+</details>
+
 - Now create a Bash Script file, name it anything - e.g., `service_max_group.sh`
   This file will generate services and then transactions for each service using the go script above.
+
+<details>
+  <summary>tx_max_group.go</summary>
+
   ```sh
   #!/usr/bin/env bash
 
@@ -203,6 +212,7 @@ For more detailed instructions you can check [apm-server dev documentation](http
 
   echo "Ending script"
   ```
+</details>
 
 - Run `sh service_max_group` to generate the data
 

x-pack/plugins/event_log/server/es/init.test.ts

@@ -455,8 +455,7 @@ describe('parseIndexAliases', () => {
   });
 });
 
-// FLAKY: https://github.com/elastic/kibana/issues/156061
-describe.skip('retries', () => {
+describe('retries', () => {
   let esContext = contextMock.create();
   // set up context APIs to return defaults indicating already created
   beforeEach(() => {
@@ -474,7 +473,7 @@ describe.skip('retries', () => {
 
     const timeStart = performance.now();
     await initializeEs(esContext);
-    const timeElapsed = performance.now() - timeStart;
+    const timeElapsed = Math.ceil(performance.now() - timeStart);
 
     expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY);