Merge branch 'main' into reenable_tests

package.json

@@ -499,7 +499,7 @@
     "deepmerge": "^4.2.2",
     "del": "^6.1.0",
     "elastic-apm-http-client": "^11.0.1",
-    "elastic-apm-node": "^3.39.0",
+    "elastic-apm-node": "^3.40.0",
     "email-addresses": "^5.0.0",
     "execa": "^4.0.2",
     "expiry-js": "0.1.7",

src/plugins/chart_expressions/expression_xy/common/expression_functions/data_decoration_config.ts

@@ -7,17 +7,11 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import type { ExpressionFunctionDefinition } from '@kbn/expressions-plugin/common';
 import { DATA_DECORATION_CONFIG } from '../constants';
-import { DataDecorationConfig, DataDecorationConfigResult } from '../types';
+import { DataDecorationConfigFn } from '../types';
 import { commonDecorationConfigArgs } from './common_y_config_args';
 
-export const dataDecorationConfigFunction: ExpressionFunctionDefinition<
-  typeof DATA_DECORATION_CONFIG,
-  null,
-  DataDecorationConfig,
-  DataDecorationConfigResult
-> = {
+export const dataDecorationConfigFunction: DataDecorationConfigFn = {
   name: DATA_DECORATION_CONFIG,
   aliases: [],
   type: DATA_DECORATION_CONFIG,

src/plugins/chart_expressions/expression_xy/common/expression_functions/event_annotations_result.ts

@@ -6,28 +6,11 @@
  * Side Public License, v 1.
  */
 
-import type { Datatable, ExpressionFunctionDefinition } from '@kbn/expressions-plugin/common';
-import { ExtendedAnnotationLayerConfigResult } from '../types';
+import { EventAnnotationResultFn } from '../types';
 import { strings } from '../i18n';
 import { EXTENDED_ANNOTATION_LAYER } from '../constants';
 
-export interface EventAnnotationResultArgs {
-  layers?: ExtendedAnnotationLayerConfigResult[];
-  datatable: Datatable;
-}
-
-export interface EventAnnotationResultResult {
-  type: 'event_annotations_result';
-  layers: ExtendedAnnotationLayerConfigResult[];
-  datatable: Datatable;
-}
-
-export function eventAnnotationsResult(): ExpressionFunctionDefinition<
-  'event_annotations_result',
-  null,
-  EventAnnotationResultArgs,
-  EventAnnotationResultResult
-> {
+export function eventAnnotationsResult(): EventAnnotationResultFn {
   return {
     name: 'event_annotations_result',
     aliases: [],

src/plugins/chart_expressions/expression_xy/common/expression_functions/extended_annotation_layer.ts

@@ -6,17 +6,11 @@
  * Side Public License, v 1.
  */
 
-import type { ExpressionFunctionDefinition } from '@kbn/expressions-plugin/common';
 import { LayerTypes, EXTENDED_ANNOTATION_LAYER } from '../constants';
-import { ExtendedAnnotationLayerConfigResult, ExtendedAnnotationLayerArgs } from '../types';
+import { ExtendedAnnotationLayerFn } from '../types';
 import { strings } from '../i18n';
 
-export function extendedAnnotationLayerFunction(): ExpressionFunctionDefinition<
-  typeof EXTENDED_ANNOTATION_LAYER,
-  null,
-  ExtendedAnnotationLayerArgs,
-  ExtendedAnnotationLayerConfigResult
-> {
+export function extendedAnnotationLayerFunction(): ExtendedAnnotationLayerFn {
   return {
     name: EXTENDED_ANNOTATION_LAYER,
     aliases: [],

src/plugins/chart_expressions/expression_xy/common/index.ts

@@ -60,4 +60,15 @@ export type {
   CommonXYReferenceLineLayerConfig,
   ReferenceLineDecorationConfigResult,
   CommonXYReferenceLineLayerConfigResult,
+  ReferenceLineDecorationConfigFn,
+  DataDecorationConfigFn,
+  ExtendedDataLayerFn,
+  ExtendedAnnotationLayerConfigResult,
+  ExtendedAnnotationLayerFn,
+  ReferenceLineLayerFn,
+  YAxisConfigFn,
+  XAxisConfigFn,
+  LegendConfigFn,
+  EventAnnotationResultFn,
+  LayeredXyVisFn,
 } from './types';

src/plugins/chart_expressions/expression_xy/common/types/expression_functions.ts

@@ -240,6 +240,24 @@ export type ExpressionAnnotationResult = ExpressionAnnotationsLayers & {
   type: 'event_annotations_result';
 };
 
+export interface EventAnnotationResultArgs {
+  layers?: ExtendedAnnotationLayerConfigResult[];
+  datatable: Datatable;
+}
+
+export interface EventAnnotationResultResult {
+  type: 'event_annotations_result';
+  layers: ExtendedAnnotationLayerConfigResult[];
+  datatable: Datatable;
+}
+
+export type EventAnnotationResultFn = ExpressionFunctionDefinition<
+  'event_annotations_result',
+  null,
+  EventAnnotationResultArgs,
+  EventAnnotationResultResult
+>;
+
 export interface LayeredXYArgs {
   legend: LegendConfigResult;
   endValue?: EndValue;
@@ -471,3 +489,10 @@ export type YAxisConfigFn = ExpressionFunctionDefinition<
   YAxisConfig,
   YAxisConfigResult
 >;
+
+export type ExtendedAnnotationLayerFn = ExpressionFunctionDefinition<
+  typeof EXTENDED_ANNOTATION_LAYER,
+  null,
+  ExtendedAnnotationLayerArgs,
+  ExtendedAnnotationLayerConfigResult
+>;

src/plugins/charts/common/expressions/palette/index.ts

@@ -9,10 +9,11 @@
 export { palette } from './palette';
 
 export { systemPalette } from './system_palette';
-export type { SystemPaletteArguments } from './system_palette';
 
 export type {
   CustomPaletteArguments,
   CustomPaletteState,
+  SystemPaletteArguments,
   PaletteExpressionFunctionDefinition,
+  SystemPaletteExpressionFunctionDefinition,
 } from './types';

src/plugins/charts/common/expressions/palette/system_palette.ts

@@ -7,20 +7,10 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import type { ExpressionFunctionDefinition } from '@kbn/expressions-plugin/common';
-import type { PaletteOutput } from './types';
+import type { SystemPaletteExpressionFunctionDefinition } from './types';
 import { paletteIds } from '../../constants';
 
-export interface SystemPaletteArguments {
-  name: string;
-}
-
-export function systemPalette(): ExpressionFunctionDefinition<
-  'system_palette',
-  null,
-  SystemPaletteArguments,
-  PaletteOutput
-> {
+export function systemPalette(): SystemPaletteExpressionFunctionDefinition {
   return {
     name: 'system_palette',
     aliases: [],

src/plugins/charts/common/expressions/palette/types.ts

@@ -41,3 +41,14 @@ export type PaletteExpressionFunctionDefinition = ExpressionFunctionDefinition<
   CustomPaletteArguments,
   Promise<PaletteOutput<CustomPaletteState>>
 >;
+
+export interface SystemPaletteArguments {
+  name: string;
+}
+
+export type SystemPaletteExpressionFunctionDefinition = ExpressionFunctionDefinition<
+  'system_palette',
+  null,
+  SystemPaletteArguments,
+  PaletteOutput
+>;

src/plugins/charts/common/index.ts

@@ -13,6 +13,7 @@ export type {
   CustomPaletteArguments,
   CustomPaletteState,
   SystemPaletteArguments,
+  SystemPaletteExpressionFunctionDefinition,
 } from './expressions/palette';
 export { palette, systemPalette } from './expressions/palette';
 

src/plugins/files/server/blob_storage_service/adapters/es/es.ts

@@ -15,6 +15,9 @@ import { Readable, Transform } from 'stream';
 import { pipeline } from 'stream/promises';
 import { promisify } from 'util';
 import { lastValueFrom, defer } from 'rxjs';
+import { PerformanceMetricEvent, reportPerformanceMetricEvent } from '@kbn/ebt-tools';
+import { FilesPlugin } from '../../../plugin';
+import { FILE_UPLOAD_PERFORMANCE_EVENT_NAME } from '../../../performance';
 import type { BlobStorageClient } from '../../types';
 import type { ReadableContentStream } from './content_stream';
 import { getReadableContentStream, getWritableContentStream } from './content_stream';
@@ -29,8 +32,14 @@ export const BLOB_STORAGE_SYSTEM_INDEX_NAME = '.kibana_blob_storage';
 
 export const MAX_BLOB_STORE_SIZE_BYTES = 50 * 1024 * 1024 * 1024; // 50 GiB
 
+interface UploadOptions {
+  transforms?: Transform[];
+  id?: string;
+}
+
 export class ElasticsearchBlobStorageClient implements BlobStorageClient {
   private static defaultSemaphore: Semaphore;
+
   /**
    * Call this function once to globally set a concurrent upload limit for
    * all {@link ElasticsearchBlobStorageClient} instances.
@@ -89,14 +98,14 @@ export class ElasticsearchBlobStorageClient implements BlobStorageClient {
     }
   });
 
-  public async upload(
-    src: Readable,
-    { transforms, id }: { transforms?: Transform[]; id?: string } = {}
-  ): Promise<{ id: string; size: number }> {
+  public async upload(src: Readable, options: UploadOptions = {}) {
+    const { transforms, id } = options;
+
     await this.createIndexIfNotExists();
 
     const processUpload = async () => {
       try {
+        const analytics = FilesPlugin.getAnalytics();
         const dest = getWritableContentStream({
           id,
           client: this.esClient,
@@ -106,14 +115,32 @@ export class ElasticsearchBlobStorageClient implements BlobStorageClient {
             maxChunkSize: this.chunkSize,
           },
         });
-        await pipeline.apply(null, [src, ...(transforms ?? []), dest] as unknown as Parameters<
-          typeof pipeline
-        >);
 
-        return {
-          id: dest.getContentReferenceId()!,
-          size: dest.getBytesWritten(),
+        const start = performance.now();
+        await pipeline([src, ...(transforms ?? []), dest]);
+        const end = performance.now();
+
+        const _id = dest.getContentReferenceId()!;
+        const size = dest.getBytesWritten();
+
+        const perfArgs: PerformanceMetricEvent = {
+          eventName: FILE_UPLOAD_PERFORMANCE_EVENT_NAME,
+          duration: end - start,
+          key1: 'size',
+          value1: size,
+          meta: {
+            datasource: 'es',
+            id: _id,
+            index: this.index,
+            chunkSize: this.chunkSize,
+          },
         };
+
+        if (analytics) {
+          reportPerformanceMetricEvent(analytics, perfArgs);
+        }
+
+        return { id: _id, size };
       } catch (e) {
         this.logger.error(`Could not write chunks to Elasticsearch for id ${id}: ${e}`);
         throw e;

src/plugins/files/server/blob_storage_service/types.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import { AnalyticsServiceStart } from '@kbn/core-analytics-server';
 import type { JsonValue } from '@kbn/utility-types';
 import type { Readable, Transform } from 'stream';
 
@@ -29,6 +30,11 @@ export interface UploadOptions {
    * And so on.
    */
   id?: string;
+
+  /**
+   * Optional analytics service client in order to report performance of upload.
+   */
+  analytics?: AnalyticsServiceStart;
 }
 
 /**

src/plugins/files/server/file_client/file_client.ts

@@ -13,6 +13,7 @@ import cuid from 'cuid';
 import { type Logger, SavedObjectsErrorHelpers } from '@kbn/core/server';
 import type { AuditLogger } from '@kbn/security-plugin/server';
 import type { UsageCounter } from '@kbn/usage-collection-plugin/server';
+
 import type {
   File,
   FileJSON,
@@ -34,6 +35,11 @@ import { createAuditEvent } from '../audit_events';
 import type { FileClient, CreateArgs, DeleteArgs, P1, ShareArgs } from './types';
 import { serializeJSON, toJSON } from '../file/to_json';
 import { createDefaultFileAttributes } from './utils';
+import {
+  PerfArgs,
+  withReportPerformanceMetric,
+  FILE_DOWNLOAD_PERFORMANCE_EVENT_NAME,
+} from '../performance';
 
 export type UploadOptions = Omit<BlobUploadOptions, 'id'>;
 
@@ -219,10 +225,21 @@ export class FileClientImpl implements FileClient {
     });
   };
 
-  public download: BlobStorageClient['download'] = (args) => {
+  public download: BlobStorageClient['download'] = async (args) => {
     this.incrementUsageCounter('DOWNLOAD');
     try {
-      return this.blobStorageClient.download(args);
+      const perf: PerfArgs = {
+        eventData: {
+          eventName: FILE_DOWNLOAD_PERFORMANCE_EVENT_NAME,
+          key1: 'size',
+          value1: args.size,
+          meta: {
+            id: args.id,
+          },
+        },
+      };
+
+      return withReportPerformanceMetric(perf, () => this.blobStorageClient.download(args));
     } catch (e) {
       this.incrementUsageCounter('DOWNLOAD_ERROR');
       throw e;

src/plugins/files/server/performance/event_names.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export const FILE_DOWNLOAD_PERFORMANCE_EVENT_NAME = 'file-download';
+export const FILE_UPLOAD_PERFORMANCE_EVENT_NAME = 'file-upload';

src/plugins/files/server/performance/index.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export type { PerfArgs } from './report_performance';
+export { withReportPerformanceMetric } from './report_performance';
+
+export {
+  FILE_DOWNLOAD_PERFORMANCE_EVENT_NAME,
+  FILE_UPLOAD_PERFORMANCE_EVENT_NAME,
+} from './event_names';

src/plugins/files/server/performance/report_performance.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { PerformanceMetricEvent, reportPerformanceMetricEvent } from '@kbn/ebt-tools';
+import { Optional } from 'utility-types';
+import { FilesPlugin } from '../plugin';
+
+export interface PerfArgs {
+  eventData: Optional<PerformanceMetricEvent, 'duration'>;
+}
+
+export async function withReportPerformanceMetric<T>(perfArgs: PerfArgs, cb: () => Promise<T>) {
+  const analytics = FilesPlugin.getAnalytics();
+
+  const start = performance.now();
+  const response = await cb();
+  const end = performance.now();
+
+  if (analytics) {
+    reportPerformanceMetricEvent(analytics, {
+      ...perfArgs.eventData,
+      duration: end - start,
+    });
+  }
+
+  return response;
+}