Merge branch 'main' into feature-fleet-agent-cypress-tests

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -511,6 +511,10 @@
       },
       "FileKind": {
         "type": "keyword"
+      },
+      "hash": {
+        "dynamic": false,
+        "properties": {}
       }
     }
   },
@@ -2728,10 +2732,6 @@
     "dynamic": false,
     "properties": {}
   },
-  "metrics-explorer-view": {
-    "dynamic": false,
-    "properties": {}
-  },
   "inventory-view": {
     "dynamic": false,
     "properties": {}
@@ -2744,6 +2744,10 @@
       }
     }
   },
+  "metrics-explorer-view": {
+    "dynamic": false,
+    "properties": {}
+  },
   "upgrade-assistant-reindex-operation": {
     "dynamic": false,
     "properties": {

packages/shared-ux/file/types/index.ts

@@ -226,6 +226,10 @@ export interface FileJSON<Meta = unknown> {
    * User data associated with this file
    */
   user?: FileMetadata['user'];
+  /**
+   * File hash information
+   */
+  hash?: BaseFileMetadata['hash'];
 }
 
 export interface FileKindBase {

src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts

@@ -88,7 +88,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "event_loop_delays_daily": "ef49e7f15649b551b458c7ea170f3ed17f89abd0",
         "exception-list": "38181294f64fc406c15f20d85ca306c8a4feb3c0",
         "exception-list-agnostic": "d527ce9d12b134cb163150057b87529043a8ec77",
-        "file": "d12998f49bc82da596a9e6c8397999930187ec6a",
+        "file": "487a562dd895407307980cc4404ca08e87e8999d",
         "file-upload-usage-collection-telemetry": "c6fcb9a7efcf19b2bb66ca6e005bfee8961f6073",
         "fileShare": "f07d346acbb724eacf139a0fb781c38dc5280115",
         "fleet-fleet-server-host": "67180a54a689111fb46403c3603c9b3a329c698d",

src/plugins/files/common/types.ts

@@ -18,6 +18,7 @@ import type {
   FileKindBase,
   FileShareJSONWithToken,
 } from '@kbn/shared-ux-file-types';
+import type { UploadOptions } from '../server/blob_storage_service';
 import type { ES_FIXED_SIZE_INDEX_BLOB_STORE } from './constants';
 
 export type {
@@ -179,7 +180,7 @@ export interface File<Meta = unknown> {
    */
   data: FileJSON<Meta>;
   /**
-   * Update a file object's metadatathat can be updated.
+   * Update a file object's metadata that can be updated.
    *
    * @param attr - The of attributes to update.
    */
@@ -190,8 +191,13 @@ export interface File<Meta = unknown> {
    *
    * @param content - The content to stream to storage.
    * @param abort$ - An observable that can be used to abort the upload at any time.
+   * @param options - additional options.
    */
-  uploadContent(content: Readable, abort$?: Observable<unknown>): Promise<File<Meta>>;
+  uploadContent(
+    content: Readable,
+    abort$?: Observable<unknown>,
+    options?: Partial<Pick<UploadOptions, 'transforms'>>
+  ): Promise<File<Meta>>;
 
   /**
    * Stream file content from storage.

src/plugins/files/server/file/file.test.ts

@@ -14,11 +14,9 @@ import {
   loggingSystemMock,
   savedObjectsServiceMock,
 } from '@kbn/core/server/mocks';
-import { Readable } from 'stream';
+import { Readable, Transform } from 'stream';
 import { promisify } from 'util';
 
-const setImmediate = promisify(global.setImmediate);
-
 import { BlobStorageService } from '../blob_storage_service';
 import { InternalFileService } from '../file_service/internal_file_service';
 import {
@@ -29,6 +27,10 @@ import {
 import { InternalFileShareService } from '../file_share_service';
 import { FileMetadataClient } from '../file_client';
 import { SavedObjectsFileMetadataClient } from '../file_client/file_metadata_client/adapters/saved_objects';
+import { File as IFile } from '../../common';
+import { createFileHashTransform } from '..';
+
+const setImmediate = promisify(global.setImmediate);
 
 describe('File', () => {
   let esClient: ElasticsearchClient;
@@ -110,4 +112,87 @@ describe('File', () => {
     expect(file.data.status).toBe('UPLOAD_ERROR');
     expect(blobStoreSpy.calledOnce).toBe(true);
   });
+
+  describe('#uploadContent() method', () => {
+    let file: IFile;
+    let fileContent: Readable;
+
+    beforeEach(async () => {
+      const fileSO = { attributes: { Status: 'AWAITING_UPLOAD' } };
+      (soClient.create as jest.Mock).mockResolvedValue(fileSO);
+      (soClient.update as jest.Mock).mockResolvedValue(fileSO);
+      (soClient.get as jest.Mock).mockResolvedValue({
+        attributes: {
+          created: '2023-04-27T19:57:19.640Z',
+          Updated: '2023-04-27T19:57:19.640Z',
+          name: 'test',
+          Status: 'DONE',
+          FileKind: fileKind,
+          hash: {
+            sha256: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
+          },
+        },
+      });
+
+      file = await fileService.createFile({ name: 'test', fileKind });
+      fileContent = Readable.from(['test']);
+    });
+
+    it('should allow custom transforms to be used', async () => {
+      let used = 0;
+      const customTransform = new Transform({
+        transform(chunk, _, next) {
+          used++;
+          next(null, chunk);
+        },
+      });
+
+      let used2 = 0;
+      const customTransform2 = new Transform({
+        transform(chunk, _, next) {
+          used2++;
+          next(null, chunk);
+        },
+      });
+
+      await file.uploadContent(fileContent, undefined, {
+        transforms: [customTransform, customTransform2],
+      });
+
+      expect(used).toBeGreaterThan(0);
+      expect(used2).toBeGreaterThan(0);
+      expect(file.data).toEqual({
+        created: expect.any(String),
+        updated: expect.any(String),
+        fileKind: 'fileKind',
+        size: 4,
+        status: 'READY',
+      });
+    });
+
+    it('should generate and store file hash when FileHashTransform is used', async () => {
+      await file.uploadContent(fileContent, undefined, {
+        transforms: [createFileHashTransform()],
+      });
+
+      expect(file.toJSON()).toEqual({
+        created: expect.any(String),
+        updated: expect.any(String),
+        fileKind: 'fileKind',
+        hash: {
+          sha256: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
+        },
+        size: 4,
+        status: 'READY',
+      });
+    });
+
+    it('should return file hash', async () => {
+      file = await fileService.getById({ id: '1' });
+
+      expect(file.data.hash).toEqual({
+        sha256: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
+      });
+    });
+  });
 });

src/plugins/files/server/file/file.ts

@@ -19,6 +19,8 @@ import {
   Observable,
   lastValueFrom,
 } from 'rxjs';
+import { isFileHashTransform } from '../file_client/stream_transforms/file_hash_transform/file_hash_transform';
+import { UploadOptions } from '../blob_storage_service';
 import type { FileShareJSON, FileShareJSONWithToken } from '../../common/types';
 import type { File as IFile, UpdatableFileMetadata, FileJSON } from '../../common';
 import { fileAttributesReducer, Action } from './file_attributes_reducer';
@@ -70,13 +72,17 @@ export class File<M = unknown> implements IFile {
     return this;
   }
 
-  private upload(content: Readable): Observable<{ size: number }> {
-    return defer(() => this.fileClient.upload(this.metadata, content));
+  private upload(
+    content: Readable,
+    options?: Partial<Pick<UploadOptions, 'transforms'>>
+  ): Observable<{ size: number }> {
+    return defer(() => this.fileClient.upload(this.metadata, content, options));
   }
 
   public async uploadContent(
     content: Readable,
-    abort$: Observable<unknown> = NEVER
+    abort$: Observable<unknown> = NEVER,
+    options?: Partial<Pick<UploadOptions, 'transforms'>>
   ): Promise<IFile<M>> {
     if (this.uploadInProgress()) {
       throw new UploadInProgressError('Upload already in progress.');
@@ -90,7 +96,7 @@ export class File<M = unknown> implements IFile {
       from(this.updateFileState({ action: 'uploading' })).pipe(
         mergeMap(() =>
           race(
-            this.upload(content),
+            this.upload(content, options),
             abort$.pipe(
               map(() => {
                 throw new AbortedUploadError(`Aborted upload of ${this.id}!`);
@@ -99,7 +105,28 @@ export class File<M = unknown> implements IFile {
           )
         ),
         mergeMap(({ size }) => {
-          return this.updateFileState({ action: 'uploaded', payload: { size } });
+          const updatedStateAction: Action & { action: 'uploaded' } = {
+            action: 'uploaded',
+            payload: { size },
+          };
+
+          if (options && options.transforms) {
+            options.transforms.some((transform) => {
+              if (isFileHashTransform(transform)) {
+                const fileHash = transform.getFileHash();
+
+                updatedStateAction.payload.hash = {
+                  [fileHash.algorithm]: fileHash.value,
+                };
+
+                return true;
+              }
+
+              return false;
+            });
+          }
+
+          return this.updateFileState(updatedStateAction);
         }),
         catchError(async (e) => {
           try {

src/plugins/files/server/file/file_attributes_reducer.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import { FileHashObj } from '../saved_objects/file';
 import { FileJSON, UpdatableFileMetadata } from '../../common';
 
 export type Action =
@@ -17,7 +18,10 @@ export type Action =
       action: 'uploading';
       payload?: undefined;
     }
-  | { action: 'uploaded'; payload: { size: number } }
+  | {
+      action: 'uploaded';
+      payload: { size: number; hash?: FileHashObj };
+    }
   | { action: 'uploadError'; payload?: undefined }
   | { action: 'updateFile'; payload: Partial<UpdatableFileMetadata> };
 

src/plugins/files/server/file/to_json.ts

@@ -10,8 +10,20 @@ import { pickBy } from 'lodash';
 import type { FileMetadata, FileJSON } from '../../common/types';
 
 export function serializeJSON<M = unknown>(attrs: Partial<FileJSON>): Partial<FileMetadata<M>> {
-  const { name, mimeType, size, created, updated, fileKind, status, alt, extension, meta, user } =
-    attrs;
+  const {
+    name,
+    mimeType,
+    size,
+    created,
+    updated,
+    fileKind,
+    status,
+    alt,
+    extension,
+    meta,
+    user,
+    hash,
+  } = attrs;
   return pickBy(
     {
       name,
@@ -25,6 +37,7 @@ export function serializeJSON<M = unknown>(attrs: Partial<FileJSON>): Partial<Fi
       Meta: meta,
       Updated: updated,
       FileKind: fileKind,
+      hash,
     },
     (v) => v != null
   );
@@ -43,6 +56,7 @@ export function toJSON<M = unknown>(id: string, attrs: FileMetadata<M>): FileJSO
     Alt,
     extension,
     Meta,
+    hash,
   } = attrs;
   return pickBy<FileJSON<M>>(
     {
@@ -58,6 +72,7 @@ export function toJSON<M = unknown>(id: string, attrs: FileMetadata<M>): FileJSO
       meta: Meta,
       updated: Updated,
       fileKind: FileKind,
+      hash,
     },
     (v) => v != null
   ) as FileJSON<M>;

src/plugins/files/server/file_client/file_client.ts

@@ -208,7 +208,7 @@ export class FileClientImpl implements FileClient {
 
   /**
    * Upload a blob
-   * @param id - The ID of the file content is associated with
+   * @param file - The file Record that the content is associated with
    * @param rs - The readable stream of the file content
    * @param options - Options for the upload
    */