[Files] Add ability to optionally generate a File Hash during upload …

…by allowing for custom Transforms to be used (#156039) ## Summary closes #154047 - Exposes reusable `Transform` that will calculate a File's hash and store it with the file's metadata. This Transform is "opt in" and not the default behaviour. - The `File.uploadContent()` method was enhanced to optionally accept `options.transforms Array<Transform>`, thus allowing consumer of the service to defined an additional set to be included in the file's processing pipeline. The upload process was also altered to recognize the use of the new `FileHashTransform` and store the file's `hash` if it is used. - Saved Object schema was also updated to include mappings for the `file.hash` property. This update also impacts the creation of indexes when the `FileClient` is created with custom indexes and those don't yet exist.
elastic · May 1, 2023 · 6e19049 · 6e19049
1 parent bdfc0d7
commit 6e19049
Show file tree

Hide file tree

Showing 15 changed files with 357 additions and 22 deletions.
diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json
@@ -511,6 +511,10 @@
       },
       "FileKind": {
         "type": "keyword"
+      },
+      "hash": {
+        "dynamic": false,
+        "properties": {}
       }
     }
   },
@@ -2728,10 +2732,6 @@
     "dynamic": false,
     "properties": {}
   },
-  "metrics-explorer-view": {
-    "dynamic": false,
-    "properties": {}
-  },
   "inventory-view": {
     "dynamic": false,
     "properties": {}
@@ -2744,6 +2744,10 @@
       }
     }
   },
+  "metrics-explorer-view": {
+    "dynamic": false,
+    "properties": {}
+  },
   "upgrade-assistant-reindex-operation": {
     "dynamic": false,
     "properties": {

diff --git a/packages/shared-ux/file/types/index.ts b/packages/shared-ux/file/types/index.ts
@@ -226,6 +226,10 @@ export interface FileJSON<Meta = unknown> {
    * User data associated with this file
    */
   user?: FileMetadata['user'];
+  /**
+   * File hash information
+   */
+  hash?: BaseFileMetadata['hash'];
 }
 
 export interface FileKindBase {

diff --git a/...e/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/...e/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts
@@ -88,7 +88,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "event_loop_delays_daily": "ef49e7f15649b551b458c7ea170f3ed17f89abd0",
         "exception-list": "38181294f64fc406c15f20d85ca306c8a4feb3c0",
         "exception-list-agnostic": "d527ce9d12b134cb163150057b87529043a8ec77",
-        "file": "d12998f49bc82da596a9e6c8397999930187ec6a",
+        "file": "487a562dd895407307980cc4404ca08e87e8999d",
         "file-upload-usage-collection-telemetry": "c6fcb9a7efcf19b2bb66ca6e005bfee8961f6073",
         "fileShare": "f07d346acbb724eacf139a0fb781c38dc5280115",
         "fleet-fleet-server-host": "67180a54a689111fb46403c3603c9b3a329c698d",

diff --git a/src/plugins/files/common/types.ts b/src/plugins/files/common/types.ts
@@ -18,6 +18,7 @@ import type {
   FileKindBase,
   FileShareJSONWithToken,
 } from '@kbn/shared-ux-file-types';
+import type { UploadOptions } from '../server/blob_storage_service';
 import type { ES_FIXED_SIZE_INDEX_BLOB_STORE } from './constants';
 
 export type {
@@ -179,7 +180,7 @@ export interface File<Meta = unknown> {
    */
   data: FileJSON<Meta>;
   /**
-   * Update a file object's metadatathat can be updated.
+   * Update a file object's metadata that can be updated.
    *
    * @param attr - The of attributes to update.
    */
@@ -190,8 +191,13 @@ export interface File<Meta = unknown> {
    *
    * @param content - The content to stream to storage.
    * @param abort$ - An observable that can be used to abort the upload at any time.
+   * @param options - additional options.
    */
-  uploadContent(content: Readable, abort$?: Observable<unknown>): Promise<File<Meta>>;
+  uploadContent(
+    content: Readable,
+    abort$?: Observable<unknown>,
+    options?: Partial<Pick<UploadOptions, 'transforms'>>
+  ): Promise<File<Meta>>;
 
   /**
    * Stream file content from storage.

diff --git a/src/plugins/files/server/file/file.test.ts b/src/plugins/files/server/file/file.test.ts
@@ -14,11 +14,9 @@ import {
   loggingSystemMock,
   savedObjectsServiceMock,
 } from '@kbn/core/server/mocks';
-import { Readable } from 'stream';
+import { Readable, Transform } from 'stream';
 import { promisify } from 'util';
 
-const setImmediate = promisify(global.setImmediate);
-
 import { BlobStorageService } from '../blob_storage_service';
 import { InternalFileService } from '../file_service/internal_file_service';
 import {
@@ -29,6 +27,10 @@ import {
 import { InternalFileShareService } from '../file_share_service';
 import { FileMetadataClient } from '../file_client';
 import { SavedObjectsFileMetadataClient } from '../file_client/file_metadata_client/adapters/saved_objects';
+import { File as IFile } from '../../common';
+import { createFileHashTransform } from '..';
+
+const setImmediate = promisify(global.setImmediate);
 
 describe('File', () => {
   let esClient: ElasticsearchClient;
@@ -110,4 +112,87 @@ describe('File', () => {
     expect(file.data.status).toBe('UPLOAD_ERROR');
     expect(blobStoreSpy.calledOnce).toBe(true);
   });
+
+  describe('#uploadContent() method', () => {
+    let file: IFile;
+    let fileContent: Readable;
+
+    beforeEach(async () => {
+      const fileSO = { attributes: { Status: 'AWAITING_UPLOAD' } };
+      (soClient.create as jest.Mock).mockResolvedValue(fileSO);
+      (soClient.update as jest.Mock).mockResolvedValue(fileSO);
+      (soClient.get as jest.Mock).mockResolvedValue({
+        attributes: {
+          created: '2023-04-27T19:57:19.640Z',
+          Updated: '2023-04-27T19:57:19.640Z',
+          name: 'test',
+          Status: 'DONE',
+          FileKind: fileKind,
+          hash: {
+            sha256: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
+          },
+        },
+      });
+
+      file = await fileService.createFile({ name: 'test', fileKind });
+      fileContent = Readable.from(['test']);
+    });
+
+    it('should allow custom transforms to be used', async () => {
+      let used = 0;
+      const customTransform = new Transform({
+        transform(chunk, _, next) {
+          used++;
+          next(null, chunk);
+        },
+      });
+
+      let used2 = 0;
+      const customTransform2 = new Transform({
+        transform(chunk, _, next) {
+          used2++;
+          next(null, chunk);
+        },
+      });
+
+      await file.uploadContent(fileContent, undefined, {
+        transforms: [customTransform, customTransform2],
+      });
+
+      expect(used).toBeGreaterThan(0);
+      expect(used2).toBeGreaterThan(0);
+      expect(file.data).toEqual({
+        created: expect.any(String),
+        updated: expect.any(String),
+        fileKind: 'fileKind',
+        size: 4,
+        status: 'READY',
+      });
+    });
+
+    it('should generate and store file hash when FileHashTransform is used', async () => {
+      await file.uploadContent(fileContent, undefined, {
+        transforms: [createFileHashTransform()],
+      });
+
+      expect(file.toJSON()).toEqual({
+        created: expect.any(String),
+        updated: expect.any(String),
+        fileKind: 'fileKind',
+        hash: {
+          sha256: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
+        },
+        size: 4,
+        status: 'READY',
+      });
+    });
+
+    it('should return file hash', async () => {
+      file = await fileService.getById({ id: '1' });
+
+      expect(file.data.hash).toEqual({
+        sha256: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
+      });
+    });
+  });
 });
diff --git a/src/plugins/files/server/file/file.ts b/src/plugins/files/server/file/file.ts
@@ -19,6 +19,8 @@ import {
   Observable,
   lastValueFrom,
 } from 'rxjs';
+import { isFileHashTransform } from '../file_client/stream_transforms/file_hash_transform/file_hash_transform';
+import { UploadOptions } from '../blob_storage_service';
 import type { FileShareJSON, FileShareJSONWithToken } from '../../common/types';
 import type { File as IFile, UpdatableFileMetadata, FileJSON } from '../../common';
 import { fileAttributesReducer, Action } from './file_attributes_reducer';
@@ -70,13 +72,17 @@ export class File<M = unknown> implements IFile {
     return this;
   }
 
-  private upload(content: Readable): Observable<{ size: number }> {
-    return defer(() => this.fileClient.upload(this.metadata, content));
+  private upload(
+    content: Readable,
+    options?: Partial<Pick<UploadOptions, 'transforms'>>
+  ): Observable<{ size: number }> {
+    return defer(() => this.fileClient.upload(this.metadata, content, options));
   }
 
   public async uploadContent(
     content: Readable,
-    abort$: Observable<unknown> = NEVER
+    abort$: Observable<unknown> = NEVER,
+    options?: Partial<Pick<UploadOptions, 'transforms'>>
   ): Promise<IFile<M>> {
     if (this.uploadInProgress()) {
       throw new UploadInProgressError('Upload already in progress.');
@@ -90,7 +96,7 @@ export class File<M = unknown> implements IFile {
       from(this.updateFileState({ action: 'uploading' })).pipe(
         mergeMap(() =>
           race(
-            this.upload(content),
+            this.upload(content, options),
             abort$.pipe(
               map(() => {
                 throw new AbortedUploadError(`Aborted upload of ${this.id}!`);
@@ -99,7 +105,28 @@ export class File<M = unknown> implements IFile {
           )
         ),
         mergeMap(({ size }) => {
-          return this.updateFileState({ action: 'uploaded', payload: { size } });
+          const updatedStateAction: Action & { action: 'uploaded' } = {
+            action: 'uploaded',
+            payload: { size },
+          };
+
+          if (options && options.transforms) {
+            options.transforms.some((transform) => {
+              if (isFileHashTransform(transform)) {
+                const fileHash = transform.getFileHash();
+
+                updatedStateAction.payload.hash = {
+                  [fileHash.algorithm]: fileHash.value,
+                };
+
+                return true;
+              }
+
+              return false;
+            });
+          }
+
+          return this.updateFileState(updatedStateAction);
         }),
         catchError(async (e) => {
           try {

diff --git a/src/plugins/files/server/file/file_attributes_reducer.ts b/src/plugins/files/server/file/file_attributes_reducer.ts
@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import { FileHashObj } from '../saved_objects/file';
 import { FileJSON, UpdatableFileMetadata } from '../../common';
 
 export type Action =
@@ -17,7 +18,10 @@ export type Action =
       action: 'uploading';
       payload?: undefined;
     }
-  | { action: 'uploaded'; payload: { size: number } }
+  | {
+      action: 'uploaded';
+      payload: { size: number; hash?: FileHashObj };
+    }
   | { action: 'uploadError'; payload?: undefined }
   | { action: 'updateFile'; payload: Partial<UpdatableFileMetadata> };
 

diff --git a/src/plugins/files/server/file/to_json.ts b/src/plugins/files/server/file/to_json.ts
@@ -10,8 +10,20 @@ import { pickBy } from 'lodash';
 import type { FileMetadata, FileJSON } from '../../common/types';
 
 export function serializeJSON<M = unknown>(attrs: Partial<FileJSON>): Partial<FileMetadata<M>> {
-  const { name, mimeType, size, created, updated, fileKind, status, alt, extension, meta, user } =
-    attrs;
+  const {
+    name,
+    mimeType,
+    size,
+    created,
+    updated,
+    fileKind,
+    status,
+    alt,
+    extension,
+    meta,
+    user,
+    hash,
+  } = attrs;
   return pickBy(
     {
       name,
@@ -25,6 +37,7 @@ export function serializeJSON<M = unknown>(attrs: Partial<FileJSON>): Partial<Fi
       Meta: meta,
       Updated: updated,
       FileKind: fileKind,
+      hash,
     },
     (v) => v != null
   );
@@ -43,6 +56,7 @@ export function toJSON<M = unknown>(id: string, attrs: FileMetadata<M>): FileJSO
     Alt,
     extension,
     Meta,
+    hash,
   } = attrs;
   return pickBy<FileJSON<M>>(
     {
@@ -58,6 +72,7 @@ export function toJSON<M = unknown>(id: string, attrs: FileMetadata<M>): FileJSO
       meta: Meta,
       updated: Updated,
       fileKind: FileKind,
+      hash,
     },
     (v) => v != null
   ) as FileJSON<M>;

diff --git a/src/plugins/files/server/file_client/file_client.ts b/src/plugins/files/server/file_client/file_client.ts
@@ -208,7 +208,7 @@ export class FileClientImpl implements FileClient {
 
   /**
    * Upload a blob
-   * @param id - The ID of the file content is associated with
+   * @param file - The file Record that the content is associated with
    * @param rs - The readable stream of the file content
    * @param options - Options for the upload
    */