Merge branch 'main' into poc/background-task-worker-utilization

package.json

@@ -97,7 +97,7 @@
  "@elastic/datemath": "5.0.3",
  "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@8.6.0-canary.3",
  "@elastic/ems-client": "8.4.0",
- "@elastic/eui": "77.1.1",
+ "@elastic/eui": "77.1.2",
  "@elastic/filesaver": "1.1.2",
  "@elastic/node-crypto": "1.2.1",
  "@elastic/numeral": "^2.5.1",

packages/core/chrome/core-chrome-browser-internal/src/ui/header/collapsible_nav.scss

@@ -1,9 +1,29 @@
+$screenHeightBreakpoint: $euiSize * 15;
+
+.kbnCollapsibleNav {
+ @media (max-height: $screenHeightBreakpoint) {
+ overflow-y: auto;
+ }
+}
+
 .kbnCollapsibleNav__recentsListGroup {
  @include euiYScroll;
  max-height: $euiSize * 10;
  margin-right: -$euiSizeS;
 }
 
+.kbnCollapsibleNav__solutions {
+ @include euiYScroll;
+
+ /**
+ * Allows the solutions nav group to be viewed on
+ * very small screen sizes and when the browser Zoom is high
+ */
+ @media (max-height: $screenHeightBreakpoint) {
+ flex: 1 0 auto;
+ }
+}
+
 /**
  * 1. Increase the hit area of the link (anchor)
  * 2. Only show the text underline when hovering on the text/anchor portion

packages/core/chrome/core-chrome-browser-internal/src/ui/header/collapsible_nav.tsx

@@ -161,6 +161,7 @@ export function CollapsibleNav({
  button={button}
  ownFocus={false}
  size={248}
+ className="kbnCollapsibleNav"
  >
  {customNavLink && (
  <>
@@ -280,7 +281,7 @@ export function CollapsibleNav({
 
  <EuiHorizontalRule margin="none" />
 
- <EuiFlexItem className="eui-yScroll">
+ <EuiFlexItem className="kbnCollapsibleNav__solutions">
  {/* Kibana, Observability, Security, and Management sections */}
  {orderedCategories.map((categoryName) => {
  const category = categoryDictionary[categoryName]!;

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -511,6 +511,10 @@
  },
  "FileKind": {
  "type": "keyword"
+ },
+ "hash": {
+ "dynamic": false,
+ "properties": {}
  }
  }
  },
@@ -2728,10 +2732,6 @@
  "dynamic": false,
  "properties": {}
  },
- "metrics-explorer-view": {
- "dynamic": false,
- "properties": {}
- },
  "inventory-view": {
  "dynamic": false,
  "properties": {}
@@ -2744,6 +2744,10 @@
  }
  }
  },
+ "metrics-explorer-view": {
+ "dynamic": false,
+ "properties": {}
+ },
  "upgrade-assistant-reindex-operation": {
  "dynamic": false,
  "properties": {

packages/shared-ux/file/types/index.ts

@@ -226,6 +226,10 @@ export interface FileJSON<Meta = unknown> {
  * User data associated with this file
  */
  user?: FileMetadata['user'];
+ /**
+ * File hash information
+ */
+ hash?: BaseFileMetadata['hash'];
 }
 
 export interface FileKindBase {

src/core/public/styles/_base.scss

@@ -34,12 +34,3 @@
 .euiPageHeader--bottomBorder:not(.euiPageHeader--tabsAtBottom):not([class*='euiPageHeader--padding']) {
  padding-bottom: $euiSizeL;
 }
-
-// Kibana's body ignores the `margin-right !important` set by react-remove-scroll-bar
-// (used by EUI's focus trap component & prevents page width jumps on full-screen overlays)
-// due to the 100% width/min-width CSS set by Kibana in other places. To work around this, we
-// grab the `--removed-body-scroll-bar-size` var added by the library & manually set `padding`
-// TODO: Use `gapMode` instead once https://github.com/theKashey/react-focus-on/issues/65 is fixed
-.kbnBody {
- padding-right: var(--removed-body-scroll-bar-size, 0);
-}

src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts

@@ -88,7 +88,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
  "event_loop_delays_daily": "ef49e7f15649b551b458c7ea170f3ed17f89abd0",
  "exception-list": "38181294f64fc406c15f20d85ca306c8a4feb3c0",
  "exception-list-agnostic": "d527ce9d12b134cb163150057b87529043a8ec77",
- "file": "d12998f49bc82da596a9e6c8397999930187ec6a",
+ "file": "487a562dd895407307980cc4404ca08e87e8999d",
  "file-upload-usage-collection-telemetry": "c6fcb9a7efcf19b2bb66ca6e005bfee8961f6073",
  "fileShare": "f07d346acbb724eacf139a0fb781c38dc5280115",
  "fleet-fleet-server-host": "67180a54a689111fb46403c3603c9b3a329c698d",

src/dev/license_checker/config.ts

@@ -85,6 +85,6 @@ export const LICENSE_OVERRIDES = {
  'jsts@1.6.2': ['Eclipse Distribution License - v 1.0'], // cf. https://github.com/bjornharrtell/jsts
  '@mapbox/jsonlint-lines-primitives@2.0.2': ['MIT'], // license in readme https://github.com/tmcw/jsonlint
  '@elastic/ems-client@8.4.0': ['Elastic License 2.0'],
- '@elastic/eui@77.1.1': ['SSPL-1.0 OR Elastic License 2.0'],
+ '@elastic/eui@77.1.2': ['SSPL-1.0 OR Elastic License 2.0'],
  'language-subtag-registry@0.3.21': ['CC-BY-4.0'], // retired ODC‑By license https://github.com/mattcg/language-subtag-registry
 };

src/plugins/files/common/types.ts

@@ -18,6 +18,7 @@ import type {
  FileKindBase,
  FileShareJSONWithToken,
 } from '@kbn/shared-ux-file-types';
+import type { UploadOptions } from '../server/blob_storage_service';
 import type { ES_FIXED_SIZE_INDEX_BLOB_STORE } from './constants';
 
 export type {
@@ -179,7 +180,7 @@ export interface File<Meta = unknown> {
  */
  data: FileJSON<Meta>;
  /**
- * Update a file object's metadatathat can be updated.
+ * Update a file object's metadata that can be updated.
  *
  * @param attr - The of attributes to update.
  */
@@ -190,8 +191,13 @@ export interface File<Meta = unknown> {
  *
  * @param content - The content to stream to storage.
  * @param abort$ - An observable that can be used to abort the upload at any time.
+ * @param options - additional options.
  */
- uploadContent(content: Readable, abort$?: Observable<unknown>): Promise<File<Meta>>;
+ uploadContent(
+ content: Readable,
+ abort$?: Observable<unknown>,
+ options?: Partial<Pick<UploadOptions, 'transforms'>>
+ ): Promise<File<Meta>>;
 
  /**
  * Stream file content from storage.

src/plugins/files/server/file/file.test.ts

@@ -14,11 +14,9 @@ import {
  loggingSystemMock,
  savedObjectsServiceMock,
 } from '@kbn/core/server/mocks';
-import { Readable } from 'stream';
+import { Readable, Transform } from 'stream';
 import { promisify } from 'util';
 
-const setImmediate = promisify(global.setImmediate);
-
 import { BlobStorageService } from '../blob_storage_service';
 import { InternalFileService } from '../file_service/internal_file_service';
 import {
@@ -29,6 +27,10 @@ import {
 import { InternalFileShareService } from '../file_share_service';
 import { FileMetadataClient } from '../file_client';
 import { SavedObjectsFileMetadataClient } from '../file_client/file_metadata_client/adapters/saved_objects';
+import { File as IFile } from '../../common';
+import { createFileHashTransform } from '..';
+
+const setImmediate = promisify(global.setImmediate);
 
 describe('File', () => {
  let esClient: ElasticsearchClient;
@@ -110,4 +112,87 @@ describe('File', () => {
  expect(file.data.status).toBe('UPLOAD_ERROR');
  expect(blobStoreSpy.calledOnce).toBe(true);
  });
+
+ describe('#uploadContent() method', () => {
+ let file: IFile;
+ let fileContent: Readable;
+
+ beforeEach(async () => {
+ const fileSO = { attributes: { Status: 'AWAITING_UPLOAD' } };
+ (soClient.create as jest.Mock).mockResolvedValue(fileSO);
+ (soClient.update as jest.Mock).mockResolvedValue(fileSO);
+ (soClient.get as jest.Mock).mockResolvedValue({
+ attributes: {
+ created: '2023-04-27T19:57:19.640Z',
+ Updated: '2023-04-27T19:57:19.640Z',
+ name: 'test',
+ Status: 'DONE',
+ FileKind: fileKind,
+ hash: {
+ sha256: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
+ },
+ },
+ });
+
+ file = await fileService.createFile({ name: 'test', fileKind });
+ fileContent = Readable.from(['test']);
+ });
+
+ it('should allow custom transforms to be used', async () => {
+ let used = 0;
+ const customTransform = new Transform({
+ transform(chunk, _, next) {
+ used++;
+ next(null, chunk);
+ },
+ });
+
+ let used2 = 0;
+ const customTransform2 = new Transform({
+ transform(chunk, _, next) {
+ used2++;
+ next(null, chunk);
+ },
+ });
+
+ await file.uploadContent(fileContent, undefined, {
+ transforms: [customTransform, customTransform2],
+ });
+
+ expect(used).toBeGreaterThan(0);
+ expect(used2).toBeGreaterThan(0);
+ expect(file.data).toEqual({
+ created: expect.any(String),
+ updated: expect.any(String),
+ fileKind: 'fileKind',
+ size: 4,
+ status: 'READY',
+ });
+ });
+
+ it('should generate and store file hash when FileHashTransform is used', async () => {
+ await file.uploadContent(fileContent, undefined, {
+ transforms: [createFileHashTransform()],
+ });
+
+ expect(file.toJSON()).toEqual({
+ created: expect.any(String),
+ updated: expect.any(String),
+ fileKind: 'fileKind',
+ hash: {
+ sha256: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
+ },
+ size: 4,
+ status: 'READY',
+ });
+ });
+
+ it('should return file hash', async () => {
+ file = await fileService.getById({ id: '1' });
+
+ expect(file.data.hash).toEqual({
+ sha256: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
+ });
+ });
+ });
 });

src/plugins/files/server/file/file.ts

@@ -19,6 +19,8 @@ import {
  Observable,
  lastValueFrom,
 } from 'rxjs';
+import { isFileHashTransform } from '../file_client/stream_transforms/file_hash_transform/file_hash_transform';
+import { UploadOptions } from '../blob_storage_service';
 import type { FileShareJSON, FileShareJSONWithToken } from '../../common/types';
 import type { File as IFile, UpdatableFileMetadata, FileJSON } from '../../common';
 import { fileAttributesReducer, Action } from './file_attributes_reducer';
@@ -70,13 +72,17 @@ export class File<M = unknown> implements IFile {
  return this;
  }
 
- private upload(content: Readable): Observable<{ size: number }> {
- return defer(() => this.fileClient.upload(this.metadata, content));
+ private upload(
+ content: Readable,
+ options?: Partial<Pick<UploadOptions, 'transforms'>>
+ ): Observable<{ size: number }> {
+ return defer(() => this.fileClient.upload(this.metadata, content, options));
  }
 
  public async uploadContent(
  content: Readable,
- abort$: Observable<unknown> = NEVER
+ abort$: Observable<unknown> = NEVER,
+ options?: Partial<Pick<UploadOptions, 'transforms'>>
  ): Promise<IFile<M>> {
  if (this.uploadInProgress()) {
  throw new UploadInProgressError('Upload already in progress.');
@@ -90,7 +96,7 @@ export class File<M = unknown> implements IFile {
  from(this.updateFileState({ action: 'uploading' })).pipe(
  mergeMap(() =>
  race(
- this.upload(content),
+ this.upload(content, options),
  abort$.pipe(
  map(() => {
  throw new AbortedUploadError(`Aborted upload of ${this.id}!`);
@@ -99,7 +105,28 @@ export class File<M = unknown> implements IFile {
  )
  ),
  mergeMap(({ size }) => {
- return this.updateFileState({ action: 'uploaded', payload: { size } });
+ const updatedStateAction: Action & { action: 'uploaded' } = {
+ action: 'uploaded',
+ payload: { size },
+ };
+
+ if (options && options.transforms) {
+ options.transforms.some((transform) => {
+ if (isFileHashTransform(transform)) {
+ const fileHash = transform.getFileHash();
+
+ updatedStateAction.payload.hash = {
+ [fileHash.algorithm]: fileHash.value,
+ };
+
+ return true;
+ }
+
+ return false;
+ });
+ }
+
+ return this.updateFileState(updatedStateAction);
  }),
  catchError(async (e) => {
  try {

src/plugins/files/server/file/file_attributes_reducer.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import { FileHashObj } from '../saved_objects/file';
 import { FileJSON, UpdatableFileMetadata } from '../../common';
 
 export type Action =
@@ -17,7 +18,10 @@ export type Action =
  action: 'uploading';
  payload?: undefined;
  }
- | { action: 'uploaded'; payload: { size: number } }
+ | {
+ action: 'uploaded';
+ payload: { size: number; hash?: FileHashObj };
+ }
  | { action: 'uploadError'; payload?: undefined }
  | { action: 'updateFile'; payload: Partial<UpdatableFileMetadata> };