From 133ea27bd73622d792b6c85cf4370635af62abf2 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 2 Jul 2020 14:12:09 -0400 Subject: [PATCH 01/32] Stateless exception list translation with improved runtime checks --- .../server/endpoint/ingest_integration.ts | 2 + .../server/endpoint/lib/artifacts/lists.ts | 114 ++++++++++-------- .../endpoint/schemas/artifacts/lists.ts | 17 ++- 3 files changed, 81 insertions(+), 52 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index 67a331f4ba677..ace5aec77ed2c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -57,6 +57,8 @@ export const getPackageConfigCreateCallback = ( try { return updatedPackageConfig; } finally { + // TODO: confirm creation of package config + // then commit. await manifestManager.commit(wrappedManifest); } }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 7fd057afdbd55..01909dfc9ebd5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -7,22 +7,20 @@ import { createHash } from 'crypto'; import { validate } from '../../../../common/validate'; -import { - Entry, - EntryNested, - EntryMatch, - EntryMatchAny, -} from '../../../../../lists/common/schemas/types/entries'; +import { Entry, EntryNested } from '../../../../../lists/common/schemas/types/entries'; import { FoundExceptionListItemSchema } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema'; import { ExceptionListClient } from '../../../../../lists/server'; import { InternalArtifactSchema, TranslatedEntry, - TranslatedEntryMatch, - TranslatedEntryMatchAny, - TranslatedEntryNested, WrappedTranslatedExceptionList, wrappedExceptionList, + TranslatedEntryNestedEntry, + translatedEntryNestedEntry, + translatedEntry, + TranslatedEntryMatcher, + translatedEntryMatchMatcher, + translatedEntryMatchAnyMatcher, } from '../../schemas'; import { ArtifactConstants } from './common'; @@ -92,66 +90,80 @@ export function translateToEndpointExceptions( exc: FoundExceptionListItemSchema, schemaVersion: string ): TranslatedEntry[] { - const translatedList: TranslatedEntry[] = []; - if (schemaVersion === '1.0.0') { - exc.data.forEach((list) => { - list.entries.forEach((entry) => { - const tEntry = translateEntry(schemaVersion, entry); - if (tEntry !== undefined) { - translatedList.push(tEntry); + return exc.data + .map( + (list): Array => { + return list.entries.map((entry: Entry | EntryNested): TranslatedEntry | undefined => { + return translateEntry(schemaVersion, entry); + }); } + ) + .reduce((acc, it) => [...acc, ...it], []) + .filter((entry: TranslatedEntry | undefined): entry is TranslatedEntry => { + return entry !== undefined && translatedEntry.is(entry); }); - }); } else { throw new Error('unsupported schemaVersion'); } - return translatedList; +} + +function getMatcherFunction(field: string, any?: boolean): TranslatedEntryMatcher { + return any + ? field.endsWith('.text') + ? 'exact_caseless_any' + : 'exact_cased_any' + : field.endsWith('.text') + ? 'exact_caseless' + : 'exact_cased'; +} + +function normalizeFieldName(field: string): string { + return field.endsWith('.text') ? field.substring(0, field.length - 5) : field; } function translateEntry( schemaVersion: string, entry: Entry | EntryNested ): TranslatedEntry | undefined { - let translatedEntry; switch (entry.type) { case 'nested': { - const e = (entry as unknown) as EntryNested; - const nestedEntries: TranslatedEntry[] = []; - for (const nestedEntry of e.entries) { - const translation = translateEntry(schemaVersion, nestedEntry); - if (translation !== undefined) { - nestedEntries.push(translation); - } - } - translatedEntry = { + const nestedEntries = entry.entries + .map((nestedEntry): TranslatedEntry | undefined => { + return translateEntry(schemaVersion, nestedEntry); + }) + .filter( + (nestedEntry: TranslatedEntry | undefined): nestedEntry is TranslatedEntryNestedEntry => { + return nestedEntry !== undefined && translatedEntryNestedEntry.is(nestedEntry); + } + ); + return { entries: nestedEntries, - field: e.field, + field: entry.field, type: 'nested', - } as TranslatedEntryNested; - break; + }; } case 'match': { - const e = (entry as unknown) as EntryMatch; - translatedEntry = { - field: e.field.endsWith('.text') ? e.field.substring(0, e.field.length - 5) : e.field, - operator: e.operator, - type: e.field.endsWith('.text') ? 'exact_caseless' : 'exact_cased', - value: e.value, - } as TranslatedEntryMatch; - break; + const matcher = getMatcherFunction(entry.field); + return translatedEntryMatchMatcher.is(matcher) + ? { + field: normalizeFieldName(entry.field), + operator: entry.operator, + type: matcher, + value: entry.value, + } + : undefined; + } + case 'match_any': { + const matcher = getMatcherFunction(entry.field, true); + return translatedEntryMatchAnyMatcher.is(matcher) + ? { + field: normalizeFieldName(entry.field), + operator: entry.operator, + type: matcher, + value: entry.value, + } + : undefined; } - case 'match_any': - { - const e = (entry as unknown) as EntryMatchAny; - translatedEntry = { - field: e.field.endsWith('.text') ? e.field.substring(0, e.field.length - 5) : e.field, - operator: e.operator, - type: e.field.endsWith('.text') ? 'exact_caseless_any' : 'exact_cased_any', - value: e.value, - } as TranslatedEntryMatchAny; - } - break; } - return translatedEntry || undefined; } diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts index 21d1105a313e7..d071896c537bf 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts @@ -20,6 +20,9 @@ export const translatedEntryMatchAny = t.exact( ); export type TranslatedEntryMatchAny = t.TypeOf; +export const translatedEntryMatchAnyMatcher = translatedEntryMatchAny.type.props.type; +export type TranslatedEntryMatchAnyMatcher = t.TypeOf; + export const translatedEntryMatch = t.exact( t.type({ field: t.string, @@ -33,11 +36,23 @@ export const translatedEntryMatch = t.exact( ); export type TranslatedEntryMatch = t.TypeOf; +export const translatedEntryMatchMatcher = translatedEntryMatch.type.props.type; +export type TranslatedEntryMatchMatcher = t.TypeOf; + +export const translatedEntryMatcher = t.union([ + translatedEntryMatchMatcher, + translatedEntryMatchAnyMatcher, +]); +export type TranslatedEntryMatcher = t.TypeOf; + +export const translatedEntryNestedEntry = t.union([translatedEntryMatch, translatedEntryMatchAny]); +export type TranslatedEntryNestedEntry = t.TypeOf; + export const translatedEntryNested = t.exact( t.type({ field: t.string, type: t.keyof({ nested: null }), - entries: t.array(t.union([translatedEntryMatch, translatedEntryMatchAny])), + entries: t.array(translatedEntryNestedEntry), }) ); export type TranslatedEntryNested = t.TypeOf; From 9e43633a746912d8aafdec58829cb8c73aff08ff Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 2 Jul 2020 22:43:33 -0400 Subject: [PATCH 02/32] use flatMap and reduce to simplify logic --- .../server/endpoint/lib/artifacts/lists.ts | 42 +++++++++---------- .../manifest_manager/manifest_manager.ts | 15 ++++--- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 01909dfc9ebd5..31a2fbd71a3df 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -17,7 +17,7 @@ import { wrappedExceptionList, TranslatedEntryNestedEntry, translatedEntryNestedEntry, - translatedEntry, + translatedEntry as translatedEntryType, TranslatedEntryMatcher, translatedEntryMatchMatcher, translatedEntryMatchAnyMatcher, @@ -92,24 +92,23 @@ export function translateToEndpointExceptions( ): TranslatedEntry[] { if (schemaVersion === '1.0.0') { return exc.data - .map( - (list): Array => { - return list.entries.map((entry: Entry | EntryNested): TranslatedEntry | undefined => { - return translateEntry(schemaVersion, entry); - }); + .flatMap((list) => { + return list.entries; + }) + .reduce((entries: TranslatedEntry[], entry) => { + const translatedEntry = translateEntry(schemaVersion, entry); + if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) { + entries.push(translatedEntry); } - ) - .reduce((acc, it) => [...acc, ...it], []) - .filter((entry: TranslatedEntry | undefined): entry is TranslatedEntry => { - return entry !== undefined && translatedEntry.is(entry); - }); + return entries; + }, []); } else { throw new Error('unsupported schemaVersion'); } } -function getMatcherFunction(field: string, any?: boolean): TranslatedEntryMatcher { - return any +function getMatcherFunction(field: string, matchAny?: boolean): TranslatedEntryMatcher { + return matchAny ? field.endsWith('.text') ? 'exact_caseless_any' : 'exact_cased_any' @@ -128,15 +127,16 @@ function translateEntry( ): TranslatedEntry | undefined { switch (entry.type) { case 'nested': { - const nestedEntries = entry.entries - .map((nestedEntry): TranslatedEntry | undefined => { - return translateEntry(schemaVersion, nestedEntry); - }) - .filter( - (nestedEntry: TranslatedEntry | undefined): nestedEntry is TranslatedEntryNestedEntry => { - return nestedEntry !== undefined && translatedEntryNestedEntry.is(nestedEntry); + const nestedEntries = entry.entries.reduce( + (entries: TranslatedEntryNestedEntry[], nestedEntry) => { + const translatedEntry = translateEntry(schemaVersion, nestedEntry); + if (nestedEntry !== undefined && translatedEntryNestedEntry.is(translatedEntry)) { + entries.push(translatedEntry); } - ); + return entries; + }, + [] + ); return { entries: nestedEntries, field: entry.field, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 33b0d5db575c6..e47a23b893b71 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -180,17 +180,15 @@ export class ManifestManager { this.logger.info(`Dispatching new manifest with diffs: ${showDiffs(wrappedManifest.diffs)}`); let paging = true; + let page = 1; let success = true; while (paging) { - const { items, total, page } = await this.packageConfigService.list( - this.savedObjectsClient, - { - page: 1, - perPage: 20, - kuery: 'ingest-package-configs.package.name:endpoint', - } - ); + const { items, total } = await this.packageConfigService.list(this.savedObjectsClient, { + page, + perPage: 20, + kuery: 'ingest-package-configs.package.name:endpoint', + }); for (const packageConfig of items) { const { id, revision, updated_at, updated_by, ...newPackageConfig } = packageConfig; @@ -222,6 +220,7 @@ export class ManifestManager { } paging = page * items.length < total; + page++; } return success ? wrappedManifest : null; From e9f74b436f5b0bd9478c99fe0aca9343b893f6a9 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sat, 4 Jul 2020 12:48:44 -0400 Subject: [PATCH 03/32] Update to new manifest format --- .../common/endpoint/schema/common.ts | 13 ++++++-- .../common/endpoint/schema/manifest.ts | 21 +++++++++--- .../server/endpoint/lib/artifacts/common.ts | 4 +-- .../server/endpoint/lib/artifacts/lists.ts | 9 ++++-- .../endpoint/lib/artifacts/manifest.test.ts | 32 +++++++++++++------ .../lib/artifacts/manifest_entry.test.ts | 18 ++++++++--- .../endpoint/lib/artifacts/manifest_entry.ts | 30 +++++++++++------ .../lib/artifacts/saved_object_mappings.ts | 24 ++++++++++---- .../server/endpoint/lib/artifacts/task.ts | 2 +- .../schemas/artifacts/saved_objects.ts | 19 ++++++++--- .../manifest_manager/manifest_manager.test.ts | 10 ++++-- .../endpoint/artifacts/api_feature/data.json | 17 ++++++---- 12 files changed, 142 insertions(+), 57 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts index 7f8c938d54feb..fdb2570314cd0 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts @@ -6,6 +6,15 @@ import * as t from 'io-ts'; +export const compressionAlgorithm = t.keyof({ + none: null, + zlib: null, +}); + +export const encryptionAlgorithm = t.keyof({ + none: null, +}); + export const identifier = t.string; export const manifestVersion = t.string; @@ -15,8 +24,8 @@ export const manifestSchemaVersion = t.keyof({ }); export type ManifestSchemaVersion = t.TypeOf; +export const relativeUrl = t.string; + export const sha256 = t.string; export const size = t.number; - -export const url = t.string; diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts index 470e9b13ef78a..2f03895d91c74 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts @@ -5,13 +5,26 @@ */ import * as t from 'io-ts'; -import { identifier, manifestSchemaVersion, manifestVersion, sha256, size, url } from './common'; +import { + compressionAlgorithm, + encryptionAlgorithm, + identifier, + manifestSchemaVersion, + manifestVersion, + relativeUrl, + sha256, + size, +} from './common'; export const manifestEntrySchema = t.exact( t.type({ - url, - sha256, - size, + relative_url: relativeUrl, + precompress_sha256: sha256, + precompress_size: size, + postcompress_sha256: sha256, + postcompress_size: size, + compression_algorithm: compressionAlgorithm, + encryption_algorithm: encryptionAlgorithm, }) ); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts index 4c3153ca0ef11..b6a5bed9078ab 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -6,12 +6,12 @@ export const ArtifactConstants = { GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist', - SAVED_OBJECT_TYPE: 'endpoint:exceptions-artifact', + SAVED_OBJECT_TYPE: 'endpoint:user-artifact', SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'], SCHEMA_VERSION: '1.0.0', }; export const ManifestConstants = { - SAVED_OBJECT_TYPE: 'endpoint:exceptions-manifest', + SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest', SCHEMA_VERSION: '1.0.0', }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 31a2fbd71a3df..2abb72234fecd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -34,11 +34,14 @@ export async function buildArtifact( return { identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, - sha256, - encoding: 'application/json', + compressionAlgorithm: 'none', + encryptionAlgorithm: 'none', + decompressedSha256: sha256, + compressedSha256: sha256, + decompressedSize: exceptionsBuffer.byteLength, + compressedSize: exceptionsBuffer.byteLength, created: Date.now(), body: exceptionsBuffer.toString('base64'), - size: exceptionsBuffer.byteLength, }; } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index 0434e3d8ffcb2..da8a449e1b026 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -55,21 +55,33 @@ describe('manifest', () => { expect(manifest1.toEndpointFormat()).toStrictEqual({ artifacts: { 'endpoint-exceptionlist-linux-1.0.0': { - sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - size: 268, - url: + compression_algorithm: 'none', + encryption_algorithm: 'none', + precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + precompress_size: 268, + postcompress_size: 268, + relative_url: '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', }, 'endpoint-exceptionlist-macos-1.0.0': { - sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - size: 268, - url: + compression_algorithm: 'none', + encryption_algorithm: 'none', + precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + precompress_size: 268, + postcompress_size: 268, + relative_url: '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', }, 'endpoint-exceptionlist-windows-1.0.0': { - sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - size: 268, - url: + compression_algorithm: 'none', + encryption_algorithm: 'none', + precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + precompress_size: 268, + postcompress_size: 268, + relative_url: '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', }, }, @@ -107,7 +119,7 @@ describe('manifest', () => { test('Manifest returns data for given artifact', async () => { const artifact = artifacts[0]; - const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.sha256}`); + const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.compressedSha256}`); expect(returned).toEqual(artifact); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts index 34bd2b0f388e1..c8cbdfc2fc5f4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -33,13 +33,17 @@ describe('manifest_entry', () => { }); test('Correct sha256 is returned', () => { - expect(manifestEntry.getSha256()).toEqual( + expect(manifestEntry.getCompressedSha256()).toEqual( + '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ); + expect(manifestEntry.getDecompressedSha256()).toEqual( '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' ); }); test('Correct size is returned', () => { - expect(manifestEntry.getSize()).toEqual(268); + expect(manifestEntry.getCompressedSize()).toEqual(268); + expect(manifestEntry.getDecompressedSize()).toEqual(268); }); test('Correct url is returned', () => { @@ -54,9 +58,13 @@ describe('manifest_entry', () => { test('Correct record is returned', () => { expect(manifestEntry.getRecord()).toEqual({ - sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - size: 268, - url: + compression_algorithm: 'none', + encryption_algorithm: 'none', + precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + precompress_size: 268, + postcompress_size: 268, + relative_url: '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts index 00fd446bf14b5..860c2d7d704b2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts @@ -15,23 +15,31 @@ export class ManifestEntry { } public getDocId(): string { - return `${this.getIdentifier()}-${this.getSha256()}`; + return `${this.getIdentifier()}-${this.getCompressedSha256()}`; } public getIdentifier(): string { return this.artifact.identifier; } - public getSha256(): string { - return this.artifact.sha256; + public getCompressedSha256(): string { + return this.artifact.compressedSha256; } - public getSize(): number { - return this.artifact.size; + public getDecompressedSha256(): string { + return this.artifact.decompressedSha256; + } + + public getCompressedSize(): number { + return this.artifact.compressedSize; + } + + public getDecompressedSize(): number { + return this.artifact.decompressedSize; } public getUrl(): string { - return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getSha256()}`; + return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getCompressedSha256()}`; } public getArtifact(): InternalArtifactSchema { @@ -40,9 +48,13 @@ export class ManifestEntry { public getRecord(): ManifestEntrySchema { return { - sha256: this.getSha256(), - size: this.getSize(), - url: this.getUrl(), + compression_algorithm: 'none', + encryption_algorithm: 'none', + precompress_sha256: this.getDecompressedSha256(), + precompress_size: this.getDecompressedSize(), + postcompress_sha256: this.getCompressedSha256(), + postcompress_size: this.getCompressedSize(), + relative_url: this.getUrl(), }; } } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts index d38026fbcbbd9..5e61b278e87e4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts @@ -16,11 +16,27 @@ export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] identifier: { type: 'keyword', }, - sha256: { + compressionAlgorithm: { type: 'keyword', + index: false, + }, + encryptionAlgorithm: { + type: 'keyword', + index: false, }, - encoding: { + compressedSha256: { type: 'keyword', + }, + compressedSize: { + type: 'long', + index: false, + }, + decompressedSha256: { + type: 'keyword', + index: false, + }, + decompressedSize: { + type: 'long', index: false, }, created: { @@ -31,10 +47,6 @@ export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] type: 'binary', index: false, }, - size: { - type: 'long', - index: false, - }, }, }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index 08d02e70dac16..78b60e9e61f3e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -14,7 +14,7 @@ import { EndpointAppContext } from '../../types'; export const ManifestTaskConstants = { TIMEOUT: '1m', - TYPE: 'securitySolution:endpoint:exceptions-packager', + TYPE: 'endpoint:user-artifact-packager', VERSION: '1.0.0', }; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts index 2e71ef98387f1..fe032586dda56 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts @@ -5,17 +5,26 @@ */ import * as t from 'io-ts'; -import { identifier, sha256, size } from '../../../../common/endpoint/schema/common'; -import { body, created, encoding } from './common'; +import { + compressionAlgorithm, + encryptionAlgorithm, + identifier, + sha256, + size, +} from '../../../../common/endpoint/schema/common'; +import { body, created } from './common'; export const internalArtifactSchema = t.exact( t.type({ identifier, - sha256, - encoding, + compressionAlgorithm, + encryptionAlgorithm, + decompressedSha256: sha256, + decompressedSize: size, + compressedSha256: sha256, + compressedSize: size, created, body, - size, }) ); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index bbb6fdfd50810..ef4f921cb537e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -38,9 +38,13 @@ describe('manifest_manager', () => { schema_version: '1.0.0', artifacts: { [artifact.identifier]: { - sha256: artifact.sha256, - size: artifact.size, - url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.sha256}`, + compression_algorithm: 'none', + encryption_algorithm: 'none', + precompress_sha256: artifact.decompressedSha256, + postcompress_sha256: artifact.compressedSha256, + precompress_size: artifact.decompressedSize, + postcompress_size: artifact.compressedSize, + relative_url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.compressedSha256}`, }, }, }); diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json index a886b60e7e0dc..178d04a7b5030 100644 --- a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -1,7 +1,7 @@ { "type": "doc", "value": { - "id": "endpoint:exceptions-artifact:endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "id": "endpoint:user-artifact:endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", "index": ".kibana", "source": { "references": [ @@ -9,12 +9,15 @@ "endpoint:exceptions-artifact": { "body": "eyJleGNlcHRpb25zX2xpc3QiOltdfQ==", "created": 1593016187465, - "encoding": "application/json", + "compressionAlgorithm": "none", + "encryptionAlgorithm": "none", "identifier": "endpoint-exceptionlist-linux-1.0.0", - "sha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", - "size": 22 + "compressedSha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "compressedSize": 22, + "decompressedSha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "decompressedSize": 22 }, - "type": "endpoint:exceptions-artifact", + "type": "endpoint:user-artifact", "updated_at": "2020-06-24T16:29:47.584Z" } } @@ -23,7 +26,7 @@ { "type": "doc", "value": { - "id": "endpoint:exceptions-manifest:endpoint-manifest-1.0.0", + "id": "endpoint:user-artifact-manifest:endpoint-manifest-1.0.0", "index": ".kibana", "source": { "references": [ @@ -36,7 +39,7 @@ "endpoint-exceptionlist-windows-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d" ] }, - "type": "endpoint:exceptions-manifest", + "type": "endpoint:user-artifact-manifest", "updated_at": "2020-06-26T15:01:39.704Z" } } From ceca5fa4c38259d7e7537d9243d237075e910e14 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sat, 4 Jul 2020 13:30:16 -0400 Subject: [PATCH 04/32] Fix test fixture SO data type --- .../es_archives/endpoint/artifacts/api_feature/data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json index 178d04a7b5030..3696f8f5a8b31 100644 --- a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -6,7 +6,7 @@ "source": { "references": [ ], - "endpoint:exceptions-artifact": { + "endpoint:user-artifact": { "body": "eyJleGNlcHRpb25zX2xpc3QiOltdfQ==", "created": 1593016187465, "compressionAlgorithm": "none", From 49d22d2044f0088df371fce91549d1ec6b426b20 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sat, 4 Jul 2020 13:55:02 -0400 Subject: [PATCH 05/32] Fix another test fixture data type --- .../es_archives/endpoint/artifacts/api_feature/data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json index 3696f8f5a8b31..b156f2f6cc7bf 100644 --- a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -31,7 +31,7 @@ "source": { "references": [ ], - "endpoint:exceptions-manifest": { + "endpoint:user-artifact-manifest": { "created": 1593183699663, "ids": [ "endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", From 9428ca55a61872c0ab9d9c3fc26616d15c0b263f Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sat, 4 Jul 2020 16:06:41 -0400 Subject: [PATCH 06/32] Fix sha256 reference in artifact_client --- .../server/endpoint/services/artifacts/artifact_client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts index 4a3dcaae1bd3d..00ae802ba6f32 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts @@ -16,7 +16,7 @@ export class ArtifactClient { } public getArtifactId(artifact: InternalArtifactSchema) { - return `${artifact.identifier}-${artifact.sha256}`; + return `${artifact.identifier}-${artifact.compressedSha256}`; } public async getArtifact(id: string): Promise> { From c3766387b30f45d005d0d05720b0cd3af0278faf Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sun, 5 Jul 2020 13:48:35 -0400 Subject: [PATCH 07/32] Refactor to remove usages of 'then' and tidy up a bit --- .../server/endpoint/ingest_integration.ts | 31 +- .../server/endpoint/lib/artifacts/task.ts | 32 +- .../manifest_manager/manifest_manager.test.ts | 30 +- .../manifest_manager/manifest_manager.ts | 310 +++++++++--------- 4 files changed, 223 insertions(+), 180 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index ace5aec77ed2c..8d1540f881d0e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -27,8 +27,19 @@ export const getPackageConfigCreateCallback = ( // follow the types/schema expected let updatedPackageConfig = newPackageConfig as NewPolicyData; - const wrappedManifest = await manifestManager.refresh({ initialize: true }); - if (wrappedManifest !== null) { + // get snapshot based on exception-list-agnostic SOs + // with diffs from last dispatched manifest, if it exists + const snapshot = await manifestManager.getSnapshot({ initialize: true }); + + if (snapshot === null) { + // TODO: log error... should not be in this state + return updatedPackageConfig; + } + + if (snapshot.diffs.length > 0) { + // create new artifacts + await manifestManager.syncArtifacts(snapshot, 'add'); + // Until we get the Default Policy Configuration in the Endpoint package, // we will add it here manually at creation time. // @ts-ignore @@ -42,7 +53,7 @@ export const getPackageConfigCreateCallback = ( streams: [], config: { artifact_manifest: { - value: wrappedManifest.manifest.toEndpointFormat(), + value: snapshot.manifest.toEndpointFormat(), }, policy: { value: policyConfigFactory(), @@ -57,9 +68,17 @@ export const getPackageConfigCreateCallback = ( try { return updatedPackageConfig; } finally { - // TODO: confirm creation of package config - // then commit. - await manifestManager.commit(wrappedManifest); + if (snapshot.diffs.length > 0) { + const created = await manifestManager.confirmPackageConfigExists(updatedPackageConfig.name); + if (created) { + await manifestManager.commit(snapshot.manifest); + + // clean up old artifacts + await manifestManager.syncArtifacts(snapshot, 'delete'); + } else { + // TODO: log error + } + } } }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index 78b60e9e61f3e..aa7f56e815d58 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -88,20 +88,22 @@ export class ManifestTask { return; } - manifestManager - .refresh() - .then((wrappedManifest) => { - if (wrappedManifest) { - return manifestManager.dispatch(wrappedManifest); - } - }) - .then((wrappedManifest) => { - if (wrappedManifest) { - return manifestManager.commit(wrappedManifest); - } - }) - .catch((err) => { - this.logger.error(err); - }); + try { + // get snapshot based on exception-list-agnostic SOs + // with diffs from last dispatched manifest + const snapshot = await manifestManager.getSnapshot(); + if (snapshot && snapshot.diffs.length > 0) { + // create new artifacts + await manifestManager.syncArtifacts(snapshot, 'add'); + // write to ingest-manager package config + await manifestManager.dispatch(snapshot.manifest); + // commit latest manifest state to user-artifact-manifest SO + await manifestManager.commit(snapshot.manifest); + // clean up old artifacts + await manifestManager.syncArtifacts(snapshot, 'delete'); + } + } catch (err) { + this.logger.error(err); + } }; } diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index ef4f921cb537e..ac04079efd6fb 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -10,26 +10,26 @@ import { getPackageConfigServiceMock, getManifestManagerMock } from './manifest_ describe('manifest_manager', () => { describe('ManifestManager sanity checks', () => { - test('ManifestManager can refresh manifest', async () => { + test('ManifestManager can snapshot manifest', async () => { const manifestManager = getManifestManagerMock(); - const manifestWrapper = await manifestManager.refresh(); - expect(manifestWrapper!.diffs).toEqual([ + const snapshot = await manifestManager.getSnapshot(); + expect(snapshot!.diffs).toEqual([ { id: 'endpoint-exceptionlist-linux-1.0.0-d34a1f6659bd86fc2023d7477aa2e5d2055c9c0fb0a0f10fae76bf8b94bebe49', type: 'add', }, ]); - expect(manifestWrapper!.manifest).toBeInstanceOf(Manifest); + expect(snapshot!.manifest).toBeInstanceOf(Manifest); }); test('ManifestManager can dispatch manifest', async () => { const packageConfigService = getPackageConfigServiceMock(); const manifestManager = getManifestManagerMock({ packageConfigService }); - const manifestWrapperRefresh = await manifestManager.refresh(); - const manifestWrapperDispatch = await manifestManager.dispatch(manifestWrapperRefresh); - expect(manifestWrapperRefresh).toEqual(manifestWrapperDispatch); - const entries = manifestWrapperDispatch!.manifest.getEntries(); + const snapshot = await manifestManager.getSnapshot(); + const dispatched = await manifestManager.dispatch(snapshot!.manifest); + expect(dispatched).toEqual(true); + const entries = snapshot!.manifest.getEntries(); const artifact = Object.values(entries)[0].getArtifact(); expect( packageConfigService.update.mock.calls[0][2].inputs[0].config.artifact_manifest.value @@ -56,15 +56,21 @@ describe('manifest_manager', () => { savedObjectsClient, }); - const manifestWrapperRefresh = await manifestManager.refresh(); - const manifestWrapperDispatch = await manifestManager.dispatch(manifestWrapperRefresh); + const snapshot = await manifestManager.getSnapshot(); + await manifestManager.syncArtifacts(snapshot!, 'add'); + const diff = { id: 'abcd', type: 'delete', }; - manifestWrapperDispatch!.diffs.push(diff); + snapshot!.diffs.push(diff); + + const dispatched = await manifestManager.dispatch(snapshot!.manifest); + expect(dispatched).toEqual(true); + + await manifestManager.commit(snapshot!.manifest); - await manifestManager.commit(manifestWrapperDispatch); + await manifestManager.syncArtifacts(snapshot!, 'delete'); // created new artifact expect(savedObjectsClient.create.mock.calls[0][0]).toEqual( diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index e47a23b893b71..c7c291f5513f0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Logger, SavedObjectsClientContract, SavedObject } from 'src/core/server'; +import { Logger, SavedObjectsClientContract } from 'src/core/server'; import { PackageConfigServiceInterface } from '../../../../../../ingest_manager/server'; import { ExceptionListClient } from '../../../../../../lists/server'; import { ManifestSchemaVersion } from '../../../../../common/endpoint/schema/common'; @@ -17,7 +17,7 @@ import { ExceptionsCache, ManifestDiff, } from '../../../lib/artifacts'; -import { InternalArtifactSchema, InternalManifestSchema } from '../../../schemas/artifacts'; +import { InternalArtifactSchema } from '../../../schemas/artifacts'; import { ArtifactClient } from '../artifact_client'; import { ManifestClient } from '../manifest_client'; @@ -30,11 +30,11 @@ export interface ManifestManagerContext { cache: ExceptionsCache; } -export interface ManifestRefreshOpts { +export interface ManifestSnapshotOpts { initialize?: boolean; } -export interface WrappedManifest { +export interface ManifestSnapshot { manifest: Manifest; diffs: ManifestDiff[]; } @@ -56,58 +56,79 @@ export class ManifestManager { this.cache = context.cache; } - private getManifestClient(schemaVersion: string): ManifestClient { + /** + * Gets a ManifestClient for the provided schemaVersion. + * + * @param schemaVersion + */ + private getManifestClient(schemaVersion: string) { return new ManifestClient(this.savedObjectsClient, schemaVersion as ManifestSchemaVersion); } - private async buildExceptionListArtifacts( - schemaVersion: string - ): Promise { - const artifacts: InternalArtifactSchema[] = []; - - for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { - const exceptionList = await getFullEndpointExceptionList( - this.exceptionListClient, - os, - schemaVersion - ); - const artifact = await buildArtifact(exceptionList, os, schemaVersion); - - artifacts.push(artifact); - } - - return artifacts; + /** + * Builds an array of artifacts (one per supported OS) based on the current + * state of exception-list-agnostic SO's. + * + * @param schemaVersion + */ + private async buildExceptionListArtifacts(schemaVersion: string) { + return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.reduce( + async (acc: Promise, os) => { + const exceptionList = await getFullEndpointExceptionList( + this.exceptionListClient, + os, + schemaVersion + ); + const artifacts = await acc; + const artifact = await buildArtifact(exceptionList, os, schemaVersion); + artifacts.push(artifact); + return Promise.resolve(artifacts); + }, + Promise.resolve([]) + ); } - private async getLastDispatchedManifest(schemaVersion: string): Promise { - return this.getManifestClient(schemaVersion) - .getManifest() - .then(async (manifestSo: SavedObject) => { - if (manifestSo.version === undefined) { - throw new Error('No version returned for manifest.'); - } - const manifest = new Manifest( - new Date(manifestSo.attributes.created), - schemaVersion, - manifestSo.version - ); + /** + * Returns the last dispatched manifest based on the current state of the + * user-artifact-manifest SO. + * + * @param schemaVersion + */ + private async getLastDispatchedManifest(schemaVersion: string) { + try { + const manifestClient = await this.getManifestClient(schemaVersion); + const manifestSo = await manifestClient.getManifest(); - for (const id of manifestSo.attributes.ids) { - const artifactSo = await this.artifactClient.getArtifact(id); - manifest.addEntry(artifactSo.attributes); - } + if (manifestSo.version === undefined) { + throw new Error('No version returned for manifest.'); + } - return manifest; - }) - .catch((err) => { - if (err.output.statusCode !== 404) { - throw err; - } - return null; + const manifest = new Manifest( + new Date(manifestSo.attributes.created), + schemaVersion, + manifestSo.version + ); + + manifestSo.attributes.ids.map(async (id) => { + const artifactSo = await this.artifactClient.getArtifact(id); + manifest.addEntry(artifactSo.attributes); }); + + return manifest; + } catch (err) { + if (err.output.statusCode !== 404) { + throw err; + } + return null; + } } - public async refresh(opts?: ManifestRefreshOpts): Promise { + /** + * Snapshots a manifest based on current state of exception-list-agnostic SOs. + * + * @param opts TODO + */ + public async getSnapshot(opts?: ManifestSnapshotOpts) { let oldManifest: Manifest | null; // Get the last-dispatched manifest @@ -133,25 +154,6 @@ export class ManifestManager { // Get diffs const diffs = newManifest.diff(oldManifest); - // Create new artifacts - for (const diff of diffs) { - if (diff.type === 'add') { - const artifact = newManifest.getArtifact(diff.id); - try { - await this.artifactClient.createArtifact(artifact); - // Cache the body of the artifact - this.cache.set(diff.id, artifact.body); - } catch (err) { - if (err.status === 409) { - // This artifact already existed... - this.logger.debug(`Tried to create artifact ${diff.id}, but it already exists.`); - } else { - throw err; - } - } - } - } - return { manifest: newManifest, diffs, @@ -159,111 +161,125 @@ export class ManifestManager { } /** - * Dispatches the manifest by writing it to the endpoint packageConfig. + * Syncs artifacts based on provided snapshot. + * + * Creates artifacts that do not yet exist and cleans up old artifacts that have been + * superceded by this snapshot. * - * @return {WrappedManifest | null} WrappedManifest if all dispatched, else null + * Can be filtered to apply one or both operations. + * + * @param snapshot + * @param diffType */ - public async dispatch(wrappedManifest: WrappedManifest | null): Promise { - if (wrappedManifest === null) { - this.logger.debug('wrappedManifest was null, aborting dispatch'); - return null; - } + public async syncArtifacts(snapshot: ManifestSnapshot, diffType?: 'add' | 'delete') { + snapshot.diffs.map(async (diff) => { + try { + if (diff.type === 'add' && (diffType === undefined || diffType === 'add')) { + const artifact = snapshot.manifest.getArtifact(diff.id); + await this.artifactClient.createArtifact(artifact); + // Cache the body of the artifact + this.cache.set(diff.id, artifact.body); + } else if (diff.type === 'delete' && (diffType === undefined || diffType === 'delete')) { + await this.artifactClient.deleteArtifact(diff.id); + // TODO: should we delete the cache entry here? + this.logger.info(`Cleaned up artifact ${diff.id}`); + } else if (!['add', 'delete'].includes(diff.type)) { + // TODO: replace with io-ts schema + throw new Error(`Unsupported diff type: ${diff.type}`); + } + } catch (err) { + if (diff.type === 'add' && err.status === 409) { + this.logger.debug(`Tried to create artifact ${diff.id}, but it already exists.`); + } else { + throw err; + } + } + }); + } - function showDiffs(diffs: ManifestDiff[]) { - return diffs.map((diff) => { - const op = diff.type === 'add' ? '(+)' : '(-)'; - return `${op}${diff.id}`; + /** + * Dispatches the manifest by writing it to the endpoint package config. + * + */ + public async dispatch(manifest: Manifest) { + let paging = true; + let page = 1; + let success = true; + + while (paging) { + const { items, total } = await this.packageConfigService.list(this.savedObjectsClient, { + page, + perPage: 20, + kuery: 'ingest-package-configs.package.name:endpoint', }); - } - - if (wrappedManifest.diffs.length > 0) { - this.logger.info(`Dispatching new manifest with diffs: ${showDiffs(wrappedManifest.diffs)}`); - - let paging = true; - let page = 1; - let success = true; - - while (paging) { - const { items, total } = await this.packageConfigService.list(this.savedObjectsClient, { - page, - perPage: 20, - kuery: 'ingest-package-configs.package.name:endpoint', - }); - - for (const packageConfig of items) { - const { id, revision, updated_at, updated_by, ...newPackageConfig } = packageConfig; - if ( - newPackageConfig.inputs.length > 0 && - newPackageConfig.inputs[0].config !== undefined - ) { - const artifactManifest = newPackageConfig.inputs[0].config.artifact_manifest ?? { - value: {}, - }; - artifactManifest.value = wrappedManifest.manifest.toEndpointFormat(); - newPackageConfig.inputs[0].config.artifact_manifest = artifactManifest; - - await this.packageConfigService - .update(this.savedObjectsClient, id, newPackageConfig) - .then((response) => { - this.logger.debug(`Updated package config ${id}`); - }) - .catch((err) => { - success = false; - this.logger.debug(`Error updating package config ${id}`); - this.logger.error(err); - }); - } else { + items.map(async (packageConfig) => { + const { id, revision, updated_at, updated_by, ...newPackageConfig } = packageConfig; + if (newPackageConfig.inputs.length > 0 && newPackageConfig.inputs[0].config !== undefined) { + const artifactManifest = newPackageConfig.inputs[0].config.artifact_manifest ?? { + value: {}, + }; + artifactManifest.value = manifest.toEndpointFormat(); + newPackageConfig.inputs[0].config.artifact_manifest = artifactManifest; + + try { + this.packageConfigService.update(this.savedObjectsClient, id, newPackageConfig); + this.logger.debug( + `Updated package config ${id} with manifest version ${manifest.getVersion()}` + ); + } catch (err) { success = false; - this.logger.debug(`Package config ${id} has no config.`); + this.logger.debug(`Error updating package config ${id}`); + this.logger.error(err); } + } else { + success = false; + this.logger.debug(`Package config ${id} has no config.`); } + }); - paging = page * items.length < total; - page++; - } - - return success ? wrappedManifest : null; - } else { - this.logger.debug('No manifest diffs [no-op]'); + paging = page * items.length < total; + page++; } - return null; + // TODO: revisit success logic + return success; } - public async commit(wrappedManifest: WrappedManifest | null) { - if (wrappedManifest === null) { - this.logger.debug('wrappedManifest was null, aborting commit'); - return; - } - - const manifestClient = this.getManifestClient(wrappedManifest.manifest.getSchemaVersion()); + /** + * Commits a manifest to indicate that it has been dispatched. + * + * @param manifest + */ + public async commit(manifest: Manifest) { + const manifestClient = this.getManifestClient(manifest.getSchemaVersion()); // Commit the new manifest - if (wrappedManifest.manifest.getVersion() === 'v0') { - await manifestClient.createManifest(wrappedManifest.manifest.toSavedObject()); + if (manifest.getVersion() === 'v0') { + await manifestClient.createManifest(manifest.toSavedObject()); } else { - const version = wrappedManifest.manifest.getVersion(); + const version = manifest.getVersion(); if (version === 'v0') { throw new Error('Updating existing manifest with baseline version. Bad state.'); } - await manifestClient.updateManifest(wrappedManifest.manifest.toSavedObject(), { + await manifestClient.updateManifest(manifest.toSavedObject(), { version, }); } - this.logger.info(`Commited manifest ${wrappedManifest.manifest.getVersion()}`); + this.logger.info(`Commited manifest ${manifest.getVersion()}`); + } - // Clean up old artifacts - for (const diff of wrappedManifest.diffs) { - try { - if (diff.type === 'delete') { - await this.artifactClient.deleteArtifact(diff.id); - this.logger.info(`Cleaned up artifact ${diff.id}`); - } - } catch (err) { - this.logger.error(err); - } - } + /** + * Confirms that a packageConfig exists with provided name. + */ + public async confirmPackageConfigExists(name: string) { + // TODO: what if there are multiple results? uh oh. + const { total } = await this.packageConfigService.list(this.savedObjectsClient, { + page: 1, + perPage: 20, + kuery: `ingest-package-configs.name:${name}`, + }); + return total > 0; } } From 13d9ec8c931954896b8c4bd7de560821ecdc7283 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Mon, 6 Jul 2020 15:29:02 -0400 Subject: [PATCH 08/32] Zlib compression --- .../server/endpoint/ingest_integration.ts | 3 ++- .../server/endpoint/lib/artifacts/lists.ts | 14 ++++++++++++++ .../artifacts/manifest_manager/manifest_manager.ts | 10 ++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index 8d1540f881d0e..d1c834158e253 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -69,7 +69,8 @@ export const getPackageConfigCreateCallback = ( return updatedPackageConfig; } finally { if (snapshot.diffs.length > 0) { - const created = await manifestManager.confirmPackageConfigExists(updatedPackageConfig.name); + // const created = await manifestManager.confirmPackageConfigExists(updatedPackageConfig.name); + const created = true; if (created) { await manifestManager.commit(snapshot.manifest); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 2abb72234fecd..5db295ba7d1f7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -5,6 +5,7 @@ */ import { createHash } from 'crypto'; +import { deflateRaw } from 'zlib'; import { validate } from '../../../../common/validate'; import { Entry, EntryNested } from '../../../../../lists/common/schemas/types/entries'; @@ -32,6 +33,7 @@ export async function buildArtifact( const exceptionsBuffer = Buffer.from(JSON.stringify(exceptions)); const sha256 = createHash('sha256').update(exceptionsBuffer.toString()).digest('hex'); + // Keep compression info empty in case its a duplicate. Lazily compress before committing if needed. return { identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, compressionAlgorithm: 'none', @@ -170,3 +172,15 @@ function translateEntry( } } } + +export function compressExceptionList(buffer: Buffer): Promise { + return new Promise((resolve, reject) => { + deflateRaw(buffer, function (err, buf) { + if (err) { + reject(err); + } else { + resolve(buf); + } + }); + }); +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index c7c291f5513f0..779b95bbaff18 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -5,6 +5,7 @@ */ import { Logger, SavedObjectsClientContract } from 'src/core/server'; +import { createHash } from 'crypto'; import { PackageConfigServiceInterface } from '../../../../../../ingest_manager/server'; import { ExceptionListClient } from '../../../../../../lists/server'; import { ManifestSchemaVersion } from '../../../../../common/endpoint/schema/common'; @@ -20,6 +21,7 @@ import { import { InternalArtifactSchema } from '../../../schemas/artifacts'; import { ArtifactClient } from '../artifact_client'; import { ManifestClient } from '../manifest_client'; +import { compressExceptionList } from '../../../lib/artifacts/lists'; export interface ManifestManagerContext { savedObjectsClient: SavedObjectsClientContract; @@ -176,6 +178,14 @@ export class ManifestManager { try { if (diff.type === 'add' && (diffType === undefined || diffType === 'add')) { const artifact = snapshot.manifest.getArtifact(diff.id); + const compressedArtifact = await compressExceptionList( + Buffer.from(artifact.body, 'base64') + ); + artifact.body = compressedArtifact.toString('base64'); + artifact.compressedSize = compressedArtifact.length; + artifact.compressionAlgorithm = 'zlib'; + artifact.compressedSha256 = createHash('sha256').update(compressedArtifact).digest('hex'); + await this.artifactClient.createArtifact(artifact); // Cache the body of the artifact this.cache.set(diff.id, artifact.body); From 47181108c06726829a2ca58e68914ca9531a85ab Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Mon, 6 Jul 2020 18:00:44 -0400 Subject: [PATCH 09/32] prefer byteLength to length --- .../services/artifacts/manifest_manager/manifest_manager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 779b95bbaff18..f15d15551c5c9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -182,7 +182,7 @@ export class ManifestManager { Buffer.from(artifact.body, 'base64') ); artifact.body = compressedArtifact.toString('base64'); - artifact.compressedSize = compressedArtifact.length; + artifact.compressedSize = compressedArtifact.byteLength; artifact.compressionAlgorithm = 'zlib'; artifact.compressedSha256 = createHash('sha256').update(compressedArtifact).digest('hex'); From feb120247df2b1337064946c03d7d22f7d3d7ca1 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 6 Jul 2020 19:21:07 -0400 Subject: [PATCH 10/32] Make ingestManager optional for security-solution startup --- x-pack/plugins/security_solution/kibana.json | 2 +- .../endpoint_app_context_services.test.ts | 4 +- .../endpoint/endpoint_app_context_services.ts | 16 ++-- .../server/endpoint/routes/metadata/index.ts | 76 ++++++++++--------- .../endpoint/routes/metadata/metadata.test.ts | 7 +- .../security_solution/server/plugin.ts | 40 ++++++---- 6 files changed, 80 insertions(+), 65 deletions(-) diff --git a/x-pack/plugins/security_solution/kibana.json b/x-pack/plugins/security_solution/kibana.json index f6f2d5171312c..745df54c62158 100644 --- a/x-pack/plugins/security_solution/kibana.json +++ b/x-pack/plugins/security_solution/kibana.json @@ -11,7 +11,6 @@ "embeddable", "features", "home", - "ingestManager", "taskManager", "inspector", "licensing", @@ -21,6 +20,7 @@ ], "optionalPlugins": [ "encryptedSavedObjects", + "ingestManager", "ml", "newsfeed", "security", diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts index 2daf259941cbf..22ea0896fec81 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts @@ -8,11 +8,11 @@ import { httpServerMock } from '../../../../../src/core/server/mocks'; import { EndpointAppContextService } from './endpoint_app_context_services'; describe('test endpoint app context services', () => { - it('should throw error on getAgentService if start is not called', async () => { + it('should return undefined on getAgentService if dependencies are not enabled', async () => { const endpointAppContextService = new EndpointAppContextService(); expect(() => endpointAppContextService.getAgentService()).toThrow(Error); }); - it('should return undefined on getManifestManager if start is not called', async () => { + it('should return undefined on getManifestManager if dependencies are not enabled', async () => { const endpointAppContextService = new EndpointAppContextService(); expect(endpointAppContextService.getManifestManager()).toEqual(undefined); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index 97a82049634c4..5d3fa24e8f51c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -12,12 +12,11 @@ import { AgentService, IngestManagerStartContract } from '../../../ingest_manage import { getPackageConfigCreateCallback } from './ingest_integration'; import { ManifestManager } from './services/artifacts'; -export type EndpointAppContextServiceStartContract = Pick< - IngestManagerStartContract, - 'agentService' +export type EndpointAppContextServiceStartContract = Partial< + Pick > & { - manifestManager?: ManifestManager | undefined; - registerIngestCallback: IngestManagerStartContract['registerExternalCallback']; + manifestManager?: ManifestManager; + registerIngestCallback?: IngestManagerStartContract['registerExternalCallback']; savedObjectsStart: SavedObjectsServiceStart; }; @@ -35,7 +34,7 @@ export class EndpointAppContextService { this.manifestManager = dependencies.manifestManager; this.savedObjectsStart = dependencies.savedObjectsStart; - if (this.manifestManager !== undefined) { + if (this.manifestManager && dependencies.registerIngestCallback) { dependencies.registerIngestCallback( 'packageConfigCreate', getPackageConfigCreateCallback(this.manifestManager) @@ -45,10 +44,7 @@ export class EndpointAppContextService { public stop() {} - public getAgentService(): AgentService { - if (!this.agentService) { - throw new Error(`must call start on ${EndpointAppContextService.name} to call getter`); - } + public getAgentService(): AgentService | undefined { return this.agentService; } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts index 235e7152b83cf..45bd3544304dd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts @@ -18,6 +18,7 @@ import { HostStatus, } from '../../../../common/endpoint/types'; import { EndpointAppContext } from '../../types'; +import { AgentService } from '../../../../../ingest_manager/server'; import { Agent, AgentStatus } from '../../../../../ingest_manager/common/types/models'; import { findAllUnenrolledAgentIds } from './support/unenroll'; @@ -26,8 +27,9 @@ interface HitSource { } interface MetadataRequestContext { + agentService: AgentService; + logger: Logger; requestHandlerContext: RequestHandlerContext; - endpointAppContext: EndpointAppContext; } const HOST_STATUS_MAPPING = new Map([ @@ -35,8 +37,12 @@ const HOST_STATUS_MAPPING = new Map([ ['offline', HostStatus.OFFLINE], ]); +const getLogger = (endpointAppContext: EndpointAppContext): Logger => { + return endpointAppContext.logFactory.get('metadata'); +}; + export function registerEndpointRoutes(router: IRouter, endpointAppContext: EndpointAppContext) { - const logger = endpointAppContext.logFactory.get('metadata'); + const logger = getLogger(endpointAppContext); router.post( { path: '/api/endpoint/metadata', @@ -66,12 +72,23 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp }) ), }, - options: { authRequired: true }, + options: { authRequired: true, tags: ['access:securitySolution'] }, }, async (context, req, res) => { + const agentService = endpointAppContext.service.getAgentService(); + if (agentService === undefined) { + return res.internalError({ body: 'agentService not available' }); + } + + const metadataRequestContext: MetadataRequestContext = { + agentService, + logger, + requestHandlerContext: context, + }; + try { const unenrolledAgentIds = await findAllUnenrolledAgentIds( - endpointAppContext.service.getAgentService(), + agentService, context.core.savedObjects.client ); @@ -89,10 +106,7 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp queryParams )) as SearchResponse; return res.ok({ - body: await mapToHostResultList(queryParams, response, { - endpointAppContext, - requestHandlerContext: context, - }), + body: await mapToHostResultList(queryParams, response, metadataRequestContext), }); } catch (err) { logger.warn(JSON.stringify(err, null, 2)); @@ -107,17 +121,16 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp validate: { params: schema.object({ id: schema.string() }), }, - options: { authRequired: true }, + options: { authRequired: true, tags: ['access:securitySolution'] }, }, async (context, req, res) => { + const agentService = endpointAppContext.service.getAgentService(); + if (agentService === undefined) { + return res.internalError({ body: 'agentService not available' }); + } + try { - const doc = await getHostData( - { - endpointAppContext, - requestHandlerContext: context, - }, - req.params.id - ); + const doc = await getHostData(context, agentService, req.params.id); if (doc) { return res.ok({ body: doc }); } @@ -164,17 +177,16 @@ async function findAgent( metadataRequestContext: MetadataRequestContext, hostMetadata: HostMetadata ): Promise { - const logger = metadataRequestContext.endpointAppContext.logFactory.get('metadata'); try { - return await metadataRequestContext.endpointAppContext.service - .getAgentService() - .getAgent( - metadataRequestContext.requestHandlerContext.core.savedObjects.client, - hostMetadata.elastic.agent.id - ); + return await metadataRequestContext.agentService.getAgent( + metadataRequestContext.requestHandlerContext.core.savedObjects.client, + hostMetadata.elastic.agent.id + ); } catch (e) { if (e.isBoom && e.output.statusCode === 404) { - logger.warn(`agent with id ${hostMetadata.elastic.agent.id} not found`); + metadataRequestContext.logger.warn( + `agent with id ${hostMetadata.elastic.agent.id} not found` + ); return undefined; } else { throw e; @@ -217,7 +229,7 @@ async function enrichHostMetadata( ): Promise { let hostStatus = HostStatus.ERROR; let elasticAgentId = hostMetadata?.elastic?.agent?.id; - const log = logger(metadataRequestContext.endpointAppContext); + const log = metadataRequestContext.logger; try { /** * Get agent status by elastic agent id if available or use the host id. @@ -228,12 +240,10 @@ async function enrichHostMetadata( log.warn(`Missing elastic agent id, using host id instead ${elasticAgentId}`); } - const status = await metadataRequestContext.endpointAppContext.service - .getAgentService() - .getAgentStatusById( - metadataRequestContext.requestHandlerContext.core.savedObjects.client, - elasticAgentId - ); + const status = await metadataRequestContext.agentService.getAgentStatusById( + metadataRequestContext.requestHandlerContext.core.savedObjects.client, + elasticAgentId + ); hostStatus = HOST_STATUS_MAPPING.get(status) || HostStatus.ERROR; } catch (e) { if (e.isBoom && e.output.statusCode === 404) { @@ -248,7 +258,3 @@ async function enrichHostMetadata( host_status: hostStatus, }; } - -const logger = (endpointAppContext: EndpointAppContext): Logger => { - return endpointAppContext.logFactory.get('metadata'); -}; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts index 668911b8d1f29..4a1b0f024474b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts @@ -47,8 +47,9 @@ describe('test endpoint route', () => { let routeHandler: RequestHandler; // eslint-disable-next-line @typescript-eslint/no-explicit-any let routeConfig: RouteConfig; - let mockAgentService: ReturnType< - typeof createMockEndpointAppContextServiceStartContract + // tests assume that ingestManager is enabled, and thus agentService is available + let mockAgentService: Required< + ReturnType >['agentService']; let endpointAppContextService: EndpointAppContextService; const noUnenrolledAgent = { @@ -70,7 +71,7 @@ describe('test endpoint route', () => { endpointAppContextService = new EndpointAppContextService(); const startContract = createMockEndpointAppContextServiceStartContract(); endpointAppContextService.start(startContract); - mockAgentService = startContract.agentService; + mockAgentService = startContract.agentService!; registerEndpointRoutes(routerMock, { logFactory: loggingSystemMock.create(), diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index a97f1eee56342..ddc32f2959fbb 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -24,7 +24,7 @@ import { ListPluginSetup } from '../../lists/server'; import { EncryptedSavedObjectsPluginSetup as EncryptedSavedObjectsSetup } from '../../encrypted_saved_objects/server'; import { SpacesPluginSetup as SpacesSetup } from '../../spaces/server'; import { LicensingPluginSetup } from '../../licensing/server'; -import { IngestManagerStartContract } from '../../ingest_manager/server'; +import { IngestManagerStartContract, ExternalCallback } from '../../ingest_manager/server'; import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server'; import { initServer } from './init_server'; import { compose } from './lib/compose/kibana'; @@ -55,14 +55,14 @@ export interface SetupPlugins { licensing: LicensingPluginSetup; security?: SecuritySetup; spaces?: SpacesSetup; - taskManager: TaskManagerSetupContract; + taskManager?: TaskManagerSetupContract; ml?: MlSetup; lists?: ListPluginSetup; } export interface StartPlugins { - ingestManager: IngestManagerStartContract; - taskManager: TaskManagerStartContract; + ingestManager?: IngestManagerStartContract; + taskManager?: TaskManagerStartContract; } // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -224,11 +224,15 @@ export class Plugin implements IPlugin { + return plugins.taskManager && plugins.lists; + }; + + if (exceptionListsSetupEnabled()) { this.lists = plugins.lists; this.manifestTask = new ManifestTask({ endpointAppContext: endpointContext, - taskManager: plugins.taskManager, + taskManager: plugins.taskManager!, }); } @@ -242,32 +246,40 @@ export class Plugin implements IPlugin void) | undefined; + + const exceptionListsStartEnabled = () => { + return this.lists && plugins.taskManager && plugins.ingestManager; + }; + + if (exceptionListsStartEnabled()) { + const exceptionListClient = this.lists!.getExceptionListClient(savedObjectsClient, 'kibana'); const artifactClient = new ArtifactClient(savedObjectsClient); + + registerIngestCallback = plugins.ingestManager!.registerExternalCallback; manifestManager = new ManifestManager({ savedObjectsClient, artifactClient, exceptionListClient, - packageConfigService: plugins.ingestManager.packageConfigService, + packageConfigService: plugins.ingestManager!.packageConfigService, logger: this.logger, cache: this.exceptionsCache, }); } this.endpointAppContextService.start({ - agentService: plugins.ingestManager.agentService, + agentService: plugins.ingestManager?.agentService, manifestManager, - registerIngestCallback: plugins.ingestManager.registerExternalCallback, + registerIngestCallback, savedObjectsStart: core.savedObjects, }); - if (this.manifestTask) { + if (exceptionListsStartEnabled() && this.manifestTask) { this.manifestTask.start({ - taskManager: plugins.taskManager, + taskManager: plugins.taskManager!, }); } else { - this.logger.debug('Manifest task not available.'); + this.logger.debug('User artifacts task not available.'); } return {}; From aecf7181c14423a8eb674d60f8d9dd2544b921b4 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 6 Jul 2020 23:40:47 -0400 Subject: [PATCH 11/32] Fix download functionality --- .../server/endpoint/lib/artifacts/cache.ts | 6 +-- .../artifacts/download_exception_list.test.ts | 14 +++--- .../artifacts/download_exception_list.ts | 9 ++-- .../endpoint/schemas/artifacts/common.ts | 2 +- .../response/download_artifact_schema.ts | 3 +- .../apis/endpoint/artifacts/index.ts | 49 ++++++++++++++++++- .../endpoint/artifacts/api_feature/data.json | 16 +++--- 7 files changed, 73 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts index b7a4c2feb6bf8..b9d3bae4e6ef9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts @@ -10,7 +10,7 @@ const DEFAULT_MAX_SIZE = 10; * FIFO cache implementation for artifact downloads. */ export class ExceptionsCache { - private cache: Map; + private cache: Map; private queue: string[]; private maxSize: number; @@ -20,7 +20,7 @@ export class ExceptionsCache { this.maxSize = maxSize || DEFAULT_MAX_SIZE; } - set(id: string, body: string) { + set(id: string, body: Buffer) { if (this.queue.length + 1 > this.maxSize) { const entry = this.queue.shift(); if (entry !== undefined) { @@ -31,7 +31,7 @@ export class ExceptionsCache { this.cache.set(id, body); } - get(id: string): string | undefined { + get(id: string): Buffer | undefined { return this.cache.get(id); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 540976134d8ae..c85021ab12018 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -77,7 +77,6 @@ describe('test alerts route', () => { let mockScopedClient: jest.Mocked; let mockSavedObjectClient: jest.Mocked; let mockResponse: jest.Mocked; - // @ts-ignore let routeConfig: RouteConfig; let routeHandler: RequestHandler; let endpointAppContextService: EndpointAppContextService; @@ -98,8 +97,9 @@ describe('test alerts route', () => { // The authentication with the Fleet Plugin needs a separate scoped SO Client ingestSavedObjectClient = savedObjectsClientMock.create(); ingestSavedObjectClient.find.mockReturnValue(Promise.resolve(mockIngestSOResponse)); - // @ts-ignore - startContract.savedObjectsStart.getScopedClient.mockReturnValue(ingestSavedObjectClient); + (startContract.savedObjectsStart.getScopedClient as jest.Mock).mockReturnValue( + ingestSavedObjectClient + ); endpointAppContextService.start(startContract); registerDownloadExceptionListRoute( @@ -147,6 +147,8 @@ describe('test alerts route', () => { path.startsWith('/api/endpoint/artifacts/download') )!; + expect(routeConfig.options).toEqual({}); + await routeHandler( ({ core: { @@ -160,8 +162,8 @@ describe('test alerts route', () => { ); const expectedHeaders = { - 'content-encoding': 'application/json', - 'content-disposition': `attachment; filename=${mockArtifactName}.json`, + 'content-encoding': 'identity', + 'content-disposition': `attachment; filename=${mockArtifactName}.zz`, }; expect(mockResponse.ok).toBeCalled(); @@ -217,7 +219,7 @@ describe('test alerts route', () => { // Add to the download cache const mockArtifact = expectedEndpointExceptions; const cacheKey = `${mockArtifactName}-${mockSha}`; - cache.set(cacheKey, JSON.stringify(mockArtifact)); + cache.set(cacheKey, Buffer.from(JSON.stringify(mockArtifact))); // TODO: add compression here [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => path.startsWith('/api/endpoint/artifacts/download') diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 337393e768a8f..e432f916bc496 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -45,7 +45,6 @@ export function registerDownloadExceptionListRoute( }, options: { tags: [] }, }, - // @ts-ignore async (context, req, res) => { let scopedSOClient: SavedObjectsClientContract; const logger = endpointContext.logFactory.get('download_exception_list'); @@ -62,12 +61,12 @@ export function registerDownloadExceptionListRoute( } } - const buildAndValidateResponse = (artName: string, body: string): IKibanaResponse => { + const buildAndValidateResponse = (artName: string, body: Buffer): IKibanaResponse => { const artifact: HttpResponseOptions = { body, headers: { - 'content-encoding': 'application/json', - 'content-disposition': `attachment; filename=${artName}.json`, + 'content-encoding': 'identity', + 'content-disposition': `attachment; filename=${artName}.zz`, }, }; @@ -90,7 +89,7 @@ export function registerDownloadExceptionListRoute( return scopedSOClient .get(ArtifactConstants.SAVED_OBJECT_TYPE, id) .then((artifact: SavedObject) => { - const body = Buffer.from(artifact.attributes.body, 'base64').toString(); + const body = Buffer.from(artifact.attributes.body, 'base64'); cache.set(id, body); return buildAndValidateResponse(artifact.attributes.identifier, body); }) diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts index 3c066e150288a..8e7fa833931f2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts @@ -11,7 +11,7 @@ export const body = t.string; export const created = t.number; // TODO: Make this into an ISO Date string check export const encoding = t.keyof({ - 'application/json': null, + identity: null, }); export const schemaVersion = t.keyof({ diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts index 537f7707889e4..8b8715c878cc0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts @@ -7,7 +7,6 @@ import * as t from 'io-ts'; import { encoding } from '../common'; -const body = t.string; const headers = t.exact( t.type({ 'content-encoding': encoding, @@ -17,7 +16,7 @@ const headers = t.exact( export const downloadArtifactResponseSchema = t.exact( t.type({ - body, + body: t.unknown, // TODO: make Buffer type headers, }) ); diff --git a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts index 2721592ba3350..29c66831c5833 100644 --- a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -5,7 +5,9 @@ */ import expect from '@kbn/expect'; +import { inflateRawSync } from 'zlib'; +import { WrappedTranslatedExceptionList } from '../../../../../plugins/security_solution/server/endpoint/schemas'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { getSupertestWithoutAuth, setupIngest } from '../../fleet/agents/services'; @@ -71,7 +73,7 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with correct hash', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/1be2ae8d19368939c6e9ecfb32d46233e975feec9611469f62dd2578bd9c4507' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) @@ -79,6 +81,51 @@ export default function (providerContext: FtrProviderContext) { .expect(200); }); + it('should download valid compressed JSON', async () => { + const { body } = await supertestWithoutAuth + .get( + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/1be2ae8d19368939c6e9ecfb32d46233e975feec9611469f62dd2578bd9c4507' + ) + .set('kbn-xsrf', 'xxx') + .set('authorization', `ApiKey ${agentAccessAPIKey}`) + .send() + .expect(200); + + const decompressedJson = inflateRawSync(body); + const decompressedBody: WrappedTranslatedExceptionList = JSON.parse( + decompressedJson.toString() + ); + + expect(decompressedBody).to.equal({ + exceptions_list: [ + { + field: 'actingProcess.file.signer', + operator: 'included', + type: 'exact_cased', + value: 'Elastic, N.V.', + }, + { + entries: [ + { + field: 'signer', + operator: 'included', + type: 'exact_cased', + value: 'Evil', + }, + { + field: 'trusted', + operator: 'included', + type: 'exact_cased', + value: 'true', + }, + ], + field: 'file.signature', + type: 'nested', + }, + ], + } as WrappedTranslatedExceptionList); + }); + it('should fail on invalid api key', async () => { await supertestWithoutAuth .get( diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json index b156f2f6cc7bf..99b186a888765 100644 --- a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -1,21 +1,21 @@ { "type": "doc", "value": { - "id": "endpoint:user-artifact:endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "id": "endpoint:user-artifact:endpoint-exceptionlist-linux-1.0.0-1be2ae8d19368939c6e9ecfb32d46233e975feec9611469f62dd2578bd9c4507", "index": ".kibana", "source": { "references": [ ], "endpoint:user-artifact": { - "body": "eyJleGNlcHRpb25zX2xpc3QiOltdfQ==", + "body": "pZDBCsIwEER/RfYc8gG9exVPXqSUkG7LwpKU7KZUSv7dGLDiVa+PmTcwO+DmcVGKQQYmUejuO0yEPEIHziuF+ZqiRxE7EaMVmgMmMBAXTE5jqjEKnvOIY6X6WLAS3Gp18E4aXB3nFz2zEyVvThd7s1DMDhg0EcrX6F8LK3ETv2WasmiL/GCrZYTSm8N2POA0J/zUA7aR0pcn", "created": 1593016187465, - "compressionAlgorithm": "none", + "compressionAlgorithm": "zlib", "encryptionAlgorithm": "none", "identifier": "endpoint-exceptionlist-linux-1.0.0", - "compressedSha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", - "compressedSize": 22, - "decompressedSha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", - "decompressedSize": 22 + "compressedSha256": "1be2ae8d19368939c6e9ecfb32d46233e975feec9611469f62dd2578bd9c4507", + "compressedSize": 153, + "decompressedSha256": "d162f0302cbf419038ade7ea978e0a7ade7aad317fedefe455ff38dfa28b7cff", + "decompressedSize": 336 }, "type": "endpoint:user-artifact", "updated_at": "2020-06-24T16:29:47.584Z" @@ -34,7 +34,7 @@ "endpoint:user-artifact-manifest": { "created": 1593183699663, "ids": [ - "endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "endpoint-exceptionlist-linux-1.0.0-1be2ae8d19368939c6e9ecfb32d46233e975feec9611469f62dd2578bd9c4507", "endpoint-exceptionlist-macos-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", "endpoint-exceptionlist-windows-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d" ] From 56886adcf187a7026ba342a8c29cded0b3835033 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 6 Jul 2020 23:59:19 -0400 Subject: [PATCH 12/32] Use eql for deep equality check --- x-pack/test/api_integration/apis/endpoint/artifacts/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts index 29c66831c5833..9e68da0ae92dd 100644 --- a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -96,7 +96,7 @@ export default function (providerContext: FtrProviderContext) { decompressedJson.toString() ); - expect(decompressedBody).to.equal({ + expect(decompressedBody).to.eql({ exceptions_list: [ { field: 'actingProcess.file.signer', From 03e75c9f25dd666871333c50b88751a217cc4e13 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 7 Jul 2020 00:09:11 -0400 Subject: [PATCH 13/32] Fix base64 download bug --- .../manifest_manager/manifest_manager.ts | 2 +- .../apis/endpoint/artifacts/index.ts | 35 ++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index e47a23b893b71..52887a38b3419 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -140,7 +140,7 @@ export class ManifestManager { try { await this.artifactClient.createArtifact(artifact); // Cache the body of the artifact - this.cache.set(diff.id, artifact.body); + this.cache.set(diff.id, Buffer.from(artifact.body, 'base64').toString()); } catch (err) { if (err.status === 409) { // This artifact already existed... diff --git a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts index 2721592ba3350..a52521b424d49 100644 --- a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -6,6 +6,7 @@ import expect from '@kbn/expect'; +import { WrappedTranslatedExceptionList } from '../../../../../plugins/security_solution/server/endpoint/schemas'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { getSupertestWithoutAuth, setupIngest } from '../../fleet/agents/services'; @@ -69,7 +70,7 @@ export default function (providerContext: FtrProviderContext) { }); it('should download an artifact with correct hash', async () => { - await supertestWithoutAuth + const { body } = await supertestWithoutAuth .get( '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d' ) @@ -77,6 +78,38 @@ export default function (providerContext: FtrProviderContext) { .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() .expect(200); + + // console.log(body); + + // const artifactObj: WrappedTranslatedExceptionList = JSON.parse(body); + expect(body).to.eql({ + exceptions_list: [ + { + field: 'actingProcess.file.signer', + operator: 'included', + type: 'exact_cased', + value: 'Elastic, N.V.', + }, + { + entries: [ + { + field: 'signer', + operator: 'included', + type: 'exact_cased', + value: 'Evil', + }, + { + field: 'trusted', + operator: 'included', + type: 'exact_cased', + value: 'true', + }, + ], + field: 'file.signature', + type: 'nested', + }, + ], + } as WrappedTranslatedExceptionList); }); it('should fail on invalid api key', async () => { From 86205107a534b0ee300eeea137eba63e97a97cfa Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 7 Jul 2020 12:37:03 -0400 Subject: [PATCH 14/32] Add test for artifact download --- .../apis/endpoint/artifacts/index.ts | 56 +++++++++---------- .../endpoint/artifacts/api_feature/data.json | 12 ++-- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts index a52521b424d49..a63c2c19dc96c 100644 --- a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -6,7 +6,6 @@ import expect from '@kbn/expect'; -import { WrappedTranslatedExceptionList } from '../../../../../plugins/security_solution/server/endpoint/schemas'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { getSupertestWithoutAuth, setupIngest } from '../../fleet/agents/services'; @@ -70,46 +69,45 @@ export default function (providerContext: FtrProviderContext) { }); it('should download an artifact with correct hash', async () => { - const { body } = await supertestWithoutAuth + await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d162f0302cbf419038ade7ea978e0a7ade7aad317fedefe455ff38dfa28b7cff' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() - .expect(200); - - // console.log(body); - - // const artifactObj: WrappedTranslatedExceptionList = JSON.parse(body); - expect(body).to.eql({ - exceptions_list: [ - { - field: 'actingProcess.file.signer', - operator: 'included', - type: 'exact_cased', - value: 'Elastic, N.V.', - }, - { - entries: [ + .expect(200) + .expect((response) => { + const artifactJson = JSON.parse(response.text); + expect(artifactJson).to.eql({ + exceptions_list: [ { - field: 'signer', + field: 'actingProcess.file.signer', operator: 'included', type: 'exact_cased', - value: 'Evil', + value: 'Elastic, N.V.', }, { - field: 'trusted', - operator: 'included', - type: 'exact_cased', - value: 'true', + entries: [ + { + field: 'signer', + operator: 'included', + type: 'exact_cased', + value: 'Evil', + }, + { + field: 'trusted', + operator: 'included', + type: 'exact_cased', + value: 'true', + }, + ], + field: 'file.signature', + type: 'nested', }, ], - field: 'file.signature', - type: 'nested', - }, - ], - } as WrappedTranslatedExceptionList); + }); + }); }); it('should fail on invalid api key', async () => { diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json index b156f2f6cc7bf..abc7c463ec96d 100644 --- a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -1,21 +1,21 @@ { "type": "doc", "value": { - "id": "endpoint:user-artifact:endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "id": "endpoint:user-artifact:endpoint-exceptionlist-linux-1.0.0-d162f0302cbf419038ade7ea978e0a7ade7aad317fedefe455ff38dfa28b7cff", "index": ".kibana", "source": { "references": [ ], "endpoint:user-artifact": { - "body": "eyJleGNlcHRpb25zX2xpc3QiOltdfQ==", + "body": "eyJleGNlcHRpb25zX2xpc3QiOlt7ImZpZWxkIjoiYWN0aW5nUHJvY2Vzcy5maWxlLnNpZ25lciIsIm9wZXJhdG9yIjoiaW5jbHVkZWQiLCJ0eXBlIjoiZXhhY3RfY2FzZWQiLCJ2YWx1ZSI6IkVsYXN0aWMsIE4uVi4ifSx7ImVudHJpZXMiOlt7ImZpZWxkIjoic2lnbmVyIiwib3BlcmF0b3IiOiJpbmNsdWRlZCIsInR5cGUiOiJleGFjdF9jYXNlZCIsInZhbHVlIjoiRXZpbCJ9LHsiZmllbGQiOiJ0cnVzdGVkIiwib3BlcmF0b3IiOiJpbmNsdWRlZCIsInR5cGUiOiJleGFjdF9jYXNlZCIsInZhbHVlIjoidHJ1ZSJ9XSwiZmllbGQiOiJmaWxlLnNpZ25hdHVyZSIsInR5cGUiOiJuZXN0ZWQifV19", "created": 1593016187465, "compressionAlgorithm": "none", "encryptionAlgorithm": "none", "identifier": "endpoint-exceptionlist-linux-1.0.0", - "compressedSha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", - "compressedSize": 22, - "decompressedSha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", - "decompressedSize": 22 + "compressedSha256": "d162f0302cbf419038ade7ea978e0a7ade7aad317fedefe455ff38dfa28b7cff", + "compressedSize": 336, + "decompressedSha256": "d162f0302cbf419038ade7ea978e0a7ade7aad317fedefe455ff38dfa28b7cff", + "decompressedSize": 336 }, "type": "endpoint:user-artifact", "updated_at": "2020-06-24T16:29:47.584Z" From 1456697298c6fdbb4c9f4205379f1b9a6a429e29 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 7 Jul 2020 13:19:46 -0400 Subject: [PATCH 15/32] Add more tests to ensure cached versions of artifacts are correct --- .../manifest_manager/manifest_manager.mock.ts | 8 ++- .../manifest_manager/manifest_manager.test.ts | 44 ++++++++++++++- .../manifest_manager/manifest_manager.ts | 1 + .../apis/endpoint/artifacts/index.ts | 55 +++++++++++++++++++ 4 files changed, 106 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index cd70b11aef305..483b3434d63f2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -84,9 +84,15 @@ export class ManifestManagerMock extends ManifestManager { } export const getManifestManagerMock = (opts?: { + cache?: ExceptionsCache; packageConfigService?: PackageConfigServiceMock; savedObjectsClient?: ReturnType; }): ManifestManagerMock => { + let cache = new ExceptionsCache(5); + if (opts?.cache !== undefined) { + cache = opts.cache; + } + let packageConfigService = getPackageConfigServiceMock(); if (opts?.packageConfigService !== undefined) { packageConfigService = opts.packageConfigService; @@ -99,7 +105,7 @@ export const getManifestManagerMock = (opts?: { const manifestManager = new ManifestManagerMock({ artifactClient: getArtifactClientMock(savedObjectsClient), - cache: new ExceptionsCache(5), + cache, // @ts-ignore packageConfigService, exceptionListClient: listMock.getExceptionListClient(), diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index ef4f921cb537e..900ad2c026956 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -5,7 +5,12 @@ */ import { savedObjectsClientMock } from 'src/core/server/mocks'; -import { ArtifactConstants, ManifestConstants, Manifest } from '../../../lib/artifacts'; +import { + ArtifactConstants, + ManifestConstants, + Manifest, + ExceptionsCache, +} from '../../../lib/artifacts'; import { getPackageConfigServiceMock, getManifestManagerMock } from './manifest_manager.mock'; describe('manifest_manager', () => { @@ -23,6 +28,43 @@ describe('manifest_manager', () => { expect(manifestWrapper!.manifest).toBeInstanceOf(Manifest); }); + test('ManifestManager populates cache properly', async () => { + const cache = new ExceptionsCache(5); + const manifestManager = getManifestManagerMock({ cache }); + const manifestWrapper = await manifestManager.refresh(); + expect(manifestWrapper!.diffs).toEqual([ + { + id: + 'endpoint-exceptionlist-linux-1.0.0-d34a1f6659bd86fc2023d7477aa2e5d2055c9c0fb0a0f10fae76bf8b94bebe49', + type: 'add', + }, + ]); + const diff = manifestWrapper!.diffs[0]; + const entry = JSON.parse(cache.get(diff!.id)!); + expect(entry).toEqual({ + exceptions_list: [ + { + entries: [ + { + field: 'nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.parentField', + type: 'nested', + }, + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + }); + }); + test('ManifestManager can dispatch manifest', async () => { const packageConfigService = getPackageConfigServiceMock(); const manifestManager = getManifestManagerMock({ packageConfigService }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 52887a38b3419..f7bc711d4bd05 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -139,6 +139,7 @@ export class ManifestManager { const artifact = newManifest.getArtifact(diff.id); try { await this.artifactClient.createArtifact(artifact); + // Cache the body of the artifact this.cache.set(diff.id, Buffer.from(artifact.body, 'base64').toString()); } catch (err) { diff --git a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts index a63c2c19dc96c..3193a4456ba37 100644 --- a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -110,6 +110,61 @@ export default function (providerContext: FtrProviderContext) { }); }); + it('should download an artifact with correct hash from cache', async () => { + await supertestWithoutAuth + .get( + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d162f0302cbf419038ade7ea978e0a7ade7aad317fedefe455ff38dfa28b7cff' + ) + .set('kbn-xsrf', 'xxx') + .set('authorization', `ApiKey ${agentAccessAPIKey}`) + .send() + .expect(200) + .expect((response) => { + JSON.parse(response.text); + }) + .then(async () => { + await supertestWithoutAuth + .get( + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d162f0302cbf419038ade7ea978e0a7ade7aad317fedefe455ff38dfa28b7cff' + ) + .set('kbn-xsrf', 'xxx') + .set('authorization', `ApiKey ${agentAccessAPIKey}`) + .send() + .expect(200) + .expect((response) => { + const artifactJson = JSON.parse(response.text); + expect(artifactJson).to.eql({ + exceptions_list: [ + { + field: 'actingProcess.file.signer', + operator: 'included', + type: 'exact_cased', + value: 'Elastic, N.V.', + }, + { + entries: [ + { + field: 'signer', + operator: 'included', + type: 'exact_cased', + value: 'Evil', + }, + { + field: 'trusted', + operator: 'included', + type: 'exact_cased', + value: 'true', + }, + ], + field: 'file.signature', + type: 'nested', + }, + ], + }); + }); + }); + }); + it('should fail on invalid api key', async () => { await supertestWithoutAuth .get( From eae1b8925b70d3fb279fe8a1c2c2b11c0f5d45e0 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 7 Jul 2020 14:17:05 -0400 Subject: [PATCH 16/32] Convert to new format --- .../common/endpoint/schema/manifest.ts | 8 +++--- .../server/endpoint/lib/artifacts/common.ts | 4 +-- .../endpoint/lib/artifacts/lists.test.ts | 12 ++++---- .../server/endpoint/lib/artifacts/lists.ts | 16 +++++------ .../endpoint/lib/artifacts/manifest.test.ts | 26 ++++++++--------- .../lib/artifacts/manifest_entry.test.ts | 16 +++++------ .../endpoint/lib/artifacts/manifest_entry.ts | 28 +++++++++---------- .../lib/artifacts/saved_object_mappings.ts | 8 +++--- .../endpoint/schemas/artifacts/lists.ts | 6 ++-- .../schemas/artifacts/saved_objects.ts | 8 +++--- .../services/artifacts/artifact_client.ts | 2 +- .../manifest_manager/manifest_manager.test.ts | 16 +++++------ .../apis/endpoint/artifacts/index.ts | 10 +++---- .../endpoint/artifacts/api_feature/data.json | 28 +++++++++---------- 14 files changed, 94 insertions(+), 94 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts index 2f03895d91c74..1c8916dfdd5bb 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts @@ -19,10 +19,10 @@ import { export const manifestEntrySchema = t.exact( t.type({ relative_url: relativeUrl, - precompress_sha256: sha256, - precompress_size: size, - postcompress_sha256: sha256, - postcompress_size: size, + decoded_sha256: sha256, + decoded_size: size, + encoded_sha256: sha256, + encoded_size: size, compression_algorithm: compressionAlgorithm, encryption_algorithm: encryptionAlgorithm, }) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts index b6a5bed9078ab..cf38147522083 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -6,12 +6,12 @@ export const ArtifactConstants = { GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist', - SAVED_OBJECT_TYPE: 'endpoint:user-artifact', + SAVED_OBJECT_TYPE: 'endpoint:user-artifact:v2', SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'], SCHEMA_VERSION: '1.0.0', }; export const ManifestConstants = { - SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest', + SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest:v2', SCHEMA_VERSION: '1.0.0', }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index 738890fb4038f..80d7bd5658363 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -21,7 +21,7 @@ describe('buildEventTypeSignal', () => { test('it should convert the exception lists response to the proper endpoint format', async () => { const expectedEndpointExceptions = { - exceptions_list: [ + entries: [ { entries: [ { @@ -57,7 +57,7 @@ describe('buildEventTypeSignal', () => { ]; const expectedEndpointExceptions = { - exceptions_list: [ + entries: [ { field: 'server.domain', operator: 'included', @@ -100,7 +100,7 @@ describe('buildEventTypeSignal', () => { ]; const expectedEndpointExceptions = { - exceptions_list: [ + entries: [ { field: 'server.domain', operator: 'included', @@ -147,7 +147,7 @@ describe('buildEventTypeSignal', () => { ]; const expectedEndpointExceptions = { - exceptions_list: [ + entries: [ { field: 'server.domain', operator: 'included', @@ -182,7 +182,7 @@ describe('buildEventTypeSignal', () => { .mockReturnValueOnce(second) .mockReturnValueOnce(third); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); - expect(resp.exceptions_list.length).toEqual(6); + expect(resp.entries.length).toEqual(6); }); test('it should handle no exceptions', async () => { @@ -191,6 +191,6 @@ describe('buildEventTypeSignal', () => { exceptionsResponse.total = 0; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(exceptionsResponse); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); - expect(resp.exceptions_list.length).toEqual(0); + expect(resp.entries.length).toEqual(0); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 2abb72234fecd..53cda8f2d2299 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -14,7 +14,7 @@ import { InternalArtifactSchema, TranslatedEntry, WrappedTranslatedExceptionList, - wrappedExceptionList, + wrappedTranslatedExceptionList, TranslatedEntryNestedEntry, translatedEntryNestedEntry, translatedEntry as translatedEntryType, @@ -36,10 +36,10 @@ export async function buildArtifact( identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, compressionAlgorithm: 'none', encryptionAlgorithm: 'none', - decompressedSha256: sha256, - compressedSha256: sha256, - decompressedSize: exceptionsBuffer.byteLength, - compressedSize: exceptionsBuffer.byteLength, + decodedSha256: sha256, + encodedSha256: sha256, + decodedSize: exceptionsBuffer.byteLength, + encodedSize: exceptionsBuffer.byteLength, created: Date.now(), body: exceptionsBuffer.toString('base64'), }; @@ -50,7 +50,7 @@ export async function getFullEndpointExceptionList( os: string, schemaVersion: string ): Promise { - const exceptions: WrappedTranslatedExceptionList = { exceptions_list: [] }; + const exceptions: WrappedTranslatedExceptionList = { entries: [] }; let numResponses = 0; let page = 1; @@ -68,7 +68,7 @@ export async function getFullEndpointExceptionList( if (response?.data !== undefined) { numResponses = response.data.length; - exceptions.exceptions_list = exceptions.exceptions_list.concat( + exceptions.entries = exceptions.entries.concat( translateToEndpointExceptions(response, schemaVersion) ); @@ -78,7 +78,7 @@ export async function getFullEndpointExceptionList( } } while (numResponses > 0); - const [validated, errors] = validate(exceptions, wrappedExceptionList); + const [validated, errors] = validate(exceptions, wrappedTranslatedExceptionList); if (errors != null) { throw new Error(errors); } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index da8a449e1b026..00a651c0b60fe 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -57,30 +57,30 @@ describe('manifest', () => { 'endpoint-exceptionlist-linux-1.0.0': { compression_algorithm: 'none', encryption_algorithm: 'none', - precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - precompress_size: 268, - postcompress_size: 268, + decoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + encoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + decoded_size: 268, + encoded_size: 268, relative_url: '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', }, 'endpoint-exceptionlist-macos-1.0.0': { compression_algorithm: 'none', encryption_algorithm: 'none', - precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - precompress_size: 268, - postcompress_size: 268, + decoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + encoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + decoded_size: 268, + encoded_size: 268, relative_url: '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', }, 'endpoint-exceptionlist-windows-1.0.0': { compression_algorithm: 'none', encryption_algorithm: 'none', - precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - precompress_size: 268, - postcompress_size: 268, + decoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + encoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + decoded_size: 268, + encoded_size: 268, relative_url: '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', }, @@ -119,7 +119,7 @@ describe('manifest', () => { test('Manifest returns data for given artifact', async () => { const artifact = artifacts[0]; - const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.compressedSha256}`); + const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.encodedSha256}`); expect(returned).toEqual(artifact); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts index c8cbdfc2fc5f4..41afd72efd366 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -33,17 +33,17 @@ describe('manifest_entry', () => { }); test('Correct sha256 is returned', () => { - expect(manifestEntry.getCompressedSha256()).toEqual( + expect(manifestEntry.getEncodedSha256()).toEqual( '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' ); - expect(manifestEntry.getDecompressedSha256()).toEqual( + expect(manifestEntry.getDecodedSha256()).toEqual( '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' ); }); test('Correct size is returned', () => { - expect(manifestEntry.getCompressedSize()).toEqual(268); - expect(manifestEntry.getDecompressedSize()).toEqual(268); + expect(manifestEntry.getEncodedSize()).toEqual(268); + expect(manifestEntry.getDecodedSize()).toEqual(268); }); test('Correct url is returned', () => { @@ -60,10 +60,10 @@ describe('manifest_entry', () => { expect(manifestEntry.getRecord()).toEqual({ compression_algorithm: 'none', encryption_algorithm: 'none', - precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - precompress_size: 268, - postcompress_size: 268, + decoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + encoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + decoded_size: 268, + encoded_size: 268, relative_url: '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts index 860c2d7d704b2..c23258c4c3ba4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts @@ -15,31 +15,31 @@ export class ManifestEntry { } public getDocId(): string { - return `${this.getIdentifier()}-${this.getCompressedSha256()}`; + return `${this.getIdentifier()}-${this.getEncodedSha256()}`; } public getIdentifier(): string { return this.artifact.identifier; } - public getCompressedSha256(): string { - return this.artifact.compressedSha256; + public getEncodedSha256(): string { + return this.artifact.encodedSha256; } - public getDecompressedSha256(): string { - return this.artifact.decompressedSha256; + public getDecodedSha256(): string { + return this.artifact.decodedSha256; } - public getCompressedSize(): number { - return this.artifact.compressedSize; + public getEncodedSize(): number { + return this.artifact.encodedSize; } - public getDecompressedSize(): number { - return this.artifact.decompressedSize; + public getDecodedSize(): number { + return this.artifact.decodedSize; } public getUrl(): string { - return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getCompressedSha256()}`; + return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getEncodedSha256()}`; } public getArtifact(): InternalArtifactSchema { @@ -50,10 +50,10 @@ export class ManifestEntry { return { compression_algorithm: 'none', encryption_algorithm: 'none', - precompress_sha256: this.getDecompressedSha256(), - precompress_size: this.getDecompressedSize(), - postcompress_sha256: this.getCompressedSha256(), - postcompress_size: this.getCompressedSize(), + decoded_sha256: this.getDecodedSha256(), + decoded_size: this.getDecodedSize(), + encoded_sha256: this.getEncodedSha256(), + encoded_size: this.getEncodedSize(), relative_url: this.getUrl(), }; } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts index 5e61b278e87e4..89e974a3d5fd3 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts @@ -24,18 +24,18 @@ export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] type: 'keyword', index: false, }, - compressedSha256: { + encodedSha256: { type: 'keyword', }, - compressedSize: { + encodedSize: { type: 'long', index: false, }, - decompressedSha256: { + decodedSha256: { type: 'keyword', index: false, }, - decompressedSize: { + decodedSize: { type: 'long', index: false, }, diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts index d071896c537bf..cdb841a8d4c82 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts @@ -72,9 +72,9 @@ export const translatedExceptionList = t.exact( ); export type TranslatedExceptionList = t.TypeOf; -export const wrappedExceptionList = t.exact( +export const wrappedTranslatedExceptionList = t.exact( t.type({ - exceptions_list: t.array(translatedEntry), + entries: t.array(translatedEntry), }) ); -export type WrappedTranslatedExceptionList = t.TypeOf; +export type WrappedTranslatedExceptionList = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts index fe032586dda56..e4cd7f48a2901 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts @@ -19,10 +19,10 @@ export const internalArtifactSchema = t.exact( identifier, compressionAlgorithm, encryptionAlgorithm, - decompressedSha256: sha256, - decompressedSize: size, - compressedSha256: sha256, - compressedSize: size, + decodedSha256: sha256, + decodedSize: size, + encodedSha256: sha256, + encodedSize: size, created, body, }) diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts index 00ae802ba6f32..e899905602c8d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts @@ -16,7 +16,7 @@ export class ArtifactClient { } public getArtifactId(artifact: InternalArtifactSchema) { - return `${artifact.identifier}-${artifact.compressedSha256}`; + return `${artifact.identifier}-${artifact.encodedSha256}`; } public async getArtifact(id: string): Promise> { diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index 900ad2c026956..e2f22a10c10e3 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -21,7 +21,7 @@ describe('manifest_manager', () => { expect(manifestWrapper!.diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-d34a1f6659bd86fc2023d7477aa2e5d2055c9c0fb0a0f10fae76bf8b94bebe49', + 'endpoint-exceptionlist-linux-1.0.0-2a2ec06c957330deb42f41835d3029001432038106f823173fb9e7ea603decb5', type: 'add', }, ]); @@ -35,14 +35,14 @@ describe('manifest_manager', () => { expect(manifestWrapper!.diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-d34a1f6659bd86fc2023d7477aa2e5d2055c9c0fb0a0f10fae76bf8b94bebe49', + 'endpoint-exceptionlist-linux-1.0.0-2a2ec06c957330deb42f41835d3029001432038106f823173fb9e7ea603decb5', type: 'add', }, ]); const diff = manifestWrapper!.diffs[0]; const entry = JSON.parse(cache.get(diff!.id)!); expect(entry).toEqual({ - exceptions_list: [ + entries: [ { entries: [ { @@ -82,11 +82,11 @@ describe('manifest_manager', () => { [artifact.identifier]: { compression_algorithm: 'none', encryption_algorithm: 'none', - precompress_sha256: artifact.decompressedSha256, - postcompress_sha256: artifact.compressedSha256, - precompress_size: artifact.decompressedSize, - postcompress_size: artifact.compressedSize, - relative_url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.compressedSha256}`, + decoded_sha256: artifact.decodedSha256, + encoded_sha256: artifact.encodedSha256, + decoded_size: artifact.decodedSize, + encoded_size: artifact.encodedSize, + relative_url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.encodedSha256}`, }, }, }); diff --git a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts index 3193a4456ba37..0a801ed237885 100644 --- a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -71,7 +71,7 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with correct hash', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d162f0302cbf419038ade7ea978e0a7ade7aad317fedefe455ff38dfa28b7cff' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/f59266b06ffb1d7250edb9dbabd946e00e98afa950f955d8ea9d8ffef0eb142a' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) @@ -80,7 +80,7 @@ export default function (providerContext: FtrProviderContext) { .expect((response) => { const artifactJson = JSON.parse(response.text); expect(artifactJson).to.eql({ - exceptions_list: [ + entries: [ { field: 'actingProcess.file.signer', operator: 'included', @@ -113,7 +113,7 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with correct hash from cache', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d162f0302cbf419038ade7ea978e0a7ade7aad317fedefe455ff38dfa28b7cff' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/f59266b06ffb1d7250edb9dbabd946e00e98afa950f955d8ea9d8ffef0eb142a' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) @@ -125,7 +125,7 @@ export default function (providerContext: FtrProviderContext) { .then(async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d162f0302cbf419038ade7ea978e0a7ade7aad317fedefe455ff38dfa28b7cff' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/f59266b06ffb1d7250edb9dbabd946e00e98afa950f955d8ea9d8ffef0eb142a' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) @@ -134,7 +134,7 @@ export default function (providerContext: FtrProviderContext) { .expect((response) => { const artifactJson = JSON.parse(response.text); expect(artifactJson).to.eql({ - exceptions_list: [ + entries: [ { field: 'actingProcess.file.signer', operator: 'included', diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json index abc7c463ec96d..565e1d619dda4 100644 --- a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -1,23 +1,23 @@ { "type": "doc", "value": { - "id": "endpoint:user-artifact:endpoint-exceptionlist-linux-1.0.0-d162f0302cbf419038ade7ea978e0a7ade7aad317fedefe455ff38dfa28b7cff", + "id": "endpoint:user-artifact:v2:endpoint-exceptionlist-linux-1.0.0-f59266b06ffb1d7250edb9dbabd946e00e98afa950f955d8ea9d8ffef0eb142a", "index": ".kibana", "source": { "references": [ ], - "endpoint:user-artifact": { - "body": "eyJleGNlcHRpb25zX2xpc3QiOlt7ImZpZWxkIjoiYWN0aW5nUHJvY2Vzcy5maWxlLnNpZ25lciIsIm9wZXJhdG9yIjoiaW5jbHVkZWQiLCJ0eXBlIjoiZXhhY3RfY2FzZWQiLCJ2YWx1ZSI6IkVsYXN0aWMsIE4uVi4ifSx7ImVudHJpZXMiOlt7ImZpZWxkIjoic2lnbmVyIiwib3BlcmF0b3IiOiJpbmNsdWRlZCIsInR5cGUiOiJleGFjdF9jYXNlZCIsInZhbHVlIjoiRXZpbCJ9LHsiZmllbGQiOiJ0cnVzdGVkIiwib3BlcmF0b3IiOiJpbmNsdWRlZCIsInR5cGUiOiJleGFjdF9jYXNlZCIsInZhbHVlIjoidHJ1ZSJ9XSwiZmllbGQiOiJmaWxlLnNpZ25hdHVyZSIsInR5cGUiOiJuZXN0ZWQifV19", + "endpoint:user-artifact:v2": { + "body": "eyJlbnRyaWVzIjpbeyJmaWVsZCI6ImFjdGluZ1Byb2Nlc3MuZmlsZS5zaWduZXIiLCJvcGVyYXRvciI6ImluY2x1ZGVkIiwidHlwZSI6ImV4YWN0X2Nhc2VkIiwidmFsdWUiOiJFbGFzdGljLCBOLlYuIn0seyJlbnRyaWVzIjpbeyJmaWVsZCI6InNpZ25lciIsIm9wZXJhdG9yIjoiaW5jbHVkZWQiLCJ0eXBlIjoiZXhhY3RfY2FzZWQiLCJ2YWx1ZSI6IkV2aWwifSx7ImZpZWxkIjoidHJ1c3RlZCIsIm9wZXJhdG9yIjoiaW5jbHVkZWQiLCJ0eXBlIjoiZXhhY3RfY2FzZWQiLCJ2YWx1ZSI6InRydWUifV0sImZpZWxkIjoiZmlsZS5zaWduYXR1cmUiLCJ0eXBlIjoibmVzdGVkIn1dfQ==", "created": 1593016187465, "compressionAlgorithm": "none", "encryptionAlgorithm": "none", "identifier": "endpoint-exceptionlist-linux-1.0.0", - "compressedSha256": "d162f0302cbf419038ade7ea978e0a7ade7aad317fedefe455ff38dfa28b7cff", - "compressedSize": 336, - "decompressedSha256": "d162f0302cbf419038ade7ea978e0a7ade7aad317fedefe455ff38dfa28b7cff", - "decompressedSize": 336 + "encodedSha256": "f59266b06ffb1d7250edb9dbabd946e00e98afa950f955d8ea9d8ffef0eb142a", + "encodedSize": 328, + "decodedSha256": "f59266b06ffb1d7250edb9dbabd946e00e98afa950f955d8ea9d8ffef0eb142a", + "decodedSize": 328 }, - "type": "endpoint:user-artifact", + "type": "endpoint:user-artifact:v2", "updated_at": "2020-06-24T16:29:47.584Z" } } @@ -26,20 +26,20 @@ { "type": "doc", "value": { - "id": "endpoint:user-artifact-manifest:endpoint-manifest-1.0.0", + "id": "endpoint:user-artifact-manifest:v2:endpoint-manifest-1.0.0", "index": ".kibana", "source": { "references": [ ], - "endpoint:user-artifact-manifest": { + "endpoint:user-artifact-manifest:v2": { "created": 1593183699663, "ids": [ - "endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", - "endpoint-exceptionlist-macos-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", - "endpoint-exceptionlist-windows-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d" + "endpoint-exceptionlist-linux-1.0.0-f59266b06ffb1d7250edb9dbabd946e00e98afa950f955d8ea9d8ffef0eb142a", + "endpoint-exceptionlist-macos-1.0.0-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658", + "endpoint-exceptionlist-windows-1.0.0-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658" ] }, - "type": "endpoint:user-artifact-manifest", + "type": "endpoint:user-artifact-manifest:v2", "updated_at": "2020-06-26T15:01:39.704Z" } } From ec6054626f14a001b217129855e12496d21c893f Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Tue, 7 Jul 2020 14:27:31 -0400 Subject: [PATCH 17/32] Deflate --- .../security_solution/server/endpoint/lib/artifacts/lists.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 5db295ba7d1f7..28d69078b3129 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -5,7 +5,7 @@ */ import { createHash } from 'crypto'; -import { deflateRaw } from 'zlib'; +import { deflate } from 'zlib'; import { validate } from '../../../../common/validate'; import { Entry, EntryNested } from '../../../../../lists/common/schemas/types/entries'; @@ -175,7 +175,7 @@ function translateEntry( export function compressExceptionList(buffer: Buffer): Promise { return new Promise((resolve, reject) => { - deflateRaw(buffer, function (err, buf) { + deflate(buffer, function (err, buf) { if (err) { reject(err); } else { From 339c887a0aadec3ba03c85fcaefbce036711779b Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 7 Jul 2020 15:44:23 -0400 Subject: [PATCH 18/32] missed some refs --- .../endpoint/lib/artifacts/manifest.test.ts | 54 +++++++++---------- .../lib/artifacts/manifest_entry.test.ts | 22 ++++---- .../artifacts/download_exception_list.test.ts | 2 +- .../endpoint/schemas/artifacts/lists.mock.ts | 2 +- .../schemas/artifacts/saved_objects.mock.ts | 2 +- 5 files changed, 41 insertions(+), 41 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index 00a651c0b60fe..1a057b526edad 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -57,32 +57,32 @@ describe('manifest', () => { 'endpoint-exceptionlist-linux-1.0.0': { compression_algorithm: 'none', encryption_algorithm: 'none', - decoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - encoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - decoded_size: 268, - encoded_size: 268, + decoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + encoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + decoded_size: 260, + encoded_size: 260, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', }, 'endpoint-exceptionlist-macos-1.0.0': { compression_algorithm: 'none', encryption_algorithm: 'none', - decoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - encoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - decoded_size: 268, - encoded_size: 268, + decoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + encoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + decoded_size: 260, + encoded_size: 260, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', }, 'endpoint-exceptionlist-windows-1.0.0': { compression_algorithm: 'none', encryption_algorithm: 'none', - decoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - encoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - decoded_size: 268, - encoded_size: 268, + decoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + encoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + decoded_size: 260, + encoded_size: 260, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', }, }, manifest_version: 'abcd', @@ -94,9 +94,9 @@ describe('manifest', () => { expect(manifest1.toSavedObject()).toStrictEqual({ created: now.getTime(), ids: [ - 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-linux-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + 'endpoint-exceptionlist-macos-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + 'endpoint-exceptionlist-windows-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', ], }); }); @@ -106,12 +106,12 @@ describe('manifest', () => { expect(diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-linux-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', type: 'delete', }, { id: - 'endpoint-exceptionlist-linux-1.0.0-69328f83418f4957470640ed6cc605be6abb5fe80e0e388fd74f9764ad7ed5d1', + 'endpoint-exceptionlist-linux-1.0.0-27cfe2fae5550d3e312ca430821a3fdd5228c486dddc4852baf694455f89fde1', type: 'add', }, ]); @@ -127,15 +127,15 @@ describe('manifest', () => { const entries = manifest1.getEntries(); const keys = Object.keys(entries); expect(keys).toEqual([ - 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-linux-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + 'endpoint-exceptionlist-macos-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + 'endpoint-exceptionlist-windows-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', ]); }); test('Manifest returns true if contains artifact', async () => { const found = manifest1.contains( - 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + 'endpoint-exceptionlist-macos-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' ); expect(found).toEqual(true); }); @@ -144,17 +144,17 @@ describe('manifest', () => { const manifest = Manifest.fromArtifacts(artifacts, '1.0.0', 'v0'); expect( manifest.contains( - 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + 'endpoint-exceptionlist-linux-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + 'endpoint-exceptionlist-macos-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + 'endpoint-exceptionlist-windows-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' ) ).toEqual(true); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts index 41afd72efd366..881a384d4054d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -24,7 +24,7 @@ describe('manifest_entry', () => { test('Correct doc_id is returned', () => { expect(manifestEntry.getDocId()).toEqual( - 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + 'endpoint-exceptionlist-windows-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' ); }); @@ -34,21 +34,21 @@ describe('manifest_entry', () => { test('Correct sha256 is returned', () => { expect(manifestEntry.getEncodedSha256()).toEqual( - '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' ); expect(manifestEntry.getDecodedSha256()).toEqual( - '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' ); }); test('Correct size is returned', () => { - expect(manifestEntry.getEncodedSize()).toEqual(268); - expect(manifestEntry.getDecodedSize()).toEqual(268); + expect(manifestEntry.getEncodedSize()).toEqual(260); + expect(manifestEntry.getDecodedSize()).toEqual(260); }); test('Correct url is returned', () => { expect(manifestEntry.getUrl()).toEqual( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' ); }); @@ -60,12 +60,12 @@ describe('manifest_entry', () => { expect(manifestEntry.getRecord()).toEqual({ compression_algorithm: 'none', encryption_algorithm: 'none', - decoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - encoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - decoded_size: 268, - encoded_size: 268, + decoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + encoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + decoded_size: 260, + encoded_size: 260, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 863a1d5037756..4f87d25032804 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -31,7 +31,7 @@ import { WrappedTranslatedExceptionList } from '../../schemas/artifacts/lists'; const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; const expectedEndpointExceptions: WrappedTranslatedExceptionList = { - exceptions_list: [ + entries: [ { entries: [ { diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts index 7354b5fd0ec4d..94a0b4b015572 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts @@ -8,7 +8,7 @@ import { WrappedTranslatedExceptionList } from './lists'; export const getTranslatedExceptionListMock = (): WrappedTranslatedExceptionList => { return { - exceptions_list: [ + entries: [ { entries: [ { diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts index 1a9cc55ca5725..183a819807ed2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -20,7 +20,7 @@ export const getInternalArtifactMockWithDiffs = async ( schemaVersion: string ): Promise => { const mock = getTranslatedExceptionListMock(); - mock.exceptions_list.pop(); + mock.entries.pop(); return buildArtifact(mock, os, schemaVersion); }; From aca7b8c2c930dad1542122deb0a4a8ab1be9c097 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 7 Jul 2020 17:46:08 -0400 Subject: [PATCH 19/32] partial fix to wrapper format --- .../endpoint/lib/artifacts/lists.test.ts | 20 ++++++++--- .../server/endpoint/lib/artifacts/lists.ts | 34 ++++++++++++------- .../artifacts/download_exception_list.test.ts | 28 +++++++++++---- .../endpoint/schemas/artifacts/lists.mock.ts | 28 +++++++++++---- .../endpoint/schemas/artifacts/lists.ts | 6 ++-- 5 files changed, 85 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index 80d7bd5658363..63021a87bbab0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -21,6 +21,7 @@ describe('buildEventTypeSignal', () => { test('it should convert the exception lists response to the proper endpoint format', async () => { const expectedEndpointExceptions = { + type: 'simple', entries: [ { entries: [ @@ -46,7 +47,9 @@ describe('buildEventTypeSignal', () => { const first = getFoundExceptionListItemSchemaMock(); mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); - expect(resp).toEqual(expectedEndpointExceptions); + expect(resp).toEqual({ + entries: [expectedEndpointExceptions], + }); }); test('it should convert simple fields', async () => { @@ -57,6 +60,7 @@ describe('buildEventTypeSignal', () => { ]; const expectedEndpointExceptions = { + type: 'simple', entries: [ { field: 'server.domain', @@ -84,7 +88,9 @@ describe('buildEventTypeSignal', () => { mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); - expect(resp).toEqual(expectedEndpointExceptions); + expect(resp).toEqual({ + entries: [expectedEndpointExceptions], + }); }); test('it should convert fields case sensitive', async () => { @@ -100,6 +106,7 @@ describe('buildEventTypeSignal', () => { ]; const expectedEndpointExceptions = { + type: 'simple', entries: [ { field: 'server.domain', @@ -127,7 +134,9 @@ describe('buildEventTypeSignal', () => { mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); - expect(resp).toEqual(expectedEndpointExceptions); + expect(resp).toEqual({ + entries: [expectedEndpointExceptions], + }); }); test('it should ignore unsupported entries', async () => { @@ -147,6 +156,7 @@ describe('buildEventTypeSignal', () => { ]; const expectedEndpointExceptions = { + type: 'simple', entries: [ { field: 'server.domain', @@ -162,7 +172,9 @@ describe('buildEventTypeSignal', () => { mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); - expect(resp).toEqual(expectedEndpointExceptions); + expect(resp).toEqual({ + entries: [expectedEndpointExceptions], + }); }); test('it should convert the exception lists response to the proper endpoint format while paging', async () => { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 53cda8f2d2299..a13781519b508 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -5,6 +5,7 @@ */ import { createHash } from 'crypto'; +import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; import { validate } from '../../../../common/validate'; import { Entry, EntryNested } from '../../../../../lists/common/schemas/types/entries'; @@ -21,6 +22,7 @@ import { TranslatedEntryMatcher, translatedEntryMatchMatcher, translatedEntryMatchAnyMatcher, + TranslatedExceptionListItem, } from '../../schemas'; import { ArtifactConstants } from './common'; @@ -92,19 +94,11 @@ export async function getFullEndpointExceptionList( export function translateToEndpointExceptions( exc: FoundExceptionListItemSchema, schemaVersion: string -): TranslatedEntry[] { +): TranslatedExceptionListItem[] { if (schemaVersion === '1.0.0') { - return exc.data - .flatMap((list) => { - return list.entries; - }) - .reduce((entries: TranslatedEntry[], entry) => { - const translatedEntry = translateEntry(schemaVersion, entry); - if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) { - entries.push(translatedEntry); - } - return entries; - }, []); + return exc.data.map((item) => { + return translateItem(schemaVersion, item); + }); } else { throw new Error('unsupported schemaVersion'); } @@ -124,6 +118,22 @@ function normalizeFieldName(field: string): string { return field.endsWith('.text') ? field.substring(0, field.length - 5) : field; } +function translateItem( + schemaVersion: string, + item: ExceptionListItemSchema +): TranslatedExceptionListItem { + return { + type: item.type, + entries: item.entries.reduce((translatedEntries: TranslatedEntry[], entry) => { + const translatedEntry = translateEntry(schemaVersion, entry); + if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) { + translatedEntries.push(translatedEntry); + } + return translatedEntries; + }, []), + }; +} + function translateEntry( schemaVersion: string, entry: Entry | EntryNested diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 4f87d25032804..fbcd3bd130dfd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -33,7 +33,20 @@ const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0 const expectedEndpointExceptions: WrappedTranslatedExceptionList = { entries: [ { + type: 'simple', entries: [ + { + entries: [ + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.field', + type: 'nested', + }, { field: 'some.not.nested.field', operator: 'included', @@ -41,14 +54,17 @@ const expectedEndpointExceptions: WrappedTranslatedExceptionList = { value: 'some value', }, ], - field: 'some.field', - type: 'nested', }, { - field: 'some.not.nested.field', - operator: 'included', - type: 'exact_cased', - value: 'some value', + type: 'simple', + entries: [ + { + field: 'some.other.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some other value', + }, + ], }, ], }; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts index 94a0b4b015572..343b192163479 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts @@ -10,7 +10,20 @@ export const getTranslatedExceptionListMock = (): WrappedTranslatedExceptionList return { entries: [ { + type: 'simple', entries: [ + { + entries: [ + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.field', + type: 'nested', + }, { field: 'some.not.nested.field', operator: 'included', @@ -18,14 +31,17 @@ export const getTranslatedExceptionListMock = (): WrappedTranslatedExceptionList value: 'some value', }, ], - field: 'some.field', - type: 'nested', }, { - field: 'some.not.nested.field', - operator: 'included', - type: 'exact_cased', - value: 'some value', + type: 'simple', + entries: [ + { + field: 'some.other.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some other value', + }, + ], }, ], }; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts index cdb841a8d4c82..b7f99fe6fe297 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts @@ -64,17 +64,17 @@ export const translatedEntry = t.union([ ]); export type TranslatedEntry = t.TypeOf; -export const translatedExceptionList = t.exact( +export const translatedExceptionListItem = t.exact( t.type({ type: t.string, entries: t.array(translatedEntry), }) ); -export type TranslatedExceptionList = t.TypeOf; +export type TranslatedExceptionListItem = t.TypeOf; export const wrappedTranslatedExceptionList = t.exact( t.type({ - entries: t.array(translatedEntry), + entries: t.array(translatedExceptionListItem), }) ); export type WrappedTranslatedExceptionList = t.TypeOf; From 04d57012779489da0726065c5fcbe4f3a2559cc0 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 7 Jul 2020 17:59:55 -0400 Subject: [PATCH 20/32] update fixtures and integration test --- .../apis/endpoint/artifacts/index.ts | 72 +++++++++++-------- .../endpoint/artifacts/api_feature/data.json | 14 ++-- 2 files changed, 48 insertions(+), 38 deletions(-) diff --git a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts index 0a801ed237885..1f1c6f27b636a 100644 --- a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -71,7 +71,7 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with correct hash', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/f59266b06ffb1d7250edb9dbabd946e00e98afa950f955d8ea9d8ffef0eb142a' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) @@ -82,28 +82,33 @@ export default function (providerContext: FtrProviderContext) { expect(artifactJson).to.eql({ entries: [ { - field: 'actingProcess.file.signer', - operator: 'included', - type: 'exact_cased', - value: 'Elastic, N.V.', - }, - { + type: 'simple', entries: [ { - field: 'signer', + field: 'actingProcess.file.signer', operator: 'included', type: 'exact_cased', - value: 'Evil', + value: 'Elastic, N.V.', }, { - field: 'trusted', - operator: 'included', - type: 'exact_cased', - value: 'true', + entries: [ + { + field: 'signer', + operator: 'included', + type: 'exact_cased', + value: 'Evil', + }, + { + field: 'trusted', + operator: 'included', + type: 'exact_cased', + value: 'true', + }, + ], + field: 'file.signature', + type: 'nested', }, ], - field: 'file.signature', - type: 'nested', }, ], }); @@ -113,7 +118,7 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with correct hash from cache', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/f59266b06ffb1d7250edb9dbabd946e00e98afa950f955d8ea9d8ffef0eb142a' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) @@ -125,7 +130,7 @@ export default function (providerContext: FtrProviderContext) { .then(async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/f59266b06ffb1d7250edb9dbabd946e00e98afa950f955d8ea9d8ffef0eb142a' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) @@ -136,28 +141,33 @@ export default function (providerContext: FtrProviderContext) { expect(artifactJson).to.eql({ entries: [ { - field: 'actingProcess.file.signer', - operator: 'included', - type: 'exact_cased', - value: 'Elastic, N.V.', - }, - { + type: 'simple', entries: [ { - field: 'signer', + field: 'actingProcess.file.signer', operator: 'included', type: 'exact_cased', - value: 'Evil', + value: 'Elastic, N.V.', }, { - field: 'trusted', - operator: 'included', - type: 'exact_cased', - value: 'true', + entries: [ + { + field: 'signer', + operator: 'included', + type: 'exact_cased', + value: 'Evil', + }, + { + field: 'trusted', + operator: 'included', + type: 'exact_cased', + value: 'true', + }, + ], + field: 'file.signature', + type: 'nested', }, ], - field: 'file.signature', - type: 'nested', }, ], }); diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json index 565e1d619dda4..3433070c08009 100644 --- a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -1,21 +1,21 @@ { "type": "doc", "value": { - "id": "endpoint:user-artifact:v2:endpoint-exceptionlist-linux-1.0.0-f59266b06ffb1d7250edb9dbabd946e00e98afa950f955d8ea9d8ffef0eb142a", + "id": "endpoint:user-artifact:v2:endpoint-exceptionlist-linux-1.0.0-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", "index": ".kibana", "source": { "references": [ ], "endpoint:user-artifact:v2": { - "body": "eyJlbnRyaWVzIjpbeyJmaWVsZCI6ImFjdGluZ1Byb2Nlc3MuZmlsZS5zaWduZXIiLCJvcGVyYXRvciI6ImluY2x1ZGVkIiwidHlwZSI6ImV4YWN0X2Nhc2VkIiwidmFsdWUiOiJFbGFzdGljLCBOLlYuIn0seyJlbnRyaWVzIjpbeyJmaWVsZCI6InNpZ25lciIsIm9wZXJhdG9yIjoiaW5jbHVkZWQiLCJ0eXBlIjoiZXhhY3RfY2FzZWQiLCJ2YWx1ZSI6IkV2aWwifSx7ImZpZWxkIjoidHJ1c3RlZCIsIm9wZXJhdG9yIjoiaW5jbHVkZWQiLCJ0eXBlIjoiZXhhY3RfY2FzZWQiLCJ2YWx1ZSI6InRydWUifV0sImZpZWxkIjoiZmlsZS5zaWduYXR1cmUiLCJ0eXBlIjoibmVzdGVkIn1dfQ==", + "body": "eyJlbnRyaWVzIjpbeyJ0eXBlIjoic2ltcGxlIiwiZW50cmllcyI6W3siZmllbGQiOiJhY3RpbmdQcm9jZXNzLmZpbGUuc2lnbmVyIiwib3BlcmF0b3IiOiJpbmNsdWRlZCIsInR5cGUiOiJleGFjdF9jYXNlZCIsInZhbHVlIjoiRWxhc3RpYywgTi5WLiJ9LHsiZW50cmllcyI6W3siZmllbGQiOiJzaWduZXIiLCJvcGVyYXRvciI6ImluY2x1ZGVkIiwidHlwZSI6ImV4YWN0X2Nhc2VkIiwidmFsdWUiOiJFdmlsIn0seyJmaWVsZCI6InRydXN0ZWQiLCJvcGVyYXRvciI6ImluY2x1ZGVkIiwidHlwZSI6ImV4YWN0X2Nhc2VkIiwidmFsdWUiOiJ0cnVlIn1dLCJmaWVsZCI6ImZpbGUuc2lnbmF0dXJlIiwidHlwZSI6Im5lc3RlZCJ9XX1dfQ==", "created": 1593016187465, "compressionAlgorithm": "none", "encryptionAlgorithm": "none", "identifier": "endpoint-exceptionlist-linux-1.0.0", - "encodedSha256": "f59266b06ffb1d7250edb9dbabd946e00e98afa950f955d8ea9d8ffef0eb142a", - "encodedSize": 328, - "decodedSha256": "f59266b06ffb1d7250edb9dbabd946e00e98afa950f955d8ea9d8ffef0eb142a", - "decodedSize": 328 + "encodedSha256": "d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", + "encodedSize": 358, + "decodedSha256": "d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", + "decodedSize": 358 }, "type": "endpoint:user-artifact:v2", "updated_at": "2020-06-24T16:29:47.584Z" @@ -34,7 +34,7 @@ "endpoint:user-artifact-manifest:v2": { "created": 1593183699663, "ids": [ - "endpoint-exceptionlist-linux-1.0.0-f59266b06ffb1d7250edb9dbabd946e00e98afa950f955d8ea9d8ffef0eb142a", + "endpoint-exceptionlist-linux-1.0.0-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", "endpoint-exceptionlist-macos-1.0.0-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658", "endpoint-exceptionlist-windows-1.0.0-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658" ] From ed87ad2d5ae4781b2d419d3d77792e2ece36bace Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Tue, 7 Jul 2020 18:05:13 -0400 Subject: [PATCH 21/32] Fixing unit tests --- .../endpoint/lib/artifacts/lists.test.ts | 2 +- .../endpoint/lib/artifacts/manifest.test.ts | 54 +++++++++---------- .../lib/artifacts/manifest_entry.test.ts | 22 ++++---- .../manifest_manager/manifest_manager.test.ts | 27 ++++++---- 4 files changed, 55 insertions(+), 50 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index 63021a87bbab0..0a1cd556e6e91 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -194,7 +194,7 @@ describe('buildEventTypeSignal', () => { .mockReturnValueOnce(second) .mockReturnValueOnce(third); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); - expect(resp.entries.length).toEqual(6); + expect(resp.entries.length).toEqual(3); }); test('it should handle no exceptions', async () => { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index 1a057b526edad..3e5fdbf9484ca 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -57,32 +57,32 @@ describe('manifest', () => { 'endpoint-exceptionlist-linux-1.0.0': { compression_algorithm: 'none', encryption_algorithm: 'none', - decoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', - encoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', - decoded_size: 260, - encoded_size: 260, + decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + decoded_size: 430, + encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }, 'endpoint-exceptionlist-macos-1.0.0': { compression_algorithm: 'none', encryption_algorithm: 'none', - decoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', - encoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', - decoded_size: 260, - encoded_size: 260, + decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + decoded_size: 430, + encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }, 'endpoint-exceptionlist-windows-1.0.0': { compression_algorithm: 'none', encryption_algorithm: 'none', - decoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', - encoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', - decoded_size: 260, - encoded_size: 260, + decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + decoded_size: 430, + encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }, }, manifest_version: 'abcd', @@ -94,9 +94,9 @@ describe('manifest', () => { expect(manifest1.toSavedObject()).toStrictEqual({ created: now.getTime(), ids: [ - 'endpoint-exceptionlist-linux-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', - 'endpoint-exceptionlist-macos-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', - 'endpoint-exceptionlist-windows-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', ], }); }); @@ -106,12 +106,12 @@ describe('manifest', () => { expect(diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', type: 'delete', }, { id: - 'endpoint-exceptionlist-linux-1.0.0-27cfe2fae5550d3e312ca430821a3fdd5228c486dddc4852baf694455f89fde1', + 'endpoint-exceptionlist-linux-1.0.0-3d3546e94f70493021ee845be32c66e36ea7a720c64b4d608d8029fe949f7e51', type: 'add', }, ]); @@ -127,15 +127,15 @@ describe('manifest', () => { const entries = manifest1.getEntries(); const keys = Object.keys(entries); expect(keys).toEqual([ - 'endpoint-exceptionlist-linux-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', - 'endpoint-exceptionlist-macos-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', - 'endpoint-exceptionlist-windows-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', ]); }); test('Manifest returns true if contains artifact', async () => { const found = manifest1.contains( - 'endpoint-exceptionlist-macos-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' + 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); expect(found).toEqual(true); }); @@ -144,17 +144,17 @@ describe('manifest', () => { const manifest = Manifest.fromArtifacts(artifacts, '1.0.0', 'v0'); expect( manifest.contains( - 'endpoint-exceptionlist-linux-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' + 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-exceptionlist-macos-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' + 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-exceptionlist-windows-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' + 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ) ).toEqual(true); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts index 881a384d4054d..a52114ad90258 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -24,7 +24,7 @@ describe('manifest_entry', () => { test('Correct doc_id is returned', () => { expect(manifestEntry.getDocId()).toEqual( - 'endpoint-exceptionlist-windows-1.0.0-339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' + 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); }); @@ -34,21 +34,21 @@ describe('manifest_entry', () => { test('Correct sha256 is returned', () => { expect(manifestEntry.getEncodedSha256()).toEqual( - '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' + '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); expect(manifestEntry.getDecodedSha256()).toEqual( - '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' + '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); }); test('Correct size is returned', () => { - expect(manifestEntry.getEncodedSize()).toEqual(260); - expect(manifestEntry.getDecodedSize()).toEqual(260); + expect(manifestEntry.getEncodedSize()).toEqual(430); + expect(manifestEntry.getDecodedSize()).toEqual(430); }); test('Correct url is returned', () => { expect(manifestEntry.getUrl()).toEqual( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); }); @@ -60,12 +60,12 @@ describe('manifest_entry', () => { expect(manifestEntry.getRecord()).toEqual({ compression_algorithm: 'none', encryption_algorithm: 'none', - decoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', - encoded_sha256: '339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', - decoded_size: 260, - encoded_size: 260, + decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + decoded_size: 430, + encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/339af4b7d15db33dfb80268d3fa0b40f7fd1806becd691d8a757f425e782db7d', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index e2f22a10c10e3..1d6dffadde61a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -21,7 +21,7 @@ describe('manifest_manager', () => { expect(manifestWrapper!.diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-2a2ec06c957330deb42f41835d3029001432038106f823173fb9e7ea603decb5', + 'endpoint-exceptionlist-linux-1.0.0-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', type: 'add', }, ]); @@ -35,7 +35,7 @@ describe('manifest_manager', () => { expect(manifestWrapper!.diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-2a2ec06c957330deb42f41835d3029001432038106f823173fb9e7ea603decb5', + 'endpoint-exceptionlist-linux-1.0.0-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', type: 'add', }, ]); @@ -44,22 +44,27 @@ describe('manifest_manager', () => { expect(entry).toEqual({ entries: [ { + type: 'simple', entries: [ { - field: 'nested.field', + entries: [ + { + field: 'nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.parentField', + type: 'nested', + }, + { + field: 'some.not.nested.field', operator: 'included', type: 'exact_cased', value: 'some value', }, ], - field: 'some.parentField', - type: 'nested', - }, - { - field: 'some.not.nested.field', - operator: 'included', - type: 'exact_cased', - value: 'some value', }, ], }); From 094c358c13950067d32730be334754ccdcc635d4 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 8 Jul 2020 12:42:06 -0400 Subject: [PATCH 22/32] small bug fixes --- .../endpoint/endpoint_app_context_services.ts | 6 ++-- .../server/endpoint/ingest_integration.ts | 6 ++-- .../server/endpoint/routes/metadata/index.ts | 29 ++++++++++++------- .../endpoint/schemas/artifacts/common.ts | 9 ++++-- .../response/download_artifact_schema.ts | 4 +-- .../schemas/artifacts/saved_objects.ts | 4 ++- .../security_solution/server/plugin.ts | 1 + 7 files changed, 39 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index 5d3fa24e8f51c..f51e8c6be1040 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -4,8 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ import { - SavedObjectsServiceStart, KibanaRequest, + Logger, + SavedObjectsServiceStart, SavedObjectsClientContract, } from 'src/core/server'; import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server'; @@ -15,6 +16,7 @@ import { ManifestManager } from './services/artifacts'; export type EndpointAppContextServiceStartContract = Partial< Pick > & { + logger: Logger; manifestManager?: ManifestManager; registerIngestCallback?: IngestManagerStartContract['registerExternalCallback']; savedObjectsStart: SavedObjectsServiceStart; @@ -37,7 +39,7 @@ export class EndpointAppContextService { if (this.manifestManager && dependencies.registerIngestCallback) { dependencies.registerIngestCallback( 'packageConfigCreate', - getPackageConfigCreateCallback(this.manifestManager) + getPackageConfigCreateCallback(dependencies.logger, this.manifestManager) ); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index d1c834158e253..03da9d59854b5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { Logger } from '../../../../../src/core/server'; import { NewPackageConfig } from '../../../ingest_manager/common/types/models'; import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config'; import { NewPolicyData } from '../../common/endpoint/types'; @@ -13,6 +14,7 @@ import { ManifestManager } from './services/artifacts'; * Callback to handle creation of PackageConfigs in Ingest Manager */ export const getPackageConfigCreateCallback = ( + logger: Logger, manifestManager: ManifestManager ): ((newPackageConfig: NewPackageConfig) => Promise) => { const handlePackageConfigCreate = async ( @@ -32,7 +34,7 @@ export const getPackageConfigCreateCallback = ( const snapshot = await manifestManager.getSnapshot({ initialize: true }); if (snapshot === null) { - // TODO: log error... should not be in this state + logger.warn('No manifest snapshot available.'); return updatedPackageConfig; } @@ -77,7 +79,7 @@ export const getPackageConfigCreateCallback = ( // clean up old artifacts await manifestManager.syncArtifacts(snapshot, 'delete'); } else { - // TODO: log error + logger.warn('Package config update failed.'); } } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts index 45bd3544304dd..4b2eb3ea1ddb0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts @@ -75,18 +75,18 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp options: { authRequired: true, tags: ['access:securitySolution'] }, }, async (context, req, res) => { - const agentService = endpointAppContext.service.getAgentService(); - if (agentService === undefined) { - return res.internalError({ body: 'agentService not available' }); - } + try { + const agentService = endpointAppContext.service.getAgentService(); + if (agentService === undefined) { + throw new Error('agentService not available'); + } - const metadataRequestContext: MetadataRequestContext = { - agentService, - logger, - requestHandlerContext: context, - }; + const metadataRequestContext: MetadataRequestContext = { + agentService, + logger, + requestHandlerContext: context, + }; - try { const unenrolledAgentIds = await findAllUnenrolledAgentIds( agentService, context.core.savedObjects.client @@ -105,6 +105,7 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp 'search', queryParams )) as SearchResponse; + return res.ok({ body: await mapToHostResultList(queryParams, response, metadataRequestContext), }); @@ -129,8 +130,14 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp return res.internalError({ body: 'agentService not available' }); } + const metadataRequestContext: MetadataRequestContext = { + agentService, + logger, + requestHandlerContext: context, + }; + try { - const doc = await getHostData(context, agentService, req.params.id); + const doc = await getHostData(metadataRequestContext, req.params.id); if (doc) { return res.ok({ body: doc }); } diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts index 8e7fa833931f2..f06fff9331b6d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts @@ -6,9 +6,14 @@ import * as t from 'io-ts'; -export const body = t.string; +export const buffer = new t.Type( + 'buffer', + (input: unknown): input is Buffer => Buffer.isBuffer(input), + (input, context) => (Buffer.isBuffer(input) ? t.success(input) : t.failure(input, context)), + t.identity +); -export const created = t.number; // TODO: Make this into an ISO Date string check +export const created = t.number; export const encoding = t.keyof({ identity: null, diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts index 8b8715c878cc0..3705062449c60 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts @@ -5,7 +5,7 @@ */ import * as t from 'io-ts'; -import { encoding } from '../common'; +import { buffer, encoding } from '../common'; const headers = t.exact( t.type({ @@ -16,7 +16,7 @@ const headers = t.exact( export const downloadArtifactResponseSchema = t.exact( t.type({ - body: t.unknown, // TODO: make Buffer type + body: buffer, headers, }) ); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts index e4cd7f48a2901..aa11f4409269a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts @@ -12,7 +12,9 @@ import { sha256, size, } from '../../../../common/endpoint/schema/common'; -import { body, created } from './common'; +import { created } from './common'; + +export const body = t.string; // base64 export const internalArtifactSchema = t.exact( t.type({ diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 8d86c6420c7fa..86b8b9b5cf61d 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -272,6 +272,7 @@ export class Plugin implements IPlugin Date: Wed, 8 Jul 2020 13:43:20 -0400 Subject: [PATCH 23/32] artifact and manifest versioning changes --- .../common/endpoint/schema/common.ts | 2 +- .../server/endpoint/lib/artifacts/common.ts | 5 +- .../endpoint/lib/artifacts/lists.test.ts | 12 ++-- .../server/endpoint/lib/artifacts/lists.ts | 2 +- .../endpoint/lib/artifacts/manifest.test.ts | 63 +++++++++--------- .../server/endpoint/lib/artifacts/manifest.ts | 3 +- .../lib/artifacts/manifest_entry.test.ts | 10 +-- .../artifacts/download_exception_list.test.ts | 4 +- .../endpoint/schemas/artifacts/common.ts | 2 +- .../artifacts/artifact_client.test.ts | 2 +- .../services/artifacts/artifact_client.ts | 2 +- .../artifacts/manifest_client.mock.ts | 4 +- .../artifacts/manifest_client.test.ts | 2 +- .../manifest_manager/manifest_manager.mock.ts | 5 +- .../manifest_manager/manifest_manager.test.ts | 10 +-- .../manifest_manager/manifest_manager.ts | 64 +++++++++++-------- 16 files changed, 104 insertions(+), 88 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts index fdb2570314cd0..84c94322b1ffe 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts @@ -20,7 +20,7 @@ export const identifier = t.string; export const manifestVersion = t.string; export const manifestSchemaVersion = t.keyof({ - '1.0.0': null, + v1: null, }); export type ManifestSchemaVersion = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts index cf38147522083..9ad4554b30203 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -8,10 +8,11 @@ export const ArtifactConstants = { GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist', SAVED_OBJECT_TYPE: 'endpoint:user-artifact:v2', SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'], - SCHEMA_VERSION: '1.0.0', + SCHEMA_VERSION: 'v1', }; export const ManifestConstants = { SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest:v2', - SCHEMA_VERSION: '1.0.0', + SCHEMA_VERSION: 'v1', + INITIAL_VERSION: 'WzAsMF0=', }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index 0a1cd556e6e91..acde455f77cb4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -46,7 +46,7 @@ describe('buildEventTypeSignal', () => { const first = getFoundExceptionListItemSchemaMock(); mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); expect(resp).toEqual({ entries: [expectedEndpointExceptions], }); @@ -87,7 +87,7 @@ describe('buildEventTypeSignal', () => { first.data[0].entries = testEntries; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); expect(resp).toEqual({ entries: [expectedEndpointExceptions], }); @@ -133,7 +133,7 @@ describe('buildEventTypeSignal', () => { first.data[0].entries = testEntries; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); expect(resp).toEqual({ entries: [expectedEndpointExceptions], }); @@ -171,7 +171,7 @@ describe('buildEventTypeSignal', () => { first.data[0].entries = testEntries; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); expect(resp).toEqual({ entries: [expectedEndpointExceptions], }); @@ -193,7 +193,7 @@ describe('buildEventTypeSignal', () => { .mockReturnValueOnce(first) .mockReturnValueOnce(second) .mockReturnValueOnce(third); - const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); expect(resp.entries.length).toEqual(3); }); @@ -202,7 +202,7 @@ describe('buildEventTypeSignal', () => { exceptionsResponse.data = []; exceptionsResponse.total = 0; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(exceptionsResponse); - const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); expect(resp.entries.length).toEqual(0); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 3e1e82ea8bbc7..556405adff62f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -97,7 +97,7 @@ export function translateToEndpointExceptions( exc: FoundExceptionListItemSchema, schemaVersion: string ): TranslatedExceptionListItem[] { - if (schemaVersion === '1.0.0') { + if (schemaVersion === 'v1') { return exc.data.map((item) => { return translateItem(schemaVersion, item); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index 3e5fdbf9484ca..d3212eb3faf4d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -10,6 +10,7 @@ import { getInternalArtifactMock, getInternalArtifactMockWithDiffs, } from '../../schemas/artifacts/saved_objects.mock'; +import { ManifestConstants } from './common'; import { Manifest } from './manifest'; describe('manifest', () => { @@ -20,41 +21,45 @@ describe('manifest', () => { let manifest2: Manifest; beforeAll(async () => { - const artifactLinux = await getInternalArtifactMock('linux', '1.0.0'); - const artifactMacos = await getInternalArtifactMock('macos', '1.0.0'); - const artifactWindows = await getInternalArtifactMock('windows', '1.0.0'); + const artifactLinux = await getInternalArtifactMock('linux', 'v1'); + const artifactMacos = await getInternalArtifactMock('macos', 'v1'); + const artifactWindows = await getInternalArtifactMock('windows', 'v1'); artifacts.push(artifactLinux); artifacts.push(artifactMacos); artifacts.push(artifactWindows); - manifest1 = new Manifest(now, '1.0.0', 'v0'); + manifest1 = new Manifest(now, 'v1', ManifestConstants.INITIAL_VERSION); manifest1.addEntry(artifactLinux); manifest1.addEntry(artifactMacos); manifest1.addEntry(artifactWindows); manifest1.setVersion('abcd'); - const newArtifactLinux = await getInternalArtifactMockWithDiffs('linux', '1.0.0'); - manifest2 = new Manifest(new Date(), '1.0.0', 'v0'); + const newArtifactLinux = await getInternalArtifactMockWithDiffs('linux', 'v1'); + manifest2 = new Manifest(new Date(), 'v1', ManifestConstants.INITIAL_VERSION); manifest2.addEntry(newArtifactLinux); manifest2.addEntry(artifactMacos); manifest2.addEntry(artifactWindows); }); test('Can create manifest with valid schema version', () => { - const manifest = new Manifest(new Date(), '1.0.0', 'v0'); + const manifest = new Manifest(new Date(), 'v1', ManifestConstants.INITIAL_VERSION); expect(manifest).toBeInstanceOf(Manifest); }); test('Cannot create manifest with invalid schema version', () => { expect(() => { - new Manifest(new Date(), 'abcd' as ManifestSchemaVersion, 'v0'); + new Manifest( + new Date(), + 'abcd' as ManifestSchemaVersion, + ManifestConstants.INITIAL_VERSION + ); }).toThrow(); }); test('Manifest transforms correctly to expected endpoint format', async () => { expect(manifest1.toEndpointFormat()).toStrictEqual({ artifacts: { - 'endpoint-exceptionlist-linux-1.0.0': { + 'endpoint-exceptionlist-linux-v1': { compression_algorithm: 'none', encryption_algorithm: 'none', decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', @@ -62,9 +67,9 @@ describe('manifest', () => { decoded_size: 430, encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }, - 'endpoint-exceptionlist-macos-1.0.0': { + 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'none', encryption_algorithm: 'none', decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', @@ -72,9 +77,9 @@ describe('manifest', () => { decoded_size: 430, encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }, - 'endpoint-exceptionlist-windows-1.0.0': { + 'endpoint-exceptionlist-windows-v1': { compression_algorithm: 'none', encryption_algorithm: 'none', decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', @@ -82,11 +87,11 @@ describe('manifest', () => { decoded_size: 430, encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }, }, manifest_version: 'abcd', - schema_version: '1.0.0', + schema_version: 'v1', }); }); @@ -94,9 +99,9 @@ describe('manifest', () => { expect(manifest1.toSavedObject()).toStrictEqual({ created: now.getTime(), ids: [ - 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-linux-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-macos-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-windows-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', ], }); }); @@ -106,12 +111,12 @@ describe('manifest', () => { expect(diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-linux-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', type: 'delete', }, { id: - 'endpoint-exceptionlist-linux-1.0.0-3d3546e94f70493021ee845be32c66e36ea7a720c64b4d608d8029fe949f7e51', + 'endpoint-exceptionlist-linux-v1-3d3546e94f70493021ee845be32c66e36ea7a720c64b4d608d8029fe949f7e51', type: 'add', }, ]); @@ -119,7 +124,7 @@ describe('manifest', () => { test('Manifest returns data for given artifact', async () => { const artifact = artifacts[0]; - const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.encodedSha256}`); + const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.decodedSha256}`); expect(returned).toEqual(artifact); }); @@ -127,34 +132,34 @@ describe('manifest', () => { const entries = manifest1.getEntries(); const keys = Object.keys(entries); expect(keys).toEqual([ - 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-linux-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-macos-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-windows-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', ]); }); test('Manifest returns true if contains artifact', async () => { const found = manifest1.contains( - 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-macos-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); expect(found).toEqual(true); }); test('Manifest can be created from list of artifacts', async () => { - const manifest = Manifest.fromArtifacts(artifacts, '1.0.0', 'v0'); + const manifest = Manifest.fromArtifacts(artifacts, 'v1', ManifestConstants.INITIAL_VERSION); expect( manifest.contains( - 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-linux-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-macos-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-windows-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ) ).toEqual(true); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index c343568226e22..c0124602ddb81 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -11,6 +11,7 @@ import { ManifestSchemaVersion, } from '../../../../common/endpoint/schema/common'; import { ManifestSchema, manifestSchema } from '../../../../common/endpoint/schema/manifest'; +import { ManifestConstants } from './common'; import { ManifestEntry } from './manifest_entry'; export interface ManifestDiff { @@ -104,7 +105,7 @@ export class Manifest { public toEndpointFormat(): ManifestSchema { const manifestObj: ManifestSchema = { - manifest_version: this.version ?? 'v0', + manifest_version: this.version ?? ManifestConstants.INITIAL_VERSION, schema_version: this.schemaVersion, artifacts: {}, }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts index a52114ad90258..002214cbd7574 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -14,7 +14,7 @@ describe('manifest_entry', () => { let manifestEntry: ManifestEntry; beforeAll(async () => { - artifact = await getInternalArtifactMock('windows', '1.0.0'); + artifact = await getInternalArtifactMock('windows', 'v1'); manifestEntry = new ManifestEntry(artifact); }); @@ -24,12 +24,12 @@ describe('manifest_entry', () => { test('Correct doc_id is returned', () => { expect(manifestEntry.getDocId()).toEqual( - 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-windows-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); }); test('Correct identifier is returned', () => { - expect(manifestEntry.getIdentifier()).toEqual('endpoint-exceptionlist-windows-1.0.0'); + expect(manifestEntry.getIdentifier()).toEqual('endpoint-exceptionlist-windows-v1'); }); test('Correct sha256 is returned', () => { @@ -48,7 +48,7 @@ describe('manifest_entry', () => { test('Correct url is returned', () => { expect(manifestEntry.getUrl()).toEqual( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); }); @@ -65,7 +65,7 @@ describe('manifest_entry', () => { decoded_size: 430, encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 2af732e43d338..87a0dce48289f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -30,7 +30,7 @@ import { createMockEndpointAppContextServiceStartContract } from '../../mocks'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; import { WrappedTranslatedExceptionList } from '../../schemas/artifacts/lists'; -const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; +const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-v1`; const expectedEndpointExceptions: WrappedTranslatedExceptionList = { entries: [ { @@ -147,7 +147,7 @@ describe('test alerts route', () => { references: [], attributes: { identifier: mockArtifactName, - schemaVersion: '1.0.0', + schemaVersion: 'v1', sha256: '123456', encoding: 'application/json', created: Date.now(), diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts index f06fff9331b6d..d5a30951e9398 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts @@ -20,5 +20,5 @@ export const encoding = t.keyof({ }); export const schemaVersion = t.keyof({ - '1.0.0': null, + v1: null, }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts index 08e29b5c6b82b..3e3b12c04d65c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts @@ -27,7 +27,7 @@ describe('artifact_client', () => { test('can create artifact', async () => { const savedObjectsClient = savedObjectsClientMock.create(); const artifactClient = getArtifactClientMock(savedObjectsClient); - const artifact = await getInternalArtifactMock('linux', '1.0.0'); + const artifact = await getInternalArtifactMock('linux', 'v1'); await artifactClient.createArtifact(artifact); expect(savedObjectsClient.create).toHaveBeenCalledWith( ArtifactConstants.SAVED_OBJECT_TYPE, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts index e899905602c8d..ca53a891c4d6b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts @@ -16,7 +16,7 @@ export class ArtifactClient { } public getArtifactId(artifact: InternalArtifactSchema) { - return `${artifact.identifier}-${artifact.encodedSha256}`; + return `${artifact.identifier}-${artifact.decodedSha256}`; } public async getArtifact(id: string): Promise> { diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts index bfeacbcedf2cb..d869ed9493abc 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts @@ -12,7 +12,7 @@ export const getManifestClientMock = ( savedObjectsClient?: SavedObjectsClientContract ): ManifestClient => { if (savedObjectsClient !== undefined) { - return new ManifestClient(savedObjectsClient, '1.0.0'); + return new ManifestClient(savedObjectsClient, 'v1'); } - return new ManifestClient(savedObjectsClientMock.create(), '1.0.0'); + return new ManifestClient(savedObjectsClientMock.create(), 'v1'); }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts index 5780c6279ee6a..fe3f193bc8ff5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts @@ -14,7 +14,7 @@ import { ManifestClient } from './manifest_client'; describe('manifest_client', () => { describe('ManifestClient sanity checks', () => { test('can create ManifestClient', () => { - const manifestClient = new ManifestClient(savedObjectsClientMock.create(), '1.0.0'); + const manifestClient = new ManifestClient(savedObjectsClientMock.create(), 'v1'); expect(manifestClient).toBeInstanceOf(ManifestClient); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index 483b3434d63f2..dfbe2572076d0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -15,6 +15,7 @@ import { buildArtifact, getFullEndpointExceptionList, } from '../../../lib/artifacts'; +import { ManifestConstants } from '../../../lib/artifacts/common'; import { InternalArtifactSchema } from '../../../schemas/artifacts'; import { getArtifactClientMock } from '../artifact_client.mock'; import { getManifestClientMock } from '../manifest_client.mock'; @@ -69,13 +70,13 @@ async function mockBuildExceptionListArtifacts( export class ManifestManagerMock extends ManifestManager { // @ts-ignore private buildExceptionListArtifacts = async () => { - return mockBuildExceptionListArtifacts('linux', '1.0.0'); + return mockBuildExceptionListArtifacts('linux', 'v1'); }; // @ts-ignore private getLastDispatchedManifest = jest .fn() - .mockResolvedValue(new Manifest(new Date(), '1.0.0', 'v0')); + .mockResolvedValue(new Manifest(new Date(), 'v1', ManifestConstants.INITIAL_VERSION)); // @ts-ignore private getManifestClient = jest diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index 3a9c0410ec9ef..b1cbc41459f15 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -22,7 +22,7 @@ describe('manifest_manager', () => { expect(snapshot!.diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', + 'endpoint-exceptionlist-linux-v1-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', type: 'add', }, ]); @@ -36,7 +36,7 @@ describe('manifest_manager', () => { expect(snapshot!.diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', + 'endpoint-exceptionlist-linux-v1-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', type: 'add', }, ]); @@ -83,8 +83,8 @@ describe('manifest_manager', () => { expect( packageConfigService.update.mock.calls[0][2].inputs[0].config.artifact_manifest.value ).toEqual({ - manifest_version: 'v0', - schema_version: '1.0.0', + manifest_version: ManifestConstants.INITIAL_VERSION, + schema_version: 'v1', artifacts: { [artifact.identifier]: { compression_algorithm: 'none', @@ -93,7 +93,7 @@ describe('manifest_manager', () => { encoded_sha256: artifact.encodedSha256, decoded_size: artifact.decodedSize, encoded_size: artifact.encodedSize, - relative_url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.encodedSha256}`, + relative_url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.decodedSha256}`, }, }, }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index b6f44c8b55fbb..c44955ce0bbb1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -117,7 +117,6 @@ export class ManifestManager { manifest.addEntry(artifactSo.attributes); }) ); - return manifest; } catch (err) { if (err.output.statusCode !== 404) { @@ -133,35 +132,44 @@ export class ManifestManager { * @param opts TODO */ public async getSnapshot(opts?: ManifestSnapshotOpts) { - let oldManifest: Manifest | null; - - // Get the last-dispatched manifest - oldManifest = await this.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION); - - if (oldManifest === null && opts !== undefined && opts.initialize) { - oldManifest = new Manifest(new Date(), ManifestConstants.SCHEMA_VERSION, 'v0'); // create empty manifest - } else if (oldManifest == null) { - this.logger.debug('Manifest does not exist yet. Waiting...'); - return null; - } + try { + let oldManifest: Manifest | null; + + // Get the last-dispatched manifest + oldManifest = await this.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION); + + if (oldManifest === null && opts !== undefined && opts.initialize) { + oldManifest = new Manifest( + new Date(), + ManifestConstants.SCHEMA_VERSION, + ManifestConstants.INITIAL_VERSION + ); // create empty manifest + } else if (oldManifest == null) { + this.logger.debug('Manifest does not exist yet. Waiting...'); + return null; + } - // Build new exception list artifacts - const artifacts = await this.buildExceptionListArtifacts(ArtifactConstants.SCHEMA_VERSION); + // Build new exception list artifacts + const artifacts = await this.buildExceptionListArtifacts(ArtifactConstants.SCHEMA_VERSION); - // Build new manifest - const newManifest = Manifest.fromArtifacts( - artifacts, - ManifestConstants.SCHEMA_VERSION, - oldManifest.getVersion() - ); + // Build new manifest + const newManifest = Manifest.fromArtifacts( + artifacts, + ManifestConstants.SCHEMA_VERSION, + oldManifest.getVersion() + ); - // Get diffs - const diffs = newManifest.diff(oldManifest); + // Get diffs + const diffs = newManifest.diff(oldManifest); - return { - manifest: newManifest, - diffs, - }; + return { + manifest: newManifest, + diffs, + }; + } catch (err) { + this.logger.error(err); + return null; + } } /** @@ -291,11 +299,11 @@ export class ManifestManager { const manifestClient = this.getManifestClient(manifest.getSchemaVersion()); // Commit the new manifest - if (manifest.getVersion() === 'v0') { + if (manifest.getVersion() === ManifestConstants.INITIAL_VERSION) { await manifestClient.createManifest(manifest.toSavedObject()); } else { const version = manifest.getVersion(); - if (version === 'v0') { + if (version === ManifestConstants.INITIAL_VERSION) { throw new Error('Updating existing manifest with baseline version. Bad state.'); } await manifestClient.updateManifest(manifest.toSavedObject(), { From ae856b588874dcd8a943aeb1dc0a4d95dfcb3b18 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 8 Jul 2020 16:15:03 -0400 Subject: [PATCH 24/32] Remove access tag from download endpoint --- .../artifacts/download_exception_list.ts | 4 ++-- .../apis/endpoint/artifacts/index.ts | 16 ++++++++-------- .../endpoint/artifacts/api_feature/data.json | 18 +++++++++--------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index c8039bd133c2f..4839f9849a924 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -43,7 +43,7 @@ export function registerDownloadExceptionListRoute( DownloadArtifactRequestParamsSchema >(downloadArtifactRequestParamsSchema), }, - options: { tags: ['access:securitySolution'] }, + options: {}, }, async (context, req, res) => { let scopedSOClient: SavedObjectsClientContract; @@ -54,7 +54,7 @@ export function registerDownloadExceptionListRoute( scopedSOClient = endpointContext.service.getScopedSavedObjectsClient(req); await authenticateAgentWithAccessToken(scopedSOClient, req); } catch (err) { - if (err.output.statusCode === 401) { + if ((err.isBoom ? err.output.statusCode : err.statusCode) === 401) { return res.unauthorized(); } else { return res.notFound(); diff --git a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts index 4c083699e4af1..8b16bda422b4c 100644 --- a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -5,7 +5,7 @@ */ import expect from '@kbn/expect'; -import { inflateRawSync } from 'zlib'; +import { inflateSync } from 'zlib'; import { WrappedTranslatedExceptionList } from '../../../../../plugins/security_solution/server/endpoint/schemas'; import { FtrProviderContext } from '../../../ftr_provider_context'; @@ -63,7 +63,7 @@ export default function (providerContext: FtrProviderContext) { it('should fail to find artifact with invalid hash', async () => { await supertestWithoutAuth - .get('/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/abcd') + .get('/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/abcd') .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() @@ -73,7 +73,7 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with correct hash', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) @@ -120,7 +120,7 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with correct hash from cache', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) @@ -132,7 +132,7 @@ export default function (providerContext: FtrProviderContext) { .then(async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) @@ -180,14 +180,14 @@ export default function (providerContext: FtrProviderContext) { it('should download valid compressed JSON', async () => { const { body } = await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/1be2ae8d19368939c6e9ecfb32d46233e975feec9611469f62dd2578bd9c4507' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() .expect(200); - const decompressedJson = inflateRawSync(body); + const decompressedJson = inflateSync(body); const decompressedBody: WrappedTranslatedExceptionList = JSON.parse( decompressedJson.toString() ); @@ -229,7 +229,7 @@ export default function (providerContext: FtrProviderContext) { it('should fail on invalid api key', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey iNvAlId`) diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json index 95fcb6d9e6f07..bd1010240f86c 100644 --- a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -1,19 +1,19 @@ { "type": "doc", "value": { - "id": "endpoint:user-artifact:v2:endpoint-exceptionlist-linux-1.0.0-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", + "id": "endpoint:user-artifact:v2:endpoint-exceptionlist-linux-v1-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", "index": ".kibana", "source": { "references": [ ], "endpoint:user-artifact:v2": { - "body": "eyJlbnRyaWVzIjpbeyJ0eXBlIjoic2ltcGxlIiwiZW50cmllcyI6W3siZmllbGQiOiJhY3RpbmdQcm9jZXNzLmZpbGUuc2lnbmVyIiwib3BlcmF0b3IiOiJpbmNsdWRlZCIsInR5cGUiOiJleGFjdF9jYXNlZCIsInZhbHVlIjoiRWxhc3RpYywgTi5WLiJ9LHsiZW50cmllcyI6W3siZmllbGQiOiJzaWduZXIiLCJvcGVyYXRvciI6ImluY2x1ZGVkIiwidHlwZSI6ImV4YWN0X2Nhc2VkIiwidmFsdWUiOiJFdmlsIn0seyJmaWVsZCI6InRydXN0ZWQiLCJvcGVyYXRvciI6ImluY2x1ZGVkIiwidHlwZSI6ImV4YWN0X2Nhc2VkIiwidmFsdWUiOiJ0cnVlIn1dLCJmaWVsZCI6ImZpbGUuc2lnbmF0dXJlIiwidHlwZSI6Im5lc3RlZCJ9XX1dfQ==", + "body": "eJylkM8KwjAMxl9Fci59gN29iicvMqR02QjUbiSpKGPvbiw6ETwpuX1/fh9kBszKhALNcQa9TQgNCJ2nhOA+vJ4wdWaGqJSHPY8RRXxPCb3QkJEtP07IQUe2GOWYSoedqU8qXq16ikGqeAmpPNRtCqIU3WbnDx4WN38d/WvhQqmCXzDlIlojP9CsjLC0bqWtHwhaGN/1jHVkae3u+6N6Sg==", "created": 1593016187465, "compressionAlgorithm": "zlib", "encryptionAlgorithm": "none", - "identifier": "endpoint-exceptionlist-linux-1.0.0", - "encodedSha256": "d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", - "encodedSize": 358, + "identifier": "endpoint-exceptionlist-linux-v1", + "encodedSha256": "5caaeabcb7864d47157fc7c28d5a7398b4f6bbaaa565d789c02ee809253b7613", + "encodedSize": 160, "decodedSha256": "d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", "decodedSize": 358 }, @@ -26,7 +26,7 @@ { "type": "doc", "value": { - "id": "endpoint:user-artifact-manifest:v2:endpoint-manifest-1.0.0", + "id": "endpoint:user-artifact-manifest:v2:endpoint-manifest-v1", "index": ".kibana", "source": { "references": [ @@ -34,9 +34,9 @@ "endpoint:user-artifact-manifest:v2": { "created": 1593183699663, "ids": [ - "endpoint-exceptionlist-linux-1.0.0-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", - "endpoint-exceptionlist-macos-1.0.0-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658", - "endpoint-exceptionlist-windows-1.0.0-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658" + "endpoint-exceptionlist-linux-v1-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", + "endpoint-exceptionlist-macos-v1-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658", + "endpoint-exceptionlist-windows-v1-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658" ] }, "type": "endpoint:user-artifact-manifest:v2", From 69f59fd422a4bf811ddfb588d2cea4539672235d Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 8 Jul 2020 16:41:03 -0400 Subject: [PATCH 25/32] Adding decompression to integration test --- .../apis/endpoint/artifacts/index.ts | 56 +------------------ 1 file changed, 3 insertions(+), 53 deletions(-) diff --git a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts index 8b16bda422b4c..ca59d396839ae 100644 --- a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -7,7 +7,6 @@ import expect from '@kbn/expect'; import { inflateSync } from 'zlib'; -import { WrappedTranslatedExceptionList } from '../../../../../plugins/security_solution/server/endpoint/schemas'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { getSupertestWithoutAuth, setupIngest } from '../../fleet/agents/services'; @@ -80,7 +79,7 @@ export default function (providerContext: FtrProviderContext) { .send() .expect(200) .expect((response) => { - const artifactJson = JSON.parse(response.text); + const artifactJson = JSON.parse(inflateSync(response.body).toString()); expect(artifactJson).to.eql({ entries: [ { @@ -127,7 +126,7 @@ export default function (providerContext: FtrProviderContext) { .send() .expect(200) .expect((response) => { - JSON.parse(response.text); + JSON.parse(inflateSync(response.body).toString()); }) .then(async () => { await supertestWithoutAuth @@ -139,7 +138,7 @@ export default function (providerContext: FtrProviderContext) { .send() .expect(200) .expect((response) => { - const artifactJson = JSON.parse(response.text); + const artifactJson = JSON.parse(inflateSync(response.body).toString()); expect(artifactJson).to.eql({ entries: [ { @@ -177,55 +176,6 @@ export default function (providerContext: FtrProviderContext) { }); }); - it('should download valid compressed JSON', async () => { - const { body } = await supertestWithoutAuth - .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' - ) - .set('kbn-xsrf', 'xxx') - .set('authorization', `ApiKey ${agentAccessAPIKey}`) - .send() - .expect(200); - - const decompressedJson = inflateSync(body); - const decompressedBody: WrappedTranslatedExceptionList = JSON.parse( - decompressedJson.toString() - ); - - expect(decompressedBody).to.eql({ - entries: [ - { - entries: [ - { - field: 'actingProcess.file.signer', - operator: 'included', - type: 'exact_cased', - value: 'Elastic, N.V.', - }, - { - entries: [ - { - field: 'signer', - operator: 'included', - type: 'exact_cased', - value: 'Evil', - }, - { - field: 'trusted', - operator: 'included', - type: 'exact_cased', - value: 'true', - }, - ], - field: 'file.signature', - type: 'nested', - }, - ], - }, - ], - } as WrappedTranslatedExceptionList); - }); - it('should fail on invalid api key', async () => { await supertestWithoutAuth .get( From 4795132ee42e7e509a8a8311e52e584d5ca546f5 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 8 Jul 2020 17:11:13 -0400 Subject: [PATCH 26/32] Removing tag from route --- .../endpoint/routes/artifacts/download_exception_list.test.ts | 2 -- .../server/endpoint/routes/artifacts/download_exception_list.ts | 1 - 2 files changed, 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 87a0dce48289f..157d3b2fe4f5b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -164,8 +164,6 @@ describe('test alerts route', () => { path.startsWith('/api/endpoint/artifacts/download') )!; - expect(routeConfig.options).toEqual({ tags: ['access:securitySolution'] }); - await routeHandler( ({ core: { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 4839f9849a924..1b364a04a4272 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -43,7 +43,6 @@ export function registerDownloadExceptionListRoute( DownloadArtifactRequestParamsSchema >(downloadArtifactRequestParamsSchema), }, - options: {}, }, async (context, req, res) => { let scopedSOClient: SavedObjectsClientContract; From c18948b40b433e8e98b8dbf680e3db289f62eaf0 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 8 Jul 2020 17:24:42 -0400 Subject: [PATCH 27/32] add try/catch in ingest callback handler --- .../server/endpoint/ingest_integration.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index 03da9d59854b5..1acec1e7c53ac 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -71,15 +71,15 @@ export const getPackageConfigCreateCallback = ( return updatedPackageConfig; } finally { if (snapshot.diffs.length > 0) { - // const created = await manifestManager.confirmPackageConfigExists(updatedPackageConfig.name); - const created = true; - if (created) { + // TODO: let's revisit the way this callback happens... use promises? + // only commit when we know the package config was created + try { await manifestManager.commit(snapshot.manifest); // clean up old artifacts await manifestManager.syncArtifacts(snapshot, 'delete'); - } else { - logger.warn('Package config update failed.'); + } catch (err) { + logger.error(err); } } } From 8bef643a51618723f9e9c974d38a773a1e9c4cf6 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 8 Jul 2020 17:29:03 -0400 Subject: [PATCH 28/32] Fixing --- .../endpoint/routes/artifacts/download_exception_list.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 87a0dce48289f..13b3aa818768a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -164,7 +164,7 @@ describe('test alerts route', () => { path.startsWith('/api/endpoint/artifacts/download') )!; - expect(routeConfig.options).toEqual({ tags: ['access:securitySolution'] }); + expect(routeConfig.options).toEqual({}); await routeHandler( ({ From a24acc80b8595a63f96f60951cb46f6da24e9920 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 8 Jul 2020 17:35:09 -0400 Subject: [PATCH 29/32] Removing last expect from unit test for tag --- .../endpoint/routes/artifacts/download_exception_list.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 13b3aa818768a..157d3b2fe4f5b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -164,8 +164,6 @@ describe('test alerts route', () => { path.startsWith('/api/endpoint/artifacts/download') )!; - expect(routeConfig.options).toEqual({}); - await routeHandler( ({ core: { From 04dbbcf3d5fde788f67a5ceda4e0a464b4d88c09 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 8 Jul 2020 17:56:32 -0400 Subject: [PATCH 30/32] type fixes --- .../common/endpoint/generate_data.ts | 4 +-- .../policy/store/policy_details/index.test.ts | 4 +-- .../endpoint/lib/artifacts/cache.test.ts | 27 +++++++++++-------- .../artifacts/download_exception_list.test.ts | 2 ++ 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 6720f3523d5c7..339e5554ccb12 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -1036,8 +1036,8 @@ export class EndpointDocGenerator { config: { artifact_manifest: { value: { - manifest_version: 'v0', - schema_version: '1.0.0', + manifest_version: 'WzAsMF0=', + schema_version: 'v1', artifacts: {}, }, }, diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts index 0bd623b27f4fb..102fd40c97672 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts @@ -43,8 +43,8 @@ describe('policy details: ', () => { config: { artifact_manifest: { value: { - manifest_version: 'v0', - schema_version: '1.0.0', + manifest_version: 'WzAsMF0=', + schema_version: 'v1', artifacts: {}, }, }, diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts index 5a0fb91345552..00c764d0b912e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts @@ -8,6 +8,7 @@ import { ExceptionsCache } from './cache'; describe('ExceptionsCache tests', () => { let cache: ExceptionsCache; + const body = Buffer.from('body'); beforeEach(() => { jest.clearAllMocks(); @@ -15,29 +16,33 @@ describe('ExceptionsCache tests', () => { }); test('it should cache', async () => { - cache.set('test', 'body'); + cache.set('test', body); const cacheResp = cache.get('test'); - expect(cacheResp).toEqual('body'); + expect(cacheResp).toEqual(body); }); test('it should handle cache miss', async () => { - cache.set('test', 'body'); + cache.set('test', body); const cacheResp = cache.get('not test'); expect(cacheResp).toEqual(undefined); }); test('it should handle cache eviction', async () => { - cache.set('1', 'a'); - cache.set('2', 'b'); - cache.set('3', 'c'); + const a = Buffer.from('a'); + const b = Buffer.from('b'); + const c = Buffer.from('c'); + const d = Buffer.from('d'); + cache.set('1', a); + cache.set('2', b); + cache.set('3', c); const cacheResp = cache.get('1'); - expect(cacheResp).toEqual('a'); + expect(cacheResp).toEqual(a); - cache.set('4', 'd'); + cache.set('4', d); const secondResp = cache.get('1'); expect(secondResp).toEqual(undefined); - expect(cache.get('2')).toEqual('b'); - expect(cache.get('3')).toEqual('c'); - expect(cache.get('4')).toEqual('d'); + expect(cache.get('2')).toEqual(b); + expect(cache.get('3')).toEqual(c); + expect(cache.get('4')).toEqual(d); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 157d3b2fe4f5b..8c6faee7f7a5d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -164,6 +164,8 @@ describe('test alerts route', () => { path.startsWith('/api/endpoint/artifacts/download') )!; + expect(routeConfig.options).toEqual(undefined); + await routeHandler( ({ core: { From 2bf4acb0a9cc26ec7664afb31f1a4fbd77489d47 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 8 Jul 2020 19:00:25 -0400 Subject: [PATCH 31/32] Add compression type to manifest --- .../common/endpoint/schema/common.ts | 1 + .../lib/artifacts/manifest_entry.test.ts | 2 + .../endpoint/lib/artifacts/manifest_entry.ts | 7 +- .../manifest_manager/manifest_manager.ts | 113 ++++++++---------- 4 files changed, 59 insertions(+), 64 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts index 84c94322b1ffe..014673ebe6398 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts @@ -10,6 +10,7 @@ export const compressionAlgorithm = t.keyof({ none: null, zlib: null, }); +export type CompressionAlgorithm = t.TypeOf; export const encryptionAlgorithm = t.keyof({ none: null, diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts index 002214cbd7574..7ea2a07210c55 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -68,5 +68,7 @@ describe('manifest_entry', () => { '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }); }); + + // TODO: add test for entry with compression }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts index f8b0d6d9e3f9f..b35e0c2b9ad6e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts @@ -5,6 +5,7 @@ */ import { InternalArtifactSchema } from '../../schemas/artifacts'; +import { CompressionAlgorithm } from '../../../../common/endpoint/schema/common'; import { ManifestEntrySchema } from '../../../../common/endpoint/schema/manifest'; export class ManifestEntry { @@ -22,6 +23,10 @@ export class ManifestEntry { return this.artifact.identifier; } + public getCompressionAlgorithm(): CompressionAlgorithm { + return this.artifact.compressionAlgorithm; + } + public getEncodedSha256(): string { return this.artifact.encodedSha256; } @@ -48,7 +53,7 @@ export class ManifestEntry { public getRecord(): ManifestEntrySchema { return { - compression_algorithm: 'none', + compression_algorithm: this.getCompressionAlgorithm(), encryption_algorithm: 'none', decoded_sha256: this.getDecodedSha256(), decoded_size: this.getDecodedSize(), diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index c44955ce0bbb1..9726e28f54186 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -111,12 +111,10 @@ export class ManifestManager { manifestSo.version ); - await Promise.all( - manifestSo.attributes.ids.map(async (id) => { - const artifactSo = await this.artifactClient.getArtifact(id); - manifest.addEntry(artifactSo.attributes); - }) - ); + for (const id of manifestSo.attributes.ids) { + const artifactSo = await this.artifactClient.getArtifact(id); + manifest.addEntry(artifactSo.attributes); + } return manifest; } catch (err) { if (err.output.statusCode !== 404) { @@ -202,38 +200,32 @@ export class ManifestManager { return diff.type === 'delete'; }); - await Promise.all( - adds.map(async (diff) => { - const artifact = snapshot.manifest.getArtifact(diff.id); - const compressedArtifact = await compressExceptionList( - Buffer.from(artifact.body, 'base64') - ); - artifact.body = compressedArtifact.toString('base64'); - artifact.encodedSize = compressedArtifact.byteLength; - artifact.compressionAlgorithm = 'zlib'; - artifact.encodedSha256 = createHash('sha256').update(compressedArtifact).digest('hex'); - - try { - await this.artifactClient.createArtifact(artifact); - } catch (err) { - if (err.status === 409) { - this.logger.debug(`Tried to create artifact ${diff.id}, but it already exists.`); - } else { - throw err; - } + for (const diff of adds) { + const artifact = snapshot.manifest.getArtifact(diff.id); + const compressedArtifact = await compressExceptionList(Buffer.from(artifact.body, 'base64')); + artifact.body = compressedArtifact.toString('base64'); + artifact.encodedSize = compressedArtifact.byteLength; + artifact.compressionAlgorithm = 'zlib'; + artifact.encodedSha256 = createHash('sha256').update(compressedArtifact).digest('hex'); + + try { + await this.artifactClient.createArtifact(artifact); + } catch (err) { + if (err.status === 409) { + this.logger.debug(`Tried to create artifact ${diff.id}, but it already exists.`); + } else { + throw err; } - // Cache the body of the artifact - this.cache.set(diff.id, Buffer.from(artifact.body, 'base64')); - }) - ); + } + // Cache the body of the artifact + this.cache.set(diff.id, Buffer.from(artifact.body, 'base64')); + } - await Promise.all( - deletes.map(async (diff) => { - await this.artifactClient.deleteArtifact(diff.id); - // TODO: should we delete the cache entry here? - this.logger.info(`Cleaned up artifact ${diff.id}`); - }) - ); + for (const diff of deletes) { + await this.artifactClient.deleteArtifact(diff.id); + // TODO: should we delete the cache entry here? + this.logger.info(`Cleaned up artifact ${diff.id}`); + } } /** @@ -252,35 +244,30 @@ export class ManifestManager { kuery: 'ingest-package-configs.package.name:endpoint', }); - await Promise.all( - items.map(async (packageConfig) => { - const { id, revision, updated_at, updated_by, ...newPackageConfig } = packageConfig; - if ( - newPackageConfig.inputs.length > 0 && - newPackageConfig.inputs[0].config !== undefined - ) { - const artifactManifest = newPackageConfig.inputs[0].config.artifact_manifest ?? { - value: {}, - }; - artifactManifest.value = manifest.toEndpointFormat(); - newPackageConfig.inputs[0].config.artifact_manifest = artifactManifest; - - try { - await this.packageConfigService.update(this.savedObjectsClient, id, newPackageConfig); - this.logger.debug( - `Updated package config ${id} with manifest version ${manifest.getVersion()}` - ); - } catch (err) { - success = false; - this.logger.debug(`Error updating package config ${id}`); - this.logger.error(err); - } - } else { + for (const packageConfig of items) { + const { id, revision, updated_at, updated_by, ...newPackageConfig } = packageConfig; + if (newPackageConfig.inputs.length > 0 && newPackageConfig.inputs[0].config !== undefined) { + const artifactManifest = newPackageConfig.inputs[0].config.artifact_manifest ?? { + value: {}, + }; + artifactManifest.value = manifest.toEndpointFormat(); + newPackageConfig.inputs[0].config.artifact_manifest = artifactManifest; + + try { + await this.packageConfigService.update(this.savedObjectsClient, id, newPackageConfig); + this.logger.debug( + `Updated package config ${id} with manifest version ${manifest.getVersion()}` + ); + } catch (err) { success = false; - this.logger.debug(`Package config ${id} has no config.`); + this.logger.debug(`Error updating package config ${id}`); + this.logger.error(err); } - }) - ); + } else { + success = false; + this.logger.debug(`Package config ${id} has no config.`); + } + } paging = page * items.length < total; page++; From 899a673f240b5f77e100cf3a286745140d5b415e Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Wed, 8 Jul 2020 22:52:32 -0400 Subject: [PATCH 32/32] Reverting ingestManager back to being required for now --- x-pack/plugins/security_solution/kibana.json | 2 +- .../endpoint_app_context_services.test.ts | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/kibana.json b/x-pack/plugins/security_solution/kibana.json index 745df54c62158..f6f2d5171312c 100644 --- a/x-pack/plugins/security_solution/kibana.json +++ b/x-pack/plugins/security_solution/kibana.json @@ -11,6 +11,7 @@ "embeddable", "features", "home", + "ingestManager", "taskManager", "inspector", "licensing", @@ -20,7 +21,6 @@ ], "optionalPlugins": [ "encryptedSavedObjects", - "ingestManager", "ml", "newsfeed", "security", diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts index 7b435f71fe4a8..7642db23812e1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts @@ -8,14 +8,14 @@ import { httpServerMock } from '../../../../../src/core/server/mocks'; import { EndpointAppContextService } from './endpoint_app_context_services'; describe('test endpoint app context services', () => { - it('should return undefined on getAgentService if dependencies are not enabled', async () => { - const endpointAppContextService = new EndpointAppContextService(); - expect(endpointAppContextService.getAgentService()).toEqual(undefined); - }); - it('should return undefined on getManifestManager if dependencies are not enabled', async () => { - const endpointAppContextService = new EndpointAppContextService(); - expect(endpointAppContextService.getManifestManager()).toEqual(undefined); - }); + // it('should return undefined on getAgentService if dependencies are not enabled', async () => { + // const endpointAppContextService = new EndpointAppContextService(); + // expect(endpointAppContextService.getAgentService()).toEqual(undefined); + // }); + // it('should return undefined on getManifestManager if dependencies are not enabled', async () => { + // const endpointAppContextService = new EndpointAppContextService(); + // expect(endpointAppContextService.getManifestManager()).toEqual(undefined); + // }); it('should throw error on getScopedSavedObjectsClient if start is not called', async () => { const endpointAppContextService = new EndpointAppContextService(); expect(() =>