Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Security solution][Endpoint] Removes zip compression when creating artifacts #101379

Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ export type CompressionAlgorithm = t.TypeOf<typeof compressionAlgorithm>;

export const compressionAlgorithmDispatch = t.keyof({
zlib: null,
none: null,
});
export type CompressionAlgorithmDispatch = t.TypeOf<typeof compressionAlgorithmDispatch>;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
*/

import { createHash } from 'crypto';
import { deflate } from 'zlib';
import type {
Entry,
EntryNested,
Expand All @@ -21,9 +20,7 @@ import {
} from '@kbn/securitysolution-list-constants';
import { ExceptionListClient } from '../../../../../lists/server';
import {
internalArtifactCompleteSchema,
InternalArtifactCompleteSchema,
InternalArtifactSchema,
TranslatedEntry,
translatedEntry as translatedEntryType,
translatedEntryMatchAnyMatcher,
Expand Down Expand Up @@ -60,28 +57,6 @@ export async function buildArtifact(
};
}

export async function maybeCompressArtifact(
uncompressedArtifact: InternalArtifactSchema
): Promise<InternalArtifactSchema> {
const compressedArtifact = { ...uncompressedArtifact };
if (internalArtifactCompleteSchema.is(uncompressedArtifact)) {
const compressedArtifactBody = await compressExceptionList(
Buffer.from(uncompressedArtifact.body, 'base64')
);
compressedArtifact.body = compressedArtifactBody.toString('base64');
compressedArtifact.encodedSize = compressedArtifactBody.byteLength;
compressedArtifact.compressionAlgorithm = 'zlib';
compressedArtifact.encodedSha256 = createHash('sha256')
.update(compressedArtifactBody)
.digest('hex');
}
return compressedArtifact;
}

export function isCompressed(artifact: InternalArtifactSchema) {
return artifact.compressionAlgorithm === 'zlib';
}

export async function getFilteredEndpointExceptionList(
eClient: ExceptionListClient,
schemaVersion: string,
Expand Down Expand Up @@ -297,15 +272,3 @@ function translateEntry(
}
}
}

export async function compressExceptionList(buffer: Buffer): Promise<Buffer> {
return new Promise((resolve, reject) => {
deflate(buffer, function (err, buf) {
if (err) {
reject(err);
} else {
resolve(buf);
}
});
});
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ describe('manifest', () => {
let ARTIFACT_COPY_TRUSTED_APPS_WINDOWS: InternalArtifactCompleteSchema;

beforeAll(async () => {
ARTIFACTS = await getMockArtifacts({ compress: true });
ARTIFACTS_COPY = await getMockArtifacts({ compress: true });
ARTIFACTS = await getMockArtifacts();
ARTIFACTS_COPY = await getMockArtifacts();
ARTIFACT_EXCEPTIONS_MACOS = ARTIFACTS[0];
ARTIFACT_EXCEPTIONS_WINDOWS = ARTIFACTS[1];
ARTIFACT_EXCEPTIONS_LINUX = ARTIFACTS[2];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
InternalArtifactSchema,
InternalManifestSchema,
InternalManifestEntrySchema,
InternalArtifactCompleteSchema,
} from '../../schemas/artifacts';
import {
ManifestSchemaVersion,
Expand Down Expand Up @@ -139,6 +140,27 @@ export class Manifest {
return this.allEntries.get(getArtifactId(artifact))?.specificTargetPolicies;
}

/**
* Replaces an artifact from all the collections.
*
* @param artifact An InternalArtifactCompleteSchema representing the artifact.
*/
public replaceArtifact(artifact: InternalArtifactCompleteSchema) {
const existingEntry = this.allEntries.get(getArtifactId(artifact));
if (existingEntry) {
existingEntry.entry = new ManifestEntry(artifact);

this.allEntries.set(getArtifactId(artifact), existingEntry);
this.defaultEntries.set(getArtifactId(artifact), existingEntry.entry);

existingEntry.specificTargetPolicies.forEach((policyId) => {
const entries = this.policySpecificEntries.get(policyId) || new Map();
entries.set(existingEntry.entry.getDocId(), existingEntry.entry);
this.policySpecificEntries.set(policyId, entries);
});
}
}

public diff(manifest: Manifest): ManifestDiff {
const diff: ManifestDiff = {
additions: [],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ describe('manifest_entry', () => {
let manifestEntry: ManifestEntry;

beforeAll(async () => {
artifact = await getInternalArtifactMock('windows', 'v1', { compress: true });
artifact = await getInternalArtifactMock('windows', 'v1');
manifestEntry = new ManifestEntry(artifact);
});

Expand All @@ -35,15 +35,15 @@ describe('manifest_entry', () => {

test('Correct sha256 is returned', () => {
expect(manifestEntry.getEncodedSha256()).toEqual(
'975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e'
'96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'
);
expect(manifestEntry.getDecodedSha256()).toEqual(
'96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'
);
});

test('Correct size is returned', () => {
expect(manifestEntry.getEncodedSize()).toEqual(147);
expect(manifestEntry.getEncodedSize()).toEqual(432);
expect(manifestEntry.getDecodedSize()).toEqual(432);
});

Expand All @@ -59,12 +59,12 @@ describe('manifest_entry', () => {

test('Correct record is returned', () => {
expect(manifestEntry.getRecord()).toEqual({
compression_algorithm: 'zlib',
compression_algorithm: 'none',
encryption_algorithm: 'none',
decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e',
encoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
decoded_size: 432,
encoded_size: 147,
encoded_size: 432,
relative_url:
'/api/fleet/artifacts/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ describe('When migrating artifacts to fleet', () => {
type: '',
id: 'abc123',
references: [],
attributes: await getInternalArtifactMock('windows', 'v1', { compress: true }),
attributes: await getInternalArtifactMock('windows', 'v1'),
},
])
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,49 +17,49 @@ import {
import { ArtifactConstants } from './common';
import { Manifest } from './manifest';

export const getMockArtifacts = async (opts?: { compress: boolean }) => {
export const getMockArtifacts = async () => {
return Promise.all([
// Exceptions items
...ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map<Promise<InternalArtifactCompleteSchema>>(
async (os) => {
return getInternalArtifactMock(os, 'v1', opts);
return getInternalArtifactMock(os, 'v1');
}
),
// Trusted Apps items
...ArtifactConstants.SUPPORTED_TRUSTED_APPS_OPERATING_SYSTEMS.map<
Promise<InternalArtifactCompleteSchema>
>(async (os) => {
return getInternalArtifactMock(os, 'v1', opts, ArtifactConstants.GLOBAL_TRUSTED_APPS_NAME);
return getInternalArtifactMock(os, 'v1', ArtifactConstants.GLOBAL_TRUSTED_APPS_NAME);
}),
]);
};

export const getMockArtifactsWithDiff = async (opts?: { compress: boolean }) => {
export const getMockArtifactsWithDiff = async () => {
return Promise.all(
ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map<Promise<InternalArtifactCompleteSchema>>(
async (os) => {
if (os === 'macos') {
return getInternalArtifactMockWithDiffs(os, 'v1');
}
return getInternalArtifactMock(os, 'v1', opts);
return getInternalArtifactMock(os, 'v1');
}
)
);
};

export const getEmptyMockArtifacts = async (opts?: { compress: boolean }) => {
export const getEmptyMockArtifacts = async () => {
return Promise.all(
ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map<Promise<InternalArtifactCompleteSchema>>(
async (os) => {
return getEmptyInternalArtifactMock(os, 'v1', opts);
return getEmptyInternalArtifactMock(os, 'v1');
}
)
);
};

export const getMockManifest = async (opts?: { compress: boolean }) => {
export const getMockManifest = async () => {
const manifest = new Manifest();
const artifacts = await getMockArtifacts(opts);
const artifacts = await getMockArtifacts();
artifacts.forEach((artifact) => manifest.addEntry(artifact));
return manifest;
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ describe('task', () => {
let ARTIFACT_TRUSTED_APPS_MACOS: InternalArtifactCompleteSchema;

beforeAll(async () => {
const artifacts = await getMockArtifacts({ compress: true });
const artifacts = await getMockArtifacts();
ARTIFACT_EXCEPTIONS_MACOS = artifacts[0];
ARTIFACT_EXCEPTIONS_WINDOWS = artifacts[1];
ARTIFACT_TRUSTED_APPS_MACOS = artifacts[2];
Expand Down Expand Up @@ -167,7 +167,7 @@ describe('task', () => {

expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([]);
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([], newManifest);
expect(manifestManager.commit).not.toHaveBeenCalled();
expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest);
expect(manifestManager.deleteArtifacts).toHaveBeenCalledWith([]);
Expand All @@ -192,10 +192,10 @@ describe('task', () => {

expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([
ARTIFACT_EXCEPTIONS_MACOS,
ARTIFACT_TRUSTED_APPS_MACOS,
]);
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith(
[ARTIFACT_EXCEPTIONS_MACOS, ARTIFACT_TRUSTED_APPS_MACOS],
newManifest
);
expect(manifestManager.commit).not.toHaveBeenCalled();
expect(manifestManager.tryDispatch).not.toHaveBeenCalled();
expect(manifestManager.deleteArtifacts).not.toHaveBeenCalled();
Expand All @@ -221,10 +221,10 @@ describe('task', () => {

expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([
ARTIFACT_EXCEPTIONS_MACOS,
ARTIFACT_TRUSTED_APPS_MACOS,
]);
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith(
[ARTIFACT_EXCEPTIONS_MACOS, ARTIFACT_TRUSTED_APPS_MACOS],
newManifest
);
expect(manifestManager.commit).toHaveBeenCalledWith(newManifest);
expect(manifestManager.tryDispatch).not.toHaveBeenCalled();
expect(manifestManager.deleteArtifacts).not.toHaveBeenCalled();
Expand All @@ -251,10 +251,10 @@ describe('task', () => {

expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([
ARTIFACT_EXCEPTIONS_MACOS,
ARTIFACT_TRUSTED_APPS_MACOS,
]);
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith(
[ARTIFACT_EXCEPTIONS_MACOS, ARTIFACT_TRUSTED_APPS_MACOS],
newManifest
);
expect(manifestManager.commit).toHaveBeenCalledWith(newManifest);
expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest);
expect(manifestManager.deleteArtifacts).not.toHaveBeenCalled();
Expand Down Expand Up @@ -284,7 +284,10 @@ describe('task', () => {

expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([ARTIFACT_TRUSTED_APPS_MACOS]);
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith(
[ARTIFACT_TRUSTED_APPS_MACOS],
newManifest
);
expect(manifestManager.commit).toHaveBeenCalledWith(newManifest);
expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest);
expect(manifestManager.deleteArtifacts).toHaveBeenCalledWith([ARTIFACT_ID_1]);
Expand Down Expand Up @@ -314,7 +317,7 @@ describe('task', () => {

expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([]);
expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([], newManifest);
expect(manifestManager.commit).toHaveBeenCalledWith(newManifest);
expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest);
expect(manifestManager.deleteArtifacts).toHaveBeenCalledWith([]);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,8 @@ export class ManifestTask {
const diff = newManifest.diff(oldManifest);

const persistErrors = await manifestManager.pushArtifacts(
diff.additions as InternalArtifactCompleteSchema[]
diff.additions as InternalArtifactCompleteSchema[],
newManifest
);
if (persistErrors.length) {
reportErrors(this.logger, persistErrors);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,33 +5,13 @@
* 2.0.
*/

import {
buildArtifact,
maybeCompressArtifact,
isCompressed,
ArtifactConstants,
} from '../../lib/artifacts';
import { buildArtifact, ArtifactConstants } from '../../lib/artifacts';
import { getTranslatedExceptionListMock } from './lists.mock';
import {
InternalManifestSchema,
internalArtifactCompleteSchema,
InternalArtifactCompleteSchema,
} from './saved_objects';

const compressArtifact = async (artifact: InternalArtifactCompleteSchema) => {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice 🎉

const compressedArtifact = await maybeCompressArtifact(artifact);
if (!isCompressed(compressedArtifact)) {
throw new Error(`Unable to compress artifact: ${artifact.identifier}`);
} else if (!internalArtifactCompleteSchema.is(compressedArtifact)) {
throw new Error(`Incomplete artifact detected: ${artifact.identifier}`);
}
return compressedArtifact;
};
import { InternalManifestSchema, InternalArtifactCompleteSchema } from './saved_objects';

export const getInternalArtifactMock = async (
os: string,
schemaVersion: string,
opts?: { compress: boolean },
artifactName: string = ArtifactConstants.GLOBAL_ALLOWLIST_NAME
): Promise<InternalArtifactCompleteSchema> => {
const artifact = await buildArtifact(
Expand All @@ -40,23 +20,21 @@ export const getInternalArtifactMock = async (
os,
artifactName
);
return opts?.compress ? compressArtifact(artifact) : artifact;
return artifact;
};

export const getEmptyInternalArtifactMock = async (
os: string,
schemaVersion: string,
opts?: { compress: boolean },
artifactName: string = ArtifactConstants.GLOBAL_ALLOWLIST_NAME
): Promise<InternalArtifactCompleteSchema> => {
const artifact = await buildArtifact({ entries: [] }, schemaVersion, os, artifactName);
return opts?.compress ? compressArtifact(artifact) : artifact;
return artifact;
};

export const getInternalArtifactMockWithDiffs = async (
os: string,
schemaVersion: string,
opts?: { compress: boolean }
schemaVersion: string
): Promise<InternalArtifactCompleteSchema> => {
const mock = getTranslatedExceptionListMock();
mock.entries.pop();
Expand All @@ -66,7 +44,7 @@ export const getInternalArtifactMockWithDiffs = async (
os,
ArtifactConstants.GLOBAL_ALLOWLIST_NAME
);
return opts?.compress ? compressArtifact(artifact) : artifact;
return artifact;
};

export const getInternalManifestMock = (): InternalManifestSchema => ({
Expand Down
Loading