diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts b/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts index 861ea0988692c..7184727fc53de 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts +++ b/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts @@ -54,8 +54,8 @@ export class KbnClient { /** * Make a direct request to the Kibana server */ - async request(options: ReqOptions) { - return await this.requester.request(options); + async request(options: ReqOptions) { + return await this.requester.request(options); } resolveUrl(relativeUrl: string) { diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index abdbbf1986bcd..ec7a49da469fe 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -110,6 +110,12 @@ const Mac: OSFields[] = []; const OS: OSFields[] = [...Windows, ...Mac, ...Linux]; +const POLICY_RESPONSE_STATUSES: HostPolicyResponseActionStatus[] = [ + HostPolicyResponseActionStatus.success, + HostPolicyResponseActionStatus.failure, + HostPolicyResponseActionStatus.warning, +]; + const APPLIED_POLICIES: Array<{ name: string; id: string; @@ -125,6 +131,11 @@ const APPLIED_POLICIES: Array<{ id: 'C2A9093E-E289-4C0A-AA44-8C32A414FA7A', status: HostPolicyResponseActionStatus.success, }, + { + name: 'Detect Malware Only', + id: '47d7965d-6869-478b-bd9c-fb0d2bb3959f', + status: HostPolicyResponseActionStatus.success, + }, ]; const FILE_OPERATIONS: string[] = ['creation', 'open', 'rename', 'execution', 'deletion']; @@ -364,15 +375,12 @@ export class EndpointDocGenerator { } /** - * Creates new random policy id for the host to simulate new policy application + * Updates the current Host common record applied Policy to a different one from the list + * of random choices and gives it a random policy response status. */ - public updatePolicyId() { - this.commonInfo.Endpoint.policy.applied.id = this.randomChoice(APPLIED_POLICIES).id; - this.commonInfo.Endpoint.policy.applied.status = this.randomChoice([ - HostPolicyResponseActionStatus.success, - HostPolicyResponseActionStatus.failure, - HostPolicyResponseActionStatus.warning, - ]); + public updateHostPolicyData() { + this.commonInfo.Endpoint.policy.applied = this.randomChoice(APPLIED_POLICIES); + this.commonInfo.Endpoint.policy.applied.status = this.randomChoice(POLICY_RESPONSE_STATUSES); } private createHostData(): HostInfo { diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index b8c2fdbe65f1e..bf3d12f231c86 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -6,25 +6,66 @@ import { Client } from '@elastic/elasticsearch'; import seedrandom from 'seedrandom'; +import { KbnClient } from '@kbn/dev-utils'; +import { AxiosResponse } from 'axios'; import { EndpointDocGenerator, TreeOptions, Event } from './generate_data'; import { firstNonNullValue } from './models/ecs_safety_helpers'; +import { + CreateAgentPolicyRequest, + CreateAgentPolicyResponse, + CreatePackagePolicyRequest, + CreatePackagePolicyResponse, + GetPackagesResponse, + PostAgentEnrollRequest, + AGENT_API_ROUTES, + AGENT_POLICY_API_ROUTES, + EPM_API_ROUTES, + PACKAGE_POLICY_API_ROUTES, + ENROLLMENT_API_KEY_ROUTES, + GetEnrollmentAPIKeysResponse, + GetOneEnrollmentAPIKeyResponse, + PostAgentEnrollResponse, + PostAgentCheckinRequest, + PostAgentCheckinResponse, + PostAgentAcksResponse, + PostAgentAcksRequest, +} from '../../../ingest_manager/common'; +import { factory as policyConfigFactory } from './models/policy_config'; +import { HostMetadata } from './types'; +import { KbnClientWithApiKeySupport } from '../../scripts/endpoint/kbn_client_with_api_key_support'; export async function indexHostsAndAlerts( client: Client, + kbnClient: KbnClientWithApiKeySupport, seed: string, numHosts: number, numDocs: number, metadataIndex: string, - policyIndex: string, + policyResponseIndex: string, eventIndex: string, alertIndex: string, alertsPerHost: number, + fleet: boolean, options: TreeOptions = {} ) { const random = seedrandom(seed); + const epmEndpointPackage = await getEndpointPackageInfo(kbnClient); + // Keep a map of host applied policy ids (fake) to real ingest package configs (policy record) + const realPolicies: Record = {}; + for (let i = 0; i < numHosts; i++) { const generator = new EndpointDocGenerator(random); - await indexHostDocs(numDocs, client, metadataIndex, policyIndex, generator); + await indexHostDocs( + numDocs, + client, + kbnClient, + realPolicies, + epmEndpointPackage, + metadataIndex, + policyResponseIndex, + fleet, + generator + ); await indexAlerts(client, eventIndex, alertIndex, generator, alertsPerHost, options); } await client.indices.refresh({ @@ -43,22 +84,78 @@ function delay(ms: number) { async function indexHostDocs( numDocs: number, client: Client, + kbnClient: KbnClientWithApiKeySupport, + realPolicies: Record, + epmEndpointPackage: GetPackagesResponse['response'][0], metadataIndex: string, - policyIndex: string, + policyResponseIndex: string, + enrollFleet: boolean, generator: EndpointDocGenerator ) { const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents const timestamp = new Date().getTime(); + let hostMetadata: HostMetadata; + let wasAgentEnrolled = false; + let enrolledAgent: undefined | PostAgentEnrollResponse['item']; + for (let j = 0; j < numDocs; j++) { generator.updateHostData(); - generator.updatePolicyId(); + generator.updateHostPolicyData(); + + hostMetadata = generator.generateHostMetadata(timestamp - timeBetweenDocs * (numDocs - j - 1)); + + if (enrollFleet) { + const { id: appliedPolicyId, name: appliedPolicyName } = hostMetadata.Endpoint.policy.applied; + + // If we don't yet have a "real" policy record, then create it now in ingest (package config) + if (!realPolicies[appliedPolicyId]) { + // eslint-disable-next-line require-atomic-updates + realPolicies[appliedPolicyId] = await createPolicy( + kbnClient, + appliedPolicyName, + epmEndpointPackage.version + ); + } + + // If we did not yet enroll an agent for this Host, do it now that we have good policy id + if (!wasAgentEnrolled) { + wasAgentEnrolled = true; + enrolledAgent = await fleetEnrollAgentForHost( + kbnClient, + hostMetadata!, + realPolicies[appliedPolicyId].policy_id + ); + } + // Update the Host metadata record with the ID of the "real" policy along with the enrolled agent id + hostMetadata = { + ...hostMetadata, + elastic: { + ...hostMetadata.elastic, + agent: { + ...hostMetadata.elastic.agent, + id: enrolledAgent?.id ?? hostMetadata.elastic.agent.id, + }, + }, + Endpoint: { + ...hostMetadata.Endpoint, + policy: { + ...hostMetadata.Endpoint.policy, + applied: { + ...hostMetadata.Endpoint.policy.applied, + id: realPolicies[appliedPolicyId].id, + }, + }, + }, + }; + } + await client.index({ index: metadataIndex, - body: generator.generateHostMetadata(timestamp - timeBetweenDocs * (numDocs - j - 1)), + body: hostMetadata, op_type: 'create', }); await client.index({ - index: policyIndex, + index: policyResponseIndex, body: generator.generatePolicyResponse(timestamp - timeBetweenDocs * (numDocs - j - 1)), op_type: 'create', }); @@ -98,3 +195,287 @@ async function indexAlerts( await client.bulk({ body, refresh: true }); } } + +const createPolicy = async ( + kbnClient: KbnClient, + policyName: string, + endpointPackageVersion: string +): Promise => { + // Create Agent Policy first + const newAgentPolicyData: CreateAgentPolicyRequest['body'] = { + name: `Policy for ${policyName}`, + description: '', + namespace: 'default', + }; + let agentPolicy; + try { + agentPolicy = (await kbnClient.request({ + path: AGENT_POLICY_API_ROUTES.CREATE_PATTERN, + method: 'POST', + body: newAgentPolicyData, + })) as AxiosResponse; + } catch (error) { + throw new Error(`create policy ${error}`); + } + + // Create Package Configuration + const newPackagePolicyData: CreatePackagePolicyRequest['body'] = { + name: policyName, + description: 'Protect the worlds data', + policy_id: agentPolicy.data.item.id, + enabled: true, + output_id: '', + inputs: [ + { + type: 'endpoint', + enabled: true, + streams: [], + config: { + policy: { + value: policyConfigFactory(), + }, + }, + }, + ], + namespace: 'default', + package: { + name: 'endpoint', + title: 'endpoint', + version: endpointPackageVersion, + }, + }; + const packagePolicy = (await kbnClient.request({ + path: PACKAGE_POLICY_API_ROUTES.CREATE_PATTERN, + method: 'POST', + body: newPackagePolicyData, + })) as AxiosResponse; + return packagePolicy.data.item; +}; + +const getEndpointPackageInfo = async ( + kbnClient: KbnClient +): Promise => { + const endpointPackage = ((await kbnClient.request({ + path: `${EPM_API_ROUTES.LIST_PATTERN}?category=security`, + method: 'GET', + })) as AxiosResponse).data.response.find( + (epmPackage) => epmPackage.name === 'endpoint' + ); + + if (!endpointPackage) { + throw new Error('EPM Endpoint package was not found!'); + } + + return endpointPackage; +}; + +const fleetEnrollAgentForHost = async ( + kbnClient: KbnClientWithApiKeySupport, + endpointHost: HostMetadata, + agentPolicyId: string +): Promise => { + // Get Enrollement key for host's applied policy + const enrollmentApiKey = await kbnClient + .request({ + path: ENROLLMENT_API_KEY_ROUTES.LIST_PATTERN, + method: 'GET', + query: { + kuery: `fleet-enrollment-api-keys.policy_id:"${agentPolicyId}"`, + }, + }) + .then((apiKeysResponse) => { + const apiKey = apiKeysResponse.data.list[0]; + + if (!apiKey) { + return Promise.reject( + new Error(`no API enrollment key found for agent policy id ${agentPolicyId}`) + ); + } + + return kbnClient + .request({ + path: ENROLLMENT_API_KEY_ROUTES.INFO_PATTERN.replace('{keyId}', apiKey.id), + method: 'GET', + }) + .catch((error) => { + // eslint-disable-next-line no-console + console.log('unable to retrieve enrollment api key for policy'); + return Promise.reject(error); + }); + }) + .then((apiKeyDetailsResponse) => { + return apiKeyDetailsResponse.data.item.api_key; + }) + .catch((error) => { + // eslint-disable-next-line no-console + console.error(error); + return ''; + }); + + if (enrollmentApiKey.length === 0) { + return; + } + + const fetchKibanaVersion = async () => { + const version = ((await kbnClient.request({ + path: '/api/status', + method: 'GET', + })) as AxiosResponse).data.version.number; + if (!version) { + // eslint-disable-next-line no-console + console.log('failed to retrieve kibana version'); + } + return version; + }; + + // Enroll an agent for the Host + const body: PostAgentEnrollRequest['body'] = { + type: 'PERMANENT', + metadata: { + local: { + elastic: { + agent: { + version: await fetchKibanaVersion(), + }, + }, + host: { + architecture: 'x86_64', + hostname: endpointHost.host, + name: endpointHost.host, + id: '1c032ec0-3a94-4d54-9ad2-c5610c0eaba4', + ip: ['fe80::703b:b9e6:887d:7f5/64', '10.0.2.15/24', '::1/128', '127.0.0.1/8'], + mac: ['08:00:27:d8:c5:c0'], + }, + os: { + family: 'windows', + kernel: '10.0.19041.388 (WinBuild.160101.0800)', + platform: 'windows', + version: '10.0', + name: 'Windows 10 Pro', + full: 'Windows 10 Pro(10.0)', + }, + }, + user_provided: { + dev_agent_version: '0.0.1', + region: 'us-east', + }, + }, + }; + + try { + // First enroll the agent + const res = await kbnClient.requestWithApiKey(AGENT_API_ROUTES.ENROLL_PATTERN, { + method: 'POST', + body: JSON.stringify(body), + headers: { + 'kbn-xsrf': 'xxx', + Authorization: `ApiKey ${enrollmentApiKey}`, + 'Content-Type': 'application/json', + }, + }); + + if (res) { + const enrollObj: PostAgentEnrollResponse = await res.json(); + if (!res.ok) { + // eslint-disable-next-line no-console + console.error('unable to enroll agent', enrollObj); + return; + } + // ------------------------------------------------ + // now check the agent in so that it can complete enrollment + const checkinBody: PostAgentCheckinRequest['body'] = { + events: [ + { + type: 'STATE', + subtype: 'RUNNING', + message: 'state changed from STOPPED to RUNNING', + timestamp: new Date().toISOString(), + payload: { + random: 'data', + state: 'RUNNING', + previous_state: 'STOPPED', + }, + agent_id: enrollObj.item.id, + }, + ], + }; + const checkinRes = await kbnClient + .requestWithApiKey( + AGENT_API_ROUTES.CHECKIN_PATTERN.replace('{agentId}', enrollObj.item.id), + { + method: 'POST', + body: JSON.stringify(checkinBody), + headers: { + 'kbn-xsrf': 'xxx', + Authorization: `ApiKey ${enrollObj.item.access_api_key}`, + 'Content-Type': 'application/json', + }, + } + ) + .catch((error) => { + return Promise.reject(error); + }); + + // Agent unenrolling? + if (checkinRes.status === 403) { + return; + } + + const checkinObj: PostAgentCheckinResponse = await checkinRes.json(); + if (!checkinRes.ok) { + // eslint-disable-next-line no-console + console.error( + `failed to checkin agent [${enrollObj.item.id}] for endpoint [${endpointHost.host.id}]` + ); + return enrollObj.item; + } + + // ------------------------------------------------ + // If we have an action to ack(), then do it now + if (checkinObj.actions.length) { + const ackActionBody: PostAgentAcksRequest['body'] = { + // @ts-ignore + events: checkinObj.actions.map((action) => { + return { + action_id: action.id, + type: 'ACTION_RESULT', + subtype: 'CONFIG', + timestamp: new Date().toISOString(), + agent_id: action.agent_id, + policy_id: agentPolicyId, + message: `endpoint generator: Endpoint Started`, + }; + }), + }; + const ackActionResp = await kbnClient.requestWithApiKey( + AGENT_API_ROUTES.ACKS_PATTERN.replace('{agentId}', enrollObj.item.id), + { + method: 'POST', + body: JSON.stringify(ackActionBody), + headers: { + 'kbn-xsrf': 'xxx', + Authorization: `ApiKey ${enrollObj.item.access_api_key}`, + 'Content-Type': 'application/json', + }, + } + ); + + const ackActionObj: PostAgentAcksResponse = await ackActionResp.json(); + if (!ackActionResp.ok) { + // eslint-disable-next-line no-console + console.error( + `failed to ACK Actions provided to agent [${enrollObj.item.id}] for endpoint [${endpointHost.host.id}]` + ); + // eslint-disable-next-line no-console + console.error(JSON.stringify(ackActionObj, null, 2)); + return enrollObj.item; + } + } + + return enrollObj.item; + } + } catch (error) { + // eslint-disable-next-line no-console + console.error(error); + } +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/kbn_client_with_api_key_support.ts b/x-pack/plugins/security_solution/scripts/endpoint/kbn_client_with_api_key_support.ts new file mode 100644 index 0000000000000..526c0eb4a9055 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/kbn_client_with_api_key_support.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { KbnClient, ToolingLog } from '@kbn/dev-utils'; +import { KibanaConfig } from '@kbn/dev-utils/target/kbn_client/kbn_client_requester'; +import fetch, { RequestInit as FetchRequestInit } from 'node-fetch'; + +export class KbnClientWithApiKeySupport extends KbnClient { + private kibanaUrlNoAuth: string; + constructor(log: ToolingLog, kibanaConfig: KibanaConfig) { + super(log, kibanaConfig); + const kibanaUrl = this.resolveUrl(kibanaConfig.url); + const matches = kibanaUrl.match(/(https?:\/\/)(.*\:.*\@)(.*)/); + // strip auth from url + this.kibanaUrlNoAuth = + matches && matches.length >= 3 + ? matches[1] + matches[3].replace('/', '') + : kibanaUrl.replace('/', ''); + } + /** + * The fleet api to enroll and agent requires an api key when you mke the request, however KbnClient currently does not support sending an api key with the request. This function allows you to send an api key with a request. + */ + requestWithApiKey(path: string, init?: RequestInit | undefined): Promise { + return (fetch( + `${this.kibanaUrlNoAuth}${path}`, + init as FetchRequestInit + ) as unknown) as Promise; + } +} diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts index cfe1c741ef3f1..1c2c4a857451b 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts @@ -4,14 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ /* eslint-disable no-console */ -import * as path from 'path'; import yargs from 'yargs'; -import * as url from 'url'; -import fetch from 'node-fetch'; import { Client, ClientOptions } from '@elastic/elasticsearch'; import { ResponseError } from '@elastic/elasticsearch/lib/errors'; +import { KbnClient, ToolingLog } from '@kbn/dev-utils'; +import { AxiosResponse } from 'axios'; import { indexHostsAndAlerts } from '../../common/endpoint/index_data'; import { ANCESTRY_LIMIT } from '../../common/endpoint/generate_data'; +import { FLEET_SETUP_API_ROUTES, SETUP_API_ROUTE } from '../../../ingest_manager/common/constants'; +import { + CreateFleetSetupResponse, + PostIngestSetupResponse, +} from '../../../ingest_manager/common/types/rest_spec'; +import { KbnClientWithApiKeySupport } from './kbn_client_with_api_key_support'; main(); @@ -35,42 +40,37 @@ async function deleteIndices(indices: string[], client: Client) { } } -async function doIngestSetup(kibanaURL: string) { +async function doIngestSetup(kbnClient: KbnClient) { + // Setup Ingest try { - const kbURL = new url.URL(kibanaURL); - // this includes the base path that is randomly generated by Kibana - const pathname = path.posix.join(path.posix.sep, kbURL.pathname, 'api/ingest_manager/setup'); - const connectURL = new url.URL(pathname, kbURL); - console.log('Calling ingest manager setup at ', connectURL.toString()); - const response = await fetch( - // wrap base url in URL class because the kibana basepath will get removed otherwise - connectURL.toString(), - { - method: 'POST', - headers: { - 'kbn-xsrf': 'blah', - }, - } - ); - if (response.status !== 200) { - console.log('POST response ', response); - console.log( - 'Request failed please check that you have the correct base path and credentials for the kibana URL' - ); - // eslint-disable-next-line no-process-exit - process.exit(1); + const setupResponse = (await kbnClient.request({ + path: SETUP_API_ROUTE, + method: 'POST', + })) as AxiosResponse; + + if (!setupResponse.data.isInitialized) { + console.error(setupResponse.data); + throw new Error('Initializing the ingest manager failed, existing'); } - const setupResponse = await response.json(); - console.log('Ingest setup response ', setupResponse); - if (!setupResponse?.isInitialized) { - console.log('Initializing the ingest manager failed, existing'); - // eslint-disable-next-line no-process-exit - process.exit(1); + } catch (error) { + console.error(error); + throw error; + } + + // Setup Fleet + try { + const setupResponse = (await kbnClient.request({ + path: FLEET_SETUP_API_ROUTES.CREATE_PATTERN, + method: 'POST', + })) as AxiosResponse; + + if (!setupResponse.data.isInitialized) { + console.error(setupResponse.data); + throw new Error('Initializing Fleet failed, existing'); } } catch (error) { - console.log(JSON.stringify(error, null, 2)); - // eslint-disable-next-line no-process-exit - process.exit(1); + console.error(error); + throw error; } } @@ -196,14 +196,25 @@ async function main() { type: 'boolean', default: false, }, + fleet: { + alias: 'f', + describe: 'enroll fleet agents for hosts', + type: 'boolean', + default: false, + }, }).argv; - await doIngestSetup(argv.kibana); + const kbnClient = new KbnClientWithApiKeySupport(new ToolingLog(), { url: argv.kibana }); - const clientOptions: ClientOptions = { - node: argv.node, - }; + try { + await doIngestSetup(kbnClient); + } catch (error) { + // eslint-disable-next-line no-process-exit + process.exit(1); + } + const clientOptions: ClientOptions = { node: argv.node }; const client = new Client(clientOptions); + if (argv.delete) { await deleteIndices( [argv.eventIndex, argv.metadataIndex, argv.policyIndex, argv.alertIndex], @@ -219,6 +230,7 @@ async function main() { const startTime = new Date().getTime(); await indexHostsAndAlerts( client, + kbnClient, seed, argv.numHosts, argv.numDocs, @@ -227,6 +239,7 @@ async function main() { argv.eventIndex, argv.alertIndex, argv.alertsPerHost, + argv.fleet, { ancestors: argv.ancestors, generations: argv.generations,