diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index eade9c21d..4a65c3f56 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -102,12 +102,14 @@ jobs: run: npm run build env: REACT_APP_API_URL: https://api.staging.crossfeed.cyber.dhs.gov + REACT_APP_FARGATE_LOG_GROUP: crossfeed-staging-worker - name: Build Production if: github.ref == 'refs/heads/production' run: npm run build env: REACT_APP_API_URL: https://api.crossfeed.cyber.dhs.gov + REACT_APP_FARGATE_LOG_GROUP: crossfeed-prod-worker - name: Deploy Staging if: github.ref == 'refs/heads/master' diff --git a/backend/Dockerfile.worker b/backend/Dockerfile.worker index 54386ba22..32f780d52 100644 --- a/backend/Dockerfile.worker +++ b/backend/Dockerfile.worker @@ -40,4 +40,4 @@ COPY --from=deps /usr/bin/findomain /usr/bin/amass /go/bin/csv2cpe /go/bin/nvdsy COPY --from=deps /etc/ssl/certs /etc/ssl/certs -CMD ["node", "worker.bundle.js"] +CMD ["node", "--unhandled-rejections=strict", "worker.bundle.js"] diff --git a/backend/env.yml b/backend/env.yml index e6f710b01..b9a59aa38 100644 --- a/backend/env.yml +++ b/backend/env.yml @@ -61,3 +61,7 @@ prod-vpc: - ${ssm:/crossfeed/prod/SG_ID} subnetIds: - ${ssm:/crossfeed/prod/SUBNET_ID} + +staging-ecs-cluster: ${ssm:/crossfeed/staging/WORKER_CLUSTER_ARN} + +prod-ecs-cluster: ${ssm:/crossfeed/prod/WORKER_CLUSTER_ARN} \ No newline at end of file diff --git a/backend/package-lock.json b/backend/package-lock.json index cc78dd13d..09199a3ca 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -15641,6 +15641,12 @@ "xml-name-validator": "^3.0.0" } }, + "wait-for-expect": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/wait-for-expect/-/wait-for-expect-3.0.2.tgz", + "integrity": "sha512-cfS1+DZxuav1aBYbaO/kE06EOS8yRw7qOFoD3XtjTkYvCvh3zUvNST8DXK/nPaeqIzIv3P3kL3lRJn8iwOiSag==", + "dev": true + }, "walker": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.7.tgz", diff --git a/backend/package.json b/backend/package.json index 96cee7d20..d2178ace8 100644 --- a/backend/package.json +++ b/backend/package.json @@ -53,6 +53,7 @@ "ts-node": "^8.10.2", "ts-node-dev": "^1.0.0-pre.52", "typescript": "^3.9.7", + "wait-for-expect": "^3.0.2", "webpack": "^4.43.0", "webpack-cli": "^3.3.12" }, diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index cb8dc9b91..24811ad67 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -3,7 +3,6 @@ import * as bodyParser from 'body-parser'; import * as cors from 'cors'; import { handler as healthcheck } from './healthcheck'; import * as auth from './auth'; -import { login } from './auth'; import * as domains from './domains'; import * as vulnerabilities from './vulnerabilities'; import * as organizations from './organizations'; @@ -11,6 +10,14 @@ import * as scans from './scans'; import * as users from './users'; import * as scanTasks from './scan-tasks'; import * as stats from './stats'; +import { listenForDockerEvents } from './docker-events'; + +if ( + (process.env.IS_OFFLINE || process.env.IS_LOCAL) && + typeof jest === 'undefined' +) { + listenForDockerEvents(); +} const handlerToExpress = (handler) => async (req, res, next) => { const { statusCode, body } = await handler( diff --git a/backend/src/api/docker-events.ts b/backend/src/api/docker-events.ts new file mode 100644 index 000000000..cfaa54f37 --- /dev/null +++ b/backend/src/api/docker-events.ts @@ -0,0 +1,51 @@ +import * as Docker from 'dockerode'; +import { + handler as updateScanTaskStatus, + EventBridgeEvent +} from '../tasks/updateScanTaskStatus'; + +/** + * Listens for Docker events and converts start / stop events to corresponding Fargate EventBridge events, + * so that they can be handled by the updateScanTaskStatus lambda function. + * + * This function is only used for runs during local development in order to simulate EventBridge events. + */ +export const listenForDockerEvents = async () => { + const docker = new Docker(); + const stream = await docker.getEvents(); + stream.on('data', async (chunk: any) => { + const message = JSON.parse(Buffer.from(chunk).toString('utf-8')); + if (message.from !== 'crossfeed-worker') { + return; + } + let payload: EventBridgeEvent; + if (message.status === 'start') { + payload = { + detail: { + stopCode: '', + stoppedReason: '', + taskArn: message.Actor.Attributes.name, + lastStatus: 'RUNNING', + containers: [{}] + } + }; + } else if (message.status === 'die') { + payload = { + detail: { + stopCode: 'EssentialContainerExited', + stoppedReason: 'Essential container in task exited', + taskArn: message.Actor.Attributes.name, + lastStatus: 'STOPPED', + containers: [ + { + exitCode: Number(message.Actor.Attributes.exitCode) + } + ] + } + }; + } else { + return; + } + await updateScanTaskStatus(payload, {} as any, () => null); + }); +}; diff --git a/backend/src/models/scan-task.ts b/backend/src/models/scan-task.ts index 7b9c5a9b4..6e8371237 100644 --- a/backend/src/models/scan-task.ts +++ b/backend/src/models/scan-task.ts @@ -47,6 +47,15 @@ export class ScanTask extends BaseEntity { @Column('text') type: 'fargate' | 'lambda'; + /** + * ARN of the associated fargate task. + */ + @Column({ + type: 'text', + nullable: true + }) + fargateTaskArn: string | null; + @Column({ type: 'text', nullable: true diff --git a/backend/src/tasks/__mocks__/ecs-client.ts b/backend/src/tasks/__mocks__/ecs-client.ts index d40ba734e..6e5b22519 100644 --- a/backend/src/tasks/__mocks__/ecs-client.ts +++ b/backend/src/tasks/__mocks__/ecs-client.ts @@ -1,7 +1,13 @@ import { CommandOptions } from '../ecs-client'; export const runCommand = jest.fn(async (commandOptions: CommandOptions) => { - return { tasks: [{}] }; + return { + tasks: [ + { + taskArn: 'mock_task_arn' + } + ] + }; }); export default jest.fn(() => ({ diff --git a/backend/src/tasks/ecs-client.ts b/backend/src/tasks/ecs-client.ts index bfce39fcb..b7a6e0934 100644 --- a/backend/src/tasks/ecs-client.ts +++ b/backend/src/tasks/ecs-client.ts @@ -1,5 +1,6 @@ import { ECS } from 'aws-sdk'; import { SCAN_SCHEMA } from '../api/scans'; +import * as Docker from 'dockerode'; export interface CommandOptions { organizationId?: string; @@ -20,14 +21,13 @@ const toSnakeCase = (input) => input.replace(/ /g, '-'); */ class ECSClient { ecs?: ECS; - docker?: any; + docker?: Docker; isLocal: boolean; constructor() { this.isLocal = process.env.IS_OFFLINE || process.env.IS_LOCAL ? true : false; if (this.isLocal) { - const Docker = require('dockerode'); this.docker = new Docker(); } else { this.ecs = new ECS(); @@ -50,13 +50,14 @@ class ECSClient { const { cpu, memory, global } = SCAN_SCHEMA[scanName]; if (this.isLocal) { try { + const containerName = toSnakeCase( + `crossfeed_worker_${ + global ? 'global' : organizationName + }_${scanName}_` + Math.floor(Math.random() * 10000000) + ); const container = await this.docker!.createContainer({ // We need to create unique container names to avoid conflicts. - name: toSnakeCase( - `crossfeed_worker_${ - global ? 'global' : organizationName - }_${scanName}_` + Math.floor(Math.random() * 10000000) - ), + name: containerName, Image: 'crossfeed-worker', Env: [ `CROSSFEED_COMMAND_OPTIONS=${JSON.stringify(commandOptions)}`, @@ -76,10 +77,14 @@ class ECSClient { // crossfeed-worker image; instead, we set NetworkMode to "host" and // connect to "localhost." NetworkMode: 'host' - }); + } as any); await container.start(); return { - tasks: [{}], + tasks: [ + { + taskArn: containerName + } + ], failures: [] }; } catch (e) { diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index ba29f6a1c..deb663dbb 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -9,3 +9,16 @@ syncdb: makeGlobalAdmin: handler: src/tasks/makeGlobalAdmin.handler + +updateScanTaskStatus: + handler: src/tasks/updateScanTaskStatus.handler + events: + - eventBridge: + pattern: + source: + - aws.ecs + detail-type: + - ECS Task State Change + detail: + clusterArn: + - ${file(env.yml):${self:provider.stage}-ecs-cluster, ''} \ No newline at end of file diff --git a/backend/src/tasks/scheduler.ts b/backend/src/tasks/scheduler.ts index a211cd013..1bffe72a1 100644 --- a/backend/src/tasks/scheduler.ts +++ b/backend/src/tasks/scheduler.ts @@ -44,9 +44,11 @@ const launchSingleScanTask = async ({ } failures.` ); } + const taskArn = result.tasks![0].taskArn; + scanTask.fargateTaskArn = taskArn; if (typeof jest === 'undefined') { console.log( - `Successfully invoked ${scan.name} scan with fargate. ` + + `Successfully invoked ${scan.name} scan with fargate, with ECS task ARN ${taskArn}. ` + (numChunks ? ` Chunk ${chunkNumber}/${numChunks}` : '') ); } diff --git a/backend/src/tasks/test/scheduler.test.ts b/backend/src/tasks/test/scheduler.test.ts index 99ab85817..5ce9bf8ce 100644 --- a/backend/src/tasks/test/scheduler.test.ts +++ b/backend/src/tasks/test/scheduler.test.ts @@ -43,6 +43,7 @@ describe('scheduler', () => { runCommand.mock.calls[0][0].scanTaskId ); expect(scanTask?.status).toEqual('requested'); + expect(scanTask?.fargateTaskArn).toEqual('mock_task_arn'); scan = (await Scan.findOne(scan.id))!; expect(scan.lastRun).toBeTruthy(); diff --git a/backend/src/tasks/test/updateScanTaskStatus.test.ts b/backend/src/tasks/test/updateScanTaskStatus.test.ts new file mode 100644 index 000000000..7e214e433 --- /dev/null +++ b/backend/src/tasks/test/updateScanTaskStatus.test.ts @@ -0,0 +1,126 @@ +import { + handler as updateScanTaskStatus, + EventBridgeEvent +} from '../updateScanTaskStatus'; +import { connectToDatabase, Scan, ScanTask } from '../../models'; + +let scan; +beforeAll(async () => { + await connectToDatabase(); + scan = await Scan.create({ + name: 'findomain', + arguments: {}, + frequency: 999 + }).save(); +}); + +const createSampleEvent = ({ + lastStatus, + taskArn, + exitCode = 0 +}): EventBridgeEvent => ({ + detail: { + attachments: [], + availabilityZone: 'us-east-1b', + clusterArn: + 'arn:aws:ecs:us-east-1:563873274798:cluster/crossfeed-staging-worker', + containers: [ + { + containerArn: + 'arn:aws:ecs:us-east-1:563873274798:container/75d926f3-c850-4722-a143-639f02ff4756', + exitCode: exitCode, + lastStatus: lastStatus, + name: 'main', + image: + '563873274798.dkr.ecr.us-east-1.amazonaws.com/crossfeed-staging-worker:latest', + imageDigest: + 'sha256:080467614c0d7d5a5b092023b762931095308ae1dec8e54481fbd951f9784391', + runtimeId: 'eeccdb34-d8cf-49e7-b379-1cf4f123c0ee-3935363592', + taskArn: + 'arn:aws:ecs:us-east-1:563873274798:task/eeccdb34-d8cf-49e7-b379-1cf4f123c0ee', + networkInterfaces: [ + { + attachmentId: '5ff58204-1180-48ab-a62e-0a65a17cc4ef', + privateIpv4Address: '10.0.3.61' + } + ], + cpu: '0' + } + ], + launchType: 'FARGATE', + cpu: '256', + memory: '512', + desiredStatus: 'STOPPED', + group: 'family:crossfeed-staging-worker', + lastStatus: lastStatus, + overrides: { + containerOverrides: [ + { + environment: [], + name: 'main' + } + ] + }, + connectivity: 'CONNECTED', + stoppedReason: 'Essential container in task exited', + stopCode: 'EssentialContainerExited', + taskArn: taskArn, + taskDefinitionArn: + 'arn:aws:ecs:us-east-1:563873274798:task-definition/crossfeed-staging-worker:2', + version: 5, + platformVersion: '1.4.0' + } +}); + +test('starting event', async () => { + const taskArn = Math.random() + ''; + let scanTask = await ScanTask.create({ + scan, + type: 'fargate', + status: 'requested', + fargateTaskArn: taskArn + }).save(); + await updateScanTaskStatus( + createSampleEvent({ lastStatus: 'RUNNING', taskArn }), + {} as any, + () => null + ); + scanTask = (await ScanTask.findOne(scanTask.id))!; + expect(scanTask.status).toEqual('started'); +}); + +test('finished event', async () => { + const taskArn = Math.random() + ''; + let scanTask = await ScanTask.create({ + scan, + type: 'fargate', + status: 'started', + fargateTaskArn: taskArn + }).save(); + await updateScanTaskStatus( + createSampleEvent({ lastStatus: 'STOPPED', taskArn, exitCode: 0 }), + {} as any, + () => null + ); + scanTask = (await ScanTask.findOne(scanTask.id))!; + expect(scanTask.status).toEqual('finished'); + expect(scanTask.finishedAt).toBeTruthy(); + expect(scanTask.output).toContain('EssentialContainerExited'); +}); + +test('failed event', async () => { + const taskArn = Math.random() + ''; + let scanTask = await ScanTask.create({ + scan, + type: 'fargate', + status: 'started', + fargateTaskArn: taskArn + }).save(); + await updateScanTaskStatus( + createSampleEvent({ lastStatus: 'STOPPED', taskArn, exitCode: 1 }), + {} as any, + () => null + ); + scanTask = (await ScanTask.findOne(scanTask.id))!; + expect(scanTask.status).toEqual('failed'); +}); diff --git a/backend/src/tasks/updateScanTaskStatus.ts b/backend/src/tasks/updateScanTaskStatus.ts new file mode 100644 index 000000000..6d5e195b9 --- /dev/null +++ b/backend/src/tasks/updateScanTaskStatus.ts @@ -0,0 +1,58 @@ +import { Handler } from 'aws-lambda'; +import { connectToDatabase, ScanTask } from '../models'; +import { Task } from 'aws-sdk/clients/ecs'; +import pRetry from 'p-retry'; + +export type EventBridgeEvent = { + detail: Task & { + stopCode?: string; + stoppedReason?: string; + taskArn: string; + lastStatus: FargateTaskStatus; + containers: { + exitCode?: number; + }[]; + }; +}; + +type FargateTaskStatus = + | 'PROVISIONING' + | 'PENDING' + | 'RUNNING' + | 'DEPROVISIONING' + | 'STOPPED'; + +export const handler: Handler = async ( + event: EventBridgeEvent +) => { + const { taskArn, lastStatus } = event.detail; + await connectToDatabase(); + const scanTask = await pRetry( + () => + ScanTask.findOne({ + fargateTaskArn: taskArn + }), + { retries: 3 } + ); + if (!scanTask) { + throw new Error(`Couldn't find scan with task arn ${taskArn}.`); + } + const oldStatus = scanTask.status; + if (lastStatus === 'RUNNING') { + scanTask.status = 'started'; + } else if (lastStatus === 'STOPPED') { + if (event.detail.containers![0]?.exitCode === 0) { + scanTask.status = 'finished'; + } else { + scanTask.status = 'failed'; + } + scanTask.output = `${event.detail.stopCode}: ${event.detail.stoppedReason}`; + scanTask.finishedAt = new Date(); + } else { + return; + } + console.log( + `Updating status of ScanTask ${scanTask.id} from ${oldStatus} to ${scanTask.status}.` + ); + await scanTask.save(); +}; diff --git a/backend/src/worker.ts b/backend/src/worker.ts index 0c7c51168..1762b4bf2 100644 --- a/backend/src/worker.ts +++ b/backend/src/worker.ts @@ -1,5 +1,4 @@ import { CommandOptions } from './tasks/ecs-client'; -import { connectToDatabase, ScanTask } from './models'; import { handler as amass } from './tasks/amass'; import { handler as censys } from './tasks/censys'; import { handler as findomain } from './tasks/findomain'; @@ -12,43 +11,27 @@ import { handler as cve } from './tasks/cve'; * Worker entrypoint. */ async function main() { - await connectToDatabase(); - const commandOptions: CommandOptions = JSON.parse( process.env.CROSSFEED_COMMAND_OPTIONS || '{}' ); console.log('commandOptions are', commandOptions); - const { scanName, scanTaskId } = commandOptions; - const scanTask = await ScanTask.findOneOrFail(scanTaskId); - - scanTask.status = 'started'; - scanTask.startedAt = new Date(); - await scanTask.save(); + const { scanName } = commandOptions; - try { - const scanFn = { - amass, - censys, - censysIpv4, - cve, - findomain, - portscanner, - wappalyzer - }[scanName]; - if (!scanFn) { - throw new Error('Invalid scan name ' + scanName); - } - await scanFn(commandOptions); - scanTask.status = 'finished'; - } catch (e) { - console.error(e); - scanTask.status = 'failed'; - scanTask.output = JSON.stringify(e); - } finally { - scanTask.finishedAt = new Date(); - await scanTask.save(); + const scanFn = { + amass, + censys, + censysIpv4, + cve, + findomain, + portscanner, + wappalyzer + }[scanName]; + if (!scanFn) { + throw new Error('Invalid scan name ' + scanName); } + + await scanFn(commandOptions); } main(); diff --git a/backend/test/docker-events.test.ts b/backend/test/docker-events.test.ts new file mode 100644 index 000000000..cb05ff8fa --- /dev/null +++ b/backend/test/docker-events.test.ts @@ -0,0 +1,99 @@ +import { listenForDockerEvents } from '../src/api/docker-events'; +import { handler as updateScanTaskStatus } from '../src/tasks/updateScanTaskStatus'; +import { Readable } from 'typeorm/platform/PlatformTools'; +import waitForExpect from 'wait-for-expect'; + +jest.mock('../src/tasks/updateScanTaskStatus', () => ({ + handler: jest.fn() +})); + +let event; +let stream: Readable; +jest.mock('dockerode', () => { + class MockDockerode { + async getEvents() { + stream = Readable.from([JSON.stringify(event)]); + return stream; + } + } + return MockDockerode; +}); + +afterEach(() => { + if (stream) { + stream.destroy(); + } +}); + +test('should listen to a start event', async () => { + event = { + status: 'start', + id: '9385a49c58efef1983ba56f5711b16b176f27b973252110e756e408894b3d0e9', + from: 'crossfeed-worker', + Type: 'container', + Action: 'start', + Actor: { + ID: '9385a49c58efef1983ba56f5711b16b176f27b973252110e756e408894b3d0e9', + Attributes: { + image: 'crossfeed-worker', + name: 'crossfeed_worker_cisa_censys_2681225' + } + }, + scope: 'local', + time: 1597458556, + timeNano: 1597458556898095000 + }; + listenForDockerEvents(); + await waitForExpect(() => { + expect(updateScanTaskStatus).toHaveBeenCalledWith( + { + detail: { + containers: [{}], + lastStatus: 'RUNNING', + stopCode: '', + stoppedReason: '', + taskArn: 'crossfeed_worker_cisa_censys_2681225' + } + }, + expect.anything(), + expect.anything() + ); + }); +}); + +test('should listen to a stop event', async () => { + event = { + status: 'die', + id: '9385a49c58efef1983ba56f5711b16b176f27b973252110e756e408894b3d0e9', + from: 'crossfeed-worker', + Type: 'container', + Action: 'die', + Actor: { + ID: '9385a49c58efef1983ba56f5711b16b176f27b973252110e756e408894b3d0e9', + Attributes: { + exitCode: '0', + image: 'crossfeed-worker', + name: 'crossfeed_worker_cisa_censys_2681225' + } + }, + scope: 'local', + time: 1597458571, + timeNano: 1597458571266194000 + }; + listenForDockerEvents(); + await waitForExpect(() => { + expect(updateScanTaskStatus).toHaveBeenCalledWith( + { + detail: { + containers: [{ exitCode: 0 }], + lastStatus: 'STOPPED', + stopCode: 'EssentialContainerExited', + stoppedReason: 'Essential container in task exited', + taskArn: 'crossfeed_worker_cisa_censys_2681225' + } + }, + expect.anything(), + expect.anything() + ); + }); +}); diff --git a/dev.env.example b/dev.env.example index fdcd2151e..ea7c0bb1d 100644 --- a/dev.env.example +++ b/dev.env.example @@ -5,6 +5,7 @@ DB_NAME=crossfeed JWT_SECRET=CHANGE_ME REACT_APP_API_URL=http://localhost:3000 +REACT_APP_FARGATE_LOG_GROUP=crossfeed-staging-worker CENSYS_API_ID= CENSYS_API_SECRET= diff --git a/docker-compose.yml b/docker-compose.yml index 099a546b2..953dcaf91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,7 @@ services: build: ./backend volumes: - ./backend/src:/app/src + - /var/run/docker.sock:/var/run/docker.sock ports: - "3000:3000" env_file: diff --git a/frontend/src/pages/Scans/ScanTasksView.tsx b/frontend/src/pages/Scans/ScanTasksView.tsx index 62dc5a3fa..279bfca56 100644 --- a/frontend/src/pages/Scans/ScanTasksView.tsx +++ b/frontend/src/pages/Scans/ScanTasksView.tsx @@ -1,5 +1,5 @@ import React, { useCallback, useState } from 'react'; -import { TableInstance, Column } from 'react-table'; +import { TableInstance, Column, CellProps } from 'react-table'; import { Query } from 'types'; import { Table, Paginator, ColumnFilter, selectFilter } from 'components'; import { ScanTask } from 'types'; @@ -103,23 +103,40 @@ export const ScanTasksView: React.FC = () => { accessor: 'output', disableFilters: true, maxWidth: 200, - Cell: ({ value }: { value: string }) => ( - 
-          {value}
-
- ) + Cell: ({ value }: CellProps) => + value && ( + 
+            {value}
+
+ ) }, { Header: 'Actions', id: 'actions', - Cell: ({ row }: { row: { index: number; original: ScanTask } }) => ( + Cell: ({ row }: CellProps) => ( <> + {row.original.fargateTaskArn && ( + <> + + Logs + +   + + )} {row.original.status !== 'finished' && row.original.status !== 'failed' && (