From 7aea62aece3ba887fbcb4773dc4ee04162891256 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 17 Mar 2022 08:10:32 -0400 Subject: [PATCH 01/27] Add Pe db credentials to stage.tfvars worker and vars.tf --- .github/workflows/backend.yml | 8 ++++---- .../worker/pe_scripts/sixgill/run_cybersixgill.py | 3 +-- infrastructure/stage.tfvars | 3 +++ infrastructure/vars.tf | 14 ++++++++++++++ infrastructure/worker.tf | 12 ++++++++++++ 5 files changed, 34 insertions(+), 6 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index cb3a69ed5..62b1c8e13 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -3,14 +3,14 @@ name: Backend Pipeline on: push: branches: - - master + - add-pe-db-creds - production paths: - 'backend/**' - '.github/workflows/backend.yml' pull_request: branches: - - master + - add-pe-db-creds - production paths: - 'backend/**' @@ -114,7 +114,7 @@ jobs: runs-on: ubuntu-latest environment: staging concurrency: 1 - if: github.event_name == 'push' && github.ref == 'refs/heads/master' + if: github.event_name == 'push' && github.ref == 'refs/heads/add-pe-db-creds' steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 @@ -129,7 +129,7 @@ jobs: ${{ runner.os }}-node- - name: Install dependencies run: npm ci - + - name: Ensure domain exists run: npx sls create_domain --stage=staging env: diff --git a/backend/worker/pe_scripts/sixgill/run_cybersixgill.py b/backend/worker/pe_scripts/sixgill/run_cybersixgill.py index 9e68f6132..96281a552 100644 --- a/backend/worker/pe_scripts/sixgill/run_cybersixgill.py +++ b/backend/worker/pe_scripts/sixgill/run_cybersixgill.py @@ -22,6 +22,7 @@ PE_DB_NAME = os.environ.get("PE_DB_NAME") PE_DB_USERNAME = os.environ.get("PE_DB_USERNAME") PE_DB_PASSWORD = os.environ.get("PE_DB_PASSWORD") +print(PE_DB_USERNAME) org_id = os.environ.get("org_id") org_name = os.environ.get("org_name") @@ -38,8 +39,6 @@ back = timedelta(days=16) from_date = (to_date - back).strftime("%Y-%m-%d %H:%M:%S") to_date = to_date.strftime("%Y-%m-%d %H:%M:%S") -print(to_date) -print(from_date) def cve(cveid): diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars index c9c6d8fa6..18e88f399 100644 --- a/infrastructure/stage.tfvars +++ b/infrastructure/stage.tfvars @@ -19,6 +19,9 @@ ssm_db_name = "/crossfeed/staging/DATABASE_NAME" ssm_db_host = "/crossfeed/staging/DATABASE_HOST" ssm_db_username = "/crossfeed/staging/DATABASE_USER" ssm_db_password = "/crossfeed/staging/DATABASE_PASSWORD" +ssm_pe_db_name = "/crossfeed/staging/PE_DATABASE_NAME" +ssm_pe_db_username = "/crossfeed/staging/PE_DATABASE_USERNAME" +ssm_pe_db_password = "/crossfeed/staging/PE_DATABASE_PASSWORD" ssm_matomo_db_password = "/crossfeed/staging/MATOMO_DATABASE_PASSWORD" ssm_worker_signature_public_key = "/crossfeed/staging/WORKER_SIGNATURE_PUBLIC_KEY" ssm_worker_signature_private_key = "/crossfeed/staging/WORKER_SIGNATURE_PRIVATE_KEY" diff --git a/infrastructure/vars.tf b/infrastructure/vars.tf index 3413b5e7a..dc7fed3fe 100644 --- a/infrastructure/vars.tf +++ b/infrastructure/vars.tf @@ -63,6 +63,20 @@ variable "ssm_db_host" { default = "/crossfeed/staging/DATABASE_HOST" } +variable "ssm_pe_db_name" { + type = string + default = "/crossfeed/staging/PE_DB_NAME" +} + +variable "ssm_pe_db_username" { + type = string + default = "/crossfeed/staging/PE_DB_USERNAME" +} + +variable "ssm_pe_db_password" { + type = string + default = "/crossfeed/staging/PE_DB_PASSWORD" +} variable "ssm_lambda_sg" { type = string default = "/crossfeed/staging/SG_ID" diff --git a/infrastructure/worker.tf b/infrastructure/worker.tf index bbb71d4ca..f370cf070 100644 --- a/infrastructure/worker.tf +++ b/infrastructure/worker.tf @@ -213,6 +213,18 @@ resource "aws_ecs_task_definition" "worker" { "name": "DB_PASSWORD", "valueFrom": "${data.aws_ssm_parameter.db_password.arn}" }, + { + "name": "PE_DB_NAME", + "valueFrom": "${data.aws_ssm_parameter.pe_db_name.arn}" + }, + { + "name": "PE_DB_USERNAME", + "valueFrom": "${data.aws_ssm_parameter.pe_db_username.arn}" + }, + { + "name": "PE_DB_PASSWORD", + "valueFrom": "${data.aws_ssm_parameter.pe_db_password.arn}" + }, { "name": "CENSYS_API_ID", "valueFrom": "${data.aws_ssm_parameter.censys_api_id.arn}" From d03af9250399fc610edcb10ea43e31f55bea7364 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 17 Mar 2022 08:52:58 -0400 Subject: [PATCH 02/27] Add sixgill logging --- backend/worker/pe_scripts/sixgill/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/worker/pe_scripts/sixgill/config.py b/backend/worker/pe_scripts/sixgill/config.py index 49b9f6a03..60d70e939 100644 --- a/backend/worker/pe_scripts/sixgill/config.py +++ b/backend/worker/pe_scripts/sixgill/config.py @@ -12,6 +12,7 @@ SIXGILL_CLIENT_ID = os.environ.get("SIXGILL_CLIENT_ID") SIXGILL_CLIENT_SECRET = os.environ.get("SIXGILL_CLIENT_SECRET") +print(SIXGILL_CLIENT_ID) def token(): From 29c90830c06c5c5dab40b3af64842b592723a0bc Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 17 Mar 2022 09:58:31 -0400 Subject: [PATCH 03/27] Add await to writefile in hibp and domMasq sync --- backend/src/tasks/peDomMasq.ts | 7 ++++--- backend/src/tasks/peHibpSync.ts | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/src/tasks/peDomMasq.ts b/backend/src/tasks/peDomMasq.ts index 6f98cb558..e36c0e426 100644 --- a/backend/src/tasks/peDomMasq.ts +++ b/backend/src/tasks/peDomMasq.ts @@ -2,7 +2,7 @@ import { CommandOptions } from './ecs-client'; import { spawnSync } from 'child_process'; import { connectToDatabase, Vulnerability } from '../models'; import * as path from 'path'; -import { writeFileSync } from 'fs'; +import { promises as fs } from 'fs'; import { getPeEnv } from './helpers/getPeEnv'; const DOM_MASQ_DIRECTORY = '/app/worker/pe_scripts/peDomMasq'; @@ -21,7 +21,8 @@ export const handler = async (commandOptions: CommandOptions) => { .andWhere("vulnerability.source = 'dnstwist'") .getRawMany(); const input_path = path.join(DOM_MASQ_DIRECTORY, organizationId + '.json'); - writeFileSync(input_path, JSON.stringify(data)); + + await fs.writeFile(input_path, JSON.stringify(data)); const child = spawnSync( 'python3', @@ -30,7 +31,7 @@ export const handler = async (commandOptions: CommandOptions) => { stdio: 'pipe', encoding: 'utf-8', env: { - ...getPeEnv(), + ...process.env, data_path: input_path, org_id: organizationId, org_name: organizationName diff --git a/backend/src/tasks/peHibpSync.ts b/backend/src/tasks/peHibpSync.ts index 5cbfbe666..2cbaeaddc 100644 --- a/backend/src/tasks/peHibpSync.ts +++ b/backend/src/tasks/peHibpSync.ts @@ -2,7 +2,7 @@ import { CommandOptions } from './ecs-client'; import { spawnSync } from 'child_process'; import { connectToDatabase, Organization, Vulnerability } from '../models'; import * as path from 'path'; -import { writeFileSync } from 'fs'; +import { promises as fs } from 'fs'; import { getPeEnv } from './helpers/getPeEnv'; const HIBP_SYNC_DIRECTORY = '/app/worker/pe_scripts/hibpSyncFiles'; @@ -21,7 +21,7 @@ export const handler = async (commandOptions: CommandOptions) => { .getRawMany(); const INPUT_PATH = path.join(HIBP_SYNC_DIRECTORY, organizationId + '.json'); - writeFileSync(INPUT_PATH, JSON.stringify(vulnerabilities)); + await fs.writeFile(INPUT_PATH, JSON.stringify(vulnerabilities)); const child = spawnSync( 'python3', ['/app/worker/pe_scripts/sync_hibp_pe.py'], @@ -29,7 +29,7 @@ export const handler = async (commandOptions: CommandOptions) => { stdio: 'pipe', encoding: 'utf-8', env: { - ...getPeEnv(), + ...process.env, data_path: INPUT_PATH, org_name: organizationName, org_id: organizationId From e8f3069c690e0af73a4a97d7fbafb74a7ed3b0cc Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 17 Mar 2022 10:24:45 -0400 Subject: [PATCH 04/27] Fix pe-scripts writefile and env passing --- backend/src/tasks/peCybersixgill.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/tasks/peCybersixgill.ts b/backend/src/tasks/peCybersixgill.ts index 3a8010d7b..7e65a7b2a 100644 --- a/backend/src/tasks/peCybersixgill.ts +++ b/backend/src/tasks/peCybersixgill.ts @@ -1,6 +1,5 @@ import { CommandOptions } from './ecs-client'; import { spawnSync } from 'child_process'; -import { getPeEnv } from './helpers/getPeEnv'; // Call the sync_dnstwist_pe.py script that fetches dnstwist results, checks // IPs using the blocklist.de api, then updates the PE db instance @@ -14,7 +13,7 @@ export const handler = async (commandOptions: CommandOptions) => { stdio: 'pipe', encoding: 'utf-8', env: { - ...getPeEnv(), + ...process.env, org_name: organizationName, org_id: organizationId } From 9fe1f4ccba24a9b328a545945c5df9bd4554b46b Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 31 Mar 2022 10:14:01 -0400 Subject: [PATCH 05/27] update mitmproxy to 8.0.0 --- backend/worker/pe_scripts/sixgill/run_cybersixgill.py | 1 + backend/worker/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/worker/pe_scripts/sixgill/run_cybersixgill.py b/backend/worker/pe_scripts/sixgill/run_cybersixgill.py index d46d090af..4d2a2ff4a 100644 --- a/backend/worker/pe_scripts/sixgill/run_cybersixgill.py +++ b/backend/worker/pe_scripts/sixgill/run_cybersixgill.py @@ -22,6 +22,7 @@ PE_DB_USERNAME = os.environ.get("PE_DB_USERNAME") PE_DB_PASSWORD = os.environ.get("PE_DB_PASSWORD") print(PE_DB_USERNAME) +print(PE_DB_NAME) org_id = os.environ.get("org_id") org_name = os.environ.get("org_name") diff --git a/backend/worker/requirements.txt b/backend/worker/requirements.txt index 3f502cd71..1681d4045 100644 --- a/backend/worker/requirements.txt +++ b/backend/worker/requirements.txt @@ -1,6 +1,6 @@ requests-http-signature==0.2.0 requests==2.24.0 -mitmproxy==7.0.3 +mitmproxy==8.0.0 cryptography==3.3.2 pytest==6.0.1 scrapy==2.6.1 From c527a19fd3d89a09e0593052ca636528c009f913 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 31 Mar 2022 10:21:41 -0400 Subject: [PATCH 06/27] Update cryptography to support mitmproxy 8.0.0 --- backend/worker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/worker/requirements.txt b/backend/worker/requirements.txt index 1681d4045..ccc5ac29d 100644 --- a/backend/worker/requirements.txt +++ b/backend/worker/requirements.txt @@ -1,7 +1,7 @@ requests-http-signature==0.2.0 requests==2.24.0 mitmproxy==8.0.0 -cryptography==3.3.2 +cryptography==36.0.2 pytest==6.0.1 scrapy==2.6.1 dnstwist==20201228 From 68e7f62a3abd1631f73df41121a07ad9fa2f9972 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 31 Mar 2022 10:52:41 -0400 Subject: [PATCH 07/27] Add cryptography dep: cargo --- backend/Dockerfile.worker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Dockerfile.worker b/backend/Dockerfile.worker index 3b0806e29..7d228669a 100644 --- a/backend/Dockerfile.worker +++ b/backend/Dockerfile.worker @@ -30,7 +30,7 @@ FROM node:14-alpine3.14 WORKDIR /app -RUN apk add --update --no-cache wget build-base curl git unzip openssl-dev linux-headers python3=3.9.5-r2 python3-dev py3-pip ruby=2.7.5-r0 ruby-dev zlib-dev libffi-dev libxml2-dev libxslt-dev postgresql-dev gcc musl-dev py3-pandas py3-scikit-learn +RUN apk add --update --no-cache wget build-base curl git unzip openssl-dev linux-headers python3=3.9.5-r2 python3-dev py3-pip ruby=2.7.5-r0 ruby-dev zlib-dev libffi-dev libxml2-dev libxslt-dev postgresql-dev gcc musl-dev py3-pandas py3-scikit-learn cargo RUN npm install -g pm2@4 wait-port@0.2.9 From 8b44b2fe0cf5e0a954727a47b5eb7678c2fab54f Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 31 Mar 2022 13:29:45 -0400 Subject: [PATCH 08/27] Create directories to store temporary json data for PE --- backend/worker/pe_scripts/hibpSyncFiles/README.md | 1 + backend/worker/pe_scripts/peDomMasq/README.md | 1 + backend/worker/pe_scripts/sixgill/run_cybersixgill.py | 1 + 3 files changed, 3 insertions(+) create mode 100644 backend/worker/pe_scripts/hibpSyncFiles/README.md create mode 100644 backend/worker/pe_scripts/peDomMasq/README.md diff --git a/backend/worker/pe_scripts/hibpSyncFiles/README.md b/backend/worker/pe_scripts/hibpSyncFiles/README.md new file mode 100644 index 000000000..cba2c784b --- /dev/null +++ b/backend/worker/pe_scripts/hibpSyncFiles/README.md @@ -0,0 +1 @@ +This folder stores temporary json data when passing data to sync_hibp_pe.py. diff --git a/backend/worker/pe_scripts/peDomMasq/README.md b/backend/worker/pe_scripts/peDomMasq/README.md new file mode 100644 index 000000000..3c8ec795f --- /dev/null +++ b/backend/worker/pe_scripts/peDomMasq/README.md @@ -0,0 +1 @@ +This folder stores temporary json data when passing data to sync_dnstwist_pe.py. diff --git a/backend/worker/pe_scripts/sixgill/run_cybersixgill.py b/backend/worker/pe_scripts/sixgill/run_cybersixgill.py index 4d2a2ff4a..7729163cb 100644 --- a/backend/worker/pe_scripts/sixgill/run_cybersixgill.py +++ b/backend/worker/pe_scripts/sixgill/run_cybersixgill.py @@ -23,6 +23,7 @@ PE_DB_PASSWORD = os.environ.get("PE_DB_PASSWORD") print(PE_DB_USERNAME) print(PE_DB_NAME) +print(DB_HOST) org_id = os.environ.get("org_id") org_name = os.environ.get("org_name") From 12c6dd27fdb1733b3a00cf1f77b5eb283542bb63 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 31 Mar 2022 14:34:36 -0400 Subject: [PATCH 09/27] Fix PE environment variables --- backend/env.yml | 9 +++++++++ backend/src/tasks/peDomMasq.ts | 2 +- backend/worker/pe_scripts/sixgill/run_cybersixgill.py | 5 ++++- backend/worker/pe_scripts/sync_dnstwist_pe.py | 3 +++ docs/src/documentation-pages/dev/pe.md | 6 +++--- infrastructure/prod.tfvars | 3 +++ infrastructure/stage.tfvars | 6 +++--- 7 files changed, 26 insertions(+), 8 deletions(-) diff --git a/backend/env.yml b/backend/env.yml index 1081ca879..1460a18a4 100644 --- a/backend/env.yml +++ b/backend/env.yml @@ -8,6 +8,9 @@ staging: PE_DB_NAME: ${ssm:/crossfeed/staging/PE_DB_NAME~true} PE_DB_USERNAME: ${ssm:/crossfeed/staging/PE_DB_USERNAME~true} PE_DB_PASSWORD: ${ssm:/crossfeed/staging/PE_DB_PASSWORD~true} + SIXGILL_CLIENT_ID: ${ssm:/crossfeed/staging/SIXGILL_CLIENT_ID~true} + SIXGILL_CLIENT_SECRET: ${ssm:/crossfeed/staging/SIXGILL_CLIENT_SECRET~true} + PE_SHODAN_API_KEYS: ${ssm:/crossfeed/staging/PE_SHODAN_API_KEYS~true} JWT_SECRET: ${ssm:/crossfeed/staging/APP_JWT_SECRET~true} LOGIN_GOV_REDIRECT_URI: ${ssm:/crossfeed/staging/LOGIN_GOV_REDIRECT_URI~true} LOGIN_GOV_BASE_URL: ${ssm:/crossfeed/staging/LOGIN_GOV_BASE_URL~true} @@ -41,6 +44,12 @@ prod: DB_NAME: ${ssm:/crossfeed/prod/DATABASE_NAME~true} DB_USERNAME: ${ssm:/crossfeed/prod/DATABASE_USER~true} DB_PASSWORD: ${ssm:/crossfeed/prod/DATABASE_PASSWORD~true} + PE_DB_NAME: ${ssm:/crossfeed/prod/PE_DB_NAME~true} + PE_DB_USERNAME: ${ssm:/crossfeed/prod/PE_DB_USERNAME~true} + PE_DB_PASSWORD: ${ssm:/crossfeed/prod/PE_DB_PASSWORD~true} + SIXGILL_CLIENT_ID: ${ssm:/crossfeed/prod/SIXGILL_CLIENT_ID~true} + SIXGILL_CLIENT_SECRET: ${ssm:/crossfeed/prod/SIXGILL_CLIENT_SECRET~true} + PE_SHODAN_API_KEYS: ${ssm:/crossfeed/staging/PE_SHODAN_API_KEYS~true} JWT_SECRET: ${ssm:/crossfeed/prod/APP_JWT_SECRET~true} LOGIN_GOV_REDIRECT_URI: ${ssm:/crossfeed/prod/LOGIN_GOV_REDIRECT_URI~true} LOGIN_GOV_BASE_URL: ${ssm:/crossfeed/prod/LOGIN_GOV_BASE_URL~true} diff --git a/backend/src/tasks/peDomMasq.ts b/backend/src/tasks/peDomMasq.ts index e36c0e426..095dfc2ce 100644 --- a/backend/src/tasks/peDomMasq.ts +++ b/backend/src/tasks/peDomMasq.ts @@ -31,7 +31,7 @@ export const handler = async (commandOptions: CommandOptions) => { stdio: 'pipe', encoding: 'utf-8', env: { - ...process.env, + ...getPeEnv(), data_path: input_path, org_id: organizationId, org_name: organizationName diff --git a/backend/worker/pe_scripts/sixgill/run_cybersixgill.py b/backend/worker/pe_scripts/sixgill/run_cybersixgill.py index 7729163cb..e719e877c 100644 --- a/backend/worker/pe_scripts/sixgill/run_cybersixgill.py +++ b/backend/worker/pe_scripts/sixgill/run_cybersixgill.py @@ -21,6 +21,9 @@ PE_DB_NAME = os.environ.get("PE_DB_NAME") PE_DB_USERNAME = os.environ.get("PE_DB_USERNAME") PE_DB_PASSWORD = os.environ.get("PE_DB_PASSWORD") +SIXGILL_CLIENT_ID = os.environ.get("SIXGILL_CLIENT_ID") +SIXGILL_CLIENT_SECRET = os.environ.get("SIXGILL_CLIENT_SECRET") +print(SIXGILL_CLIENT_ID) print(PE_DB_USERNAME) print(PE_DB_NAME) print(DB_HOST) @@ -110,7 +113,7 @@ def getDataSource(conn, source): print("Failed fetching Cybersixgill aliases.") print(traceback.format_exc()) -"""Insert/Update Aliases into PE databse instance""" +"""Insert/Update Aliases into PE database instance""" try: # aliases_list = json.loads(aliases.replace("'", '"')) alias_df = pd.DataFrame(aliases, columns=["alias"]) diff --git a/backend/worker/pe_scripts/sync_dnstwist_pe.py b/backend/worker/pe_scripts/sync_dnstwist_pe.py index 366f6a052..8c3b774be 100644 --- a/backend/worker/pe_scripts/sync_dnstwist_pe.py +++ b/backend/worker/pe_scripts/sync_dnstwist_pe.py @@ -11,6 +11,9 @@ PE_DB_USERNAME = os.environ.get("PE_DB_USERNAME") PE_DB_PASSWORD = os.environ.get("PE_DB_PASSWORD") +print(DB_HOST) +print(PE_DB_NAME) + org_name = os.environ.get("org_name") data_path = os.environ.get("data_path") diff --git a/docs/src/documentation-pages/dev/pe.md b/docs/src/documentation-pages/dev/pe.md index 70530f3dc..9f119a2fa 100644 --- a/docs/src/documentation-pages/dev/pe.md +++ b/docs/src/documentation-pages/dev/pe.md @@ -18,9 +18,9 @@ The local database will contain the entire schema. The only table with any data Before deploying. Generate a secure secret value for a database password, then run the following commands on the terraformer instance: ``` -aws ssm put-parameter --name "/crossfeed/staging/PE_DATABASE_NAME" --value "pe" --type "SecureString" -aws ssm put-parameter --name "/crossfeed/staging/PE_DATABASE_USER" --value "pe" --type "SecureString" -aws ssm put-parameter --name "/crossfeed/staging/PE_DATABASE_PASSWORD" --value "[generated secret password]" --type "SecureString" +aws ssm put-parameter --name "/crossfeed/staging/PE_DB_NAME" --value "pe" --type "SecureString" +aws ssm put-parameter --name "/crossfeed/staging/PE_DB_USER" --value "pe" --type "SecureString" +aws ssm put-parameter --name "/crossfeed/staging/PE_DB_PASSWORD" --value "[generated secret password]" --type "SecureString" ``` ### Sync DB diff --git a/infrastructure/prod.tfvars b/infrastructure/prod.tfvars index 6aa46436d..d866e2cb3 100644 --- a/infrastructure/prod.tfvars +++ b/infrastructure/prod.tfvars @@ -19,6 +19,9 @@ ssm_db_name = "/crossfeed/prod/DATABASE_NAME" ssm_db_host = "/crossfeed/prod/DATABASE_HOST" ssm_db_username = "/crossfeed/prod/DATABASE_USER" ssm_db_password = "/crossfeed/prod/DATABASE_PASSWORD" +ssm_pe_db_name = "/crossfeed/prod/PE_DB_NAME" +ssm_pe_db_username = "/crossfeed/prod/PE_DB_USERNAME" +ssm_pe_db_password = "/crossfeed/prod/PE_DB_PASSWORD" ssm_matomo_db_password = "/crossfeed/prod/MATOMO_DATABASE_PASSWORD" ssm_worker_signature_public_key = "/crossfeed/prod/WORKER_SIGNATURE_PUBLIC_KEY" ssm_worker_signature_private_key = "/crossfeed/prod/WORKER_SIGNATURE_PRIVATE_KEY" diff --git a/infrastructure/stage.tfvars b/infrastructure/stage.tfvars index 7a8a82e1d..dd7ead258 100644 --- a/infrastructure/stage.tfvars +++ b/infrastructure/stage.tfvars @@ -19,9 +19,9 @@ ssm_db_name = "/crossfeed/staging/DATABASE_NAME" ssm_db_host = "/crossfeed/staging/DATABASE_HOST" ssm_db_username = "/crossfeed/staging/DATABASE_USER" ssm_db_password = "/crossfeed/staging/DATABASE_PASSWORD" -ssm_pe_db_name = "/crossfeed/staging/PE_DATABASE_NAME" -ssm_pe_db_username = "/crossfeed/staging/PE_DATABASE_USERNAME" -ssm_pe_db_password = "/crossfeed/staging/PE_DATABASE_PASSWORD" +ssm_pe_db_name = "/crossfeed/staging/PE_DB_NAME" +ssm_pe_db_username = "/crossfeed/staging/PE_DB_USERNAME" +ssm_pe_db_password = "/crossfeed/staging/PE_DB_PASSWORD" ssm_matomo_db_password = "/crossfeed/staging/MATOMO_DATABASE_PASSWORD" ssm_worker_signature_public_key = "/crossfeed/staging/WORKER_SIGNATURE_PUBLIC_KEY" ssm_worker_signature_private_key = "/crossfeed/staging/WORKER_SIGNATURE_PRIVATE_KEY" From 407f6fb2ab9118d71071551ca817d159624587fe Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 31 Mar 2022 15:23:38 -0400 Subject: [PATCH 10/27] Directly pass env vars for PE scripts --- backend/src/tasks/peCybersixgill.ts | 5 +++++ backend/src/tasks/peDomMasq.ts | 3 +++ backend/src/tasks/peHibpSync.ts | 5 ++++- backend/src/tasks/peShodan.ts | 5 ++++- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/backend/src/tasks/peCybersixgill.ts b/backend/src/tasks/peCybersixgill.ts index 7e65a7b2a..8d2bf7401 100644 --- a/backend/src/tasks/peCybersixgill.ts +++ b/backend/src/tasks/peCybersixgill.ts @@ -14,6 +14,11 @@ export const handler = async (commandOptions: CommandOptions) => { encoding: 'utf-8', env: { ...process.env, + PE_DB_USERNAME: process.env.PE_DB_USERNAME, + PE_DB_PASSWORD: process.env.PE_DB_PASSWORD, + PE_DB_NAME: process.env.PE_DB_NAME, + SIXGILL_CLIENT_ID: process.env.SIXGILL_CLIENT_ID, + SIXGILL_CLIENT_SECRET: process.env.SIXGILL_CLIENT_SECRET, org_name: organizationName, org_id: organizationId } diff --git a/backend/src/tasks/peDomMasq.ts b/backend/src/tasks/peDomMasq.ts index 095dfc2ce..322fff349 100644 --- a/backend/src/tasks/peDomMasq.ts +++ b/backend/src/tasks/peDomMasq.ts @@ -32,6 +32,9 @@ export const handler = async (commandOptions: CommandOptions) => { encoding: 'utf-8', env: { ...getPeEnv(), + PE_DB_USERNAME: process.env.PE_DB_USERNAME, + PE_DB_PASSWORD: process.env.PE_DB_PASSWORD, + PE_DB_NAME: process.env.PE_DB_NAME, data_path: input_path, org_id: organizationId, org_name: organizationName diff --git a/backend/src/tasks/peHibpSync.ts b/backend/src/tasks/peHibpSync.ts index 2cbaeaddc..4af04ed63 100644 --- a/backend/src/tasks/peHibpSync.ts +++ b/backend/src/tasks/peHibpSync.ts @@ -32,7 +32,10 @@ export const handler = async (commandOptions: CommandOptions) => { ...process.env, data_path: INPUT_PATH, org_name: organizationName, - org_id: organizationId + org_id: organizationId, + PE_DB_USERNAME: process.env.PE_DB_USERNAME, + PE_DB_PASSWORD: process.env.PE_DB_PASSWORD, + PE_DB_NAME: process.env.PE_DB_NAME } } ); diff --git a/backend/src/tasks/peShodan.ts b/backend/src/tasks/peShodan.ts index 3f09f3255..bb933a3cf 100644 --- a/backend/src/tasks/peShodan.ts +++ b/backend/src/tasks/peShodan.ts @@ -47,7 +47,10 @@ const create_child = async (APIkey, org_list, thread_num): Promise => { ...getPeEnv(), org_list: JSON.stringify(org_list), key: APIkey, - thread_num + thread_num, + PE_DB_USERNAME: process.env.PE_DB_USERNAME, + PE_DB_PASSWORD: process.env.PE_DB_PASSWORD, + PE_DB_NAME: process.env.PE_DB_NAME } } ); From 56d908a2fef12ad8d787b285823317a572a71f48 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 31 Mar 2022 16:25:38 -0400 Subject: [PATCH 11/27] Simplify env passing --- backend/src/tasks/peCybersixgill.ts | 7 ++----- infrastructure/worker.tf | 6 ++++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/backend/src/tasks/peCybersixgill.ts b/backend/src/tasks/peCybersixgill.ts index 8d2bf7401..4d2f30b5f 100644 --- a/backend/src/tasks/peCybersixgill.ts +++ b/backend/src/tasks/peCybersixgill.ts @@ -6,6 +6,8 @@ import { spawnSync } from 'child_process'; export const handler = async (commandOptions: CommandOptions) => { const { organizationId, organizationName } = commandOptions; + console.log(process.env.SIXGILL_CLIENT_ID); + console.log(process.env.PE_DB_USERNAME); const child = spawnSync( 'python3', ['/app/worker/pe_scripts/sixgill/run_cybersixgill.py'], @@ -14,11 +16,6 @@ export const handler = async (commandOptions: CommandOptions) => { encoding: 'utf-8', env: { ...process.env, - PE_DB_USERNAME: process.env.PE_DB_USERNAME, - PE_DB_PASSWORD: process.env.PE_DB_PASSWORD, - PE_DB_NAME: process.env.PE_DB_NAME, - SIXGILL_CLIENT_ID: process.env.SIXGILL_CLIENT_ID, - SIXGILL_CLIENT_SECRET: process.env.SIXGILL_CLIENT_SECRET, org_name: organizationName, org_id: organizationId } diff --git a/infrastructure/worker.tf b/infrastructure/worker.tf index a91986ff5..e901baf02 100644 --- a/infrastructure/worker.tf +++ b/infrastructure/worker.tf @@ -320,6 +320,12 @@ data "aws_ssm_parameter" "sixgill_client_id" { name = var.ssm_sixgill_client_id data "aws_ssm_parameter" "sixgill_client_secret" { name = var.ssm_sixgill_client_secret } +data "aws_ssm_parameter" "pe_db_name" { name = var.ssm_pe_db_name } + +data "aws_ssm_parameter" "pe_db_username" { name = var.ssm_pe_db_username } + +data "aws_ssm_parameter" "pe_db_password" { name = var.ssm_pe_db_password } + data "aws_ssm_parameter" "lg_api_key" { name = var.ssm_lg_api_key } data "aws_ssm_parameter" "lg_workspace_name" { name = var.ssm_lg_workspace_name } From e64dcd3fee9506a6b06ac5f4a22d1b84dcadba9f Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 31 Mar 2022 17:42:51 -0400 Subject: [PATCH 12/27] Increase hibp and dnstwist memory --- backend/src/api/scans.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/src/api/scans.ts b/backend/src/api/scans.ts index 1b744a6d0..d429cee93 100644 --- a/backend/src/api/scans.ts +++ b/backend/src/api/scans.ts @@ -169,6 +169,8 @@ export const SCAN_SCHEMA: ScanSchema = { type: 'fargate', isPassive: true, global: false, + cpu: '2048', + memory: '16384', description: 'Finds emails that have appeared in breaches related to a given domain' }, @@ -182,6 +184,8 @@ export const SCAN_SCHEMA: ScanSchema = { type: 'fargate', isPassive: true, global: false, + cpu: '2048', + memory: '16384', description: 'Domain name permutation engine for detecting similar registered domains.' }, From 96214e04921ce1e11edfd9d59ba27699d3a6e4a0 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Fri, 1 Apr 2022 11:44:26 -0400 Subject: [PATCH 13/27] Clean up env var passing and logging --- backend/src/tasks/peCybersixgill.ts | 3 ++- backend/src/tasks/peDomMasq.ts | 3 --- backend/src/tasks/peHibpSync.ts | 7 ++----- backend/src/tasks/peShodan.ts | 5 +---- backend/worker/pe_scripts/sixgill/run_cybersixgill.py | 6 ------ backend/worker/pe_scripts/sync_dnstwist_pe.py | 3 --- 6 files changed, 5 insertions(+), 22 deletions(-) diff --git a/backend/src/tasks/peCybersixgill.ts b/backend/src/tasks/peCybersixgill.ts index 4d2f30b5f..ae3497bb9 100644 --- a/backend/src/tasks/peCybersixgill.ts +++ b/backend/src/tasks/peCybersixgill.ts @@ -1,5 +1,6 @@ import { CommandOptions } from './ecs-client'; import { spawnSync } from 'child_process'; +import { getPeEnv } from './helpers/getPeEnv'; // Call the sync_dnstwist_pe.py script that fetches dnstwist results, checks // IPs using the blocklist.de api, then updates the PE db instance @@ -15,7 +16,7 @@ export const handler = async (commandOptions: CommandOptions) => { stdio: 'pipe', encoding: 'utf-8', env: { - ...process.env, + ...getPeEnv(), org_name: organizationName, org_id: organizationId } diff --git a/backend/src/tasks/peDomMasq.ts b/backend/src/tasks/peDomMasq.ts index 322fff349..095dfc2ce 100644 --- a/backend/src/tasks/peDomMasq.ts +++ b/backend/src/tasks/peDomMasq.ts @@ -32,9 +32,6 @@ export const handler = async (commandOptions: CommandOptions) => { encoding: 'utf-8', env: { ...getPeEnv(), - PE_DB_USERNAME: process.env.PE_DB_USERNAME, - PE_DB_PASSWORD: process.env.PE_DB_PASSWORD, - PE_DB_NAME: process.env.PE_DB_NAME, data_path: input_path, org_id: organizationId, org_name: organizationName diff --git a/backend/src/tasks/peHibpSync.ts b/backend/src/tasks/peHibpSync.ts index 4af04ed63..ebc90c3bf 100644 --- a/backend/src/tasks/peHibpSync.ts +++ b/backend/src/tasks/peHibpSync.ts @@ -29,13 +29,10 @@ export const handler = async (commandOptions: CommandOptions) => { stdio: 'pipe', encoding: 'utf-8', env: { - ...process.env, + ...getPeEnv(), data_path: INPUT_PATH, org_name: organizationName, - org_id: organizationId, - PE_DB_USERNAME: process.env.PE_DB_USERNAME, - PE_DB_PASSWORD: process.env.PE_DB_PASSWORD, - PE_DB_NAME: process.env.PE_DB_NAME + org_id: organizationId } } ); diff --git a/backend/src/tasks/peShodan.ts b/backend/src/tasks/peShodan.ts index bb933a3cf..3f09f3255 100644 --- a/backend/src/tasks/peShodan.ts +++ b/backend/src/tasks/peShodan.ts @@ -47,10 +47,7 @@ const create_child = async (APIkey, org_list, thread_num): Promise => { ...getPeEnv(), org_list: JSON.stringify(org_list), key: APIkey, - thread_num, - PE_DB_USERNAME: process.env.PE_DB_USERNAME, - PE_DB_PASSWORD: process.env.PE_DB_PASSWORD, - PE_DB_NAME: process.env.PE_DB_NAME + thread_num } } ); diff --git a/backend/worker/pe_scripts/sixgill/run_cybersixgill.py b/backend/worker/pe_scripts/sixgill/run_cybersixgill.py index e719e877c..101999b54 100644 --- a/backend/worker/pe_scripts/sixgill/run_cybersixgill.py +++ b/backend/worker/pe_scripts/sixgill/run_cybersixgill.py @@ -21,12 +21,6 @@ PE_DB_NAME = os.environ.get("PE_DB_NAME") PE_DB_USERNAME = os.environ.get("PE_DB_USERNAME") PE_DB_PASSWORD = os.environ.get("PE_DB_PASSWORD") -SIXGILL_CLIENT_ID = os.environ.get("SIXGILL_CLIENT_ID") -SIXGILL_CLIENT_SECRET = os.environ.get("SIXGILL_CLIENT_SECRET") -print(SIXGILL_CLIENT_ID) -print(PE_DB_USERNAME) -print(PE_DB_NAME) -print(DB_HOST) org_id = os.environ.get("org_id") org_name = os.environ.get("org_name") diff --git a/backend/worker/pe_scripts/sync_dnstwist_pe.py b/backend/worker/pe_scripts/sync_dnstwist_pe.py index 8c3b774be..366f6a052 100644 --- a/backend/worker/pe_scripts/sync_dnstwist_pe.py +++ b/backend/worker/pe_scripts/sync_dnstwist_pe.py @@ -11,9 +11,6 @@ PE_DB_USERNAME = os.environ.get("PE_DB_USERNAME") PE_DB_PASSWORD = os.environ.get("PE_DB_PASSWORD") -print(DB_HOST) -print(PE_DB_NAME) - org_name = os.environ.get("org_name") data_path = os.environ.get("data_path") From 52a6af33242981c3f3d036f8750a70837dd8b4e4 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 4 Apr 2022 12:59:56 -0400 Subject: [PATCH 14/27] Add pe db creds to worker.tf --- infrastructure/worker.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/infrastructure/worker.tf b/infrastructure/worker.tf index e901baf02..565dde640 100644 --- a/infrastructure/worker.tf +++ b/infrastructure/worker.tf @@ -70,6 +70,9 @@ resource "aws_iam_role_policy" "worker_task_execution_role_policy" { "${aws_ssm_parameter.crossfeed_send_db_name.arn}", "${data.aws_ssm_parameter.db_username.arn}", "${data.aws_ssm_parameter.db_password.arn}", + "${data.aws_ssm_parameter.pe_db_name.arn}", + "${data.aws_ssm_parameter.pe_db_username.arn}", + "${data.aws_ssm_parameter.pe_db_password.arn}", "${data.aws_ssm_parameter.worker_signature_public_key.arn}", "${data.aws_ssm_parameter.worker_signature_private_key.arn}", "${data.aws_ssm_parameter.censys_api_id.arn}", From 9e7b05c6989ff1a9853d175500b65059a3c06517 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 4 Apr 2022 14:21:20 -0400 Subject: [PATCH 15/27] Update main.tf for Terraform AWS Provider version 4.0.0 --- infrastructure/main.tf | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/infrastructure/main.tf b/infrastructure/main.tf index 119c06253..ec3159d41 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -26,28 +26,35 @@ resource "aws_ssm_parameter" "stage_api_domain" { resource "aws_s3_bucket" "logging_bucket" { bucket = var.logging_bucket_name - acl = "private" + tags = { + Project = var.project + Stage = var.stage + } +} - server_side_encryption_configuration { - rule { +resource "aws_s3_bucket_acl" "logging_bucket" { + bucket = aws_s3_bucket.logging_bucket.id + acl = "private" +} +resource "aws_s3_bucket_server_side_encryption_configuration" "logging_bucket" { + bucket = aws_s3_bucket.logging_bucket.id + rule { apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } + sse_algorithm = "AES256" } } +} - versioning { - enabled = true - mfa_delete = false - } - - logging { - target_bucket = var.logging_bucket_name - target_prefix = "logging_bucket/" +resource "aws_s3_bucket_versioning" "logging_bucket" { + bucket = aws_s3_bucket.logging_bucket.id + versioning_configuration { + status = "Enabled" } +} - tags = { - Project = var.project - Stage = var.stage - } +resource "aws_s3_bucket_logging" "logging_resource" { + target_bucket = aws_s3_bucket.logging_bucket + target_prefix = "logging_bucket/" } + + From d5de54d8459f3ce4200ab8e3726401744a9e0d99 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 4 Apr 2022 14:35:51 -0400 Subject: [PATCH 16/27] Update main.tf to handle terraform aws provider 4.0 --- infrastructure/main.tf | 41 +++++++++++++++++++++++----------------- infrastructure/worker.tf | 21 ++++++++++++++++++++ 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/infrastructure/main.tf b/infrastructure/main.tf index 119c06253..ec3159d41 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -26,28 +26,35 @@ resource "aws_ssm_parameter" "stage_api_domain" { resource "aws_s3_bucket" "logging_bucket" { bucket = var.logging_bucket_name - acl = "private" + tags = { + Project = var.project + Stage = var.stage + } +} - server_side_encryption_configuration { - rule { +resource "aws_s3_bucket_acl" "logging_bucket" { + bucket = aws_s3_bucket.logging_bucket.id + acl = "private" +} +resource "aws_s3_bucket_server_side_encryption_configuration" "logging_bucket" { + bucket = aws_s3_bucket.logging_bucket.id + rule { apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } + sse_algorithm = "AES256" } } +} - versioning { - enabled = true - mfa_delete = false - } - - logging { - target_bucket = var.logging_bucket_name - target_prefix = "logging_bucket/" +resource "aws_s3_bucket_versioning" "logging_bucket" { + bucket = aws_s3_bucket.logging_bucket.id + versioning_configuration { + status = "Enabled" } +} - tags = { - Project = var.project - Stage = var.stage - } +resource "aws_s3_bucket_logging" "logging_resource" { + target_bucket = aws_s3_bucket.logging_bucket + target_prefix = "logging_bucket/" } + + diff --git a/infrastructure/worker.tf b/infrastructure/worker.tf index b30561178..565dde640 100644 --- a/infrastructure/worker.tf +++ b/infrastructure/worker.tf @@ -70,6 +70,9 @@ resource "aws_iam_role_policy" "worker_task_execution_role_policy" { "${aws_ssm_parameter.crossfeed_send_db_name.arn}", "${data.aws_ssm_parameter.db_username.arn}", "${data.aws_ssm_parameter.db_password.arn}", + "${data.aws_ssm_parameter.pe_db_name.arn}", + "${data.aws_ssm_parameter.pe_db_username.arn}", + "${data.aws_ssm_parameter.pe_db_password.arn}", "${data.aws_ssm_parameter.worker_signature_public_key.arn}", "${data.aws_ssm_parameter.worker_signature_private_key.arn}", "${data.aws_ssm_parameter.censys_api_id.arn}", @@ -216,6 +219,18 @@ resource "aws_ecs_task_definition" "worker" { "name": "DB_PASSWORD", "valueFrom": "${data.aws_ssm_parameter.db_password.arn}" }, + { + "name": "PE_DB_NAME", + "valueFrom": "${data.aws_ssm_parameter.pe_db_name.arn}" + }, + { + "name": "PE_DB_USERNAME", + "valueFrom": "${data.aws_ssm_parameter.pe_db_username.arn}" + }, + { + "name": "PE_DB_PASSWORD", + "valueFrom": "${data.aws_ssm_parameter.pe_db_password.arn}" + }, { "name": "CENSYS_API_ID", "valueFrom": "${data.aws_ssm_parameter.censys_api_id.arn}" @@ -308,6 +323,12 @@ data "aws_ssm_parameter" "sixgill_client_id" { name = var.ssm_sixgill_client_id data "aws_ssm_parameter" "sixgill_client_secret" { name = var.ssm_sixgill_client_secret } +data "aws_ssm_parameter" "pe_db_name" { name = var.ssm_pe_db_name } + +data "aws_ssm_parameter" "pe_db_username" { name = var.ssm_pe_db_username } + +data "aws_ssm_parameter" "pe_db_password" { name = var.ssm_pe_db_password } + data "aws_ssm_parameter" "lg_api_key" { name = var.ssm_lg_api_key } data "aws_ssm_parameter" "lg_workspace_name" { name = var.ssm_lg_workspace_name } From 8619c81905a8b5ac015502ec36855c9bf174e16f Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 4 Apr 2022 14:40:43 -0400 Subject: [PATCH 17/27] fix typo --- infrastructure/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/main.tf b/infrastructure/main.tf index ec3159d41..094836966 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -52,8 +52,8 @@ resource "aws_s3_bucket_versioning" "logging_bucket" { } } -resource "aws_s3_bucket_logging" "logging_resource" { - target_bucket = aws_s3_bucket.logging_bucket +resource "aws_s3_bucket_logging" "logging_bucket" { + target_bucket = aws_s3_bucket.logging_bucket.id target_prefix = "logging_bucket/" } From c7ecae3d1266aa5af34608ac2f8080c71a13b4c9 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 4 Apr 2022 14:51:47 -0400 Subject: [PATCH 18/27] hibpSyncFiles folder --- backend/worker/pe_scripts/hibpSyncFiles/README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 backend/worker/pe_scripts/hibpSyncFiles/README.md diff --git a/backend/worker/pe_scripts/hibpSyncFiles/README.md b/backend/worker/pe_scripts/hibpSyncFiles/README.md new file mode 100644 index 000000000..cba2c784b --- /dev/null +++ b/backend/worker/pe_scripts/hibpSyncFiles/README.md @@ -0,0 +1 @@ +This folder stores temporary json data when passing data to sync_hibp_pe.py. From cc9196aabb7e4f96e93294c1f672def1299c3736 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 4 Apr 2022 15:09:25 -0400 Subject: [PATCH 19/27] make version 4 (tf v4) to worker and frontend --- infrastructure/frontend.tf | 43 +++++++++++++++++++--------------- infrastructure/main.tf | 2 +- infrastructure/worker.tf | 47 +++++++++++++++++++++++--------------- 3 files changed, 53 insertions(+), 39 deletions(-) diff --git a/infrastructure/frontend.tf b/infrastructure/frontend.tf index 545923545..c99fbbc60 100644 --- a/infrastructure/frontend.tf +++ b/infrastructure/frontend.tf @@ -1,29 +1,34 @@ resource "aws_s3_bucket" "frontend_bucket" { bucket = var.frontend_bucket - acl = "private" + tags = { + Project = var.project + Stage = var.stage + } +} - server_side_encryption_configuration { - rule { +resource "aws_s3_bucket_acl" "frontend_bucket" { + bucket = aws_s3_bucket.frontend_bucket.id + acl = "private" +} +resource "aws_s3_bucket_server_side_encryption_configuration" "frontend_bucket" { + bucket = aws_s3_bucket.frontend_bucket.id + rule { apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } + sse_algorithm = "AES256" } } - - versioning { - enabled = true - mfa_delete = false - } - - logging { - target_bucket = aws_s3_bucket.logging_bucket.id - target_prefix = "frontend_bucket/" +} +resource "aws_s3_bucket_versioning" "frontend_bucket" { + bucket = aws_s3_bucket.frontend_bucket.id + versioning_configuration { + status = "Enabled" } +} - tags = { - Project = var.project - Stage = var.stage - } +resource "aws_s3_bucket_logging" "frontend_bucket" { + bucket = aws_s3_bucket.frontend_bucket.id + target_bucket = aws_s3_bucket.logging_bucket.id + target_prefix = "frontend_bucket/" } data "template_file" "policy_file" { @@ -35,7 +40,7 @@ data "template_file" "policy_file" { resource "aws_s3_bucket_policy" "b" { bucket = aws_s3_bucket.frontend_bucket.id - + policy = data.template_file.policy_file.rendered } diff --git a/infrastructure/main.tf b/infrastructure/main.tf index 094836966..384d94269 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -53,7 +53,7 @@ resource "aws_s3_bucket_versioning" "logging_bucket" { } resource "aws_s3_bucket_logging" "logging_bucket" { - target_bucket = aws_s3_bucket.logging_bucket.id + bucket = aws_s3_bucket.logging_bucket.id target_prefix = "logging_bucket/" } diff --git a/infrastructure/worker.tf b/infrastructure/worker.tf index 565dde640..2accd8614 100644 --- a/infrastructure/worker.tf +++ b/infrastructure/worker.tf @@ -339,32 +339,41 @@ data "aws_ssm_parameter" "worker_signature_private_key" { name = var.ssm_worker_ resource "aws_s3_bucket" "export_bucket" { bucket = var.export_bucket_name - acl = "private" + tags = { + Project = var.project + Stage = var.stage + } +} - server_side_encryption_configuration { - rule { +resource "aws_s3_bucket_acl" "export_bucket" { + bucket = aws_s3_bucket.export_bucket.id + acl = "private" +} +resource "aws_s3_bucket_server_side_encryption_configuration" "export_bucket" { + bucket = aws_s3_bucket.export_bucket.id + rule { apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } + sse_algorithm = "AES256" } } +} - versioning { - enabled = true - mfa_delete = false - } - - logging { - target_bucket = aws_s3_bucket.logging_bucket.id - target_prefix = "export_bucket/" - } - - tags = { - Project = var.project - Stage = var.stage +resource "aws_s3_bucket_versioning" "export_bucket" { + bucket = aws_s3_bucket.export_bucket.id + versioning_configuration { + status = "Enabled" } +} - lifecycle_rule { +resource "aws_s3_bucket_logging" "export_bucket" { + bucket = aws_s3_bucket.export_bucket.id + target_bucket = aws_s3_bucket.logging_bucket.id + target_prefix = "export_bucket/" +} + +resource "aws_s3_bucket_lifecycle_configuration" "export_bucket" { + bucket = aws_s3_bucket.export_bucket.id + rule { id = "all_files" enabled = true expiration { From 26253181b80431ccfd620ec0544d9f494d0e589f Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 4 Apr 2022 15:18:57 -0400 Subject: [PATCH 20/27] Run terraform formatting --- infrastructure/frontend.tf | 8 ++++---- infrastructure/main.tf | 9 +++++---- infrastructure/worker.tf | 12 ++++++------ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/infrastructure/frontend.tf b/infrastructure/frontend.tf index c99fbbc60..cd7d66492 100644 --- a/infrastructure/frontend.tf +++ b/infrastructure/frontend.tf @@ -13,8 +13,8 @@ resource "aws_s3_bucket_acl" "frontend_bucket" { resource "aws_s3_bucket_server_side_encryption_configuration" "frontend_bucket" { bucket = aws_s3_bucket.frontend_bucket.id rule { - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" } } } @@ -26,7 +26,7 @@ resource "aws_s3_bucket_versioning" "frontend_bucket" { } resource "aws_s3_bucket_logging" "frontend_bucket" { - bucket = aws_s3_bucket.frontend_bucket.id + bucket = aws_s3_bucket.frontend_bucket.id target_bucket = aws_s3_bucket.logging_bucket.id target_prefix = "frontend_bucket/" } @@ -40,7 +40,7 @@ data "template_file" "policy_file" { resource "aws_s3_bucket_policy" "b" { bucket = aws_s3_bucket.frontend_bucket.id - + policy = data.template_file.policy_file.rendered } diff --git a/infrastructure/main.tf b/infrastructure/main.tf index 384d94269..d0ae3c7cc 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -39,8 +39,8 @@ resource "aws_s3_bucket_acl" "logging_bucket" { resource "aws_s3_bucket_server_side_encryption_configuration" "logging_bucket" { bucket = aws_s3_bucket.logging_bucket.id rule { - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" } } } @@ -53,8 +53,9 @@ resource "aws_s3_bucket_versioning" "logging_bucket" { } resource "aws_s3_bucket_logging" "logging_bucket" { - bucket = aws_s3_bucket.logging_bucket.id + bucket = aws_s3_bucket.logging_bucket.id + target_bucket = aws_s3_bucket.logging_bucket.id target_prefix = "logging_bucket/" } - + diff --git a/infrastructure/worker.tf b/infrastructure/worker.tf index 2accd8614..0caaad1e9 100644 --- a/infrastructure/worker.tf +++ b/infrastructure/worker.tf @@ -352,8 +352,8 @@ resource "aws_s3_bucket_acl" "export_bucket" { resource "aws_s3_bucket_server_side_encryption_configuration" "export_bucket" { bucket = aws_s3_bucket.export_bucket.id rule { - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" } } } @@ -366,16 +366,16 @@ resource "aws_s3_bucket_versioning" "export_bucket" { } resource "aws_s3_bucket_logging" "export_bucket" { - bucket = aws_s3_bucket.export_bucket.id + bucket = aws_s3_bucket.export_bucket.id target_bucket = aws_s3_bucket.logging_bucket.id target_prefix = "export_bucket/" } - + resource "aws_s3_bucket_lifecycle_configuration" "export_bucket" { bucket = aws_s3_bucket.export_bucket.id rule { - id = "all_files" - enabled = true + id = "all_files" + status = "Enabled" expiration { days = 1 } From 61313a30bb0a7c1e7e84a8c645626431125d6d51 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 4 Apr 2022 15:46:18 -0400 Subject: [PATCH 21/27] Use aws_iam_role_policy_attachment instead of aws_iam_policy_attachment To define multiple policies. aws_iam_policy_attachment cannot be declared twice: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment Also removed PE variables from prod --- backend/env.yml | 6 ------ infrastructure/database.tf | 6 ++---- infrastructure/prod.tfvars | 6 ------ 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/backend/env.yml b/backend/env.yml index 1460a18a4..03439fc42 100644 --- a/backend/env.yml +++ b/backend/env.yml @@ -44,12 +44,6 @@ prod: DB_NAME: ${ssm:/crossfeed/prod/DATABASE_NAME~true} DB_USERNAME: ${ssm:/crossfeed/prod/DATABASE_USER~true} DB_PASSWORD: ${ssm:/crossfeed/prod/DATABASE_PASSWORD~true} - PE_DB_NAME: ${ssm:/crossfeed/prod/PE_DB_NAME~true} - PE_DB_USERNAME: ${ssm:/crossfeed/prod/PE_DB_USERNAME~true} - PE_DB_PASSWORD: ${ssm:/crossfeed/prod/PE_DB_PASSWORD~true} - SIXGILL_CLIENT_ID: ${ssm:/crossfeed/prod/SIXGILL_CLIENT_ID~true} - SIXGILL_CLIENT_SECRET: ${ssm:/crossfeed/prod/SIXGILL_CLIENT_SECRET~true} - PE_SHODAN_API_KEYS: ${ssm:/crossfeed/staging/PE_SHODAN_API_KEYS~true} JWT_SECRET: ${ssm:/crossfeed/prod/APP_JWT_SECRET~true} LOGIN_GOV_REDIRECT_URI: ${ssm:/crossfeed/prod/LOGIN_GOV_REDIRECT_URI~true} LOGIN_GOV_BASE_URL: ${ssm:/crossfeed/prod/LOGIN_GOV_BASE_URL~true} diff --git a/infrastructure/database.tf b/infrastructure/database.tf index 6f12e8abe..f1e3a9df4 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -88,14 +88,12 @@ resource "aws_iam_instance_profile" "db_accessor" { } #Attach Policies to Instance Role -resource "aws_iam_policy_attachment" "db_accessor_1" { - name = "crossfeed-db-accessor-${var.stage}" +resource "aws_iam_role_policy_attachment" "db_accessor_1" { roles = [aws_iam_role.db_accessor.id] policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } -resource "aws_iam_policy_attachment" "db_accessor_2" { - name = "crossfeed-db-accessor-${var.stage}" +resource "aws_iam_role_policy_attachment" "db_accessor_2" { roles = [aws_iam_role.db_accessor.id] policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM" } diff --git a/infrastructure/prod.tfvars b/infrastructure/prod.tfvars index 02d35270e..21875ff02 100644 --- a/infrastructure/prod.tfvars +++ b/infrastructure/prod.tfvars @@ -19,9 +19,6 @@ ssm_db_name = "/crossfeed/prod/DATABASE_NAME" ssm_db_host = "/crossfeed/prod/DATABASE_HOST" ssm_db_username = "/crossfeed/prod/DATABASE_USER" ssm_db_password = "/crossfeed/prod/DATABASE_PASSWORD" -ssm_pe_db_name = "/crossfeed/prod/PE_DB_NAME" -ssm_pe_db_username = "/crossfeed/prod/PE_DB_USERNAME" -ssm_pe_db_password = "/crossfeed/prod/PE_DB_PASSWORD" ssm_matomo_db_password = "/crossfeed/prod/MATOMO_DATABASE_PASSWORD" ssm_worker_signature_public_key = "/crossfeed/prod/WORKER_SIGNATURE_PUBLIC_KEY" ssm_worker_signature_private_key = "/crossfeed/prod/WORKER_SIGNATURE_PRIVATE_KEY" @@ -29,9 +26,6 @@ ssm_censys_api_id = "/crossfeed/prod/CENSYS_API_ID" ssm_censys_api_secret = "/crossfeed/prod/CENSYS_API_SECRET" ssm_shodan_api_key = "/crossfeed/prod/SHODAN_API_KEY" ssm_hibp_api_key = "/crossfeed/prod/HIBP_API_KEY" -ssm_pe_shodan_api_keys = "/crossfeed/prod/PE_SHODAN_API_KEYS" -ssm_sixgill_client_id = "/crossfeed/prod/SIXGILL_CLIENT_ID" -ssm_sixgill_client_secret = "/crossfeed/prod/SIXGILL_CLIENT_SECRET" ssm_lg_api_key = "/crossfeed/prod/LG_API_KEY" ssm_lg_workspace_name = "/crossfeed/prod/LG_WORKSPACE_NAME" cloudfront_name = "Crossfeed Prod Frontend" From 6c02a3b9abd62fa32f18007372a2390dcb5c06cd Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Tue, 5 Apr 2022 14:06:36 -0400 Subject: [PATCH 22/27] Update to Terraform AWS Provider v4 To stop aws_instance form being destroyed: https://stackoverflow.com/questions/55809698/ec2-instances-recreated-by-terraform-when-new-ami-released --- infrastructure/database.tf | 11 +++++++---- infrastructure/matomo.tf | 10 +++++++--- infrastructure/provider.tf | 15 ++++++++++----- infrastructure/worker.tf | 10 ++++++++-- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/infrastructure/database.tf b/infrastructure/database.tf index f1e3a9df4..35ba111d3 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -25,7 +25,7 @@ resource "aws_db_instance" "db" { iam_database_authentication_enabled = true // database information - name = var.db_table_name + db_name = var.db_table_name username = data.aws_ssm_parameter.db_username.value password = data.aws_ssm_parameter.db_password.value port = var.db_port @@ -88,12 +88,14 @@ resource "aws_iam_instance_profile" "db_accessor" { } #Attach Policies to Instance Role -resource "aws_iam_role_policy_attachment" "db_accessor_1" { +resource "aws_iam_policy_attachment" "db_accessor_1" { + name = "crossfeed-db-accessor-${var.stage}" roles = [aws_iam_role.db_accessor.id] policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } -resource "aws_iam_role_policy_attachment" "db_accessor_2" { +resource "aws_iam_policy_attachment" "db_accessor_2" { + name = "crossfeed-db-accessor-${var.stage}" roles = [aws_iam_role.db_accessor.id] policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM" } @@ -118,7 +120,8 @@ resource "aws_instance" "db_accessor" { user_data = file("./ssm-agent-install.sh") lifecycle { - # prevent_destroy = true + prevent_destroy = true + ignore_changes = [ami] } } diff --git a/infrastructure/matomo.tf b/infrastructure/matomo.tf index 105bebda5..4d8002174 100644 --- a/infrastructure/matomo.tf +++ b/infrastructure/matomo.tf @@ -1,6 +1,5 @@ resource "aws_ecs_cluster" "matomo" { - name = var.matomo_ecs_cluster_name - capacity_providers = ["FARGATE"] + name = var.matomo_ecs_cluster_name setting { name = "containerInsights" @@ -13,6 +12,11 @@ resource "aws_ecs_cluster" "matomo" { } } +resource "aws_ecs_cluster_capacity_providers" "motomo" { + cluster_name = aws_ecs_cluster.matomo.name + capacity_providers = ["FARGATE"] +} + resource "aws_iam_role" "matomo_task_execution_role" { name = var.matomo_ecs_role_name assume_role_policy = < Date: Tue, 5 Apr 2022 14:20:42 -0400 Subject: [PATCH 23/27] remove source from aws provider --- infrastructure/provider.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/infrastructure/provider.tf b/infrastructure/provider.tf index 8e1f14577..9e9d8275d 100644 --- a/infrastructure/provider.tf +++ b/infrastructure/provider.tf @@ -4,16 +4,15 @@ terraform { required_providers { aws = { - source = "hashicorp/aws" version = "~> 4.0" } } } provider "aws" { - shared_credentials_files = ["$HOME/.aws/credentials"] alias = "virginia" region = "us-east-1" + shared_credentials_files = ["$HOME/.aws/credentials"] } terraform { From c6a3225a0dba110341bad892b1dd991202b061bb Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Tue, 5 Apr 2022 14:26:05 -0400 Subject: [PATCH 24/27] remove alias setting from aws provider --- infrastructure/provider.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/infrastructure/provider.tf b/infrastructure/provider.tf index 9e9d8275d..a102bc3af 100644 --- a/infrastructure/provider.tf +++ b/infrastructure/provider.tf @@ -10,7 +10,6 @@ terraform { } provider "aws" { - alias = "virginia" region = "us-east-1" shared_credentials_files = ["$HOME/.aws/credentials"] } From 64b8febdf10a94feba43489c789f07276fe05534 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Tue, 5 Apr 2022 15:06:06 -0400 Subject: [PATCH 25/27] use for_each to attach multiple policies to accessor instance --- infrastructure/database.tf | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/infrastructure/database.tf b/infrastructure/database.tf index 35ba111d3..4ec4d96eb 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -89,15 +89,13 @@ resource "aws_iam_instance_profile" "db_accessor" { #Attach Policies to Instance Role resource "aws_iam_policy_attachment" "db_accessor_1" { + for_each = toset([ + "arn:aws:iam::aws:policy/AmazonEC2FullAccess", + "arn:aws:iam::aws:policy/AmazonS3FullAccess" + ]) name = "crossfeed-db-accessor-${var.stage}" roles = [aws_iam_role.db_accessor.id] - policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" -} - -resource "aws_iam_policy_attachment" "db_accessor_2" { - name = "crossfeed-db-accessor-${var.stage}" - roles = [aws_iam_role.db_accessor.id] - policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM" + policy_arn = each.value } resource "aws_instance" "db_accessor" { From 3df713d36da4ae1f2d7872a8e4d31e2f2eb67841 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Tue, 5 Apr 2022 15:59:40 -0400 Subject: [PATCH 26/27] Revert back to old policy config --- infrastructure/database.tf | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/infrastructure/database.tf b/infrastructure/database.tf index 4ec4d96eb..35ba111d3 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -89,13 +89,15 @@ resource "aws_iam_instance_profile" "db_accessor" { #Attach Policies to Instance Role resource "aws_iam_policy_attachment" "db_accessor_1" { - for_each = toset([ - "arn:aws:iam::aws:policy/AmazonEC2FullAccess", - "arn:aws:iam::aws:policy/AmazonS3FullAccess" - ]) name = "crossfeed-db-accessor-${var.stage}" roles = [aws_iam_role.db_accessor.id] - policy_arn = each.value + policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + +resource "aws_iam_policy_attachment" "db_accessor_2" { + name = "crossfeed-db-accessor-${var.stage}" + roles = [aws_iam_role.db_accessor.id] + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM" } resource "aws_instance" "db_accessor" { From e537722fe7b48c73a102bbe0996939dda0557ab2 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 6 Apr 2022 09:31:26 -0400 Subject: [PATCH 27/27] Fix prod output failure and remove prevent_destroy --- infrastructure/database.tf | 4 ++-- infrastructure/output.tf | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/infrastructure/database.tf b/infrastructure/database.tf index 35ba111d3..fb75e29cb 100644 --- a/infrastructure/database.tf +++ b/infrastructure/database.tf @@ -120,8 +120,8 @@ resource "aws_instance" "db_accessor" { user_data = file("./ssm-agent-install.sh") lifecycle { - prevent_destroy = true - ignore_changes = [ami] + # prevent_destroy = true + ignore_changes = [ami] } } diff --git a/infrastructure/output.tf b/infrastructure/output.tf index 1770f0f97..db42a7ea6 100644 --- a/infrastructure/output.tf +++ b/infrastructure/output.tf @@ -2,6 +2,6 @@ output "worker_ecs_repository_url" { value = aws_ecr_repository.worker.repository_url } -output "db_accessor_instance_id" { - value = aws_instance.db_accessor[0].id -} +# output "db_accessor_instance_id" { +# value = try(aws_instance.db_accessor[0].id, null) +# }