Add P&E database credentials, update dependencies, upgrade aws terraform provider

backend/Dockerfile.worker

@@ -30,7 +30,7 @@ FROM node:14-alpine3.14
 
 WORKDIR /app
 
-RUN apk add --update --no-cache wget build-base curl git unzip openssl-dev linux-headers python3=3.9.5-r2 python3-dev py3-pip ruby=2.7.5-r0 ruby-dev zlib-dev libffi-dev libxml2-dev libxslt-dev postgresql-dev gcc musl-dev py3-pandas py3-scikit-learn
+RUN apk add --update --no-cache wget build-base curl git unzip openssl-dev linux-headers python3=3.9.5-r2 python3-dev py3-pip ruby=2.7.5-r0 ruby-dev zlib-dev libffi-dev libxml2-dev libxslt-dev postgresql-dev gcc musl-dev py3-pandas py3-scikit-learn cargo
 
 RUN npm install -g pm2@4 wait-port@0.2.9
 

backend/env.yml

@@ -8,6 +8,9 @@ staging:
   PE_DB_NAME: ${ssm:/crossfeed/staging/PE_DB_NAME~true}
   PE_DB_USERNAME: ${ssm:/crossfeed/staging/PE_DB_USERNAME~true}
   PE_DB_PASSWORD: ${ssm:/crossfeed/staging/PE_DB_PASSWORD~true}
+  SIXGILL_CLIENT_ID: ${ssm:/crossfeed/staging/SIXGILL_CLIENT_ID~true}
+  SIXGILL_CLIENT_SECRET: ${ssm:/crossfeed/staging/SIXGILL_CLIENT_SECRET~true}
+  PE_SHODAN_API_KEYS: ${ssm:/crossfeed/staging/PE_SHODAN_API_KEYS~true}
   JWT_SECRET: ${ssm:/crossfeed/staging/APP_JWT_SECRET~true}
   LOGIN_GOV_REDIRECT_URI: ${ssm:/crossfeed/staging/LOGIN_GOV_REDIRECT_URI~true}
   LOGIN_GOV_BASE_URL: ${ssm:/crossfeed/staging/LOGIN_GOV_BASE_URL~true}

backend/src/api/scans.ts

@@ -169,6 +169,8 @@ export const SCAN_SCHEMA: ScanSchema = {
     type: 'fargate',
     isPassive: true,
     global: false,
+    cpu: '2048',
+    memory: '16384',
     description:
       'Finds emails that have appeared in breaches related to a given domain'
   },
@@ -182,6 +184,8 @@ export const SCAN_SCHEMA: ScanSchema = {
     type: 'fargate',
     isPassive: true,
     global: false,
+    cpu: '2048',
+    memory: '16384',
     description:
       'Domain name permutation engine for detecting similar registered domains.'
   },

backend/src/tasks/peDomMasq.ts

@@ -2,7 +2,7 @@ import { CommandOptions } from './ecs-client';
 import { spawnSync } from 'child_process';
 import { connectToDatabase, Vulnerability } from '../models';
 import * as path from 'path';
-import { writeFileSync } from 'fs';
+import { promises as fs } from 'fs';
 import { getPeEnv } from './helpers/getPeEnv';
 
 const DOM_MASQ_DIRECTORY = '/app/worker/pe_scripts/peDomMasq';
@@ -21,7 +21,8 @@ export const handler = async (commandOptions: CommandOptions) => {
     .andWhere("vulnerability.source = 'dnstwist'")
     .getRawMany();
   const input_path = path.join(DOM_MASQ_DIRECTORY, organizationId + '.json');
-  writeFileSync(input_path, JSON.stringify(data));
+
+  await fs.writeFile(input_path, JSON.stringify(data));
 
   const child = spawnSync(
     'python3',

backend/worker/pe_scripts/hibpSyncFiles/README.md

@@ -0,0 +1 @@
+This folder stores temporary json data when passing data to sync_hibp_pe.py.

backend/worker/pe_scripts/peDomMasq/README.md

@@ -0,0 +1 @@
+This folder stores temporary json data when passing data to sync_dnstwist_pe.py.

backend/worker/pe_scripts/sixgill/run_cybersixgill.py

@@ -36,8 +36,6 @@
 back = timedelta(days=16)
 from_date = (to_date - back).strftime("%Y-%m-%d %H:%M:%S")
 to_date = to_date.strftime("%Y-%m-%d %H:%M:%S")
-print(to_date)
-print(from_date)
 
 
 def cve(cveid):
@@ -109,7 +107,7 @@ def getDataSource(conn, source):
     print("Failed fetching Cybersixgill aliases.")
     print(traceback.format_exc())
 
-"""Insert/Update Aliases into PE databse instance"""
+"""Insert/Update Aliases into PE database instance"""
 try:
     # aliases_list = json.loads(aliases.replace("'", '"'))
     alias_df = pd.DataFrame(aliases, columns=["alias"])

backend/worker/requirements.txt

@@ -1,7 +1,7 @@
 requests-http-signature==0.2.0
 requests==2.24.0
-mitmproxy==7.0.3
-cryptography==3.3.2
+mitmproxy==8.0.0
+cryptography==36.0.2
 pytest==6.0.1
 scrapy==2.6.1
 dnstwist==20201228

docs/src/documentation-pages/dev/pe.md

@@ -18,9 +18,9 @@ The local database will contain the entire schema. The only table with any data
 Before deploying. Generate a secure secret value for a database password, then run the following commands on the terraformer instance:
 
 ```
-aws ssm put-parameter --name "/crossfeed/staging/PE_DATABASE_NAME" --value "pe" --type "SecureString"
-aws ssm put-parameter --name "/crossfeed/staging/PE_DATABASE_USER" --value "pe" --type "SecureString"
-aws ssm put-parameter --name "/crossfeed/staging/PE_DATABASE_PASSWORD" --value "[generated secret password]" --type "SecureString"
+aws ssm put-parameter --name "/crossfeed/staging/PE_DB_NAME" --value "pe" --type "SecureString"
+aws ssm put-parameter --name "/crossfeed/staging/PE_DB_USER" --value "pe" --type "SecureString"
+aws ssm put-parameter --name "/crossfeed/staging/PE_DB_PASSWORD" --value "[generated secret password]" --type "SecureString"
 ```
 
 ### Sync DB

infrastructure/database.tf

@@ -25,7 +25,7 @@ resource "aws_db_instance" "db" {
   iam_database_authentication_enabled = true
 
   // database information
-  name     = var.db_table_name
+  db_name  = var.db_table_name
   username = data.aws_ssm_parameter.db_username.value
   password = data.aws_ssm_parameter.db_password.value
   port     = var.db_port
@@ -121,6 +121,7 @@ resource "aws_instance" "db_accessor" {
 
   lifecycle {
     # prevent_destroy = true
+    ignore_changes = [ami]
   }
 }
 

infrastructure/frontend.tf

@@ -1,29 +1,34 @@
 resource "aws_s3_bucket" "frontend_bucket" {
   bucket = var.frontend_bucket
-  acl    = "private"
-
-  server_side_encryption_configuration {
-    rule {
-      apply_server_side_encryption_by_default {
-        sse_algorithm = "AES256"
-      }
-    }
+  tags = {
+    Project = var.project
+    Stage   = var.stage
   }
+}
 
-  versioning {
-    enabled    = true
-    mfa_delete = false
+resource "aws_s3_bucket_acl" "frontend_bucket" {
+  bucket = aws_s3_bucket.frontend_bucket.id
+  acl    = "private"
+}
+resource "aws_s3_bucket_server_side_encryption_configuration" "frontend_bucket" {
+  bucket = aws_s3_bucket.frontend_bucket.id
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
   }
-
-  logging {
-    target_bucket = aws_s3_bucket.logging_bucket.id
-    target_prefix = "frontend_bucket/"
+}
+resource "aws_s3_bucket_versioning" "frontend_bucket" {
+  bucket = aws_s3_bucket.frontend_bucket.id
+  versioning_configuration {
+    status = "Enabled"
   }
+}
 
-  tags = {
-    Project = var.project
-    Stage   = var.stage
-  }
+resource "aws_s3_bucket_logging" "frontend_bucket" {
+  bucket        = aws_s3_bucket.frontend_bucket.id
+  target_bucket = aws_s3_bucket.logging_bucket.id
+  target_prefix = "frontend_bucket/"
 }
 
 data "template_file" "policy_file" {

infrastructure/main.tf

@@ -26,28 +26,36 @@ resource "aws_ssm_parameter" "stage_api_domain" {
 
 resource "aws_s3_bucket" "logging_bucket" {
   bucket = var.logging_bucket_name
-  acl    = "private"
-
-  server_side_encryption_configuration {
-    rule {
-      apply_server_side_encryption_by_default {
-        sse_algorithm = "AES256"
-      }
-    }
+  tags = {
+    Project = var.project
+    Stage   = var.stage
   }
+}
 
-  versioning {
-    enabled    = true
-    mfa_delete = false
+resource "aws_s3_bucket_acl" "logging_bucket" {
+  bucket = aws_s3_bucket.logging_bucket.id
+  acl    = "private"
+}
+resource "aws_s3_bucket_server_side_encryption_configuration" "logging_bucket" {
+  bucket = aws_s3_bucket.logging_bucket.id
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
   }
+}
 
-  logging {
-    target_bucket = var.logging_bucket_name
-    target_prefix = "logging_bucket/"
+resource "aws_s3_bucket_versioning" "logging_bucket" {
+  bucket = aws_s3_bucket.logging_bucket.id
+  versioning_configuration {
+    status = "Enabled"
   }
+}
 
-  tags = {
-    Project = var.project
-    Stage   = var.stage
-  }
+resource "aws_s3_bucket_logging" "logging_bucket" {
+  bucket        = aws_s3_bucket.logging_bucket.id
+  target_bucket = aws_s3_bucket.logging_bucket.id
+  target_prefix = "logging_bucket/"
 }
+
+

infrastructure/matomo.tf

@@ -1,6 +1,5 @@
 resource "aws_ecs_cluster" "matomo" {
-  name               = var.matomo_ecs_cluster_name
-  capacity_providers = ["FARGATE"]
+  name = var.matomo_ecs_cluster_name
 
   setting {
     name  = "containerInsights"
@@ -13,6 +12,11 @@ resource "aws_ecs_cluster" "matomo" {
   }
 }
 
+resource "aws_ecs_cluster_capacity_providers" "motomo" {
+  cluster_name       = aws_ecs_cluster.matomo.name
+  capacity_providers = ["FARGATE"]
+}
+
 resource "aws_iam_role" "matomo_task_execution_role" {
   name               = var.matomo_ecs_role_name
   assume_role_policy = <<EOF
@@ -193,7 +197,7 @@ resource "aws_db_instance" "matomo_db" {
   storage_encrypted       = true
 
   // database information
-  name     = "matomo"
+  db_name  = "matomo"
   username = "matomo"
   password = random_password.matomo_db_password.result
 

infrastructure/output.tf

@@ -2,6 +2,6 @@ output "worker_ecs_repository_url" {
   value = aws_ecr_repository.worker.repository_url
 }
 
-output "db_accessor_instance_id" {
-  value = aws_instance.db_accessor[0].id
-}
+# output "db_accessor_instance_id" {
+#   value = try(aws_instance.db_accessor[0].id, null)
+# }

infrastructure/prod.tfvars

@@ -26,9 +26,6 @@ ssm_censys_api_id                 = "/crossfeed/prod/CENSYS_API_ID"
 ssm_censys_api_secret             = "/crossfeed/prod/CENSYS_API_SECRET"
 ssm_shodan_api_key                = "/crossfeed/prod/SHODAN_API_KEY"
 ssm_hibp_api_key                  = "/crossfeed/prod/HIBP_API_KEY"
-ssm_pe_shodan_api_keys            = "/crossfeed/prod/PE_SHODAN_API_KEYS"
-ssm_sixgill_client_id             = "/crossfeed/prod/SIXGILL_CLIENT_ID"
-ssm_sixgill_client_secret         = "/crossfeed/prod/SIXGILL_CLIENT_SECRET"
 ssm_lg_api_key                    = "/crossfeed/prod/LG_API_KEY"
 ssm_lg_workspace_name             = "/crossfeed/prod/LG_WORKSPACE_NAME"
 cloudfront_name                   = "Crossfeed Prod Frontend"

infrastructure/provider.tf

@@ -1,14 +1,17 @@
 # provider.tf
 
 # Specify the provider and access details
-provider "aws" {
-  shared_credentials_file = "$HOME/.aws/credentials"
-  region                  = var.aws_region
+terraform {
+  required_providers {
+    aws = {
+      version = "~> 4.0"
+    }
+  }
 }
 
 provider "aws" {
-  alias  = "virginia"
-  region = "us-east-1"
+  region                   = "us-east-1"
+  shared_credentials_files = ["$HOME/.aws/credentials"]
 }
 
 terraform {

infrastructure/stage.tfvars

@@ -19,6 +19,9 @@ ssm_db_name                       = "/crossfeed/staging/DATABASE_NAME"
 ssm_db_host                       = "/crossfeed/staging/DATABASE_HOST"
 ssm_db_username                   = "/crossfeed/staging/DATABASE_USER"
 ssm_db_password                   = "/crossfeed/staging/DATABASE_PASSWORD"
+ssm_pe_db_name                    = "/crossfeed/staging/PE_DB_NAME"
+ssm_pe_db_username                = "/crossfeed/staging/PE_DB_USERNAME"
+ssm_pe_db_password                = "/crossfeed/staging/PE_DB_PASSWORD"
 ssm_matomo_db_password            = "/crossfeed/staging/MATOMO_DATABASE_PASSWORD"
 ssm_worker_signature_public_key   = "/crossfeed/staging/WORKER_SIGNATURE_PUBLIC_KEY"
 ssm_worker_signature_private_key  = "/crossfeed/staging/WORKER_SIGNATURE_PRIVATE_KEY"

infrastructure/vars.tf

@@ -63,6 +63,20 @@ variable "ssm_db_host" {
   default = "/crossfeed/staging/DATABASE_HOST"
 }
 
+variable "ssm_pe_db_name" {
+  type    = string
+  default = "/crossfeed/staging/PE_DB_NAME"
+}
+
+variable "ssm_pe_db_username" {
+  type    = string
+  default = "/crossfeed/staging/PE_DB_USERNAME"
+}
+
+variable "ssm_pe_db_password" {
+  type    = string
+  default = "/crossfeed/staging/PE_DB_PASSWORD"
+}
 variable "ssm_lambda_sg" {
   type    = string
   default = "/crossfeed/staging/SG_ID"