Merge pull request #2014 from GSA-TTS/main

.github/workflows/deploy-application.yml

@@ -78,3 +78,12 @@ jobs:
           cf_manifest: backend/manifests/manifest-fac.yml
           cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml
           command: bin/ops/deploy.sh
+
+      - name: Load historical data
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf run-task gsa-fac -k 6G -m 1G --name load_data --command ./load_data.sh

backend/.profile

@@ -56,3 +56,6 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then
     python manage.py collectstatic --noinput &&
     echo 'Finished collectstatic'
 fi
+
+# Make psql usable by scripts, for debugging, etc.
+alias psql='/home/vcap/deps/0/apt/usr/lib/postgresql/*/bin/psql'

backend/apt.yml

@@ -0,0 +1,3 @@
+---
+packages:
+- postgresql-client

backend/cypress/e2e/full-submission.cy.js

@@ -16,7 +16,8 @@ import {
   testWorkbookFindingsText,
   testWorkbookCorrectiveActionPlan,
   testWorkbookAdditionalUEIs,
-  testWorkbookSecondaryAuditors
+  testWorkbookSecondaryAuditors,
+  testWorkbookAdditionalEINs
 } from '../support/workbook-uploads.js';
 
 const LOGIN_TEST_EMAIL_AUDITEE = Cypress.env('LOGIN_TEST_EMAIL_AUDITEE');
@@ -83,9 +84,9 @@ describe('Full audit submission', () => {
     cy.get(".usa-link").contains("Secondary Auditors").click();
     testWorkbookSecondaryAuditors(false);
 
-    //uncomment this once there is a valid Additional EINs workbook & import testWorkbookAdditionalEINs
-    /*cy.get(".usa-link").contains("Additional EINs").click();
-    testWorkbookAdditionalEINs(false);*/
+
+    cy.get(".usa-link").contains("Additional EINs").click();
+    testWorkbookAdditionalEINs(false);
 
     // Complete the audit information form
     cy.get(".usa-link").contains("Audit Information form").click();

backend/cypress/support/general-info.js

@@ -19,7 +19,7 @@ export function testValidGeneralInfo() {
 	cy.get('label[for=multiple-ueis-yes]').click();
 	cy.get('#ein').type('546000173');
 	cy.get('label[for=ein_not_an_ssn_attestation]').click();
-	cy.get('label[for=multiple-eins-no]').click();
+	cy.get('label[for=multiple-eins-yes]').click();
 	cy.get('#auditee_contact_name').type('John Doe');
 	cy.get('#auditee_contact_title').type('Keymaster');
 	cy.get('#auditee_phone').type('5558675309');

backend/cypress/support/workbook-uploads.js

@@ -88,11 +88,11 @@ export function testWorkbookSecondaryAuditors(will_intercept = true) {
   )
 }
 
-/*export function testWorkbookAdditionalEINs(will_intercept = true) {
+export function testWorkbookAdditionalEINs(will_intercept = true) {
   testWorkbookUpload(
     '/audit/excel/additional-eins/',
     '#file-input-additional-eins-xlsx',
     'test_workbooks/additional-eins-workbook.xlsx',
     will_intercept
   )
-}*/
+}

backend/load_data.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# The following lines will enable extended glob, and allow us to remove /app, except load_data.sh in the event we want to reduce the task disk quota
+#shopt -s extglob
+#rm -rf /home/vcap/app/!(load_data.sh)
+git config --global http.proxy "$https_proxy"
+git clone https://github.com/GSA-TTS/fac-historic-public-csvs.git
+./fac-historic-public-csvs/create-dumps.sh
+./fac-historic-public-csvs/wait-and-load.sh

backend/manifests/manifest-fac.yml

@@ -2,6 +2,7 @@
 applications:
   - name: ((app_name))
     buildpacks:
+      - https://github.com/cloudfoundry/apt-buildpack
       - python_buildpack
     memory: ((mem_amount))
     path: ../

terraform/shared/modules/env/https-proxy.tf

@@ -15,16 +15,19 @@ module "https-proxy" {
       "api.sam.gov:443",
 
       # New Relic telemetry (https://docs.newrelic.com/docs/new-relic-solutions/get-started/networks/#data-ingest)
-      # 
+      #
       # Note: New Relic says that APM agent data ingests via `collector*.newrelic.com`, but we can't specify that
       # to the Caddy forwardproxy (https://github.com/caddyserver/forwardproxy/blob/caddy2/README.md#access-control)
       # because it only allows subdomain wildcards in `*.` as a prefix. So this wildcard is a little broader than
       # we would prefer, but realistically the tightest domain mask we can specify given our current solution.
       # We put in an upstream issue about this: https://github.com/caddyserver/forwardproxy/issues/102
       "*.newrelic.com:443",
 
-      # Login.gov sanbox
+      # Login.gov sandbox
       "idp.int.identitysandbox.gov:443",
+
+      # Git
+      "*.github.com:443"
     ],
     # The parens here make Terraform understand that the key below is a reference
     # Solution from https://stackoverflow.com/a/57401750

terraform/shared/modules/env/postgrest.tf

@@ -1,6 +1,5 @@
 locals {
   postgrest_name = "postgrest"
-  data_load_name = "dataload"
 }
 
 resource "cloudfoundry_route" "postgrest" {
@@ -18,10 +17,6 @@ data "docker_registry_image" "postgrest" {
   name = "ghcr.io/gsa-tts/fac/postgrest:latest"
 }
 
-data "docker_registry_image" "data_load" {
-  name = "ghcr.io/gsa-tts/fac-historic-public-csvs/load-historic-public-data:latest"
-}
-
 resource "cloudfoundry_app" "postgrest" {
   name         = local.postgrest_name
   space        = data.cloudfoundry_space.apps.id
@@ -43,21 +38,3 @@ resource "cloudfoundry_app" "postgrest" {
     PGRST_DB_MAX_ROWS : 20000
   }
 }
-
-resource "cloudfoundry_app" "data_load" {
-  name         = local.data_load_name
-  space        = data.cloudfoundry_space.apps.id
-  docker_image = "ghcr.io/gsa-tts/fac-historic-public-csvs/load-historic-public-data@${data.docker_registry_image.data_load.sha256_digest}"
-  timeout      = 180
-  memory       = 64
-  disk_quota   = 64
-  instances    = 1
-  strategy     = "rolling"
-
-  environment = {
-    DATABASE_URL : cloudfoundry_service_key.postgrest.credentials.uri
-  }
-  depends_on = [
-    cloudfoundry_app.postgrest
-  ]
-}