diff --git a/.github/workflows/deploy-application.yml b/.github/workflows/deploy-application.yml index f5a1397f59..6567dbe964 100644 --- a/.github/workflows/deploy-application.yml +++ b/.github/workflows/deploy-application.yml @@ -78,3 +78,12 @@ jobs: cf_manifest: backend/manifests/manifest-fac.yml cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml command: bin/ops/deploy.sh + + - name: Load historical data + uses: cloud-gov/cg-cli-tools@main + with: + cf_username: ${{ secrets.CF_USERNAME }} + cf_password: ${{ secrets.CF_PASSWORD }} + cf_org: gsa-tts-oros-fac + cf_space: ${{ env.space }} + command: cf run-task gsa-fac -k 6G -m 1G --name load_data --command ./load_data.sh diff --git a/backend/.profile b/backend/.profile index 236a22316f..37fd1d1c5f 100644 --- a/backend/.profile +++ b/backend/.profile @@ -56,3 +56,6 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then python manage.py collectstatic --noinput && echo 'Finished collectstatic' fi + +# Make psql usable by scripts, for debugging, etc. +alias psql='/home/vcap/deps/0/apt/usr/lib/postgresql/*/bin/psql' diff --git a/backend/apt.yml b/backend/apt.yml new file mode 100644 index 0000000000..27c7d72c62 --- /dev/null +++ b/backend/apt.yml @@ -0,0 +1,3 @@ +--- +packages: +- postgresql-client diff --git a/backend/cypress/e2e/full-submission.cy.js b/backend/cypress/e2e/full-submission.cy.js index 6a6d155d0a..5f9cd40192 100644 --- a/backend/cypress/e2e/full-submission.cy.js +++ b/backend/cypress/e2e/full-submission.cy.js @@ -16,7 +16,8 @@ import { testWorkbookFindingsText, testWorkbookCorrectiveActionPlan, testWorkbookAdditionalUEIs, - testWorkbookSecondaryAuditors + testWorkbookSecondaryAuditors, + testWorkbookAdditionalEINs } from '../support/workbook-uploads.js'; const LOGIN_TEST_EMAIL_AUDITEE = Cypress.env('LOGIN_TEST_EMAIL_AUDITEE'); @@ -83,9 +84,9 @@ describe('Full audit submission', () => { cy.get(".usa-link").contains("Secondary Auditors").click(); testWorkbookSecondaryAuditors(false); - //uncomment this once there is a valid Additional EINs workbook & import testWorkbookAdditionalEINs - /*cy.get(".usa-link").contains("Additional EINs").click(); - testWorkbookAdditionalEINs(false);*/ + + cy.get(".usa-link").contains("Additional EINs").click(); + testWorkbookAdditionalEINs(false); // Complete the audit information form cy.get(".usa-link").contains("Audit Information form").click(); diff --git a/backend/cypress/fixtures/test_workbooks/additional-eins-workbook.xlsx b/backend/cypress/fixtures/test_workbooks/additional-eins-workbook.xlsx index c413a20191..893567adb4 100644 Binary files a/backend/cypress/fixtures/test_workbooks/additional-eins-workbook.xlsx and b/backend/cypress/fixtures/test_workbooks/additional-eins-workbook.xlsx differ diff --git a/backend/cypress/fixtures/test_workbooks/additional-ueis-workbook.xlsx b/backend/cypress/fixtures/test_workbooks/additional-ueis-workbook.xlsx index 690f463504..6535a80346 100644 Binary files a/backend/cypress/fixtures/test_workbooks/additional-ueis-workbook.xlsx and b/backend/cypress/fixtures/test_workbooks/additional-ueis-workbook.xlsx differ diff --git a/backend/cypress/fixtures/test_workbooks/audit-findings-text-workbook.xlsx b/backend/cypress/fixtures/test_workbooks/audit-findings-text-workbook.xlsx index b2977e2182..958b284d94 100644 Binary files a/backend/cypress/fixtures/test_workbooks/audit-findings-text-workbook.xlsx and b/backend/cypress/fixtures/test_workbooks/audit-findings-text-workbook.xlsx differ diff --git a/backend/cypress/fixtures/test_workbooks/corrective-action-plan-workbook.xlsx b/backend/cypress/fixtures/test_workbooks/corrective-action-plan-workbook.xlsx index 3a116c35ec..ede74a6953 100644 Binary files a/backend/cypress/fixtures/test_workbooks/corrective-action-plan-workbook.xlsx and b/backend/cypress/fixtures/test_workbooks/corrective-action-plan-workbook.xlsx differ diff --git a/backend/cypress/fixtures/test_workbooks/federal-awards-audit-findings-workbook.xlsx b/backend/cypress/fixtures/test_workbooks/federal-awards-audit-findings-workbook.xlsx index f7a2b491f9..9fea30edcd 100644 Binary files a/backend/cypress/fixtures/test_workbooks/federal-awards-audit-findings-workbook.xlsx and b/backend/cypress/fixtures/test_workbooks/federal-awards-audit-findings-workbook.xlsx differ diff --git a/backend/cypress/fixtures/test_workbooks/federal-awards-workbook.xlsx b/backend/cypress/fixtures/test_workbooks/federal-awards-workbook.xlsx index e4ee1f2c05..63a7a2a9cd 100644 Binary files a/backend/cypress/fixtures/test_workbooks/federal-awards-workbook.xlsx and b/backend/cypress/fixtures/test_workbooks/federal-awards-workbook.xlsx differ diff --git a/backend/cypress/fixtures/test_workbooks/notes-to-sefa-workbook.xlsx b/backend/cypress/fixtures/test_workbooks/notes-to-sefa-workbook.xlsx index b470b25817..c3435d19af 100644 Binary files a/backend/cypress/fixtures/test_workbooks/notes-to-sefa-workbook.xlsx and b/backend/cypress/fixtures/test_workbooks/notes-to-sefa-workbook.xlsx differ diff --git a/backend/cypress/fixtures/test_workbooks/secondary-auditors-workbook.xlsx b/backend/cypress/fixtures/test_workbooks/secondary-auditors-workbook.xlsx index 676db5ecc5..b52884d6a3 100644 Binary files a/backend/cypress/fixtures/test_workbooks/secondary-auditors-workbook.xlsx and b/backend/cypress/fixtures/test_workbooks/secondary-auditors-workbook.xlsx differ diff --git a/backend/cypress/support/general-info.js b/backend/cypress/support/general-info.js index fc4bf9f3d1..9f9201b093 100644 --- a/backend/cypress/support/general-info.js +++ b/backend/cypress/support/general-info.js @@ -19,7 +19,7 @@ export function testValidGeneralInfo() { cy.get('label[for=multiple-ueis-yes]').click(); cy.get('#ein').type('546000173'); cy.get('label[for=ein_not_an_ssn_attestation]').click(); - cy.get('label[for=multiple-eins-no]').click(); + cy.get('label[for=multiple-eins-yes]').click(); cy.get('#auditee_contact_name').type('John Doe'); cy.get('#auditee_contact_title').type('Keymaster'); cy.get('#auditee_phone').type('5558675309'); diff --git a/backend/cypress/support/workbook-uploads.js b/backend/cypress/support/workbook-uploads.js index 4d70447b73..6a39e53c4a 100644 --- a/backend/cypress/support/workbook-uploads.js +++ b/backend/cypress/support/workbook-uploads.js @@ -88,11 +88,11 @@ export function testWorkbookSecondaryAuditors(will_intercept = true) { ) } -/*export function testWorkbookAdditionalEINs(will_intercept = true) { +export function testWorkbookAdditionalEINs(will_intercept = true) { testWorkbookUpload( '/audit/excel/additional-eins/', '#file-input-additional-eins-xlsx', 'test_workbooks/additional-eins-workbook.xlsx', will_intercept ) -}*/ +} diff --git a/backend/load_data.sh b/backend/load_data.sh new file mode 100644 index 0000000000..510142d429 --- /dev/null +++ b/backend/load_data.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# The following lines will enable extended glob, and allow us to remove /app, except load_data.sh in the event we want to reduce the task disk quota +#shopt -s extglob +#rm -rf /home/vcap/app/!(load_data.sh) +git config --global http.proxy "$https_proxy" +git clone https://github.com/GSA-TTS/fac-historic-public-csvs.git +./fac-historic-public-csvs/create-dumps.sh +./fac-historic-public-csvs/wait-and-load.sh diff --git a/backend/manifests/manifest-fac.yml b/backend/manifests/manifest-fac.yml index 8aa6cffe4c..f746b3a6ac 100644 --- a/backend/manifests/manifest-fac.yml +++ b/backend/manifests/manifest-fac.yml @@ -2,6 +2,7 @@ applications: - name: ((app_name)) buildpacks: + - https://github.com/cloudfoundry/apt-buildpack - python_buildpack memory: ((mem_amount)) path: ../ diff --git a/terraform/shared/modules/env/https-proxy.tf b/terraform/shared/modules/env/https-proxy.tf index 02e77ede9e..b8028757bf 100644 --- a/terraform/shared/modules/env/https-proxy.tf +++ b/terraform/shared/modules/env/https-proxy.tf @@ -15,7 +15,7 @@ module "https-proxy" { "api.sam.gov:443", # New Relic telemetry (https://docs.newrelic.com/docs/new-relic-solutions/get-started/networks/#data-ingest) - # + # # Note: New Relic says that APM agent data ingests via `collector*.newrelic.com`, but we can't specify that # to the Caddy forwardproxy (https://github.com/caddyserver/forwardproxy/blob/caddy2/README.md#access-control) # because it only allows subdomain wildcards in `*.` as a prefix. So this wildcard is a little broader than @@ -23,8 +23,11 @@ module "https-proxy" { # We put in an upstream issue about this: https://github.com/caddyserver/forwardproxy/issues/102 "*.newrelic.com:443", - # Login.gov sanbox + # Login.gov sandbox "idp.int.identitysandbox.gov:443", + + # Git + "*.github.com:443" ], # The parens here make Terraform understand that the key below is a reference # Solution from https://stackoverflow.com/a/57401750 diff --git a/terraform/shared/modules/env/postgrest.tf b/terraform/shared/modules/env/postgrest.tf index dab8da943e..9fde251541 100644 --- a/terraform/shared/modules/env/postgrest.tf +++ b/terraform/shared/modules/env/postgrest.tf @@ -1,6 +1,5 @@ locals { postgrest_name = "postgrest" - data_load_name = "dataload" } resource "cloudfoundry_route" "postgrest" { @@ -18,10 +17,6 @@ data "docker_registry_image" "postgrest" { name = "ghcr.io/gsa-tts/fac/postgrest:latest" } -data "docker_registry_image" "data_load" { - name = "ghcr.io/gsa-tts/fac-historic-public-csvs/load-historic-public-data:latest" -} - resource "cloudfoundry_app" "postgrest" { name = local.postgrest_name space = data.cloudfoundry_space.apps.id @@ -43,21 +38,3 @@ resource "cloudfoundry_app" "postgrest" { PGRST_DB_MAX_ROWS : 20000 } } - -resource "cloudfoundry_app" "data_load" { - name = local.data_load_name - space = data.cloudfoundry_space.apps.id - docker_image = "ghcr.io/gsa-tts/fac-historic-public-csvs/load-historic-public-data@${data.docker_registry_image.data_load.sha256_digest}" - timeout = 180 - memory = 64 - disk_quota = 64 - instances = 1 - strategy = "rolling" - - environment = { - DATABASE_URL : cloudfoundry_service_key.postgrest.credentials.uri - } - depends_on = [ - cloudfoundry_app.postgrest - ] -}