3213 clean up inconsistent section naming (#4134)

* #3213 Renamed field for consistency

* #3213 Renamed section to enforce consistency

* #3213 More renaming for consistency

* #3213 Enforcing consistency accross federal awards section name

* #3213 Updated version as a result of updating federal awards section name

* #3213 Bug fix - renamed section name in federal awards workbook

* #3213 Fix linting

* #3213 Reverted a previous field name change as this might affect the existing sac records in the DB

* #3213 Bug fix

backend/audit/cross_validation/naming.py

@@ -2,6 +2,7 @@
 from typing import NamedTuple
 
 from audit.models.submission_event import SubmissionEvent
+from audit.fixtures.excel import FORM_SECTIONS
 
 # We need a canonical source of the different versions of each name.
 
@@ -33,7 +34,7 @@ class SectionBabelFish(NamedTuple):
 SECTION_NAMES = {
     "additional_eins": SectionBabelFish(
         all_caps="ADDITIONAL_EINS",
-        camel_case="AdditionalEINs",
+        camel_case="AdditionalEINs",  # We can't use FORM_SECTIONS.ADDITIONAL_EINS here because "AdditionalEINs" is used as a field name in the JSON schema
         friendly="Additional EINs",
         friendly_title="Additional EINs",
         reverse_url_for_file_upload="report_submission:additional-eins",
@@ -46,7 +47,7 @@ class SectionBabelFish(NamedTuple):
     ),
     "additional_ueis": SectionBabelFish(
         all_caps="ADDITIONAL_UEIS",
-        camel_case="AdditionalUEIs",
+        camel_case="AdditionalUEIs",  # We can't use FORM_SECTIONS.ADDITIONAL_UEIS here because "AdditionalUEIs" is used as a field name in the JSON schema
         friendly="Additional UEIs",
         friendly_title="Additional UEIs",
         reverse_url_for_file_upload="report_submission:additional-ueis",
@@ -72,7 +73,7 @@ class SectionBabelFish(NamedTuple):
     ),
     "corrective_action_plan": SectionBabelFish(
         all_caps="CORRECTIVE_ACTION_PLAN",
-        camel_case="CorrectiveActionPlan",
+        camel_case=FORM_SECTIONS.CORRECTIVE_ACTION_PLAN,
         friendly="Corrective Action Plan",
         friendly_title="Corrective Action Plan",
         snake_case="corrective_action_plan",
@@ -85,7 +86,7 @@ class SectionBabelFish(NamedTuple):
     ),
     "federal_awards": SectionBabelFish(
         all_caps="FEDERAL_AWARDS",
-        camel_case="FederalAwards",
+        camel_case=FORM_SECTIONS.FEDERAL_AWARDS,
         friendly="Federal Awards",
         friendly_title="Federal Awards",
         reverse_url_for_file_upload="report_submission:federal-awards",
@@ -98,7 +99,7 @@ class SectionBabelFish(NamedTuple):
     ),
     "findings_text": SectionBabelFish(
         all_caps="FINDINGS_TEXT",
-        camel_case="FindingsText",
+        camel_case=FORM_SECTIONS.FINDINGS_TEXT,
         friendly="Federal Awards Audit Findings Text",
         friendly_title="Federal Awards Audit Findings Text",
         reverse_url_for_file_upload="report_submission:audit-findings-text",
@@ -111,7 +112,7 @@ class SectionBabelFish(NamedTuple):
     ),
     "findings_uniform_guidance": SectionBabelFish(
         all_caps="FINDINGS_UNIFORM_GUIDANCE",
-        camel_case="FindingsUniformGuidance",
+        camel_case=FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE,
         friendly="Findings Uniform Guidance",
         friendly_title="Federal Awards Audit Findings",
         reverse_url_for_file_upload="report_submission:audit-findings",
@@ -137,7 +138,7 @@ class SectionBabelFish(NamedTuple):
     ),
     "notes_to_sefa": SectionBabelFish(
         all_caps="NOTES_TO_SEFA",
-        camel_case="NotesToSefa",
+        camel_case=FORM_SECTIONS.NOTES_TO_SEFA,
         friendly="Notes to SEFA",
         friendly_title="Notes to SEFA",
         reverse_url_for_file_upload="report_submission:notes-to-sefa",
@@ -163,7 +164,7 @@ class SectionBabelFish(NamedTuple):
     ),
     "secondary_auditors": SectionBabelFish(
         all_caps="SECONDARY_AUDITORS",
-        camel_case="SecondaryAuditors",
+        camel_case=FORM_SECTIONS.SECONDARY_AUDITORS,
         friendly="Secondary Auditors",
         friendly_title="Secondary Auditors",
         reverse_url_for_file_upload="report_submission:secondary-auditors",

backend/audit/fixtures/excel.py

@@ -86,7 +86,7 @@
     "FormSections",
     (
         "CORRECTIVE_ACTION_PLAN",
-        "FEDERAL_AWARDS_EXPENDED",
+        "FEDERAL_AWARDS",
         "FINDINGS_TEXT",
         "FINDINGS_UNIFORM_GUIDANCE",
         "ADDITIONAL_UEIS",
@@ -95,10 +95,10 @@
         "NOTES_TO_SEFA",
     ),
 )
-# Note: we turn these into hyphenated lowercase for URLs, e.g. federal-awards-expended
+# Note: we turn these into hyphenated lowercase for URLs, e.g. federal-awards
 FORM_SECTIONS = FormSections(
     CORRECTIVE_ACTION_PLAN="CorrectiveActionPlan",
-    FEDERAL_AWARDS_EXPENDED="FederalAwardsExpended",
+    FEDERAL_AWARDS="FederalAwards",
     FINDINGS_TEXT="FindingsText",
     FINDINGS_UNIFORM_GUIDANCE="FindingsUniformGuidance",
     ADDITIONAL_UEIS="AdditionalUeis",

backend/audit/fixtures/single_audit_checklist.py

@@ -161,7 +161,7 @@ def _post_create_federal_awards(this_sac, this_user):
 
     if (
         ExcelFile.objects.filter(
-            sac_id=this_sac.id, form_section=FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED
+            sac_id=this_sac.id, form_section=FORM_SECTIONS.FEDERAL_AWARDS
         ).exists()
         and this_sac.federal_awards is not None
     ):
@@ -183,7 +183,7 @@ def _post_create_federal_awards(this_sac, this_user):
         filename="temp",
         user=this_user,
         sac_id=this_sac.id,
-        form_section=FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED,
+        form_section=FORM_SECTIONS.FEDERAL_AWARDS,
     )
     excel_file.full_clean()
     excel_file.save()

backend/audit/intakelib/checks/check_has_all_the_named_ranges.py

@@ -20,7 +20,7 @@
     FORM_SECTIONS.ADDITIONAL_UEIS: ADDITIONAL_UEIS_JSONSCHEMA,
     FORM_SECTIONS.NOTES_TO_SEFA: NOTES_TO_SEFA_JSONSCHEMA,
     FORM_SECTIONS.ADDITIONAL_EINS: ADDITIONAL_EINS_JSONSCHEMA,
-    FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: FEDERAL_AWARDS_JSONSCHEMA,
+    FORM_SECTIONS.FEDERAL_AWARDS: FEDERAL_AWARDS_JSONSCHEMA,
     FORM_SECTIONS.FINDINGS_TEXT: FINDINGS_TEXT_JSONSCHEMA,
     FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: CORRECTIVE_ACTION_PLAN_JSONSCHEMA,
     FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: FINDINGS_UNIFORM_GUIDANCE_JSONSCHEMA,

backend/audit/intakelib/checks/check_is_right_workbook.py

@@ -5,9 +5,12 @@
     get_range_by_name,
 )
 from audit.intakelib.common import get_message, build_cell_error_tuple
+from audit.fixtures.excel import FORM_SECTIONS
 
 logger = logging.getLogger(__name__)
 
+FEDERAL_AWARDS_EXPENDED = "FederalAwardsExpended"
+
 
 # DESCRIPTION
 # Makes sure we're looking at the right workbook.
@@ -17,14 +20,23 @@
 def is_right_workbook(what_this_section_should_be):
     def _check_ir(ir):
         section_name = get_range_values_by_name(ir, "section_name")
-        if (section_name) and (what_this_section_should_be not in section_name):
+        expected_section_name = what_this_section_should_be
+        # If the expected section name is FEDERAL_AWARDS, we need to check the version number
+        # to determine if we should be looking at FEDERAL_AWARDS or FEDERAL_AWARDS_EXPENDED
+        if expected_section_name == FORM_SECTIONS.FEDERAL_AWARDS:
+            version_range = get_range_by_name(ir, "version")
+            version = version_range["values"][0]
+            if int(version.replace(".", "")) < 112:
+                expected_section_name = FEDERAL_AWARDS_EXPENDED
+
+        if (section_name) and (expected_section_name not in section_name):
             raise ValidationError(
                 build_cell_error_tuple(
                     ir,
                     get_range_by_name(ir, "section_name"),
                     0,
                     get_message("check_is_right_workbook").format(
-                        what_this_section_should_be
+                        expected_section_name
                     ),
                 )
             )

backend/audit/intakelib/checks/check_missing_required_fields.py

@@ -6,7 +6,7 @@
 logger = logging.getLogger(__name__)
 
 map_required_field_ranges_to_workbook = {
-    FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: {
+    FORM_SECTIONS.FEDERAL_AWARDS: {
         "award_reference": "check_missing_award_numbers",
         "federal_agency_prefix": "check_missing_federal_agency_prefix",
         "three_digit_extension": "check_missing_aln_three_digit_extension",

backend/audit/intakelib/checks/check_version_number.py

@@ -16,6 +16,7 @@
     "1.0.5",
     "1.1.0",
     "1.1.1",
+    "1.1.2",
 }
 
 

backend/audit/intakelib/checks/check_y_or_n__fields.py

@@ -5,7 +5,7 @@
 logger = logging.getLogger(__name__)
 
 map_yorn_field_ranges_to_workbook = {
-    FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: [
+    FORM_SECTIONS.FEDERAL_AWARDS: [
         "is_guaranteed",
         "is_direct",
         "is_passed",

backend/audit/intakelib/checks/runners.py

@@ -71,10 +71,10 @@
 ]
 
 federal_awards_checks = general_checks + [
-    is_right_workbook(FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED),
-    has_all_the_named_ranges(FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED),
-    has_all_required_fields(FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED),
-    has_invalid_yorn_field(FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED),
+    is_right_workbook(FORM_SECTIONS.FEDERAL_AWARDS),
+    has_all_the_named_ranges(FORM_SECTIONS.FEDERAL_AWARDS),
+    has_all_required_fields(FORM_SECTIONS.FEDERAL_AWARDS),
+    has_invalid_yorn_field(FORM_SECTIONS.FEDERAL_AWARDS),
     award_references_pattern,
     federal_award_amount_passed_through_optional,
     check_cluster_names,

backend/audit/intakelib/intermediate_representation.py

@@ -20,7 +20,10 @@ def _extract_generic_data(ir, params) -> dict:
     result: dict = {}
     try:
         _extract_generic_meta_and_field_data(ir, params, result)
-        if result.get("Meta", {}).get(SECTION_NAME) == params.section:
+        if result.get("Meta", {}).get(SECTION_NAME) in [
+            params.section,
+            "FederalAwardsExpended",  # This is necessary to maintain compatibility with the old template
+        ]:
             _extract_generic_column_data(ir, result, params)
         return result
 

backend/audit/intakelib/mapping_federal_awards.py

@@ -45,13 +45,13 @@ def extract_federal_awards(file, is_gsa_migration=False, auditee_uei=None):
         federal_awards_field_mapping,
         federal_awards_column_mapping,
         meta_mapping,
-        FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED,
+        FORM_SECTIONS.FEDERAL_AWARDS,
         template["title_row"],
     )
 
     ir = extract_workbook_as_ir(file)
     run_all_general_checks(
-        ir, FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED, is_gsa_migration, auditee_uei
+        ir, FORM_SECTIONS.FEDERAL_AWARDS, is_gsa_migration, auditee_uei
     )
     new_ir = run_all_federal_awards_transforms(ir)
     run_all_federal_awards_checks(new_ir, is_gsa_migration)

backend/audit/test_validators.py

@@ -82,7 +82,7 @@ def test_validation_is_applied(self):
         Empty Federal Awards should fail, simple case should pass.
         """
         invalid = json.loads(
-            f'{{"Meta":{{"section_name":"{FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED}"}},"FederalAwards":{{}}}}'
+            f'{{"Meta":{{"section_name":"{FORM_SECTIONS.FEDERAL_AWARDS}"}},"FederalAwards":{{}}}}'
         )
         expected_msg = "[\"'Federal Awards' is a required property.\"]"
         self.assertRaisesRegex(

backend/audit/test_views.py

@@ -798,7 +798,7 @@ def test_invalid_file_upload_returns_400(self):
     #     workbook = load_workbook(FEDERAL_AWARDS_TEMPLATE, data_only=True)
     #     _set_by_name(workbook, "total_amount_expended", test_data[0]["amount_expended"])
     #     _set_by_name(workbook, "auditee_uei", ExcelFileHandlerViewTests.GOOD_UEI)
-    #     _set_by_name(workbook, "section_name", FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED)
+    #     _set_by_name(workbook, "section_name", FORM_SECTIONS.FEDERAL_AWARDS)
     #     _add_entry(workbook, 0, test_data[0])
 
     #     with NamedTemporaryFile(suffix=".xlsx") as tmp:
@@ -808,10 +808,10 @@ def test_invalid_file_upload_returns_400(self):
     #         with open(tmp.name, "rb") as excel_file:
     #             response = self.client.post(
     #                 reverse(
-    #                     f"audit:{FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED}",
+    #                     f"audit:{FORM_SECTIONS.FEDERAL_AWARDS}",
     #                     kwargs={
     #                         "report_id": sac.report_id,
-    #                         "form_section": FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED,
+    #                         "form_section": FORM_SECTIONS.FEDERAL_AWARDS,
     #                     },
     #                 ),
     #                 data={"FILES": excel_file},
@@ -1270,7 +1270,7 @@ def test_late_file_upload(self, mock_scan_file):
             (
                 FEDERAL_AWARDS_ENTRY_FIXTURES,
                 FEDERAL_AWARDS_TEMPLATE,
-                FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED,
+                FORM_SECTIONS.FEDERAL_AWARDS,
             ),
             (
                 CORRECTIVE_ACTION_PLAN_ENTRY_FIXTURES,

backend/audit/views/views.py

@@ -126,7 +126,7 @@ def get(self, request, *args, **kwargs):
 
 class ExcelFileHandlerView(SingleAuditChecklistAccessRequiredMixin, generic.View):
     FORM_SECTION_HANDLERS = {
-        FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: {
+        FORM_SECTIONS.FEDERAL_AWARDS: {
             "extractor": extract_federal_awards,
             "field_name": "federal_awards",
             "validator": validate_federal_award_json,
@@ -185,7 +185,7 @@ def _event_type(self, form_section):
             FORM_SECTIONS.ADDITIONAL_EINS: SubmissionEvent.EventType.ADDITIONAL_EINS_UPDATED,
             FORM_SECTIONS.ADDITIONAL_UEIS: SubmissionEvent.EventType.ADDITIONAL_UEIS_UPDATED,
             FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: SubmissionEvent.EventType.CORRECTIVE_ACTION_PLAN_UPDATED,
-            FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: SubmissionEvent.EventType.FEDERAL_AWARDS_UPDATED,
+            FORM_SECTIONS.FEDERAL_AWARDS: SubmissionEvent.EventType.FEDERAL_AWARDS_UPDATED,
             FORM_SECTIONS.FINDINGS_TEXT: SubmissionEvent.EventType.FEDERAL_AWARDS_AUDIT_FINDINGS_TEXT_UPDATED,
             FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: SubmissionEvent.EventType.FINDINGS_UNIFORM_GUIDANCE_UPDATED,
             FORM_SECTIONS.NOTES_TO_SEFA: SubmissionEvent.EventType.NOTES_TO_SEFA_UPDATED,

backend/census_historical_migration/workbooklib/federal_awards.py

@@ -919,9 +919,7 @@ def generate_federal_awards(audit_header, outfile):
         f"--- generate federal awards {audit_header.DBKEY} {audit_header.AUDITYEAR} ---"
     )
 
-    wb = pyxl.load_workbook(
-        sections_to_template_paths[FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED]
-    )
+    wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.FEDERAL_AWARDS])
     uei = xform_retrieve_uei(audit_header.UEI)
     set_workbook_uei(wb, uei)
     audits = get_audits(audit_header.DBKEY, audit_header.AUDITYEAR)

backend/census_historical_migration/workbooklib/post_upload_utils.py

@@ -19,7 +19,7 @@
 
 
 def get_field_by_section(sac, section):
-    if section == FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED:
+    if section == FORM_SECTIONS.FEDERAL_AWARDS:
         return sac.federal_awards
     elif section == FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE:
         return sac.findings_uniform_guidance
@@ -36,7 +36,7 @@ def get_field_by_section(sac, section):
 
 
 extract_mapping = {
-    FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: extract_federal_awards,
+    FORM_SECTIONS.FEDERAL_AWARDS: extract_federal_awards,
     FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: extract_findings_uniform_guidance,
     FORM_SECTIONS.FINDINGS_TEXT: extract_findings_text,
     FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: extract_corrective_action_plan,
@@ -47,7 +47,7 @@ def get_field_by_section(sac, section):
 }
 
 validator_mapping = {
-    FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: audit.validators.validate_federal_award_json,
+    FORM_SECTIONS.FEDERAL_AWARDS: audit.validators.validate_federal_award_json,
     FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: audit.validators.validate_findings_uniform_guidance_json,
     FORM_SECTIONS.FINDINGS_TEXT: audit.validators.validate_findings_text_json,
     FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: audit.validators.validate_corrective_action_plan_json,
@@ -89,7 +89,7 @@ def post_upload_workbook(this_sac, section, xlsx_file):
     audit_data = extract_mapping[section](xlsx_file, is_gsa_migration=True)
     validator_mapping[section](audit_data)
 
-    if section == FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED:
+    if section == FORM_SECTIONS.FEDERAL_AWARDS:
         this_sac.federal_awards = audit_data
     elif section == FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE:
         this_sac.findings_uniform_guidance = audit_data

backend/census_historical_migration/workbooklib/templates.py

@@ -15,7 +15,7 @@
     FORM_SECTIONS.ADDITIONAL_EINS: ADDITIONAL_EINS_TEMPLATE,
     FORM_SECTIONS.ADDITIONAL_UEIS: ADDITIONAL_UEIS_TEMPLATE,
     FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: CORRECTIVE_ACTION_PLAN_TEMPLATE,
-    FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: FEDERAL_AWARDS_TEMPLATE,
+    FORM_SECTIONS.FEDERAL_AWARDS: FEDERAL_AWARDS_TEMPLATE,
     FORM_SECTIONS.FINDINGS_TEXT: FINDINGS_TEXT_TEMPLATE,
     FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: FINDINGS_UNIFORM_GUIDANCE_TEMPLATE,
     FORM_SECTIONS.NOTES_TO_SEFA: NOTES_TO_SEFA_TEMPLATE,

backend/census_historical_migration/workbooklib/workbook_section_handlers.py

@@ -26,7 +26,7 @@
     FORM_SECTIONS.ADDITIONAL_UEIS: generate_additional_ueis,
     FORM_SECTIONS.ADDITIONAL_UEIS: generate_additional_ueis,
     FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: generate_corrective_action_plan,
-    FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: generate_federal_awards,
+    FORM_SECTIONS.FEDERAL_AWARDS: generate_federal_awards,
     FORM_SECTIONS.FINDINGS_TEXT: generate_findings_text,
     FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: generate_findings,
     FORM_SECTIONS.NOTES_TO_SEFA: generate_notes_to_sefa,

backend/cypress/e2e/workbook-uploads.cy.js

@@ -44,7 +44,7 @@ describe('Workbook upload successful', () => {
 
   describe('Workbook upload fail', () => {
     it('unsuccessful upload Federal Awards', () => {
-      cy.intercept('POST', '/audit/excel/federal-awards-expended/*', {
+      cy.intercept('POST', '/audit/excel/federal-awards/*', {
         statusCode: 400,
         fixture: 'fail-res.json',
       }).as('uploadFail');

backend/cypress/support/workbook-uploads.js

@@ -1,6 +1,6 @@
 // re-usable code for workbook uploads
 
-// testWorkbookUpload('/audit/excel/federal-awards-expended/*', '#file-input-federal-awards-xlsx', 'federal-awards-expended-UPDATE.xlsx')
+// testWorkbookUpload('/audit/excel/federal-awards/*', '#file-input-federal-awards-xlsx', 'federal-awards-expended-UPDATE.xlsx')
 // assumes you are on the appropriate upload page already
 function testWorkbookUpload(interceptUrl, uploadSelector, filename, will_intercept = true) {
   if (will_intercept) {
@@ -28,7 +28,7 @@ function testWorkbookUpload(interceptUrl, uploadSelector, filename, will_interce
 
 export function testWorkbookFederalAwards(will_intercept = true) {
   testWorkbookUpload(
-    '/audit/excel/federal-awards-expended/*',
+    '/audit/excel/federal-awards/*',
     '#file-input-federal-awards-xlsx',
     'test_workbooks/federal-awards-workbook.xlsx',
     will_intercept

backend/data_fixtures/audit/excel_schema_test_files/federal-awards-gsa-migration.json

@@ -1,6 +1,6 @@
 {
   "Meta": {
-    "section_name":"FederalAwardsExpended"
+    "section_name":"FederalAwards"
   },
   "FederalAwards": {
     "auditee_uei": "GSA_MIGRATION",