Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#2282 Fixed schema validation bug for federal awards section #2362

Merged
merged 3 commits into from
Oct 3, 2023
Merged

#2282 Fixed schema validation bug for federal awards section #2362

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6e9b59d
#2282 Fixed schema validation bug for federal awards section
sambodeme Oct 3, 2023
f361997
#2282 Updated FederalAward schema to allow negative numbers and appli…
sambodeme Oct 3, 2023
33c0f4a
Merge branch 'main' into 2282-hd-a-federal-awards-workbook-that-fails…
sambodeme Oct 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 46 additions & 6 deletions backend/audit/excel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -508,16 +508,54 @@ def _extract_column_data(workbook, result, params):
set_fn(result, f"{parent_target}", entries)


def _has_only_one_field_with_value_0(my_dict, field_name):
"""Check if the dictionary has exactly one field with the provided name and its value is 0"""
return len(my_dict) == 1 and my_dict.get(field_name) == 0


def _remove_empty_award_entries(data):
"""Removes empty award entries from the data"""
indexed_awards = []
awards = []
for award in data.get("FederalAwards", {}).get("federal_awards", []):
if "program" in award:
program = award["program"]
if FEDERAL_AGENCY_PREFIX in program:
indexed_awards.append(award)
if not all(
[
"direct_or_indirect_award" not in award,
"loan_or_loan_guarantee" not in award,
"subrecipients" not in award,
"program" in award
and _has_only_one_field_with_value_0(
award["program"], "federal_program_total"
),
"cluster" in award
and _has_only_one_field_with_value_0(award["cluster"], "cluster_total"),
]
):
awards.append(award)
if "FederalAwards" in data:
# Update the federal_awards with the valid awards
data["FederalAwards"]["federal_awards"] = awards

return data


def _add_required_fields(data):
"""Adds empty parent fields to the json object to allow for proper schema validation / indexing"""
indexed_awards = []
for award in data.get("FederalAwards", {}).get("federal_awards", []):
if "cluster" not in award:
award["cluster"] = {}
if "direct_or_indirect_award" not in award:
award["direct_or_indirect_award"] = {}
if "loan_or_loan_guarantee" not in award:
award["loan_or_loan_guarantee"] = {}
if "program" not in award:
award["program"] = {}
if "subrecipients" not in award:
award["subrecipients"] = {}
indexed_awards.append(award)

if "FederalAwards" in data:
# Update the federal_awards with all required fields
data["FederalAwards"]["federal_awards"] = indexed_awards

return data
Expand All @@ -536,7 +574,9 @@ def extract_federal_awards(file):
template["title_row"],
)
result = _extract_data(file, params)
return _remove_empty_award_entries(result)
result = _remove_empty_award_entries(result)
result = _add_required_fields(result)
return result


def extract_corrective_action_plan(file):
Expand Down
15 changes: 2 additions & 13 deletions backend/audit/validators.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import logging
from jsonschema import Draft7Validator, FormatChecker, validate
from jsonschema.exceptions import ValidationError as JSONSchemaValidationError
from jsonschema.exceptions import SchemaError as JSONSchemaError

from django.core.exceptions import ValidationError

Expand Down Expand Up @@ -209,18 +208,8 @@ def validate_federal_award_json(value):
schema_path = settings.SECTION_SCHEMA_DIR / "FederalAwards.schema.json"
schema = json.loads(schema_path.read_text(encoding="utf-8"))

# FIXME: For some classes of error, this approach
# will go into an infinite/recursive loop in the iterator.
# There needs to either be a way to solve that, or we need
# to not use this approach.
# validator = Draft7Validator(schema)
# errors = list(validator.iter_errors(value))
# The side-effect is that now I only do one error at a time...
errors = []
try:
validate(schema=schema, instance=value)
except (JSONSchemaValidationError, JSONSchemaError) as e:
errors = [e]
validator = Draft7Validator(schema)
errors = list(validator.iter_errors(value))
if len(errors) > 0:
raise ValidationError(message=_federal_awards_json_error(errors))

Expand Down
44 changes: 22 additions & 22 deletions backend/schemas/output/excel/json/federal-awards-workbook.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,8 @@
"format": "dollar",
"formula": "=SUM('Form'!F$FIRSTROW:F$LASTROW)",
"help": {
"link": "https://fac.gov/documentation/validation/#positive_number",
"text": "The number must be zero or greater"
"link": "https://fac.gov/documentation/validation/#any_number",
"text": "Must be a number"
},
"keep_locked": true,
"range_cell": "B5",
Expand All @@ -99,9 +99,9 @@
"title_cell": "A5",
"type": "single_cell",
"validation": {
"custom_error": "This cell must be a positive number",
"custom_title": "Positive numbers",
"formula1": "=AND(ISNUMBER(FIRSTCELLREF),OR(SIGN(FIRSTCELLREF)=0,SIGN(FIRSTCELLREF)=1))",
"custom_error": "This cell must be a number",
"custom_title": "Numbers",
"formula1": "=ISNUMBER(FIRSTCELLREF)",
"type": "custom"
},
"width": 36
Expand Down Expand Up @@ -197,17 +197,17 @@
{
"format": "dollar",
"help": {
"link": "https://fac.gov/documentation/validation/#positive_number",
"text": "The number must be zero or greater"
"link": "https://fac.gov/documentation/validation/#any_number",
"text": "Must be a number"
},
"range_name": "amount_expended",
"title": "Amount Expended",
"title_cell": "F1",
"type": "open_range",
"validation": {
"custom_error": "This cell must be a positive number",
"custom_title": "Positive numbers",
"formula1": "=AND(ISNUMBER(FIRSTCELLREF),OR(SIGN(FIRSTCELLREF)=0,SIGN(FIRSTCELLREF)=1))",
"custom_error": "This cell must be a number",
"custom_title": "Numbers",
"formula1": "=ISNUMBER(FIRSTCELLREF)",
"type": "custom"
}
},
Expand Down Expand Up @@ -259,37 +259,37 @@
"format": "dollar",
"formula": "=SUMIFS(amount_expended,cfda_key,V{0})",
"help": {
"link": "https://fac.gov/documentation/validation/#positive_number",
"text": "The number must be zero or greater"
"link": "https://fac.gov/documentation/validation/#any_number",
"text": "Must be a number"
},
"keep_locked": true,
"range_name": "federal_program_total",
"title": "Federal Program Total",
"title_cell": "J1",
"type": "open_range",
"validation": {
"custom_error": "This cell must be a positive number",
"custom_title": "Positive numbers",
"formula1": "=AND(ISNUMBER(FIRSTCELLREF),OR(SIGN(FIRSTCELLREF)=0,SIGN(FIRSTCELLREF)=1))",
"custom_error": "This cell must be a number",
"custom_title": "Numbers",
"formula1": "=ISNUMBER(FIRSTCELLREF)",
"type": "custom"
}
},
{
"format": "dollar",
"formula": "=IF(G{0}=\"OTHER CLUSTER NOT LISTED ABOVE\",SUMIFS(amount_expended,uniform_other_cluster_name,X{0}), IF(AND(OR(G{0}=\"N/A\",G{0}=\"\"),H{0}=\"\"),0,IF(G{0}=\"STATE CLUSTER\",SUMIFS(amount_expended,uniform_state_cluster_name,W{0}),SUMIFS(amount_expended,cluster_name,G{0}))))",
"help": {
"link": "https://fac.gov/documentation/validation/#positive_number",
"text": "The number must be zero or greater"
"link": "https://fac.gov/documentation/validation/#any_number",
"text": "Must be a number"
},
"keep_locked": true,
"range_name": "cluster_total",
"title": "Cluster Total",
"title_cell": "K1",
"type": "open_range",
"validation": {
"custom_error": "This cell must be a positive number",
"custom_title": "Positive numbers",
"formula1": "=AND(ISNUMBER(FIRSTCELLREF),OR(SIGN(FIRSTCELLREF)=0,SIGN(FIRSTCELLREF)=1))",
"custom_error": "This cell must be a number",
"custom_title": "Numbers",
"formula1": "=ISNUMBER(FIRSTCELLREF)",
"type": "custom"
}
},
Expand Down Expand Up @@ -393,8 +393,8 @@
{
"format": "dollar",
"help": {
"link": "https://fac.gov/documentation/validation/#positive_number",
"text": "The number must be zero or greater"
"link": "https://fac.gov/documentation/validation/#any_number",
"text": "Must be a number"
},
"range_name": "subrecipient_amount",
"title": "If yes (Passed Through), Amount Passed Through to Subrecipients",
Expand Down
Binary file modified backend/schemas/output/excel/xlsx/additional-eins-workbook.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified backend/schemas/output/excel/xlsx/additional-ueis-workbook.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified backend/schemas/output/excel/xlsx/audit-findings-text-workbook.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified backend/schemas/output/excel/xlsx/corrective-action-plan-workbook.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified backend/schemas/output/excel/xlsx/federal-awards-audit-findings-workbook.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified backend/schemas/output/excel/xlsx/federal-awards-workbook.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified backend/schemas/output/excel/xlsx/notes-to-sefa-workbook.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified backend/schemas/output/excel/xlsx/secondary-auditors-workbook.xlsx
View file
Open in desktop
Binary file not shown.
59 changes: 32 additions & 27 deletions backend/schemas/output/sections/FederalAwards.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@
},
"cluster": {
"allOf": [
{
"required": [
"cluster_name",
"cluster_total"
]
},
{
"if": {
"properties": {
Expand Down Expand Up @@ -165,7 +171,6 @@
"type": "string"
},
"cluster_total": {
"minimum": 0,
"type": "number"
},
"other_cluster_name": {
Expand All @@ -175,15 +180,16 @@
"type": "string"
}
},
"required": [
"cluster_name",
"cluster_total"
],
"type": "object"
},
"direct_or_indirect_award": {
"additionalProperties": false,
"allOf": [
{
"required": [
"is_direct"
]
},
{
"if": {
"properties": {
Expand Down Expand Up @@ -243,14 +249,16 @@
"type": "string"
}
},
"required": [
"is_direct"
],
"type": "object"
},
"loan_or_loan_guarantee": {
"additionalProperties": false,
"allOf": [
{
"required": [
"is_guaranteed"
]
},
{
"if": {
"properties": {
Expand Down Expand Up @@ -307,14 +315,22 @@
]
}
},
"required": [
"is_guaranteed"
],
"type": "object"
},
"program": {
"additionalProperties": false,
"allOf": [
{
"required": [
"program_name",
"federal_agency_prefix",
"three_digit_extension",
"is_major",
"number_of_audit_findings",
"federal_program_total",
"amount_expended"
]
},
{
"properties": {
"number_of_audit_findings": {
Expand Down Expand Up @@ -380,7 +396,6 @@
"type": "string"
},
"amount_expended": {
"minimum": 0,
"type": "number"
},
"audit_report_type": {
Expand All @@ -407,7 +422,6 @@
"type": "string"
},
"federal_program_total": {
"minimum": 0,
"type": "number"
},
"is_major": {
Expand All @@ -428,20 +442,16 @@
"type": "string"
}
},
"required": [
"program_name",
"federal_agency_prefix",
"three_digit_extension",
"is_major",
"number_of_audit_findings",
"federal_program_total",
"amount_expended"
],
"type": "object"
},
"subrecipients": {
"additionalProperties": false,
"allOf": [
{
"required": [
"is_passed"
]
},
{
"if": {
"properties": {
Expand Down Expand Up @@ -482,13 +492,9 @@
"type": "string"
},
"subrecipient_amount": {
"minimum": 0,
"type": "number"
}
},
"required": [
"is_passed"
],
"type": "object"
}
},
Expand All @@ -505,7 +511,6 @@
"type": "array"
},
"total_amount_expended": {
"minimum": 0,
"type": "number"
}
},
Expand Down
2 changes: 1 addition & 1 deletion backend/schemas/output/sections/TribalAccess.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,5 @@
],
"title": "TribalAccess",
"type": "object",
"version": null
"version": 20230927
}
4 changes: 4 additions & 0 deletions backend/schemas/source/excel/libs/Help.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ local make_url = function(anchor)
text: 'The number must be zero or greater',
link: make_url('positive_number'),
},
any_number: {
text: 'Must be a number',
link: make_url('any_number'),
},
prior_references: {
text: 'Must be a comma-separated list of reference numbers (YYYY-NNN) or N/A.',
link: make_url('prior_references'),
Expand Down
9 changes: 9 additions & 0 deletions backend/schemas/source/excel/libs/SheetValidations.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,14 @@ local PositiveNumberValidation = {
custom_title: 'Positive numbers',
};

local NumberValidation = {
type: 'custom',
// Is it a number ?
formula1: '=ISNUMBER(FIRSTCELLREF)',
custom_error: 'This cell must be a number',
custom_title: 'Numbers',
};

local ReferenceNumberValidation = {
type: 'custom',
//It is neccessary to allow blank otherwise user cannot delete the value
Expand Down Expand Up @@ -87,6 +95,7 @@ local AwardReferenceValidation = {
NoValidation: { type: 'NOVALIDATION' },
FAPPrefixValidation: FAPPrefixValidation,
PositiveNumberValidation: PositiveNumberValidation,
NumberValidation: NumberValidation,
LookupValidation: LookupValidation,
RangeLookupValidation: RangeLookupValidation,
StringOfLengthNine: StringOfSize(9),
Expand Down
Loading