diff --git a/server/api/batch_inventory.py b/server/api/batch_inventory.py index 085756acb..eeb3fedd4 100644 --- a/server/api/batch_inventory.py +++ b/server/api/batch_inventory.py @@ -136,7 +136,15 @@ def process_dominion(): # Parse out all the initial metadata _election_name = next(cvrs)[0] - contests_row = [" ".join(contest.splitlines()) for contest in next(cvrs)] + contests_row_uncleaned = [ + " ".join(contest.splitlines()) for contest in next(cvrs) + ] + # We've encountered files with extra spaces in between the contest name and the number of + # votes allowed. Remove these extra spaces so that the contest headers are what we expect. + contests_row = [ + re.sub(r"\s+\(Vote For=", " (Vote For=", contest) + for contest in contests_row_uncleaned + ] contest_choices_row = next(cvrs) headers_and_affiliations = next(cvrs) diff --git a/server/tests/batch_comparison/snapshots/snap_test_batch_inventory.py b/server/tests/batch_comparison/snapshots/snap_test_batch_inventory.py index a5eb9597a..fd22127ff 100644 --- a/server/tests/batch_comparison/snapshots/snap_test_batch_inventory.py +++ b/server/tests/batch_comparison/snapshots/snap_test_batch_inventory.py @@ -175,6 +175,48 @@ Tabulator 2 - BATCH2,2,0,0\r """ +snapshots[ + "test_batch_inventory_happy_path_cvrs_with_extra_spaces 1" +] = """Batch Inventory Worksheet\r +\r +Section 1: Check Ballot Groups\r +1. Compare the CVR Ballot Count for each ballot group to your voter check-in data.\r +2. Ensure that the numbers reconcile. If there is a large discrepancy contact your SOS liaison.\r +\r +Ballot Group,CVR Ballot Count,Checked? (Type Yes/No)\r +Election Day,13,\r +Mail,2,\r +\r +Section 2: Check Batches\r +1. Locate each batch in storage.\r +2. Confirm the CVR Ballot Count is correct using associated documentation. Do NOT count the ballots. If there is a large discrepancy contact your SOS liaison.\r +3. Make sure there are no batches missing from this worksheet.\r +\r +Batch,CVR Ballot Count,Checked? (Type Yes/No)\r +Tabulator 1 - BATCH1,3,\r +Tabulator 1 - BATCH2,3,\r +Tabulator 2 - BATCH1,3,\r +Tabulator 2 - BATCH2,6,\r +""" + +snapshots[ + "test_batch_inventory_happy_path_cvrs_with_extra_spaces 2" +] = """Container,Batch Name,Number of Ballots\r +Election Day,Tabulator 1 - BATCH1,3\r +Election Day,Tabulator 1 - BATCH2,3\r +Mail,Tabulator 2 - BATCH1,3\r +Election Day,Tabulator 2 - BATCH2,6\r +""" + +snapshots[ + "test_batch_inventory_happy_path_cvrs_with_extra_spaces 3" +] = """Batch Name,Choice 1-1,Choice 1-2,Write-In\r +Tabulator 1 - BATCH1,1,1,1\r +Tabulator 1 - BATCH2,2,1,0\r +Tabulator 2 - BATCH1,2,1,0\r +Tabulator 2 - BATCH2,2,0,0\r +""" + snapshots[ "test_batch_inventory_happy_path_cvrs_with_leading_equal_signs 1" ] = """Batch Inventory Worksheet\r diff --git a/server/tests/batch_comparison/test_batch_inventory.py b/server/tests/batch_comparison/test_batch_inventory.py index c3a0aba05..2bf19755c 100644 --- a/server/tests/batch_comparison/test_batch_inventory.py +++ b/server/tests/batch_comparison/test_batch_inventory.py @@ -55,6 +55,27 @@ ="15",="TABULATOR2",="BATCH2",="6",="2-2-6",Election Day,12345,CITY,,,,1,0,1,0 """ +TEST_CVRS_WITH_EXTRA_SPACES = """Test Audit CVR Upload,5.2.16.1,,,,,,,,,,,,,,, +,,,,,,,,Contest 1 (Vote For=1),Contest 1 (Vote For=1),Contest 1 (Vote For=1),Contest 2 (Vote For=2),Contest 2 (Vote For=2),Contest 2 (Vote For=2),Contest 2 (Vote For=2) +,,,,,,,,Choice 1-1,Choice 1-2,Write-In,Choice 2-1,Choice 2-2,Choice 2-3,Write-In +CvrNumber,TabulatorNum,BatchId,RecordId,ImprintedId,CountingGroup,PrecinctPortion,BallotType,REP,DEM,LBR,IND,, +1,TABULATOR1,BATCH1,1,1-1-1,Election Day,12345,COUNTY,0,0,1,1,1,0,0 +2,TABULATOR1,BATCH1,2,1-1-2,Election Day,12345,COUNTY,1,0,0,1,0,1,0 +3,TABULATOR1,BATCH1,3,1-1-3,Election Day,12345,COUNTY,0,1,0,1,1,0,0 +4,TABULATOR1,BATCH2,1,1-2-1,Election Day,12345,COUNTY,1,0,0,1,0,1,0 +5,TABULATOR1,BATCH2,2,1-2-2,Election Day,12345,COUNTY,0,1,0,1,1,0,0 +6,TABULATOR1,BATCH2,3,1-2-3,Election Day,12345,COUNTY,1,0,0,1,0,1,0 +7,TABULATOR2,BATCH1,1,2-1-1,Election Day,12345,COUNTY,0,1,0,1,1,0,0 +8,TABULATOR2,BATCH1,2,2-1-2,Mail,12345,COUNTY,1,0,0,1,0,1,0 +9,TABULATOR2,BATCH1,3,2-1-3,Mail,12345,COUNTY,1,0,0,1,1,0,0 +10,TABULATOR2,BATCH2,1,2-2-1,Election Day,12345,COUNTY,1,0,0,1,0,1,0 +11,TABULATOR2,BATCH2,2,2-2-2,Election Day,12345,COUNTY,1,1,0,1,1,0,0 +12,TABULATOR2,BATCH2,3,2-2-3,Election Day,12345,COUNTY,1,0,0,1,0,1,0 +13,TABULATOR2,BATCH2,4,2-2-4,Election Day,12345,CITY,,,,1,0,1,0 +14,TABULATOR2,BATCH2,5,2-2-5,Election Day,12345,CITY,,,,1,1,0,0 +15,TABULATOR2,BATCH2,6,2-2-6,Election Day,12345,CITY,,,,1,0,1,0 +""" + TEST_TABULATOR_STATUS = """ @@ -561,6 +582,217 @@ def test_batch_inventory_happy_path_cvrs_with_leading_equal_signs( assert rv.data.decode("utf-8") == TEST_TABULATOR_STATUS +def test_batch_inventory_happy_path_cvrs_with_extra_spaces( + client: FlaskClient, + election_id: str, + jurisdiction_ids: List[str], + contest_id: str, # pylint: disable=unused-argument + snapshot, +): + set_logged_in_user( + client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) + ) + + # Load batch inventory starting state (simulate JA loading the page) + + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/system-type" + ) + response = json.loads(rv.data) + assert response == dict(systemType=None) + + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/cvr" + ) + cvr = json.loads(rv.data) + assert cvr == dict(file=None, processing=None) + + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/tabulator-status" + ) + tabulator_status = json.loads(rv.data) + assert tabulator_status == dict(file=None, processing=None) + + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/sign-off" + ) + sign_off = json.loads(rv.data) + assert sign_off == dict(signedOffAt=None) + + # Set system type + rv = put_json( + client, + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/system-type", + {"systemType": CvrFileType.DOMINION}, + ) + assert_ok(rv) + + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/system-type" + ) + compare_json(json.loads(rv.data), {"systemType": CvrFileType.DOMINION}) + + # Upload CVR file + rv = upload_batch_inventory_cvr( + client, + io.BytesIO(TEST_CVRS_WITH_EXTRA_SPACES.encode()), + election_id, + jurisdiction_ids[0], + ) + assert_ok(rv) + + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/cvr" + ) + compare_json( + json.loads(rv.data), + { + "file": { + "name": asserts_startswith("batch_inventory_cvrs"), + "uploadedAt": assert_is_date, + }, + "processing": { + "status": ProcessingStatus.PROCESSED, + "startedAt": assert_is_date, + "completedAt": assert_is_date, + "error": None, + }, + }, + ) + + # Upload tabulator status file + rv = upload_batch_inventory_tabulator_status( + client, + io.BytesIO(TEST_TABULATOR_STATUS.encode()), + election_id, + jurisdiction_ids[0], + ) + assert_ok(rv) + + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/tabulator-status" + ) + compare_json( + json.loads(rv.data), + { + "file": { + "name": asserts_startswith("batch_inventory_tabulator_status"), + "uploadedAt": assert_is_date, + }, + "processing": { + "status": ProcessingStatus.PROCESSED, + "startedAt": assert_is_date, + "completedAt": assert_is_date, + "error": None, + }, + }, + ) + + # Download worksheet + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/worksheet" + ) + snapshot.assert_match(rv.data.decode("utf-8")) + + # Sign off + rv = client.post( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/sign-off" + ) + assert_ok(rv) + + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/sign-off" + ) + compare_json(json.loads(rv.data), {"signedOffAt": assert_is_date}) + batch_inventory_data = BatchInventoryData.query.get(jurisdiction_ids[0]) + assert ( + batch_inventory_data.sign_off_user_id + == User.query.filter_by(email=default_ja_email(election_id)).one().id + ) + + # Download manifest + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/ballot-manifest" + ) + ballot_manifest = rv.data.decode("utf-8") + snapshot.assert_match(ballot_manifest) + + # Download batch tallies + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/batch-tallies" + ) + batch_tallies = rv.data.decode("utf-8") + snapshot.assert_match(batch_tallies) + + # Upload manifest + rv = upload_ballot_manifest( + client, + io.BytesIO(ballot_manifest.encode()), + election_id, + jurisdiction_ids[0], + ) + assert_ok(rv) + + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest" + ) + compare_json( + json.loads(rv.data), + { + "file": { + "name": asserts_startswith("manifest"), + "uploadedAt": assert_is_date, + }, + "processing": { + "status": ProcessingStatus.PROCESSED, + "startedAt": assert_is_date, + "completedAt": assert_is_date, + "error": None, + }, + }, + ) + + # Upload batch tallies + rv = upload_batch_tallies( + client, + io.BytesIO(batch_tallies.encode()), + election_id, + jurisdiction_ids[0], + ) + assert_ok(rv) + + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-tallies" + ) + compare_json( + json.loads(rv.data), + { + "file": { + "name": asserts_startswith("batch_tallies"), + "uploadedAt": assert_is_date, + }, + "processing": { + "status": ProcessingStatus.PROCESSED, + "startedAt": assert_is_date, + "completedAt": assert_is_date, + "error": None, + }, + }, + ) + + # Download CVR file + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/cvr/file" + ) + assert rv.data.decode("utf-8") == TEST_CVRS_WITH_EXTRA_SPACES + + # Download tabulator status file + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/tabulator-status/file" + ) + assert rv.data.decode("utf-8") == TEST_TABULATOR_STATUS + + def test_batch_inventory_happy_path_multi_contest_batch_comparison( client: FlaskClient, election_id: str,