Handle extra spaces in Dominion CVR contest headers (#2053)

server/api/batch_inventory.py

@@ -136,7 +136,15 @@ def process_dominion():
 
         # Parse out all the initial metadata
         _election_name = next(cvrs)[0]
-        contests_row = [" ".join(contest.splitlines()) for contest in next(cvrs)]
+        contests_row_uncleaned = [
+            " ".join(contest.splitlines()) for contest in next(cvrs)
+        ]
+        # We've encountered files with extra spaces in between the contest name and the number of
+        # votes allowed. Remove these extra spaces so that the contest headers are what we expect.
+        contests_row = [
+            re.sub(r"\s+\(Vote For=", " (Vote For=", contest)
+            for contest in contests_row_uncleaned
+        ]
         contest_choices_row = next(cvrs)
         headers_and_affiliations = next(cvrs)
 

server/tests/batch_comparison/snapshots/snap_test_batch_inventory.py

@@ -175,6 +175,48 @@
 Tabulator 2 - BATCH2,2,0,0\r
 """
 
+snapshots[
+    "test_batch_inventory_happy_path_cvrs_with_extra_spaces 1"
+] = """Batch Inventory Worksheet\r
+\r
+Section 1: Check Ballot Groups\r
+1. Compare the CVR Ballot Count for each ballot group to your voter check-in data.\r
+2. Ensure that the numbers reconcile. If there is a large discrepancy contact your SOS liaison.\r
+\r
+Ballot Group,CVR Ballot Count,Checked? (Type Yes/No)\r
+Election Day,13,\r
+Mail,2,\r
+\r
+Section 2: Check Batches\r
+1. Locate each batch in storage.\r
+2. Confirm the CVR Ballot Count is correct using associated documentation. Do NOT count the ballots. If there is a large discrepancy contact your SOS liaison.\r
+3. Make sure there are no batches missing from this worksheet.\r
+\r
+Batch,CVR Ballot Count,Checked? (Type Yes/No)\r
+Tabulator 1 - BATCH1,3,\r
+Tabulator 1 - BATCH2,3,\r
+Tabulator 2 - BATCH1,3,\r
+Tabulator 2 - BATCH2,6,\r
+"""
+
+snapshots[
+    "test_batch_inventory_happy_path_cvrs_with_extra_spaces 2"
+] = """Container,Batch Name,Number of Ballots\r
+Election Day,Tabulator 1 - BATCH1,3\r
+Election Day,Tabulator 1 - BATCH2,3\r
+Mail,Tabulator 2 - BATCH1,3\r
+Election Day,Tabulator 2 - BATCH2,6\r
+"""
+
+snapshots[
+    "test_batch_inventory_happy_path_cvrs_with_extra_spaces 3"
+] = """Batch Name,Choice 1-1,Choice 1-2,Write-In\r
+Tabulator 1 - BATCH1,1,1,1\r
+Tabulator 1 - BATCH2,2,1,0\r
+Tabulator 2 - BATCH1,2,1,0\r
+Tabulator 2 - BATCH2,2,0,0\r
+"""
+
 snapshots[
     "test_batch_inventory_happy_path_cvrs_with_leading_equal_signs 1"
 ] = """Batch Inventory Worksheet\r

server/tests/batch_comparison/test_batch_inventory.py

@@ -55,6 +55,27 @@
 ="15",="TABULATOR2",="BATCH2",="6",="2-2-6",Election Day,12345,CITY,,,,1,0,1,0
 """
 
+TEST_CVRS_WITH_EXTRA_SPACES = """Test Audit CVR Upload,5.2.16.1,,,,,,,,,,,,,,,
+,,,,,,,,Contest 1  (Vote For=1),Contest 1  (Vote For=1),Contest 1  (Vote For=1),Contest 2  (Vote For=2),Contest 2  (Vote For=2),Contest 2  (Vote For=2),Contest 2  (Vote For=2)
+,,,,,,,,Choice 1-1,Choice 1-2,Write-In,Choice 2-1,Choice 2-2,Choice 2-3,Write-In
+CvrNumber,TabulatorNum,BatchId,RecordId,ImprintedId,CountingGroup,PrecinctPortion,BallotType,REP,DEM,LBR,IND,,
+1,TABULATOR1,BATCH1,1,1-1-1,Election Day,12345,COUNTY,0,0,1,1,1,0,0
+2,TABULATOR1,BATCH1,2,1-1-2,Election Day,12345,COUNTY,1,0,0,1,0,1,0
+3,TABULATOR1,BATCH1,3,1-1-3,Election Day,12345,COUNTY,0,1,0,1,1,0,0
+4,TABULATOR1,BATCH2,1,1-2-1,Election Day,12345,COUNTY,1,0,0,1,0,1,0
+5,TABULATOR1,BATCH2,2,1-2-2,Election Day,12345,COUNTY,0,1,0,1,1,0,0
+6,TABULATOR1,BATCH2,3,1-2-3,Election Day,12345,COUNTY,1,0,0,1,0,1,0
+7,TABULATOR2,BATCH1,1,2-1-1,Election Day,12345,COUNTY,0,1,0,1,1,0,0
+8,TABULATOR2,BATCH1,2,2-1-2,Mail,12345,COUNTY,1,0,0,1,0,1,0
+9,TABULATOR2,BATCH1,3,2-1-3,Mail,12345,COUNTY,1,0,0,1,1,0,0
+10,TABULATOR2,BATCH2,1,2-2-1,Election Day,12345,COUNTY,1,0,0,1,0,1,0
+11,TABULATOR2,BATCH2,2,2-2-2,Election Day,12345,COUNTY,1,1,0,1,1,0,0
+12,TABULATOR2,BATCH2,3,2-2-3,Election Day,12345,COUNTY,1,0,0,1,0,1,0
+13,TABULATOR2,BATCH2,4,2-2-4,Election Day,12345,CITY,,,,1,0,1,0
+14,TABULATOR2,BATCH2,5,2-2-5,Election Day,12345,CITY,,,,1,1,0,0
+15,TABULATOR2,BATCH2,6,2-2-6,Election Day,12345,CITY,,,,1,0,1,0
+"""
+
 TEST_TABULATOR_STATUS = """<?xml version="1.0" standalone="yes"?>
 <ExportName>
    <Terminology Subdivision="District" Subdivisions="Districts" PollingSubdivision="Precinct" PollingSubdivisions="Precincts" ParentSubdivision="Parent District" MultiPollingSubdivisionCollection="Multi-Precinct Collection" />
@@ -561,6 +582,217 @@ def test_batch_inventory_happy_path_cvrs_with_leading_equal_signs(
     assert rv.data.decode("utf-8") == TEST_TABULATOR_STATUS
 
 
+def test_batch_inventory_happy_path_cvrs_with_extra_spaces(
+    client: FlaskClient,
+    election_id: str,
+    jurisdiction_ids: List[str],
+    contest_id: str,  # pylint: disable=unused-argument
+    snapshot,
+):
+    set_logged_in_user(
+        client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id)
+    )
+
+    # Load batch inventory starting state (simulate JA loading the page)
+
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/system-type"
+    )
+    response = json.loads(rv.data)
+    assert response == dict(systemType=None)
+
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/cvr"
+    )
+    cvr = json.loads(rv.data)
+    assert cvr == dict(file=None, processing=None)
+
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/tabulator-status"
+    )
+    tabulator_status = json.loads(rv.data)
+    assert tabulator_status == dict(file=None, processing=None)
+
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/sign-off"
+    )
+    sign_off = json.loads(rv.data)
+    assert sign_off == dict(signedOffAt=None)
+
+    # Set system type
+    rv = put_json(
+        client,
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/system-type",
+        {"systemType": CvrFileType.DOMINION},
+    )
+    assert_ok(rv)
+
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/system-type"
+    )
+    compare_json(json.loads(rv.data), {"systemType": CvrFileType.DOMINION})
+
+    # Upload CVR file
+    rv = upload_batch_inventory_cvr(
+        client,
+        io.BytesIO(TEST_CVRS_WITH_EXTRA_SPACES.encode()),
+        election_id,
+        jurisdiction_ids[0],
+    )
+    assert_ok(rv)
+
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/cvr"
+    )
+    compare_json(
+        json.loads(rv.data),
+        {
+            "file": {
+                "name": asserts_startswith("batch_inventory_cvrs"),
+                "uploadedAt": assert_is_date,
+            },
+            "processing": {
+                "status": ProcessingStatus.PROCESSED,
+                "startedAt": assert_is_date,
+                "completedAt": assert_is_date,
+                "error": None,
+            },
+        },
+    )
+
+    # Upload tabulator status file
+    rv = upload_batch_inventory_tabulator_status(
+        client,
+        io.BytesIO(TEST_TABULATOR_STATUS.encode()),
+        election_id,
+        jurisdiction_ids[0],
+    )
+    assert_ok(rv)
+
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/tabulator-status"
+    )
+    compare_json(
+        json.loads(rv.data),
+        {
+            "file": {
+                "name": asserts_startswith("batch_inventory_tabulator_status"),
+                "uploadedAt": assert_is_date,
+            },
+            "processing": {
+                "status": ProcessingStatus.PROCESSED,
+                "startedAt": assert_is_date,
+                "completedAt": assert_is_date,
+                "error": None,
+            },
+        },
+    )
+
+    # Download worksheet
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/worksheet"
+    )
+    snapshot.assert_match(rv.data.decode("utf-8"))
+
+    # Sign off
+    rv = client.post(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/sign-off"
+    )
+    assert_ok(rv)
+
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/sign-off"
+    )
+    compare_json(json.loads(rv.data), {"signedOffAt": assert_is_date})
+    batch_inventory_data = BatchInventoryData.query.get(jurisdiction_ids[0])
+    assert (
+        batch_inventory_data.sign_off_user_id
+        == User.query.filter_by(email=default_ja_email(election_id)).one().id
+    )
+
+    # Download manifest
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/ballot-manifest"
+    )
+    ballot_manifest = rv.data.decode("utf-8")
+    snapshot.assert_match(ballot_manifest)
+
+    # Download batch tallies
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/batch-tallies"
+    )
+    batch_tallies = rv.data.decode("utf-8")
+    snapshot.assert_match(batch_tallies)
+
+    # Upload manifest
+    rv = upload_ballot_manifest(
+        client,
+        io.BytesIO(ballot_manifest.encode()),
+        election_id,
+        jurisdiction_ids[0],
+    )
+    assert_ok(rv)
+
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest"
+    )
+    compare_json(
+        json.loads(rv.data),
+        {
+            "file": {
+                "name": asserts_startswith("manifest"),
+                "uploadedAt": assert_is_date,
+            },
+            "processing": {
+                "status": ProcessingStatus.PROCESSED,
+                "startedAt": assert_is_date,
+                "completedAt": assert_is_date,
+                "error": None,
+            },
+        },
+    )
+
+    # Upload batch tallies
+    rv = upload_batch_tallies(
+        client,
+        io.BytesIO(batch_tallies.encode()),
+        election_id,
+        jurisdiction_ids[0],
+    )
+    assert_ok(rv)
+
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-tallies"
+    )
+    compare_json(
+        json.loads(rv.data),
+        {
+            "file": {
+                "name": asserts_startswith("batch_tallies"),
+                "uploadedAt": assert_is_date,
+            },
+            "processing": {
+                "status": ProcessingStatus.PROCESSED,
+                "startedAt": assert_is_date,
+                "completedAt": assert_is_date,
+                "error": None,
+            },
+        },
+    )
+
+    # Download CVR file
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/cvr/file"
+    )
+    assert rv.data.decode("utf-8") == TEST_CVRS_WITH_EXTRA_SPACES
+
+    # Download tabulator status file
+    rv = client.get(
+        f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/tabulator-status/file"
+    )
+    assert rv.data.decode("utf-8") == TEST_TABULATOR_STATUS
+
+
 def test_batch_inventory_happy_path_multi_contest_batch_comparison(
     client: FlaskClient,
     election_id: str,