From f0588bd7d4c7957f89617574c42dfefe90066931 Mon Sep 17 00:00:00 2001 From: Nikhil Bhatia Date: Wed, 30 Oct 2024 16:52:26 +0000 Subject: [PATCH 1/4] Server endpoint for surfacing discrepacies --- server/api/jurisdictions.py | 97 ++++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/server/api/jurisdictions.py b/server/api/jurisdictions.py index 8b6159ebb..a0caae153 100644 --- a/server/api/jurisdictions.py +++ b/server/api/jurisdictions.py @@ -1,4 +1,5 @@ -from collections import Counter +from collections import Counter, defaultdict +from dataclasses import dataclass import logging from typing import Dict, List, Optional, Mapping, cast as typing_cast import enum @@ -727,3 +728,97 @@ def get_discrepancy_counts_by_jurisdiction(election: Election): for jurisdiction in election.jurisdictions } ) + + +@dataclass +class Discrepancy: + audit_count: int + reported_count: int + candidate_name: str + + +@dataclass +class ContestDiscrepancies: + contest_id: str + jurisdiction_id: str + batch_name: str + discrepancies: List[Discrepancy] + + +@api.route("/election//discrepancy", methods=["GET"]) +@restrict_access([UserType.AUDIT_ADMIN]) +def get_discrepancies_by_jurisdiction(election: Election): + current_round = get_current_round(election) + if not current_round: + raise Conflict("Audit not started") + + discrepancies_by_jurisdiction: Dict[str, List[ContestDiscrepancies]] = defaultdict() + + if election.audit_type == AuditType.BATCH_COMPARISON: + discrepancies_by_jurisdiction = ( + get_batch_comparison_audit_discrepancies_by_jurisdiction( + election, current_round.id + ) + ) + else: + raise Conflict("Discrepancies are only implemented for batch comparison audits") + + return jsonify(discrepancies_by_jurisdiction) + + +def get_batch_comparison_audit_discrepancies_by_jurisdiction( + election: Election, round_id: str +): + discrepancies_by_jurisdiction: Dict[str, List[ContestDiscrepancies]] = defaultdict() + + batch_keys_in_round = set( + SampledBatchDraw.query.filter_by(round_id=round_id) + .join(Batch) + .join(Jurisdiction) + .with_entities(Jurisdiction.name, Batch.name) + .all() + ) + jurisdiction_name_to_id = dict( + Jurisdiction.query.filter_by(election_id=election.id).with_entities( + Jurisdiction.name, Jurisdiction.id + ) + ) + + for contest in list(election.contests): + audited_batch_results = sampled_batch_results( + contest, include_non_rla_batches=True + ) + reported_batch_results = batch_tallies(contest) + + for batch_key, audited_batch_result in audited_batch_results.items(): + if batch_key not in batch_keys_in_round: + continue + + vote_deltas = batch_vote_deltas( + reported_batch_results[batch_key][contest.id], + audited_batch_result[contest.id], + ) + if not vote_deltas: + continue + + jurisdiction_name, batch_name = batch_key + jurisdiction_id = jurisdiction_name_to_id[jurisdiction_name] + if jurisdiction_id not in discrepancies_by_jurisdiction: + discrepancies_by_jurisdiction[jurisdiction_id] = [] + + discrepancies: List[Discrepancy] = [] + for choice in contest.choices: + audit_count = audited_batch_result[contest.id][choice.id] + reported_count = reported_batch_results[batch_key][contest.id][ + choice.id + ] + if audit_count == reported_count: + continue + discrepancy = Discrepancy(audit_count, reported_count, choice.name) + discrepancies.append(discrepancy) + + contest_discrepancies = ContestDiscrepancies( + contest.id, jurisdiction_id, batch_name, discrepancies + ) + discrepancies_by_jurisdiction[jurisdiction_id].append(contest_discrepancies) + return discrepancies_by_jurisdiction From 8e5ffa3400f29c29801b5c56b50a45b30e9f4d6f Mon Sep 17 00:00:00 2001 From: Nikhil Bhatia Date: Thu, 31 Oct 2024 16:42:19 +0000 Subject: [PATCH 2/4] PR feedback: update return model --- server/api/jurisdictions.py | 58 ++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/server/api/jurisdictions.py b/server/api/jurisdictions.py index a0caae153..4975be169 100644 --- a/server/api/jurisdictions.py +++ b/server/api/jurisdictions.py @@ -1,5 +1,4 @@ from collections import Counter, defaultdict -from dataclasses import dataclass import logging from typing import Dict, List, Optional, Mapping, cast as typing_cast import enum @@ -730,19 +729,22 @@ def get_discrepancy_counts_by_jurisdiction(election: Election): ) -@dataclass -class Discrepancy: - audit_count: int - reported_count: int - candidate_name: str +DiscrepanciesByJurisdiction = Dict[str, Dict[str, Dict[str, Dict[str, int]]]] +# DiscrepanciesByJurisdiction = { +# jurisdictionID: { +# batchName: { +# contestID: { +# reportedVotes: {choiceID: int}, +# auditedVotes: {choiceID: int}, +# discrepancies: {choiceID: int}, +# } +# } -@dataclass -class ContestDiscrepancies: - contest_id: str - jurisdiction_id: str - batch_name: str - discrepancies: List[Discrepancy] +def create_nested_discrepancies_dict(): + return defaultdict( + lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(int))) + ) @api.route("/election//discrepancy", methods=["GET"]) @@ -752,7 +754,7 @@ def get_discrepancies_by_jurisdiction(election: Election): if not current_round: raise Conflict("Audit not started") - discrepancies_by_jurisdiction: Dict[str, List[ContestDiscrepancies]] = defaultdict() + discrepancies_by_jurisdiction: DiscrepanciesByJurisdiction = {} if election.audit_type == AuditType.BATCH_COMPARISON: discrepancies_by_jurisdiction = ( @@ -769,8 +771,9 @@ def get_discrepancies_by_jurisdiction(election: Election): def get_batch_comparison_audit_discrepancies_by_jurisdiction( election: Election, round_id: str ): - discrepancies_by_jurisdiction: Dict[str, List[ContestDiscrepancies]] = defaultdict() + discrepancies_by_jurisdiction = create_nested_discrepancies_dict() + # TODO: Add support for combined batches batch_keys_in_round = set( SampledBatchDraw.query.filter_by(round_id=round_id) .join(Batch) @@ -794,31 +797,20 @@ def get_batch_comparison_audit_discrepancies_by_jurisdiction( if batch_key not in batch_keys_in_round: continue + audited_contest_result = audited_batch_result[contest.id] + reported_contest_result = reported_batch_results[batch_key][contest.id] vote_deltas = batch_vote_deltas( - reported_batch_results[batch_key][contest.id], - audited_batch_result[contest.id], + reported_contest_result, audited_contest_result ) if not vote_deltas: continue jurisdiction_name, batch_name = batch_key jurisdiction_id = jurisdiction_name_to_id[jurisdiction_name] - if jurisdiction_id not in discrepancies_by_jurisdiction: - discrepancies_by_jurisdiction[jurisdiction_id] = [] - - discrepancies: List[Discrepancy] = [] - for choice in contest.choices: - audit_count = audited_batch_result[contest.id][choice.id] - reported_count = reported_batch_results[batch_key][contest.id][ - choice.id - ] - if audit_count == reported_count: - continue - discrepancy = Discrepancy(audit_count, reported_count, choice.name) - discrepancies.append(discrepancy) + discrepancies_by_jurisdiction[jurisdiction_id][batch_name][contest.id] = { + "reportedVotes": reported_contest_result, + "auditedVotes": audited_contest_result, + "discrepancies": vote_deltas, + } - contest_discrepancies = ContestDiscrepancies( - contest.id, jurisdiction_id, batch_name, discrepancies - ) - discrepancies_by_jurisdiction[jurisdiction_id].append(contest_discrepancies) return discrepancies_by_jurisdiction From 0678197476e7a55fe5a60e1c5c1bfe2fcec2bde3 Mon Sep 17 00:00:00 2001 From: Nikhil Bhatia Date: Thu, 31 Oct 2024 18:38:58 +0000 Subject: [PATCH 3/4] Tests for batch comparison discrepancies --- server/tests/api/test_jurisdictions.py | 35 ++++++++++++ .../batch_comparison/test_batch_comparison.py | 29 +++++++++- .../test_multi_contest_batch_comparison.py | 55 +++++++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) diff --git a/server/tests/api/test_jurisdictions.py b/server/tests/api/test_jurisdictions.py index f776cecb2..dcbbc3e01 100644 --- a/server/tests/api/test_jurisdictions.py +++ b/server/tests/api/test_jurisdictions.py @@ -429,3 +429,38 @@ def test_discrepancy_counts_before_audit_launch( } ] } + + +def test_discrepancy_before_audit_launch( + client: FlaskClient, + election_id: str, + jurisdiction_ids: List[str], # pylint: disable=unused-argument +): + rv = client.get(f"/api/election/{election_id}/discrepancy") + assert rv.status_code == 409 + assert json.loads(rv.data) == { + "errors": [ + { + "errorType": "Conflict", + "message": "Audit not started", + } + ] + } + + +def test_discrepancy_non_batch_comparison_enabled( + client: FlaskClient, + election_id: str, + jurisdiction_ids: List[str], # pylint: disable=unused-argument + round_1_id: str, # pylint: disable=unused-argument +): + rv = client.get(f"/api/election/{election_id}/discrepancy") + assert rv.status_code == 409 + assert json.loads(rv.data) == { + "errors": [ + { + "errorType": "Conflict", + "message": "Discrepancies are only implemented for batch comparison audits", + } + ] + } diff --git a/server/tests/batch_comparison/test_batch_comparison.py b/server/tests/batch_comparison/test_batch_comparison.py index 305e72081..edfc1806a 100644 --- a/server/tests/batch_comparison/test_batch_comparison.py +++ b/server/tests/batch_comparison/test_batch_comparison.py @@ -328,6 +328,28 @@ def test_batch_comparison_round_2( # In J2, single sampled batch hasn't been audited yet. The frontend won't show this to the user. assert discrepancy_counts[jurisdictions[1]["id"]] == 1 + # Check discrepancies + rv = client.get(f"/api/election/{election_id}/discrepancy") + discrepancies = json.loads(rv.data) + assert ( + discrepancies[jurisdictions[0]["id"]]["Batch 1"][contests[0]["id"]][ + "reportedVotes" + ][choice_ids[0]] + == 500 + ) + assert ( + discrepancies[jurisdictions[0]["id"]]["Batch 1"][contests[0]["id"]][ + "auditedVotes" + ][choice_ids[0]] + == 400 + ) + assert ( + discrepancies[jurisdictions[0]["id"]]["Batch 1"][contests[0]["id"]][ + "discrepancies" + ][choice_ids[0]] + == 100 + ) + # Now do the second jurisdiction set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) @@ -652,10 +674,15 @@ def test_batch_comparison_batches_sampled_multiple_times( # Check discrepancy counts rv = client.get(f"/api/election/{election_id}/discrepancy-counts") discrepancy_counts = json.loads(rv.data) - print(discrepancy_counts) assert discrepancy_counts[jurisdictions[0]["id"]] == 0 assert discrepancy_counts[jurisdictions[1]["id"]] == 0 + # Check discrepancies + rv = client.get(f"/api/election/{election_id}/discrepancy") + discrepancies = json.loads(rv.data) + assert jurisdictions[0]["id"] not in discrepancies + assert jurisdictions[1]["id"] not in discrepancies + # End the round rv = client.post(f"/api/election/{election_id}/round/current/finish") assert_ok(rv) diff --git a/server/tests/batch_comparison/test_multi_contest_batch_comparison.py b/server/tests/batch_comparison/test_multi_contest_batch_comparison.py index 3fc160241..4b4fe230a 100644 --- a/server/tests/batch_comparison/test_multi_contest_batch_comparison.py +++ b/server/tests/batch_comparison/test_multi_contest_batch_comparison.py @@ -692,6 +692,28 @@ def test_multi_contest_batch_comparison_end_to_end( assert discrepancy_counts[jurisdictions[1]["id"]] == 0 assert discrepancy_counts[jurisdictions[2]["id"]] == 1 + # Check discrepancies + rv = client.get(f"/api/election/{election_id}/discrepancy") + discrepancies = json.loads(rv.data) + assert ( + discrepancies[jurisdictions[0]["id"]]["Batch 6"][contest_ids[0]][ + "reportedVotes" + ][contest_1_choice_ids[0]] + == 50 + ) + assert ( + discrepancies[jurisdictions[0]["id"]]["Batch 6"][contest_ids[0]][ + "auditedVotes" + ][contest_1_choice_ids[0]] + == 49 + ) + assert ( + discrepancies[jurisdictions[0]["id"]]["Batch 6"][contest_ids[0]][ + "discrepancies" + ][contest_1_choice_ids[0]] + == 1 + ) + # # Finish audit # @@ -880,6 +902,31 @@ def test_multi_contest_batch_comparison_round_2( assert discrepancy_counts[jurisdiction_ids[1]] == 0 assert discrepancy_counts[jurisdiction_ids[2]] == 0 + # Check discrepancies + rv = client.get(f"/api/election/{election_id}/discrepancy") + discrepancies = json.loads(rv.data) + + assert ( + discrepancies[jurisdiction_ids[0]]["Batch 7"][contest_ids[0]]["reportedVotes"][ + contest_1_choice_ids[0] + ] + == 50 + ) + assert ( + discrepancies[jurisdiction_ids[0]]["Batch 7"][contest_ids[0]]["auditedVotes"][ + contest_1_choice_ids[0] + ] + == 0 + ) + assert ( + discrepancies[jurisdiction_ids[0]]["Batch 7"][contest_ids[0]]["discrepancies"][ + contest_1_choice_ids[0] + ] + == 50 + ) + assert jurisdiction_ids[1] not in discrepancies + assert jurisdiction_ids[2] not in discrepancies + # # End round 1 # @@ -1012,6 +1059,14 @@ def test_multi_contest_batch_comparison_round_2( assert discrepancy_counts[jurisdiction_ids[1]] == 0 assert discrepancy_counts[jurisdiction_ids[2]] == 0 + # Check discrepancies + rv = client.get(f"/api/election/{election_id}/discrepancy") + discrepancies = json.loads(rv.data) + + assert jurisdiction_ids[0] not in discrepancies + assert jurisdiction_ids[1] not in discrepancies + assert jurisdiction_ids[2] not in discrepancies + # # End round 2 / finish audit # From 466ac216ff8293c6690b50ad6d07feda35f699c7 Mon Sep 17 00:00:00 2001 From: Nikhil Bhatia Date: Thu, 31 Oct 2024 20:43:33 +0000 Subject: [PATCH 4/4] PR feedback: simplify dict init --- server/api/jurisdictions.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/server/api/jurisdictions.py b/server/api/jurisdictions.py index 4975be169..8184c4b84 100644 --- a/server/api/jurisdictions.py +++ b/server/api/jurisdictions.py @@ -729,7 +729,7 @@ def get_discrepancy_counts_by_jurisdiction(election: Election): ) -DiscrepanciesByJurisdiction = Dict[str, Dict[str, Dict[str, Dict[str, int]]]] +DiscrepanciesByJurisdiction = Dict[str, Dict[str, Dict[str, Dict[str, Dict[str, int]]]]] # DiscrepanciesByJurisdiction = { # jurisdictionID: { # batchName: { @@ -741,12 +741,6 @@ def get_discrepancy_counts_by_jurisdiction(election: Election): # } -def create_nested_discrepancies_dict(): - return defaultdict( - lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(int))) - ) - - @api.route("/election//discrepancy", methods=["GET"]) @restrict_access([UserType.AUDIT_ADMIN]) def get_discrepancies_by_jurisdiction(election: Election): @@ -770,8 +764,10 @@ def get_discrepancies_by_jurisdiction(election: Election): def get_batch_comparison_audit_discrepancies_by_jurisdiction( election: Election, round_id: str -): - discrepancies_by_jurisdiction = create_nested_discrepancies_dict() +) -> DiscrepanciesByJurisdiction: + discrepancies_by_jurisdiction: DiscrepanciesByJurisdiction = defaultdict( + lambda: defaultdict(dict) + ) # TODO: Add support for combined batches batch_keys_in_round = set(