From 699b65919c0dbc7f9405be54adf237ba9548bf4d Mon Sep 17 00:00:00 2001 From: carolinemodic Date: Wed, 30 Oct 2024 10:27:08 -0700 Subject: [PATCH] Support large(r) file uploads by directly uploading to S3 (#2012) * update client code to get upload url, post to upload, post to finalize * restrict all cvr files uploads to 1 file * client tests passing * add file helper and parsing utils * update server endpoints to break upload into 3 steps * handle unwrapped hart zip * add helper functions to manage test uploads * add tests for public file upload endpoint * add tests for get upload url endpoints * refactor server tests to call helper functions to upload files and handle error testing * update test missed in rebase * change files to file in client useCSV * update upload complete calls to use json not form data * nit changes to file utils * dont create subdir for cvrs * make upload file require logged in user * address various pr nit comments in api * rename test helpers * fix server to expect json params on upload complete * test csp on server --- client/src/components/Atoms/CSVForm.tsx | 35 +- .../src/components/Atoms/FileUpload.test.tsx | 176 ++++- .../AuditAdmin/AuditAdminView.test.tsx | 14 +- .../Progress/JurisdictionDetail.test.tsx | 128 +--- .../Progress/JurisdictionDetail.tsx | 5 +- .../Setup/Participants/Participants.test.tsx | 94 ++- .../Setup/Participants/Participants.tsx | 8 +- .../JurisdictionAdmin/BatchInventory.test.tsx | 83 ++- .../JurisdictionAdmin/BatchInventory.tsx | 10 +- .../JurisdictionAdminView.test.tsx | 132 +--- .../JurisdictionAdminView.tsx | 6 +- client/src/components/_mocks.ts | 244 +++++-- client/src/components/testUtilities.tsx | 4 + client/src/components/useCSV.ts | 87 ++- client/src/components/useFileUpload.ts | 87 ++- server/api/ballot_manifest.py | 59 +- server/api/batch_inventory.py | 109 +-- server/api/batch_tallies.py | 59 +- server/api/cvrs.py | 160 +++-- server/api/jurisdictions.py | 40 +- server/api/public.py | 25 +- server/api/standardized_contests.py | 67 +- server/app.py | 1 + server/auth/auth_helpers.py | 17 + server/config.py | 10 +- server/tests/api/test_activity.py | 31 +- server/tests/api/test_ballot_manifest.py | 182 +++-- server/tests/api/test_ballots.py | 23 +- server/tests/api/test_jurisdictions.py | 34 +- server/tests/api/test_jurisdictions_file.py | 307 ++++---- server/tests/api/test_public.py | 83 +++ server/tests/ballot_comparison/conftest.py | 130 ++-- .../test_ballot_comparison.py | 287 ++++---- .../test_ballot_comparison_manifests.py | 171 +++-- .../test_contest_name_standardizations.py | 15 +- server/tests/ballot_comparison/test_cvrs.py | 661 ++++++++++-------- .../test_standardized_contests.py | 317 ++++----- server/tests/batch_comparison/conftest.py | 91 ++- .../batch_comparison/test_batch_comparison.py | 10 +- .../batch_comparison/test_batch_inventory.py | 552 ++++++++------- .../batch_comparison/test_batch_tallies.py | 371 +++++----- server/tests/batch_comparison/test_batches.py | 38 +- .../test_multi_contest_batch_comparison.py | 33 +- ..._sample_extra_batches_by_counting_group.py | 247 +++---- server/tests/conftest.py | 76 +- server/tests/helpers.py | 180 ++++- server/tests/hybrid/conftest.py | 84 +-- server/tests/hybrid/test_hybrid.py | 137 ++-- server/tests/hybrid/test_hybrid_manifests.py | 91 ++- server/tests/test_full_hand_tally.py | 54 +- server/tests/util/test_csv_parse.py | 21 +- server/tests/util/test_file_storage.py | 169 ++++- server/util/csv_parse.py | 14 +- server/util/file.py | 89 ++- 54 files changed, 3431 insertions(+), 2727 deletions(-) diff --git a/client/src/components/Atoms/CSVForm.tsx b/client/src/components/Atoms/CSVForm.tsx index 327e17faa..aee7008b4 100644 --- a/client/src/components/Atoms/CSVForm.tsx +++ b/client/src/components/Atoms/CSVForm.tsx @@ -15,7 +15,6 @@ import { CvrFileType, IFileInfo, FileProcessingStatus } from '../useCSV' import FormWrapper from './Form/FormWrapper' import { FormSectionDescription } from './Form/FormSection' import { ErrorLabel, SuccessLabel } from './Form/_helpers' -import { sum } from '../../utils/number' import FormButton from './Form/FormButton' import AsyncButton from './AsyncButton' @@ -30,13 +29,13 @@ const schema = Yup.object().shape({ }) interface IValues { - csv: File[] | null + csv: File | null cvrFileType?: CvrFileType } interface IProps { csvFile: IFileInfo - uploadCSVFiles: (csvs: File[], cvrFileType?: CvrFileType) => Promise + uploadCSVFile: (csv: File, cvrFileType?: CvrFileType) => Promise deleteCSVFile?: () => Promise title?: string description: string @@ -47,7 +46,7 @@ interface IProps { const CSVFile: React.FC = ({ csvFile, - uploadCSVFiles, + uploadCSVFile, deleteCSVFile, title, description, @@ -62,7 +61,7 @@ const CSVFile: React.FC = ({ return ( = ({ validateOnBlur={false} onSubmit={async (values: IValues) => { if (values.csv) { - await uploadCSVFiles(values.csv, values.cvrFileType) + await uploadCSVFile(values.csv, values.cvrFileType) setIsEditing(false) } }} @@ -117,34 +116,30 @@ const CSVFile: React.FC = ({ { const { files } = e.currentTarget setFieldValue( 'csv', - files && files.length > 0 ? Array.from(files) : null + files && files.length === 1 ? files[0] : null ) }} hasSelection={!!values.csv} text={(() => { if (!values.csv) { - return values.cvrFileType === CvrFileType.ESS || - values.cvrFileType === CvrFileType.HART - ? 'Select files...' - : 'Select a file...' + return 'Select a file...' } - if (values.csv.length === 1) return values.csv[0].name - return `${values.csv.length} files selected` + return values.csv.name })()} onBlur={handleBlur} disabled={isSubmitting || isProcessing || !enabled} @@ -180,7 +175,7 @@ const CSVFile: React.FC = ({ {upload && // Only show upload progress for large sets of files (over 1 MB), // otherwise it will just flash on the screen - sum(upload.files.map(f => f.size)) >= 1000 * 1000 && ( + upload.file.size >= 1000 * 1000 && ( <> Uploading... diff --git a/client/src/components/Atoms/FileUpload.test.tsx b/client/src/components/Atoms/FileUpload.test.tsx index bff06a2ed..e1a975c57 100644 --- a/client/src/components/Atoms/FileUpload.test.tsx +++ b/client/src/components/Atoms/FileUpload.test.tsx @@ -36,11 +36,7 @@ const TestFileUpload = ({ const fileUpload: IFileUpload = { uploadedFile, uploadFiles: files => { - const formData = new FormData() - for (const file of files) { - formData.append('files', file, file.name) - } - return uploadFiles.mutateAsync(formData) + return uploadFiles.mutateAsync({ file: files[0] }) }, uploadProgress: uploadFiles.progress, deleteFile: () => deleteFile.mutateAsync(), @@ -66,16 +62,50 @@ const render = (element: React.ReactElement) => const testFile = new File(['test content'], 'test-file.csv', { type: 'text/csv', }) -const formData = new FormData() -formData.append('files', testFile, testFile.name) +const uploadFormData = new FormData() +uploadFormData.append('key', 'path/to/file/file.csv') +uploadFormData.append('otherField', 'canBePassedThrough') +uploadFormData.append('Content-Type', testFile.type) +uploadFormData.append('file', testFile, testFile.name) + +const uploadCompleteJSONData = { + fileName: testFile.name, + fileType: testFile.type, + storagePathKey: 'path/to/file/file.csv', +} + +const getUploadUrlMock = { + url: '/test/file-upload', + fields: { + key: 'path/to/file/file.csv', + otherField: 'canBePassedThrough', + }, +} describe('FileUpload + useFileUpload', () => { it('when no file is uploaded, shows a form to upload a file', async () => { const expectedCalls = [ { url: '/test', response: fileInfoMocks.empty }, { - url: '/test', - options: { method: 'PUT', body: formData }, + url: '/test/upload-url', + options: { + method: 'GET', + params: { fileType: testFile.type }, + }, + response: getUploadUrlMock, + }, + { + url: '/test/file-upload', + options: { method: 'POST', body: uploadFormData }, + response: { status: 'ok' }, + }, + { + url: '/test/upload-complete', + options: { + method: 'POST', + body: (uploadCompleteJSONData as unknown) as BodyInit, + headers: { 'Content-Type': 'application/json' }, + }, response: { status: 'ok' }, }, { url: '/test', response: fileInfoMocks.processing }, @@ -138,8 +168,25 @@ describe('FileUpload + useFileUpload', () => { const expectedCalls = [ { url: '/test', response: fileInfoMocks.empty }, { - url: '/test', - options: { method: 'PUT', body: formData }, + url: '/test/upload-url', + options: { + method: 'GET', + params: { fileType: testFile.type }, + }, + response: getUploadUrlMock, + }, + { + url: '/test/file-upload', + options: { method: 'POST', body: uploadFormData }, + response: { status: 'ok' }, + }, + { + url: '/test/upload-complete', + options: { + method: 'POST', + body: (uploadCompleteJSONData as unknown) as BodyInit, + headers: { 'Content-Type': 'application/json' }, + }, response: { status: 'ok' }, }, { url: '/test', response: fileInfoMocks.errored }, @@ -175,8 +222,25 @@ describe('FileUpload + useFileUpload', () => { const expectedCalls = [ { url: '/test', response: fileInfoMocks.empty }, { - url: '/test', - options: { method: 'PUT', body: formData2 }, + url: '/test/upload-url', + options: { + method: 'GET', + params: { fileType: testFile.type }, + }, + response: getUploadUrlMock, + }, + { + url: '/test/file-upload', + options: { method: 'POST', body: uploadFormData }, + response: { status: 'ok' }, + }, + { + url: '/test/upload-complete', + options: { + method: 'POST', + body: (uploadCompleteJSONData as unknown) as BodyInit, + headers: { 'Content-Type': 'application/json' }, + }, response: { status: 'ok' }, }, { url: '/test', response: fileInfoMocks.processed }, @@ -232,14 +296,88 @@ describe('FileUpload + useFileUpload', () => { }) }) - it('handles an API error on put', async () => { + it('handles an API error on get', async () => { const expectedCalls = [ { url: '/test', response: fileInfoMocks.empty }, - serverError('putFile', { - url: '/test', + serverError('getFile', { + url: '/test/upload-url', + options: { + method: 'GET', + params: { fileType: testFile.type }, + } as RequestInit, + }), + ] + await withMockFetch(expectedCalls, async () => { + render( + <> + + + + ) + await screen.findByText('Test File') + userEvent.upload(screen.getByLabelText('Select a file...'), testFile) + await screen.findByText('test-file.csv') + userEvent.click(screen.getByRole('button', { name: /Upload/ })) + await findAndCloseToast('getFile') + }) + }) + + it('handles an API error on file upload', async () => { + const expectedCalls = [ + { url: '/test', response: fileInfoMocks.empty }, + { + url: '/test/upload-url', + options: { + method: 'GET', + params: { fileType: testFile.type }, + }, + response: getUploadUrlMock, + }, + serverError('postFileUpload', { + url: '/test/file-upload', + options: { + method: 'POST', + body: uploadFormData, + }, + }), + ] + await withMockFetch(expectedCalls, async () => { + render( + <> + + + + ) + await screen.findByText('Test File') + userEvent.upload(screen.getByLabelText('Select a file...'), testFile) + await screen.findByText('test-file.csv') + userEvent.click(screen.getByRole('button', { name: /Upload/ })) + await findAndCloseToast('postFileUpload') + }) + }) + + it('handles an API error on file upload completion', async () => { + const expectedCalls = [ + { url: '/test', response: fileInfoMocks.empty }, + { + url: '/test/upload-url', + options: { + method: 'GET', + params: { fileType: testFile.type }, + }, + response: getUploadUrlMock, + }, + { + url: '/test/file-upload', + options: { method: 'POST', body: uploadFormData }, + response: { status: 'ok' }, + }, + serverError('postFileComplete', { + url: '/test/upload-complete', options: { - method: 'PUT', - body: formData, + method: 'POST', + body: (uploadCompleteJSONData as unknown) as BodyInit, + headers: { 'Content-Type': 'application/json' }, }, }), ] @@ -254,7 +392,7 @@ describe('FileUpload + useFileUpload', () => { userEvent.upload(screen.getByLabelText('Select a file...'), testFile) await screen.findByText('test-file.csv') userEvent.click(screen.getByRole('button', { name: /Upload/ })) - await findAndCloseToast('putFile') + await findAndCloseToast('postFileComplete') }) }) diff --git a/client/src/components/AuditAdmin/AuditAdminView.test.tsx b/client/src/components/AuditAdmin/AuditAdminView.test.tsx index 1f60e9e88..2f0d0f77d 100644 --- a/client/src/components/AuditAdmin/AuditAdminView.test.tsx +++ b/client/src/components/AuditAdmin/AuditAdminView.test.tsx @@ -127,7 +127,9 @@ describe('AA setup flow', () => { aaApiCalls.getContests(contestMocks.filledTargeted), aaApiCalls.getSettings(auditSettingsMocks.all), aaApiCalls.getJurisdictionFileWithResponse(jurisdictionFileMocks.empty), - aaApiCalls.putJurisdictionFile, + aaApiCalls.uploadJurisdictionFileGetUrl, + aaApiCalls.uploadJurisdictionFilePostFile, + aaApiCalls.uploadJurisdictionFileUploadComplete, aaApiCalls.getJurisdictionFileWithResponse( jurisdictionFileMocks.processed ), @@ -160,7 +162,9 @@ describe('AA setup flow', () => { aaApiCalls.getContests(contestMocks.filledTargeted), aaApiCalls.getSettings(auditSettingsMocks.all), aaApiCalls.getJurisdictionFileWithResponse(jurisdictionFileMocks.empty), - aaApiCalls.putJurisdictionErrorFile, + aaApiCalls.uploadJurisdictionFileGetUrl, + aaApiCalls.uploadJurisdictionFilePostFile, + aaApiCalls.uploadJurisdictionFileUploadCompleteError, aaApiCalls.getJurisdictionFileWithResponse(jurisdictionFileMocks.errored), aaApiCalls.getJurisdictions, ] @@ -197,7 +201,9 @@ describe('AA setup flow', () => { aaApiCalls.getStandardizedContestsFileWithResponse( standardizedContestsFileMocks.empty ), - aaApiCalls.putStandardizedContestsFile, + aaApiCalls.uploadStandardizedContestsFileGetUrl, + aaApiCalls.uploadStandardizedContestsFilePostFile, + aaApiCalls.uploadStandardizedContestsFileUploadComplete, aaApiCalls.getStandardizedContestsFileWithResponse( standardizedContestsFileMocks.processed ), @@ -320,7 +326,7 @@ describe('AA setup flow', () => { aaApiCalls.getSettings(auditSettingsMocks.all), aaApiCalls.getMapData, jaApiCalls.getBallotManifestFile(manifestMocks.empty), - jaApiCalls.putManifest, + ...jaApiCalls.uploadManifestCalls, jaApiCalls.getBallotManifestFile(manifestMocks.processed), { ...aaApiCalls.getJurisdictions, diff --git a/client/src/components/AuditAdmin/Progress/JurisdictionDetail.test.tsx b/client/src/components/AuditAdmin/Progress/JurisdictionDetail.test.tsx index 84394c993..7afa13088 100644 --- a/client/src/components/AuditAdmin/Progress/JurisdictionDetail.test.tsx +++ b/client/src/components/AuditAdmin/Progress/JurisdictionDetail.test.tsx @@ -69,7 +69,7 @@ describe('JurisdictionDetail', () => { it('before launch, shows manifest for ballot polling audit', async () => { const expectedCalls = [ jaApiCalls.getBallotManifestFile(manifestMocks.empty), - jaApiCalls.putManifest, + ...jaApiCalls.uploadManifestCalls, jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.deleteManifest, jaApiCalls.getBallotManifestFile(manifestMocks.empty), @@ -112,15 +112,15 @@ describe('JurisdictionDetail', () => { const expectedCalls = [ jaApiCalls.getBallotManifestFile(manifestMocks.empty), jaApiCalls.getCVRSfile(cvrsMocks.empty), - jaApiCalls.putManifest, + ...jaApiCalls.uploadManifestCalls, jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.getCVRSfile(cvrsMocks.empty), - jaApiCalls.putCVRs, + ...jaApiCalls.uploadCVRsCalls, jaApiCalls.getCVRSfile(cvrsMocks.processed), jaApiCalls.deleteManifest, jaApiCalls.getBallotManifestFile(manifestMocks.empty), jaApiCalls.getCVRSfile(cvrsMocks.processed), - jaApiCalls.putManifest, + ...jaApiCalls.uploadManifestCalls, jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.getCVRSfile(cvrsMocks.errored), jaApiCalls.deleteCVRs, @@ -213,68 +213,15 @@ describe('JurisdictionDetail', () => { }) }) - it('before launch, accepts multiple files for ES&S CVR uploads', async () => { - const cvrsFormData: FormData = new FormData() - const cvrsFile1 = new File(['test cvr data'], 'cvrs1.csv', { - type: 'text/csv', - }) - const cvrsFile2 = new File(['test cvr data'], 'cvrs2.csv', { - type: 'text/csv', - }) - cvrsFormData.append('cvrs', cvrsFile1, cvrsFile1.name) - cvrsFormData.append('cvrs', cvrsFile2, cvrsFile2.name) - cvrsFormData.append('cvrFileType', 'ESS') - - const expectedCalls = [ - jaApiCalls.getBallotManifestFile(manifestMocks.processed), - jaApiCalls.getCVRSfile(cvrsMocks.empty), - { - ...jaApiCalls.putCVRs, - options: { ...jaApiCalls.putCVRs.options, body: cvrsFormData }, - }, - jaApiCalls.getCVRSfile(cvrsMocks.processed), - ] - await withMockFetch(expectedCalls, async () => { - render({ - jurisdiction: { - ...jurisdictionMocks.allManifests[0], - cvrs: cvrsMocks.empty, - }, - auditSettings: auditSettingsMocks.ballotComparisonAll, - }) - - const cvrsCard = ( - await screen.findByRole('heading', { name: 'Cast Vote Records (CVR)' }) - ).closest('.bp3-card') as HTMLElement - userEvent.selectOptions( - within(cvrsCard).getByLabelText('CVR File Type:'), - within(cvrsCard).getByRole('option', { name: 'ES&S' }) - ) - userEvent.upload( - await within(cvrsCard).findByLabelText('Select files...'), - [cvrsFile1, cvrsFile2] - ) - await within(cvrsCard).findByText('2 files selected') - userEvent.click(screen.getByRole('button', { name: /Upload/ })) - await within(cvrsCard).findByText('Uploaded') - }) - }) - it('before launch, accepts Hart CVR ZIP file', async () => { - const cvrsFormData: FormData = new FormData() const cvrsZip = new File(['test cvr data'], 'cvrs.zip', { type: 'application/zip', }) - cvrsFormData.append('cvrs', cvrsZip, cvrsZip.name) - cvrsFormData.append('cvrFileType', 'HART') const expectedCalls = [ jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.getCVRSfile(cvrsMocks.empty), - { - ...jaApiCalls.putCVRs, - options: { ...jaApiCalls.putCVRs.options, body: cvrsFormData }, - }, + ...jaApiCalls.uploadCVRZipCalls, jaApiCalls.getCVRSfile(cvrsMocks.processed), ] await withMockFetch(expectedCalls, async () => { @@ -294,8 +241,8 @@ describe('JurisdictionDetail', () => { within(cvrsCard).getByRole('option', { name: 'Hart' }) ) userEvent.upload( - await within(cvrsCard).findByLabelText('Select files...'), - [cvrsZip] + await within(cvrsCard).findByLabelText('Select a file...'), + cvrsZip ) await within(cvrsCard).findByText('cvrs.zip') userEvent.click(screen.getByRole('button', { name: /Upload/ })) @@ -303,74 +250,19 @@ describe('JurisdictionDetail', () => { }) }) - it('before launch, accepts Hart CVR ZIP file and scanned ballot information CSV', async () => { - const cvrsFormData: FormData = new FormData() - const cvrsZip = new File(['test cvr data'], 'cvrs.zip', { - type: 'application/zip', - }) - const scannedBallotInformationCsv = new File( - ['test cvr data'], - 'scanned-ballot-information.csv', - { - type: 'text/csv', - } - ) - cvrsFormData.append('cvrs', cvrsZip, cvrsZip.name) - cvrsFormData.append( - 'cvrs', - scannedBallotInformationCsv, - scannedBallotInformationCsv.name - ) - cvrsFormData.append('cvrFileType', 'HART') - - const expectedCalls = [ - jaApiCalls.getBallotManifestFile(manifestMocks.processed), - jaApiCalls.getCVRSfile(cvrsMocks.empty), - { - ...jaApiCalls.putCVRs, - options: { ...jaApiCalls.putCVRs.options, body: cvrsFormData }, - }, - jaApiCalls.getCVRSfile(cvrsMocks.processed), - ] - await withMockFetch(expectedCalls, async () => { - render({ - jurisdiction: { - ...jurisdictionMocks.allManifests[0], - cvrs: cvrsMocks.empty, - }, - auditSettings: auditSettingsMocks.ballotComparisonAll, - }) - - const cvrsCard = ( - await screen.findByRole('heading', { name: 'Cast Vote Records (CVR)' }) - ).closest('.bp3-card') as HTMLElement - userEvent.selectOptions( - within(cvrsCard).getByLabelText('CVR File Type:'), - within(cvrsCard).getByRole('option', { name: 'Hart' }) - ) - userEvent.upload( - await within(cvrsCard).findByLabelText('Select files...'), - [cvrsZip, scannedBallotInformationCsv] - ) - await within(cvrsCard).findByText('2 files selected') - userEvent.click(screen.getByRole('button', { name: /Upload/ })) - await within(cvrsCard).findByText('Uploaded') - }) - }) - it('before launch, shows manifest and batch tallies for batch comparison audit', async () => { const expectedCalls = [ jaApiCalls.getBallotManifestFile(manifestMocks.empty), jaApiCalls.getBatchTalliesFile(talliesMocks.empty), - jaApiCalls.putManifest, + ...jaApiCalls.uploadManifestCalls, jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.getBatchTalliesFile(talliesMocks.empty), - jaApiCalls.putTallies, + ...jaApiCalls.uploadTalliesCalls, jaApiCalls.getBatchTalliesFile(talliesMocks.processed), jaApiCalls.deleteManifest, jaApiCalls.getBallotManifestFile(manifestMocks.empty), jaApiCalls.getBatchTalliesFile(talliesMocks.processed), - jaApiCalls.putManifest, + ...jaApiCalls.uploadManifestCalls, jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.getBatchTalliesFile(talliesMocks.errored), jaApiCalls.deleteTallies, diff --git a/client/src/components/AuditAdmin/Progress/JurisdictionDetail.tsx b/client/src/components/AuditAdmin/Progress/JurisdictionDetail.tsx index 7fd0cda5a..b2b0121f4 100644 --- a/client/src/components/AuditAdmin/Progress/JurisdictionDetail.tsx +++ b/client/src/components/AuditAdmin/Progress/JurisdictionDetail.tsx @@ -165,10 +165,7 @@ const CvrsFileUpload = ({ acceptFileTypes={ selectedCvrFileType === CvrFileType.HART ? ['zip', 'csv'] : ['csv'] } - allowMultipleFiles={ - selectedCvrFileType === CvrFileType.ESS || - selectedCvrFileType === CvrFileType.HART - } + allowMultipleFiles={false} uploadDisabled={uploadDisabled || (!cvrs.file && !selectedCvrFileType)} deleteDisabled={deleteDisabled} additionalFields={ diff --git a/client/src/components/AuditAdmin/Setup/Participants/Participants.test.tsx b/client/src/components/AuditAdmin/Setup/Participants/Participants.test.tsx index fa028cba3..58cc817c5 100644 --- a/client/src/components/AuditAdmin/Setup/Participants/Participants.test.tsx +++ b/client/src/components/AuditAdmin/Setup/Participants/Participants.test.tsx @@ -6,6 +6,10 @@ import Participants, { IParticipantsProps } from './Participants' import { jurisdictionFile } from './_mocks' import { withMockFetch, createQueryClient } from '../../../testUtilities' import { IFileInfo, FileProcessingStatus } from '../../../useCSV' +import { + getMockFormDataForFileUpload, + getMockJsonDataForUploadComplete, +} from '../../../_mocks' jest.mock('axios') @@ -75,29 +79,63 @@ const apiCalls = { url: '/api/election/1/standardized-contests/file', response, }), - putJurisdictionsFile: (file: File) => { - const formData: FormData = new FormData() - formData.append('jurisdictions', file, file.name) - return { - url: '/api/election/1/jurisdiction/file', - options: { - method: 'PUT', - body: formData, + uploadJurisdictionFileCalls: (file: File) => { + return [ + { + url: '/api/election/1/jurisdiction/file/upload-url', + options: { + method: 'GET', + params: { fileType: file.type }, + }, + response: { url: '/api/upload', fields: { key: '/path/to/file' } }, }, - response: { status: 'ok' }, - } + { + url: '/api/upload', + options: { + method: 'POST', + body: getMockFormDataForFileUpload(file), + }, + response: { status: 'ok' }, + }, + { + url: '/api/election/1/jurisdiction/file/upload-complete', + options: { + method: 'POST', + body: getMockJsonDataForUploadComplete(file), + headers: { 'Content-Type': 'application/json' }, + }, + response: { status: 'ok' }, + }, + ] }, - putStandardizedContestsFile: (file: File) => { - const formData: FormData = new FormData() - formData.append('standardized-contests', file, file.name) - return { - url: '/api/election/1/standardized-contests/file', - options: { - method: 'PUT', - body: formData, + uploadStandardizedContestsFileCalls: (file: File) => { + return [ + { + url: '/api/election/1/standardized-contests/file/upload-url', + options: { + method: 'GET', + params: { fileType: file.type }, + }, + response: { url: '/api/upload', fields: { key: '/path/to/file' } }, }, - response: { status: 'ok' }, - } + { + url: '/api/upload', + options: { + method: 'POST', + body: getMockFormDataForFileUpload(file), + }, + response: { status: 'ok' }, + }, + { + url: '/api/election/1/standardized-contests/file/upload-complete', + options: { + method: 'POST', + body: getMockJsonDataForUploadComplete(file), + headers: { 'Content-Type': 'application/json' }, + }, + response: { status: 'ok' }, + }, + ] }, } @@ -117,9 +155,9 @@ describe('Audit Setup > Participants', () => { const anotherFile = new File([], 'another file') const expectedCalls = [ apiCalls.getJurisdictionsFile(fileMocks.empty), - apiCalls.putJurisdictionsFile(jurisdictionFile), + ...apiCalls.uploadJurisdictionFileCalls(jurisdictionFile), apiCalls.getJurisdictionsFile(fileMocks.processed), - apiCalls.putJurisdictionsFile(anotherFile), + ...apiCalls.uploadJurisdictionFileCalls(anotherFile), apiCalls.getJurisdictionsFile(fileMocks.processed), ] await withMockFetch(expectedCalls, async () => { @@ -156,10 +194,10 @@ describe('Audit Setup > Participants', () => { const expectedCalls = [ apiCalls.getJurisdictionsFile(fileMocks.empty), apiCalls.getStandardizedContestsFile(fileMocks.empty), - apiCalls.putJurisdictionsFile(jurisdictionFile), + ...apiCalls.uploadJurisdictionFileCalls(jurisdictionFile), apiCalls.getJurisdictionsFile(fileMocks.processed), apiCalls.getStandardizedContestsFile(fileMocks.empty), - apiCalls.putStandardizedContestsFile(contestsFile), + ...apiCalls.uploadStandardizedContestsFileCalls(contestsFile), apiCalls.getStandardizedContestsFile(fileMocks.processed), ] await withMockFetch(expectedCalls, async () => { @@ -212,10 +250,10 @@ describe('Audit Setup > Participants', () => { const expectedCalls = [ apiCalls.getJurisdictionsFile(fileMocks.empty), apiCalls.getStandardizedContestsFile(fileMocks.empty), - apiCalls.putJurisdictionsFile(jurisdictionFile), + ...apiCalls.uploadJurisdictionFileCalls(jurisdictionFile), apiCalls.getJurisdictionsFile(fileMocks.processed), apiCalls.getStandardizedContestsFile(fileMocks.empty), - apiCalls.putStandardizedContestsFile(contestsFile), + ...apiCalls.uploadStandardizedContestsFileCalls(contestsFile), apiCalls.getStandardizedContestsFile(fileMocks.processed), ] await withMockFetch(expectedCalls, async () => { @@ -331,7 +369,7 @@ describe('Audit Setup > Participants', () => { const expectedCalls = [ apiCalls.getJurisdictionsFile(fileMocks.processed), apiCalls.getStandardizedContestsFile(fileMocks.processed), - apiCalls.putStandardizedContestsFile(contestsFile), + ...apiCalls.uploadStandardizedContestsFileCalls(contestsFile), apiCalls.getStandardizedContestsFile(fileMocks.processing), apiCalls.getStandardizedContestsFile(fileMocks.errored), ] @@ -361,7 +399,7 @@ describe('Audit Setup > Participants', () => { const expectedCalls = [ apiCalls.getJurisdictionsFile(fileMocks.processed), apiCalls.getStandardizedContestsFile(fileMocks.processed), - apiCalls.putJurisdictionsFile(jurisdictionFile), + ...apiCalls.uploadJurisdictionFileCalls(jurisdictionFile), apiCalls.getJurisdictionsFile(fileMocks.processing), apiCalls.getJurisdictionsFile(fileMocks.processed), apiCalls.getStandardizedContestsFile(fileMocks.errored), diff --git a/client/src/components/AuditAdmin/Setup/Participants/Participants.tsx b/client/src/components/AuditAdmin/Setup/Participants/Participants.tsx index baed8043e..42e4e0c2b 100644 --- a/client/src/components/AuditAdmin/Setup/Participants/Participants.tsx +++ b/client/src/components/AuditAdmin/Setup/Participants/Participants.tsx @@ -47,8 +47,8 @@ const Participants: React.FC = ({ > { - await jurisdictionsFileUpload.uploadFiles(files) + uploadCSVFile={async file => { + await jurisdictionsFileUpload.uploadFiles([file]) return true }} title={ @@ -64,8 +64,8 @@ const Participants: React.FC = ({
{ - await standardizedContestsFileUpload.uploadFiles(files) + uploadCSVFile={async file => { + await standardizedContestsFileUpload.uploadFiles([file]) return true }} title="Standardized Contests" diff --git a/client/src/components/JurisdictionAdmin/BatchInventory.test.tsx b/client/src/components/JurisdictionAdmin/BatchInventory.test.tsx index fa0ade232..be55ea68e 100644 --- a/client/src/components/JurisdictionAdmin/BatchInventory.test.tsx +++ b/client/src/components/JurisdictionAdmin/BatchInventory.test.tsx @@ -10,7 +10,11 @@ import { createQueryClient, } from '../testUtilities' import { CvrFileType, IFileInfo } from '../useCSV' -import { fileInfoMocks } from '../_mocks' +import { + fileInfoMocks, + getMockFormDataForFileUpload, + getMockJsonDataForUploadComplete, +} from '../_mocks' jest.mock('axios') jest.mock('../useFeatureFlag', (): typeof import('../useFeatureFlag') => ({ @@ -21,8 +25,6 @@ jest.mock('../useFeatureFlag', (): typeof import('../useFeatureFlag') => ({ const testCvrFile = new File([''], 'test-cvr.csv', { type: 'text/csv', }) -const cvrFormData = new FormData() -cvrFormData.append('cvr', testCvrFile, testCvrFile.name) const testTabulatorStatusFile = new File([''], 'test-tabulator-status.xml', { type: 'application/xml', @@ -76,23 +78,64 @@ const apiCalls = { }, response: { status: 'ok' }, }), - putCvr: { - url: '/api/election/1/jurisdiction/jurisdiction-id-1/batch-inventory/cvr', - options: { - method: 'PUT', - body: cvrFormData, + uploadCvrCalls: [ + { + url: + '/api/election/1/jurisdiction/jurisdiction-id-1/batch-inventory/cvr/upload-url', + options: { + method: 'GET', + params: { fileType: testCvrFile.type }, + }, + response: { url: '/api/upload', fields: { key: '/path/to/file' } }, }, - response: { status: 'ok' }, - }, - putTabulatorStatus: { - url: - '/api/election/1/jurisdiction/jurisdiction-id-1/batch-inventory/tabulator-status', - options: { - method: 'PUT', - body: tabulatorStatusFormData, + { + url: '/api/upload', + options: { + method: 'POST', + body: getMockFormDataForFileUpload(testCvrFile), + }, + response: { status: 'ok' }, }, - response: { status: 'ok' }, - }, + { + url: + '/api/election/1/jurisdiction/jurisdiction-id-1/batch-inventory/cvr/upload-complete', + options: { + method: 'POST', + body: getMockJsonDataForUploadComplete(testCvrFile), + headers: { 'Content-Type': 'application/json' }, + }, + response: { status: 'ok' }, + }, + ], + uploadTabulatorStatusCalls: [ + { + url: + '/api/election/1/jurisdiction/jurisdiction-id-1/batch-inventory/tabulator-status/upload-url', + options: { + method: 'GET', + params: { fileType: testTabulatorStatusFile.type }, + }, + response: { url: '/api/upload', fields: { key: '/path/to/file' } }, + }, + { + url: '/api/upload', + options: { + method: 'POST', + body: getMockFormDataForFileUpload(testTabulatorStatusFile), + }, + response: { status: 'ok' }, + }, + { + url: + '/api/election/1/jurisdiction/jurisdiction-id-1/batch-inventory/tabulator-status/upload-complete', + options: { + method: 'POST', + body: getMockJsonDataForUploadComplete(testTabulatorStatusFile), + headers: { 'Content-Type': 'application/json' }, + }, + response: { status: 'ok' }, + }, + ], postSignOff: { url: '/api/election/1/jurisdiction/jurisdiction-id-1/batch-inventory/sign-off', @@ -138,12 +181,12 @@ describe('BatchInventory', () => { apiCalls.getCvr(fileInfoMocks.empty), apiCalls.getTabulatorStatus(fileInfoMocks.empty), apiCalls.getSignOff(null), - apiCalls.putCvr, + ...apiCalls.uploadCvrCalls, apiCalls.getCvr(cvrProcessed), apiCalls.getTabulatorStatus(fileInfoMocks.empty), apiCalls.getSignOff(null), apiCalls.getSignOff(null), - apiCalls.putTabulatorStatus, + ...apiCalls.uploadTabulatorStatusCalls, apiCalls.getTabulatorStatus(tabulatorStatusProcessed), apiCalls.getSignOff(null), ] diff --git a/client/src/components/JurisdictionAdmin/BatchInventory.tsx b/client/src/components/JurisdictionAdmin/BatchInventory.tsx index b4312458e..1a741e08a 100644 --- a/client/src/components/JurisdictionAdmin/BatchInventory.tsx +++ b/client/src/components/JurisdictionAdmin/BatchInventory.tsx @@ -147,9 +147,7 @@ const useBatchInventoryCVR = ( ), }), uploadFiles: files => { - const formData = new FormData() - formData.append('cvr', files[0], files[0].name) - return uploadFiles.mutateAsync(formData) + return uploadFiles.mutateAsync({ file: files[0] }) }, uploadProgress: uploadFiles.progress, deleteFile: () => deleteFile.mutateAsync(), @@ -175,9 +173,7 @@ const useBatchInventoryTabulatorStatus = ( ), }), uploadFiles: files => { - const formData = new FormData() - formData.append('tabulatorStatus', files[0], files[0].name) - return uploadFiles.mutateAsync(formData) + return uploadFiles.mutateAsync({ file: files[0] }) }, uploadProgress: uploadFiles.progress, deleteFile: () => deleteFile.mutateAsync(), @@ -237,7 +233,7 @@ const SelectSystemStep: React.FC<{ {!systemType && } diff --git a/client/src/components/JurisdictionAdmin/JurisdictionAdminView.test.tsx b/client/src/components/JurisdictionAdmin/JurisdictionAdminView.test.tsx index 9a3a799be..7ac305149 100644 --- a/client/src/components/JurisdictionAdmin/JurisdictionAdminView.test.tsx +++ b/client/src/components/JurisdictionAdmin/JurisdictionAdminView.test.tsx @@ -76,7 +76,7 @@ describe('JA setup', () => { jaApiCalls.getSettings(auditSettingsMocks.all), jaApiCalls.getRounds([]), jaApiCalls.getBallotManifestFile(manifestMocks.empty), - jaApiCalls.putManifest, + ...jaApiCalls.uploadManifestCalls, jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.deleteManifest, jaApiCalls.getBallotManifestFile(manifestMocks.empty), @@ -112,7 +112,7 @@ describe('JA setup', () => { jaApiCalls.getRounds([]), jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.getBatchTalliesFile(talliesMocks.empty), - jaApiCalls.putTallies, + ...jaApiCalls.uploadTalliesCalls, jaApiCalls.getBatchTalliesFile(talliesMocks.processed), ] await withMockFetch(expectedCalls, async () => { @@ -157,7 +157,9 @@ describe('JA setup', () => { jaApiCalls.getRounds([]), jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.getBatchTalliesFile(talliesMocks.empty), - serverError('putTallies', jaApiCalls.putTallies), + jaApiCalls.uploadTalliesCalls[0], + jaApiCalls.uploadTalliesCalls[1], + serverError('postTallies', jaApiCalls.uploadTalliesCalls[2]), ] await withMockFetch(expectedCalls, async () => { renderView() @@ -165,7 +167,7 @@ describe('JA setup', () => { userEvent.upload(screen.getByLabelText('Select a file...'), talliesFile) userEvent.click(screen.getByRole('button', { name: 'Upload File' })) const toast = await screen.findByRole('alert') - expect(toast).toHaveTextContent('something went wrong: putTallies') + expect(toast).toHaveTextContent('something went wrong: postTallies') }) }) @@ -176,7 +178,7 @@ describe('JA setup', () => { jaApiCalls.getRounds([]), jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.getBatchTalliesFile(talliesMocks.processed), - jaApiCalls.putTallies, + ...jaApiCalls.uploadTalliesCalls, jaApiCalls.getBatchTalliesFile(talliesMocks.errored), ] await withMockFetch(expectedCalls, async () => { @@ -206,7 +208,7 @@ describe('JA setup', () => { jaApiCalls.getRounds([]), jaApiCalls.getBallotManifestFile(manifestMocks.empty), jaApiCalls.getBatchTalliesFile(talliesMocks.processed), - jaApiCalls.putManifest, + ...jaApiCalls.uploadManifestCalls, jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.getBatchTalliesFile(talliesMocks.errored), ] @@ -235,7 +237,7 @@ describe('JA setup', () => { jaApiCalls.getRounds([]), jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.getCVRSfile(cvrsMocks.empty), - jaApiCalls.putCVRs, + ...jaApiCalls.uploadCVRsCalls, jaApiCalls.getCVRSfile(cvrsMocks.processing), jaApiCalls.getCVRSfile(cvrsMocks.processed), ] @@ -304,7 +306,7 @@ describe('JA setup', () => { jaApiCalls.getRounds([]), jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.getCVRSfile(cvrsMocks.processed), - jaApiCalls.putCVRs, + ...jaApiCalls.uploadCVRsCalls, jaApiCalls.getCVRSfile(cvrsMocks.errored), ] await withMockFetch(expectedCalls, async () => { @@ -327,108 +329,10 @@ describe('JA setup', () => { }) }) - it('allows multiple CVR files to be uploaded for ES&S', async () => { - const cvrsFormData: FormData = new FormData() - const cvrsFile1 = new File(['test cvr data'], 'cvrs1.csv', { - type: 'text/csv', - }) - const cvrsFile2 = new File(['test cvr data'], 'cvrs2.csv', { - type: 'text/csv', - }) - // Make the combined CVR files large enough to trigger an "Uploading..." progress bar - Object.defineProperty(cvrsFile1, 'size', { value: 500 * 1000 }) - Object.defineProperty(cvrsFile2, 'size', { value: 500 * 1000 }) - cvrsFormData.append('cvrs', cvrsFile1, cvrsFile1.name) - cvrsFormData.append('cvrs', cvrsFile2, cvrsFile2.name) - cvrsFormData.append('cvrFileType', 'ESS') - - const expectedCalls = [ - jaApiCalls.getUser, - jaApiCalls.getSettings(auditSettingsMocks.ballotComparisonAll), - jaApiCalls.getRounds([]), - jaApiCalls.getBallotManifestFile(manifestMocks.processed), - jaApiCalls.getCVRSfile(cvrsMocks.empty), - { - ...jaApiCalls.putCVRs, - options: { ...jaApiCalls.putCVRs.options, body: cvrsFormData }, - }, - jaApiCalls.getCVRSfile(cvrsMocks.processed), - ] - await withMockFetch(expectedCalls, async () => { - renderView() - await screen.findByText('Audit Setup') - - userEvent.selectOptions( - screen.getByLabelText(/CVR File Type:/), - screen.getByRole('option', { name: 'ES&S' }) - ) - - const fileSelect = screen.getByLabelText('Select files...') - userEvent.upload(fileSelect, [cvrsFile1, cvrsFile2]) - await screen.findByLabelText('2 files selected') - userEvent.click(screen.getByRole('button', { name: 'Upload File' })) - - await screen.findByText('Uploading...') - await screen.findByText('Uploaded at 11/18/2020, 9:39:14 PM.') - }) - }) - it('allows CVRs ZIP file to be uploaded for Hart', async () => { - const cvrsFormData: FormData = new FormData() - const cvrsZip = new File(['test cvr data'], 'cvrs.zip', { - type: 'application/zip', - }) - cvrsFormData.append('cvrs', cvrsZip, cvrsZip.name) - cvrsFormData.append('cvrFileType', 'HART') - - const expectedCalls = [ - jaApiCalls.getUser, - jaApiCalls.getSettings(auditSettingsMocks.ballotComparisonAll), - jaApiCalls.getRounds([]), - jaApiCalls.getBallotManifestFile(manifestMocks.processed), - jaApiCalls.getCVRSfile(cvrsMocks.empty), - { - ...jaApiCalls.putCVRs, - options: { ...jaApiCalls.putCVRs.options, body: cvrsFormData }, - }, - jaApiCalls.getCVRSfile(cvrsMocks.processed), - ] - await withMockFetch(expectedCalls, async () => { - renderView() - await screen.findByText('Audit Setup') - - userEvent.selectOptions( - screen.getByLabelText(/CVR File Type:/), - screen.getByRole('option', { name: 'Hart' }) - ) - - const fileSelect = screen.getByLabelText('Select files...') - userEvent.upload(fileSelect, [cvrsZip]) - await screen.findByLabelText('cvrs.zip') - userEvent.click(screen.getByRole('button', { name: 'Upload File' })) - await screen.findByText('Uploaded at 11/18/2020, 9:39:14 PM.') - }) - }) - - it('allows CVRs ZIP file and scanned ballot information CSV to be uploaded for Hart', async () => { - const cvrsFormData: FormData = new FormData() const cvrsZip = new File(['test cvr data'], 'cvrs.zip', { type: 'application/zip', }) - const scannedBallotInformationCsv = new File( - ['test cvr data'], - 'scanned-ballot-information.csv', - { - type: 'text/csv', - } - ) - cvrsFormData.append('cvrs', cvrsZip, cvrsZip.name) - cvrsFormData.append( - 'cvrs', - scannedBallotInformationCsv, - scannedBallotInformationCsv.name - ) - cvrsFormData.append('cvrFileType', 'HART') const expectedCalls = [ jaApiCalls.getUser, @@ -436,10 +340,7 @@ describe('JA setup', () => { jaApiCalls.getRounds([]), jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.getCVRSfile(cvrsMocks.empty), - { - ...jaApiCalls.putCVRs, - options: { ...jaApiCalls.putCVRs.options, body: cvrsFormData }, - }, + ...jaApiCalls.uploadCVRZipCalls, jaApiCalls.getCVRSfile(cvrsMocks.processed), ] await withMockFetch(expectedCalls, async () => { @@ -451,9 +352,8 @@ describe('JA setup', () => { screen.getByRole('option', { name: 'Hart' }) ) - const fileSelect = screen.getByLabelText('Select files...') - userEvent.upload(fileSelect, [cvrsZip, scannedBallotInformationCsv]) - await screen.findByLabelText('2 files selected') + const fileSelect = screen.getByLabelText('Select a file...') + userEvent.upload(fileSelect, cvrsZip) userEvent.click(screen.getByRole('button', { name: 'Upload File' })) await screen.findByText('Uploaded at 11/18/2020, 9:39:14 PM.') }) @@ -466,7 +366,7 @@ describe('JA setup', () => { jaApiCalls.getRounds([]), jaApiCalls.getBallotManifestFile(manifestMocks.empty), jaApiCalls.getCVRSfile(cvrsMocks.processed), - jaApiCalls.putManifest, + ...jaApiCalls.uploadManifestCalls, jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.getCVRSfile(cvrsMocks.errored), ] @@ -495,7 +395,7 @@ describe('JA setup', () => { jaApiCalls.getRounds([]), jaApiCalls.getBallotManifestFile(manifestMocks.empty), jaApiCalls.getBatchTalliesFile(talliesMocks.empty), - jaApiCalls.putManifest, + ...jaApiCalls.uploadManifestCalls, jaApiCalls.getBallotManifestFile(manifestMocks.errored), jaApiCalls.getBatchTalliesFile(talliesMocks.empty), ] @@ -528,7 +428,7 @@ describe('JA setup', () => { jaApiCalls.getRounds([]), jaApiCalls.getBallotManifestFile(manifestMocks.processed), jaApiCalls.getBatchTalliesFile(talliesMocks.empty), - jaApiCalls.putManifest, + ...jaApiCalls.uploadManifestCalls, jaApiCalls.getBallotManifestFile(manifestMocks.processed), ] await withMockFetch(expectedCalls, async () => { diff --git a/client/src/components/JurisdictionAdmin/JurisdictionAdminView.tsx b/client/src/components/JurisdictionAdmin/JurisdictionAdminView.tsx index 20201b572..e01948f48 100644 --- a/client/src/components/JurisdictionAdmin/JurisdictionAdminView.tsx +++ b/client/src/components/JurisdictionAdmin/JurisdictionAdminView.tsx @@ -131,7 +131,7 @@ const JurisdictionAdminView: React.FC = () => { { ballotManifest.processing.status === FileProcessingStatus.PROCESSED } - uploadCSVFiles={uploadBatchTallies} + uploadCSVFile={uploadBatchTallies} deleteCSVFile={deleteBatchTallies} title="Candidate Totals by Batch" description='Click "Browse" to choose the appropriate Candidate @@ -198,7 +198,7 @@ const JurisdictionAdminView: React.FC = () => { ballotManifest.processing.status === FileProcessingStatus.PROCESSED } - uploadCSVFiles={uploadCVRS} + uploadCSVFile={uploadCVRS} deleteCSVFile={deleteCVRS} title={ isHybrid diff --git a/client/src/components/_mocks.ts b/client/src/components/_mocks.ts index e1f9e6b72..582dc5e00 100644 --- a/client/src/components/_mocks.ts +++ b/client/src/components/_mocks.ts @@ -1464,34 +1464,31 @@ export const auditBoardMocks = mocksOfType()({ ], }) -const jurisdictionFormData: FormData = new FormData() -jurisdictionFormData.append( - 'jurisdictions', - jurisdictionFile, - jurisdictionFile.name -) -const jurisdictionErrorFormData: FormData = new FormData() -jurisdictionErrorFormData.append( - 'jurisdictions', - jurisdictionFile, - jurisdictionFile.name -) -const standardizedContestsFormData: FormData = new FormData() -standardizedContestsFormData.append( - 'standardized-contests', - standardizedContestsFile, - standardizedContestsFile.name -) +export const getMockFormDataForFileUpload = (file: File): FormData => { + const formData = new FormData() + formData.append('key', '/path/to/file') + formData.append('Content-Type', file.type) + formData.append('file', file, file.name) + return formData +} + +export const getMockJsonDataForUploadComplete = ( + file: File, + cvrFileType?: CvrFileType +): BodyInit => { + return ({ + fileName: file.name, + fileType: file.type, + ...(cvrFileType && { cvrFileType }), + storagePathKey: '/path/to/file', + } as unknown) as BodyInit +} -const manifestFormData: FormData = new FormData() -manifestFormData.append('manifest', manifestFile, manifestFile.name) -const talliesFormData: FormData = new FormData() -talliesFormData.append('batchTallies', talliesFile, talliesFile.name) -const cvrsFormData: FormData = new FormData() // Make the mock CVR file large enough to trigger an "Uploading..." progress bar Object.defineProperty(cvrsFile, 'size', { value: 1000 * 1000 }) -cvrsFormData.append('cvrs', cvrsFile, cvrsFile.name) -cvrsFormData.append('cvrFileType', 'CLEARBALLOT') +const cvrsZip = new File(['test cvr data'], 'cvrs.zip', { + type: 'application/zip', +}) export const apiCalls = { serverError: ( @@ -1637,30 +1634,126 @@ export const jaApiCalls = { url: '/api/election/1/jurisdiction/jurisdiction-id-1/settings', response, }), - putManifest: { - url: '/api/election/1/jurisdiction/jurisdiction-id-1/ballot-manifest', - options: { - method: 'PUT', - body: manifestFormData, + uploadManifestCalls: [ + { + url: + '/api/election/1/jurisdiction/jurisdiction-id-1/ballot-manifest/upload-url', + options: { + method: 'GET', + params: { fileType: manifestFile.type }, + }, + response: { url: '/api/upload', fields: { key: '/path/to/file' } }, }, - response: { status: 'ok' }, - }, - putTallies: { - url: '/api/election/1/jurisdiction/jurisdiction-id-1/batch-tallies', - options: { - method: 'PUT', - body: talliesFormData, + { + url: '/api/upload', + options: { + method: 'POST', + body: getMockFormDataForFileUpload(manifestFile), + }, + response: { status: 'ok' }, }, - response: { status: 'ok' }, - }, - putCVRs: { - url: '/api/election/1/jurisdiction/jurisdiction-id-1/cvrs', - options: { - method: 'PUT', - body: cvrsFormData, + { + url: + '/api/election/1/jurisdiction/jurisdiction-id-1/ballot-manifest/upload-complete', + options: { + method: 'POST', + body: getMockJsonDataForUploadComplete(manifestFile), + headers: { 'Content-Type': 'application/json' }, + }, + response: { status: 'ok' }, }, - response: { status: 'ok' }, - }, + ], + uploadTalliesCalls: [ + { + url: + '/api/election/1/jurisdiction/jurisdiction-id-1/batch-tallies/upload-url', + options: { + method: 'GET', + params: { fileType: talliesFile.type }, + }, + response: { url: '/api/upload', fields: { key: '/path/to/file' } }, + }, + { + url: '/api/upload', + options: { + method: 'POST', + body: getMockFormDataForFileUpload(talliesFile), + }, + response: { status: 'ok' }, + }, + { + url: + '/api/election/1/jurisdiction/jurisdiction-id-1/batch-tallies/upload-complete', + options: { + method: 'POST', + body: getMockJsonDataForUploadComplete(talliesFile), + headers: { 'Content-Type': 'application/json' }, + }, + response: { status: 'ok' }, + }, + ], + uploadCVRsCalls: [ + { + url: '/api/election/1/jurisdiction/jurisdiction-id-1/cvrs/upload-url', + options: { + method: 'GET', + params: { + fileType: cvrsFile.type, + cvrFileType: CvrFileType.CLEARBALLOT, + }, + }, + response: { url: '/api/upload', fields: { key: '/path/to/file' } }, + }, + { + url: '/api/upload', + options: { + method: 'POST', + body: getMockFormDataForFileUpload(cvrsFile), + }, + response: { status: 'ok' }, + }, + { + url: + '/api/election/1/jurisdiction/jurisdiction-id-1/cvrs/upload-complete', + options: { + method: 'POST', + body: getMockJsonDataForUploadComplete( + cvrsFile, + CvrFileType.CLEARBALLOT + ), + headers: { 'Content-Type': 'application/json' }, + }, + response: { status: 'ok' }, + }, + ], + uploadCVRZipCalls: [ + { + url: '/api/election/1/jurisdiction/jurisdiction-id-1/cvrs/upload-url', + options: { + method: 'GET', + params: { fileType: cvrsZip.type, cvrFileType: CvrFileType.HART }, + }, + response: { url: '/api/upload', fields: { key: '/path/to/file' } }, + }, + { + url: '/api/upload', + options: { + method: 'POST', + body: getMockFormDataForFileUpload(cvrsZip), + }, + response: { status: 'ok' }, + }, + { + url: + '/api/election/1/jurisdiction/jurisdiction-id-1/cvrs/upload-complete', + options: { + method: 'POST', + body: getMockJsonDataForUploadComplete(cvrsZip, CvrFileType.HART), + headers: { 'Content-Type': 'application/json' }, + }, + response: { status: 'ok' }, + }, + ], deleteCVRs: { url: '/api/election/1/jurisdiction/jurisdiction-id-1/cvrs', options: { method: 'DELETE' }, @@ -1972,19 +2065,37 @@ export const aaApiCalls = { url: '/api/election/1/sample-sizes/1', response, }), - putJurisdictionFile: { - url: '/api/election/1/jurisdiction/file', + uploadJurisdictionFileGetUrl: { + url: '/api/election/1/jurisdiction/file/upload-url', options: { - method: 'PUT', - body: jurisdictionFormData, + method: 'GET', + params: { fileType: jurisdictionFile.type }, + }, + response: { url: '/api/upload', fields: { key: '/path/to/file' } }, + }, + uploadJurisdictionFilePostFile: { + url: '/api/upload', + options: { + method: 'POST', + body: getMockFormDataForFileUpload(jurisdictionFile), }, response: { status: 'ok' }, }, - putJurisdictionErrorFile: { - url: '/api/election/1/jurisdiction/file', + uploadJurisdictionFileUploadComplete: { + url: '/api/election/1/jurisdiction/file/upload-complete', options: { - method: 'PUT', - body: jurisdictionErrorFormData, + method: 'POST', + body: getMockJsonDataForUploadComplete(jurisdictionFile), + headers: { 'Content-Type': 'application/json' }, + }, + response: { status: 'ok' }, + }, + uploadJurisdictionFileUploadCompleteError: { + url: '/api/election/1/jurisdiction/file/upload-complete', + options: { + method: 'POST', + body: getMockJsonDataForUploadComplete(jurisdictionFile), + headers: { 'Content-Type': 'application/json' }, }, response: { status: 'ok' }, }, @@ -1992,11 +2103,28 @@ export const aaApiCalls = { url: '/api/election/1/jurisdiction/file', response, }), - putStandardizedContestsFile: { - url: '/api/election/1/standardized-contests/file', + uploadStandardizedContestsFileGetUrl: { + url: '/api/election/1/standardized-contests/file/upload-url', options: { - method: 'PUT', - body: standardizedContestsFormData, + method: 'GET', + params: { fileType: standardizedContestsFile.type }, + }, + response: { url: '/api/upload', fields: { key: '/path/to/file' } }, + }, + uploadStandardizedContestsFilePostFile: { + url: '/api/upload', + options: { + method: 'POST', + body: getMockFormDataForFileUpload(standardizedContestsFile), + }, + response: { status: 'ok' }, + }, + uploadStandardizedContestsFileUploadComplete: { + url: '/api/election/1/standardized-contests/file/upload-complete', + options: { + method: 'POST', + body: getMockJsonDataForUploadComplete(standardizedContestsFile), + headers: { 'Content-Type': 'application/json' }, }, response: { status: 'ok' }, }, diff --git a/client/src/components/testUtilities.tsx b/client/src/components/testUtilities.tsx index b85e59114..387b0ed51 100644 --- a/client/src/components/testUtilities.tsx +++ b/client/src/components/testUtilities.tsx @@ -183,6 +183,10 @@ export const withMockFetch = async ( } throw error } + return { + ...response, + data: await response.json(), + } } ) } diff --git a/client/src/components/useCSV.ts b/client/src/components/useCSV.ts index 9299ab3c2..a5f8e7bec 100644 --- a/client/src/components/useCSV.ts +++ b/client/src/components/useCSV.ts @@ -37,7 +37,7 @@ export interface IFileInfo { } interface IUpload { - files: File[] + file: File progress: number } @@ -47,29 +47,70 @@ export const isFileProcessed = (file: IFileInfo): boolean => const loadCSVFile = async (url: string): Promise => api(url) -const putCSVFiles = async ( +const putCSVFile = async ( url: string, - files: File[], + file: File, formKey: string, trackProgress: (progress: number) => void, cvrFileType?: CvrFileType ): Promise => { - const formData: FormData = new FormData() - for (const f of files) formData.append(formKey, f, f.name) - if (cvrFileType) formData.append('cvrFileType', cvrFileType) try { + // Get the signed s3 URL for the file upload + const params = cvrFileType + ? { + fileType: file.type, + cvrFileType, + } + : { + fileType: file.type, + } + const getUploadResponse = await axios( + `/api${url}/upload-url`, + addCSRFToken({ + method: 'GET', + params, + }) as AxiosRequestConfig + ) + + // Upload the file to s3 + const uploadFileFormData = new FormData() + Object.entries(getUploadResponse.data.fields).forEach(([key, value]) => { + uploadFileFormData.append(key, value as string) + }) + uploadFileFormData.append('Content-Type', file.type) + uploadFileFormData.append('file', file, file.name) + await axios( - `/api${url}`, + getUploadResponse.data.url, addCSRFToken({ - method: 'PUT', - data: formData, + method: 'POST', + data: uploadFileFormData, onUploadProgress: progress => trackProgress(progress.loaded / progress.total), }) as AxiosRequestConfig ) + + // Tell the server that the upload has finished to save the file path reference and kick off processing + const jsonData = { + fileName: file.name, + fileType: file.type, + ...(cvrFileType && { cvrFileType }), + storagePathKey: getUploadResponse.data.fields.key, + } + + await axios( + `/api${url}/upload-complete`, + addCSRFToken({ + method: 'POST', + data: jsonData, + headers: { + 'Content-Type': 'application/json', + }, + }) as AxiosRequestConfig + ) return true } catch (error) { - const { errors } = error.response.data + const { errors } = error.response ? error.response.data : error const message = errors && errors.length ? errors[0].message : error.response.statusText toast.error(message) @@ -89,7 +130,7 @@ const useCSV = ( dependencyFile?: IFileInfo | null ): [ IFileInfo | null, - (csv: File[]) => Promise, + (csv: File) => Promise, () => Promise ] => { const [csv, setCSV] = useState(null) @@ -119,18 +160,18 @@ const useCSV = ( dependencyNotProcessing, ]) - const uploadCSVs = async ( - files: File[], + const uploadCSV = async ( + file: File, cvrFileType?: CvrFileType ): Promise => { if (!shouldFetch) return false - setUpload({ files, progress: 0 }) + setUpload({ file, progress: 0 }) if ( - await putCSVFiles( + await putCSVFile( url, - files, + file, formKey, - progress => setUpload({ files, progress }), + progress => setUpload({ file, progress }), cvrFileType ) ) { @@ -160,12 +201,12 @@ const useCSV = ( shouldPoll ? 1000 : null ) - return [csv && { ...csv, upload }, uploadCSVs, deleteCSV] + return [csv && { ...csv, upload }, uploadCSV, deleteCSV] } export const useJurisdictionsFile = ( electionId: string -): [IFileInfo | null, (csv: File[]) => Promise] => { +): [IFileInfo | null, (csv: File) => Promise] => { const [csv, uploadCSV] = useCSV( `/election/${electionId}/jurisdiction/file`, 'jurisdictions' @@ -178,7 +219,7 @@ export const useStandardizedContestsFile = ( electionId: string, auditType: IAuditSettings['auditType'], jurisdictionsFile?: IFileInfo | null -): [IFileInfo | null, (csv: File[]) => Promise] => { +): [IFileInfo | null, (csv: File) => Promise] => { const [csv, uploadCSV] = useCSV( `/election/${electionId}/standardized-contests/file`, 'standardized-contests', @@ -194,7 +235,7 @@ export const useBallotManifest = ( jurisdictionId: string ): [ IFileInfo | null, - (csv: File[]) => Promise, + (csv: File) => Promise, () => Promise ] => useCSV( @@ -209,7 +250,7 @@ export const useBatchTallies = ( ballotManifest: IFileInfo | null ): [ IFileInfo | null, - (csv: File[]) => Promise, + (csv: File) => Promise, () => Promise ] => useCSV( @@ -226,7 +267,7 @@ export const useCVRs = ( ballotManifest: IFileInfo | null ): [ IFileInfo | null, - (csv: File[]) => Promise, + (csv: File) => Promise, () => Promise ] => useCSV( diff --git a/client/src/components/useFileUpload.ts b/client/src/components/useFileUpload.ts index d8cb5b0b4..7aa6e42c7 100644 --- a/client/src/components/useFileUpload.ts +++ b/client/src/components/useFileUpload.ts @@ -57,21 +57,71 @@ export const useUploadedFile = ( }) } +type CompleteFileUploadArgs = { + file: File + cvrFileType?: CvrFileType +} + // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types export const useUploadFiles = (key: string[], url: string) => { const [progress, setProgress] = useState() - const putFiles = async (formData: FormData) => { + const completeFileUpload = async ({ + file, + cvrFileType, + }: CompleteFileUploadArgs): Promise => { try { + // Get the signed s3 URL for the file upload + const params = cvrFileType + ? { + fileType: file.type, + cvrFileType, + } + : { + fileType: file.type, + } + const getUploadResponse = await axios( + `${url}/upload-url`, + addCSRFToken({ + method: 'GET', + params, + }) as AxiosRequestConfig + ) + + // Upload the file to s3 + const uploadFileFormData = new FormData() + Object.entries(getUploadResponse.data.fields).forEach(([k, v]) => { + uploadFileFormData.append(k, v as string) + }) + uploadFileFormData.append('Content-Type', file.type) + uploadFileFormData.append('file', file, file.name) + await axios( - url, + getUploadResponse.data.url, addCSRFToken({ - method: 'PUT', - data: formData, - onUploadProgress: progressEvent => - setProgress(progressEvent.loaded / progressEvent.total), + method: 'POST', + data: uploadFileFormData, + onUploadProgress: p => setProgress(p.loaded / p.total), }) as AxiosRequestConfig ) + + const jsonData = { + fileName: file.name, + fileType: file.type, + ...(cvrFileType && { cvrFileType }), + storagePathKey: getUploadResponse.data.fields.key, + } + await axios( + `${url}/upload-complete`, + addCSRFToken({ + method: 'POST', + data: jsonData, + headers: { + 'Content-Type': 'application/json', + }, + }) as AxiosRequestConfig + ) + return } catch (error) { const { errors } = error.response.data const message = @@ -85,7 +135,7 @@ export const useUploadFiles = (key: string[], url: string) => { const queryClient = useQueryClient() return { - ...useMutation(putFiles, { + ...useMutation(completeFileUpload, { onSuccess: () => queryClient.invalidateQueries(key), }), progress, @@ -121,9 +171,7 @@ export const useJurisdictionsFile = (electionId: string): IFileUpload => { }, }), uploadFiles: async (files: File[]) => { - const formData = new FormData() - formData.append('jurisdictions', files[0]) - await uploadFiles.mutateAsync(formData) + await uploadFiles.mutateAsync({ file: files[0] }) }, uploadProgress: uploadFiles.progress, deleteFile: () => deleteFile.mutateAsync(), @@ -151,9 +199,7 @@ export const useStandardizedContestsFile = ( }, }), uploadFiles: async (files: File[]) => { - const formData = new FormData() - formData.append('standardized-contests', files[0]) - await uploadFiles.mutateAsync(formData) + await uploadFiles.mutateAsync({ file: files[0] }) }, uploadProgress: uploadFiles.progress, deleteFile: () => deleteFile.mutateAsync(), @@ -189,9 +235,7 @@ export const useBallotManifest = ( }, }), uploadFiles: files => { - const formData = new FormData() - formData.append('manifest', files[0], files[0].name) - return uploadFiles.mutateAsync(formData) + return uploadFiles.mutateAsync({ file: files[0] }) }, uploadProgress: uploadFiles.progress, deleteFile: () => deleteFile.mutateAsync(), @@ -222,9 +266,7 @@ export const useBatchTallies = ( }, }), uploadFiles: files => { - const formData = new FormData() - formData.append('batchTallies', files[0], files[0].name) - return uploadFiles.mutateAsync(formData) + return uploadFiles.mutateAsync({ file: files[0] }) }, uploadProgress: uploadFiles.progress, deleteFile: () => deleteFile.mutateAsync(), @@ -259,12 +301,7 @@ export const useCVRs = ( }, }), uploadFiles: (files, cvrFileType) => { - const formData = new FormData() - for (const file of files) { - formData.append('cvrs', file, file.name) - } - formData.append('cvrFileType', cvrFileType) - return uploadFiles.mutateAsync(formData) + return uploadFiles.mutateAsync({ file: files[0], cvrFileType }) }, uploadProgress: uploadFiles.progress, deleteFile: () => deleteFile.mutateAsync(), diff --git a/server/api/ballot_manifest.py b/server/api/ballot_manifest.py index 0e0179f83..c0a4dc580 100644 --- a/server/api/ballot_manifest.py +++ b/server/api/ballot_manifest.py @@ -2,7 +2,7 @@ import uuid import logging from datetime import datetime -from flask import request, jsonify, Request, session +from flask import request, jsonify, session from werkzeug.exceptions import BadRequest, NotFound from sqlalchemy import func from sqlalchemy.orm import Session @@ -16,10 +16,11 @@ create_background_task, ) from ..util.file import ( + get_file_upload_url, + get_standard_file_upload_request_params, retrieve_file, serialize_file, serialize_file_processing, - store_file, timestamp_filename, ) from ..util.csv_download import csv_response @@ -228,23 +229,12 @@ def process() -> None: session.commit() -# Raises if invalid -def validate_ballot_manifest_upload(request: Request): - if "manifest" not in request.files: - raise BadRequest("Missing required file parameter 'manifest'") - - validate_csv_mimetype(request.files["manifest"]) - - -def save_ballot_manifest_file(manifest, jurisdiction: Jurisdiction): - storage_path = store_file( - manifest.stream, - f"audits/{jurisdiction.election_id}/jurisdictions/{jurisdiction.id}/" - + timestamp_filename("manifest", "csv"), - ) +def save_ballot_manifest_file( + storage_path: str, file_name: str, jurisdiction: Jurisdiction +): jurisdiction.manifest_file = File( id=str(uuid.uuid4()), - name=manifest.filename, + name=file_name, storage_path=storage_path, uploaded_at=datetime.now(timezone.utc), ) @@ -268,17 +258,42 @@ def clear_ballot_manifest_file(jurisdiction: Jurisdiction): @api.route( - "/election//jurisdiction//ballot-manifest", - methods=["PUT"], + "/election//jurisdiction//ballot-manifest/upload-url", + methods=["GET"], +) +@restrict_access([UserType.AUDIT_ADMIN, UserType.JURISDICTION_ADMIN]) +def start_upload_for_ballot_manifest( + election: Election, # pylint: disable=unused-argument + jurisdiction: Jurisdiction, +): + file_type = request.args.get("fileType") + if file_type is None: + raise BadRequest("Missing expected query parameter: fileType") + + storage_path_prefix = ( + f"audits/{jurisdiction.election_id}/jurisdictions/{jurisdiction.id}" + ) + filename = timestamp_filename("manifest", "csv") + + return jsonify(get_file_upload_url(storage_path_prefix, filename, file_type)) + + +@api.route( + "/election//jurisdiction//ballot-manifest/upload-complete", + methods=["POST"], ) @restrict_access([UserType.AUDIT_ADMIN, UserType.JURISDICTION_ADMIN]) -def upload_ballot_manifest( +def complete_upload_for_ballot_manifest( election: Election, # pylint: disable=unused-argument jurisdiction: Jurisdiction, ): - validate_ballot_manifest_upload(request) + (storage_path, filename, file_type) = get_standard_file_upload_request_params( + request + ) + validate_csv_mimetype(file_type) + clear_ballot_manifest_file(jurisdiction) - save_ballot_manifest_file(request.files["manifest"], jurisdiction) + save_ballot_manifest_file(storage_path, filename, jurisdiction) db_session.commit() return jsonify(status="ok") diff --git a/server/api/batch_inventory.py b/server/api/batch_inventory.py index 2d6797a39..32f75434d 100644 --- a/server/api/batch_inventory.py +++ b/server/api/batch_inventory.py @@ -33,18 +33,20 @@ ) from ..models import * # pylint: disable=wildcard-import from ..util.csv_parse import ( - does_file_have_csv_mimetype, - does_file_have_zip_mimetype, - INVALID_CSV_ERROR, validate_comma_delimited, + is_filetype_csv_mimetype, + validate_csv_mimetype, ) from ..util.file import ( + get_file_upload_url, + get_standard_file_upload_request_params, retrieve_file, serialize_file, serialize_file_processing, - store_file, timestamp_filename, unzip_files, + validate_csv_or_zip_mimetype, + validate_xml_mimetype, ) from ..worker.tasks import UserError, background_task, create_background_task from ..util.csv_download import csv_response, jurisdiction_timestamp_name @@ -624,11 +626,33 @@ def get_batch_inventory_system_type( @api.route( - "/election//jurisdiction//batch-inventory/cvr", - methods=["PUT"], + "/election//jurisdiction//batch-inventory/cvr/upload-url", + methods=["GET"], ) @restrict_access([UserType.JURISDICTION_ADMIN]) -def upload_batch_inventory_cvr(election: Election, jurisdiction: Jurisdiction): +def start_upload_for_batch_inventory_cvr( + election: Election, jurisdiction: Jurisdiction # pylint: disable=unused-argument +): + file_type = request.args.get("fileType") + if file_type is None: + raise BadRequest("Missing expected query parameter: fileType") + + is_csv = is_filetype_csv_mimetype(file_type) + + storage_path_prefix = f"audits/{election.id}/jurisdictions/{jurisdiction.id}" + filename = timestamp_filename("batch-inventory-cvrs", "csv" if is_csv else "zip") + + return jsonify(get_file_upload_url(storage_path_prefix, filename, file_type)) + + +@api.route( + "/election//jurisdiction//batch-inventory/cvr/upload-complete", + methods=["POST"], +) +@restrict_access([UserType.JURISDICTION_ADMIN]) +def complete_upload_for_batch_inventory_cvr( + election: Election, jurisdiction: Jurisdiction # pylint: disable=unused-argument +): if len(list(jurisdiction.contests)) == 0: raise Conflict("Jurisdiction does not have any contests assigned.") @@ -636,34 +660,21 @@ def upload_batch_inventory_cvr(election: Election, jurisdiction: Jurisdiction): if not batch_inventory_data or not batch_inventory_data.system_type: raise Conflict("Must select system type before uploading CVR file.") - file = request.files["cvr"] - file_type = ( - "csv" - if does_file_have_csv_mimetype(file) - else "zip" if does_file_have_zip_mimetype(file) else "other" - ) - - if batch_inventory_data.system_type == CvrFileType.DOMINION and file_type != "csv": - raise BadRequest(INVALID_CSV_ERROR) - elif ( - batch_inventory_data.system_type == CvrFileType.ESS - and file_type != "csv" - and file_type != "zip" - ): - raise BadRequest("Please submit a valid CSV or ZIP file.") - - assert file_type != "other" - - file_name: str = file.filename # type: ignore - storage_path = store_file( - file.stream, - f"audits/{election.id}/jurisdictions/{jurisdiction.id}/" - + timestamp_filename("batch-inventory-cvrs", file_type), + (storage_path, filename, file_type) = get_standard_file_upload_request_params( + request ) + if batch_inventory_data.system_type == CvrFileType.DOMINION: + validate_csv_mimetype(file_type) + elif batch_inventory_data.system_type == CvrFileType.ESS: + validate_csv_or_zip_mimetype(file_type) + else: + raise Conflict( + f"Batch Inventory CVR uploads not supported for cvr file type: {batch_inventory_data.system_type}" + ) batch_inventory_data.cvr_file = File( id=str(uuid.uuid4()), - name=file_name, + name=filename, storage_path=storage_path, uploaded_at=datetime.now(timezone.utc), ) @@ -738,27 +749,45 @@ def download_batch_inventory_cvr( @api.route( - "/election//jurisdiction//batch-inventory/tabulator-status", - methods=["PUT"], + "/election//jurisdiction//batch-inventory/tabulator-status/upload-url", + methods=["GET"], ) @restrict_access([UserType.JURISDICTION_ADMIN]) -def upload_batch_inventory_tabulator_status( +def start_upload_for_batch_inventory_tabulator_status( election: Election, jurisdiction: Jurisdiction ): + + file_type = request.args.get("fileType") + if file_type is None: + raise BadRequest("Missing expected query parameter: fileType") + + storage_path_prefix = f"audits/{election.id}/jurisdictions/{jurisdiction.id}" + filename = timestamp_filename("batch-inventory-tabulator-status", "xml") + + return jsonify(get_file_upload_url(storage_path_prefix, filename, file_type)) + + +@api.route( + "/election//jurisdiction//batch-inventory/tabulator-status/upload-complete", + methods=["POST"], +) +@restrict_access([UserType.JURISDICTION_ADMIN]) +def complete_upload_for_batch_inventory_tabulator_status( + election: Election, jurisdiction: Jurisdiction # pylint: disable=unused-argument +): + batch_inventory_data = BatchInventoryData.query.get(jurisdiction.id) if not batch_inventory_data or not batch_inventory_data.cvr_file_id: raise Conflict("Must upload CVR file before uploading tabulator status file.") - file_name: str = request.files["tabulatorStatus"].filename # type: ignore - storage_path = store_file( - request.files["tabulatorStatus"].stream, - f"audits/{election.id}/jurisdictions/{jurisdiction.id}/" - + timestamp_filename("batch-inventory-tabulator-status", "xml"), + (storage_path, filename, file_type) = get_standard_file_upload_request_params( + request ) + validate_xml_mimetype(file_type) batch_inventory_data.tabulator_status_file = File( id=str(uuid.uuid4()), - name=file_name, + name=filename, storage_path=storage_path, uploaded_at=datetime.now(timezone.utc), ) diff --git a/server/api/batch_tallies.py b/server/api/batch_tallies.py index f4876e1ad..be4d3fb6c 100644 --- a/server/api/batch_tallies.py +++ b/server/api/batch_tallies.py @@ -4,7 +4,7 @@ import csv import io import uuid -from flask import request, jsonify, Request, session +from flask import request, jsonify, session from werkzeug.exceptions import BadRequest, NotFound, Conflict from sqlalchemy.orm import Session @@ -18,10 +18,11 @@ create_background_task, ) from ..util.file import ( + get_file_upload_url, + get_standard_file_upload_request_params, retrieve_file, serialize_file, serialize_file_processing, - store_file, timestamp_filename, ) from ..util.csv_download import ( @@ -185,9 +186,7 @@ def process() -> None: # Raises if invalid -def validate_batch_tallies_upload( - request: Request, election: Election, jurisdiction: Jurisdiction -): +def validate_batch_tallies_upload(election: Election, jurisdiction: Jurisdiction): if election.audit_type != AuditType.BATCH_COMPARISON: raise Conflict( "Can only upload batch tallies file for batch comparison audits." @@ -199,11 +198,6 @@ def validate_batch_tallies_upload( if not jurisdiction.manifest_file_id: raise Conflict("Must upload ballot manifest before uploading batch tallies.") - if "batchTallies" not in request.files: - raise BadRequest("Missing required file parameter 'batchTallies'") - - validate_csv_mimetype(request.files["batchTallies"]) - def clear_batch_tallies_data(jurisdiction: Jurisdiction): jurisdiction.batch_tallies = None @@ -227,27 +221,47 @@ def reprocess_batch_tallies_file_if_uploaded( @api.route( - "/election//jurisdiction//batch-tallies", - methods=["PUT"], + "/election//jurisdiction//batch-tallies/upload-url", + methods=["GET"], ) @restrict_access([UserType.AUDIT_ADMIN, UserType.JURISDICTION_ADMIN]) -def upload_batch_tallies( +def start_upload_for_batch_tallies( + election: Election, # pylint: disable=unused-argument + jurisdiction: Jurisdiction, +): + file_type = request.args.get("fileType") + if file_type is None: + raise BadRequest("Missing expected query parameter: fileType") + + storage_path_prefix = ( + f"audits/{jurisdiction.election_id}/jurisdictions/{jurisdiction.id}" + ) + filename = timestamp_filename("batch_tallies", "csv") + + return jsonify(get_file_upload_url(storage_path_prefix, filename, file_type)) + + +@api.route( + "/election//jurisdiction//batch-tallies/upload-complete", + methods=["POST"], +) +@restrict_access([UserType.AUDIT_ADMIN, UserType.JURISDICTION_ADMIN]) +def complete_upload_for_batch_tallies( election: Election, - jurisdiction: Jurisdiction, # pylint: disable=unused-argument + jurisdiction: Jurisdiction, ): - validate_batch_tallies_upload(request, election, jurisdiction) + validate_batch_tallies_upload(election, jurisdiction) + + (storage_path, filename, file_type) = get_standard_file_upload_request_params( + request + ) + validate_csv_mimetype(file_type) clear_batch_tallies_data(jurisdiction) - batch_tallies = request.files["batchTallies"] - storage_path = store_file( - batch_tallies.stream, - f"audits/{jurisdiction.election_id}/jurisdictions/{jurisdiction.id}/" - + timestamp_filename("batch_tallies", "csv"), - ) jurisdiction.batch_tallies_file = File( id=str(uuid.uuid4()), - name=batch_tallies.filename, # type: ignore + name=filename, storage_path=storage_path, uploaded_at=datetime.now(timezone.utc), ) @@ -259,7 +273,6 @@ def upload_batch_tallies( support_user_email=get_support_user(session), ), ) - db_session.commit() return jsonify(status="ok") diff --git a/server/api/cvrs.py b/server/api/cvrs.py index 600f252c0..905c865e0 100644 --- a/server/api/cvrs.py +++ b/server/api/cvrs.py @@ -44,23 +44,23 @@ create_background_task, ) from ..util.file import ( + get_file_upload_url, + get_standard_file_upload_request_params, retrieve_file, serialize_file, serialize_file_processing, - store_file, timestamp_filename, unzip_files, - zip_files, + validate_zip_mimetype, ) from ..util.csv_download import csv_response from ..util.csv_parse import ( CSVIterator, decode_csv, - does_file_have_csv_mimetype, reject_no_rows, validate_comma_delimited, - validate_csv_mimetype, validate_not_empty, + validate_csv_mimetype, ) from ..util.collections import find_first_duplicate from ..audit_math.suite import HybridPair @@ -1032,19 +1032,31 @@ def parse_hart_cvrs( cvr_zip_files: Dict[str, BinaryIO] = {} # { file_name: file } scanned_ballot_information_files: List[BinaryIO] = [] + nonCsvZipFiles = [] for file_name in file_names: if file_name.lower().endswith(".zip"): # pylint: disable=consider-using-with cvr_zip_files[file_name] = open( os.path.join(working_directory, file_name), "rb" ) - if file_name.lower().endswith(".csv"): + elif file_name.lower().endswith(".csv"): scanned_ballot_information_files.append( # pylint: disable=consider-using-with open(os.path.join(working_directory, file_name), "rb") ) + else: + nonCsvZipFiles.append(file_name) - assert len(cvr_zip_files) != 0 # Validated during file upload + # If there are no zip files inside the "wrapper" we assume it was not a wrapper and there was only one cvr zip file uploaded, unwrapped. + if len(cvr_zip_files) == 0 and len(scanned_ballot_information_files) == 0: + cvr_zip_files[jurisdiction.cvr_file.name] = retrieve_file( + jurisdiction.cvr_file.storage_path + ) + # If the wrapper was a "wrapper" zip it should only contain csv and zip files + elif len(nonCsvZipFiles) > 0: + raise UserError( + f"Unsupported file type. Expected either a ZIP file or a CSV file, but found {(','.join(nonCsvZipFiles))}." + ) def parse_scanned_ballot_information_file( scanned_ballot_information_file: BinaryIO, @@ -1540,81 +1552,22 @@ def validate_cvr_upload( if not jurisdiction.manifest_file_id: raise Conflict("Must upload ballot manifest before uploading CVR file.") - if "cvrs" not in request.files: - raise BadRequest("Missing required file parameter 'cvrs'") + data = request.get_json() + cvr_file_type = data.get("cvrFileType") if data else None + if cvr_file_type is None: + raise BadRequest("CVR file type is required") - cvr_file_type = request.form.get("cvrFileType") if cvr_file_type not in [cvr_file_type.value for cvr_file_type in CvrFileType]: raise BadRequest("Invalid file type") - if cvr_file_type == CvrFileType.HART: - - def is_zip_file(file): - return file.mimetype in ["application/zip", "application/x-zip-compressed"] - - files = request.files.getlist("cvrs") - if not all( - is_zip_file(file) or does_file_have_csv_mimetype(file) for file in files - ): - raise BadRequest("Please submit only ZIP files and CSVs.") - if not any(is_zip_file(file) for file in files): - raise BadRequest("Please submit at least one ZIP file.") - - else: - validate_csv_mimetype(request.files["cvrs"]) - def clear_cvr_contests_metadata(jurisdiction: Jurisdiction): jurisdiction.cvr_contests_metadata = None -@background_task -def clear_cvr_ballots(jurisdiction_id: str): - # Note that this query can be slow due to the query planner sometimes - # choosing to not use the relevant index on CvrBallot.batch_id. So it should - # only be run in background tasks. - CvrBallot.query.filter( - CvrBallot.batch_id.in_( - Batch.query.filter_by(jurisdiction_id=jurisdiction_id) - .with_entities(Batch.id) - .subquery() - ) - ).delete(synchronize_session=False) - - -@api.route( - "/election//jurisdiction//cvrs", - methods=["PUT"], -) -@restrict_access([UserType.AUDIT_ADMIN, UserType.JURISDICTION_ADMIN]) -def upload_cvrs( - election: Election, - jurisdiction: Jurisdiction, # pylint: disable=unused-argument +def finalize_cvr_upload( + storage_path: str, file_name: str, cvr_file_type: str, jurisdiction: Jurisdiction ): - validate_cvr_upload(request, election, jurisdiction) - clear_cvr_contests_metadata(jurisdiction) - - if request.form["cvrFileType"] in [CvrFileType.ESS, CvrFileType.HART]: - file_name = "cvr-files.zip" - zip_file = zip_files( - { - file.filename: file.stream # type: ignore - for file in request.files.getlist("cvrs") - } - ) - storage_path = store_file( - zip_file, - f"audits/{election.id}/jurisdictions/{jurisdiction.id}/" - + timestamp_filename("cvrs", "zip"), - ) - else: - file_name = request.files["cvrs"].filename # type: ignore - file_extension = "csv" - storage_path = store_file( - request.files["cvrs"].stream, - f"audits/{election.id}/jurisdictions/{jurisdiction.id}/" - + timestamp_filename("cvrs", file_extension), - ) jurisdiction.cvr_file = File( id=str(uuid.uuid4()), @@ -1622,7 +1575,7 @@ def upload_cvrs( storage_path=storage_path, uploaded_at=datetime.now(timezone.utc), ) - jurisdiction.cvr_file_type = request.form["cvrFileType"] + jurisdiction.cvr_file_type = cvr_file_type jurisdiction.cvr_file.task = create_background_task( process_cvr_file, dict( @@ -1631,8 +1584,20 @@ def upload_cvrs( support_user_email=get_support_user(session), ), ) - db_session.commit() - return jsonify(status="ok") + + +@background_task +def clear_cvr_ballots(jurisdiction_id: str): + # Note that this query can be slow due to the query planner sometimes + # choosing to not use the relevant index on CvrBallot.batch_id. So it should + # only be run in background tasks. + CvrBallot.query.filter( + CvrBallot.batch_id.in_( + Batch.query.filter_by(jurisdiction_id=jurisdiction_id) + .with_entities(Batch.id) + .subquery() + ) + ).delete(synchronize_session=False) @api.route( @@ -1650,6 +1615,53 @@ def get_cvrs( ) +@api.route( + "/election//jurisdiction//cvrs/upload-url", + methods=["GET"], +) +@restrict_access([UserType.AUDIT_ADMIN, UserType.JURISDICTION_ADMIN]) +def start_upload_for_cvrs(election: Election, jurisdiction: Jurisdiction): + file_type = request.args.get("fileType") + if file_type is None: + raise BadRequest("Missing expected query parameter: fileType") + + storage_path_prefix = f"audits/{election.id}/jurisdictions/{jurisdiction.id}" + + cvr_file_type = request.args.get("cvrFileType") + if cvr_file_type in [CvrFileType.ESS, CvrFileType.HART]: + filename = timestamp_filename("cvrs", "zip") + else: + filename = timestamp_filename("cvrs", "csv") + + return jsonify(get_file_upload_url(storage_path_prefix, filename, file_type)) + + +@api.route( + "/election//jurisdiction//cvrs/upload-complete", + methods=["POST"], +) +@restrict_access([UserType.AUDIT_ADMIN, UserType.JURISDICTION_ADMIN]) +def complete_upload_for_cvrs( + election: Election, jurisdiction: Jurisdiction # pylint: disable=unused-argument +): + validate_cvr_upload(request, election, jurisdiction) + + (storage_path, filename, file_type) = get_standard_file_upload_request_params( + request + ) + data = request.get_json() + cvr_file_type = data.get("cvrFileType") if data else None + if cvr_file_type in [CvrFileType.ESS, CvrFileType.HART]: + validate_zip_mimetype(file_type) + else: + validate_csv_mimetype(file_type) + + clear_cvr_contests_metadata(jurisdiction) + finalize_cvr_upload(storage_path, filename, cvr_file_type, jurisdiction) # type: ignore + db_session.commit() + return jsonify(status="ok") + + @api.route( "/election//jurisdiction//cvrs/csv", methods=["GET"], diff --git a/server/api/jurisdictions.py b/server/api/jurisdictions.py index 9bd3baef2..8b6159ebb 100644 --- a/server/api/jurisdictions.py +++ b/server/api/jurisdictions.py @@ -33,10 +33,11 @@ create_background_task, ) from ..util.file import ( + get_file_upload_url, + get_standard_file_upload_request_params, retrieve_file, serialize_file, serialize_file_processing, - store_file, timestamp_filename, ) from ..util.jsonschema import JSONDict @@ -582,35 +583,42 @@ def download_jurisdictions_file(election: Election): ADMIN_EMAIL = "Admin Email" -@api.route("/election//jurisdiction/file", methods=["PUT"]) +@api.route("/election//jurisdiction/file/upload-url", methods=["GET"]) @restrict_access([UserType.AUDIT_ADMIN]) -def update_jurisdictions_file(election: Election): - if len(list(election.rounds)) > 0: - raise Conflict("Cannot update jurisdictions after audit has started.") +def start_upload_for_jurisdictions_file(election: Election): + file_type = request.args.get("fileType") + if file_type is None: + raise BadRequest("Missing expected query parameter: fileType") + + storage_path_prefix = f"audits/{election.id}" + file_name = timestamp_filename("participating_jurisdictions", "csv") - if "jurisdictions" not in request.files: - raise BadRequest("Missing required file parameter 'jurisdictions'") + return jsonify(get_file_upload_url(storage_path_prefix, file_name, file_type)) - validate_csv_mimetype(request.files["jurisdictions"]) - jurisdictions_file = request.files["jurisdictions"] - storage_path = store_file( - jurisdictions_file.stream, - f"audits/{election.id}/" - + timestamp_filename("participating_jurisdictions", "csv"), +@api.route( + "/election//jurisdiction/file/upload-complete", methods=["POST"] +) +@restrict_access([UserType.AUDIT_ADMIN]) +def complete_upload_for_jurisdictions_file(election: Election): + if len(list(election.rounds)) > 0: + raise Conflict("Cannot update jurisdictions after audit has started.") + + (storage_path, filename, file_type) = get_standard_file_upload_request_params( + request ) + validate_csv_mimetype(file_type) + election.jurisdictions_file = File( id=str(uuid.uuid4()), - name=jurisdictions_file.filename, # type: ignore + name=filename, storage_path=storage_path, uploaded_at=datetime.datetime.now(timezone.utc), ) election.jurisdictions_file.task = create_background_task( process_jurisdictions_file, dict(election_id=election.id) ) - db_session.commit() - return jsonify(status="ok") diff --git a/server/api/public.py b/server/api/public.py index 1e8c69948..6f394a773 100644 --- a/server/api/public.py +++ b/server/api/public.py @@ -1,16 +1,17 @@ import math from typing import Any -from werkzeug.exceptions import Conflict +from werkzeug.exceptions import Conflict, BadRequest from flask import jsonify, request from . import api -from ..auth.auth_helpers import allow_public_access +from ..auth.auth_helpers import allow_public_access, allow_any_logged_in_user_access from ..audit_math import bravo, sampler_contest, supersimple from ..util.jsonschema import validate from ..models import * # pylint: disable=wildcard-import from ..util.get_json import safe_get_json_dict +from ..util.file import store_file # Leave enough buffer to support an election of galactic scale while making it hard for users to @@ -151,3 +152,23 @@ def compute_batch_comparison_sample_size( + 2 ) return min(sample_size, contest.ballots) + + +@api.route( + "/file-upload", + methods=["POST"], +) +@allow_any_logged_in_user_access +def upload_file_to_local_filesystem(): + file = request.files.get("file") + storage_key = request.form.get("key") + if storage_key is None: + raise BadRequest("Missing required form parameter 'key'") + if file is None: + raise BadRequest("Missing required form parameter 'file'") + + store_file( + file.stream, + storage_key, + ) + return jsonify(status="ok") diff --git a/server/api/standardized_contests.py b/server/api/standardized_contests.py index 2802ef475..ab4473f30 100644 --- a/server/api/standardized_contests.py +++ b/server/api/standardized_contests.py @@ -4,7 +4,7 @@ from datetime import datetime from typing import Dict, Optional from collections import defaultdict -from flask import request, jsonify, Request +from flask import request, jsonify from werkzeug.exceptions import BadRequest, Conflict from . import api @@ -25,10 +25,11 @@ ) from ..util.csv_download import csv_response from ..util.file import ( + get_file_upload_url, + get_standard_file_upload_request_params, retrieve_file, serialize_file, serialize_file_processing, - store_file, timestamp_filename, ) @@ -177,43 +178,57 @@ def process_standardized_contests_file(election_id: str): set_contest_metadata(election) -def validate_standardized_contests_upload(request: Request, election: Election): - if election.audit_type not in [AuditType.BALLOT_COMPARISON, AuditType.HYBRID]: - raise Conflict("Can't upload CVR file for this audit type.") - - if len(list(election.jurisdictions)) == 0: - raise Conflict( - "Must upload jurisdictions file before uploading standardized contests file." - ) - - if "standardized-contests" not in request.files: - raise BadRequest("Missing required file parameter 'standardized-contests'") +@api.route( + "/election//standardized-contests/file/upload-url", methods=["GET"] +) +@restrict_access([UserType.AUDIT_ADMIN]) +def start_upload_for_standardized_contests_file(election: Election): + file_type = request.args.get("fileType") + if file_type is None: + raise BadRequest("Missing expected query parameter: fileType") - validate_csv_mimetype(request.files["standardized-contests"]) + storage_path_prefix = f"audits/{election.id}" + filename = timestamp_filename("standardized_contests", "csv") + return jsonify(get_file_upload_url(storage_path_prefix, filename, file_type)) -@api.route("/election//standardized-contests/file", methods=["PUT"]) -@restrict_access([UserType.AUDIT_ADMIN]) -def upload_standardized_contests_file(election: Election): - validate_standardized_contests_upload(request, election) - election.standardized_contests = None - file = request.files["standardized-contests"] - storage_path = store_file( - file.stream, - f"audits/{election.id}/" + timestamp_filename("standardized_contests", "csv"), - ) +def save_standardized_contests_file( + election: Election, storage_path: str, filename: str +): election.standardized_contests_file = File( id=str(uuid.uuid4()), - name=file.filename, # type: ignore + name=filename, storage_path=storage_path, uploaded_at=datetime.now(timezone.utc), ) election.standardized_contests_file.task = create_background_task( process_standardized_contests_file, dict(election_id=election.id) ) - db_session.commit() + +@api.route( + "/election//standardized-contests/file/upload-complete", + methods=["POST"], +) +@restrict_access([UserType.AUDIT_ADMIN]) +def complete_upload_for_standardized_contests_file(election: Election): + if election.audit_type not in [AuditType.BALLOT_COMPARISON, AuditType.HYBRID]: + raise Conflict("Can't upload standardized contests file for this audit type.") + + if len(list(election.jurisdictions)) == 0: + raise Conflict( + "Must upload jurisdictions file before uploading standardized contests file." + ) + + (storage_path, filename, file_type) = get_standard_file_upload_request_params( + request + ) + validate_csv_mimetype(file_type) + + election.standardized_contests = None + save_standardized_contests_file(election, storage_path, filename) + db_session.commit() return jsonify(status="ok") diff --git a/server/app.py b/server/app.py index 00826fa63..5d51401d0 100644 --- a/server/app.py +++ b/server/app.py @@ -37,6 +37,7 @@ "default-src": "'self'", "script-src": "'self' 'unsafe-inline'", "style-src": "'self' 'unsafe-inline'", + "connect-src": "'self' https://*.s3.amazonaws.com", }, ) csrf = SeaSurf(app) diff --git a/server/auth/auth_helpers.py b/server/auth/auth_helpers.py index 4cafcbbf7..47fd81725 100644 --- a/server/auth/auth_helpers.py +++ b/server/auth/auth_helpers.py @@ -260,3 +260,20 @@ def wrapper(*args, **kwargs): wrapper.has_access_control = True # type: ignore return wrapper + + +def allow_any_logged_in_user_access(route: Callable): + """ + Flask route decorator that allows public access to a route. + """ + + @functools.wraps(route) + def wrapper(*args, **kwargs): + _, user_key = get_loggedin_user(session) + if not user_key: + raise Unauthorized("Please log in to access Arlo") + return route(*args, **kwargs) + + wrapper.has_access_control = True # type: ignore + + return wrapper diff --git a/server/config.py b/server/config.py index 55f475082..5f51b3773 100644 --- a/server/config.py +++ b/server/config.py @@ -142,10 +142,14 @@ def parse_bool(value: str) -> bool: env_defaults=dict(development="/tmp/arlo", test="/tmp/arlo-test"), ) # If using S3, AWS credentials are required as well -AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY = ( - (read_env_var("AWS_ACCESS_KEY_ID"), read_env_var("AWS_SECRET_ACCESS_KEY")) +AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_DEFAULT_REGION = ( + ( + read_env_var("AWS_ACCESS_KEY_ID"), + read_env_var("AWS_SECRET_ACCESS_KEY"), + read_env_var("AWS_DEFAULT_REGION", default="us-west-1"), + ) if FILE_UPLOAD_STORAGE_PATH.startswith("s3://") - else (None, None) + else (None, None, None) ) # Configure round size growth from ARLO_MINERVA_MULTIPLE (a float) if given, otherwise 1.5 diff --git a/server/tests/api/test_activity.py b/server/tests/api/test_activity.py index ca6c8cb48..3b81541d0 100644 --- a/server/tests/api/test_activity.py +++ b/server/tests/api/test_activity.py @@ -318,20 +318,16 @@ def test_file_upload_errors( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={"manifest": (io.BytesIO(b"invalid"), "manifest.csv")}, + rv = upload_ballot_manifest( + client, io.BytesIO(b"invalid"), election_id, jurisdiction_ids[0] ) assert_ok(rv) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO(b"Batch Name,Number of Ballots\n" b"A,1"), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO(b"Batch Name,Number of Ballots\n" b"A,1"), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -339,9 +335,8 @@ def test_file_upload_errors( election.audit_type = AuditType.BATCH_COMPARISON db_session.commit() - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-tallies", - data={"batchTallies": (io.BytesIO(b"invalid"), "tallies.csv")}, + rv = upload_batch_tallies( + client, io.BytesIO(b"invalid"), election_id, jurisdiction_ids[0] ) assert rv.status_code == 200 @@ -349,12 +344,8 @@ def test_file_upload_errors( election.audit_type = AuditType.BALLOT_COMPARISON db_session.commit() - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": (io.BytesIO(b""), "cvrs.csv"), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, io.BytesIO(b""), election_id, jurisdiction_ids[0], "DOMINION" ) assert_ok(rv) diff --git a/server/tests/api/test_ballot_manifest.py b/server/tests/api/test_ballot_manifest.py index 3bb951cf4..94c58ebd4 100644 --- a/server/tests/api/test_ballot_manifest.py +++ b/server/tests/api/test_ballot_manifest.py @@ -12,16 +12,11 @@ def test_ballot_manifest_upload( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Batch Name,Number of Ballots\n" b"1,23\n" b"12,100\n" b"6,0\n" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO(b"Batch Name,Number of Ballots\n" b"1,23\n" b"12,100\n" b"6,0\n"), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -32,7 +27,7 @@ def test_ballot_manifest_upload( json.loads(rv.data), { "file": { - "name": "manifest.csv", + "name": asserts_startswith("manifest"), "uploadedAt": assert_is_date, }, "processing": { @@ -62,14 +57,11 @@ def test_ballot_manifest_clear( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO(b"Batch Name,Number of Ballots\n" b"1,23\n"), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO(b"Batch Name,Number of Ballots\n" b"1,23\n"), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -98,29 +90,21 @@ def test_ballot_manifest_replace_as_audit_admin( ): # Check that AA can also get/put/clear manifest set_logged_in_user(client, UserType.AUDIT_ADMIN, DEFAULT_AA_EMAIL) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Batch Name,Number of Ballots\n" b"1,23\n" b"12,100\n" b"6,0,,\n" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO(b"Batch Name,Number of Ballots\n" b"1,23\n" b"12,100\n" b"6,0,,\n"), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) file_id = Jurisdiction.query.get(jurisdiction_ids[0]).manifest_file_id - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO(b"Batch Name,Number of Ballots\n" b"1,23\n" b"12,6\n"), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO(b"Batch Name,Number of Ballots\n" b"1,23\n" b"12,6\n"), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -150,22 +134,22 @@ def test_ballot_manifest_replace_as_audit_admin( assert json.loads(rv.data) == {"file": None, "processing": None} -def test_ballot_manifest_upload_missing_file( +def test_ballot_manifest_upload_missing_file_path( client: FlaskClient, election_id: str, jurisdiction_ids: List[str] ): set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={}, + rv = client.post( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest/upload-complete", + json={}, ) assert rv.status_code == 400 assert json.loads(rv.data) == { "errors": [ { "errorType": "Bad Request", - "message": "Missing required file parameter 'manifest'", + "message": "Missing required JSON parameter: storagePathKey", } ] } @@ -177,9 +161,24 @@ def test_ballot_manifest_upload_bad_csv( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={"manifest": (io.BytesIO(b"not a CSV file"), "random.txt")}, + rv = client.post( + "/api/file-upload", + data={ + "file": ( + io.BytesIO(b"not a CSV file"), + "random.txt", + ), + "key": "test_dir/random.txt", + }, + ) + assert_ok(rv) + rv = client.post( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest/upload-complete", + json={ + "storagePathKey": "test_dir/random.txt", + "fileName": "random.txt", + "fileType": "text/plain", + }, ) assert rv.status_code == 400 assert json.loads(rv.data) == { @@ -202,14 +201,11 @@ def test_ballot_manifest_upload_missing_field( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO(header_row.encode() + b"\n1,2,3"), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO(header_row.encode() + b"\n1,2,3"), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -220,7 +216,7 @@ def test_ballot_manifest_upload_missing_field( json.loads(rv.data), { "file": { - "name": "manifest.csv", + "name": asserts_startswith("manifest"), "uploadedAt": assert_is_date, }, "processing": { @@ -239,14 +235,12 @@ def test_ballot_manifest_upload_invalid_num_ballots( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO(b"Batch Name,Number of Ballots\n" b"1,not a number\n"), - "manifest.csv", - ) - }, + + rv = upload_ballot_manifest( + client, + io.BytesIO(b"Batch Name,Number of Ballots\n" b"1,not a number\n"), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -257,7 +251,7 @@ def test_ballot_manifest_upload_invalid_num_ballots( json.loads(rv.data), { "file": { - "name": "manifest.csv", + "name": asserts_startswith("manifest"), "uploadedAt": assert_is_date, }, "processing": { @@ -276,16 +270,11 @@ def test_ballot_manifest_upload_duplicate_batch_name( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Batch Name,Number of Ballots\n" b"12,23\n" b"12,100\n" b"6,0\n" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO(b"Batch Name,Number of Ballots\n" b"12,23\n" b"12,100\n" b"6,0\n"), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -296,7 +285,7 @@ def test_ballot_manifest_upload_duplicate_batch_name( json.loads(rv.data), { "file": { - "name": "manifest.csv", + "name": asserts_startswith("manifest"), "uploadedAt": assert_is_date, }, "processing": { @@ -307,3 +296,48 @@ def test_ballot_manifest_upload_duplicate_batch_name( }, }, ) + + +def test_ballot_manifest_get_upload_url_missing_file_type( + client: FlaskClient, election_id: str, jurisdiction_ids: List[str] +): + set_logged_in_user( + client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) + ) + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest/upload-url" + ) + assert rv.status_code == 400 + assert json.loads(rv.data) == { + "errors": [ + { + "errorType": "Bad Request", + "message": "Missing expected query parameter: fileType", + } + ] + } + + +def test_ballot_manifest_get_upload_url( + client: FlaskClient, election_id: str, jurisdiction_ids: List[str] +): + allowed_users = [ + (UserType.JURISDICTION_ADMIN, default_ja_email(election_id)), + (UserType.AUDIT_ADMIN, DEFAULT_AA_EMAIL), + ] + for user, email in allowed_users: + set_logged_in_user(client, user, email) + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest/upload-url", + query_string={"fileType": "text/csv"}, + ) + assert rv.status_code == 200 + + response_data = json.loads(rv.data) + expected_url = "/api/file-upload" + + assert response_data["url"] == expected_url + assert response_data["fields"]["key"].startswith( + f"audits/{election_id}/jurisdictions/{jurisdiction_ids[0]}/manifest_" + ) + assert response_data["fields"]["key"].endswith(".csv") diff --git a/server/tests/api/test_ballots.py b/server/tests/api/test_ballots.py index 387332d40..bf3fe8a8f 100644 --- a/server/tests/api/test_ballots.py +++ b/server/tests/api/test_ballots.py @@ -1248,19 +1248,16 @@ def test_ballots_human_sort_order( "Batch 2", "Batch 10", ] - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - ( - "Batch Name,Number of Ballots\n" - + "\n".join(f"{batch},10" for batch in human_ordered_batches) - ).encode() - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO( + ( + "Batch Name,Number of Ballots\n" + + "\n".join(f"{batch},10" for batch in human_ordered_batches) + ).encode() + ), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) diff --git a/server/tests/api/test_jurisdictions.py b/server/tests/api/test_jurisdictions.py index 72c31a8cd..f776cecb2 100644 --- a/server/tests/api/test_jurisdictions.py +++ b/server/tests/api/test_jurisdictions.py @@ -73,14 +73,11 @@ def test_jurisdictions_list_with_manifest( manifest = ( b"Batch Name,Number of Ballots\n" b"1,23\n" b"2,101\n" b"3,122\n" b"4,400" ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO(manifest), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO(manifest), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -94,7 +91,7 @@ def test_jurisdictions_list_with_manifest( "name": "J1", "ballotManifest": { "file": { - "name": "manifest.csv", + "name": asserts_startswith("manifest"), "uploadedAt": assert_is_date, }, "processing": { @@ -140,7 +137,9 @@ def test_jurisdictions_list_with_manifest( rv = client.get( f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest/csv" ) - assert rv.headers["Content-Disposition"] == 'attachment; filename="manifest.csv"' + assert rv.headers["Content-Disposition"].startswith( + 'attachment; filename="manifest' + ) assert rv.data == manifest @@ -155,14 +154,11 @@ def test_duplicate_batch_name(client, election_id, jurisdiction_ids): set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO(b"Batch Name,Number of Ballots\n" b"1,23\n" b"1,101\n"), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO(b"Batch Name,Number of Ballots\n" b"1,23\n" b"1,101\n"), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -176,7 +172,7 @@ def test_duplicate_batch_name(client, election_id, jurisdiction_ids): "name": "J1", "ballotManifest": { "file": { - "name": "manifest.csv", + "name": asserts_startswith("manifest"), "uploadedAt": assert_is_date, }, "processing": { diff --git a/server/tests/api/test_jurisdictions_file.py b/server/tests/api/test_jurisdictions_file.py index f745f6179..bddc94878 100644 --- a/server/tests/api/test_jurisdictions_file.py +++ b/server/tests/api/test_jurisdictions_file.py @@ -6,12 +6,14 @@ def test_missing_file(client: FlaskClient, election_id: str): - rv = client.put(f"/api/election/{election_id}/jurisdiction/file") + rv = client.post( + f"/api/election/{election_id}/jurisdiction/file/upload-complete", json={} + ) assert rv.status_code == 400 assert json.loads(rv.data) == { "errors": [ { - "message": "Missing required file parameter 'jurisdictions'", + "message": "Missing required JSON parameter: storagePathKey", "errorType": "Bad Request", } ] @@ -19,9 +21,18 @@ def test_missing_file(client: FlaskClient, election_id: str): def test_bad_csv_file(client: FlaskClient, election_id: str): - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={"jurisdictions": (io.BytesIO(b"not a CSV file"), "random.txt")}, + rv = upload_jurisdictions_file( + client, + io.BytesIO(b"not a CSV file"), + election_id, + ) + rv = client.post( + f"/api/election/{election_id}/jurisdiction/file/upload-complete", + json={ + "storagePathKey": "test_dir/random.txt", + "fileName": "random.txt", + "fileType": "text/plain", + }, ) assert rv.status_code == 400 assert json.loads(rv.data) == { @@ -35,14 +46,10 @@ def test_bad_csv_file(client: FlaskClient, election_id: str): def test_missing_one_csv_field(client, election_id): - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO(b"Jurisdiction\nJurisdiction #1"), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO(b"Jurisdiction\nJurisdiction #1"), + election_id, ) assert_ok(rv) @@ -50,7 +57,10 @@ def test_missing_one_csv_field(client, election_id): compare_json( json.loads(rv.data), { - "file": {"name": "jurisdictions.csv", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("jurisdictions"), + "uploadedAt": assert_is_date, + }, "processing": { "status": ProcessingStatus.ERRORED, "startedAt": assert_is_date, @@ -66,21 +76,22 @@ def test_jurisdictions_file_metadata(client, election_id): assert json.loads(rv.data) == {"file": None, "processing": None} contents = "Jurisdiction,Admin Email\nJ1,ja@example.com" - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={"jurisdictions": (io.BytesIO(contents.encode()), "jurisdictions.csv")}, + rv = upload_jurisdictions_file( + client, + io.BytesIO(contents.encode()), + election_id, ) assert_ok(rv) election = Election.query.filter_by(id=election_id).one() - assert election.jurisdictions_file.name == "jurisdictions.csv" + assert election.jurisdictions_file.name.startswith("jurisdictions") assert election.jurisdictions_file.uploaded_at rv = client.get(f"/api/election/{election_id}/jurisdiction/file") response = json.loads(rv.data) file = response["file"] processing = response["processing"] - assert file["name"] == "jurisdictions.csv" + assert file["name"].startswith("jurisdictions") assert file["uploadedAt"] assert processing["status"] == ProcessingStatus.PROCESSED assert processing["startedAt"] @@ -88,26 +99,20 @@ def test_jurisdictions_file_metadata(client, election_id): assert processing["error"] is None rv = client.get(f"/api/election/{election_id}/jurisdiction/file/csv") - assert ( - rv.headers["Content-Disposition"] == 'attachment; filename="jurisdictions.csv"' + assert rv.headers["Content-Disposition"].startswith( + 'attachment; filename="jurisdictions' ) assert rv.data.decode("utf-8") == contents def test_replace_jurisdictions_file(client, election_id): # Create the initial file. - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO( - b"Jurisdiction,Admin Email\n" - b"J1,ja@example.com\n" - b"J2,ja2@example.com" - ), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO( + b"Jurisdiction,Admin Email\n" b"J1,ja@example.com\n" b"J2,ja2@example.com" + ), + election_id, ) assert_ok(rv) @@ -123,18 +128,12 @@ def test_replace_jurisdictions_file(client, election_id): file_id = election.jurisdictions_file_id # Replace it with another file. - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO( - b"Jurisdiction,Admin Email\n" - b"J2,ja2@example.com\n" - b"J3,ja3@example.com" - ), - "jurisdictions2.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO( + b"Jurisdiction,Admin Email\n" b"J2,ja2@example.com\n" b"J3,ja3@example.com" + ), + election_id, ) assert_ok(rv) @@ -143,7 +142,7 @@ def test_replace_jurisdictions_file(client, election_id): ) assert rv.status_code == 200 response = json.loads(rv.data) - assert response["file"]["name"] == "jurisdictions2.csv" + assert response["file"]["name"].startswith("jurisdictions") assert File.query.get(file_id) is None, "the old file should have been deleted" @@ -158,14 +157,10 @@ def test_replace_jurisdictions_file(client, election_id): def test_no_jurisdiction(client, election_id): - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO(b"Jurisdiction,Admin Email"), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO(b"Jurisdiction,Admin Email"), + election_id, ) assert_ok(rv) @@ -174,14 +169,10 @@ def test_no_jurisdiction(client, election_id): def test_single_jurisdiction_single_admin(client, election_id): - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO(b"Jurisdiction,Admin Email\nJ1,a1@example.com"), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO(b"Jurisdiction,Admin Email\nJ1,a1@example.com"), + election_id, ) assert_ok(rv) @@ -195,16 +186,10 @@ def test_single_jurisdiction_single_admin(client, election_id): def test_single_jurisdiction_multiple_admins(client, election_id): - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO( - b"Jurisdiction,Admin Email\nJ1,a1@example.com\nJ1,a2@example.com" - ), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO(b"Jurisdiction,Admin Email\nJ1,a1@example.com\nJ1,a2@example.com"), + election_id, ) assert_ok(rv) @@ -219,16 +204,10 @@ def test_single_jurisdiction_multiple_admins(client, election_id): def test_multiple_jurisdictions_single_admin(client, election_id): - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO( - b"Jurisdiction,Admin Email\nJ1,a1@example.com\nJ2,a1@example.com" - ), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO(b"Jurisdiction,Admin Email\nJ1,a1@example.com\nJ2,a1@example.com"), + election_id, ) assert_ok(rv) @@ -247,19 +226,15 @@ def test_download_jurisdictions_file_not_found(client, election_id): def test_convert_emails_to_lowercase(client, election_id): - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO( - b"Jurisdiction,Admin Email\n" - b"J1,lowecase@example.com\n" - b"J2,UPPERCASE@EXAMPLE.COM\n" - b"J3,MiXeDcAsE@eXaMpLe.CoM\n" - ), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO( + b"Jurisdiction,Admin Email\n" + b"J1,lowecase@example.com\n" + b"J2,UPPERCASE@EXAMPLE.COM\n" + b"J3,MiXeDcAsE@eXaMpLe.CoM\n" + ), + election_id, ) assert_ok(rv) @@ -274,14 +249,10 @@ def test_upload_jurisdictions_file_after_audit_starts( election_id: str, round_1_id: str, # pylint: disable=unused-argument ): - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO(b"Jurisdiction,Admin Email\n" b"J1,j1@example.com\n"), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO(b"Jurisdiction,Admin Email\n" b"J1,j1@example.com\n"), + election_id, ) assert rv.status_code == 409 assert json.loads(rv.data) == { @@ -298,18 +269,12 @@ def test_upload_jurisdictions_file_duplicate_row( client: FlaskClient, election_id: str, ): - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO( - b"Jurisdiction,Admin Email\n" - b"J1,j1@example.com\n" - b"J1,j1@example.com" - ), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO( + b"Jurisdiction,Admin Email\n" b"J1,j1@example.com\n" b"J1,j1@example.com" + ), + election_id, ) assert_ok(rv) @@ -317,7 +282,10 @@ def test_upload_jurisdictions_file_duplicate_row( compare_json( json.loads(rv.data), { - "file": {"name": "jurisdictions.csv", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("jurisdictions"), + "uploadedAt": assert_is_date, + }, "processing": { "status": ProcessingStatus.ERRORED, "startedAt": assert_is_date, @@ -340,38 +308,26 @@ def test_jurisdictions_file_dont_clobber_other_elections( ) # Add jurisdictions. - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO(b"Jurisdiction,Admin Email\n" b"J1,j1@example.com\n"), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO(b"Jurisdiction,Admin Email\n" b"J1,j1@example.com\n"), + election_id, ) assert_ok(rv) # Add jurisdictions for other election - rv = client.put( - f"/api/election/{other_election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO(b"Jurisdiction,Admin Email\n" b"J2,j2@example.com\n"), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO(b"Jurisdiction,Admin Email\n" b"J2,j2@example.com\n"), + other_election_id, ) assert_ok(rv) # Now change them - rv = client.put( - f"/api/election/{other_election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO(b"Jurisdiction,Admin Email\n" b"J3,j3@example.com\n"), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO(b"Jurisdiction,Admin Email\n" b"J3,j3@example.com\n"), + other_election_id, ) assert_ok(rv) @@ -387,26 +343,22 @@ def test_jurisdictions_file_dont_clobber_other_elections( def test_jurisdictions_file_expected_num_ballots(client: FlaskClient, election_id: str): - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO( - b"Jurisdiction,Admin Email,Expected Number of Ballots\n" - # If multiple rows for the same jurisdiction, the last one wins - b"J1,a1@example.com,10\n" - b"J1,a2@example.com,20\n" - # Value is optional - b"J2,a1@example.com,\n" - # If no value in some rows for jurisdiction, the last value wins - b"J3,a1@example.com,10\n" - b"J3,a2@example.com,\n" - b"J4,a1@example.com,\n" - b"J4,a2@example.com,500\n" - ), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO( + b"Jurisdiction,Admin Email,Expected Number of Ballots\n" + # If multiple rows for the same jurisdiction, the last one wins + b"J1,a1@example.com,10\n" + b"J1,a2@example.com,20\n" + # Value is optional + b"J2,a1@example.com,\n" + # If no value in some rows for jurisdiction, the last value wins + b"J3,a1@example.com,10\n" + b"J3,a2@example.com,\n" + b"J4,a1@example.com,\n" + b"J4,a2@example.com,500\n" + ), + election_id, ) assert_ok(rv) @@ -415,3 +367,38 @@ def test_jurisdictions_file_expected_num_ballots(client: FlaskClient, election_i assert [ (j["name"], j["expectedBallotManifestNumBallots"]) for j in jurisdictions ] == [("J1", 20), ("J2", None), ("J3", 10), ("J4", 500)] + + +def test_jurisdiction_file_get_upload_url_missing_file_type( + client: FlaskClient, election_id: str +): + set_logged_in_user(client, UserType.AUDIT_ADMIN, user_key=DEFAULT_AA_EMAIL) + rv = client.get(f"/api/election/{election_id}/jurisdiction/file/upload-url") + assert rv.status_code == 400 + assert json.loads(rv.data) == { + "errors": [ + { + "errorType": "Bad Request", + "message": "Missing expected query parameter: fileType", + } + ] + } + + +def test_jurisdiction_file_get_upload_url(client: FlaskClient, election_id: str): + set_logged_in_user(client, UserType.AUDIT_ADMIN, user_key=DEFAULT_AA_EMAIL) + + rv = client.get( + f"/api/election/{election_id}/jurisdiction/file/upload-url", + query_string={"fileType": "text/csv"}, + ) + assert rv.status_code == 200 + + response_data = json.loads(rv.data) + expected_url = "/api/file-upload" + + assert response_data["url"] == expected_url + assert response_data["fields"]["key"].startswith( + f"audits/{election_id}/participating_jurisdictions_" + ) + assert response_data["fields"]["key"].endswith(".csv") diff --git a/server/tests/api/test_public.py b/server/tests/api/test_public.py index 42a2ef1fb..0fde9a6d6 100644 --- a/server/tests/api/test_public.py +++ b/server/tests/api/test_public.py @@ -1,7 +1,11 @@ import json +import io +import tempfile from typing import Any, Dict, List, TypedDict from flask.testing import FlaskClient +from ..helpers import * # pylint: disable=wildcard-import +from ... import config def copy_dict_and_remove_key(input: Dict, key: str): @@ -548,3 +552,82 @@ class TestCase(TypedDict): assert rv.status_code == 200 response = json.loads(rv.data) snapshot.assert_match(response, test_case["description"]) + + +def test_public_file_upload(client: FlaskClient): + set_logged_in_user(client, UserType.AUDIT_ADMIN, DEFAULT_AA_EMAIL) + with tempfile.TemporaryDirectory() as temp_dir: + config.FILE_UPLOAD_STORAGE_PATH = temp_dir + rv = client.post( + "/api/file-upload", + data={ + "file": ( + io.BytesIO(b"hello, I am a file"), + "random.txt", + ), + "key": "test_dir/random.txt", + }, + ) + assert_ok(rv) + with open(f"{temp_dir}/test_dir/random.txt", "rb") as stored_file: + assert stored_file.read() == b"hello, I am a file" + + +def test_public_file_upload_unauthorized(client: FlaskClient): + rv = client.post( + "/api/file-upload", + data={ + "file": ( + io.BytesIO(b"hello, I am a file"), + "random.txt", + ), + "key": "test_dir/random.txt", + }, + ) + assert rv.status_code == 401 + assert json.loads(rv.data) == { + "errors": [ + { + "errorType": "Unauthorized", + "message": "Please log in to access Arlo", + } + ] + } + + +def test_public_file_upload_error(client: FlaskClient): + set_logged_in_user(client, UserType.AUDIT_ADMIN, DEFAULT_AA_EMAIL) + rv = client.post( + "/api/file-upload", + data={ + "key": "test_dir/random.txt", + }, + ) + assert rv.status_code == 400 + assert json.loads(rv.data) == { + "errors": [ + { + "errorType": "Bad Request", + "message": "Missing required form parameter 'file'", + } + ] + } + + rv = client.post( + "/api/file-upload", + data={ + "file": ( + io.BytesIO(b"hello, I am a file"), + "random.txt", + ), + }, + ) + assert rv.status_code == 400 + assert json.loads(rv.data) == { + "errors": [ + { + "errorType": "Bad Request", + "message": "Missing required form parameter 'key'", + } + ] + } diff --git a/server/tests/ballot_comparison/conftest.py b/server/tests/ballot_comparison/conftest.py index 0410f94f5..989a57a28 100644 --- a/server/tests/ballot_comparison/conftest.py +++ b/server/tests/ballot_comparison/conftest.py @@ -87,36 +87,30 @@ def manifests(client: FlaskClient, election_id: str, jurisdiction_ids: List[str] set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Tabulator,Batch Name,Number of Ballots\n" - b"TABULATOR1,BATCH1,3\n" - b"TABULATOR1,BATCH2,3\n" - b"TABULATOR2,BATCH1,3\n" - b"TABULATOR2,BATCH2,6" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO( + b"Tabulator,Batch Name,Number of Ballots\n" + b"TABULATOR1,BATCH1,3\n" + b"TABULATOR1,BATCH2,3\n" + b"TABULATOR2,BATCH1,3\n" + b"TABULATOR2,BATCH2,6" + ), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Tabulator,Batch Name,Number of Ballots\n" - b"TABULATOR1,BATCH1,3\n" - b"TABULATOR1,BATCH2,3\n" - b"TABULATOR2,BATCH1,3\n" - b"TABULATOR2,BATCH2,6" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO( + b"Tabulator,Batch Name,Number of Ballots\n" + b"TABULATOR1,BATCH1,3\n" + b"TABULATOR1,BATCH2,3\n" + b"TABULATOR2,BATCH1,3\n" + b"TABULATOR2,BATCH2,6" + ), + election_id, + jurisdiction_ids[1], ) assert_ok(rv) @@ -127,20 +121,17 @@ def ess_manifests(client: FlaskClient, election_id: str, jurisdiction_ids: List[ set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_id}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Tabulator,Batch Name,Number of Ballots\n" - b"0001,BATCH1,3\n" - b"0001,BATCH2,3\n" - b"0002,BATCH1,3\n" - b"0002,BATCH2,6" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO( + b"Tabulator,Batch Name,Number of Ballots\n" + b"0001,BATCH1,3\n" + b"0001,BATCH2,3\n" + b"0002,BATCH1,3\n" + b"0002,BATCH2,6" + ), + election_id, + jurisdiction_id, ) assert_ok(rv) @@ -151,20 +142,17 @@ def hart_manifests(client: FlaskClient, election_id: str, jurisdiction_ids: List set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_id}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Tabulator,Batch Name,Number of Ballots\n" - b"TABULATOR1,BATCH1,3\n" - b"TABULATOR1,BATCH2,3\n" - b"TABULATOR2,BATCH3,3\n" - b"TABULATOR2,BATCH4,6" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO( + b"Tabulator,Batch Name,Number of Ballots\n" + b"TABULATOR1,BATCH1,3\n" + b"TABULATOR1,BATCH2,3\n" + b"TABULATOR2,BATCH3,3\n" + b"TABULATOR2,BATCH4,6" + ), + election_id, + jurisdiction_id, ) assert_ok(rv) @@ -179,25 +167,19 @@ def cvrs( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(TEST_CVRS.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(TEST_CVRS.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert_ok(rv) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(TEST_CVRS.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(TEST_CVRS.encode()), + election_id, + jurisdiction_ids[1], + "DOMINION", ) assert_ok(rv) diff --git a/server/tests/ballot_comparison/test_ballot_comparison.py b/server/tests/ballot_comparison/test_ballot_comparison.py index a2a8290c3..fe8ef72f6 100644 --- a/server/tests/ballot_comparison/test_ballot_comparison.py +++ b/server/tests/ballot_comparison/test_ballot_comparison.py @@ -88,21 +88,14 @@ def test_set_contest_metadata_on_manifest_and_cvr_upload( assert contest["totalBallotsCast"] is None assert contest["votesAllowed"] is None - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Tabulator,Batch Name,Number of Ballots\n" - b"TABULATOR1,BATCH1,3\n" - b"TABULATOR1,BATCH2,3\n" - b"TABULATOR2,BATCH1,3\n" - b"TABULATOR2,BATCH2,6" - ), - "manifest.csv", - ) - }, - ) + file_content = io.BytesIO( + b"Tabulator,Batch Name,Number of Ballots\n" + b"TABULATOR1,BATCH1,3\n" + b"TABULATOR1,BATCH2,3\n" + b"TABULATOR2,BATCH1,3\n" + b"TABULATOR2,BATCH2,6" + ) + rv = upload_ballot_manifest(client, file_content, election_id, jurisdiction_ids[0]) assert_ok(rv) # Contest total ballots isn't set when only some manifests uploaded @@ -112,20 +105,17 @@ def test_set_contest_metadata_on_manifest_and_cvr_upload( assert contest["totalBallotsCast"] is None assert contest["votesAllowed"] is None - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Tabulator,Batch Name,Number of Ballots\n" - b"TABULATOR1,BATCH1,3\n" - b"TABULATOR1,BATCH2,3\n" - b"TABULATOR2,BATCH1,3\n" - b"TABULATOR2,BATCH2,6" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO( + b"Tabulator,Batch Name,Number of Ballots\n" + b"TABULATOR1,BATCH1,3\n" + b"TABULATOR1,BATCH2,3\n" + b"TABULATOR2,BATCH1,3\n" + b"TABULATOR2,BATCH2,6" + ), + election_id, + jurisdiction_ids[1], ) assert_ok(rv) @@ -136,15 +126,12 @@ def test_set_contest_metadata_on_manifest_and_cvr_upload( assert contest["totalBallotsCast"] == 30 assert contest["votesAllowed"] is None - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(TEST_CVRS.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(TEST_CVRS.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert_ok(rv) @@ -155,15 +142,12 @@ def test_set_contest_metadata_on_manifest_and_cvr_upload( assert contest["totalBallotsCast"] == 30 assert contest["votesAllowed"] is None - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(TEST_CVRS.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(TEST_CVRS.encode()), + election_id, + jurisdiction_ids[1], + "DOMINION", ) assert_ok(rv) @@ -185,32 +169,26 @@ def test_set_contest_metadata_on_manifest_and_cvr_upload( # Contest metadata changes on new manifest/CVR upload # - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Tabulator,Batch Name,Number of Ballots\n" - b"TABULATOR1,BATCH1,3\n" - b"TABULATOR1,BATCH2,3\n" - b"TABULATOR2,BATCH1,3" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO( + b"Tabulator,Batch Name,Number of Ballots\n" + b"TABULATOR1,BATCH1,3\n" + b"TABULATOR1,BATCH2,3\n" + b"TABULATOR2,BATCH1,3" + ), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) new_cvr = "\n".join(TEST_CVRS.splitlines()[:10]) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(new_cvr.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(new_cvr.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert_ok(rv) @@ -255,15 +233,12 @@ def test_cvr_choice_name_validation( contest = json.loads(rv.data)["contests"][0] assert "cvrChoiceNameConsistencyError" not in contest - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(TEST_CVRS.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(TEST_CVRS.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert_ok(rv) @@ -271,15 +246,12 @@ def test_cvr_choice_name_validation( contest = json.loads(rv.data)["contests"][0] assert "cvrChoiceNameConsistencyError" not in contest - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(TEST_CVRS.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(TEST_CVRS.encode()), + election_id, + jurisdiction_ids[1], + "DOMINION", ) assert_ok(rv) @@ -288,15 +260,12 @@ def test_cvr_choice_name_validation( assert "cvrChoiceNameConsistencyError" not in contest modified_cvrs = TEST_CVRS.replace("Choice", "CHOICE") - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(modified_cvrs.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(modified_cvrs.encode()), + election_id, + jurisdiction_ids[1], + "DOMINION", ) assert_ok(rv) @@ -314,15 +283,12 @@ def test_cvr_choice_name_validation( } modified_cvrs = TEST_CVRS.replace("Choice 1-1", "CHOICE 1-1") - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(modified_cvrs.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(modified_cvrs.encode()), + election_id, + jurisdiction_ids[1], + "DOMINION", ) assert_ok(rv) @@ -340,15 +306,12 @@ def test_cvr_choice_name_validation( } modified_cvrs = TEST_CVRS_WITH_CHOICE_REMOVED - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(modified_cvrs.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(modified_cvrs.encode()), + election_id, + jurisdiction_ids[1], + "DOMINION", ) assert_ok(rv) @@ -357,15 +320,12 @@ def test_cvr_choice_name_validation( assert "cvrChoiceNameConsistencyError" not in contest modified_cvrs = TEST_CVRS_WITH_EXTRA_CHOICE - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(modified_cvrs.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(modified_cvrs.encode()), + election_id, + jurisdiction_ids[1], + "DOMINION", ) assert_ok(rv) @@ -407,20 +367,16 @@ def test_set_contest_metadata_on_jurisdiction_change( # Upload new jurisdictions, removing J1 set_logged_in_user(client, UserType.AUDIT_ADMIN, DEFAULT_AA_EMAIL) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO( - ( - "Jurisdiction,Admin Email\n" - f"J2,{default_ja_email(election_id)}\n" - f"J3,j3-{election_id}@example.com\n" - ).encode() - ), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO( + ( + "Jurisdiction,Admin Email\n" + f"J2,{default_ja_email(election_id)}\n" + f"J3,j3-{election_id}@example.com\n" + ).encode() + ), + election_id, ) assert_ok(rv) @@ -733,21 +689,16 @@ def test_ballot_comparison_two_rounds( snapshot, ): set_logged_in_user(client, UserType.AUDIT_ADMIN, DEFAULT_AA_EMAIL) - # AA uploads standardized contests file - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO( - b"Contest Name,Jurisdictions\n" - b'Contest 1,"J1,J2"\n' - b'Contest 2,"J1,J2"\n' - b"Contest 3,J2\n" - ), - "standardized-contests.csv", - ) - }, + rv = upload_standardized_contests( + client, + io.BytesIO( + b"Contest Name,Jurisdictions\n" + b'Contest 1,"J1,J2"\n' + b'Contest 2,"J1,J2"\n' + b"Contest 3,J2\n" + ), + election_id, ) assert_ok(rv) @@ -1405,10 +1356,10 @@ def test_ballot_comparison_ess( 13,p,bs,Choice 1-1,Choice 2-3 15,p,bs,Choice 1-1,Choice 2-3 """ - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": [ + rv = upload_cvrs( + client, + zip_cvrs( + [ ( io.BytesIO(ESS_BALLOTS_1.encode()), "ess_ballots_1.csv", @@ -1421,15 +1372,18 @@ def test_ballot_comparison_ess( io.BytesIO(j1_cvr.encode()), "ess_cvr.csv", ), - ], - "cvrFileType": "ESS", - }, + ] + ), + election_id, + jurisdiction_ids[0], + "ESS", + "application/zip", ) assert_ok(rv) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/cvrs", - data={ - "cvrs": [ + rv = upload_cvrs( + client, + zip_cvrs( + [ ( io.BytesIO(ESS_BALLOTS_1.encode()), "ess_ballots_1.csv", @@ -1442,9 +1396,12 @@ def test_ballot_comparison_ess( io.BytesIO(j2_cvr.encode()), "ess_cvr.csv", ), - ], - "cvrFileType": "ESS", - }, + ] + ), + election_id, + jurisdiction_ids[1], + "ESS", + "application/zip", ) assert_ok(rv) diff --git a/server/tests/ballot_comparison/test_ballot_comparison_manifests.py b/server/tests/ballot_comparison/test_ballot_comparison_manifests.py index 8435b86bd..d3f26028c 100644 --- a/server/tests/ballot_comparison/test_ballot_comparison_manifests.py +++ b/server/tests/ballot_comparison/test_ballot_comparison_manifests.py @@ -19,50 +19,42 @@ def manifests(client: FlaskClient, election_id: str, jurisdiction_ids: List[str] set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Container,Tabulator,Batch Name,Number of Ballots\n" - b"CONTAINER1,TABULATOR1,BATCH1,50\n" - b"CONTAINER1,TABULATOR1,BATCH2,50\n" - b"CONTAINER1,TABULATOR2,BATCH1,50\n" - b"CONTAINER1,TABULATOR2,BATCH2,50\n" - b"CONTAINER2,TABULATOR1,BATCH3,50\n" - b"CONTAINER2,TABULATOR1,BATCH4,50\n" - b"CONTAINER2,TABULATOR2,BATCH3,50\n" - b"CONTAINER2,TABULATOR2,BATCH4,50\n" - b"CONTAINER3,TABULATOR1,BATCH5,50\n" - b"CONTAINER4,TABULATOR1,BATCH6,50\n" - b"CONTAINER5,TABULATOR2,BATCH5,50\n" - b"CONTAINER6,TABULATOR2,BATCH6,50\n" - b"CONTAINER7,TABULATOR1,BATCH7,50\n" - b"CONTAINER8,TABULATOR1,BATCH8,50\n" - b"CONTAINER9,TABULATOR2,BATCH7,50\n" - b"CONTAINER0,TABULATOR2,BATCH8,50\n" - ), - "manifest.csv", - ) - }, + upload_ballot_manifest( + client, + io.BytesIO( + b"Container,Tabulator,Batch Name,Number of Ballots\n" + b"CONTAINER1,TABULATOR1,BATCH1,50\n" + b"CONTAINER1,TABULATOR1,BATCH2,50\n" + b"CONTAINER1,TABULATOR2,BATCH1,50\n" + b"CONTAINER1,TABULATOR2,BATCH2,50\n" + b"CONTAINER2,TABULATOR1,BATCH3,50\n" + b"CONTAINER2,TABULATOR1,BATCH4,50\n" + b"CONTAINER2,TABULATOR2,BATCH3,50\n" + b"CONTAINER2,TABULATOR2,BATCH4,50\n" + b"CONTAINER3,TABULATOR1,BATCH5,50\n" + b"CONTAINER4,TABULATOR1,BATCH6,50\n" + b"CONTAINER5,TABULATOR2,BATCH5,50\n" + b"CONTAINER6,TABULATOR2,BATCH6,50\n" + b"CONTAINER7,TABULATOR1,BATCH7,50\n" + b"CONTAINER8,TABULATOR1,BATCH8,50\n" + b"CONTAINER9,TABULATOR2,BATCH7,50\n" + b"CONTAINER0,TABULATOR2,BATCH8,50\n" + ), + election_id, + jurisdiction_ids[0], ) - assert_ok(rv) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Tabulator,Batch Name,Number of Ballots\n" - b"TABULATOR1,BATCH1,50\n" - b"TABULATOR1,BATCH2,50\n" - b"TABULATOR2,BATCH1,50\n" - b"TABULATOR2,BATCH2,50" - ), - "manifest.csv", - ) - }, + upload_ballot_manifest( + client, + io.BytesIO( + b"Tabulator,Batch Name,Number of Ballots\n" + b"TABULATOR1,BATCH1,50\n" + b"TABULATOR1,BATCH2,50\n" + b"TABULATOR2,BATCH1,50\n" + b"TABULATOR2,BATCH2,50" + ), + election_id, + jurisdiction_ids[1], ) - assert_ok(rv) @pytest.fixture @@ -89,17 +81,13 @@ def cvrs( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(j1_cvr.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + upload_cvrs( + client, + io.BytesIO(j1_cvr.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) - assert_ok(rv) j2_cvr_lines = [ f"TABULATOR{tabulator},BATCH{batch},{ballot},{tabulator}-{batch}-{ballot},x,x,0,0,0,0,0" @@ -115,17 +103,13 @@ def cvrs( [f"{i},{line}" for i, line in enumerate(j2_cvr_lines)] ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(j2_cvr.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + upload_cvrs( + client, + io.BytesIO(j2_cvr.encode()), + election_id, + jurisdiction_ids[1], + "DOMINION", ) - assert_ok(rv) def test_ballot_comparison_container_manifest( @@ -298,25 +282,23 @@ def test_ballot_comparison_manifest_missing_tabulator( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Container,Batch Name,Number of Ballots\n" - b"A,1,20\n" - b"A,2,20\n" - b"A,1,20\n" - b"A,2,20\n" - b"B,3,20\n" - b"B,4,20\n" - b"B,3,20\n" - b"B,4,20" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO( + b"Container,Batch Name,Number of Ballots\n" + b"A,1,20\n" + b"A,2,20\n" + b"A,1,20\n" + b"A,2,20\n" + b"B,3,20\n" + b"B,4,20\n" + b"B,3,20\n" + b"B,4,20" + ), + election_id, + jurisdiction_ids[0], ) + assert_ok(rv) rv = client.get( @@ -325,7 +307,10 @@ def test_ballot_comparison_manifest_missing_tabulator( compare_json( json.loads(rv.data), { - "file": {"name": "manifest.csv", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("manifest"), + "uploadedAt": assert_is_date, + }, "processing": { "completedAt": assert_is_date, "error": "Missing required column: Tabulator.", @@ -344,17 +329,14 @@ def test_ballot_comparison_manifest_unexpected_cvr_column( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Container,Tabulator,Batch Name,Number of Ballots,CVR\n" - b"CONTAINER1,TABULATOR1,BATCH1,50,Yes\n" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO( + b"Container,Tabulator,Batch Name,Number of Ballots,CVR\n" + b"CONTAINER1,TABULATOR1,BATCH1,50,Yes\n" + ), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -364,7 +346,10 @@ def test_ballot_comparison_manifest_unexpected_cvr_column( compare_json( json.loads(rv.data), { - "file": {"name": "manifest.csv", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("manifest"), + "uploadedAt": assert_is_date, + }, "processing": { "completedAt": assert_is_date, "error": "Found unexpected columns. Allowed columns: Batch Name, Container, Number of Ballots, Tabulator.", diff --git a/server/tests/ballot_comparison/test_contest_name_standardizations.py b/server/tests/ballot_comparison/test_contest_name_standardizations.py index e39043cb1..541f2a8e6 100644 --- a/server/tests/ballot_comparison/test_contest_name_standardizations.py +++ b/server/tests/ballot_comparison/test_contest_name_standardizations.py @@ -237,15 +237,12 @@ def test_standardize_contest_names_cvr_change( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(TEST_CVRS.replace("Contest 1", "Contest A").encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(TEST_CVRS.replace("Contest 1", "Contest A").encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert_ok(rv) diff --git a/server/tests/ballot_comparison/test_cvrs.py b/server/tests/ballot_comparison/test_cvrs.py index ff8759e37..c6dc0fe1c 100644 --- a/server/tests/ballot_comparison/test_cvrs.py +++ b/server/tests/ballot_comparison/test_cvrs.py @@ -1,10 +1,9 @@ import io, json -from typing import BinaryIO, Dict, List, TypedDict, Tuple +from typing import List, TypedDict, Tuple from flask.testing import FlaskClient from ...models import * # pylint: disable=wildcard-import from ..helpers import * # pylint: disable=wildcard-import -from ...util.file import zip_files from .conftest import TEST_CVRS @@ -34,15 +33,12 @@ def test_dominion_cvr_upload( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(TEST_CVRS.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(TEST_CVRS.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert_ok(rv) @@ -56,7 +52,7 @@ def test_dominion_cvr_upload( json.loads(rv.data), { "file": { - "name": "cvrs.csv", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, "cvrFileType": "DOMINION", }, @@ -103,7 +99,7 @@ def test_dominion_cvr_upload( jurisdictions[0]["cvrs"], { "file": { - "name": "cvrs.csv", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, "cvrFileType": "DOMINION", }, @@ -124,7 +120,7 @@ def test_dominion_cvr_upload( f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs/csv" ) assert rv.status_code == 200 - assert rv.headers["Content-Disposition"] == 'attachment; filename="cvrs.csv"' + assert rv.headers["Content-Disposition"].startswith('attachment; filename="cvrs') assert rv.data == TEST_CVRS.encode() @@ -160,15 +156,12 @@ def test_cvrs_counting_group( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(COUNTING_GROUP_CVR.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(COUNTING_GROUP_CVR.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert_ok(rv) @@ -187,7 +180,7 @@ def test_cvrs_counting_group( json.loads(rv.data), { "file": { - "name": "cvrs.csv", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, "cvrFileType": "DOMINION", }, @@ -258,15 +251,12 @@ def test_dominion_cvr_unique_voting_identifier( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(DOMINION_UNIQUE_VOTING_IDENTIFIER_CVR.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(DOMINION_UNIQUE_VOTING_IDENTIFIER_CVR.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert_ok(rv) @@ -285,7 +275,7 @@ def test_dominion_cvr_unique_voting_identifier( json.loads(rv.data), { "file": { - "name": "cvrs.csv", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, "cvrFileType": "DOMINION", }, @@ -354,15 +344,12 @@ def test_dominion_cvrs_with_leading_equal_signs( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(DOMINION_CVRS_WITH_LEADING_EQUAL_SIGNS.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(DOMINION_CVRS_WITH_LEADING_EQUAL_SIGNS.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert_ok(rv) @@ -377,11 +364,12 @@ def test_dominion_cvrs_with_leading_equal_signs( rv = client.get( f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs" ) + print(json.loads(rv.data)) compare_json( json.loads(rv.data), { "file": { - "name": "cvrs.csv", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, "cvrFileType": "DOMINION", }, @@ -406,15 +394,12 @@ def test_cvrs_clear( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(TEST_CVRS.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(TEST_CVRS.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert_ok(rv) @@ -457,29 +442,23 @@ def test_cvrs_replace_as_audit_admin( ): # Check that AA can also get/put/clear batch tallies set_logged_in_user(client, UserType.AUDIT_ADMIN, DEFAULT_AA_EMAIL) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(TEST_CVRS.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(TEST_CVRS.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert_ok(rv) file_id = Jurisdiction.query.get(jurisdiction_ids[0]).cvr_file_id - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO("\n".join(TEST_CVRS.splitlines()[:-2]).encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO("\n".join(TEST_CVRS.splitlines()[:-2]).encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert_ok(rv) @@ -517,16 +496,16 @@ def test_cvrs_upload_missing_file( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={}, + rv = client.post( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs/upload-complete", + json={}, ) assert rv.status_code == 400 assert json.loads(rv.data) == { "errors": [ { "errorType": "Bad Request", - "message": "Missing required file parameter 'cvrs'", + "message": "CVR file type is required", } ] } @@ -541,10 +520,12 @@ def test_cvrs_upload_bad_csv( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": (io.BytesIO(b"not a CSV file"), "random.txt"), + rv = client.post( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs/upload-complete", + json={ + "storagePathKey": "random.txt", + "fileName": "random.txt", + "fileType": "text/plain", "cvrFileType": "DOMINION", }, ) @@ -575,14 +556,12 @@ def test_cvrs_wrong_audit_type( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(TEST_CVRS.encode()), - "cvrs.csv", - ) - }, + rv = upload_cvrs( + client, + io.BytesIO(TEST_CVRS.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert rv.status_code == 409 assert json.loads(rv.data) == { @@ -603,14 +582,12 @@ def test_cvrs_before_manifests( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(TEST_CVRS.encode()), - "cvrs.csv", - ) - }, + rv = upload_cvrs( + client, + io.BytesIO(TEST_CVRS.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert rv.status_code == 409 assert json.loads(rv.data) == { @@ -657,15 +634,12 @@ def test_cvrs_newlines( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(NEWLINE_CVR.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(NEWLINE_CVR.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert_ok(rv) @@ -684,7 +658,7 @@ def test_cvrs_newlines( json.loads(rv.data), { "file": { - "name": "cvrs.csv", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, "cvrFileType": "DOMINION", }, @@ -1006,15 +980,12 @@ def test_invalid_cvrs( ] for invalid_cvr, expected_error, cvr_file_type in invalid_cvrs: - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(invalid_cvr.encode()), - "cvrs.csv", - ), - "cvrFileType": cvr_file_type, - }, + rv = upload_cvrs( + client, + io.BytesIO(invalid_cvr.encode()), + election_id, + jurisdiction_ids[0], + cvr_file_type, ) assert_ok(rv) @@ -1025,7 +996,7 @@ def test_invalid_cvrs( json.loads(rv.data), { "file": { - "name": "cvrs.csv", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, "cvrFileType": cvr_file_type, }, @@ -1059,20 +1030,18 @@ def test_cvr_reprocess_after_manifest_reupload( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Tabulator,Batch Name,Number of Ballots\n" - b"TABULATOR2,BATCH2,6\n" - b"TABULATOR1,BATCH1,3\n" - b"TABULATOR1,BATCH2,3" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO( + b"Tabulator,Batch Name,Number of Ballots\n" + b"TABULATOR2,BATCH2,6\n" + b"TABULATOR1,BATCH1,3\n" + b"TABULATOR1,BATCH2,3" + ), + election_id, + jurisdiction_ids[0], ) + assert_ok(rv) set_logged_in_user(client, UserType.AUDIT_ADMIN, DEFAULT_AA_EMAIL) @@ -1091,7 +1060,7 @@ def test_cvr_reprocess_after_manifest_reupload( json.loads(rv.data), { "file": { - "name": "cvrs.csv", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, "cvrFileType": "DOMINION", }, @@ -1115,20 +1084,17 @@ def test_cvr_reprocess_after_manifest_reupload( assert Jurisdiction.query.get(jurisdiction_ids[0]).cvr_contests_metadata is None # Fix the manifest - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Tabulator,Batch Name,Number of Ballots\n" - b"TABULATOR1,BATCH1,3\n" - b"TABULATOR1,BATCH2,3\n" - b"TABULATOR2,BATCH1,3\n" - b"TABULATOR2,BATCH2,6" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO( + b"Tabulator,Batch Name,Number of Ballots\n" + b"TABULATOR1,BATCH1,3\n" + b"TABULATOR1,BATCH2,3\n" + b"TABULATOR2,BATCH1,3\n" + b"TABULATOR2,BATCH2,6" + ), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -1148,7 +1114,7 @@ def test_cvr_reprocess_after_manifest_reupload( json.loads(rv.data), { "file": { - "name": "cvrs.csv", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, "cvrFileType": "DOMINION", }, @@ -1208,15 +1174,12 @@ def test_clearballot_cvr_upload( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(CLEARBALLOT_CVRS.encode()), - "cvrs.csv", - ), - "cvrFileType": "CLEARBALLOT", - }, + rv = upload_cvrs( + client, + io.BytesIO(CLEARBALLOT_CVRS.encode()), + election_id, + jurisdiction_ids[0], + "CLEARBALLOT", ) assert_ok(rv) @@ -1235,7 +1198,7 @@ def test_clearballot_cvr_upload( json.loads(rv.data), { "file": { - "name": "cvrs.csv", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, "cvrFileType": "CLEARBALLOT", }, @@ -1284,15 +1247,12 @@ def test_clearballot_cvr_upload_invalid( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(CLEARBALLOT_CVRS_INVALID.encode()), - "cvrs.csv", - ), - "cvrFileType": "CLEARBALLOT", - }, + rv = upload_cvrs( + client, + io.BytesIO(CLEARBALLOT_CVRS_INVALID.encode()), + election_id, + jurisdiction_ids[0], + "CLEARBALLOT", ) assert_ok(rv) @@ -1312,7 +1272,7 @@ def test_clearballot_cvr_upload_invalid( { "file": { "cvrFileType": "CLEARBALLOT", - "name": "cvrs.csv", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, }, "processing": { @@ -1495,9 +1455,13 @@ def test_ess_cvr_upload( ) for cvrs in test_cases: - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={"cvrs": cvrs, "cvrFileType": "ESS"}, + rv = upload_cvrs( + client, + zip_cvrs(cvrs), + election_id, + jurisdiction_ids[0], + "ESS", + "application/zip", ) assert_ok(rv) @@ -1509,7 +1473,7 @@ def test_ess_cvr_upload( { "file": { "cvrFileType": "ESS", - "name": "cvr-files.zip", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, }, "processing": { @@ -1843,9 +1807,13 @@ def replace_line(string: str, line: int, new_line: str) -> str: ) for invalid_cvrs, expected_error in test_cases: - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={"cvrs": invalid_cvrs, "cvrFileType": "ESS"}, + rv = upload_cvrs( + client, + zip_cvrs(invalid_cvrs), + election_id, + jurisdiction_ids[0], + "ESS", + "application/zip", ) assert_ok(rv) @@ -1857,7 +1825,7 @@ def replace_line(string: str, line: int, new_line: str) -> str: { "file": { "cvrFileType": "ESS", - "name": "cvr-files.zip", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, }, "processing": { @@ -1902,9 +1870,13 @@ def test_ess_cvr_upload_cvr_file_with_tabulator_cvr_column( (io.BytesIO(ESS_BALLOTS_2.encode()), "ess_ballots_2.csv"), ] - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={"cvrs": cvrs, "cvrFileType": "ESS"}, + rv = upload_cvrs( + client, + zip_cvrs(cvrs), + election_id, + jurisdiction_ids[0], + "ESS", + "application/zip", ) assert_ok(rv) @@ -1916,7 +1888,7 @@ def test_ess_cvr_upload_cvr_file_with_tabulator_cvr_column( { "file": { "cvrFileType": "ESS", - "name": "cvr-files.zip", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, }, "processing": { @@ -1932,9 +1904,13 @@ def test_ess_cvr_upload_cvr_file_with_tabulator_cvr_column( }, ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={"cvrs": cvrs_with_override_cvr_file_name, "cvrFileType": "ESS"}, + rv = upload_cvrs( + client, + zip_cvrs(cvrs_with_override_cvr_file_name), + election_id, + jurisdiction_ids[0], + "ESS", + "application/zip", ) assert_ok(rv) @@ -1946,7 +1922,7 @@ def test_ess_cvr_upload_cvr_file_with_tabulator_cvr_column( { "file": { "cvrFileType": "ESS", - "name": "cvr-files.zip", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, }, "processing": { @@ -2170,16 +2146,6 @@ def build_contest(contest_name: str, choice_names: List[str]): """ -def zip_hart_cvrs(cvrs: List[str]): - files: Dict[str, BinaryIO] = { - f"cvr-{i}.xml": io.BytesIO(cvr.encode()) for i, cvr in enumerate(cvrs) - } - # There's usually a WriteIns directory in the zip file - simulate that to - # make sure it gets skipped - files["WriteIns"] = io.BytesIO() - return io.BytesIO(zip_files(files).read()) - - def test_hart_cvr_upload( client: FlaskClient, election_id: str, @@ -2188,12 +2154,13 @@ def test_hart_cvr_upload( snapshot, ): # Upload CVRs - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": [(zip_hart_cvrs(HART_CVRS), "cvrs.zip")], - "cvrFileType": "HART", - }, + rv = upload_cvrs( + client, + zip_hart_cvrs(HART_CVRS), + election_id, + jurisdiction_ids[0], + "HART", + "application/zip", ) assert_ok(rv) @@ -2212,7 +2179,7 @@ def test_hart_cvr_upload( json.loads(rv.data), { "file": { - "name": "cvr-files.zip", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, "cvrFileType": "HART", }, @@ -2366,21 +2333,25 @@ class TestCase(TypedDict): ] for test_case in test_cases: + print("on test case ", test_case) scanned_ballot_information_files = [ (string_to_bytes_io(file_contents), f"scanned-ballot-information-{i}.csv") for i, file_contents in enumerate( test_case["scanned_ballot_information_file_contents"] ) ] - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": [ + rv = upload_cvrs( + client, + zip_cvrs( + [ (zip_hart_cvrs(HART_CVRS), "cvrs.zip"), *scanned_ballot_information_files, - ], - "cvrFileType": "HART", - }, + ] + ), + election_id, + jurisdiction_ids[0], + "HART", + "application/zip", ) assert_ok(rv) @@ -2392,7 +2363,7 @@ class TestCase(TypedDict): { "file": { "cvrFileType": "HART", - "name": "cvr-files.zip", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, }, "processing": { @@ -2453,7 +2424,7 @@ def test_hart_cvr_upload_with_duplicate_batch_names( ) class TestCase(TypedDict): - files: List[Tuple[BinaryIO, str]] + files: List[Tuple[io.BytesIO, str]] expected_processing_status: ProcessingStatus expected_processing_error: Optional[str] @@ -2540,12 +2511,13 @@ class TestCase(TypedDict): ] for test_case in test_cases: - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": test_case["files"], - "cvrFileType": "HART", - }, + rv = upload_cvrs( + client, + zip_cvrs(test_case["files"]), + election_id, + jurisdiction_ids[0], + "HART", + "application/zip", ) assert_ok(rv) @@ -2557,7 +2529,7 @@ class TestCase(TypedDict): { "file": { "cvrFileType": "HART", - "name": "cvr-files.zip", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, }, "processing": { @@ -2623,12 +2595,14 @@ def test_hart_cvr_upload_no_batch_match( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) for invalid_cvr, expected_error in invalid_cvrs: - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": [(zip_hart_cvrs(invalid_cvr), "cvrs.zip")], - "cvrFileType": "HART", - }, + rv = upload_cvrs( + client, + zip_hart_cvrs(invalid_cvr), + election_id, + jurisdiction_ids[0], + "HART", + "application/zip", + "cvrs.zip", ) assert_ok(rv) @@ -2639,7 +2613,7 @@ def test_hart_cvr_upload_no_batch_match( json.loads(rv.data), { "file": { - "name": "cvr-files.zip", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, "cvrFileType": "HART", }, @@ -2706,12 +2680,13 @@ def test_hart_cvr_upload_no_tabulator_plus_batch_match( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) for cvr_upload, expected_error in cvr_uploads: - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": cvr_upload, - "cvrFileType": "HART", - }, + rv = upload_cvrs( + client, + zip_cvrs(cvr_upload), + election_id, + jurisdiction_ids[0], + "HART", + "application/zip", ) assert_ok(rv) @@ -2722,7 +2697,7 @@ def test_hart_cvr_upload_no_tabulator_plus_batch_match( json.loads(rv.data), { "file": { - "name": "cvr-files.zip", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, "cvrFileType": "HART", }, @@ -2749,80 +2724,122 @@ def test_hart_cvr_upload_basic_input_validation( ) class TestCase(TypedDict): - cvrs: List + cvrs: io.BytesIO + file_type: str expected_status_code: int expected_response: Any test_cases: List[TestCase] = [ { - "cvrs": [(zip_hart_cvrs(HART_CVRS), "cvrs.csv")], + "cvrs": zip_hart_cvrs(HART_CVRS), + "file_type": "text/csv", "expected_status_code": 400, "expected_response": { "errors": [ { "errorType": "Bad Request", - "message": "Please submit at least one ZIP file.", + "message": "Please submit a valid ZIP file.", } ] }, }, { - "cvrs": [ - (zip_hart_cvrs(HART_CVRS), "cvrs.csv"), - ( - string_to_bytes_io(HART_SCANNED_BALLOT_INFORMATION), - "scanned-ballot-information.csv", - ), - ], - "expected_status_code": 400, - "expected_response": { - "errors": [ - { - "errorType": "Bad Request", - "message": "Please submit at least one ZIP file.", - } - ] - }, + "cvrs": zip_hart_cvrs(HART_CVRS), + "file_type": "application/x-zip-compressed", # Verify that the Windows ZIP mimetype works + "expected_status_code": 200, + "expected_response": {"status": "ok"}, }, + ] + + for test_case in test_cases: + rv = upload_cvrs( + client, + test_case["cvrs"], + election_id, + jurisdiction_ids[0], + "HART", + test_case["file_type"], + ) + assert rv.status_code == test_case["expected_status_code"] + assert json.loads(rv.data) == test_case["expected_response"] + + +def test_hart_cvr_upload_processing_validation( + client: FlaskClient, + election_id: str, + jurisdiction_ids: List[str], + hart_manifests, # pylint: disable=unused-argument +): + set_logged_in_user( + client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) + ) + + class TestCase(TypedDict): + cvrs: io.BytesIO + expected_status_code: int + expected_response: Any + + test_cases: List[TestCase] = [ { - "cvrs": [ - (zip_hart_cvrs(HART_CVRS), "cvrs.zip"), - ( - string_to_bytes_io(HART_SCANNED_BALLOT_INFORMATION), - "scanned-ballot-information.jpg", - ), - ], - "expected_status_code": 400, - "expected_response": { - "errors": [ - { - "errorType": "Bad Request", - "message": "Please submit only ZIP files and CSVs.", - } + "cvrs": zip_cvrs( + [ + (zip_hart_cvrs(HART_CVRS), "cvrs.csv"), + ( + string_to_bytes_io(HART_SCANNED_BALLOT_INFORMATION), + "scanned-ballot-information.csv", + ), ] - }, + ), + "expected_status_code": 400, + "expected_response": "Expected first line of scanned ballot information CSV to contain '#FormatVersion'.", }, { - "cvrs": [ - ( - zip_hart_cvrs(HART_CVRS), - "cvrs.zip", - # Verify that the Windows ZIP mimetype works - "application/x-zip-compressed", - ), - ], - "expected_status_code": 200, - "expected_response": {"status": "ok"}, + "cvrs": zip_cvrs( + [ + (zip_hart_cvrs(HART_CVRS), "cvrs.zip"), + ( + string_to_bytes_io(HART_SCANNED_BALLOT_INFORMATION), + "scanned-ballot-information.jpg", + ), + ] + ), + "expected_status_code": 400, + "expected_response": "Unsupported file type. Expected either a ZIP file or a CSV file, but found scanned-ballot-information.jpg.", }, ] for test_case in test_cases: - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={"cvrFileType": "HART", "cvrs": test_case["cvrs"]}, + rv = upload_cvrs( + client, + test_case["cvrs"], + election_id, + jurisdiction_ids[0], + "HART", + "application/zip", + ) + # these test cases will fail when being processed not uploaded + assert_ok(rv) + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs" + ) + compare_json( + json.loads(rv.data), + { + "file": { + "name": asserts_startswith("cvrs"), + "uploadedAt": assert_is_date, + "cvrFileType": "HART", + }, + "processing": { + "status": ProcessingStatus.ERRORED, + "startedAt": assert_is_date, + "completedAt": assert_is_date, + "error": test_case["expected_response"], + "workProgress": 0, + "workTotal": assert_is_int, + }, + }, ) - assert rv.status_code == test_case["expected_status_code"] - assert json.loads(rv.data) == test_case["expected_response"] def test_cvrs_unexpected_error( @@ -2847,15 +2864,12 @@ def test_cvrs_unexpected_error( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(cvrs.encode()), - "cvrs.csv", - ), - "cvrFileType": "DOMINION", - }, + rv = upload_cvrs( + client, + io.BytesIO(cvrs.encode()), + election_id, + jurisdiction_ids[0], + "DOMINION", ) assert_ok(rv) @@ -2866,7 +2880,7 @@ def test_cvrs_unexpected_error( json.loads(rv.data), { "file": { - "name": "cvrs.csv", + "name": asserts_startswith("cvrs"), "uploadedAt": assert_is_date, "cvrFileType": "DOMINION", }, @@ -2899,15 +2913,12 @@ def test_cvr_invalid_file_type( set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs", - data={ - "cvrs": ( - io.BytesIO(TEST_CVRS.encode()), - "cvrs.csv", - ), - "cvrFileType": "WRONG", - }, + rv = upload_cvrs( + client, + io.BytesIO(TEST_CVRS.encode()), + election_id, + jurisdiction_ids[0], + "WRONG", ) assert rv.status_code == 400 assert json.loads(rv.data) == { @@ -2918,3 +2929,63 @@ def test_cvr_invalid_file_type( } ] } + + +def test_cvrs_get_upload_url_missing_file_type( + client: FlaskClient, election_id: str, jurisdiction_ids: List[str] +): + set_logged_in_user( + client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) + ) + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs/upload-url" + ) + assert rv.status_code == 400 + assert json.loads(rv.data) == { + "errors": [ + { + "errorType": "Bad Request", + "message": "Missing expected query parameter: fileType", + } + ] + } + + +def test_cvrs_get_upload_url( + client: FlaskClient, election_id: str, jurisdiction_ids: List[str] +): + allowed_users = [ + (UserType.JURISDICTION_ADMIN, default_ja_email(election_id)), + (UserType.AUDIT_ADMIN, DEFAULT_AA_EMAIL), + ] + for user, email in allowed_users: + set_logged_in_user(client, user, email) + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs/upload-url", + query_string={"fileType": "text/csv", "cvrFileType": "DOMINION"}, + ) + assert rv.status_code == 200 + + response_data = json.loads(rv.data) + expected_url = "/api/file-upload" + + assert response_data["url"] == expected_url + assert response_data["fields"]["key"].startswith( + f"audits/{election_id}/jurisdictions/{jurisdiction_ids[0]}/cvrs_" + ) + assert response_data["fields"]["key"].endswith(".csv") + + rv = client.get( + f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/cvrs/upload-url", + query_string={"fileType": "text/csv", "cvrFileType": "HART"}, + ) + assert rv.status_code == 200 + + response_data = json.loads(rv.data) + expected_url = "/api/file-upload" + + assert response_data["url"] == expected_url + assert response_data["fields"]["key"].startswith( + f"audits/{election_id}/jurisdictions/{jurisdiction_ids[0]}/cvrs_" + ) + assert response_data["fields"]["key"].endswith(".zip") diff --git a/server/tests/ballot_comparison/test_standardized_contests.py b/server/tests/ballot_comparison/test_standardized_contests.py index 072cfec3e..f25584820 100644 --- a/server/tests/ballot_comparison/test_standardized_contests.py +++ b/server/tests/ballot_comparison/test_standardized_contests.py @@ -15,14 +15,10 @@ def test_upload_standardized_contests( 'Contest 2,"J1, J3"\n' "Contest 3,J2 \n" ) - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO(standardized_contests_file.encode()), - "standardized-contests.csv", - ) - }, + rv = upload_standardized_contests( + client, + io.BytesIO(standardized_contests_file.encode()), + election_id, ) assert_ok(rv) @@ -31,7 +27,7 @@ def test_upload_standardized_contests( json.loads(rv.data), { "file": { - "name": "standardized-contests.csv", + "name": asserts_startswith("standardizedContests"), "uploadedAt": assert_is_date, }, "processing": { @@ -54,9 +50,8 @@ def test_upload_standardized_contests( ] rv = client.get(f"/api/election/{election_id}/standardized-contests/file/csv") - assert ( - rv.headers["Content-Disposition"] - == 'attachment; filename="standardized-contests.csv"' + assert rv.headers["Content-Disposition"].startswith( + 'attachment; filename="standardizedContests' ) assert rv.data.decode("utf-8") == standardized_contests_file @@ -71,19 +66,15 @@ def test_download_standardized_contests_file_before_upload( def test_standardized_contests_replace( client: FlaskClient, election_id: str, jurisdiction_ids: List[str] ): - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO( - b"Contest Name,Jurisdictions\n" - b"Contest 1,all\n" - b'Contest 2,"J1, J3"\n' - b"Contest 3,J2 \n" - ), - "standardized-contests.csv", - ) - }, + rv = upload_standardized_contests( + client, + io.BytesIO( + b"Contest Name,Jurisdictions\n" + b"Contest 1,all\n" + b'Contest 2,"J1, J3"\n' + b"Contest 3,J2 \n" + ), + election_id, ) assert_ok(rv) @@ -91,14 +82,10 @@ def test_standardized_contests_replace( file_id = election.standardized_contests_file_id standardized_contests = election.standardized_contests - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO(b"Contest Name,Jurisdictions\n" b"Contest 4,all\n"), - "standardized-contests.csv", - ) - }, + rv = upload_standardized_contests( + client, + io.BytesIO(b"Contest Name,Jurisdictions\n" b"Contest 4,all\n"), + election_id, ) assert_ok(rv) @@ -120,17 +107,13 @@ def test_standardized_contests_bad_jurisdiction( election_id: str, jurisdiction_ids: List[str], # pylint: disable=unused-argument ): - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO( - b"Contest Name,Jurisdictions\n" - b'Contest 1,"J1,not a real jurisdiction,another bad one"\n"' - ), - "standardized-contests.csv", - ) - }, + rv = upload_standardized_contests( + client, + io.BytesIO( + b"Contest Name,Jurisdictions\n" + b'Contest 1,"J1,not a real jurisdiction,another bad one"\n"' + ), + election_id, ) assert_ok(rv) @@ -139,7 +122,7 @@ def test_standardized_contests_bad_jurisdiction( json.loads(rv.data), { "file": { - "name": "standardized-contests.csv", + "name": asserts_startswith("standardizedContests"), "uploadedAt": assert_is_date, }, "processing": { @@ -160,14 +143,10 @@ def test_standardized_contests_no_jurisdictions( election_id: str, jurisdiction_ids: List[str], # pylint: disable=unused-argument ): - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO(b"Contest Name,Jurisdictions\n" b"Contest 1,"), - "standardized-contests.csv", - ) - }, + rv = upload_standardized_contests( + client, + io.BytesIO(b"Contest Name,Jurisdictions\n" b"Contest 1,"), + election_id, ) assert_ok(rv) @@ -176,7 +155,7 @@ def test_standardized_contests_no_jurisdictions( json.loads(rv.data), { "file": { - "name": "standardized-contests.csv", + "name": asserts_startswith("standardizedContests"), "uploadedAt": assert_is_date, }, "processing": { @@ -197,16 +176,16 @@ def test_standardized_contests_missing_file( election_id: str, jurisdiction_ids: List[str], # pylint: disable=unused-argument ): - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={}, + rv = client.post( + f"/api/election/{election_id}/standardized-contests/file/upload-complete", + json={}, ) assert rv.status_code == 400 assert json.loads(rv.data) == { "errors": [ { "errorType": "Bad Request", - "message": "Missing required file parameter 'standardized-contests'", + "message": "Missing required JSON parameter: storagePathKey", } ] } @@ -217,13 +196,12 @@ def test_standardized_contests_bad_csv( election_id: str, jurisdiction_ids: List[str], # pylint: disable=unused-argument ): - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO(b"not a csv"), - "standardized-contests.txt", - ) + rv = client.post( + f"/api/election/{election_id}/standardized-contests/file/upload-complete", + json={ + "storagePathKey": "test_dir/random.txt", + "fileName": "random.txt", + "fileType": "text/plain", }, ) assert rv.status_code == 400 @@ -249,21 +227,17 @@ def test_standardized_contests_wrong_audit_type( db_session.add(election) db_session.commit() - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO(b"Contest Name,Jurisdictions\n" b"Contest 1,all\n"), - "standardized-contests.csv", - ) - }, + rv = upload_standardized_contests( + client, + io.BytesIO(b"Contest Name,Jurisdictions\n" b"Contest 1,all\n"), + election_id, ) assert rv.status_code == 409 assert json.loads(rv.data) == { "errors": [ { "errorType": "Conflict", - "message": "Can't upload CVR file for this audit type.", + "message": "Can't upload standardized contests file for this audit type.", } ] } @@ -272,14 +246,10 @@ def test_standardized_contests_wrong_audit_type( def test_standardized_contests_before_jurisdictions( client: FlaskClient, election_id: str ): - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO(b"Contest Name,Jurisdictions\n" b"Contest 1,all\n"), - "standardized-contests.csv", - ) - }, + rv = upload_standardized_contests( + client, + io.BytesIO(b"Contest Name,Jurisdictions\n" b"Contest 1,all\n"), + election_id, ) assert rv.status_code == 409 assert json.loads(rv.data) == { @@ -295,19 +265,15 @@ def test_standardized_contests_before_jurisdictions( def test_standardized_contests_newlines( client: FlaskClient, election_id: str, jurisdiction_ids: List[str] ): - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO( - b"Contest Name,Jurisdictions\n" - b'"Contest\r\n1",all\n' - b'Contest 2,"J1, J3"\n' - b"Contest 3,J2\n" - ), - "standardized-contests.csv", - ) - }, + rv = upload_standardized_contests( + client, + io.BytesIO( + b"Contest Name,Jurisdictions\n" + b'"Contest\r\n1",all\n' + b'Contest 2,"J1, J3"\n' + b"Contest 3,J2\n" + ), + election_id, ) assert_ok(rv) @@ -325,19 +291,15 @@ def test_standardized_contests_newlines( def test_standardized_contests_dominion_vote_for( client: FlaskClient, election_id: str, jurisdiction_ids: List[str] ): - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO( - b"Contest Name,Jurisdictions\n" - b'"Contest\r\n1 (Vote For=2)",all\n' - b'Contest 2,"J1, J3"\n' - b"Contest 3,J2\n" - ), - "standardized-contests.csv", - ) - }, + rv = upload_standardized_contests( + client, + io.BytesIO( + b"Contest Name,Jurisdictions\n" + b'"Contest\r\n1 (Vote For=2)",all\n' + b'Contest 2,"J1, J3"\n' + b"Contest 3,J2\n" + ), + election_id, ) assert_ok(rv) @@ -361,14 +323,10 @@ def test_standardized_contests_change_jurisdictions_file( 'Contest 2,"J1, J3"\n' "Contest 3,all \n" ) - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO(standardized_contests_file.encode()), - "standardized-contests.csv", - ) - }, + rv = upload_standardized_contests( + client, + io.BytesIO(standardized_contests_file.encode()), + election_id, ) assert_ok(rv) @@ -388,20 +346,16 @@ def test_standardized_contests_change_jurisdictions_file( assert_ok(rv) # Remove a jurisdiction that isn't referenced directly in standardized contests - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO( - ( - "Jurisdiction,Admin Email\n" - f"J3,j3-{election_id}@example.com\n" - f"J1,{default_ja_email(election_id)}\n" - ).encode() - ), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO( + ( + "Jurisdiction,Admin Email\n" + f"J3,j3-{election_id}@example.com\n" + f"J1,{default_ja_email(election_id)}\n" + ).encode() + ), + election_id, ) assert_ok(rv) @@ -423,19 +377,14 @@ def test_standardized_contests_change_jurisdictions_file( ] # Now remove a jurisdiction that is referenced directly in standardized contests - rv = client.put( - f"/api/election/{election_id}/jurisdiction/file", - data={ - "jurisdictions": ( - io.BytesIO( - ( - "Jurisdiction,Admin Email\n" - f"J1,{default_ja_email(election_id)}\n" - ).encode() - ), - "jurisdictions.csv", - ) - }, + rv = upload_jurisdictions_file( + client, + io.BytesIO( + ( + "Jurisdiction,Admin Email\n" f"J1,{default_ja_email(election_id)}\n" + ).encode() + ), + election_id, ) assert_ok(rv) @@ -449,7 +398,7 @@ def test_standardized_contests_change_jurisdictions_file( json.loads(rv.data), { "file": { - "name": "standardized-contests.csv", + "name": asserts_startswith("standardizedContests"), "uploadedAt": assert_is_date, }, "processing": { @@ -468,14 +417,10 @@ def test_standardized_contests_parse_all( standardized_contests_file = ( "Contest Name,Jurisdictions\n" + "Contest 1,All\n" + "Contest 2, aLL \n" ) - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO(standardized_contests_file.encode()), - "standardized-contests.csv", - ) - }, + rv = upload_standardized_contests( + client, + io.BytesIO(standardized_contests_file.encode()), + election_id, ) assert_ok(rv) @@ -504,14 +449,10 @@ def test_reupload_standardized_contests_after_contests_selected( 'Contest 2,"J1, J3"\n' "Contest 3,J2 \n" ) - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO(standardized_contests_file.encode()), - "standardized-contests.csv", - ) - }, + rv = upload_standardized_contests( + client, + io.BytesIO(standardized_contests_file.encode()), + election_id, ) assert_ok(rv) @@ -577,14 +518,10 @@ def test_reupload_standardized_contests_after_contests_selected( standardized_contests_file = ( "Contest Name,Jurisdictions\n" + 'Contest 1,"J1,J2"\n' + "Contest 3,J2 \n" ) - rv = client.put( - f"/api/election/{election_id}/standardized-contests/file", - data={ - "standardized-contests": ( - io.BytesIO(standardized_contests_file.encode()), - "standardized-contests.csv", - ) - }, + rv = upload_standardized_contests( + client, + io.BytesIO(standardized_contests_file.encode()), + election_id, ) assert_ok(rv) @@ -610,3 +547,47 @@ def test_reupload_standardized_contests_after_contests_selected( ] }, ) + + +def test_standardized_contests_get_upload_url_missing_file_type( + client: FlaskClient, election_id: str +): + set_logged_in_user( + client, + UserType.AUDIT_ADMIN, + DEFAULT_AA_EMAIL, + ) + rv = client.get( + f"/api/election/{election_id}/standardized-contests/file/upload-url" + ) + assert rv.status_code == 400 + assert json.loads(rv.data) == { + "errors": [ + { + "errorType": "Bad Request", + "message": "Missing expected query parameter: fileType", + } + ] + } + + +def test_standardized_contests_get_upload_url(client: FlaskClient, election_id: str): + set_logged_in_user( + client, + UserType.AUDIT_ADMIN, + DEFAULT_AA_EMAIL, + ) + rv = client.get( + f"/api/election/{election_id}/standardized-contests/file/upload-url", + query_string={"fileType": "text/csv"}, + ) + assert rv.status_code == 200 + + response_data = json.loads(rv.data) + expected_url = "/api/file-upload" + + assert response_data["url"] == expected_url + assert response_data["fields"]["key"].startswith( + f"audits/{election_id}/standardized_contests_" + ) + assert response_data["fields"]["key"].endswith(".csv") diff --git a/server/tests/batch_comparison/conftest.py b/server/tests/batch_comparison/conftest.py index af06e6e10..92740dcfa 100644 --- a/server/tests/batch_comparison/conftest.py +++ b/server/tests/batch_comparison/conftest.py @@ -66,43 +66,37 @@ def manifests(client: FlaskClient, election_id: str, jurisdiction_ids: List[str] set_logged_in_user( client, UserType.JURISDICTION_ADMIN, default_ja_email(election_id) ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Batch Name,Number of Ballots\n" - b"Batch 1,500\n" - b"Batch 2,500\n" - b"Batch 3,500\n" - b"Batch 4,500\n" - b"Batch 5,100\n" - b"Batch 6,100\n" - b"Batch 7,100\n" - b"Batch 8,100\n" - b"Batch 9,100\n" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO( + b"Batch Name,Number of Ballots\n" + b"Batch 1,500\n" + b"Batch 2,500\n" + b"Batch 3,500\n" + b"Batch 4,500\n" + b"Batch 5,100\n" + b"Batch 6,100\n" + b"Batch 7,100\n" + b"Batch 8,100\n" + b"Batch 9,100\n" + ), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO( - b"Batch Name,Number of Ballots\n" - b"Batch 1,500\n" - b"Batch 2,500\n" - b"Batch 3,500\n" - b"Batch 4,500\n" - b"Batch 5,250\n" - b"Batch 6,250\n" - ), - "manifest.csv", - ) - }, + rv = upload_ballot_manifest( + client, + io.BytesIO( + b"Batch Name,Number of Ballots\n" + b"Batch 1,500\n" + b"Batch 2,500\n" + b"Batch 3,500\n" + b"Batch 4,500\n" + b"Batch 5,250\n" + b"Batch 6,250\n" + ), + election_id, + jurisdiction_ids[1], ) assert_ok(rv) @@ -130,15 +124,13 @@ def batch_tallies( b"Batch 8,100,50,50\n" b"Batch 9,100,50,50\n" ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-tallies", - data={ - "batchTallies": ( - io.BytesIO(batch_tallies_file), - "batchTallies.csv", - ) - }, + rv = upload_batch_tallies( + client, + io.BytesIO(batch_tallies_file), + election_id, + jurisdiction_ids[0], ) + assert_ok(rv) batch_tallies_file = ( b"Batch Name,candidate 1,candidate 2,candidate 3\n" b"Batch 1,500,250,250\n" @@ -148,14 +140,11 @@ def batch_tallies( b"Batch 5,100,50,50\n" b"Batch 6,100,50,50\n" ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/batch-tallies", - data={ - "batchTallies": ( - io.BytesIO(batch_tallies_file), - "batchTallies.csv", - ) - }, + rv = upload_batch_tallies( + client, + io.BytesIO(batch_tallies_file), + election_id, + jurisdiction_ids[1], ) assert_ok(rv) diff --git a/server/tests/batch_comparison/test_batch_comparison.py b/server/tests/batch_comparison/test_batch_comparison.py index 6d319d464..305e72081 100644 --- a/server/tests/batch_comparison/test_batch_comparison.py +++ b/server/tests/batch_comparison/test_batch_comparison.py @@ -103,14 +103,8 @@ def test_batch_comparison_too_many_votes( b"Batch 5,100,50,50\n" b"Batch 6,100,50,50\n" ) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[1]}/batch-tallies", - data={ - "batchTallies": ( - io.BytesIO(batch_tallies_file), - "batchTallies.csv", - ) - }, + rv = upload_batch_tallies( + client, io.BytesIO(batch_tallies_file), election_id, jurisdiction_ids[1] ) assert_ok(rv) diff --git a/server/tests/batch_comparison/test_batch_inventory.py b/server/tests/batch_comparison/test_batch_inventory.py index ba624f1b7..845b86c33 100644 --- a/server/tests/batch_comparison/test_batch_inventory.py +++ b/server/tests/batch_comparison/test_batch_inventory.py @@ -179,14 +179,11 @@ def test_batch_inventory_happy_path( compare_json(json.loads(rv.data), {"systemType": CvrFileType.DOMINION}) # Upload CVR file - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/cvr", - data={ - "cvr": ( - io.BytesIO(TEST_CVR.encode()), - "cvrs.csv", - ), - }, + rv = upload_batch_inventory_cvr( + client, + io.BytesIO(TEST_CVR.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -196,7 +193,10 @@ def test_batch_inventory_happy_path( compare_json( json.loads(rv.data), { - "file": {"name": "cvrs.csv", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("batchInventoryCvr"), + "uploadedAt": assert_is_date, + }, "processing": { "status": ProcessingStatus.PROCESSED, "startedAt": assert_is_date, @@ -207,14 +207,11 @@ def test_batch_inventory_happy_path( ) # Upload tabulator status file - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/tabulator-status", - data={ - "tabulatorStatus": ( - io.BytesIO(TEST_TABULATOR_STATUS.encode()), - "tabulator-status.xml", - ), - }, + rv = upload_batch_inventory_tabulator_status( + client, + io.BytesIO(TEST_TABULATOR_STATUS.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -224,7 +221,10 @@ def test_batch_inventory_happy_path( compare_json( json.loads(rv.data), { - "file": {"name": "tabulator-status.xml", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("tabulator-status"), + "uploadedAt": assert_is_date, + }, "processing": { "status": ProcessingStatus.PROCESSED, "startedAt": assert_is_date, @@ -271,14 +271,11 @@ def test_batch_inventory_happy_path( snapshot.assert_match(batch_tallies) # Upload manifest - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO(ballot_manifest.encode()), - "ballot-manifest.csv", - ), - }, + rv = upload_ballot_manifest( + client, + io.BytesIO(ballot_manifest.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -288,7 +285,10 @@ def test_batch_inventory_happy_path( compare_json( json.loads(rv.data), { - "file": {"name": "ballot-manifest.csv", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("manifest"), + "uploadedAt": assert_is_date, + }, "processing": { "status": ProcessingStatus.PROCESSED, "startedAt": assert_is_date, @@ -299,14 +299,11 @@ def test_batch_inventory_happy_path( ) # Upload batch tallies - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-tallies", - data={ - "batchTallies": ( - io.BytesIO(batch_tallies.encode()), - "batch-tallies.csv", - ) - }, + rv = upload_batch_tallies( + client, + io.BytesIO(batch_tallies.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -316,7 +313,10 @@ def test_batch_inventory_happy_path( compare_json( json.loads(rv.data), { - "file": {"name": "batch-tallies.csv", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("batchTallies"), + "uploadedAt": assert_is_date, + }, "processing": { "status": ProcessingStatus.PROCESSED, "startedAt": assert_is_date, @@ -390,14 +390,11 @@ def test_batch_inventory_happy_path_cvrs_with_leading_equal_signs( compare_json(json.loads(rv.data), {"systemType": CvrFileType.DOMINION}) # Upload CVR file - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/cvr", - data={ - "cvr": ( - io.BytesIO(TEST_CVRS_WITH_LEADING_EQUAL_SIGNS.encode()), - "cvrs.csv", - ), - }, + rv = upload_batch_inventory_cvr( + client, + io.BytesIO(TEST_CVRS_WITH_LEADING_EQUAL_SIGNS.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -407,7 +404,10 @@ def test_batch_inventory_happy_path_cvrs_with_leading_equal_signs( compare_json( json.loads(rv.data), { - "file": {"name": "cvrs.csv", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("batchInventoryCvr"), + "uploadedAt": assert_is_date, + }, "processing": { "status": ProcessingStatus.PROCESSED, "startedAt": assert_is_date, @@ -418,14 +418,11 @@ def test_batch_inventory_happy_path_cvrs_with_leading_equal_signs( ) # Upload tabulator status file - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/tabulator-status", - data={ - "tabulatorStatus": ( - io.BytesIO(TEST_TABULATOR_STATUS.encode()), - "tabulator-status.xml", - ), - }, + rv = upload_batch_inventory_tabulator_status( + client, + io.BytesIO(TEST_TABULATOR_STATUS.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -435,7 +432,10 @@ def test_batch_inventory_happy_path_cvrs_with_leading_equal_signs( compare_json( json.loads(rv.data), { - "file": {"name": "tabulator-status.xml", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("tabulator-status"), + "uploadedAt": assert_is_date, + }, "processing": { "status": ProcessingStatus.PROCESSED, "startedAt": assert_is_date, @@ -482,14 +482,11 @@ def test_batch_inventory_happy_path_cvrs_with_leading_equal_signs( snapshot.assert_match(batch_tallies) # Upload manifest - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO(ballot_manifest.encode()), - "ballot-manifest.csv", - ), - }, + rv = upload_ballot_manifest( + client, + io.BytesIO(ballot_manifest.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -499,7 +496,10 @@ def test_batch_inventory_happy_path_cvrs_with_leading_equal_signs( compare_json( json.loads(rv.data), { - "file": {"name": "ballot-manifest.csv", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("manifest"), + "uploadedAt": assert_is_date, + }, "processing": { "status": ProcessingStatus.PROCESSED, "startedAt": assert_is_date, @@ -510,14 +510,11 @@ def test_batch_inventory_happy_path_cvrs_with_leading_equal_signs( ) # Upload batch tallies - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-tallies", - data={ - "batchTallies": ( - io.BytesIO(batch_tallies.encode()), - "batch-tallies.csv", - ) - }, + rv = upload_batch_tallies( + client, + io.BytesIO(batch_tallies.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -527,7 +524,10 @@ def test_batch_inventory_happy_path_cvrs_with_leading_equal_signs( compare_json( json.loads(rv.data), { - "file": {"name": "batch-tallies.csv", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("batchTallies"), + "uploadedAt": assert_is_date, + }, "processing": { "status": ProcessingStatus.PROCESSED, "startedAt": assert_is_date, @@ -601,14 +601,11 @@ def test_batch_inventory_happy_path_multi_contest_batch_comparison( compare_json(json.loads(rv.data), {"systemType": CvrFileType.DOMINION}) # Upload CVR file - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/cvr", - data={ - "cvr": ( - io.BytesIO(TEST_CVR.encode()), - "cvrs.csv", - ), - }, + rv = upload_batch_inventory_cvr( + client, + io.BytesIO(TEST_CVR.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -618,7 +615,10 @@ def test_batch_inventory_happy_path_multi_contest_batch_comparison( compare_json( json.loads(rv.data), { - "file": {"name": "cvrs.csv", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("batchInventoryCvr"), + "uploadedAt": assert_is_date, + }, "processing": { "status": ProcessingStatus.PROCESSED, "startedAt": assert_is_date, @@ -629,14 +629,11 @@ def test_batch_inventory_happy_path_multi_contest_batch_comparison( ) # Upload tabulator status file - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/tabulator-status", - data={ - "tabulatorStatus": ( - io.BytesIO(TEST_TABULATOR_STATUS.encode()), - "tabulator-status.xml", - ), - }, + rv = upload_batch_inventory_tabulator_status( + client, + io.BytesIO(TEST_TABULATOR_STATUS.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -646,7 +643,10 @@ def test_batch_inventory_happy_path_multi_contest_batch_comparison( compare_json( json.loads(rv.data), { - "file": {"name": "tabulator-status.xml", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("tabulator-status"), + "uploadedAt": assert_is_date, + }, "processing": { "status": ProcessingStatus.PROCESSED, "startedAt": assert_is_date, @@ -693,14 +693,11 @@ def test_batch_inventory_happy_path_multi_contest_batch_comparison( snapshot.assert_match(batch_tallies) # Upload manifest - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/ballot-manifest", - data={ - "manifest": ( - io.BytesIO(ballot_manifest.encode()), - "ballot-manifest.csv", - ), - }, + rv = upload_ballot_manifest( + client, + io.BytesIO(ballot_manifest.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -710,7 +707,10 @@ def test_batch_inventory_happy_path_multi_contest_batch_comparison( compare_json( json.loads(rv.data), { - "file": {"name": "ballot-manifest.csv", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("manifest"), + "uploadedAt": assert_is_date, + }, "processing": { "status": ProcessingStatus.PROCESSED, "startedAt": assert_is_date, @@ -721,14 +721,11 @@ def test_batch_inventory_happy_path_multi_contest_batch_comparison( ) # Upload batch tallies - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-tallies", - data={ - "batchTallies": ( - io.BytesIO(batch_tallies.encode()), - "batch-tallies.csv", - ) - }, + rv = upload_batch_tallies( + client, + io.BytesIO(batch_tallies.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -738,7 +735,10 @@ def test_batch_inventory_happy_path_multi_contest_batch_comparison( compare_json( json.loads(rv.data), { - "file": {"name": "batch-tallies.csv", "uploadedAt": assert_is_date}, + "file": { + "name": asserts_startswith("batchTallies"), + "uploadedAt": assert_is_date, + }, "processing": { "status": ProcessingStatus.PROCESSED, "startedAt": assert_is_date, @@ -826,14 +826,11 @@ def test_batch_inventory_invalid_file_uploads( ), ] for invalid_cvr, expected_error in invalid_cvrs: - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/cvr", - data={ - "cvr": ( - io.BytesIO(invalid_cvr.encode()), - "cvrs.csv", - ) - }, + rv = upload_batch_inventory_cvr( + client, + io.BytesIO(invalid_cvr.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -850,14 +847,11 @@ def test_batch_inventory_invalid_file_uploads( assert_ok(rv) # Upload valid CVR file - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/cvr", - data={ - "cvr": ( - io.BytesIO(TEST_CVR.encode()), - "cvrs.csv", - ), - }, + rv = upload_batch_inventory_cvr( + client, + io.BytesIO(TEST_CVR.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -868,18 +862,15 @@ def test_batch_inventory_invalid_file_uploads( assert cvr["processing"]["status"] == ProcessingStatus.PROCESSED # Upload tabulator status file with missing tabulator - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/tabulator-status", - data={ - "tabulatorStatus": ( - io.BytesIO( - TEST_TABULATOR_STATUS.replace( - '', "" - ).encode() - ), - "tabulator-status.xml", - ), - }, + rv = upload_batch_inventory_tabulator_status( + client, + io.BytesIO( + TEST_TABULATOR_STATUS.replace( + '', "" + ).encode() + ), + election_id, + jurisdiction_ids[0], ) rv = client.get( f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/tabulator-status" @@ -897,25 +888,22 @@ def test_batch_inventory_invalid_file_uploads( ) assert_ok(rv) - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/cvr", - data={ - "cvr": ( - io.BytesIO( - TEST_CVR.replace( - """1,TABULATOR1,BATCH1,1,1-1-1,Election Day,12345,COUNTY,0,1,1,1,0 + rv = upload_batch_inventory_cvr( + client, + io.BytesIO( + TEST_CVR.replace( + """1,TABULATOR1,BATCH1,1,1-1-1,Election Day,12345,COUNTY,0,1,1,1,0 2,TABULATOR1,BATCH1,2,1-1-2,Election Day,12345,COUNTY,1,0,1,0,1 3,TABULATOR1,BATCH1,3,1-1-3,Election Day,12345,COUNTY,0,1,1,1,0 4,TABULATOR1,BATCH2,1,1-2-1,Election Day,12345,COUNTY,1,0,1,0,1 5,TABULATOR1,BATCH2,2,1-2-2,Election Day,12345,COUNTY,0,1,1,1,0 6,TABULATOR1,BATCH2,3,1-2-3,Election Day,12345,COUNTY,1,0,1,0,1 """, - "", - ).encode() - ), - "cvrs.csv", - ), - }, + "", + ).encode() + ), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -967,14 +955,11 @@ def test_batch_inventory_missing_data_multi_contest_batch_comparison( ), ] for invalid_cvr, expected_error in invalid_cvrs: - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/cvr", - data={ - "cvr": ( - io.BytesIO(invalid_cvr.encode()), - "cvrs.csv", - ) - }, + rv = upload_batch_inventory_cvr( + client, + io.BytesIO(invalid_cvr.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) @@ -1011,24 +996,19 @@ def test_batch_inventory_excel_tabulator_status_file( assert_ok(rv) # Upload CVR file - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/cvr", - data={ - "cvr": ( - io.BytesIO(TEST_CVR.encode()), - "cvrs.csv", - ), - }, + rv = upload_batch_inventory_cvr( + client, + io.BytesIO(TEST_CVR.encode()), + election_id, + jurisdiction_ids[0], ) assert_ok(rv) # Upload tabulator status "To Excel" version - rv = client.put( - f"/api/election/{election_id}/jurisdiction/{jurisdiction_ids[0]}/batch-inventory/tabulator-status", - data={ - "tabulatorStatus": ( - io.BytesIO( - b""" + rv = upload_batch_inventory_tabulator_status( + client, + io.BytesIO( + b"""