From 511184c75569080077886cc9992f92912a7a7632 Mon Sep 17 00:00:00 2001 From: George Helman Date: Tue, 2 Apr 2024 17:34:38 -0400 Subject: [PATCH] Allow combining of batches to support multiple sources of records (CIPRS and Portal) --- .eslintrc.json | 1 + dear_petition/petition/api/viewsets.py | 19 ++- dear_petition/petition/conftest.py | 23 +-- dear_petition/petition/etl/__init__.py | 2 +- dear_petition/petition/etl/refresh.py | 1 - .../petition/etl/tests/test_transform.py | 51 ++++++- dear_petition/petition/etl/transform.py | 40 ++++- dear_petition/petition/tests/factories.py | 104 +++++++------ src/features/ExistingPetitions.jsx | 138 ++++++++++++++---- src/service/api.js | 4 + vite.config.js | 3 + 11 files changed, 292 insertions(+), 94 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index 3712514a..38cff895 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -40,6 +40,7 @@ ], "rules": { // Disabled default rules + "no-debugger": "off", "no-plusplus": "off", "no-continue": "off", "react/jsx-props-no-spreading": "off", diff --git a/dear_petition/petition/api/viewsets.py b/dear_petition/petition/api/viewsets.py index b5a4d274..2ca6bdb7 100644 --- a/dear_petition/petition/api/viewsets.py +++ b/dear_petition/petition/api/viewsets.py @@ -11,13 +11,14 @@ from django.contrib.auth.models import update_last_login from django.http import FileResponse, HttpResponse from django_filters.rest_framework import DjangoFilterBackend -from rest_framework import filters, generics, parsers, permissions, status, viewsets +from rest_framework import filters, generics, parsers, permissions, status, viewsets, views from rest_framework.decorators import action from rest_framework.response import Response from rest_framework_simplejwt import exceptions from rest_framework_simplejwt import views as simplejwt_views from rest_framework_simplejwt.serializers import TokenRefreshSerializer + from dear_petition.petition import constants from dear_petition.petition import models as pm from dear_petition.petition import resources @@ -27,6 +28,7 @@ from dear_petition.petition.etl import ( import_ciprs_records, recalculate_petitions, + combine_batches, assign_agencies_to_documents, ) from dear_petition.petition.export import ( @@ -328,6 +330,21 @@ def generate_summary(self, request, pk): ] = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" resp["Content-Disposition"] = 'attachment; filename="Records Summary.docx"' return resp + + + @action( + detail=False, + methods=[ + "post", + ], + ) + def combine_batches(self, request): + batch_ids = request.data['batchIds'] + label = request.data['label'] + user_id = request.data['user_id'] + + new_batch = combine_batches(batch_ids, label, user_id) + return Response(self.get_serializer(new_batch).data) class MyInboxView(generics.ListAPIView): diff --git a/dear_petition/petition/conftest.py b/dear_petition/petition/conftest.py index 53d3738d..39952449 100644 --- a/dear_petition/petition/conftest.py +++ b/dear_petition/petition/conftest.py @@ -1,13 +1,12 @@ -import io import string from datetime import datetime import pytest -from django.core.files.uploadedfile import InMemoryUploadedFile from dear_petition.petition.constants import CHARGED, CONVICTED, FEMALE from dear_petition.petition.tests.factories import ( BatchFactory, + BatchFileFactory, CIPRSRecordFactory, ClientFactory, PetitionFactory, @@ -16,6 +15,7 @@ ContactFactory, OffenseFactory, OffenseRecordFactory, + fake_file, ) from dear_petition.petition.types import dismissed from dear_petition.petition import constants @@ -95,20 +95,6 @@ def offense_record2(offense1, petition, petition_document): yield record -def fake_file(filename, content_type): - output = io.StringIO("blahblah") - stream = io.BytesIO(output.getvalue().encode("utf-8")) - file_ = InMemoryUploadedFile( - file=stream, - field_name=None, - name=filename, - content_type=content_type, - size=stream.getbuffer().nbytes, - charset=None, - ) - return file_ - - @pytest.fixture def fake_pdf(): return fake_file("sample.pdf", "pdf") @@ -119,6 +105,11 @@ def fake_pdf2(): return fake_file("sample2.pdf", "pdf") +@pytest.fixture +def batch_file(batch): + return BatchFileFactory(batch=batch) + + @pytest.fixture def petition(batch): return PetitionFactory(batch=batch) diff --git a/dear_petition/petition/etl/__init__.py b/dear_petition/petition/etl/__init__.py index 227d4982..086ac7bf 100644 --- a/dear_petition/petition/etl/__init__.py +++ b/dear_petition/petition/etl/__init__.py @@ -1,3 +1,3 @@ from .extract import transform_ciprs_document, parse_ciprs_document -from .transform import recalculate_petitions +from .transform import recalculate_petitions, combine_batches from .load import import_ciprs_records, create_documents, assign_agencies_to_documents diff --git a/dear_petition/petition/etl/refresh.py b/dear_petition/petition/etl/refresh.py index 8a6c66c7..2736721e 100644 --- a/dear_petition/petition/etl/refresh.py +++ b/dear_petition/petition/etl/refresh.py @@ -57,7 +57,6 @@ def refresh_offenses(record): """Create Offense and OffenseRecords in each jurisdiction for this record.""" for jurisdiction, header in constants.OFFENSE_HEADERS: offenses = record.data.get(header, {}) - # delete existing offenses in this jurisdiction record.offenses.filter(jurisdiction=jurisdiction).delete() for data_offense in offenses: diff --git a/dear_petition/petition/etl/tests/test_transform.py b/dear_petition/petition/etl/tests/test_transform.py index 15590cb3..102d96d7 100644 --- a/dear_petition/petition/etl/tests/test_transform.py +++ b/dear_petition/petition/etl/tests/test_transform.py @@ -1,14 +1,17 @@ import pytest from dear_petition.petition import constants +from dear_petition.petition import models as pm from dear_petition.petition.tests.factories import ( + BatchFactory, + BatchFileFactory, OffenseFactory, OffenseRecordFactory, PetitionFactory, CIPRSRecordFactory, ) from dear_petition.petition.etl.load import link_offense_records, create_documents -from dear_petition.petition.etl.transform import recalculate_petitions +from dear_petition.petition.etl.transform import recalculate_petitions, combine_batches pytestmark = pytest.mark.django_db @@ -43,3 +46,49 @@ def test_recalculate_petitions(petition): petition.offense_records.filter(petitionoffenserecord__active=True).count() == 5 ) assert not petition.has_attachments() + + +def test_combine_batches(batch, batch_file, fake_pdf): + batch_label = batch.label + record = CIPRSRecordFactory( + batch=batch, jurisdiction=constants.DISTRICT_COURT, county="DURHAM" + ) + offense = OffenseFactory( + disposition_method="PROBATION OTHER", + ciprs_record=record, + jurisdiction=constants.DISTRICT_COURT, + ) + offense_record = OffenseRecordFactory(offense=offense, action="CHARGED") + + second_batch = BatchFactory() + second_batch_file = BatchFileFactory(batch=second_batch, file=fake_pdf) + second_batch_label = second_batch.label + second_record = CIPRSRecordFactory( + batch=second_batch, batch_file=second_batch_file, jurisdiction=constants.DISTRICT_COURT, county="DURHAM" + ) + second_offense = OffenseFactory( + disposition_method="PROBATION OTHER", + ciprs_record=second_record, + jurisdiction=constants.DISTRICT_COURT, + ) + second_offense_record = OffenseRecordFactory(offense=second_offense, action="CHARGED") + third_offense = OffenseFactory( + disposition_method="PROBATION OTHER", + ciprs_record=second_record, + jurisdiction=constants.SUPERIOR_COURT, + ) + third_offense_record = OffenseRecordFactory(offense=third_offense, action="CHARGED") + + assert batch.records.count() == 1 + assert pm.Offense.objects.filter(ciprs_record__batch__id=batch.id).count() == 1 + assert pm.Offense.objects.filter(ciprs_record__batch__id=second_batch.id).count() == 2 + assert batch.files.count() == 1 + + new_label = "Combined Batch" + new_batch = combine_batches([batch.id, second_batch.id], label=new_label, user_id=1) + + assert new_batch.records.count() == 2 + assert pm.Offense.objects.filter(ciprs_record__batch__id=new_batch.id).count() == 2 + assert new_batch.files.count() == 2 + + \ No newline at end of file diff --git a/dear_petition/petition/etl/transform.py b/dear_petition/petition/etl/transform.py index 3ef7ba79..36645834 100644 --- a/dear_petition/petition/etl/transform.py +++ b/dear_petition/petition/etl/transform.py @@ -1,10 +1,11 @@ +import os +from typing import List from django.db import transaction from django.db.models import Q from dear_petition.petition import models as pm -from dear_petition.petition.constants import ATTACHMENT -from .load import create_documents, assign_agencies_to_documents +from .load import create_batch_petitions, create_documents, assign_agencies_to_documents def recalculate_petitions(petition_id, offense_record_ids): @@ -21,3 +22,38 @@ def recalculate_petitions(petition_id, offense_record_ids): petition = assign_agencies_to_documents(petition) return petition + + +def combine_batches(batch_ids: List[int], label: str, user_id: int): + + with transaction.atomic(): + new_batch = pm.Batch.objects.create(label=label, user_id=user_id) + batches = pm.Batch.objects.filter(id__in=batch_ids) + + saved_file_nos = [] + saved_batch_files = {} + for batch in batches: + for record in batch.records.iterator(): + + if record.batch_file: + file = record.batch_file.file.file + file_name = os.path.basename(file.name) + + if saved_batch_files.get(file_name): + new_batch_file = saved_batch_files[file_name] + else: + file.name = file_name + new_batch_file = new_batch.files.create(file=file) + saved_batch_files[file_name] = new_batch_file + else: + new_batch_file=None + + new_record = new_batch.records.create(batch=new_batch, batch_file=new_batch_file, data=record.data) + # Pass file numbers of CIPRS records that have already been saved in this batch of CIPRS records. + # If this CIPRS record is in the list, it will not be saved again. + new_record.refresh_record_from_data(saved_file_nos) + saved_file_nos.append(record.file_no) + + create_batch_petitions(new_batch) + + return new_batch diff --git a/dear_petition/petition/tests/factories.py b/dear_petition/petition/tests/factories.py index ec10931e..6cd8e671 100644 --- a/dear_petition/petition/tests/factories.py +++ b/dear_petition/petition/tests/factories.py @@ -1,34 +1,19 @@ +import io import random -from pytz import timezone import factory - -from dear_petition.petition.models import ( - CIPRSRecord, - Batch, - Offense, - OffenseRecord, - Petition, - PetitionOffenseRecord, - PetitionDocument, - Contact, - GeneratedPetition, -) +from dear_petition.petition.models import (Batch, BatchFile, CIPRSRecord, + Contact, GeneratedPetition, Offense, + OffenseRecord, Petition, + PetitionDocument, + PetitionOffenseRecord) from dear_petition.users.tests.factories import UserFactory +from django.core.files.uploadedfile import InMemoryUploadedFile +from pytz import timezone -from ..constants import ( - CHARGED, - CONVICTED, - DISMISSED, - DISTRICT_COURT, - SUPERIOR_COURT, - DURHAM_COUNTY, - DISTRICT_COURT_WITHOUT_DA_LEAVE, - FEMALE, - MALE, - NOT_AVAILABLE, - UNKNOWN, -) +from ..constants import (CHARGED, CONVICTED, DISMISSED, DISTRICT_COURT, + DISTRICT_COURT_WITHOUT_DA_LEAVE, DURHAM_COUNTY, + FEMALE, MALE, NOT_AVAILABLE, SUPERIOR_COURT, UNKNOWN) class BatchFactory(factory.django.DjangoModelFactory): @@ -39,6 +24,29 @@ class Meta: model = Batch +def fake_file(filename, content_type): + output = io.StringIO("blahblah") + stream = io.BytesIO(output.getvalue().encode("utf-8")) + file_ = InMemoryUploadedFile( + file=stream, + field_name=None, + name=filename, + content_type=content_type, + size=stream.getbuffer().nbytes, + charset=None, + ) + return file_ + + +class BatchFileFactory(factory.django.DjangoModelFactory): + batch = factory.SubFactory(BatchFactory) + date_uploaded = factory.Faker("date_time", tzinfo=timezone("US/Eastern")) + file = fake_file("fakefile.pdf", "pdf") + + class Meta: + model = BatchFile + + def record_data(idx): return { "General": {"County": "DURHAM", "File No": "00GR000000"}, @@ -52,31 +60,35 @@ def record_data(idx): "Race": "WHITE", "Sex": "MALE", }, - "Offense Record": { - "Records": [ - { - "Action": "CHARGED", - "Description": "SPEEDING(80 mph in a 65 mph zone)", - "Severity": "TRAFFIC", - "Law": "20-141(J1)", - "Code": "4450", - }, - { - "Action": "ARRAIGNED", - "Description": "SPEEDING(80 mph in a 65 mph zone)", - "Severity": "INFRACTION", - "Law": "G.S. 20-141(B)", - "Code": "4450", - }, - ], - "Disposed On": "2018-02-01", - "Disposition Method": "DISPOSED BY JUDGE", - }, + "District Court Offense Information": [ + { + "Records": [ + { + "Action": "CHARGED", + "Description": "SPEEDING(80 mph in a 65 mph zone)", + "Severity": "TRAFFIC", + "Law": "20-141(J1)", + "Code": "4450", + }, + { + "Action": "ARRAIGNED", + "Description": "SPEEDING(80 mph in a 65 mph zone)", + "Severity": "INFRACTION", + "Law": "G.S. 20-141(B)", + "Code": "4450", + }, + ], + "Disposed On": "2018-02-01", + "Disposition Method": "DISPOSED BY JUDGE", + } + ], + "Superior Court Offense Information": [], } class CIPRSRecordFactory(factory.django.DjangoModelFactory): batch = factory.SubFactory(BatchFactory) + batch_file = factory.SubFactory(BatchFileFactory) label = factory.Faker("name") data = factory.Sequence(record_data) offense_date = factory.Faker("date_time", tzinfo=timezone("US/Eastern")) diff --git a/src/features/ExistingPetitions.jsx b/src/features/ExistingPetitions.jsx index bb1de83b..6cb2d1ab 100644 --- a/src/features/ExistingPetitions.jsx +++ b/src/features/ExistingPetitions.jsx @@ -7,14 +7,110 @@ import { manualAxiosRequest } from '../service/axios'; import { Button, ModalButton } from '../components/elements/Button'; import { Table, TableBody, TableCell, TableHeader, TableRow } from '../components/elements/Table'; import { Tooltip } from '../components/elements/Tooltip/Tooltip'; -import { useDeleteBatchMutation, useGetUserBatchesQuery } from '../service/api'; +import { useDeleteBatchMutation, useGetUserBatchesQuery, useCombineBatchesMutation } from '../service/api'; import useAuth from '../hooks/useAuth'; import { DownloadDocumentsModal } from './DownloadDocuments'; import { hasValidationsErrors } from '../util/errors'; import { downloadFile } from '../util/downloadFile'; -import { CAUTION, NEUTRAL } from '../components/elements/Button/Button'; +import { POSITIVE, CAUTION, NEUTRAL } from '../components/elements/Button/Button'; import { useModalContext } from '../components/elements/Button/ModalButton'; +const finishCombineModal = ( + batch, + batchIds, + setCombineWithBatchId, + setBatchIdsToCombine, + triggerCombine, + rowData, + setRowData, +) => { + let postData = { + batchIds: batchIds, + label: batch.label, + user_id: batch.user, + }; + + triggerCombine(postData).then((newBatch) => { + console.log(newBatch.data); + setRowData((prevList) => [newBatch.data, ...prevList]); + + setCombineWithBatchId(null); + setBatchIdsToCombine([]); + }); +}; + +const CombineBatchModal = ({ + batch, + combineWithBatchId, + setCombineWithBatchId, + batchIdsToCombine, + setBatchIdsToCombine, + rowData, + setRowData, +}) => { + const [triggerCombine] = useCombineBatchesMutation(); + let batchId = batch.pk; + if (!combineWithBatchId) { + return ( + + ); + } else if (batchId != combineWithBatchId) { + let alreadyIncluded = batchIdsToCombine.includes(batchId); + if (alreadyIncluded) { + return ( + + ); + } else { + return ( + + ); + } + } else { + return ( + + ); + } +}; + const DeleteBatchModal = ({ batch }) => { const [triggerDelete] = useDeleteBatchMutation(); const { closeModal } = useModalContext(); @@ -41,8 +137,11 @@ const DeleteBatchModal = ({ batch }) => { // TODO: Rename batches to "Petition Groups" export const ExistingPetitions = () => { const { user } = useAuth(); - const { data } = useGetUserBatchesQuery({ user: user.pk }); + const data = useGetUserBatchesQuery({ user: user.pk }); const [downloadDocumentBatch, setDownloadDocumentBatch] = useState(null); + const [combineWithBatchId, setCombineWithBatchId] = useState(null); + const [batchIdsToCombine, setBatchIdsToCombine] = useState([]); + const [rowData, setRowData] = useState(data?.data?.results); return (
@@ -71,7 +170,7 @@ export const ExistingPetitions = () => { - {data?.results?.map((batch) => ( + {rowData?.map((batch) => ( {batch.label} @@ -96,28 +195,6 @@ export const ExistingPetitions = () => { > Download - {/* - Legal team requested this be temporarily removed from UI - - - */}