Allow combining of batches to support multiple sources of records (CI…

…PRS and Portal)

.eslintrc.json

@@ -40,6 +40,7 @@
   ],
   "rules": {
     // Disabled default rules
+    "no-debugger": "off",
     "no-plusplus": "off",
     "no-continue": "off",
     "react/jsx-props-no-spreading": "off",

dear_petition/petition/api/viewsets.py

@@ -11,13 +11,14 @@
 from django.contrib.auth.models import update_last_login
 from django.http import FileResponse, HttpResponse
 from django_filters.rest_framework import DjangoFilterBackend
-from rest_framework import filters, generics, parsers, permissions, status, viewsets
+from rest_framework import filters, generics, parsers, permissions, status, viewsets, views
 from rest_framework.decorators import action
 from rest_framework.response import Response
 from rest_framework_simplejwt import exceptions
 from rest_framework_simplejwt import views as simplejwt_views
 from rest_framework_simplejwt.serializers import TokenRefreshSerializer
 
+
 from dear_petition.petition import constants
 from dear_petition.petition import models as pm
 from dear_petition.petition import resources
@@ -27,6 +28,7 @@
 from dear_petition.petition.etl import (
     import_ciprs_records,
     recalculate_petitions,
+    combine_batches,
     assign_agencies_to_documents,
 )
 from dear_petition.petition.export import (
@@ -328,6 +330,21 @@ def generate_summary(self, request, pk):
             ] = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
             resp["Content-Disposition"] = 'attachment; filename="Records Summary.docx"'
             return resp
+
+
+    @action(
+        detail=False,
+        methods=[
+            "post",
+        ],
+    )
+    def combine_batches(self, request):
+        batch_ids = request.data['batchIds']
+        label = request.data['label']
+        user_id = request.data['user_id']
+
+        new_batch = combine_batches(batch_ids, label, user_id)
+        return Response(self.get_serializer(new_batch).data)
 
 
 class MyInboxView(generics.ListAPIView):

dear_petition/petition/conftest.py

@@ -1,13 +1,12 @@
-import io
 import string
 from datetime import datetime
 
 import pytest
-from django.core.files.uploadedfile import InMemoryUploadedFile
 
 from dear_petition.petition.constants import CHARGED, CONVICTED, FEMALE
 from dear_petition.petition.tests.factories import (
     BatchFactory,
+    BatchFileFactory,
     CIPRSRecordFactory,
     ClientFactory,
     PetitionFactory,
@@ -16,6 +15,7 @@
     ContactFactory,
     OffenseFactory,
     OffenseRecordFactory,
+    fake_file,
 )
 from dear_petition.petition.types import dismissed
 from dear_petition.petition import constants
@@ -95,20 +95,6 @@ def offense_record2(offense1, petition, petition_document):
     yield record
 
 
-def fake_file(filename, content_type):
-    output = io.StringIO("blahblah")
-    stream = io.BytesIO(output.getvalue().encode("utf-8"))
-    file_ = InMemoryUploadedFile(
-        file=stream,
-        field_name=None,
-        name=filename,
-        content_type=content_type,
-        size=stream.getbuffer().nbytes,
-        charset=None,
-    )
-    return file_
-
-
 @pytest.fixture
 def fake_pdf():
     return fake_file("sample.pdf", "pdf")
@@ -119,6 +105,11 @@ def fake_pdf2():
     return fake_file("sample2.pdf", "pdf")
 
 
+@pytest.fixture
+def batch_file(batch):
+    return BatchFileFactory(batch=batch)
+
+
 @pytest.fixture
 def petition(batch):
     return PetitionFactory(batch=batch)

dear_petition/petition/etl/__init__.py

@@ -1,3 +1,3 @@
 from .extract import transform_ciprs_document, parse_ciprs_document
-from .transform import recalculate_petitions
+from .transform import recalculate_petitions, combine_batches
 from .load import import_ciprs_records, create_documents, assign_agencies_to_documents

dear_petition/petition/etl/refresh.py

@@ -57,7 +57,6 @@ def refresh_offenses(record):
     """Create Offense and OffenseRecords in each jurisdiction for this record."""
     for jurisdiction, header in constants.OFFENSE_HEADERS:
         offenses = record.data.get(header, {})
-
         # delete existing offenses in this jurisdiction
         record.offenses.filter(jurisdiction=jurisdiction).delete()
         for data_offense in offenses:

dear_petition/petition/etl/tests/test_transform.py

@@ -1,14 +1,17 @@
 import pytest
 
 from dear_petition.petition import constants
+from dear_petition.petition import models as pm
 from dear_petition.petition.tests.factories import (
+    BatchFactory,
+    BatchFileFactory,
     OffenseFactory,
     OffenseRecordFactory,
     PetitionFactory,
     CIPRSRecordFactory,
 )
 from dear_petition.petition.etl.load import link_offense_records, create_documents
-from dear_petition.petition.etl.transform import recalculate_petitions
+from dear_petition.petition.etl.transform import recalculate_petitions, combine_batches
 
 pytestmark = pytest.mark.django_db
 
@@ -43,3 +46,49 @@ def test_recalculate_petitions(petition):
         petition.offense_records.filter(petitionoffenserecord__active=True).count() == 5
     )
     assert not petition.has_attachments()
+
+
+def test_combine_batches(batch, batch_file, fake_pdf):
+    batch_label = batch.label
+    record = CIPRSRecordFactory(
+        batch=batch, jurisdiction=constants.DISTRICT_COURT, county="DURHAM"
+    )
+    offense = OffenseFactory(
+        disposition_method="PROBATION OTHER",
+        ciprs_record=record,
+        jurisdiction=constants.DISTRICT_COURT,
+    )
+    offense_record = OffenseRecordFactory(offense=offense, action="CHARGED")
+
+    second_batch = BatchFactory()
+    second_batch_file = BatchFileFactory(batch=second_batch, file=fake_pdf)
+    second_batch_label = second_batch.label
+    second_record = CIPRSRecordFactory(
+        batch=second_batch, batch_file=second_batch_file, jurisdiction=constants.DISTRICT_COURT, county="DURHAM"
+    )
+    second_offense = OffenseFactory(
+        disposition_method="PROBATION OTHER",
+        ciprs_record=second_record,
+        jurisdiction=constants.DISTRICT_COURT,
+    )
+    second_offense_record = OffenseRecordFactory(offense=second_offense, action="CHARGED")
+    third_offense = OffenseFactory(
+        disposition_method="PROBATION OTHER",
+        ciprs_record=second_record,
+        jurisdiction=constants.SUPERIOR_COURT,
+    )
+    third_offense_record = OffenseRecordFactory(offense=third_offense, action="CHARGED")
+
+    assert batch.records.count() == 1
+    assert pm.Offense.objects.filter(ciprs_record__batch__id=batch.id).count() == 1
+    assert pm.Offense.objects.filter(ciprs_record__batch__id=second_batch.id).count() == 2
+    assert batch.files.count() == 1
+
+    new_label = "Combined Batch"
+    new_batch = combine_batches([batch.id, second_batch.id], label=new_label, user_id=1)
+
+    assert new_batch.records.count() == 2
+    assert pm.Offense.objects.filter(ciprs_record__batch__id=new_batch.id).count() == 2
+    assert new_batch.files.count() == 2
+
+

dear_petition/petition/etl/transform.py

@@ -1,10 +1,11 @@
+import os
+from typing import List
 from django.db import transaction
 from django.db.models import Q
 
 from dear_petition.petition import models as pm
-from dear_petition.petition.constants import ATTACHMENT
 
-from .load import create_documents, assign_agencies_to_documents
+from .load import create_batch_petitions, create_documents, assign_agencies_to_documents
 
 
 def recalculate_petitions(petition_id, offense_record_ids):
@@ -21,3 +22,38 @@ def recalculate_petitions(petition_id, offense_record_ids):
         petition = assign_agencies_to_documents(petition)
 
     return petition
+
+
+def combine_batches(batch_ids: List[int], label: str, user_id: int):
+
+    with transaction.atomic():
+        new_batch = pm.Batch.objects.create(label=label, user_id=user_id)
+        batches = pm.Batch.objects.filter(id__in=batch_ids)
+
+        saved_file_nos = []
+        saved_batch_files = {}
+        for batch in batches:
+            for record in batch.records.iterator():
+
+                if record.batch_file:
+                    file = record.batch_file.file.file
+                    file_name = os.path.basename(file.name)
+
+                    if saved_batch_files.get(file_name):
+                        new_batch_file = saved_batch_files[file_name]
+                    else:
+                        file.name = file_name
+                        new_batch_file = new_batch.files.create(file=file)
+                        saved_batch_files[file_name] = new_batch_file
+                else:
+                    new_batch_file=None
+
+                new_record = new_batch.records.create(batch=new_batch, batch_file=new_batch_file, data=record.data)
+                # Pass file numbers of CIPRS records that have already been saved in this batch of CIPRS records.
+                # If this CIPRS record is in the list, it will not be saved again.
+                new_record.refresh_record_from_data(saved_file_nos)
+                saved_file_nos.append(record.file_no)
+
+        create_batch_petitions(new_batch)
+
+        return new_batch

dear_petition/petition/tests/factories.py

@@ -1,34 +1,19 @@
+import io
 import random
-from pytz import timezone
 
 import factory
-
-from dear_petition.petition.models import (
-    CIPRSRecord,
-    Batch,
-    Offense,
-    OffenseRecord,
-    Petition,
-    PetitionOffenseRecord,
-    PetitionDocument,
-    Contact,
-    GeneratedPetition,
-)
+from dear_petition.petition.models import (Batch, BatchFile, CIPRSRecord,
+                                           Contact, GeneratedPetition, Offense,
+                                           OffenseRecord, Petition,
+                                           PetitionDocument,
+                                           PetitionOffenseRecord)
 from dear_petition.users.tests.factories import UserFactory
+from django.core.files.uploadedfile import InMemoryUploadedFile
+from pytz import timezone
 
-from ..constants import (
-    CHARGED,
-    CONVICTED,
-    DISMISSED,
-    DISTRICT_COURT,
-    SUPERIOR_COURT,
-    DURHAM_COUNTY,
-    DISTRICT_COURT_WITHOUT_DA_LEAVE,
-    FEMALE,
-    MALE,
-    NOT_AVAILABLE,
-    UNKNOWN,
-)
+from ..constants import (CHARGED, CONVICTED, DISMISSED, DISTRICT_COURT,
+                         DISTRICT_COURT_WITHOUT_DA_LEAVE, DURHAM_COUNTY,
+                         FEMALE, MALE, NOT_AVAILABLE, SUPERIOR_COURT, UNKNOWN)
 
 
 class BatchFactory(factory.django.DjangoModelFactory):
@@ -39,6 +24,29 @@ class Meta:
         model = Batch
 
 
+def fake_file(filename, content_type):
+    output = io.StringIO("blahblah")
+    stream = io.BytesIO(output.getvalue().encode("utf-8"))
+    file_ = InMemoryUploadedFile(
+        file=stream,
+        field_name=None,
+        name=filename,
+        content_type=content_type,
+        size=stream.getbuffer().nbytes,
+        charset=None,
+    )
+    return file_
+
+
+class BatchFileFactory(factory.django.DjangoModelFactory):
+    batch = factory.SubFactory(BatchFactory)
+    date_uploaded = factory.Faker("date_time", tzinfo=timezone("US/Eastern"))
+    file = fake_file("fakefile.pdf", "pdf")
+
+    class Meta:
+        model = BatchFile
+
+
 def record_data(idx):
     return {
         "General": {"County": "DURHAM", "File No": "00GR000000"},
@@ -52,31 +60,35 @@ def record_data(idx):
             "Race": "WHITE",
             "Sex": "MALE",
         },
-        "Offense Record": {
-            "Records": [
-                {
-                    "Action": "CHARGED",
-                    "Description": "SPEEDING(80 mph in a 65 mph zone)",
-                    "Severity": "TRAFFIC",
-                    "Law": "20-141(J1)",
-                    "Code": "4450",
-                },
-                {
-                    "Action": "ARRAIGNED",
-                    "Description": "SPEEDING(80 mph in a 65 mph zone)",
-                    "Severity": "INFRACTION",
-                    "Law": "G.S. 20-141(B)",
-                    "Code": "4450",
-                },
-            ],
-            "Disposed On": "2018-02-01",
-            "Disposition Method": "DISPOSED BY JUDGE",
-        },
+        "District Court Offense Information": [
+            {
+                "Records": [
+                    {
+                        "Action": "CHARGED",
+                        "Description": "SPEEDING(80 mph in a 65 mph zone)",
+                        "Severity": "TRAFFIC",
+                        "Law": "20-141(J1)",
+                        "Code": "4450",
+                    },
+                    {
+                        "Action": "ARRAIGNED",
+                        "Description": "SPEEDING(80 mph in a 65 mph zone)",
+                        "Severity": "INFRACTION",
+                        "Law": "G.S. 20-141(B)",
+                        "Code": "4450",
+                    },
+                ],
+                "Disposed On": "2018-02-01",
+                "Disposition Method": "DISPOSED BY JUDGE",
+            }
+        ],
+        "Superior Court Offense Information": [],
     }
 
 
 class CIPRSRecordFactory(factory.django.DjangoModelFactory):
     batch = factory.SubFactory(BatchFactory)
+    batch_file = factory.SubFactory(BatchFileFactory)
     label = factory.Faker("name")
     data = factory.Sequence(record_data)
     offense_date = factory.Faker("date_time", tzinfo=timezone("US/Eastern"))