✨ Allow users to redact description

froide/foirequest/api_views.py

@@ -290,7 +290,8 @@ class FoiRequestListSerializer(serializers.HyperlinkedModelSerializer):
         read_only=True, view_name="api:campaign-detail", lookup_field="pk"
     )
     tags = TagListField()
-    description = serializers.CharField(source="get_description")
+
+    redacted_description = serializers.SerializerMethodField()
 
     class Meta:
         model = FoiRequest
@@ -307,6 +308,7 @@ class Meta:
             "public",
             "law",
             "description",
+            "redacted_description",
             "summary",
             "same_as_count",
             "same_as",
@@ -336,6 +338,13 @@ def get_user(self, obj):
             return None
         return obj.user.pk
 
+    def get_redacted_description(self, obj):
+        request = self.context["request"]
+        authenticated_read = can_read_foirequest_authenticated(
+            obj, request, allow_code=False
+        )
+        return obj.get_redacted_description(authenticated_read)
+
 
 class FoiRequestDetailSerializer(FoiRequestListSerializer):
     public_body = PublicBodySerializer(read_only=True)

froide/foirequest/forms/message.py

@@ -16,8 +16,7 @@
 from froide.account.services import AccountService
 from froide.helper.fields import MultipleFileField
 from froide.helper.storage import make_filename, make_unique_filename
-from froide.helper.text_diff import get_diff_chunks
-from froide.helper.text_utils import redact_subject
+from froide.helper.text_utils import apply_user_redaction, redact_subject
 from froide.helper.widgets import (
     BootstrapCheckboxInput,
     BootstrapFileInput,
@@ -809,32 +808,9 @@ def clean(self):
             if not self.cleaned_data.get("subject_length"):
                 raise forms.ValidationError
 
-    def redact_part(self, original, instructions, length):
-        REDACTION_MARKER = str(_("[redacted]"))
-
-        if not instructions:
-            return original
-
-        chunks = get_diff_chunks(original)
-
-        # Sanity check chunk length
-        if len(chunks) != length:
-            raise IndexError
-
-        for index in instructions:
-            chunks[index] = REDACTION_MARKER
-
-        redacted = "".join(chunks)
-        # Replace multiple connecting redactions with one
-        return re.sub(
-            "{marker}(?: {marker})+".format(marker=re.escape(REDACTION_MARKER)),
-            REDACTION_MARKER,
-            redacted,
-        )
-
     def save(self, message):
         try:
-            redacted_subject = self.redact_part(
+            redacted_subject = apply_user_redaction(
                 message.subject,
                 self.cleaned_data["subject"],
                 self.cleaned_data["subject_length"],
@@ -844,7 +820,7 @@ def save(self, message):
             logging.warning(e)
 
         try:
-            redacted_content = self.redact_part(
+            redacted_content = apply_user_redaction(
                 message.plaintext,
                 self.cleaned_data["content"],
                 self.cleaned_data["content_length"],

froide/foirequest/forms/request.py

@@ -3,6 +3,8 @@
 
 from django import forms
 from django.conf import settings
+from django.contrib import messages
+from django.http import HttpRequest
 from django.urls import reverse_lazy
 from django.utils import timezone
 from django.utils.html import escape
@@ -16,7 +18,7 @@
 from froide.helper.auth import get_read_queryset
 from froide.helper.form_utils import JSONMixin
 from froide.helper.forms import TagObjectForm
-from froide.helper.text_utils import redact_plaintext, slugify
+from froide.helper.text_utils import apply_user_redaction, redact_plaintext, slugify
 from froide.helper.widgets import BootstrapRadioSelect, BootstrapSelect, PriceInput
 from froide.publicbody.models import PublicBody
 from froide.publicbody.widgets import PublicBodySelect
@@ -449,3 +451,43 @@ def save(self) -> List[str]:
             m for m in trigger.apply_actions(self.foirequest, self.request) if m
         ]
         return messages
+
+
+class RedactDescriptionForm(forms.Form):
+    description = forms.CharField(required=False)
+    description_length = forms.IntegerField()
+
+    def clean_description(self):
+        val = self.cleaned_data["description"]
+        if not val:
+            return []
+        try:
+            val = [int(x) for x in val.split(",")]
+        except ValueError:
+            raise forms.ValidationError("Bad value")
+        return val
+
+    def save(self, request: HttpRequest, foirequest: FoiRequest):
+        redacted_description = apply_user_redaction(
+            foirequest.description,
+            self.cleaned_data["description"],
+            self.cleaned_data["description_length"],
+        )
+        first_message = foirequest.first_outgoing_message
+        if foirequest.description_redacted in first_message.plaintext_redacted:
+            first_message.plaintext_redacted = first_message.plaintext_redacted.replace(
+                foirequest.description_redacted, redacted_description
+            )
+            first_message.save()
+        else:
+            messages.add_message(
+                request,
+                messages.WARNING,
+                _(
+                    "Could not automatically redact first message. Please check the message manually."
+                ),
+            )
+
+        foirequest.description_redacted = redacted_description
+
+        foirequest.save()

froide/foirequest/migrations/0065_foirequest_description_redacted.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.4 on 2023-10-05 16:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("foirequest", "0064_foimessage_confirmation_sent"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="foirequest",
+            name="description_redacted",
+            field=models.TextField(blank=True, verbose_name="Redacted Description"),
+        ),
+    ]

froide/foirequest/models/event.py

@@ -66,6 +66,8 @@ class EventName(models.TextChoices):
     PUBLIC_BODY_SUGGESTED = "public_body_suggested", _("a public body was suggested")
     REQUEST_REDIRECTED = "request_redirected", _("the request was redirected")
 
+    DESCRIPTION_REDACTED = "description_redacted", _("the description was redacted")
+
 
 EVENT_KEYS = dict(EventName.choices).keys()
 

froide/foirequest/models/request.py

@@ -2,7 +2,7 @@
 import re
 from collections import namedtuple
 from datetime import timedelta
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, List, Optional, Tuple
 
 import django.dispatch
 from django.conf import settings
@@ -22,6 +22,7 @@
 
 from froide.campaign.models import Campaign
 from froide.helper.email_utils import make_address
+from froide.helper.text_diff import get_differences
 from froide.helper.text_utils import redact_plaintext
 from froide.publicbody.models import FoiLaw, Jurisdiction, PublicBody
 from froide.team.models import Team
@@ -295,6 +296,7 @@ class FoiRequest(models.Model):
     title = models.CharField(_("Title"), max_length=255)
     slug = models.SlugField(_("Slug"), max_length=255, unique=True)
     description = models.TextField(_("Description"), blank=True)
+    description_redacted = models.TextField(_("Redacted Description"), blank=True)
     summary = models.TextField(_("Summary"), blank=True)
 
     public_body = models.ForeignKey(
@@ -627,8 +629,27 @@ def get_redaction_regexes(self):
         return json.dumps([a.strip() for a in all_regexes if a.strip()])
 
     def get_description(self):
-        user_replacements = self.user.get_redactions()
-        return redact_plaintext(self.description, user_replacements=user_replacements)
+        if not self.description_redacted:
+            user_replacements = self.user.get_redactions()
+            self.description_redacted = redact_plaintext(
+                self.description, user_replacements=user_replacements
+            )
+            self.save(update_fields=["description_redacted"])
+        return self.description_redacted
+
+    def get_redacted_description(self, auth: bool) -> List[Tuple[bool, str]]:
+        if auth:
+            show, hide = (
+                self.description,
+                self.get_description(),
+            )
+        else:
+            show, hide = (
+                self.get_description(),
+                self.description,
+            )
+
+        return [x for x in get_differences(show, hide)]
 
     def response_messages(self):
         return list(filter(lambda m: m.is_response, self.messages))

froide/foirequest/services.py

@@ -16,7 +16,7 @@
 from froide.helper.db_utils import save_obj_with_slug
 from froide.helper.email_parsing import ParsedEmail
 from froide.helper.storage import make_unique_filename
-from froide.helper.text_utils import redact_subject
+from froide.helper.text_utils import redact_plaintext, redact_subject
 from froide.problem.models import ProblemReport
 
 from .hooks import registry
@@ -166,13 +166,17 @@ def create_project(self):
     def create_request(self, publicbody, sequence=0):
         data = self.data
         user = data["user"]
+        user_replacements = user.get_redactions()
 
         now = timezone.now()
         request = FoiRequest(
             title=data["subject"],
             public_body=publicbody,
             user=data["user"],
             description=data["body"],
+            description_redacted=redact_plaintext(
+                data["body"], user_replacements=user_replacements
+            ),
             public=data["public"],
             language=data.get("language", ""),
             site=Site.objects.get_current(),
@@ -219,7 +223,6 @@ def create_request(self, publicbody, sequence=0):
             request.tags.add(*[t[:100] for t in data["tags"]])
 
         subject = "%s [#%s]" % (request.title, request.pk)
-        user_replacements = user.get_redactions()
         message = FoiMessage(
             request=request,
             sent=False,