Migrate mailing lists to their own API endpoints

Add a new model for the bot to store its mailing list state in, as
opposed to the current JSON blob in the BotSetting table. Migrate the
existing settings from the BotSetting table into the new model.

pydis_site/apps/api/migrations/0093_add_mailing_lists.py

@@ -0,0 +1,36 @@
+# Generated by Django 5.0 on 2023-12-14 13:24
+
+import django.db.models.deletion
+import pydis_site.apps.api.models.mixins
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0092_remove_redirect_filter_list'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='MailingList',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(help_text='A short identifier for the mailing list.', max_length=50)),
+            ],
+            bases=(pydis_site.apps.api.models.mixins.ModelReprMixin, models.Model),
+        ),
+        migrations.CreateModel(
+            name='MailingListSeenItem',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('hash', models.CharField(help_text='A hash, or similar identifier, of the content that was seen.', max_length=100)),
+                ('list', models.ForeignKey(help_text='The mailing list from which this seen item originates.', on_delete=django.db.models.deletion.CASCADE, related_name='seen_items', to='api.mailinglist')),
+            ],
+            bases=(pydis_site.apps.api.models.mixins.ModelReprMixin, models.Model),
+        ),
+        migrations.AddConstraint(
+            model_name='mailinglistseenitem',
+            constraint=models.UniqueConstraint(fields=('list', 'hash'), name='unique_list_and_hash'),
+        ),
+    ]

pydis_site/apps/api/migrations/0094_migrate_mailing_listdata.py

@@ -0,0 +1,41 @@
+# Generated by Django 5.0 on 2023-12-13 07:03
+
+from django.db import migrations
+
+
+def migrate_mailing_lists_into_new_model(apps, schema_editor):
+    """Move the bot's mailing list information from the BotSetting to the new MailingList model."""
+
+    BotSetting = apps.get_model('api', 'BotSetting')
+    MailingList = apps.get_model('api', 'MailingList')
+    MailingListSeenItem = apps.get_model('api', 'MailingListSeenItem')
+    try:
+        setting = BotSetting.objects.get(name='news')
+    except BotSetting.DoesNotExist:
+        return
+
+    # Field format:
+    # {
+    #   "pep": [
+    #     "644",
+    #     "8102",
+    #     ...
+    for list_name, item_hashes in setting.data.items():
+        (mailing_list, _created) = MailingList.objects.get_or_create(name=list_name)
+        MailingListSeenItem.objects.bulk_create(
+            MailingListSeenItem(list=mailing_list, hash=item_hash)
+            for item_hash in item_hashes
+        )
+
+    setting.delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0093_add_mailing_lists'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_mailing_lists_into_new_model, elidable=True)
+    ]

pydis_site/apps/api/models/__init__.py

@@ -7,6 +7,8 @@
     DocumentationLink,
     DeletedMessage,
     Infraction,
+    MailingList,
+    MailingListSeenItem,
     Message,
     MessageDeletionContext,
     Nomination,

pydis_site/apps/api/models/bot/__init__.py

@@ -8,6 +8,8 @@
 from .message import Message
 from .aoc_completionist_block import AocCompletionistBlock
 from .aoc_link import AocAccountLink
+from .mailing_list import MailingList
+from .mailing_list_seen_item import MailingListSeenItem
 from .message_deletion_context import MessageDeletionContext
 from .nomination import Nomination, NominationEntry
 from .off_topic_channel_name import OffTopicChannelName

pydis_site/apps/api/models/bot/mailing_list.py

@@ -0,0 +1,12 @@
+from django.db import models
+
+from pydis_site.apps.api.models.mixins import ModelReprMixin
+
+
+class MailingList(ModelReprMixin, models.Model):
+    """A mailing list that the bot is following."""
+
+    name = models.CharField(
+        max_length=50,
+        help_text="A short identifier for the mailing list."
+    )

pydis_site/apps/api/models/bot/mailing_list_seen_item.py

@@ -0,0 +1,29 @@
+from django.db import models
+
+from pydis_site.apps.api.models.mixins import ModelReprMixin
+from .mailing_list import MailingList
+
+
+class MailingListSeenItem(ModelReprMixin, models.Model):
+    """An item in a mailing list that the bot has consumed and mirrored elsewhere."""
+
+    list = models.ForeignKey(
+        MailingList,
+        on_delete=models.CASCADE,
+        related_name='seen_items',
+        help_text="The mailing list from which this seen item originates."
+    )
+    hash = models.CharField(
+        max_length=100,
+        help_text="A hash, or similar identifier, of the content that was seen."
+    )
+
+    class Meta:
+        """Prevent adding the same hash to the same list multiple times."""
+
+        constraints = (
+            models.UniqueConstraint(
+                fields=('list', 'hash'),
+                name='unique_list_and_hash',
+            ),
+        )

pydis_site/apps/api/serializers.py

@@ -26,6 +26,8 @@
     Filter,
     FilterList,
     Infraction,
+    MailingList,
+    MailingListSeenItem,
     MessageDeletionContext,
     Nomination,
     NominationEntry,
@@ -741,3 +743,37 @@ class Meta:
         model = OffensiveMessage
         fields = ('id', 'channel_id', 'delete_date')
         frozen_fields = ('id', 'channel_id')
+
+
+class MailingListSeenItemListSerializer(ListSerializer):
+    """A class providing (de-)serialization of `MailingListSeenItem` instances as a list."""
+
+    def to_representation(self, objects: list[MailingListSeenItem]) -> list[str]:
+        """Return the hashes of each seen mailing list item."""
+        return [obj['hash'] for obj in objects.values('hash')]
+
+
+class MailingListSeenItemSerializer(ModelSerializer):
+    """A class providing (de-)serialization of `MailingListSeenItem` instances."""
+
+    class Meta:
+        """Metadata defined for the Django REST Framework."""
+
+        model = MailingListSeenItem
+        # Since this is only exposed on the parent mailing list model,
+        # we don't need information about the list or even the ID.
+        fields = ('hash',)
+        list_serializer_class = MailingListSeenItemListSerializer
+
+
+class MailingListSerializer(FrozenFieldsMixin, ModelSerializer):
+    """A class providing (de-)serialization of `MailingList` instances."""
+
+    seen_items = MailingListSeenItemSerializer(many=True)
+
+    class Meta:
+        """Metadata defined for the Django REST Framework."""
+
+        model = MailingList
+        fields = ('id', 'name', 'seen_items')
+        frozen_fields = ('name',)

pydis_site/apps/api/tests/test_mailing_list.py

@@ -0,0 +1,80 @@
+from django.urls import reverse
+
+from .base import AuthenticatedAPITestCase
+from pydis_site.apps.api.models import MailingList, MailingListSeenItem
+
+
+class EmptyMailingListTests(AuthenticatedAPITestCase):
+    @classmethod
+    def setUpTestData(cls):
+        cls.list = MailingList.objects.create(name='erlang-dev')
+
+    def test_get_all_mailing_lists(self):
+        url = reverse('api:bot:mailinglist-list')
+        response = self.client.get(url)
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json(), [
+            {'id': self.list.id, 'name': self.list.name, 'seen_items': []}
+        ])
+
+    def test_get_single_mailing_list(self):
+        url = reverse('api:bot:mailinglist-detail', args=(self.list.name,))
+        response = self.client.get(url)
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json(), {
+            'id': self.list.id, 'name': self.list.name, 'seen_items': []
+        })
+
+    def test_add_seen_item_to_mailing_list(self):
+        data = 'PEP-123'
+        url = reverse('api:bot:mailinglist-seen-items', args=(self.list.name,))
+        response = self.client.post(url, data=data)
+
+        self.assertEqual(response.status_code, 204)
+        self.list.refresh_from_db()
+        self.assertEqual(self.list.seen_items.first().hash, data)
+
+    def test_invalid_request_body(self):
+        data = [
+            "Dinoman, such tiny hands",
+            "He couldn't even ride a bike",
+            "He couldn't even dance",
+            "With the girl that he liked",
+            "He lived in tiny villages",
+            "And prayed to tiny god",
+            "He couldn't go to gameshow",
+            "Cause he could not applaud...",
+        ]
+        url = reverse('api:bot:mailinglist-seen-items', args=(self.list.name,))
+        response = self.client.post(url, data=data)
+        self.assertEqual(response.status_code, 400)
+        self.assertEqual(response.json(), {
+            'non_field_errors': ["The request body must be a string"]
+        })
+
+
+class MailingListWithSeenItemsTests(AuthenticatedAPITestCase):
+    @classmethod
+    def setUpTestData(cls):
+        cls.list = MailingList.objects.create(name='erlang-dev')
+        cls.seen_item = MailingListSeenItem.objects.create(hash='12345', list=cls.list)
+
+    def test_get_mailing_list(self):
+        url = reverse('api:bot:mailinglist-detail', args=(self.list.name,))
+        response = self.client.get(url)
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json(), {
+            'id': self.list.id, 'name': self.list.name, 'seen_items': [self.seen_item.hash]
+        })
+
+    def test_prevents_duplicate_addition_of_seen_item(self):
+        url = reverse('api:bot:mailinglist-seen-items', args=(self.list.name,))
+        response = self.client.post(url, data=self.seen_item.hash)
+
+        self.assertEqual(response.status_code, 400)
+        self.assertEqual(response.json(), {
+            'non_field_errors': ["Seen item already known."]
+        })

pydis_site/apps/api/urls.py

@@ -17,6 +17,7 @@
     FilterListViewSet,
     FilterViewSet,
     InfractionViewSet,
+    MailingListViewSet,
     NominationViewSet,
     OffTopicChannelNameViewSet,
     OffensiveMessageViewSet,
@@ -67,6 +68,10 @@
     'infractions',
     InfractionViewSet
 )
+bot_router.register(
+    'mailing-lists',
+    MailingListViewSet
+)
 bot_router.register(
     'nominations',
     NominationViewSet

pydis_site/apps/api/viewsets/__init__.py

@@ -1,17 +1,18 @@
 # flake8: noqa
 from .bot import (
+    AocAccountLinkViewSet,
+    AocCompletionistBlockViewSet,
     BotSettingViewSet,
     BumpedThreadViewSet,
     DeletedMessageViewSet,
     DocumentationLinkViewSet,
     FilterListViewSet,
-    InfractionViewSet,
     FilterListViewSet,
     FilterViewSet,
+    InfractionViewSet,
+    MailingListViewSet,
     NominationViewSet,
     OffensiveMessageViewSet,
-    AocAccountLinkViewSet,
-    AocCompletionistBlockViewSet,
     OffTopicChannelNameViewSet,
     ReminderViewSet,
     RoleViewSet,

pydis_site/apps/api/viewsets/bot/__init__.py

@@ -1,18 +1,16 @@
 # flake8: noqa
-from .filters import (
-    FilterListViewSet,
-    FilterViewSet
-)
+from .aoc_completionist_block import AocCompletionistBlockViewSet
+from .aoc_link import AocAccountLinkViewSet
 from .bot_setting import BotSettingViewSet
 from .bumped_thread import BumpedThreadViewSet
 from .deleted_message import DeletedMessageViewSet
 from .documentation_link import DocumentationLinkViewSet
+from .filters import FilterListViewSet, FilterViewSet
 from .infraction import InfractionViewSet
+from .mailing_list import MailingListViewSet
 from .nomination import NominationViewSet
 from .off_topic_channel_name import OffTopicChannelNameViewSet
 from .offensive_message import OffensiveMessageViewSet
-from .aoc_link import AocAccountLinkViewSet
-from .aoc_completionist_block import AocCompletionistBlockViewSet
 from .reminder import ReminderViewSet
 from .role import RoleViewSet
 from .user import UserViewSet