Admin: add a filter to ignore bulk created GPS groups and memberships

itou/gps/admin.py

@@ -8,25 +8,41 @@
 class MemberInline(admin.TabularInline):
     model = models.FollowUpGroup.members.through
 
-    fields = ["is_referent", "is_active", "member", "creator"]
+    fields = ["is_referent", "is_active", "creator"]
+    raw_id_fields = [
+        "member",
+    ]
 
-    readonly_fields = ["creator"]
+    readonly_fields = [
+        "creator",
+        "created_at",
+        "updated_at",
+    ]
 
 
 @admin.register(models.FollowUpGroupMembership)
 class FollowUpGroupMembershipAdmin(ItouModelAdmin):
-    list_display = ("created_at", "member", "follow_up_group", "is_referent")
-    list_filter = ("is_referent",)
+    list_display = ("created_at", "updated_at", "member", "follow_up_group", "is_referent")
+    list_filter = (
+        "is_referent",
+        "created_in_bulk",
+    )
     raw_id_fields = ["follow_up_group"]
-    readonly_fields = ["member", "creator", "created_at", "ended_at"]
+    readonly_fields = ["member", "creator", "created_at", "updated_at", "ended_at", "created_in_bulk"]
     ordering = ["-created_at"]
 
 
 @admin.register(models.FollowUpGroup)
 class FollowUpGroupAdmin(ItouModelAdmin):
     list_display = ("created_at", "updated_at", "beneficiary", "display_members")
-
-    fields = ["beneficiary"]
+    readonly_fields = [
+        "created_in_bulk",
+    ]
+    list_filter = ("created_in_bulk",)
+
+    raw_id_fields = [
+        "beneficiary",
+    ]
 
     inlines = (MemberInline,)
 

itou/gps/migrations/0001_initial.py

@@ -48,7 +48,7 @@ class Migration(migrations.Migration):
                     "created_at",
                     models.DateTimeField(default=django.utils.timezone.now, verbose_name="date de création"),
                 ),
-                ("ended_at", models.DateTimeField(null=True)),
+                ("ended_at", models.DateTimeField(null=True, verbose_name="date de désactivation")),
                 ("updated_at", models.DateTimeField(auto_now=True, verbose_name="date de modification")),
                 (
                     "creator",

itou/gps/migrations/0002_followupgroup_created_in_bulk_and_more.py

@@ -0,0 +1,37 @@
+# Generated by Django 5.0.6 on 2024-06-28 13:12
+
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("gps", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="followupgroup",
+            name="created_in_bulk",
+            field=models.BooleanField(db_index=True, default=False, verbose_name="créé massivement"),
+        ),
+        migrations.AddField(
+            model_name="followupgroupmembership",
+            name="created_in_bulk",
+            field=models.BooleanField(db_index=True, default=False, verbose_name="créé massivement"),
+        ),
+        migrations.AlterField(
+            model_name="followupgroup",
+            name="created_at",
+            field=models.DateTimeField(
+                db_index=True, default=django.utils.timezone.now, verbose_name="date de création"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="followupgroupmembership",
+            name="created_at",
+            field=models.DateTimeField(
+                db_index=True, default=django.utils.timezone.now, verbose_name="date de création"
+            ),
+        ),
+    ]

itou/gps/migrations/0003_fill_groups_created_in_bulk.py

@@ -0,0 +1,55 @@
+# Generated by Django 5.0.6 on 2024-06-27 12:53
+import datetime
+import logging
+import time
+
+from django.conf import settings
+from django.db import migrations
+from django.db.models import Q
+
+
+logger = logging.getLogger(__name__)
+
+
+def _bulk_created_lookup():
+    created_at_as_dt = datetime.datetime.combine(
+        settings.GPS_GROUPS_CREATED_AT_DATE, datetime.time(15, 0, 0), tzinfo=datetime.UTC
+    )
+    return Q(created_at__lte=created_at_as_dt)
+
+
+def _update_follow_up_groups_created_in_bulk(apps, schema_editor):
+    FollowUpGroup = apps.get_model("gps", "FollowUpGroup")
+    groups = FollowUpGroup.objects.filter(_bulk_created_lookup()).exclude(created_in_bulk=True)
+
+    count = 0
+    start = time.perf_counter()
+    while batch_groups := groups[:10000]:
+        count += FollowUpGroup.objects.filter(pk__in=batch_groups.values_list("pk", flat=True)).update(
+            created_in_bulk=True
+        )
+        logger.info(f"{count} groups migrated in {time.perf_counter() - start:.2f} sec")
+
+
+def _update_groups_memberships_created_in_bulk(apps, schema_editor):
+    FollowUpGroupMembership = apps.get_model("gps", "FollowUpGroupMembership")
+    memberships = FollowUpGroupMembership.objects.filter(_bulk_created_lookup()).exclude(created_in_bulk=True)
+
+    count = 0
+    start = time.perf_counter()
+    while batch_memberships := memberships[:10000]:
+        count += FollowUpGroupMembership.objects.filter(pk__in=batch_memberships.values_list("pk", flat=True)).update(
+            created_in_bulk=True
+        )
+        logger.info(f"{count} memberships migrated in {time.perf_counter() - start:.2f} sec")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("gps", "0002_followupgroup_created_in_bulk_and_more"),
+    ]
+
+    operations = [
+        migrations.RunPython(_update_follow_up_groups_created_in_bulk, migrations.RunPython.noop, elidable=True),
+        migrations.RunPython(_update_groups_memberships_created_in_bulk, migrations.RunPython.noop, elidable=True),
+    ]

itou/gps/migrations/0004_alter_followupgroup_created_at_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.0.6 on 2024-06-28 13:13
+
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("gps", "0003_fill_groups_created_in_bulk"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="followupgroup",
+            name="created_at",
+            field=models.DateTimeField(default=django.utils.timezone.now, verbose_name="date de création"),
+        ),
+        migrations.AlterField(
+            model_name="followupgroupmembership",
+            name="created_at",
+            field=models.DateTimeField(default=django.utils.timezone.now, verbose_name="date de création"),
+        ),
+    ]

itou/gps/models.py

@@ -1,11 +1,18 @@
-from django.conf import settings
 from django.contrib.postgres.aggregates import ArrayAgg
 from django.db import models, transaction
 from django.utils import timezone
 
 from itou.users.models import User
 
 
+class BulkCreatedAtQuerysetProxy:
+    def bulk_created(self):
+        return self.filter(created_in_bulk=True)
+
+    def not_bulk_created(self):
+        return self.exclude(created_in_bulk=True)
+
+
 class FollowUpGroupManager(models.Manager):
     def follow_beneficiary(self, beneficiary, user, is_referent=False):
         with transaction.atomic():
@@ -23,10 +30,15 @@ def follow_beneficiary(self, beneficiary, user, is_referent=False):
                 )
 
 
+class FollowUpGroupQueryset(BulkCreatedAtQuerysetProxy, models.QuerySet):
+    pass
+
+
 class FollowUpGroup(models.Model):
     created_at = models.DateTimeField(verbose_name="date de création", default=timezone.now)
+    created_in_bulk = models.BooleanField(verbose_name="créé massivement", default=False, db_index=True)
 
-    objects = FollowUpGroupManager()
+    objects = FollowUpGroupManager.from_queryset(FollowUpGroupQueryset)()
 
     updated_at = models.DateTimeField(verbose_name="date de modification", auto_now=True)
 
@@ -54,7 +66,7 @@ def __str__(self):
         return "Groupe de " + self.beneficiary.get_full_name()
 
 
-class FollowUpGroupMembershipQueryset(models.QuerySet):
+class FollowUpGroupMembershipQueryset(BulkCreatedAtQuerysetProxy, models.QuerySet):
     def with_members_organizations_names(self):
         qs = self.annotate(
             prescriber_org_names=ArrayAgg(
@@ -82,9 +94,10 @@ class Meta:
     is_active = models.BooleanField(default=True, verbose_name="actif")
 
     created_at = models.DateTimeField(verbose_name="date de création", default=timezone.now)
+    created_in_bulk = models.BooleanField(verbose_name="créé massivement", default=False, db_index=True)
 
     # Keep track of when the membership was ended
-    ended_at = models.DateTimeField(null=True)
+    ended_at = models.DateTimeField(verbose_name="date de désactivation", null=True)
 
     updated_at = models.DateTimeField(verbose_name="date de modification", auto_now=True)
 
@@ -118,7 +131,3 @@ def __str__(self):
     @property
     def organization_name(self):
         return next((name for name in (*self.prescriber_org_names, *self.companies_names) if name), None)
-
-    @property
-    def is_from_bulk_creation(self):
-        return self.created_at.date() == settings.GPS_GROUPS_CREATED_AT_DATE

itou/templates/gps/my_groups.html

@@ -81,7 +81,7 @@ <h3>{{ membership.follow_up_group.beneficiary.get_full_name }}</h3>
                                     <div>
                                         {% with membership.nb_members|add:"-1" as counter %}
                                             <div>
-                                                {% if not membership.is_from_bulk_creation %}
+                                                {% if not membership.created_in_bulk %}
                                                     <div>
                                                         {# djlint:off #}
                                                         {# Don't let djlint add a newline before the . or it will add a space after référent and . #}

tests/gps/factories.py

@@ -2,6 +2,7 @@
 
 import factory.fuzzy
 from django.conf import settings
+from django.utils import timezone
 
 from itou.gps.models import FollowUpGroup, FollowUpGroupMembership
 from tests.users.factories import JobSeekerFactory, PrescriberFactory
@@ -15,17 +16,20 @@ class Meta:
         skip_postgeneration_save = True
 
     class Params:
-        created_in_bulk = factory.Trait(
-            created_at=(
-                datetime.datetime.combine(settings.GPS_GROUPS_CREATED_AT_DATE, datetime.time(), tzinfo=datetime.UTC)
-            )
-        )
         for_snapshot = factory.Trait(
             beneficiary__for_snapshot=True,
             created_at=datetime.datetime(2024, 6, 21, 0, 0, 0, tzinfo=datetime.UTC),
         )
 
     beneficiary = factory.SubFactory(JobSeekerFactory)
+    created_at = factory.LazyAttribute(
+        lambda o: datetime.datetime.combine(
+            settings.GPS_GROUPS_CREATED_AT_DATE, datetime.time(12, 0, 0), tzinfo=datetime.UTC
+        )
+        if o.created_in_bulk
+        else timezone.now()
+    )
+    created_in_bulk = False
 
     @factory.post_generation
     def memberships(self, create, extracted, **kwargs):
@@ -41,20 +45,24 @@ def memberships(self, create, extracted, **kwargs):
                     creator=PrescriberFactory(),
                     follow_up_group=self,
                     is_referent=True if i == 0 else False,
+                    created_at=self.created_at,
+                    created_in_bulk=self.created_in_bulk,
                 )
 
 
 class FollowUpGroupMembershipFactory(factory.django.DjangoModelFactory):
     class Meta:
         model = FollowUpGroupMembership
 
-    class Params:
-        created_in_bulk = factory.Trait(
-            created_at=(
-                datetime.datetime.combine(settings.GPS_GROUPS_CREATED_AT_DATE, datetime.time(), tzinfo=datetime.UTC)
-            )
+    created_at = factory.LazyAttribute(
+        lambda o: datetime.datetime.combine(
+            settings.GPS_GROUPS_CREATED_AT_DATE, datetime.time(12, 0, 0), tzinfo=datetime.UTC
         )
+        if o.created_in_bulk
+        else timezone.now()
+    )
 
+    created_in_bulk = False
     follow_up_group = factory.SubFactory(FollowUpGroupFactory)
     member = factory.SubFactory(PrescriberFactory)
     creator = factory.SubFactory(PrescriberFactory)

tests/gps/test_models.py

@@ -1,12 +1,9 @@
-import datetime
-
 import pytest
-from django.conf import settings
 from pytest_django.asserts import assertNumQueries
 
 from itou.gps.models import FollowUpGroup, FollowUpGroupMembership
 from tests.companies.factories import CompanyMembershipFactory
-from tests.gps.factories import FollowUpGroupFactory, FollowUpGroupMembershipFactory
+from tests.gps.factories import FollowUpGroupFactory
 from tests.prescribers.factories import PrescriberMembershipFactory
 from tests.users.factories import (
     EmployerFactory,
@@ -15,14 +12,14 @@
 )
 
 
-def test_membership_is_from_bulk_creation():
-    membership = FollowUpGroupMembershipFactory()
-    assert not membership.is_from_bulk_creation
+def test_bulk_created():
+    FollowUpGroupFactory.create_batch(2, memberships=2)  # 4 memberships
+    FollowUpGroupFactory.create_batch(3, created_in_bulk=True, memberships=2)  # 6 memberships
+    assert FollowUpGroup.objects.not_bulk_created().count() == 2
+    assert FollowUpGroup.objects.bulk_created().count() == 3
 
-    membership.created_at = datetime.datetime.combine(
-        settings.GPS_GROUPS_CREATED_AT_DATE, datetime.time(), tzinfo=datetime.UTC
-    )
-    assert membership.is_from_bulk_creation
+    assert FollowUpGroupMembership.objects.not_bulk_created().count() == 4
+    assert FollowUpGroupMembership.objects.bulk_created().count() == 6
 
 
 def test_follow_beneficiary():

tests/gps/test_views.py

@@ -202,11 +202,13 @@ def test_my_groups(snapshot, client):
         beneficiary__last_name="de Lucia",
         created_in_bulk=True,
     )
+
     FollowUpGroup.objects.follow_beneficiary(beneficiary=group.beneficiary, user=user, is_referent=True)
     membership = group.memberships.get(member=user)
     membership.created_at = datetime.datetime.combine(
         settings.GPS_GROUPS_CREATED_AT_DATE, datetime.time(), tzinfo=datetime.UTC
     )
+    membership.created_in_bulk = True
     membership.save()
 
     response = client.get(reverse("gps:my_groups"))