From a0bc470d11f3eda4f31254dc09ee638968c98630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9line=20MS?= Date: Mon, 24 Jun 2024 17:55:50 +0200 Subject: [PATCH] Admin: add a filter to ignore bulk created GPS groups and memberships --- itou/gps/admin.py | 30 +++++++--- itou/gps/migrations/0001_initial.py | 2 +- ..._followupgroup_created_in_bulk_and_more.py | 37 +++++++++++++ .../0003_fill_groups_created_in_bulk.py | 55 +++++++++++++++++++ ...alter_followupgroup_created_at_and_more.py | 23 ++++++++ itou/gps/models.py | 25 ++++++--- itou/templates/gps/my_groups.html | 2 +- tests/gps/factories.py | 28 ++++++---- tests/gps/test_models.py | 19 +++---- tests/gps/test_views.py | 2 + 10 files changed, 185 insertions(+), 38 deletions(-) create mode 100644 itou/gps/migrations/0002_followupgroup_created_in_bulk_and_more.py create mode 100644 itou/gps/migrations/0003_fill_groups_created_in_bulk.py create mode 100644 itou/gps/migrations/0004_alter_followupgroup_created_at_and_more.py diff --git a/itou/gps/admin.py b/itou/gps/admin.py index 2ee4710a452..a88e568991c 100644 --- a/itou/gps/admin.py +++ b/itou/gps/admin.py @@ -8,25 +8,41 @@ class MemberInline(admin.TabularInline): model = models.FollowUpGroup.members.through - fields = ["is_referent", "is_active", "member", "creator"] + fields = ["is_referent", "is_active", "creator"] + raw_id_fields = [ + "member", + ] - readonly_fields = ["creator"] + readonly_fields = [ + "creator", + "created_at", + "updated_at", + ] @admin.register(models.FollowUpGroupMembership) class FollowUpGroupMembershipAdmin(ItouModelAdmin): - list_display = ("created_at", "member", "follow_up_group", "is_referent") - list_filter = ("is_referent",) + list_display = ("created_at", "updated_at", "member", "follow_up_group", "is_referent") + list_filter = ( + "is_referent", + "created_in_bulk", + ) raw_id_fields = ["follow_up_group"] - readonly_fields = ["member", "creator", "created_at", "ended_at"] + readonly_fields = ["member", "creator", "created_at", "updated_at", "ended_at", "created_in_bulk"] ordering = ["-created_at"] @admin.register(models.FollowUpGroup) class FollowUpGroupAdmin(ItouModelAdmin): list_display = ("created_at", "updated_at", "beneficiary", "display_members") - - fields = ["beneficiary"] + readonly_fields = [ + "created_in_bulk", + ] + list_filter = ("created_in_bulk",) + + raw_id_fields = [ + "beneficiary", + ] inlines = (MemberInline,) diff --git a/itou/gps/migrations/0001_initial.py b/itou/gps/migrations/0001_initial.py index aa41611ca29..8c75da1dfb4 100644 --- a/itou/gps/migrations/0001_initial.py +++ b/itou/gps/migrations/0001_initial.py @@ -48,7 +48,7 @@ class Migration(migrations.Migration): "created_at", models.DateTimeField(default=django.utils.timezone.now, verbose_name="date de création"), ), - ("ended_at", models.DateTimeField(null=True)), + ("ended_at", models.DateTimeField(null=True, verbose_name="date de désactivation")), ("updated_at", models.DateTimeField(auto_now=True, verbose_name="date de modification")), ( "creator", diff --git a/itou/gps/migrations/0002_followupgroup_created_in_bulk_and_more.py b/itou/gps/migrations/0002_followupgroup_created_in_bulk_and_more.py new file mode 100644 index 00000000000..eadec501806 --- /dev/null +++ b/itou/gps/migrations/0002_followupgroup_created_in_bulk_and_more.py @@ -0,0 +1,37 @@ +# Generated by Django 5.0.6 on 2024-06-28 13:12 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("gps", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="followupgroup", + name="created_in_bulk", + field=models.BooleanField(db_index=True, default=False, verbose_name="créé massivement"), + ), + migrations.AddField( + model_name="followupgroupmembership", + name="created_in_bulk", + field=models.BooleanField(db_index=True, default=False, verbose_name="créé massivement"), + ), + migrations.AlterField( + model_name="followupgroup", + name="created_at", + field=models.DateTimeField( + db_index=True, default=django.utils.timezone.now, verbose_name="date de création" + ), + ), + migrations.AlterField( + model_name="followupgroupmembership", + name="created_at", + field=models.DateTimeField( + db_index=True, default=django.utils.timezone.now, verbose_name="date de création" + ), + ), + ] diff --git a/itou/gps/migrations/0003_fill_groups_created_in_bulk.py b/itou/gps/migrations/0003_fill_groups_created_in_bulk.py new file mode 100644 index 00000000000..a1d66d194fe --- /dev/null +++ b/itou/gps/migrations/0003_fill_groups_created_in_bulk.py @@ -0,0 +1,55 @@ +# Generated by Django 5.0.6 on 2024-06-27 12:53 +import datetime +import logging +import time + +from django.conf import settings +from django.db import migrations +from django.db.models import Q + + +logger = logging.getLogger(__name__) + + +def _bulk_created_lookup(): + created_at_as_dt = datetime.datetime.combine( + settings.GPS_GROUPS_CREATED_AT_DATE, datetime.time(15, 0, 0), tzinfo=datetime.UTC + ) + return Q(created_at__lte=created_at_as_dt) + + +def _update_follow_up_groups_created_in_bulk(apps, schema_editor): + FollowUpGroup = apps.get_model("gps", "FollowUpGroup") + groups = FollowUpGroup.objects.filter(_bulk_created_lookup()).exclude(created_in_bulk=True) + + count = 0 + start = time.perf_counter() + while batch_groups := groups[:10000]: + count += FollowUpGroup.objects.filter(pk__in=batch_groups.values_list("pk", flat=True)).update( + created_in_bulk=True + ) + logger.info(f"{count} groups migrated in {time.perf_counter() - start:.2f} sec") + + +def _update_groups_memberships_created_in_bulk(apps, schema_editor): + FollowUpGroupMembership = apps.get_model("gps", "FollowUpGroupMembership") + memberships = FollowUpGroupMembership.objects.filter(_bulk_created_lookup()).exclude(created_in_bulk=True) + + count = 0 + start = time.perf_counter() + while batch_memberships := memberships[:10000]: + count += FollowUpGroupMembership.objects.filter(pk__in=batch_memberships.values_list("pk", flat=True)).update( + created_in_bulk=True + ) + logger.info(f"{count} memberships migrated in {time.perf_counter() - start:.2f} sec") + + +class Migration(migrations.Migration): + dependencies = [ + ("gps", "0002_followupgroup_created_in_bulk_and_more"), + ] + + operations = [ + migrations.RunPython(_update_follow_up_groups_created_in_bulk, migrations.RunPython.noop, elidable=True), + migrations.RunPython(_update_groups_memberships_created_in_bulk, migrations.RunPython.noop, elidable=True), + ] diff --git a/itou/gps/migrations/0004_alter_followupgroup_created_at_and_more.py b/itou/gps/migrations/0004_alter_followupgroup_created_at_and_more.py new file mode 100644 index 00000000000..16db482db86 --- /dev/null +++ b/itou/gps/migrations/0004_alter_followupgroup_created_at_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 5.0.6 on 2024-06-28 13:13 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("gps", "0003_fill_groups_created_in_bulk"), + ] + + operations = [ + migrations.AlterField( + model_name="followupgroup", + name="created_at", + field=models.DateTimeField(default=django.utils.timezone.now, verbose_name="date de création"), + ), + migrations.AlterField( + model_name="followupgroupmembership", + name="created_at", + field=models.DateTimeField(default=django.utils.timezone.now, verbose_name="date de création"), + ), + ] diff --git a/itou/gps/models.py b/itou/gps/models.py index 32864a970f8..e66ca280a32 100644 --- a/itou/gps/models.py +++ b/itou/gps/models.py @@ -1,4 +1,3 @@ -from django.conf import settings from django.contrib.postgres.aggregates import ArrayAgg from django.db import models, transaction from django.utils import timezone @@ -6,6 +5,14 @@ from itou.users.models import User +class BulkCreatedAtQuerysetProxy: + def bulk_created(self): + return self.filter(created_in_bulk=True) + + def not_bulk_created(self): + return self.exclude(created_in_bulk=True) + + class FollowUpGroupManager(models.Manager): def follow_beneficiary(self, beneficiary, user, is_referent=False): with transaction.atomic(): @@ -23,10 +30,15 @@ def follow_beneficiary(self, beneficiary, user, is_referent=False): ) +class FollowUpGroupQueryset(BulkCreatedAtQuerysetProxy, models.QuerySet): + pass + + class FollowUpGroup(models.Model): created_at = models.DateTimeField(verbose_name="date de création", default=timezone.now) + created_in_bulk = models.BooleanField(verbose_name="créé massivement", default=False, db_index=True) - objects = FollowUpGroupManager() + objects = FollowUpGroupManager.from_queryset(FollowUpGroupQueryset)() updated_at = models.DateTimeField(verbose_name="date de modification", auto_now=True) @@ -54,7 +66,7 @@ def __str__(self): return "Groupe de " + self.beneficiary.get_full_name() -class FollowUpGroupMembershipQueryset(models.QuerySet): +class FollowUpGroupMembershipQueryset(BulkCreatedAtQuerysetProxy, models.QuerySet): def with_members_organizations_names(self): qs = self.annotate( prescriber_org_names=ArrayAgg( @@ -82,9 +94,10 @@ class Meta: is_active = models.BooleanField(default=True, verbose_name="actif") created_at = models.DateTimeField(verbose_name="date de création", default=timezone.now) + created_in_bulk = models.BooleanField(verbose_name="créé massivement", default=False, db_index=True) # Keep track of when the membership was ended - ended_at = models.DateTimeField(null=True) + ended_at = models.DateTimeField(verbose_name="date de désactivation", null=True) updated_at = models.DateTimeField(verbose_name="date de modification", auto_now=True) @@ -118,7 +131,3 @@ def __str__(self): @property def organization_name(self): return next((name for name in (*self.prescriber_org_names, *self.companies_names) if name), None) - - @property - def is_from_bulk_creation(self): - return self.created_at.date() == settings.GPS_GROUPS_CREATED_AT_DATE diff --git a/itou/templates/gps/my_groups.html b/itou/templates/gps/my_groups.html index 1ff562c62d0..0d70cc16e26 100644 --- a/itou/templates/gps/my_groups.html +++ b/itou/templates/gps/my_groups.html @@ -81,7 +81,7 @@

{{ membership.follow_up_group.beneficiary.get_full_name }}

{% with membership.nb_members|add:"-1" as counter %}
- {% if not membership.is_from_bulk_creation %} + {% if not membership.created_in_bulk %}
{# djlint:off #} {# Don't let djlint add a newline before the . or it will add a space after référent and . #} diff --git a/tests/gps/factories.py b/tests/gps/factories.py index 77675296aca..d6740f30b8d 100644 --- a/tests/gps/factories.py +++ b/tests/gps/factories.py @@ -2,6 +2,7 @@ import factory.fuzzy from django.conf import settings +from django.utils import timezone from itou.gps.models import FollowUpGroup, FollowUpGroupMembership from tests.users.factories import JobSeekerFactory, PrescriberFactory @@ -15,17 +16,20 @@ class Meta: skip_postgeneration_save = True class Params: - created_in_bulk = factory.Trait( - created_at=( - datetime.datetime.combine(settings.GPS_GROUPS_CREATED_AT_DATE, datetime.time(), tzinfo=datetime.UTC) - ) - ) for_snapshot = factory.Trait( beneficiary__for_snapshot=True, created_at=datetime.datetime(2024, 6, 21, 0, 0, 0, tzinfo=datetime.UTC), ) beneficiary = factory.SubFactory(JobSeekerFactory) + created_at = factory.LazyAttribute( + lambda o: datetime.datetime.combine( + settings.GPS_GROUPS_CREATED_AT_DATE, datetime.time(12, 0, 0), tzinfo=datetime.UTC + ) + if o.created_in_bulk + else timezone.now() + ) + created_in_bulk = False @factory.post_generation def memberships(self, create, extracted, **kwargs): @@ -41,6 +45,8 @@ def memberships(self, create, extracted, **kwargs): creator=PrescriberFactory(), follow_up_group=self, is_referent=True if i == 0 else False, + created_at=self.created_at, + created_in_bulk=self.created_in_bulk, ) @@ -48,13 +54,15 @@ class FollowUpGroupMembershipFactory(factory.django.DjangoModelFactory): class Meta: model = FollowUpGroupMembership - class Params: - created_in_bulk = factory.Trait( - created_at=( - datetime.datetime.combine(settings.GPS_GROUPS_CREATED_AT_DATE, datetime.time(), tzinfo=datetime.UTC) - ) + created_at = factory.LazyAttribute( + lambda o: datetime.datetime.combine( + settings.GPS_GROUPS_CREATED_AT_DATE, datetime.time(12, 0, 0), tzinfo=datetime.UTC ) + if o.created_in_bulk + else timezone.now() + ) + created_in_bulk = False follow_up_group = factory.SubFactory(FollowUpGroupFactory) member = factory.SubFactory(PrescriberFactory) creator = factory.SubFactory(PrescriberFactory) diff --git a/tests/gps/test_models.py b/tests/gps/test_models.py index 721bb874c12..9af01c05f9f 100644 --- a/tests/gps/test_models.py +++ b/tests/gps/test_models.py @@ -1,12 +1,9 @@ -import datetime - import pytest -from django.conf import settings from pytest_django.asserts import assertNumQueries from itou.gps.models import FollowUpGroup, FollowUpGroupMembership from tests.companies.factories import CompanyMembershipFactory -from tests.gps.factories import FollowUpGroupFactory, FollowUpGroupMembershipFactory +from tests.gps.factories import FollowUpGroupFactory from tests.prescribers.factories import PrescriberMembershipFactory from tests.users.factories import ( EmployerFactory, @@ -15,14 +12,14 @@ ) -def test_membership_is_from_bulk_creation(): - membership = FollowUpGroupMembershipFactory() - assert not membership.is_from_bulk_creation +def test_bulk_created(): + FollowUpGroupFactory.create_batch(2, memberships=2) # 4 memberships + FollowUpGroupFactory.create_batch(3, created_in_bulk=True, memberships=2) # 6 memberships + assert FollowUpGroup.objects.not_bulk_created().count() == 2 + assert FollowUpGroup.objects.bulk_created().count() == 3 - membership.created_at = datetime.datetime.combine( - settings.GPS_GROUPS_CREATED_AT_DATE, datetime.time(), tzinfo=datetime.UTC - ) - assert membership.is_from_bulk_creation + assert FollowUpGroupMembership.objects.not_bulk_created().count() == 4 + assert FollowUpGroupMembership.objects.bulk_created().count() == 6 def test_follow_beneficiary(): diff --git a/tests/gps/test_views.py b/tests/gps/test_views.py index 847eb012066..ee4cb9d6b8a 100644 --- a/tests/gps/test_views.py +++ b/tests/gps/test_views.py @@ -202,11 +202,13 @@ def test_my_groups(snapshot, client): beneficiary__last_name="de Lucia", created_in_bulk=True, ) + FollowUpGroup.objects.follow_beneficiary(beneficiary=group.beneficiary, user=user, is_referent=True) membership = group.memberships.get(member=user) membership.created_at = datetime.datetime.combine( settings.GPS_GROUPS_CREATED_AT_DATE, datetime.time(), tzinfo=datetime.UTC ) + membership.created_in_bulk = True membership.save() response = client.get(reverse("gps:my_groups"))