From a9780ae185a5dce07b21b2491a5186d117ae55df Mon Sep 17 00:00:00 2001 From: Julian Dehm Date: Wed, 15 Feb 2023 16:59:54 +0100 Subject: [PATCH 1/3] codes: delete in background tasks --- meinberlin/apps/votes/admin.py | 15 +- meinberlin/apps/votes/exports.py | 33 ++-- .../votes/migrations/0006_token_packages.py | 145 ++++++++++++++++++ meinberlin/apps/votes/models.py | 42 +++-- meinberlin/apps/votes/tasks.py | 57 +++---- .../token_export_dashboard.html | 30 ++-- meinberlin/apps/votes/views.py | 43 ++---- meinberlin/fixtures/votes.json | 39 ++++- meinberlin/test/factories/votes.py | 11 +- tests/budgeting/conftest.py | 1 + tests/votes/conftest.py | 1 + .../dashboard_components/test_token_export.py | 55 ++++--- .../test_token_generation.py | 102 ++++++------ 13 files changed, 398 insertions(+), 176 deletions(-) create mode 100644 meinberlin/apps/votes/migrations/0006_token_packages.py diff --git a/meinberlin/apps/votes/admin.py b/meinberlin/apps/votes/admin.py index 5eb542e7cf..53d7b394f0 100644 --- a/meinberlin/apps/votes/admin.py +++ b/meinberlin/apps/votes/admin.py @@ -1,5 +1,6 @@ from django.contrib import admin +from .models import TokenPackage from .models import TokenVote from .models import VotingToken @@ -11,19 +12,25 @@ class VotingTokenAdmin(admin.ModelAdmin): "token_hash", "module", "is_active", - "package_number", + "package", "allowed_votes", ) - readonly_fields = ("token", "token_hash", "package_number") + readonly_fields = ("token", "token_hash", "package") list_display = ("pk", "__str__", "project", "module", "module_name", "is_active") list_filter = ("module__project",) + def delete_model(self, request, obj): + obj.package.size -= 1 + obj.package.save() + super().delete_model(request, obj) + def module_name(self, token): return token.module.name def save_model(self, request, obj, form, change): - if obj.package_number is None: - obj.package_number = VotingToken.next_package_number(obj.module) + if not hasattr(obj, "package"): + obj.package = TokenPackage.objects.create(module=obj.module, size=1) + if not obj.token_hash: obj.token_hash = VotingToken.hash_token(obj.token, obj.module) super().save_model(request, obj, form, change) diff --git a/meinberlin/apps/votes/exports.py b/meinberlin/apps/votes/exports.py index 80e5448197..5d7aef6940 100644 --- a/meinberlin/apps/votes/exports.py +++ b/meinberlin/apps/votes/exports.py @@ -5,7 +5,9 @@ from adhocracy4.exports.views import AbstractXlsxExportView from adhocracy4.projects.mixins import ProjectMixin +from meinberlin.apps.votes.models import TokenPackage from meinberlin.apps.votes.models import VotingToken +from meinberlin.apps.votes.tasks import delete_plain_codes class TokenExportView( @@ -20,30 +22,36 @@ class TokenExportView( def get_permission_object(self): return self.module.project - def get_package_number(self): - package_number = self.request.GET.get("package") or 0 + def get_package(self): try: - return int(package_number) + pk = int(self.request.GET.get("package")) + return TokenPackage.objects.get(pk=pk) except ValueError: return None def get_queryset(self): """Filter QS to only include active tokens from module.""" - package_number = self.get_package_number() - if package_number is None: + package = self.get_package() + if package is None: return None return ( super() .get_queryset() - .filter(module=self.module, is_active=True, package_number=package_number) + .filter( + module=self.module, + is_active=True, + package=package, + ) .exclude(token="") .order_by("pk") ) def get_base_filename(self): - package_number = self.get_package_number() - if not package_number: - package_number = 0 + package = self.get_package() + package_number = ( + TokenPackage.objects.filter(module=self.module, pk__lt=package.pk).count() + + 1 + ) return "%s_package_%s" % (self.project.slug, package_number) def get_header(self): @@ -55,11 +63,14 @@ def export_rows(self): queryset = self.get_queryset() # raise BadRequest as either package has been downloaded # already or wrong package number - if queryset is None or queryset.count() == 0: + package = self.get_package() + if queryset is None or queryset.count() == 0 or not package.is_created: raise BadRequest for item in queryset: yield [self.get_token_data(item)] - queryset.update(token="") + package.downloaded = True + package.save() + delete_plain_codes(package.pk) def get_token_data(self, item): """Add dashes like in string method.""" diff --git a/meinberlin/apps/votes/migrations/0006_token_packages.py b/meinberlin/apps/votes/migrations/0006_token_packages.py new file mode 100644 index 0000000000..f1dbfd22f6 --- /dev/null +++ b/meinberlin/apps/votes/migrations/0006_token_packages.py @@ -0,0 +1,145 @@ +# Generated by Django 3.2.17 on 2023-02-17 11:35 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import meinberlin.apps.votes.models + + +class Migration(migrations.Migration): + dependencies = [ + ("a4modules", "0006_module_blueprint_type"), + ("contenttypes", "0002_remove_content_type_name"), + ("meinberlin_votes", "0005_alter_votingtoken_token"), + ] + + operations = [ + migrations.DeleteModel( + name="VotingToken", + ), + migrations.DeleteModel( + name="TokenVote", + ), + migrations.CreateModel( + name="TokenPackage", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("size", models.PositiveIntegerField()), + ("downloaded", models.BooleanField(default=False)), + ( + "module", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="a4modules.module", + ), + ), + ], + options={ + "ordering": ["pk"], + }, + ), + migrations.CreateModel( + name="VotingToken", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "token", + models.CharField( + blank=True, + default=meinberlin.apps.votes.models.get_token_16, + editable=False, + max_length=40, + ), + ), + ( + "token_hash", + models.CharField(editable=False, max_length=128, unique=True), + ), + ("allowed_votes", models.PositiveSmallIntegerField(default=5)), + ( + "is_active", + models.BooleanField( + default=True, + help_text="Designates whether this token should be treated as active. Unselect this instead of deleting tokens.", + ), + ), + ( + "module", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="a4modules.module", + ), + ), + ( + "package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="meinberlin_votes.tokenpackage", + ), + ), + ], + ), + migrations.CreateModel( + name="TokenVote", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + models.DateTimeField( + default=django.utils.timezone.now, + editable=False, + verbose_name="Created", + ), + ), + ( + "modified", + models.DateTimeField( + blank=True, editable=False, null=True, verbose_name="Modified" + ), + ), + ("object_pk", models.PositiveIntegerField()), + ( + "content_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="contenttypes.contenttype", + ), + ), + ( + "token", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="meinberlin_votes.votingtoken", + ), + ), + ], + options={ + "unique_together": {("content_type", "object_pk", "token")}, + "index_together": {("content_type", "object_pk")}, + }, + ), + ] diff --git a/meinberlin/apps/votes/models.py b/meinberlin/apps/votes/models.py index 03a71f809c..fc0a8edb88 100644 --- a/meinberlin/apps/votes/models.py +++ b/meinberlin/apps/votes/models.py @@ -6,6 +6,7 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError from django.db import models +from django.db.models import Sum from django.db.utils import IntegrityError from django.utils.translation import gettext_lazy as _ @@ -56,6 +57,34 @@ def get_or_create_salt_for_module(module): return token_salt.salt +class TokenPackage(models.Model): + module = models.ForeignKey(Module, on_delete=models.CASCADE) + size = models.PositiveIntegerField() + downloaded = models.BooleanField(default=False) + + class Meta: + ordering = ["pk"] + + @property + def num_tokens(self): + return VotingToken.objects.filter(package=self).count() + + @property + def is_created(self): + return self.num_tokens == self.size + + @staticmethod + def get_sum_token(module): + num_tokens = ( + TokenPackage.objects.filter(module=module) + .aggregate(Sum("size")) + .get("size__sum") + ) + if num_tokens is None: + num_tokens = 0 + return num_tokens + + class VotingToken(models.Model): token = models.CharField( max_length=40, default=get_token_16, blank=True, editable=False @@ -63,7 +92,7 @@ class VotingToken(models.Model): token_hash = models.CharField(max_length=128, editable=False, unique=True) module = models.ForeignKey(Module, on_delete=models.CASCADE) allowed_votes = models.PositiveSmallIntegerField(default=5) - package_number = models.PositiveIntegerField() + package = models.ForeignKey(TokenPackage, on_delete=models.CASCADE) is_active = models.BooleanField( default=True, help_text=_( @@ -108,17 +137,6 @@ def project(self): def is_valid_for_item(self, item): return item.module == self.module - @staticmethod - def next_package_number(module): - token = ( - VotingToken.objects.filter(module=module) - .order_by("-package_number") - .first() - ) - if token: - return token.package_number + 1 - return 0 - @staticmethod def hash_token(token, module): salt = TokenSalt.get_or_create_salt_for_module(module) diff --git a/meinberlin/apps/votes/tasks.py b/meinberlin/apps/votes/tasks.py index dc1c20304e..034e4a6c20 100644 --- a/meinberlin/apps/votes/tasks.py +++ b/meinberlin/apps/votes/tasks.py @@ -1,79 +1,72 @@ from background_task import background from adhocracy4.modules.models import Module +from meinberlin.apps.votes.models import TokenPackage from meinberlin.apps.votes.models import VotingToken from meinberlin.apps.votes.models import get_token_16 # Number of tokens to insert into database per bulk_create -BATCH_SIZE = int(1e3) +BATCH_SIZE = int(1e5) # Max number of tokens in one download / package -PACKAGE_SIZE = int(1e4) +PACKAGE_SIZE = int(1e6) -def generate_voting_tokens(module_id, number_of_tokens, existing_tokens): +def generate_voting_tokens(module_id, number_of_tokens): module = Module.objects.get(pk=module_id) - module_name = module.name - project_id = module.project.id - project_name = module.project.name number_to_generate = number_of_tokens - # for votes.models.VotingToken.package_number, used for download - package_number = VotingToken.next_package_number(module) # determine when to go to next package_number package_number_limit = 0 - + package_size = number_of_tokens if number_of_tokens > PACKAGE_SIZE: package_number_limit = number_of_tokens - PACKAGE_SIZE + package_size = PACKAGE_SIZE + package = TokenPackage.objects.create(module=module, size=package_size) + while number_to_generate > 0: if number_to_generate >= BATCH_SIZE: generate_voting_tokens_batch( module_id, BATCH_SIZE, - package_number, - number_of_tokens, - module_name, - project_id, - project_name, - existing_tokens, + package.pk, ) number_to_generate = number_to_generate - BATCH_SIZE else: generate_voting_tokens_batch( module_id, number_to_generate, - package_number, - number_of_tokens, - module_name, - project_id, - project_name, - existing_tokens, + package.pk, ) number_to_generate = 0 - if package_number_limit >= number_to_generate: - package_number += 1 + if package_number_limit > 0 and package_number_limit >= number_to_generate: package_number_limit = package_number_limit - PACKAGE_SIZE + if package_number_limit < PACKAGE_SIZE: + package_size = number_to_generate + package = TokenPackage.objects.create(module=module, size=package_size) @background(schedule=1) def generate_voting_tokens_batch( module_id, batch_size, - package_number, - number_of_tokens, - module_name, - project_id, - project_name, - existing_tokens, + package_id, ): module = Module.objects.get(pk=module_id) + package = TokenPackage.objects.get(pk=package_id) VotingToken.objects.bulk_create( - [get_token_and_hash(module, package_number) for i in range(batch_size)] + [get_token_and_hash(module, package) for i in range(batch_size)] ) -def get_token_and_hash(module, package_number): +def get_token_and_hash(module, package): token = get_token_16() token_hash = VotingToken.hash_token(token, module) return VotingToken( - token=token, token_hash=token_hash, module=module, package_number=package_number + token=token, token_hash=token_hash, module=module, package=package ) + + +@background(schedule=1) +def delete_plain_codes(package_pk): + queryset = VotingToken.objects.filter(package__pk=package_pk) + queryset.update(token="") diff --git a/meinberlin/apps/votes/templates/meinberlin_votes/token_export_dashboard.html b/meinberlin/apps/votes/templates/meinberlin_votes/token_export_dashboard.html index e215bf1c6b..69dfdbf3e2 100644 --- a/meinberlin/apps/votes/templates/meinberlin_votes/token_export_dashboard.html +++ b/meinberlin/apps/votes/templates/meinberlin_votes/token_export_dashboard.html @@ -1,5 +1,5 @@ {% extends "a4dashboard/base_dashboard_project.html" %} -{% load i18n static %} +{% load i18n static humanize %} {% block dashboard_project_content %}

{% translate 'Download voting codes' %}

@@ -8,26 +8,26 @@

{% translate 'Download voting codes' %}

{% blocktranslate %}Number of generated codes: {{ number_of_module_tokens }}.{% endblocktranslate %}