Upgrade to Django 3.2

.flake8

@@ -0,0 +1,6 @@
+[flake8]
+max-line-length = 88
+max-complexity = 18
+select = B,C,E,F,W,T4,B9,
+ignore = E203, E266, E501, W503, F403, F401, E231, E123, B950, B306, C416, C901, F405, E501, W291
+extend-ignore = E203

backend/.coveragerc

@@ -0,0 +1,6 @@
+[run]
+omit =
+	# omit all the migrations
+	*migrations*
+	# omit all the t snapshots
+	*snapshots*

backend/hct_mis_api/apps/account/admin.py

@@ -3,7 +3,6 @@
 import re
 from collections import defaultdict, namedtuple
 from functools import cached_property
-from import_export.widgets import ForeignKeyWidget, ManyToManyWidget
 from urllib.parse import unquote
 
 from django import forms
@@ -18,11 +17,10 @@
 from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
 from django.contrib.auth.forms import UserCreationForm, UsernameField
 from django.contrib.auth.models import Group, Permission
-from django.contrib.postgres.fields import JSONField
 from django.core.exceptions import ValidationError
 from django.core.mail import send_mail
 from django.db import router, transaction
-from django.db.models import Q
+from django.db.models import JSONField, Q
 from django.db.transaction import atomic
 from django.forms import EmailField, ModelChoiceField, MultipleChoiceField
 from django.forms.models import BaseInlineFormSet, ModelForm
@@ -40,8 +38,9 @@
 from adminfilters.autocomplete import AutoCompleteFilter
 from adminfilters.filters import AllValuesComboFilter
 from constance import config
-from import_export import resources, fields
+from import_export import fields, resources
 from import_export.admin import ImportExportModelAdmin
+from import_export.widgets import ForeignKeyWidget, ManyToManyWidget
 from jsoneditor.forms import JSONEditor
 from requests import HTTPError
 from smart_admin.decorators import smart_register
@@ -118,9 +117,9 @@ def clean(self):
                     form_two.cleaned_data["role"].name
                     for form_two in self.forms
                     if form_two.cleaned_data
-                       and not form_two.cleaned_data.get("DELETE")
-                       and form_two.cleaned_data["business_area"] == business_area
-                       and form_two.cleaned_data["role"].id in incompatible_roles
+                    and not form_two.cleaned_data.get("DELETE")
+                    and form_two.cleaned_data["business_area"] == business_area
+                    and form_two.cleaned_data["role"].id in incompatible_roles
                 ]
                 if error_forms:
                     if "role" not in form._errors:
@@ -257,8 +256,7 @@ def list_users(self, q=""):
             if matches[0] == last_match:
                 break
             last_match = matches[0]
-            for m in matches:
-                yield m
+            yield from matches
 
             page += 1
 
@@ -272,7 +270,10 @@ def get_csrfmiddlewaretoken(self):
 
     def delete_user(self, username, pk):
         self.login()
-        for url in [f"{self.admin_url_kc}auth/user/{pk}/delete/", f"{self.admin_url}auth/user/{pk}/delete/"]:
+        for url in [
+            f"{self.admin_url_kc}auth/user/{pk}/delete/",
+            f"{self.admin_url}auth/user/{pk}/delete/",
+        ]:
             self._get(url)
             self.assert_response([200, 404, 302], custom_error=url)
             if self._last_response.status_code == 302 and "/login/" in self._last_response.headers["Location"]:
@@ -453,7 +454,11 @@ def delete_view(self, request, object_id, extra_context=None):
                 api = DjAdminManager()
                 api.login(request)
                 extra_context["kobo_pk"] = kobo_pk
-                self.message_user(request, "This action will also delete linked Kobo account", messages.WARNING)
+                self.message_user(
+                    request,
+                    "This action will also delete linked Kobo account",
+                    messages.WARNING,
+                )
             except Exception as e:
                 extra_context["kobo_failed"] = True
                 self.message_user(request, str(e), messages.ERROR)
@@ -492,7 +497,7 @@ def privileges(self, request, pk):
         return TemplateResponse(request, "admin/account/user/privileges.html", context)
 
     def get_actions(self, request):
-        actions = super(UserAdmin, self).get_actions(request)
+        actions = super().get_actions(request)
         if not request.user.has_perm("account.can_create_kobo_user"):
             if "create_kobo_user_qs" in actions:
                 del actions["create_kobo_user_qs"]
@@ -548,7 +553,7 @@ def add_business_area_role(self, request, queryset):
     add_business_area_role.short_description = "Add/Remove Business Area roles"
 
     def _grant_kobo_accesss_to_user(self, user, notify=True, sync=True):
-        password = get_random_string()
+        password = get_random_string(length=12)
         url = f"{settings.KOBO_KF_URL}/authorized_application/users/"
         username = get_valid_kobo_username(user)
         res = requests.post(
@@ -592,9 +597,16 @@ def create_kobo_user_qs(self, request, queryset):
             except Exception as e:
                 logger.exception(e)
                 self.message_user(request, f"{e.__class__.__name__}: {str(e)}", messages.ERROR)
-        self.message_user(request, f"User successfully `{user.username}` created on Kobo", messages.SUCCESS)
+        self.message_user(
+            request,
+            f"User successfully `{user.username}` created on Kobo",
+            messages.SUCCESS,
+        )
 
-    @button(permission="account.can_create_kobo_user", visible=lambda o, r: not o.custom_fields.get("kobo_username"))
+    @button(
+        permission="account.can_create_kobo_user",
+        visible=lambda o, r: not o.custom_fields.get("kobo_username"),
+    )
     def create_kobo_user(self, request, pk):
         try:
             self._grant_kobo_accesss_to_user(self.get_queryset(request).get(pk=pk))
@@ -603,7 +615,10 @@ def create_kobo_user(self, request, pk):
             logger.exception(e)
             self.message_user(request, f"{e.__class__.__name__}: {str(e)}", messages.ERROR)
 
-    @button(permission="account.can_create_kobo_user", visible=lambda o, r: o.custom_fields.get("kobo_username"))
+    @button(
+        permission="account.can_create_kobo_user",
+        visible=lambda o, r: o.custom_fields.get("kobo_username"),
+    )
     def remove_kobo_access(self, request, pk):
         try:
             obj = self.get_object(request, pk)
@@ -612,14 +627,19 @@ def remove_kobo_access(self, request, pk):
             obj.custom_fields["kobo_username"] = None
             obj.custom_fields["kobo_pk"] = None
             obj.save()
-            self.message_user(request, f"Kobo Access removed from {settings.KOBO_KF_URL}", messages.WARNING)
+            self.message_user(
+                request,
+                f"Kobo Access removed from {settings.KOBO_KF_URL}",
+                messages.WARNING,
+            )
         except Exception as e:
             logger.exception(e)
             self.message_user(request, f"{e.__class__.__name__}: {str(e)}", messages.ERROR)
 
     @button(label="Import CSV", permission="account.can_upload_to_kobo")
     def import_csv(self, request):
         from django.contrib.admin.helpers import AdminForm
+
         context = self.get_common_context(request, processed=False)
         if request.method == "GET":
             form = ImportCSVForm(initial={"partner": Partner.objects.first()})
@@ -656,13 +676,20 @@ def import_csv(self, request):
                                 except Exception as e:
                                     raise Exception(f"{e.__class__.__name__}: {e} on `{row}`")
 
-                                user_info = {"email": email, "is_new": False, "kobo": False, "error": ""}
+                                user_info = {
+                                    "email": email,
+                                    "is_new": False,
+                                    "kobo": False,
+                                    "error": "",
+                                }
                                 if "username" in row:
                                     username = row["username"].strip()
                                 else:
                                     username = row["email"].replace("@", "_").replace(".", "_").lower()
                                 u, isnew = account_models.User.objects.get_or_create(
-                                    email=email, partner=partner, defaults={"username": username}
+                                    email=email,
+                                    partner=partner,
+                                    defaults={"username": username},
                                 )
                                 if isnew:
                                     ur = u.user_roles.create(business_area=business_area, role=role)
@@ -674,7 +701,11 @@ def import_csv(self, request):
                                         u.user_roles.get_or_create(business_area=business_area, role=role)
                                         self.log_addition(request, ur, "User Role added")
                                     except ValidationError as e:
-                                        self.message_user(request, f"Error on {u}: {e}", messages.ERROR)
+                                        self.message_user(
+                                            request,
+                                            f"Error on {u}: {e}",
+                                            messages.ERROR,
+                                        )
 
                                 if enable_kobo:
                                     self._grant_kobo_accesss_to_user(u, sync=False)
@@ -709,7 +740,10 @@ def kobo_users_sync(self, request):
                         email=entry[2],
                         defaults={
                             "username": entry[1],
-                            "custom_fields": {"kobo_pk": entry[0], "kobo_username": entry[1]},
+                            "custom_fields": {
+                                "kobo_pk": entry[0],
+                                "kobo_username": entry[1],
+                            },
                         },
                     )
                     local.custom_fields["kobo_pk"] = entry[0]
@@ -765,9 +799,17 @@ def sync_multi(self, request):
                 except Http404:
                     not_found.append(str(user))
             if not_found:
-                self.message_user(request, f"These users were not found: {', '.join(not_found)}", messages.WARNING)
+                self.message_user(
+                    request,
+                    f"These users were not found: {', '.join(not_found)}",
+                    messages.WARNING,
+                )
             else:
-                self.message_user(request, "Active Directory data successfully fetched", messages.SUCCESS)
+                self.message_user(
+                    request,
+                    "Active Directory data successfully fetched",
+                    messages.SUCCESS,
+                )
         except Exception as e:
             logger.exception(e)
             self.message_user(request, str(e), messages.ERROR)
@@ -833,7 +875,9 @@ def load_ad_users(self, request):
                                 if global_business_area and basic_role:
                                     users_role_to_bulk_create.append(
                                         account_models.UserRole(
-                                            business_area=global_business_area, user=user, role=basic_role
+                                            business_area=global_business_area,
+                                            user=user,
+                                            role=basic_role,
                                         )
                                     )
                                 results.created.append(user)
@@ -877,7 +921,7 @@ def queryset(self, request, queryset):
 class RoleResource(resources.ModelResource):
     class Meta:
         model = account_models.Role
-        fields = ('name', 'subsystem', 'permissions')
+        fields = ("name", "subsystem", "permissions")
         import_id_fields = ("name", "subsystem")
 
 
@@ -987,14 +1031,12 @@ class IncompatibleRolesAdmin(HOPEModelAdminBase):
 
 
 class GroupResource(resources.ModelResource):
-    permissions = fields.Field(widget=ManyToManyWidget(Permission,
-                                                       field='codename'),
-                               attribute='permissions')
+    permissions = fields.Field(widget=ManyToManyWidget(Permission, field="codename"), attribute="permissions")
 
     class Meta:
         model = Group
-        fields = ('name', 'permissions')
-        import_id_fields = ("name", )
+        fields = ("name", "permissions")
+        import_id_fields = ("name",)
 
 
 @smart_register(Group)

backend/hct_mis_api/apps/account/authentication.py

@@ -1,12 +1,13 @@
 import logging
 
 from django.contrib.auth import get_user_model
+
 from social_core.exceptions import InvalidEmail
 from social_core.pipeline import social_auth
 from social_core.pipeline import user as social_core_user
 
 from hct_mis_api.apps.account.microsoft_graph import MicrosoftGraphAPI
-from hct_mis_api.apps.account.models import UserRole, Role, ACTIVE
+from hct_mis_api.apps.account.models import ACTIVE, Role, UserRole
 from hct_mis_api.apps.core.models import BusinessArea
 
 logger = logging.getLogger(__name__)

backend/hct_mis_api/apps/account/fixtures.py

@@ -17,7 +17,7 @@ class Meta:
 
 
 class BusinessAreaFactory(factory.DjangoModelFactory):
-    name = factory.Sequence(lambda x: "BusinessArea%s" % x)
+    name = factory.Sequence(lambda x: "BusinessArea{}".format(x))
 
     class Meta:
         model = BusinessArea

backend/hct_mis_api/apps/account/models.py

@@ -4,18 +4,19 @@
 from django import forms
 from django.contrib.auth import get_user_model
 from django.contrib.auth.models import AbstractUser
-from django.contrib.postgres.fields import ArrayField, CICharField, JSONField
+from django.contrib.postgres.fields import ArrayField, CICharField
 from django.core.exceptions import ValidationError
 from django.core.validators import (
     MaxLengthValidator,
     MinLengthValidator,
     ProhibitNullCharactersValidator,
 )
 from django.db import models
+from django.db.models import JSONField
 from django.db.models.signals import post_save, pre_delete, pre_save
 from django.dispatch import receiver
 from django.utils import timezone
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from model_utils import Choices
 from model_utils.models import UUIDModel
@@ -72,7 +73,9 @@ class User(AbstractUser, UUIDModel):
         default=None, null=True, blank=True, help_text="Timestamp of last sync with CA"
     )
     doap_hash = models.TextField(
-        editable=False, default="", help_text="System field used to check if changes need to be sent to CA"
+        editable=False,
+        default="",
+        help_text="System field used to check if changes need to be sent to CA",
     )
 
     def __str__(self):
@@ -89,17 +92,20 @@ def save(self, *args, **kwargs):
 
     def permissions_in_business_area(self, business_area_slug):
         all_roles_permissions_list = list(
-            Role.objects.filter(user_roles__user=self, user_roles__business_area__slug=business_area_slug).values_list(
-                "permissions", flat=True
-            )
+            Role.objects.filter(
+                user_roles__user=self,
+                user_roles__business_area__slug=business_area_slug,
+            ).values_list("permissions", flat=True)
         )
         return [
             permission for roles_permissions in all_roles_permissions_list for permission in roles_permissions or []
         ]
 
     def has_permission(self, permission, business_area, write=False):
         query = Role.objects.filter(
-            permissions__contains=[permission], user_roles__user=self, user_roles__business_area=business_area
+            permissions__contains=[permission],
+            user_roles__user=self,
+            user_roles__business_area=business_area,
         )
         return query.count() > 0
 
@@ -159,7 +165,9 @@ class Role(TimeStampedUUIDModel):
     )
     subsystem = models.CharField(choices=SUBSYSTEMS, max_length=30, default=HOPE)
     permissions = ChoiceArrayField(
-        models.CharField(choices=Permissions.choices(), max_length=255), null=True, blank=True
+        models.CharField(choices=Permissions.choices(), max_length=255),
+        null=True,
+        blank=True,
     )
 
     def clean(self):
@@ -254,10 +262,15 @@ def clean(self):
             raise ValidationError(_("Choose two different roles."))
         failing_users = set()
 
-        for role_pair in [(self.role_one, self.role_two), (self.role_two, self.role_one)]:
+        for role_pair in [
+            (self.role_one, self.role_two),
+            (self.role_two, self.role_one),
+        ]:
             for userrole in UserRole.objects.filter(role=role_pair[0]):
                 if UserRole.objects.filter(
-                    user=userrole.user, business_area=userrole.business_area, role=role_pair[1]
+                    user=userrole.user,
+                    business_area=userrole.business_area,
+                    role=role_pair[1],
                 ).exists():
                     failing_users.add(userrole.user.email)