Upgrade Django to v4.2 (#1113)

* Upgrade Django to v4.2

* refresh lockfile

* replace is_ajax calls

* upgrade to pyscopg3 (recommended by Django) and update storage settings.

* transfer django-recaptcha2 to designsafe org

* update pytest-django for django 4 compatibility

* fix imports and add more stable debugging

* replace deprecated smart_text import

---------

Co-authored-by: Jake Rosenberg <jrosenberg@wireless-10-145-48-201.public.utexas.edu>
Co-authored-by: Jake Rosenberg <jrosenberg@MacBook-Pro.lan>

bin/run-django.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 # run django dev server as designsafe community account
-python manage.py runserver 0.0.0.0:8000
+python -m debugpy --listen 0.0.0.0:5678 manage.py runserver 0.0.0.0:8000

designsafe/apps/accounts/forms.py

@@ -3,7 +3,7 @@
 from django import forms
 from django.contrib.auth import get_user_model
 from django.urls import reverse
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from django.utils.safestring import mark_safe
 from django.utils.html import escape
 from snowpenguin.django.recaptcha2.fields import ReCaptchaField

designsafe/apps/accounts/models.py

@@ -1,7 +1,7 @@
 from django.conf import settings
 from django.db import models
 from django.db import connections, DatabaseError
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from django.core.mail import send_mail
 import logging
 import six

designsafe/apps/accounts/urls.py

@@ -1,6 +1,6 @@
-from django.conf.urls import include, url
+from django.urls import include, re_path as url
 from django.urls import reverse
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from designsafe.apps.accounts import views
 
 urlpatterns = [

designsafe/apps/accounts/views.py

@@ -6,7 +6,7 @@
 from django.urls import reverse
 from django.db.models import Q
 from django.http import HttpResponse, HttpResponseRedirect
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from designsafe.apps.accounts import forms, integrations
 from designsafe.apps.accounts.models import (NEESUser, DesignSafeProfile,
                                              NotificationPreferences)

designsafe/apps/api/datafiles/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import url
+from django.urls import re_path as url
 from designsafe.apps.api.datafiles.views import DataFilesView, TransferFilesView, MicrosurveyView
 
 urlpatterns = [

designsafe/apps/api/licenses/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import url
+from django.urls import re_path as url
 from designsafe.apps.api.licenses.views import LicenseView
 
 urlpatterns = [

designsafe/apps/api/notifications/signals.py

@@ -1,4 +1,4 @@
 from django.dispatch import Signal
 
-ds_event = Signal(providing_args=['session_id', 'event_type', 'event_data'])
-generic_event = Signal(providing_args=['event_type', 'event_data', 'event_users'])
+ds_event = Signal()
+generic_event = Signal()

designsafe/apps/api/notifications/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import include, url
+from django.urls import re_path as url
 
 from designsafe.apps.api.notifications.views.api import ManageNotificationsView, NotificationsBadgeView
 from designsafe.apps.api.notifications.views.webhooks import JobsWebhookView, FilesWebhookView

designsafe/apps/api/projects/urls.py

@@ -10,7 +10,7 @@
                                  More specific action should live in the body.
 
 """
-from django.conf.urls import url, include
+from django.urls import re_path as url
 from designsafe.apps.api.projects.views import (ProjectListingView,
                                                 ProjectCollectionView,
                                                 ProjectDataView,

designsafe/apps/api/projects/views.py

@@ -96,7 +96,7 @@ def post(self, request, **kwargs):
         """
         Publish a project or version a publication
         """
-        if request.is_ajax():
+        if request.headers.get('x-requested-with') == 'XMLHttpRequest':
             data = json.loads(request.body)
         else:
             data = request.POST
@@ -173,7 +173,7 @@ def put(self, request, **kwargs):
         """
         Amend a Publication
         """
-        if request.is_ajax():
+        if request.headers.get('x-requested-with') == 'XMLHttpRequest':
             data = json.loads(request.body)
         else:
             data = request.PUT
@@ -289,7 +289,7 @@ def post(self, request):
         # portal service account needs to create the objects on behalf of the user
         sa_client = get_service_account_client()
 
-        if request.is_ajax():
+        if request.headers.get('x-requested-with') == 'XMLHttpRequest':
             post_data = json.loads(request.body)
         else:
             post_data = request.POST.copy()
@@ -449,7 +449,7 @@ def post(self, request, project_id):
         should be prevented anyways). Once that data is deleted, then we should update
         the type of project.
         """
-        if request.is_ajax():
+        if request.headers.get('x-requested-with') == 'XMLHttpRequest':
             post_data = json.loads(request.body)
         else:
             post_data = request.POST.copy()
@@ -606,7 +606,7 @@ def post(self, request, project_id, name):
         """
         sa_client = get_service_account_client()
 
-        if request.is_ajax():
+        if request.headers.get('x-requested-with') == 'XMLHttpRequest':
             post_data = json.loads(request.body)
         else:
             post_data = request.POST.copy()
@@ -658,7 +658,7 @@ def put(self, request, uuid):
         """
         client = request.user.agave_oauth.client
 
-        if request.is_ajax():
+        if request.headers.get('x-requested-with') == 'XMLHttpRequest':
             post_data = json.loads(request.body)
         else:
             post_data = request.POST.copy()

designsafe/apps/api/publications/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import url
+from django.urls import re_path as url
 from designsafe.apps.api.publications.views import PublicationListingView, PublicationDetailView, PublicationDataCiteView
 from django.http import JsonResponse
 

designsafe/apps/api/search/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import url
+from django.urls import re_path as url
 from designsafe.apps.api.search.views import SearchView
 
 """

designsafe/apps/api/urls.py

@@ -1,5 +1,5 @@
 # pylint: disable=missing-docstring
-from django.conf.urls import url, include
+from django.urls import re_path as url, include
 from designsafe.apps.api.views import LoggerApi
 from django.http import JsonResponse
 

designsafe/apps/api/users/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import url
+from django.urls import re_path as url
 from designsafe.apps.api.users.views import SearchView, AuthenticatedView, UsageView, PublicView
 
 urlpatterns = [

designsafe/apps/applications/urls.py

@@ -1,6 +1,6 @@
-from django.conf.urls import include, url
+from django.urls import include, re_path as url
 from django.urls import reverse
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from designsafe.apps.applications import views
 
 

designsafe/apps/auth/urls.py

@@ -1,6 +1,6 @@
-from django.conf.urls import include, url
+from django.urls import re_path as url
 from django.urls import reverse
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from designsafe.apps.auth import views
 
 urlpatterns = [

designsafe/apps/box_integration/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import url
+from django.urls import re_path as url
 from designsafe.apps.box_integration import views
 
 urlpatterns = [

designsafe/apps/cms_plugins/cms_plugins.py

@@ -7,7 +7,7 @@
 from cms.plugin_pool import plugin_pool
 from designsafe.apps.cms_plugins.models import ResponsiveEmbedPlugin
 from django.conf import settings
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 
 
 class CMSResponsiveEmbedPlugin(CMSPluginBase):

designsafe/apps/cms_plugins/menu.py

@@ -3,7 +3,7 @@
 from menus.menu_pool import menu_pool
 from django.conf import settings
 from importlib import import_module
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 import logging
 
 logger = logging.getLogger(__name__)

designsafe/apps/dashboard/urls.py

@@ -1,6 +1,6 @@
-from django.conf.urls import include, url
+from django.urls import include, re_path as url
 from django.urls import reverse
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from designsafe.apps.dashboard import views
 
 urlpatterns = [

designsafe/apps/data/urls.py

@@ -1,12 +1,12 @@
-from django.conf.urls import url
+from django.urls import re_path as url
 from designsafe.apps.data.views.base import (
     DataDepotView,
     FileMediaView,
     DataDepotPublishedView,
     DataDepotLegacyPublishedView
 )
 from django.urls import reverse
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 urlpatterns = [
     url(

designsafe/apps/djangoRT/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import url
+from django.urls import re_path as url
 
 from designsafe.apps.djangoRT import views
 

designsafe/apps/dropbox_integration/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import url
+from django.urls import re_path as url
 from designsafe.apps.dropbox_integration import views
 
 urlpatterns = [

designsafe/apps/geo/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import include, url
+from django.urls import include, re_path as url
 from django.urls import reverse
 from designsafe.apps.geo import views
 

designsafe/apps/googledrive_integration/models.py

@@ -56,7 +56,7 @@ def get_prep_value(self, value):
         if value is None:
             return None
         else:
-            return encoding.smart_text(
+            return encoding.smart_str(
                 base64.b64encode(jsonpickle.encode(value).encode()))
 
     def value_to_string(self, obj):

designsafe/apps/googledrive_integration/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import url
+from django.urls import re_path as url
 from designsafe.apps.googledrive_integration import views
 
 urlpatterns = [

designsafe/apps/nco/api_urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import url
+from django.urls import re_path as url
 from designsafe.apps.nco.views import ProjectsListView, FiltersListView, TtcGrantsView, TtcFacilitiesView
 
 """

designsafe/apps/nco/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import url
+from django.urls import re_path as url
 from designsafe.apps.nco.views import NcoIndexView, NcoTtcGrantsView
 
 urlpatterns = [

designsafe/apps/notifications/urls.py

@@ -1,6 +1,6 @@
-from django.conf.urls import include, url
+from django.urls import include, re_path as url
 from django.urls import reverse
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from designsafe.apps.notifications import views
 
 

designsafe/apps/rapid/urls.py

@@ -1,4 +1,4 @@
-from django.conf.urls import include, url
+from django.urls import include, re_path as url
 from django.urls import reverse
 from designsafe.apps.rapid import views
 

designsafe/apps/search/search_indexes.py

@@ -9,7 +9,7 @@
 from django.test import RequestFactory
 
 from django.utils.html import strip_tags
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 
 from cms.models import Title, CMSPlugin, Page
 # from cms.toolbar.toolbar import CMSToolbar
@@ -24,7 +24,7 @@ def _strip_tags(value):
     whitespace in between replaced tags to make sure words are not erroneously
     concatenated.
     """
-    return re.sub(r'<[^>]*?>', ' ', force_text(value))
+    return re.sub(r'<[^>]*?>', ' ', force_str(value))
 
 
 class TextPluginIndex(indexes.SearchIndex, indexes.Indexable):
@@ -70,7 +70,7 @@ def prepare(self, obj):
                 # this is an empty plugin
                 continue
             if hasattr(instance, 'search_fields'):
-                text += ' '.join(force_text(strip_tags(getattr(instance, field, ''))) for field in instance.search_fields)
+                text += ' '.join(force_str(strip_tags(getattr(instance, field, ''))) for field in instance.search_fields)
             if getattr(instance, 'search_fulltext', False) or getattr(plugin_type, 'search_fulltext', False):
                 text += _strip_tags(instance.render_plugin(context=RequestContext(request))) + ' '
         text += page.get_meta_description() or ''

designsafe/apps/search/urls.py

@@ -1,6 +1,6 @@
-from django.conf.urls import include, url
+from django.urls import include, re_path as url
 from django.urls import reverse
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from designsafe.apps.search import views
 
 urlpatterns = [

designsafe/apps/signals/signals.py

@@ -1,4 +1,4 @@
 from django.dispatch import Signal
 
-ds_event = Signal(providing_args=['session_id', 'event_type', 'event_data'])
-generic_event = Signal(providing_args=['event_type', 'event_data', 'event_users'])
+ds_event = Signal()
+generic_event = Signal()

designsafe/apps/workspace/urls.py

@@ -1,6 +1,6 @@
-from django.conf.urls import url
+from django.urls import re_path as url
 from django.urls import reverse
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from designsafe.apps.workspace import views
 
 # TODO look at linking directly into an app in the workspace

designsafe/celery.py

@@ -35,7 +35,7 @@
             'schedule': crontab(hour=0, minute=0)
         },
         'clear_old_notifications': {
-            'task': 'designsafe.auth.tasks.clear_old_notifications',
+            'task': 'designsafe.apps.auth.tasks.clear_old_notifications',
             'schedule': crontab(hour=0, minute=1)
         }
     }

designsafe/conftest.py

@@ -8,7 +8,7 @@ def mock_agave_client(mocker):
 
 
 @pytest.fixture
-def regular_user(django_user_model, django_db_reset_sequences, mock_agave_client):
+def regular_user(django_user_model, mock_agave_client):
     django_user_model.objects.create_user(username="username",
                                           password="password",
                                           first_name="Firstname",

designsafe/settings/common_settings.py

@@ -142,8 +142,8 @@
     'django.middleware.security.SecurityMiddleware',
     'cms.middleware.user.CurrentUserMiddleware',
     'cms.middleware.page.CurrentPageMiddleware',
-    'cms.middleware.toolbar.ToolbarMiddleware',
     'cms.middleware.language.LanguageCookieMiddleware',
+    'cms.middleware.toolbar.ToolbarMiddleware',
     'impersonate.middleware.ImpersonateMiddleware',
     'designsafe.middleware.DesignSafeTermsMiddleware',
     'designsafe.middleware.DesignsafeProfileUpdateMiddleware',
@@ -273,7 +273,14 @@
     ('vendor/d3plus', os.path.join(BASE_DIR, 'node_modules', 'd3plus')),
 ]
 
-STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage'
+STORAGES = {
+    "default": {
+        "BACKEND": "django.core.files.storage.FileSystemStorage",
+    },
+    "staticfiles": {
+        "BACKEND": "django.contrib.staticfiles.storage.ManifestStaticFilesStorage",
+    },
+}
 STATICFILES_FINDERS = (
     'django.contrib.staticfiles.finders.FileSystemFinder',
     'django.contrib.staticfiles.finders.AppDirectoriesFinder',

designsafe/urls.py

@@ -23,7 +23,7 @@
 """
 
 from django.conf import settings
-from django.conf.urls import include, url
+from django.urls import include, re_path as url
 from django.conf.urls.static import static
 from django.contrib import admin
 from django.views.generic import RedirectView, TemplateView

designsafe/webhooks.py

@@ -2,7 +2,7 @@
 DesignSafe-CI Webhook URLs
 """
 from django.conf import settings
-from django.conf.urls import include, url
+from django.urls import include, re_path as url
 from designsafe.apps.notifications import views
 from designsafe.apps.box_integration import webhooks