[Python3 and Django2.2 upgrade] part 1 - Use Python 3 & Django 1.11

conftest.py

@@ -20,16 +20,10 @@
     'peter',
 ]
 
-try:
-    INTERRUPTED = pytest.ExitCode.INTERRUPTED  # pytest 5
-except AttributeError:
-    INTERRUPTED = 2
-
 
 def stderr_prompt(message):
     sys.stderr.write(message)
-    # FIXME: Python 3 compatibility
-    return raw_input().strip()
+    return input().strip()
 
 
 def toggle_capturing(capture_manager, stop):
@@ -63,7 +57,7 @@ def setup(request):
             if response.lower() != 'yes':
                 if is_global_capturing:
                     toggle_capturing(capture_manager, stop=False)
-                pytest.exit('User interrupted tests', INTERRUPTED)
+                pytest.exit('User interrupted tests', pytest.ExitCode.INTERRUPTED)
 
     if 'instances' in settings.MONGO_DB.collection_names():
         response = stderr_prompt(
@@ -77,7 +71,7 @@ def setup(request):
         else:
             if is_global_capturing:
                 toggle_capturing(capture_manager, stop=False)
-            pytest.exit('User interrupted tests', INTERRUPTED)
+            pytest.exit('User interrupted tests', pytest.ExitCode.INTERRUPTED)
 
     if is_global_capturing:
         toggle_capturing(capture_manager, stop=False)

docker/dev_wrong_port_warning.py

@@ -5,20 +5,22 @@
 provides a helpful message to anyone still trying to use port 8000
 """
 
-import BaseHTTPServer
 import sys
+from http.server import BaseHTTPRequestHandler, HTTPServer
 
-class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
+
+class Handler(BaseHTTPRequestHandler):
     def do_GET(self):
         self.send_response(503)
         self.end_headers()
         self.wfile.write(
-            'Your development environment is trying to connect to the KoBoCAT '
-            'container on port 8000 instead of 8001. Please change this. See '
-            'https://github.com/kobotoolbox/kobo-docker/issues/301 '
-            'for more details.'
+            b'Your development environment is trying to connect to the KoBoCAT '
+            b'container on port 8000 instead of 8001. Please change this. See '
+            b'https://github.com/kobotoolbox/kobo-docker/issues/301 '
+            b'for more details.'
         )
 
+
 server_address = ('', int(sys.argv[1]))
-httpd = BaseHTTPServer.HTTPServer(server_address, Handler)
+httpd = HTTPServer(server_address, Handler)
 httpd.serve_forever()

kobocat-template/templates/base.html

@@ -99,7 +99,7 @@
 {% block javascript %}
 
     <!-- Le javascript -->
-    <script type="text/javascript" src="{% url "django.views.i18n.javascript_catalog" %}"></script>
+    <script type="text/javascript" src="{% url "javascript-catalog" %}"></script>
     <script src="{% static "js/jquery.min.js" %}"></script>
     <script src="{% static "js/jquery-migrate.min.js" %}"></script>
     <script src="{% static "js/jquery.tablesorter.min.js" %}"></script>

kobocat-template/templates/export_list.html

@@ -7,7 +7,7 @@
 
 <div class="sub-header-bar">
   <div class="container__wide">
-    <a class="sub-header__back" href="{% url "onadata.apps.main.views.show" xform.user.username xform.id_string %}"><i class="fa fa-chevron-left"></i> {% trans "Return to" %} {{ xform.title }}</a>
+    <a class="sub-header__back" href="{% url "show_form" xform.user.username xform.id_string %}"><i class="fa fa-chevron-left"></i> {% trans "Return to" %} {{ xform.title }}</a>
   </div>
 </div>
 
@@ -25,7 +25,7 @@ <h1>{{ export_type_name|upper }} {% blocktrans %}Exports{% endblocktrans %}</h1>
 <section id="export-downloads">
 
     {% if user.is_authenticated %}
-    <form action="{% url "onadata.apps.viewer.views.create_export" username xform.id_string export_type %}" method="post" enctype="application/x-www-form-urlencoded">
+    <form action="{% url "create_export" username xform.id_string export_type %}" method="post" enctype="application/x-www-form-urlencoded">
         {% csrf_token %}
             <span style='float:middle;'>
               <div>
@@ -40,7 +40,7 @@ <h1>{{ export_type_name|upper }} {% blocktrans %}Exports{% endblocktrans %}</h1>
     {% endif %}
 
     <div id="advanced-export-modal" class="modal hide fade">
-      <form action="{% url "onadata.apps.viewer.views.create_export" username xform.id_string export_type %}" method="post" enctype="application/x-www-form-urlencoded">
+      <form action="{% url "create_export" username xform.id_string export_type %}" method="post" enctype="application/x-www-form-urlencoded">
         {% csrf_token %}
         <div class="modal-header">
           <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
@@ -80,7 +80,7 @@ <h3>{% trans "Advanced Export" %}</h3>
         <td>
             {% if not export.is_pending %}
                 {% if export.is_successful %}
-                    <a href="{% url "onadata.apps.viewer.views.export_download" username xform.id_string export.export_type export.filename %}">{{ export.filename }}</a>
+                    <a href="{% url "export_download" username xform.id_string export.export_type export.filename %}">{{ export.filename }}</a>
                 {% else %}
                     Failed
                 {% endif %}
@@ -91,7 +91,7 @@ <h3>{% trans "Advanced Export" %}</h3>
         </td>
         <td>{{ export.created_on }}</td>
         <td>
-            <form action="{% url "onadata.apps.viewer.views.delete_export" username xform.id_string export_type %}" method="post" enctype="application/x-www-form-urlencoded">
+            <form action="{% url "delete_export" username xform.id_string export_type %}" method="post" enctype="application/x-www-form-urlencoded">
               {% csrf_token %}
                 <div id="delete-{{ export.id|stringformat:"d" }}" class="modal hide fade">
                     <div class="modal-header">
@@ -132,6 +132,6 @@ <h3>{% trans "Delete Export" %}</h3>
 <script type="text/javascript" charset="utf-8" src="{% static "js/underscore-min.js" %}"></script>
 <script type="text/javascript" charset="utf-8" src="{% static "js/export_list.js" %}"></script>
 <script type="text/javascript">
-    var progress_url = '{% url "onadata.apps.viewer.views.export_progress" username xform.id_string export_type %}';
+    var progress_url = '{% url "export_progress" username xform.id_string export_type %}';
 </script>
 {% endblock %}

kobocat-template/templates/map.html

@@ -18,7 +18,7 @@
 
     <body>
     {% include "topbar.html" %}
-    <div class="map__back"><a href="{% url "onadata.apps.main.views.show" xform.user.username xform.id_string %}">back
+    <div class="map__back"><a href="{% url "show" xform.user.username xform.id_string %}">back
         to project</a></div>
 
     <ul class="nav pull-right map__viewby">
@@ -75,7 +75,7 @@
             var mongoAPIUrl = "{{ mongo_api_url }}";
             var deleteAPIUrl = "{{ delete_data_url }}";
             var language_code = "{{LANGUAGE_CODE}}";
-            var attachmentsBaseUrl = "{% url "onadata.apps.viewer.views.attachment_url" %}";
+            var attachmentsBaseUrl = "{% url "attachment_url" %}";
             var userName = "{{ content_user }}";
             var modalVisible = null;
             var idToDelete = null;

kobocat-template/templates/published_forms__list.html

@@ -21,7 +21,7 @@
       <tr class="published_forms__form">
         <td></td>
         <td class="published_forms__cell published_forms__cell--project">
-            <a href="{% url "onadata.apps.main.views.show" xform.user.username xform.id_string %}">{{ xform.title }}</a> 
+            <a href="{% url "show_form" xform.user.username xform.id_string %}">{{ xform.title }}</a>
             {% if xform.description %}
             <div class="project__description">
               {{ xform.description }}
@@ -58,4 +58,4 @@
 
 
   </tbody>
-</table>
+</table>

kobocat-template/templates/show.html

@@ -31,14 +31,10 @@ <h2 class="dashboard__group-label">
             </a>
 
             <div class="dashboard__download__drop hidden">
-                <a href="{% url "onadata.apps.viewer.views.export_list" content_user.username xform.id_string 'xls' %}"
-                   class="download__drop-button">XLS</a>
-                <a href="{% url "onadata.apps.viewer.views.export_list" content_user.username xform.id_string 'csv' %}"
-                   class="download__drop-button">CSV</a>
-                <a href="{% url "onadata.apps.viewer.views.export_list" content_user.username xform.id_string 'zip' %}"
-                   class="download__drop-button">ZIP</a>
-                <a href="{% url "onadata.apps.viewer.views.export_list" content_user.username xform.id_string 'kml' %}"
-                   class="download__drop-button">KML</a>
+                <a href="{% url "export_list" content_user.username xform.id_string 'xls' %}" class="download__drop-button">XLS</a>
+                <a href="{% url "export_list" content_user.username xform.id_string 'csv' %}" class="download__drop-button">CSV</a>
+                <a href="{% url "export_list" content_user.username xform.id_string 'zip' %}" class="download__drop-button">ZIP</a>
+                <a href="{% url "export_list" content_user.username xform.id_string 'kml'%}"  class="download__drop-button">KML</a>
             </div>
         </div>
 
@@ -49,12 +45,12 @@ <h2 class="dashboard__group-label">
             </h2>
 
             <a title="{% trans "View Photos" %}"
-               href="{% url "onadata.apps.main.views.form_photos" content_user.username xform.id_string %}"
+               href="{% url "form_photos" content_user.username xform.id_string %}"
                class="dashboard__button dashboard__button-view-gallery">
                 <i class="fa fa-eye"></i>
                 <span>View gallery</span>
             </a>
-            <a href="{% url "onadata.apps.viewer.views.export_list" content_user.username xform.id_string 'zip' %}"
+            <a href="{% url "export_list" content_user.username xform.id_string 'zip' %}"
                class="dashboard__button dashboard__button-twoline dashboard__button-download-all-photos">
                 <i class="fa fa-download"></i>
                 <span>Download all photos</span>
@@ -77,7 +73,7 @@ <h2 class="dashboard__group-label">
         {% if xform.has_instances_with_geopoints %}
             <div class="dashboard__gps">
                 <h2 class="dashboard__group-label">GPS Points</h2>
-                <a href="{% url "onadata.apps.viewer.views.export_list" content_user.username xform.id_string 'kml' %}"
+                <a href="{% url "export_list" content_user.username xform.id_string 'kml' %}"
                    class="dashboard__button dashboard__button-twoline dashboard__button-download-gps-points">
                     <i class="fa fa-download"></i>
                     <span>Download GPS points</span>

kobocat-template/templates/show_form_settings.html

@@ -27,9 +27,9 @@ <h2 class="settings__group-label">
             {% for media in media_upload %}
               <li class="{% cycle 'odd' 'even' %}">
                 <div>{{ media.data_value }}</div>
-                <a href="{% url "onadata.apps.main.views.download_media_data" content_user.username xform.id_string media.id %}" class="settings__files-download" {% trans "download" %}><i class="fa fa-download"></i></a>
+                <a href="{% url "download_media_data" content_user.username xform.id_string media.id %}" class="settings__files-download" {% trans "download" %}><i class="fa fa-download"></i></a>
                 {% if can_edit %}
-                <a href="{% url "onadata.apps.main.views.download_media_data" content_user.username xform.id_string media.id %}?del=true" class="settings__files-delete" title="{% trans "remove" %}"><i class="fa fa-trash-o"></i></a>
+                <a href="{% url "download_media_data" content_user.username xform.id_string media.id %}?del=true" class="settings__files-delete" title="{% trans "remove" %}"><i class="fa fa-trash-o"></i></a>
                 {% endif %}
               </li>
             {% endfor %}
@@ -38,7 +38,7 @@ <h2 class="settings__group-label">
           {% if can_edit %}
           <button class="single-project__button single-project__add-media">+ Add Document</button>
 
-          <form action="{% url "onadata.apps.main.views.edit" content_user.username xform.id_string %}" method="post" enctype="multipart/form-data" class="single-project__media-upload hidden">
+          <form action="{% url "edit_form" content_user.username xform.id_string %}" method="post" enctype="multipart/form-data" class="single-project__media-upload hidden">
               {% csrf_token %}
               <ul class="nav nav-tabs">
                   <li class="active"><a href="#media_upload" data-toggle="tab">Media Upload</a></li>

onadata/apps/api/migrations/0001_initial.py

@@ -21,9 +21,9 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='OrganizationProfile',
             fields=[
-                ('userprofile_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UserProfile')),
+                ('userprofile_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='main.UserProfile', on_delete=models.CASCADE)),
                 ('is_organization', models.BooleanField(default=True)),
-                ('creator', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
+                ('creator', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
             options={
                 'permissions': (('can_add_xform', 'Can add/upload an xform to organization'), ('view_organizationprofile', 'Can view organization profile')),
@@ -39,8 +39,8 @@ class Migration(migrations.Migration):
                 ('shared', models.BooleanField(default=False)),
                 ('date_created', models.DateTimeField(auto_now_add=True)),
                 ('date_modified', models.DateTimeField(auto_now=True)),
-                ('created_by', models.ForeignKey(related_name='project_creator', to=settings.AUTH_USER_MODEL)),
-                ('organization', models.ForeignKey(related_name='project_organization', to=settings.AUTH_USER_MODEL)),
+                ('created_by', models.ForeignKey(related_name='project_creator', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
+                ('organization', models.ForeignKey(related_name='project_organization', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
                 ('tags', taggit.managers.TaggableManager(to='taggit.Tag', through='taggit.TaggedItem', help_text='A comma-separated list of tags.', verbose_name='Tags')),
                 ('user_stars', models.ManyToManyField(to=settings.AUTH_USER_MODEL)),
             ],
@@ -52,19 +52,19 @@ class Migration(migrations.Migration):
             name='ProjectXForm',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('created_by', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
-                ('project', models.ForeignKey(to='api.Project')),
-                ('xform', models.ForeignKey(to='logger.XForm')),
+                ('created_by', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
+                ('project', models.ForeignKey(to='api.Project', on_delete=models.CASCADE)),
+                ('xform', models.ForeignKey(to='logger.XForm', on_delete=models.CASCADE)),
             ],
         ),
         migrations.CreateModel(
             name='Team',
             fields=[
-                ('group_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='auth.Group')),
+                ('group_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='auth.Group', on_delete=models.CASCADE)),
                 ('date_created', models.DateTimeField(auto_now_add=True, null=True)),
                 ('date_modified', models.DateTimeField(auto_now=True, null=True)),
-                ('created_by', models.ForeignKey(related_name='team_creator', blank=True, to=settings.AUTH_USER_MODEL, null=True)),
-                ('organization', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
+                ('created_by', models.ForeignKey(related_name='team_creator', blank=True, to=settings.AUTH_USER_MODEL, null=True, on_delete=models.CASCADE)),
+                ('organization', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
                 ('projects', models.ManyToManyField(to='api.Project')),
             ],
             options={

onadata/apps/api/mongo_helper.py

@@ -1,21 +1,22 @@
 from __future__ import unicode_literals, print_function, division, absolute_import
 import re
-import base64
+
 
 from onadata.libs.utils.common_tags import NESTED_RESERVED_ATTRIBUTES
+from onadata.libs.utils.string import base64_encodestring
 
 
 class MongoHelper(object):
 
     KEY_WHITELIST = ['$or', '$and', '$exists', '$in', '$gt', '$gte',
                      '$lt', '$lte', '$regex', '$options', '$all']
     ENCODING_SUBSTITUTIONS = [
-        (re.compile(r'^\$'), base64.encodestring('$').strip()),
-        (re.compile(r'\.'), base64.encodestring('.').strip()),
+        (re.compile(r'^\$'), base64_encodestring('$').strip()),
+        (re.compile(r'\.'), base64_encodestring('.').strip()),
     ]
     DECODING_SUBSTITUTIONS = [
-        (re.compile(r'^' + base64.encodestring('$').strip()), '$'),
-        (re.compile(base64.encodestring('.').strip()), '.'),
+        (re.compile(r'^' + base64_encodestring('$').strip()), '$'),
+        (re.compile(base64_encodestring('.').strip()), '.'),
     ]
 
     @classmethod

onadata/apps/api/permissions.py

@@ -130,7 +130,7 @@ def has_permission(self, request, view):
         perms = self.get_required_permissions(request.method, model_cls)
 
         if (request.user and
-                (request.user.is_authenticated() or
+                (request.user.is_authenticated or
                  not self.authenticated_users_only) and
                 request.user.has_perms(perms)):