Convert to Python 3

environment.yml

@@ -1,9 +1,8 @@
 name: pb_env
 dependencies:
-- python =2.7.14
+- python =3.6.5
 - taxcalc =0.17.0
 - btax =0.2.2
-- ogusa =0.5.8
 - numba >=0.33.0
 - pandas >=0.22.0
 - bokeh =0.12.7
@@ -16,4 +15,4 @@ channels:
   - ospc
   - ospc/label/dev
   - defaults
-  - conda-forge
+  - conda-forge

requirements.txt

@@ -5,7 +5,6 @@ dj-static==0.0.6
 django-toolbelt==0.0.1
 waitress
 static3==0.5.1
-wsgiref==0.1.2
 -e git+https://github.com/funkybob/django-flatblocks.git#egg=django-flatblocks
 sendgrid-django
 psycopg2>=2.5.4
@@ -26,4 +25,4 @@ django-storages
 django-htmlmin
 pyparsing
 python-dateutil
-toolz
+toolz

webapp/apps/btax/bubble_plot/bubble_plot_tabs.py

@@ -14,8 +14,8 @@
 from bokeh.resources import CDN
 
 # import styles and callback
-from styles import (PLOT_FORMATS, TITLE_FORMATS, RED, BLUE)
-from controls_callback_script import CONTROLS_CALLBACK_SCRIPT
+from .styles import (PLOT_FORMATS, TITLE_FORMATS, RED, BLUE)
+from .controls_callback_script import CONTROLS_CALLBACK_SCRIPT
 
 
 def bubble_plot_tabs(dataframes):

webapp/apps/btax/compute.py

@@ -21,10 +21,10 @@
 
 def package_up_vars(self, user_mods, first_budget_year):
     # TODO - is first_budget_year important here?
-    user_mods = {k: v for k, v in user_mods.iteritems()
+    user_mods = {k: v for k, v in user_mods.items()
                  if k.startswith(('btax_', 'start_year'))}
     user_mods = {k: (v[0] if hasattr(v, '__getitem__') else v)
-                 for k, v in user_mods.iteritems()}
+                 for k, v in user_mods.items()}
     return user_mods
 
 

webapp/apps/btax/forms.py

@@ -29,7 +29,7 @@ def __init__(self, first_year, *args, **kwargs):
         # those outs here in the init because the user may
         # have chosen a different start year
         all_defaults = []
-        for param in self._default_params.values():
+        for param in list(self._default_params.values()):
             for field in param.col_fields:
                 all_defaults.append((field.id, field.default_value))
         for _id, default in all_defaults:
@@ -76,7 +76,7 @@ def do_btax_validations(self):
         are detected.
         """
 
-        for param_id, param in self._default_params.iteritems():
+        for param_id, param in self._default_params.items():
             if any(token in param_id for token in ('gds', 'ads', 'tax')):
                 param.col_fields[0].values[0] = make_bool(param.col_fields[0].values[0])
             if param.max is None and param.min is None:
@@ -112,11 +112,11 @@ def do_btax_validations(self):
                         if value > maxes[i]:
                             if len(col_values) == 1:
                                 self.add_error(col_field.id,
-                                               u"Must be \u2264 {0} of {1}".
+                                               "Must be \u2264 {0} of {1}".
                                                format(source, maxes[i]))
                             else:
                                 self.add_error(col_field.id,
-                                               u"{0} value must be \u2264 \
+                                               "{0} value must be \u2264 \
                                                {1}'s {0} value of {2}".format(
                                                    int_to_nth(i + 1),
                                                    source, maxes[i]))
@@ -131,11 +131,11 @@ def do_btax_validations(self):
                         if value < mins[i]:
                             if len(col_values) == 1:
                                 self.add_error(col_field.id,
-                                               u"Must be \u2265 {0} of {1}".
+                                               "Must be \u2265 {0} of {1}".
                                                format(source, mins[i]))
                             else:
                                 self.add_error(col_field.id,
-                                               u"{0} value must be \u2265 \
+                                               "{0} value must be \u2265 \
                                                {1}'s {0} value of {2}".format(
                                                    int_to_nth(i + 1),
                                                    source, mins[i]))
@@ -146,7 +146,7 @@ class Meta:
         widgets = {}
         labels = {}
 
-        for param in BTAX_DEFAULTS.values():
+        for param in list(BTAX_DEFAULTS.values()):
             for field in param.col_fields:
                 attrs = {
                     'class': 'form-control',

webapp/apps/btax/helpers.py

@@ -133,7 +133,7 @@ def get_btax_defaults(start_year=START_YEAR):
     defaults = dict(DEFAULTS)
     # Set Bogus default for now
     defaults['btax_betr_pass']['value'] = [0.0]
-    for k,v in defaults.items():
+    for k,v in list(defaults.items()):
         v['col_label'] = ['']
     BTAX_DEFAULTS = {}
 

webapp/apps/btax/migrations/0001_initial.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
+
 
 from django.db import models, migrations
 import datetime

webapp/apps/btax/migrations/0002_btaxoutputurl_webapp_vers.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
+
 
 from django.db import models, migrations
 

webapp/apps/btax/migrations/0003_auto_20180205_2151.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.9 on 2018-02-05 21:51
-from __future__ import unicode_literals
+
 
 from django.db import migrations, models
 import uuid

webapp/apps/btax/migrations/0004_btaxsaveinputs_data_source.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.9 on 2018-03-06 20:12
-from __future__ import unicode_literals
+
 
 from django.db import migrations, models
 

webapp/apps/btax/tests/test_views.py

@@ -13,18 +13,18 @@
 
 from ...constants import START_YEAR
 
-OK_POST_DATA = {u'btax_betr_pass': 0.33,
-                u'btax_depr_5yr': u'btax_depr_5yr_gds_Switch',
-                u'btax_depr_27_5yr_exp': 0.4,
-                u'has_errors': [u'False'],
-                u'start_year': unicode(START_YEAR),
-                u'csrfmiddlewaretoken': u'abc123'}
+OK_POST_DATA = {'btax_betr_pass': 0.33,
+                'btax_depr_5yr': 'btax_depr_5yr_gds_Switch',
+                'btax_depr_27_5yr_exp': 0.4,
+                'has_errors': ['False'],
+                'start_year': str(START_YEAR),
+                'csrfmiddlewaretoken': 'abc123'}
 
 class BTaxViewsTests(TestCase):
     ''' Test the views of this app. '''
-    expected_results_tokens = ['cost of capital', 'change from reform',
-                               'baseline', 'reform',
-                               'industry', 'asset', 'accommodation',
+    expected_results_tokens = ['Cost of Capital', 'Change from reform',
+                               'Baseline', 'Reform',
+                               'industry', 'asset', 'Accommodation',
                                'typically financed', 'were generated by']
     def setUp(self):
         # Every test needs a client.
@@ -58,7 +58,7 @@ def test_btax_post(self):
         self.assertEqual(response.status_code, 302)
         # Go to results page
         link_idx = response.url[:-1].rfind('/')
-        self.failUnless(response.url[:link_idx+1].endswith("ccc/"))
+        self.assertTrue(response.url[:link_idx+1].endswith("ccc/"))
 
     def test_btax_nodes_down(self):
         #Monkey patch to mock out running of compute jobs
@@ -72,7 +72,7 @@ def test_btax_nodes_down(self):
         # Check that redirect happens
         self.assertEqual(response.status_code, 302)
         link_idx = response.url[:-1].rfind('/')
-        self.failUnless(response.url[:link_idx+1].endswith("ccc/"))
+        self.assertTrue(response.url[:link_idx+1].endswith("ccc/"))
         # One more redirect
         print(dir(response))
         response = self.client.get(response.url)
@@ -88,20 +88,18 @@ def test_btax_failed_job(self):
         # Check that redirect happens
         self.assertEqual(response.status_code, 302)
         link_idx = response.url[:-1].rfind('/')
-        print '302 response info', response.url, link_idx, str(response), response.url[:link_idx + 1]
-        self.failUnless(response.url[:link_idx+1].endswith("ccc/"))
+        print('302 response info', response.url, link_idx, str(response), response.url[:link_idx + 1])
+        self.assertTrue(response.url[:link_idx+1].endswith("ccc/"))
         response = self.client.get(response.url)
         # Make sure the failure message is in the response
-        response = str(response)
-        print 'test_btax_failed_job response', response
-        self.failUnless("Your calculation failed" in response)
+        assert "Your calculation failed" in response.content.decode('utf-8')
 
     def test_mocked_results_table(self):
         response = self.client.post('/ccc/mock-ccc-results')
         self.assertEqual(response.status_code, 200)
-        response = str(response).lower()
+        content = response.content.decode('utf-8')
         for expected in self.expected_results_tokens:
-            self.assertIn(expected, response)
+            assert expected in content
 
     def test_btax_submit_to_single_host(self):
         """
@@ -125,7 +123,7 @@ def test_btax_submit_to_single_host(self):
         self.assertEqual(response.status_code, 302)
         # Go to results page
         link_idx = response.url[:-1].rfind('/')
-        self.failUnless(response.url[:link_idx+1].endswith("ccc/"))
+        self.assertTrue(response.url[:link_idx+1].endswith("ccc/"))
 
         # Submit another job, which would error if we incremented dropq_offset
         # with the submit
@@ -140,17 +138,17 @@ def test_btax_edit_ccc_switches_show_correctly(self):
         from webapp.apps.btax import views as webapp_views
         webapp_views.dropq_compute = MockComputeBtax()
 
-        data = { u'has_errors': [u'False'],
-                u'start_year': unicode(START_YEAR),
-                u'btax_depr_5yr': u'btax_depr_5yr_ads_Switch',
-                'csrfmiddlewaretoken':'abc123', u'start_year': u'2016'}
+        data = { 'has_errors': ['False'],
+                'start_year': str(START_YEAR),
+                'btax_depr_5yr': 'btax_depr_5yr_ads_Switch',
+                'csrfmiddlewaretoken':'abc123', 'start_year': '2016'}
 
         response = self.client.post('/ccc/', data)
         # Check that redirect happens
         self.assertEqual(response.status_code, 302)
         # Go to results page
         link_idx = response.url[:-1].rfind('/')
-        self.failUnless(response.url[:link_idx+1].endswith("ccc/"))
+        self.assertTrue(response.url[:link_idx+1].endswith("ccc/"))
         model_num = response.url[link_idx+1:-1]
         edit_ccc = '/ccc/edit/{0}/?start_year={1}'.format(model_num, START_YEAR)
         edit_page = self.client.get(edit_ccc)
@@ -159,7 +157,7 @@ def test_btax_edit_ccc_switches_show_correctly(self):
         # Get results model
         out = BTaxOutputUrl.objects.get(pk=model_num)
         bsi = BTaxSaveInputs.objects.get(pk=out.model_pk)
-        assert bsi.btax_depr_5yr == u'btax_depr_5yr_ads_Switch'
+        assert bsi.btax_depr_5yr == 'btax_depr_5yr_ads_Switch'
         assert bsi.btax_depr_5yr_ads_Switch == 'True'
 
     def test_get_not_avail_page_renders(self):

webapp/apps/btax/update_mock_json.py

@@ -12,9 +12,9 @@
         'Major Grouping 5',
         'Major Grouping 6']
 
-for h in data.keys():
+for h in list(data.keys()):
     if h.startswith('asset') or h.startswith('industry'):
-        for i in data[h].keys():
+        for i in list(data[h].keys()):
             for j in data[h][i]['rows']:
                 j['major_grouping'] = random.choice(cats)
                 j['summary_c'] = -999

webapp/apps/btax/views.py

@@ -15,7 +15,7 @@
 import datetime
 import logging
 from os import path
-from urlparse import urlparse, parse_qs
+from urllib.parse import urlparse, parse_qs
 from ipware.ip import get_real_ip
 
 from django.core import serializers
@@ -70,7 +70,7 @@
 
 
 def make_bool_gds_ads(form_btax_input):
-    for k, v in vars(form_btax_input).items():
+    for k, v in list(vars(form_btax_input).items()):
         if any(token in k for token in ('gds', 'ads', 'tax')):
             setattr(getattr(form_btax_input, k), 'value', make_bool(v))
     return form_btax_input
@@ -108,7 +108,7 @@ def btax_results(request):
         has_errors = make_bool(request.POST['has_errors'])
         fields = dict(request.GET)
         fields.update(dict(request.POST))
-        fields = {k: v[0] if isinstance(v, list) else v for k, v in fields.items()}
+        fields = {k: v[0] if isinstance(v, list) else v for k, v in list(fields.items())}
         start_year = fields.get('start_year', START_YEAR)
         # TODO: migrate first_year to start_year to get rid of weird stuff like
         # this
@@ -125,7 +125,7 @@ def btax_results(request):
         # time
 
         if btax_inputs.is_valid() or has_errors:
-            print 'is valid'
+            print('is valid')
             stored_errors = None
             if has_errors and btax_inputs.errors:
                 msg = ("Form has validation errors, but allowing the user "
@@ -149,13 +149,13 @@ def btax_results(request):
             # prepare taxcalc params from BTaxSaveInputs model
             curr_dict = dict(model.__dict__)
 
-            for key, value in curr_dict.items():
-                if type(value) == type(unicode()):
+            for key, value in list(curr_dict.items()):
+                if type(value) == type(str()):
                     curr_dict[key] = [convert_val(x) for x in value.split(',') if x]
                 else:
-                    print "missing this: ", key
+                    print("missing this: ", key)
 
-            worker_data = {k:v for k, v in curr_dict.items() if not (v == [] or v == None)}
+            worker_data = {k:v for k, v in list(curr_dict.items()) if not (v == [] or v == None)}
 
             #Non corp entity fix up:
             if 'btax_betr_pass' not in worker_data:
@@ -165,15 +165,15 @@ def btax_results(request):
             ip = get_real_ip(request)
             if ip is not None:
                 # we have a real, public ip address for user
-                print "BEGIN DROPQ WORK FROM: ", ip
+                print("BEGIN DROPQ WORK FROM: ", ip)
             else:
                 # we don't have a real, public ip address for user
-                print "BEGIN DROPQ WORK FROM: unknown IP"
+                print("BEGIN DROPQ WORK FROM: unknown IP")
 
             # start calc job
             submitted_ids, max_q_length = dropq_compute.submit_btax_calculation(worker_data, start_year)
 
-            print 'submitted_ids', submitted_ids, max_q_length
+            print('submitted_ids', submitted_ids, max_q_length)
             if not submitted_ids:
                 no_inputs = True
                 form_btax_input = btax_inputs
@@ -209,7 +209,7 @@ def btax_results(request):
                 return redirect(unique_url)
 
         else:
-            print 'invalid inputs'
+            print('invalid inputs')
             # received POST but invalid results, return to form with errors
             form_btax_input = btax_inputs
 
@@ -417,7 +417,7 @@ def output_detail(request, pk):
         try:
             jobs_ready = dropq_compute.dropq_results_ready(jobs_to_check)
         except JobFailError as jfe:
-            print jfe
+            print(jfe)
             return render_to_response('taxbrain/failed.html',
                                      context={'is_btax': True})
 
@@ -439,15 +439,15 @@ def output_detail(request, pk):
             tax_result = dropq_compute.btax_get_results(normalize(job_ids))
             model.tax_result = json.dumps(tax_result)
             model.creation_date = datetime.datetime.now()
-            print 'ready'
+            print('ready')
             model.save()
             return redirect(url)
         else:
             jobs_not_ready = [sub_id for (sub_id, job_ready) in
                                 zip(jobs_to_check, jobs_ready) if not job_ready == 'YES']
             jobs_not_ready = denormalize(jobs_not_ready)
             model.jobs_not_ready = jobs_not_ready
-            print 'not ready', jobs_not_ready
+            print('not ready', jobs_not_ready)
             model.save()
             if request.method == 'POST':
                 # if not ready yet, insert number of minutes remaining
@@ -464,7 +464,7 @@ def output_detail(request, pk):
                     return JsonResponse({'eta': exp_num_minutes,'wait_interval': 7000}, status=200)
 
             else:
-                print "rendering not ready yet"
+                print("rendering not ready yet")
                 context = {'eta': '100', 'is_btax': True, 'wait_interval': 7000}
                 context.update(context_vers_disp)
                 return render_to_response('btax/not_ready.html', context,