Merge pull request #59 from UN-OCHA/taas/85-python2-to-3

[TAAS-85] Upgrade to Python3

.travis.yml

@@ -2,12 +2,7 @@
 language: python
 
 python:
-  - 2.7
-
-# The secure section is our CODECLIMATE_REPO_TOKEN, which is modestly private.
-env:
-  - secure: >
-      wxCU9KU/XefZL9TjWXz4F+ukgYlFRKzsuK8IwVUJODQIDLraQ0OMVOd2K/1XUuM1x0/aqWqC0/NVK+8z9rTaqwy8lU1BGK51qJl/BhWBz2nDgs+0wFq5EJXpR3MG0uSVVby7K8UySNQFRMbsnPdN1AER4lknAvrbVik2pEesGfsR3YGIe+2IeYA2Skn7yQfnZj+GBRl2i6jcDoZeTOY6oTAEt2py5YHpV6GPMpuvjsqVtJ317zkmQlItvMnMoh8xKJwa31fn29305XcAMqOwpowjfknUfhV2UPWtcwIN06osUb5h9GEu/+h8hvGkbYtFUNJjMtylyvjNDqDnvBmoYek25MjMdj+GDzyjn4O4aLlhU3NN2eM9eNJ0UT3bwG/jdLl9xqNjQ07YVUkLEDIgDHfe4RmONgNUHtzj4kwWW9zs7AHHlEeAJsFj2pXVTxyyp68TNY+rl1Fyh96x4UQqi9DRaFOV2EXjjMiEY8tFljOwjJYg0JxpZUc8iIiTWxbh8d54t8l6gPEiM+q3zJqAkQFw78AJ+DWNvbAvTF6YQ7fv5lnjZZ43JEsQJRBYzb4ZMwkaLWOWPf/ocSu7N3E1W+MRgizW1l8ErF64t6Qeh+ifKogcGSplBnDSNhFZ1vSXjejNzNSXs4jGSC26W6EsMizB8pHPWfp9gBDpMeugSFM=
+  - 3.6
 
 install:
   - make

Dockerfile

@@ -1,4 +1,4 @@
-FROM unocha/alpine-base-s6-python2:3.8
+FROM unocha/alpine-base-s6-python3:3.8
 
 MAINTAINER Paul Fenwick "paul@humanitarianresponse.info"
 
@@ -23,7 +23,7 @@ RUN apk add --update --no-cache \
       py-virtualenv \
       # These are needed so python can build its things.
       gcc \
-      python-dev \
+      python3-dev \
       musl-dev \
       linux-headers \
       libffi-dev \

Makefile

@@ -14,7 +14,7 @@ GIT=git
 SED=sed
 
 venv: requirements.txt
-	virtualenv --system-site-packages venv
+	virtualenv --python=python3.6 --system-site-packages venv
 	venv/bin/pip install --upgrade pip
 	venv/bin/pip install --upgrade appdirs
 	venv/bin/pip install --upgrade --editable .
@@ -28,7 +28,6 @@ test: venv FORCE
 # Reports tests to our CI providers and friends
 test-report: test FORCE
 	venv/bin/coveralls
-	venv/bin/codeclimate-test-reporter
 
 update:	venv FORCE
 	venv/bin/gss2json --push

requirements.txt

@@ -1,9 +1,8 @@
 appdirs>=1.4.2
 backports.tempfile==1.0rc1
-cffi==1.9.1
+cffi==1.12.3
 click==6.7
-codeclimate-test-reporter==0.2.1
-coverage==4.0.3
+coverage==4.5.3
 cryptography==2.3.1
 enum34==1.1.6
 Flask>=0.12.3
@@ -24,7 +23,7 @@ pyparsing==2.2.0
 pytest==3.0.6
 pytest-cov==2.4.0
 python-coveralls==2.9.0
-PyYAML>=4.2b1
+PyYAML<5.0,>=4.2b1
 requests>=2.20.0
 six==1.10.0
 Werkzeug==0.15.4

taas/__init__.py

@@ -4,7 +4,9 @@
 import os
 import re
 import sys
-import urllib
+import urllib.request
+import urllib.parse
+import urllib.error
 import yaml
 
 from collections import namedtuple
@@ -87,7 +89,7 @@ def _merge_config(new, base):
 
     # If we've got two dicts, merge them.
     if isinstance(new, dict) and isinstance(base, dict):
-        for k, v in base.iteritems():
+        for k, v in base.items():
             if k not in new:
                 new[k] = v
             else:
@@ -182,7 +184,7 @@ def save_google_sheet(name, key, gid, file_format="csv", directory=None):
     debug("Saving {} to {}\n".format(url, filename))
 
     # TODO: How does urllib handle errors? Can we trust it to throw on failure?
-    urllib.urlretrieve(url, filename)
+    urllib.request.urlretrieve(url, filename)
 
 
 def insert_nested_value(dictionary, path, value):
@@ -240,7 +242,7 @@ def normalise_sheet(raw, mapping):
 
     for row in raw:
         cooked_row = {}
-        for label, processor in fieldmap.items():
+        for label, processor in list(fieldmap.items()):
             # Get the processed value for this field
             result = processor.emit(row)
 
@@ -434,7 +436,7 @@ def process_source(source_name, source):
     # TODO: We should have a service class definition, rather than trusting our
     #       config file is in the right format.
 
-    for version, version_config in source.iteritems():
+    for version, version_config in source.items():
 
         # Skip our options block
         if version == 'options':
@@ -472,7 +474,7 @@ def get_default_sources(config):
 
     sources = []
 
-    for source, src_config in config['sources'].iteritems():
+    for source, src_config in config['sources'].items():
         if src_config.get('options', {}).get('build_by_default', True):
             sources.append(source)
 

taas/cmdline.py

@@ -14,7 +14,7 @@ def run(cmdlist):
         change how we run things in the future, we can do so.
     """
 
-    print "$ {}\n".format(" ".join(cmdlist))
+    print("$ {}\n".format(" ".join(cmdlist)))
     subprocess.check_call(cmdlist)
 
 

taas/config.d/coordination_hubs.yml

@@ -21,7 +21,9 @@ sources:
                     optional: True
 
                 address.premise:
-                    field: Address (free text to include building name/number, street & Postcode)
+                    field: >
+                      Address (free text to include building name/number,
+                      street & Postcode)
                     optional: True
 
                 address.locality:

taas/config.d/functional_roles.yml

@@ -12,8 +12,8 @@ sources:
                 # Our numeric ID
                 id: ID
 
-                # Our rich JSON-LD data. Ideally context would be a top-level field, as opposed
-                # to being attached to each element.
+                # Our rich JSON-LD data. Ideally context would be a top-level
+                # field, as opposed to being attached to each element.
 
                 "@context":
                     type: literal

tests/test_taas.py

@@ -49,7 +49,7 @@ def test_read_config(self):
     def test_read_config_d(self):
         config = taas.read_config(self.test_config_d)
 
-        print config
+        print(config)
 
         self.assertEqual(config['food']['vegemite']['type'], 'breakfast spread')
         self.assertEqual(config['food']['durian']['type'], 'fruit')
@@ -103,7 +103,7 @@ def test_key_gid_clash(self):
                 "MySource", {"key": "foo", "gid": 0, "url": "http://example.com/"}
             )
 
-        self.assertRegexpMatches(ex.exception.message, "MySource")
+        self.assertRegex(ex.exception.args[0], "MySource")
 
     def test_fragments(self):
         config = taas.read_config(self.test_config)
@@ -147,7 +147,7 @@ def test_fragments(self):
 
                 self.assertEqual(bulbasaur['id'], '1')
                 self.assertEqual(bulbasaur['name']['en'], 'Bulbasaur')
-                self.assertEqual(bulbasaur['name']['jp'], u'フシギソウ')
+                self.assertEqual(bulbasaur['name']['jp'], 'フシギソウ')
                 self.assertEqual(bulbasaur['characteristic']['weight']['kg'], '6.9')
                 self.assertEqual(bulbasaur['type'], 'grass')