python: make code compatible with Python3.7

* add Pipfile
* de-lint with `flake8`

.flake8

@@ -0,0 +1,6 @@
+[flake8]
+exclude=.venv
+max-line-length=250
+import-order-style=google
+ignore=E111
+

.gitignore

@@ -12,3 +12,5 @@ setup/*/host_vars/release-*
 ansible/host_vars/*
 !ansible/host_vars/README.md
 !ansible/host_vars/*-template
+.venv
+Pipfile.lock

Pipfile

@@ -0,0 +1,14 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+ansible = "*"
+"flake8" = "*"
+"flake8-import-order" = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.7"

ansible/plugins/inventory/nodejs_yaml.py

@@ -23,35 +23,33 @@
 #
 
 from __future__ import print_function
+
 import argparse
-try:
-    import configparser
-except ImportError:
-    import ConfigParser as configparser
-try:
-    from future_builtins import filter # Python 2
-except ImportError:
-    pass # Python 3
 import json
-import yaml
 import os
-import sys
 import subprocess
+import sys
 
+import yaml
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
 
 valid = {
-  # taken from nodejs/node.git: ./configure
-  'arch': ('armv6l', 'armv7l', 'arm64', 'ia32', 'mips', 'mipsel', 'ppc',
-           'ppc64', 'x32', 'x64', 'x86', 's390', 's390x'),
-
-  # valid roles - add as necessary
-  'type': ('infra', 'release', 'test'),
-
-  # providers - validated for consistency
-  'provider': ('azure', 'digitalocean', 'joyent', 'ibm', 'linuxonecc',
-               'macstadium', 'marist', 'mininodes', 'msft', 'osuosl',
-               'rackspace', 'requireio', 'scaleway', 'softlayer', 'voxer',
-               'packetnet', 'nearform')
+    # taken from nodejs/node.git: ./configure
+    'arch': ('armv6l', 'armv7l', 'arm64', 'ia32', 'mips', 'mipsel', 'ppc', 'ppc64', 'x32', 'x64', 'x86', 's390', 's390x'),
+
+    # valid roles - add as necessary
+    'type': ('infra', 'release', 'test'),
+
+    # providers - validated for consistency
+    'provider': (
+        'azure', 'digitalocean', 'joyent', 'ibm', 'linuxonecc',
+        'macstadium', 'marist', 'mininodes', 'msft', 'osuosl',
+        'rackspace', 'requireio', 'scaleway', 'softlayer', 'voxer',
+        'packetnet', 'nearform'
+    )
 }
 DECRYPT_TOOL = "gpg"
 INVENTORY_FILENAME = "inventory.yml"
@@ -97,15 +95,16 @@ def main():
 # https://stackoverflow.com/a/7205107
 def merge(a, b, path=None):
     "merges b into a"
-    if path is None: path = []
+    if path is None:
+      path = []
     for key in b:
         if key in a:
             if isinstance(a[key], dict) and isinstance(b[key], dict):
                 merge(a[key], b[key], path + [str(key)])
             elif isinstance(a[key], list) and isinstance(b[key], list):
                 a[key] = sorted(set(a[key]).union(b[key]))
             elif a[key] == b[key]:
-                pass # same leaf value
+                pass  # same leaf value
             else:
                 raise Exception('Conflict at %s' % '.'.join(path + [str(key)]))
         else:
@@ -167,7 +166,7 @@ def load_yaml_file(file_name):
     # get inventory
     with open(file_name, 'r') as stream:
         try:
-            hosts = yaml.load(stream)
+            hosts = yaml.safe_load(stream)
 
         except yaml.YAMLError as exc:
             print(exc)
@@ -186,11 +185,11 @@ def load_yaml_secrets(file_name):
         print("WARNING: cannot load %s" % file_name, file=sys.stderr)
         return None
 
-    return yaml.load(stdout)
+    return yaml.safe_load(stdout)
 
 
 def parse_yaml(hosts, config):
-    """Parses host information from the output of yaml.load"""
+    """Parses host information from the output of yaml.safe_load"""
 
     export = {'_meta': {'hostvars': {}}}
 
@@ -210,7 +209,7 @@ def parse_yaml(hosts, config):
 
                         # some hosts have metadata appended to provider
                         # which requires underscore
-                        delimiter = "_" if host.count('-') is 3 else "-"
+                        delimiter = "_" if host.count('-') == 3 else "-"
                         hostname = '{}-{}{}{}'.format(host_type, provider_name,
                                                       delimiter, host)
 
@@ -265,7 +264,7 @@ def parse_host(host):
 
     expected = ['type', 'provider', 'os', 'arch', 'uid']
 
-    if len(info) is not 5:
+    if len(info) != 5:
         raise Exception('Host format is invalid: %s,' % host)
 
     for key, item in enumerate(expected):
@@ -279,9 +278,11 @@ def parse_host(host):
 
 
 def has_metadata(info):
-    """Checks for metadata in variables. These are separated from the "key"
-       metadata by underscore. Not used anywhere at the moment for anything
-       other than descriptiveness"""
+    """
+    Checks for metadata in variables. These are separated from the "key"
+    metadata by underscore. Not used anywhere at the moment for anything
+    other than descriptiveness
+    """
 
     metadata = info.split('_', 1)
 
@@ -296,9 +297,8 @@ def has_metadata(info):
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--list', action='store_true')
-    parser.add_argument('--host', action='store')
-    args = parser.parse_args()
-
+    # parser = argparse.ArgumentParser()
+    # parser.add_argument('--list', action='store_true')
+    # parser.add_argument('--host', action='store')
+    # args = parser.parse_args()
     main()

ansible/plugins/library/remmina_config.py

@@ -23,15 +23,17 @@
 #
 
 from __future__ import print_function
-from ansible.module_utils.basic import *
-from jinja2 import Environment
+
+import base64
 import os
+
+from ansible.module_utils.basic import AnsibleModule
+from Crypto.Cipher import DES3
+from jinja2 import Environment
 try:
-    import configparser # Python 3
+    import configparser  # Python 3
 except ImportError:
-    import ConfigParser as configparser # Python 2
-import base64
-from Crypto.Cipher import DES3
+    import ConfigParser as configparser  # Python 2
 
 
 host_template = \

ansible/plugins/library/ssh_config.py

@@ -22,11 +22,13 @@
 # IN THE SOFTWARE.
 #
 
-from ansible.module_utils.basic import *
-from jinja2 import Environment
 import os
 import re
 
+from ansible.module_utils.basic import AnsibleModule
+from jinja2 import Environment
+
+
 pre_match = '# begin: node.js template'
 post_match = '# end: node.js template'
 match = re.compile(r'^' + re.escape(pre_match) + '(.*)' + re.escape(post_match),
@@ -99,8 +101,7 @@ def main():
                          path)
 
     if not is_templatable(path, contents):
-        module.fail_json(msg='Your ssh config lacks template stubs. ' +
-                             'Check README.md for instructions.')
+        module.fail_json(msg='Your ssh config lacks template stubs. Check README.md for instructions.')
 
     rendered = '{}{}{}'.format(
         pre_match,

jenkins/scripts/coverage/generate-index-html.py

@@ -6,9 +6,9 @@
 with open('out/index.csv') as index:
   index_csv = filter(lambda line: line, index.read().split('\n'))
 
+# noqa
 with open('out/index.html', 'w') as out:
-  out.write(
-'''
+  out.write('''
 <!DOCTYPE html>
 <html>
   <head>
@@ -96,6 +96,7 @@
               <div><div class="cell-header">JS Coverage</div><div class="cell-value"><a href="coverage-{1}/index.html">{2:05.2f}&nbsp;%</a></div></div>
               <div><div class="cell-header">C++ Coverage</div><div class="cell-value"><a href="coverage-{1}/cxxcoverage.html">{3:05.2f}&nbsp;%</a></div></div>
             </div>'''.format(date, sha, float(jscov), float(cxxcov)))
+
   out.write('''
           </div>
         </div>

setup/www/host_vars/.gitignore

@@ -1 +1,2 @@
-infra-*
+infra-*
+!node-www.tmpl

setup/www/tools/metrics/country-lookup.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python
 
-import sys
 import csv
-import geoip2.database
 import os
+import sys
+
+import geoip2.database
 
 reader = geoip2.database.Reader(os.path.dirname(os.path.realpath(__file__)) + '/GeoLite2-City.mmdb')
 
@@ -28,11 +29,10 @@
         country = georec.country.iso_code
       if georec.subdivisions.most_specific.iso_code:
         region = georec.subdivisions.most_specific.iso_code
-  except:
+  except Exception:
     pass
 
   row.insert(1, country.encode('utf-8'))
   row.insert(2, region.encode('utf-8'))
 
   logFileWriter.writerow(row)
-