From f7bc1a7654369271c99ccbaf17a1191221f69281 Mon Sep 17 00:00:00 2001 From: Refael Ackermann Date: Thu, 12 Jul 2018 15:12:17 -0400 Subject: [PATCH] python: make code compatible with Python3.7 * add Pipfile * de-lint with `flake8` --- ansible/.editorconfig => .editorconfig | 0 .flake8 | 6 ++ .gitignore | 2 + Pipfile | 14 ++++ ansible/plugins/inventory/nodejs_yaml.py | 74 +++++++++---------- ansible/plugins/library/remmina_config.py | 14 ++-- ansible/plugins/library/ssh_config.py | 9 ++- .../scripts/coverage/generate-index-html.py | 5 +- setup/www/host_vars/.gitignore | 3 +- setup/www/tools/metrics/country-lookup.py | 8 +- 10 files changed, 81 insertions(+), 54 deletions(-) rename ansible/.editorconfig => .editorconfig (100%) create mode 100644 .flake8 create mode 100644 Pipfile diff --git a/ansible/.editorconfig b/.editorconfig similarity index 100% rename from ansible/.editorconfig rename to .editorconfig diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..0892f3d60 --- /dev/null +++ b/.flake8 @@ -0,0 +1,6 @@ +[flake8] +exclude=.venv +max-line-length=250 +import-order-style=google +ignore=E111 + diff --git a/.gitignore b/.gitignore index 6fd7c95dc..c6fd7d207 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ setup/*/host_vars/release-* ansible/host_vars/* !ansible/host_vars/README.md !ansible/host_vars/*-template +.venv +Pipfile.lock diff --git a/Pipfile b/Pipfile new file mode 100644 index 000000000..f2907985c --- /dev/null +++ b/Pipfile @@ -0,0 +1,14 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +ansible = "*" +"flake8" = "*" +"flake8-import-order" = "*" + +[dev-packages] + +[requires] +python_version = "3.7" diff --git a/ansible/plugins/inventory/nodejs_yaml.py b/ansible/plugins/inventory/nodejs_yaml.py index cbd437daf..26888bb0d 100755 --- a/ansible/plugins/inventory/nodejs_yaml.py +++ b/ansible/plugins/inventory/nodejs_yaml.py @@ -23,35 +23,33 @@ # from __future__ import print_function + import argparse -try: - import configparser -except ImportError: - import ConfigParser as configparser -try: - from future_builtins import filter # Python 2 -except ImportError: - pass # Python 3 import json -import yaml import os -import sys import subprocess +import sys +import yaml +try: + import configparser +except ImportError: + import ConfigParser as configparser valid = { - # taken from nodejs/node.git: ./configure - 'arch': ('armv6l', 'armv7l', 'arm64', 'ia32', 'mips', 'mipsel', 'ppc', - 'ppc64', 'x32', 'x64', 'x86', 's390', 's390x'), - - # valid roles - add as necessary - 'type': ('infra', 'release', 'test'), - - # providers - validated for consistency - 'provider': ('azure', 'digitalocean', 'joyent', 'ibm', 'linuxonecc', - 'macstadium', 'marist', 'mininodes', 'msft', 'osuosl', - 'rackspace', 'requireio', 'scaleway', 'softlayer', 'voxer', - 'packetnet', 'nearform') + # taken from nodejs/node.git: ./configure + 'arch': ('armv6l', 'armv7l', 'arm64', 'ia32', 'mips', 'mipsel', 'ppc', 'ppc64', 'x32', 'x64', 'x86', 's390', 's390x'), + + # valid roles - add as necessary + 'type': ('infra', 'release', 'test'), + + # providers - validated for consistency + 'provider': ( + 'azure', 'digitalocean', 'joyent', 'ibm', 'linuxonecc', + 'macstadium', 'marist', 'mininodes', 'msft', 'osuosl', + 'rackspace', 'requireio', 'scaleway', 'softlayer', 'voxer', + 'packetnet', 'nearform' + ) } DECRYPT_TOOL = "gpg" INVENTORY_FILENAME = "inventory.yml" @@ -97,7 +95,8 @@ def main(): # https://stackoverflow.com/a/7205107 def merge(a, b, path=None): "merges b into a" - if path is None: path = [] + if path is None: + path = [] for key in b: if key in a: if isinstance(a[key], dict) and isinstance(b[key], dict): @@ -105,7 +104,7 @@ def merge(a, b, path=None): elif isinstance(a[key], list) and isinstance(b[key], list): a[key] = sorted(set(a[key]).union(b[key])) elif a[key] == b[key]: - pass # same leaf value + pass # same leaf value else: raise Exception('Conflict at %s' % '.'.join(path + [str(key)])) else: @@ -167,7 +166,7 @@ def load_yaml_file(file_name): # get inventory with open(file_name, 'r') as stream: try: - hosts = yaml.load(stream) + hosts = yaml.safe_load(stream) except yaml.YAMLError as exc: print(exc) @@ -186,11 +185,11 @@ def load_yaml_secrets(file_name): print("WARNING: cannot load %s" % file_name, file=sys.stderr) return None - return yaml.load(stdout) + return yaml.safe_load(stdout) def parse_yaml(hosts, config): - """Parses host information from the output of yaml.load""" + """Parses host information from the output of yaml.safe_load""" export = {'_meta': {'hostvars': {}}} @@ -210,7 +209,7 @@ def parse_yaml(hosts, config): # some hosts have metadata appended to provider # which requires underscore - delimiter = "_" if host.count('-') is 3 else "-" + delimiter = "_" if host.count('-') == 3 else "-" hostname = '{}-{}{}{}'.format(host_type, provider_name, delimiter, host) @@ -265,7 +264,7 @@ def parse_host(host): expected = ['type', 'provider', 'os', 'arch', 'uid'] - if len(info) is not 5: + if len(info) != 5: raise Exception('Host format is invalid: %s,' % host) for key, item in enumerate(expected): @@ -279,9 +278,11 @@ def parse_host(host): def has_metadata(info): - """Checks for metadata in variables. These are separated from the "key" - metadata by underscore. Not used anywhere at the moment for anything - other than descriptiveness""" + """ + Checks for metadata in variables. These are separated from the "key" + metadata by underscore. Not used anywhere at the moment for anything + other than descriptiveness + """ metadata = info.split('_', 1) @@ -296,9 +297,8 @@ def has_metadata(info): if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--list', action='store_true') - parser.add_argument('--host', action='store') - args = parser.parse_args() - + # parser = argparse.ArgumentParser() + # parser.add_argument('--list', action='store_true') + # parser.add_argument('--host', action='store') + # args = parser.parse_args() main() diff --git a/ansible/plugins/library/remmina_config.py b/ansible/plugins/library/remmina_config.py index a1864846e..882bef6f3 100755 --- a/ansible/plugins/library/remmina_config.py +++ b/ansible/plugins/library/remmina_config.py @@ -23,15 +23,17 @@ # from __future__ import print_function -from ansible.module_utils.basic import * -from jinja2 import Environment + +import base64 import os + +from ansible.module_utils.basic import AnsibleModule +from Crypto.Cipher import DES3 +from jinja2 import Environment try: - import configparser # Python 3 + import configparser # Python 3 except ImportError: - import ConfigParser as configparser # Python 2 -import base64 -from Crypto.Cipher import DES3 + import ConfigParser as configparser # Python 2 host_template = \ diff --git a/ansible/plugins/library/ssh_config.py b/ansible/plugins/library/ssh_config.py index 66c07b4f7..c7485fb84 100755 --- a/ansible/plugins/library/ssh_config.py +++ b/ansible/plugins/library/ssh_config.py @@ -22,11 +22,13 @@ # IN THE SOFTWARE. # -from ansible.module_utils.basic import * -from jinja2 import Environment import os import re +from ansible.module_utils.basic import AnsibleModule +from jinja2 import Environment + + pre_match = '# begin: node.js template' post_match = '# end: node.js template' match = re.compile(r'^' + re.escape(pre_match) + '(.*)' + re.escape(post_match), @@ -99,8 +101,7 @@ def main(): path) if not is_templatable(path, contents): - module.fail_json(msg='Your ssh config lacks template stubs. ' + - 'Check README.md for instructions.') + module.fail_json(msg='Your ssh config lacks template stubs. Check README.md for instructions.') rendered = '{}{}{}'.format( pre_match, diff --git a/jenkins/scripts/coverage/generate-index-html.py b/jenkins/scripts/coverage/generate-index-html.py index 695b5978c..5020e8c59 100755 --- a/jenkins/scripts/coverage/generate-index-html.py +++ b/jenkins/scripts/coverage/generate-index-html.py @@ -6,9 +6,9 @@ with open('out/index.csv') as index: index_csv = filter(lambda line: line, index.read().split('\n')) +# noqa with open('out/index.html', 'w') as out: - out.write( -''' + out.write(''' @@ -96,6 +96,7 @@
JS Coverage
{2:05.2f} %
C++ Coverage
{3:05.2f} %
'''.format(date, sha, float(jscov), float(cxxcov))) + out.write(''' diff --git a/setup/www/host_vars/.gitignore b/setup/www/host_vars/.gitignore index 22cdad5a0..7157cb3bb 100644 --- a/setup/www/host_vars/.gitignore +++ b/setup/www/host_vars/.gitignore @@ -1 +1,2 @@ -infra-* \ No newline at end of file +infra-* +!node-www.tmpl diff --git a/setup/www/tools/metrics/country-lookup.py b/setup/www/tools/metrics/country-lookup.py index 2cb63df7b..6a132f63c 100755 --- a/setup/www/tools/metrics/country-lookup.py +++ b/setup/www/tools/metrics/country-lookup.py @@ -1,9 +1,10 @@ #!/usr/bin/env python -import sys import csv -import geoip2.database import os +import sys + +import geoip2.database reader = geoip2.database.Reader(os.path.dirname(os.path.realpath(__file__)) + '/GeoLite2-City.mmdb') @@ -28,11 +29,10 @@ country = georec.country.iso_code if georec.subdivisions.most_specific.iso_code: region = georec.subdivisions.most_specific.iso_code - except: + except Exception: pass row.insert(1, country.encode('utf-8')) row.insert(2, region.encode('utf-8')) logFileWriter.writerow(row) -