From e50eeaece536844b628cf682dfb75f5b4bd4d906 Mon Sep 17 00:00:00 2001 From: pavelbotsman Date: Wed, 30 Jan 2019 12:47:04 +0300 Subject: [PATCH] [#686] fix/disable high priority warnings --- legion/legion/containers/docker.py | 12 +++++----- legion/legion/docker_bootup.py | 2 +- legion/legion/edi/deploy.py | 6 ++--- legion/legion/edi/security.py | 4 ++-- legion/legion/edi/server.py | 15 +++++++----- legion/legion/external/edi.py | 14 ++++++----- legion/legion/external/grafana.py | 5 ++-- legion/legion/http.py | 23 +++++++++---------- legion/legion/k8s/definitions.py | 2 +- legion/legion/pymodel/model.py | 6 ++--- legion/legion/utils.py | 10 ++++---- legion/tests/legion_test_utils.py | 8 +++---- legion/tests/test_utils_external_file.py | 4 ++-- .../legion_airflow/auth/dex_auth.py | 7 ++---- legion_test/legion_test/robot/dex_client.py | 3 ++- 15 files changed, 62 insertions(+), 59 deletions(-) diff --git a/legion/legion/containers/docker.py b/legion/legion/containers/docker.py index dea239000..2139f5c4e 100644 --- a/legion/legion/containers/docker.py +++ b/legion/legion/containers/docker.py @@ -32,7 +32,7 @@ LOGGER = logging.getLogger(__name__) -def build_docker_client(args=None): +def build_docker_client(): """ Create docker client @@ -55,7 +55,7 @@ def get_docker_container_id_from_cgroup_line(line): parts = line.split('/') try: - if 'docker' in parts: + if 'docker' in parts: # pylint: disable=R1705 docker_pos = parts.index('docker') return parts[docker_pos + 1] elif 'kubepods' in parts: @@ -73,8 +73,8 @@ def get_current_docker_container_id(): :return: str -- current container id """ - with open('/proc/self/cgroup') as f: - lines = [line.strip('\n') for line in f] + with open('/proc/self/cgroup') as file: + lines = [line.strip('\n') for line in file] longest_line = max(lines, key=len) return get_docker_container_id_from_cgroup_line(longest_line) @@ -208,7 +208,7 @@ def build_docker_image(client, model_id, model_file, labels, LOGGER.info('Building docker image in folder {}'.format(temp_directory.path)) try: - image, logs = client.images.build( + image, _ = client.images.build( tag=docker_image_tag, nocache=True, path=temp_directory.path, @@ -224,7 +224,7 @@ def build_docker_image(client, model_id, model_file, labels, return image -def generate_docker_labels_for_image(model_file, model_id, args): +def generate_docker_labels_for_image(model_file, model_id): """ Generate docker image labels from model file diff --git a/legion/legion/docker_bootup.py b/legion/legion/docker_bootup.py index caeaef565..119c3a012 100644 --- a/legion/legion/docker_bootup.py +++ b/legion/legion/docker_bootup.py @@ -28,7 +28,7 @@ LOGGER.debug('Docker bootup got configuration: host={!r} port={!r}'.format(host, port)) if host and port: - import pydevd + import pydevd # pylint: disable=F0401 try: port = int(port) diff --git a/legion/legion/edi/deploy.py b/legion/legion/edi/deploy.py index 782512769..af90352b3 100644 --- a/legion/legion/edi/deploy.py +++ b/legion/legion/edi/deploy.py @@ -47,7 +47,7 @@ def build_model(args): :type args: :py:class:`argparse.Namespace` :return: :py:class:`docker.model.Image` docker image """ - client = legion.containers.docker.build_docker_client(args) + client = legion.containers.docker.build_docker_client() with ExternalFileReader(args.model_file) as external_reader: if not os.path.exists(external_reader.path): @@ -56,7 +56,7 @@ def build_model(args): container = legion.pymodel.Model.load(external_reader.path) model_id = container.model_id - image_labels = legion.containers.docker.generate_docker_labels_for_image(external_reader.path, model_id, args) + image_labels = legion.containers.docker.generate_docker_labels_for_image(external_reader.path, model_id) LOGGER.info('Building docker image...') image = legion.containers.docker.build_docker_image( @@ -77,7 +77,7 @@ def build_model(args): return image -def inspect_kubernetes(args): +def inspect_kubernetes(args): # pylint: disable=R0912 """ Inspect kubernetes diff --git a/legion/legion/edi/security.py b/legion/legion/edi/security.py index eb16b5c64..95270e012 100644 --- a/legion/legion/edi/security.py +++ b/legion/legion/edi/security.py @@ -130,8 +130,8 @@ def get_security_params_from_config(): config_parser.read(str(config_path)) return dict(config_parser['security']) - except Exception as e: - LOG.debug('Exception during parsing of legion config {}'.format(e), exc_info=True) + except Exception as exception: + LOG.debug('Exception during parsing of legion config {}'.format(exception), exc_info=True) return {} diff --git a/legion/legion/edi/server.py b/legion/legion/edi/server.py index b1aa10b89..0f9c211f5 100644 --- a/legion/legion/edi/server.py +++ b/legion/legion/edi/server.py @@ -22,19 +22,22 @@ from datetime import datetime, timedelta import jwt + +from flask import Flask, Blueprint, render_template, request +from flask import current_app as app + import legion.config import legion.external.grafana import legion.http import legion.k8s import legion.model -from flask import Flask, Blueprint, render_template, request -from flask import current_app as app + LOGGER = logging.getLogger(__name__) -blueprint = Blueprint('apiserver', __name__) +blueprint = Blueprint('apiserver', __name__) # pylint: disable=C0103 TEMPLATES_FOLDER = os.path.abspath(os.path.join( os.path.dirname(__file__), os.pardir, 'templates', 'edi') -) +) # pylint: disable=C0330 AUTH_COOKIE_NAME = '_oauth2_proxy' EDI_VERSION = '1.0' @@ -81,7 +84,7 @@ def authenticate(user, password): return False - # TODO: Add LDAP authorisation + # TODO: Add LDAP authorisation # pylint:disable=W0511 return False @@ -293,7 +296,7 @@ def inspect(model=None, version=None): try: model_api_info['result'] = model_client.info() model_api_ok = True - except Exception as model_api_exception: + except Exception as model_api_exception: # pylint: disable=W0703 LOGGER.error('Cannot connect to model <{}> endpoint to get info: {}'.format(model_service, model_api_exception)) model_api_info['exception'] = str(model_api_exception) diff --git a/legion/legion/external/edi.py b/legion/legion/external/edi.py index f0f7b8d16..9951065e4 100644 --- a/legion/legion/external/edi.py +++ b/legion/legion/external/edi.py @@ -22,11 +22,12 @@ import os from urllib.parse import urlparse +import requests +import requests.exceptions + import legion.k8s import legion.edi.server import legion.config -import requests -import requests.exceptions from legion.edi import security LOGGER = logging.getLogger(__name__) @@ -53,7 +54,7 @@ def __init__(self, base, token=None, retries=3): self._version = legion.edi.server.EDI_VERSION self._retries = retries - def _request(self, action, url, data=None, headers=None, cookies=None): + def _request(self, action, url, data=None, headers=None, cookies=None): # pylint: disable=R0201, R0913 """ Make HTTP request :param action: request action, e.g. get / post / delete @@ -173,7 +174,8 @@ def info(self): """ return self._query(legion.edi.server.EDI_INFO) - def deploy(self, image, model_iam_role=None, count=1, livenesstimeout=2, readinesstimeout=2): + def deploy(self, image, model_iam_role=None, count=1, livenesstimeout=2, # pylint: disable=R0913 + readinesstimeout=2): """ Deploy API endpoint @@ -250,7 +252,7 @@ def scale(self, model, count, version=None): return self.parse_deployments(self._query(legion.edi.server.EDI_SCALE, action='POST', payload=payload)) - def get_token(self, model_id, model_version): + def get_token(self, model_id, model_version): # pylint: disable=R1710 """ Get API token @@ -322,7 +324,7 @@ def build_client(args=None): if not host: try: host = legion.k8s.Enclave(os.environ.get("NAMESPACE")).edi_service.url - except Exception: + except Exception: # pylint: disable=W0703 LOGGER.warning('Cannot get EDI URL from K8S API') client = EdiClient(host, token) diff --git a/legion/legion/external/grafana.py b/legion/legion/external/grafana.py index aa755e938..b06951ea5 100644 --- a/legion/legion/external/grafana.py +++ b/legion/legion/external/grafana.py @@ -21,10 +21,11 @@ import logging import os +import requests + import legion.config from legion.utils import render_template -import requests LOGGER = logging.getLogger(__name__) @@ -79,7 +80,7 @@ def _query(self, url, payload=None, action='GET'): if resp.status_code != 200: raise Exception('Wrong answer for [url="{}", status="{}", text="{}"]'.format( full_url, resp.status_code, resp.text) - ) + ) # pylint: disable=C0330 answer = resp.json() diff --git a/legion/legion/http.py b/legion/legion/http.py index aca7af857..9e4a00e54 100644 --- a/legion/legion/http.py +++ b/legion/legion/http.py @@ -19,15 +19,16 @@ import functools import os import logging +from urllib.parse import parse_qs + +import flask +from requests.compat import urlencode +from requests.utils import to_key_val_list import legion.config import legion.containers.headers import legion.utils -import flask -from requests.compat import urlencode -from requests.utils import to_key_val_list -from urllib.parse import parse_qs LOGGER = logging.getLogger(__name__) @@ -44,14 +45,13 @@ def encode_http_params(data): return urlencode(data) elif hasattr(data, '__iter__'): result = [] - for k, vs in to_key_val_list(data): - if vs is not None: + for key, value in to_key_val_list(data): + if value is not None: result.append( - (k.encode('utf-8') if isinstance(k, str) else k, - vs.encode('utf-8') if isinstance(vs, str) else vs)) + (key.encode('utf-8') if isinstance(key, str) else key, + value.encode('utf-8') if isinstance(value, str) else value)) return urlencode(result, doseq=True) - else: - raise ValueError('Invalid argument') + raise ValueError('Invalid argument') def parse_multi_dict(multi_dict, map_func=None): @@ -129,8 +129,7 @@ def parse_request(input_request): **parse_multi_dict(input_request.form), **parse_multi_dict(input_request.files, lambda file: file.read()) } - else: - raise ValueError('Unexpected http method: {}'.format(input_request.method)) + raise ValueError('Unexpected http method: {}'.format(input_request.method)) def prepare_response(response_data, model_id=None, model_version=None, model_endpoint=None): diff --git a/legion/legion/k8s/definitions.py b/legion/legion/k8s/definitions.py index b598bc1c4..6a552a1bf 100644 --- a/legion/legion/k8s/definitions.py +++ b/legion/legion/k8s/definitions.py @@ -73,7 +73,7 @@ def __init__(self, model_id, model_version): self._version = model_version @property - def id(self): + def id(self): # pylint: disable=C0103 """ Get model ID diff --git a/legion/legion/pymodel/model.py b/legion/legion/pymodel/model.py index 50e3602bd..3a65b632c 100644 --- a/legion/legion/pymodel/model.py +++ b/legion/legion/pymodel/model.py @@ -95,10 +95,10 @@ def invoke(self, input_vector): data_frame = legion.model.types.build_df(self.column_types, input_vector, not self.use_df) LOGGER.info('Running prepare with DataFrame: %r' % data_frame) - data_frame = self.prepare(data_frame) + data_frame = self.prepare(data_frame) # pylint: disable=E1102 LOGGER.info('Applying function with DataFrame: %s' % str(data_frame)) - response = self.apply(data_frame) + response = self.apply(data_frame) # pylint: disable=E1102 LOGGER.info('Returning response: %s' % str(response)) return response @@ -439,7 +439,7 @@ def _export(self, apply_func, prepare_func, column_types, use_df, endpoint_name) raise Exception('Bad param_types / input_data_frame provided') if prepare_func is None: - def prepare_func(input_dict): + def prepare_func(input_dict): # pylint: disable=E0102 """ Return input value (default prepare function) :param input_dict: dict of values diff --git a/legion/legion/utils.py b/legion/legion/utils.py index 4f78410df..cb99b600e 100644 --- a/legion/legion/utils.py +++ b/legion/legion/utils.py @@ -20,7 +20,7 @@ import contextlib import os import getpass -import distutils.dir_util +import distutils.dir_util # pylint: disable=E0611,E0401 import re import logging import shutil @@ -33,13 +33,13 @@ import zipfile import inspect -import legion.config -import legion.containers.headers - import requests import requests.auth from jinja2 import Environment, PackageLoader, select_autoescape +import legion.config +import legion.containers.headers + KUBERNETES_STRING_LENGTH_LIMIT = 63 LOGGER = logging.getLogger(__name__) @@ -593,7 +593,7 @@ def get_installed_packages(): import pkg_resources return sorted([ (item.key, item.version) - for item in pkg_resources.working_set + for item in pkg_resources.working_set # pylint: disable=E1133 ], key=lambda item: item[0]) diff --git a/legion/tests/legion_test_utils.py b/legion/tests/legion_test_utils.py index 41d8b3797..b373c1717 100644 --- a/legion/tests/legion_test_utils.py +++ b/legion/tests/legion_test_utils.py @@ -219,7 +219,7 @@ def response_catcher(*args, **kwargs): # This helps to emulate kubernetes exception if isinstance(current_test_resource_name, BaseException): - raise current_test_resource_name + raise current_test_resource_name # pylint: disable=E0702 searched_files = glob.glob('{}/{}.*.{}.json'.format(TEST_RESPONSES_LOCATION, function, current_test_resource_name)) @@ -264,7 +264,7 @@ def __init__(self, image, port): :type port: int """ - self._docker_client = legion.containers.docker.build_docker_client(None) + self._docker_client = legion.containers.docker.build_docker_client() self._image = image self._port = port @@ -373,7 +373,7 @@ def __init__(self, self._docker_tag = 'legion/base-python-image:{}'.format(self._docker_image_version) self._docker_base_image = None self._docker_container = None - self._docker_client = legion.containers.docker.build_docker_client(None) + self._docker_client = legion.containers.docker.build_docker_client() self._docker_volume = None # Legion python package @@ -782,7 +782,7 @@ def __init__(self, model_image): :type model_image: str """ self._image_id = model_image - self._docker_client = legion.containers.docker.build_docker_client(None) + self._docker_client = legion.containers.docker.build_docker_client() self._image = self._docker_client.images.get(self._image_id) diff --git a/legion/tests/test_utils_external_file.py b/legion/tests/test_utils_external_file.py index 140d3b3e7..3c169ac2f 100644 --- a/legion/tests/test_utils_external_file.py +++ b/legion/tests/test_utils_external_file.py @@ -102,10 +102,10 @@ def test_resource_type_detection(self): with self.assertRaisesRegex(Exception, 'Unknown or unavailable resource'): utils.is_local_resource(path) - @responses.activate + @responses.activate # pylint: disable=E1101 def _test_external_file_reader(self, url): body = 'Example' * 200 - responses.add('GET', url, body=body, stream=True) + responses.add('GET', url, body=body, stream=True) # pylint: disable=E1101 with utils.ExternalFileReader(url) as reader: self.assertTrue(os.path.exists(reader.path)) diff --git a/legion_airflow/legion_airflow/auth/dex_auth.py b/legion_airflow/legion_airflow/auth/dex_auth.py index fe1915eae..606cf7d49 100644 --- a/legion_airflow/legion_airflow/auth/dex_auth.py +++ b/legion_airflow/legion_airflow/auth/dex_auth.py @@ -19,11 +19,8 @@ from sys import version_info -import flask_login -from flask_login import (current_user, - logout_user, - login_required, - login_user) +import flask_login # pylint: disable=E0401 +from flask_login import current_user, logout_user, login_required, login_user # pylint: disable=W0611,E0401 from flask import flash diff --git a/legion_test/legion_test/robot/dex_client.py b/legion_test/legion_test/robot/dex_client.py index fa005d1e0..09ad99505 100644 --- a/legion_test/legion_test/robot/dex_client.py +++ b/legion_test/legion_test/robot/dex_client.py @@ -78,7 +78,8 @@ def init_session_id(login: str, password: str, cluster_host: str) -> None: data = {PARAM_NAME_LOGIN: login, PARAM_NAME_PASSWORD: password} response = session.post(url, data) if response.status_code != 200: - raise IOError('Unable to authorise, got {} http code from {} for the query to {} with data, response {}' + raise IOError('Unable to authorise, got {} http code from {} ' # pylint: disable=E1305 + 'for the query to {} with data, response {}' .format(response.status_code, auth_endpoint_url.format(cluster_host), url, data, response.text))