diff --git a/Makefile b/Makefile index deb5d6b32..bec4698c6 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ MODULE=cwltool # `SHELL=bash` doesn't work for some, so don't use BASH-isms like # `[[` conditional expressions. PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py -DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 flake8 pytest +DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 flake8 pytest isort DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pep257 sloccount python-flake8 VERSION=1.0.$(shell date +%Y%m%d%H%M%S --date=`git log --first-parent \ --max-count=1 --format=format:%cI`) @@ -65,6 +65,11 @@ clean: FORCE rm -Rf .coverage rm -f diff-cover.html +# Linting and code style related targets +## sorting imports using isort: https://github.com/timothycrosley/isort +sort_imports: + isort ${MODULE}/*.py tests/*.py setup.py + ## pep8 : check Python code style pep8: $(PYSOURCES) pep8 --exclude=_version.py --show-source --show-pep8 $^ || true diff --git a/cwltool/builder.py b/cwltool/builder.py index 658c4d383..52693dbcd 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -1,17 +1,19 @@ import copy +from typing import Any, Callable, Text, Type, Union + +from six import iteritems, string_types import avro import schema_salad.validate as validate from schema_salad.sourceline import SourceLine -from typing import Any, Callable, Text, Type, Union -from six import string_types, iteritems from . import expression from .errors import WorkflowException -from .pathmapper import PathMapper, normalizeFilesDirs, get_listing, visit_class +from .mutation import MutationManager +from .pathmapper import (PathMapper, get_listing, normalizeFilesDirs, + visit_class) from .stdfsaccess import StdFsAccess from .utils import aslist -from .mutation import MutationManager CONTENT_LIMIT = 64 * 1024 diff --git a/cwltool/cwlrdf.py b/cwltool/cwlrdf.py index e72b0c799..7b0f6f8fd 100644 --- a/cwltool/cwlrdf.py +++ b/cwltool/cwlrdf.py @@ -1,7 +1,9 @@ +from typing import IO, Any, Dict, Text + from rdflib import Graph + from schema_salad.jsonld_context import makerdf from schema_salad.ref_resolver import ContextType -from typing import Any, Dict, IO, Text from six.moves import urllib from .process import Process diff --git a/cwltool/docker.py b/cwltool/docker.py index d849cd199..c2ada773f 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -4,9 +4,9 @@ import subprocess import sys import tempfile +from typing import Text import requests -from typing import Text from .errors import WorkflowException diff --git a/cwltool/docker_uid.py b/cwltool/docker_uid.py index e5cca4250..fad871152 100644 --- a/cwltool/docker_uid.py +++ b/cwltool/docker_uid.py @@ -1,6 +1,6 @@ from __future__ import print_function -import subprocess +import subprocess from typing import Text diff --git a/cwltool/draft2tool.py b/cwltool/draft2tool.py index 5a52836c2..e226e9e47 100644 --- a/cwltool/draft2tool.py +++ b/cwltool/draft2tool.py @@ -6,24 +6,26 @@ import re import shutil import tempfile -from six.moves import urllib -from six import string_types, u from functools import partial +from typing import Any, Callable, Dict, Generator, Optional, Text, Union, cast + +from six import string_types, u import schema_salad.validate as validate import shellescape from schema_salad.ref_resolver import file_uri, uri_file_path from schema_salad.sourceline import SourceLine, indent -from typing import Any, Callable, cast, Generator, Optional, Text, Union, Dict +from six.moves import urllib -from .builder import CONTENT_LIMIT, substitute, Builder -from .pathmapper import adjustFileObjs, adjustDirObjs, visit_class +from .builder import CONTENT_LIMIT, Builder, substitute from .errors import WorkflowException -from .job import JobBase, CommandLineJob, DockerCommandLineJob -from .pathmapper import PathMapper, get_listing, trim_listing -from .process import (Process, shortname, uniquename, normalizeFilesDirs, - compute_checksums, _logger_validation_warnings, - UnsupportedRequirement) +from .flatten import flatten +from .job import CommandLineJob, DockerCommandLineJob, JobBase +from .pathmapper import (PathMapper, adjustDirObjs, adjustFileObjs, + get_listing, trim_listing, visit_class) +from .process import (Process, UnsupportedRequirement, + _logger_validation_warnings, compute_checksums, + normalizeFilesDirs, shortname, uniquename) from .stdfsaccess import StdFsAccess from .utils import aslist @@ -31,7 +33,6 @@ ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*") # Accept anything ACCEPTLIST_RE = ACCEPTLIST_EN_STRICT_RE -from .flatten import flatten _logger = logging.getLogger("cwltool") diff --git a/cwltool/expression.py b/cwltool/expression.py index 575d53b66..becb6f900 100644 --- a/cwltool/expression.py +++ b/cwltool/expression.py @@ -2,8 +2,8 @@ import json import logging import re +from typing import Any, AnyStr, Dict, List, Text, Union -from typing import Any, AnyStr, Union, Text, Dict, List from six import u from . import sandboxjs diff --git a/cwltool/factory.py b/cwltool/factory.py index cb1dccd88..3f7447d63 100644 --- a/cwltool/factory.py +++ b/cwltool/factory.py @@ -1,11 +1,8 @@ import os - -from typing import Any, Text, Union, Tuple from typing import Callable as tCallable +from typing import Any, Text, Tuple, Union -from . import load_tool -from . import main -from . import workflow +from . import load_tool, main, workflow from .process import Process diff --git a/cwltool/flatten.py b/cwltool/flatten.py index 0e3e437ce..477444aad 100644 --- a/cwltool/flatten.py +++ b/cwltool/flatten.py @@ -1,6 +1,5 @@ from typing import Any, Callable, List, cast - # http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html diff --git a/cwltool/job.py b/cwltool/job.py index 918522be8..60573cbad 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -8,18 +8,18 @@ import subprocess import sys import tempfile +from typing import (IO, Any, Callable, Iterable, List, MutableMapping, Text, + Tuple, Union, cast) import shellescape -from typing import (Any, Callable, Union, Iterable, MutableMapping, - IO, Text, Tuple, cast, List) from . import docker from .builder import Builder from .docker_uid import docker_vm_uid from .errors import WorkflowException from .pathmapper import PathMapper -from .process import (get_feature, empty_subtree, stageFiles, - UnsupportedRequirement) +from .process import (UnsupportedRequirement, empty_subtree, get_feature, + stageFiles) _logger = logging.getLogger("cwltool") diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 7f727d3b6..822cca8de 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -5,20 +5,20 @@ import os import re import uuid +from typing import Any, Callable, Dict, Text, Tuple, Union, cast import requests.sessions +from six import itervalues, string_types + import schema_salad.schema as schema from avro.schema import Names -from ruamel.yaml.comments import CommentedSeq, CommentedMap -from schema_salad.ref_resolver import Loader, Fetcher, file_uri +from ruamel.yaml.comments import CommentedMap, CommentedSeq +from schema_salad.ref_resolver import Fetcher, Loader, file_uri from schema_salad.sourceline import cmap from schema_salad.validate import ValidationException -from typing import Any, Callable, cast, Dict, Text, Tuple, Union from six.moves import urllib -from six import itervalues, string_types -from . import process -from . import update +from . import process, update from .errors import WorkflowException from .process import Process, shortname diff --git a/cwltool/main.py b/cwltool/main.py index 9f755b690..f627d3ae2 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -8,29 +8,31 @@ import os import sys import tempfile +from typing import (IO, Any, AnyStr, Callable, Dict, Sequence, Text, Tuple, + Union, cast) import pkg_resources # part of setuptools import requests + import ruamel.yaml as yaml import schema_salad.validate as validate -from schema_salad.ref_resolver import Loader, Fetcher, file_uri, uri_file_path +from schema_salad.ref_resolver import Fetcher, Loader, file_uri, uri_file_path from schema_salad.sourceline import strip_dup_lineno -from typing import (Union, Any, AnyStr, cast, Callable, Dict, Sequence, Text, - Tuple, IO) - -from . import draft2tool -from . import workflow -from .pathmapper import adjustDirObjs, get_listing, adjustFileObjs, trim_listing, visit_class -from .cwlrdf import printrdf, printdot -from .errors import WorkflowException, UnsupportedRequirement -from .load_tool import fetch_document, validate_document, make_tool + +from . import draft2tool, workflow +from .cwlrdf import printdot, printrdf +from .errors import UnsupportedRequirement, WorkflowException +from .load_tool import fetch_document, make_tool, validate_document +from .mutation import MutationManager from .pack import pack -from .process import (shortname, Process, relocateOutputs, cleanIntermediate, - scandeps, normalizeFilesDirs, use_custom_schema, use_standard_schema) -from .resolver import tool_resolver, ga4gh_tool_registries +from .pathmapper import (adjustDirObjs, adjustFileObjs, get_listing, + trim_listing, visit_class) +from .process import (Process, cleanIntermediate, normalizeFilesDirs, + relocateOutputs, scandeps, shortname, use_custom_schema, + use_standard_schema) +from .resolver import ga4gh_tool_registries, tool_resolver from .stdfsaccess import StdFsAccess -from .mutation import MutationManager -from .update import UPDATES, ALLUPDATES +from .update import ALLUPDATES, UPDATES _logger = logging.getLogger("cwltool") diff --git a/cwltool/mutation.py b/cwltool/mutation.py index a91e3cd31..0d389f046 100644 --- a/cwltool/mutation.py +++ b/cwltool/mutation.py @@ -1,6 +1,5 @@ from collections import namedtuple - -from typing import Any, Callable, cast, Generator, Iterable, List, Text, Union +from typing import Any, Callable, Generator, Iterable, List, Text, Union, cast from .errors import WorkflowException diff --git a/cwltool/pack.py b/cwltool/pack.py index f4b532ce1..bf1114b65 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -1,7 +1,7 @@ import copy +from typing import Any, Callable, Dict, Text, Union, cast from schema_salad.ref_resolver import Loader -from typing import Union, Any, cast, Callable, Dict, Text from six.moves import urllib from .process import shortname, uniquename diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 3faef33e8..2c1b4ee19 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -4,14 +4,14 @@ import stat import uuid from functools import partial +from typing import Any, Callable, Iterable, Set, Text, Tuple, Union import schema_salad.validate as validate from schema_salad.ref_resolver import uri_file_path from schema_salad.sourceline import SourceLine -from typing import Any, Callable, Set, Text, Tuple, Union, Iterable from six.moves import urllib -from .stdfsaccess import abspath, StdFsAccess +from .stdfsaccess import StdFsAccess, abspath _logger = logging.getLogger("cwltool") diff --git a/cwltool/process.py b/cwltool/process.py index 4b0942e52..245c9f817 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -1,6 +1,7 @@ import abc import copy import errno +import functools import hashlib import json import logging @@ -11,28 +12,28 @@ import urlparse import uuid from collections import Iterable -import functools +from typing import (Any, AnyStr, Callable, Dict, Generator, List, Text, Tuple, + Union, cast) + +from pkg_resources import resource_stream +from rdflib import Graph, URIRef +from rdflib.namespace import OWL, RDFS import avro.schema import schema_salad.schema import schema_salad.validate as validate -from pkg_resources import resource_stream -from rdflib import Graph -from rdflib import URIRef -from rdflib.namespace import RDFS, OWL -from ruamel.yaml.comments import CommentedSeq, CommentedMap +from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.ref_resolver import Loader, file_uri from schema_salad.sourceline import SourceLine -from typing import (Any, AnyStr, Callable, cast, Dict, List, Generator, Text, - Tuple, Union) from .builder import Builder -from .pathmapper import adjustDirObjs, get_listing -from .errors import WorkflowException, UnsupportedRequirement -from .pathmapper import PathMapper, normalizeFilesDirs, visit_class +from .errors import UnsupportedRequirement, WorkflowException +from .pathmapper import (PathMapper, adjustDirObjs, get_listing, + normalizeFilesDirs, visit_class) from .stdfsaccess import StdFsAccess from .utils import aslist, get_feature + class LogAsDebugFilter(logging.Filter): def __init__(self, name, parent): # type: (str, logging.Logger) -> None super(LogAsDebugFilter, self).__init__(name) diff --git a/cwltool/sandboxjs.py b/cwltool/sandboxjs.py index 48bd0f58e..0cab56a0b 100644 --- a/cwltool/sandboxjs.py +++ b/cwltool/sandboxjs.py @@ -6,9 +6,9 @@ import subprocess import threading from io import BytesIO +from typing import Any, Dict, List, Mapping, Text, Tuple, Union from pkg_resources import resource_stream -from typing import Any, Dict, List, Mapping, Text, Union, Tuple class JavascriptException(Exception): diff --git a/cwltool/stdfsaccess.py b/cwltool/stdfsaccess.py index 3be90f70f..843bb784e 100644 --- a/cwltool/stdfsaccess.py +++ b/cwltool/stdfsaccess.py @@ -1,9 +1,10 @@ import glob import os import urllib +from typing import BinaryIO, Text from schema_salad.ref_resolver import file_uri, uri_file_path -from typing import BinaryIO, Text + def abspath(src, basedir): # type: (Text, Text) -> Text if src.startswith(u"file://"): diff --git a/cwltool/update.py b/cwltool/update.py index f549d1a65..bb6a60843 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -3,11 +3,12 @@ import re import traceback import urlparse +from typing import (Any, Callable, Dict, Text, # pylint: disable=unused-import + Tuple, Union) import schema_salad.validate -from ruamel.yaml.comments import CommentedSeq, CommentedMap +from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.ref_resolver import Loader -from typing import Any, Callable, Dict, Text, Tuple, Union # pylint: disable=unused-import from .utils import aslist diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 3ceb6b1fb..4ab5b2bbc 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -5,14 +5,13 @@ import random import tempfile from collections import namedtuple -from ruamel.yaml.comments import CommentedSeq, CommentedMap +from typing import Any, Callable, Generator, Iterable, List, Text, Union, cast import schema_salad.validate as validate +from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.sourceline import SourceLine, cmap -from typing import Any, Callable, cast, Generator, Iterable, List, Text, Union -from . import draft2tool -from . import expression +from . import draft2tool, expression from .errors import WorkflowException from .load_tool import load_tool from .process import Process, shortname, uniquename diff --git a/tests/test_check.py b/tests/test_check.py index 4a03cabc1..83a49ab9d 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -5,9 +5,11 @@ import cwltool.pathmapper import cwltool.process import cwltool.workflow -from .util import get_data from cwltool.main import main +from .util import get_data + + class TestCheck(unittest.TestCase): def test_output_checking(self): self.assertEquals(main([get_data('tests/wf/badout1.cwl')]), 1) diff --git a/tests/test_examples.py b/tests/test_examples.py index 1d288c34d..e85ad7217 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -6,9 +6,10 @@ import cwltool.process import cwltool.workflow import schema_salad.validate +from cwltool.main import main from .util import get_data -from cwltool.main import main + class TestParamMatching(unittest.TestCase): def test_params(self): diff --git a/tests/test_ext.py b/tests/test_ext.py index d2d47fcba..4c17e2d45 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -1,16 +1,18 @@ -import unittest -import tempfile import os import shutil +import tempfile +import unittest import cwltool.expression as expr import cwltool.factory import cwltool.pathmapper import cwltool.process import cwltool.workflow -from .util import get_data from cwltool.main import main +from .util import get_data + + class TestListing(unittest.TestCase): def test_missing_enable_ext(self): # Require that --enable-ext is provided. diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 7845fb9b6..664adb8ba 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -3,7 +3,6 @@ import schema_salad.main import schema_salad.ref_resolver import schema_salad.schema - from cwltool.load_tool import load_tool from cwltool.main import main from cwltool.workflow import defaultMakeTool diff --git a/tests/test_js_sandbox.py b/tests/test_js_sandbox.py index 5abe79ed8..bf32a7acf 100644 --- a/tests/test_js_sandbox.py +++ b/tests/test_js_sandbox.py @@ -1,8 +1,10 @@ import unittest -from mock import Mock, patch # we should modify the subprocess imported from cwltool.sandboxjs -from cwltool.sandboxjs import check_js_threshold_version, subprocess, minimum_node_version_str +from cwltool.sandboxjs import (check_js_threshold_version, + minimum_node_version_str, subprocess) +from mock import Mock, patch + class Javascript_Sanity_Checks(unittest.TestCase): diff --git a/tests/test_pack.py b/tests/test_pack.py index e042154a7..fca1d45c3 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -7,9 +7,11 @@ import cwltool.workflow from cwltool.load_tool import fetch_document, validate_document from cwltool.main import makeRelative -from cwltool.pathmapper import adjustFileObjs, adjustDirObjs +from cwltool.pathmapper import adjustDirObjs, adjustFileObjs + from .util import get_data + class TestPack(unittest.TestCase): def test_pack(self): self.maxDiff = None diff --git a/tests/test_toolargparse.py b/tests/test_toolargparse.py index 4b61aa442..8e0d5bb2c 100644 --- a/tests/test_toolargparse.py +++ b/tests/test_toolargparse.py @@ -2,8 +2,10 @@ from tempfile import NamedTemporaryFile from cwltool.main import main + from .util import get_data + class ToolArgparse(unittest.TestCase): script = ''' #!/usr/bin/env cwl-runner diff --git a/tests/util.py b/tests/util.py index 8a1355ebc..8bad5157b 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,6 +1,9 @@ -from pkg_resources import Requirement, resource_filename, ResolutionError # type: ignore import os +from pkg_resources import (Requirement, ResolutionError, # type: ignore + resource_filename) + + def get_data(filename): filepath = None try: