diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..ecc32fec7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*pyc +.tox +cwltool.egg-info/ \ No newline at end of file diff --git a/cwltool/builder.py b/cwltool/builder.py index 5352f915a..fa28dd018 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -3,15 +3,18 @@ import expression import avro import schema_salad.validate as validate +from .errors import WorkflowException CONTENT_LIMIT = 64 * 1024 + def substitute(value, replace): if replace[0] == "^": return substitute(value[0:value.rindex('.')], replace[1:]) else: return value + replace + class Builder(object): def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]): diff --git a/cwltool/cwlrdf.py b/cwltool/cwlrdf.py index 02b42bf91..59a9b8c71 100644 --- a/cwltool/cwlrdf.py +++ b/cwltool/cwlrdf.py @@ -1,24 +1,25 @@ import json -from rdflib import Graph, plugin -from rdflib.serializer import Serializer +from rdflib import Graph + def printrdf(workflow, wf, ctx, sr): wf["@context"] = ctx g = Graph().parse(data=json.dumps(wf), format='json-ld', location=workflow) print(g.serialize(format=sr)) + def printdot(workflow, wf, ctx, sr): wf["@context"] = ctx g = Graph().parse(data=json.dumps(wf), format='json-ld', location=workflow) print "digraph {" - #g.namespace_manager.qname(predicate) + # g.namespace_manager.qname(predicate) def lastpart(uri): uri = str(uri) if "/" in uri: - return uri[uri.rindex("/")+1:] + return uri[uri.rindex("/") + 1:] else: return uri @@ -76,5 +77,4 @@ def lastpart(uri): for (inp,) in qres: print '"%s" [shape=octagon]' % (lastpart(inp)) - print "}" diff --git a/cwltool/cwltest.py b/cwltool/cwltest.py index 919f5b8d8..219e5da64 100755 --- a/cwltool/cwltest.py +++ b/cwltool/cwltest.py @@ -17,6 +17,7 @@ UNSUPPORTED_FEATURE = 33 + def compare(a, b): try: if isinstance(a, dict): @@ -47,6 +48,7 @@ def compare(a, b): except: return False + def run_test(args, i, t): out = {} outdir = None @@ -92,7 +94,7 @@ def run_test(args, i, t): _logger.error(outstr) _logger.error("Parse error %s", str(e)) - pwd = os.path.abspath(os.path.dirname(t["job"])) + os.path.abspath(os.path.dirname(t["job"])) # t["args"] = map(lambda x: x.replace("$PWD", pwd), t["args"]) # if "stdin" in t: # t["stdin"] = t["stdin"].replace("$PWD", pwd) @@ -110,9 +112,9 @@ def run_test(args, i, t): _logger.warn(t.get("doc")) failed = True _logger.warn("%s expected %s\n%s got %s", key, - t.get(key), - " " * len(key), - out.get(key)) + t.get(key), + " " * len(key), + out.get(key)) if outdir: shutil.rmtree(outdir) @@ -144,14 +146,14 @@ def main(): if args.n is not None: sys.stderr.write("\rTest [%i/%i] " % (args.n, len(tests))) - rt = run_test(args, args.n-1, tests[args.n-1]) + rt = run_test(args, args.n - 1, tests[args.n - 1]) if rt == 1: failures += 1 elif rt == UNSUPPORTED_FEATURE: unsupported += 1 else: for i, t in enumerate(tests): - sys.stderr.write("\rTest [%i/%i] " % (i+1, len(tests))) + sys.stderr.write("\rTest [%i/%i] " % (i + 1, len(tests))) sys.stderr.flush() rt = run_test(args, i, t) if rt == 1: @@ -160,8 +162,8 @@ def main(): unsupported += 1 if failures == 0 and unsupported == 0: - _logger.info("All tests passed") - return 0 + _logger.info("All tests passed") + return 0 else: _logger.warn("%i failures, %i unsupported features", failures, unsupported) return 1 diff --git a/cwltool/docker.py b/cwltool/docker.py index e9a56f418..4790f563c 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -9,6 +9,7 @@ _logger = logging.getLogger("cwltool") + def get_image(dockerRequirement, pull_image, dry_run=False): found = False diff --git a/cwltool/draft2tool.py b/cwltool/draft2tool.py index 75ba80d72..37452077d 100644 --- a/cwltool/draft2tool.py +++ b/cwltool/draft2tool.py @@ -1,4 +1,3 @@ -import avro.schema import json import copy from flatten import flatten @@ -7,23 +6,19 @@ from pathmapper import PathMapper, DockerPathMapper from job import CommandLineJob import yaml -import glob import logging import hashlib -import random from process import Process, shortname from errors import WorkflowException import schema_salad.validate as validate from aslist import aslist -import expression -import re -import urlparse import tempfile from builder import CONTENT_LIMIT, substitute import shellescape _logger = logging.getLogger("cwltool") + class ExpressionTool(Process): def __init__(self, toolpath_object, **kwargs): super(ExpressionTool, self).__init__(toolpath_object, **kwargs) @@ -106,12 +101,11 @@ def job(self, joborder, input_basedir, output_callback, **kwargs): j.hints = self.hints _logger.debug("[job %s] initializing from %s%s", - id(j), - self.tool.get("id", ""), - " as part of %s" % kwargs["part_of"] if "part_of" in kwargs else "") + id(j), + self.tool.get("id", ""), + " as part of %s" % kwargs["part_of"] if "part_of" in kwargs else "") _logger.debug("[job %s] %s", id(j), json.dumps(joborder, indent=4)) - builder.pathmapper = None if self.tool.get("stdin"): @@ -246,7 +240,6 @@ def collect_output(self, schema, builder, outdir): if not builder.fs_access.exists(sf["path"]): raise WorkflowException("Missing secondary file of '%s' of primary file '%s'" % (sf["path"], r["path"])) - if not r and schema["type"] == "record": r = {} for f in schema["fields"]: diff --git a/cwltool/expression.py b/cwltool/expression.py index a5f8b7e5e..b0bf76fa1 100644 --- a/cwltool/expression.py +++ b/cwltool/expression.py @@ -3,22 +3,20 @@ import json from aslist import aslist import logging -import os from errors import WorkflowException import process -import yaml -import schema_salad.validate as validate import schema_salad.ref_resolver _logger = logging.getLogger("cwltool") + def exeval(ex, jobinput, requirements, outdir, tmpdir, context, pull_image): if ex["engine"] == "https://w3id.org/cwl/cwl#JsonPointer": try: obj = {"job": jobinput, "context": context, "outdir": outdir, "tmpdir": tmpdir} return schema_salad.ref_resolver.resolve_json_pointer(obj, ex["script"]) except ValueError as v: - raise WorkflowException("%s in %s" % (v, obj)) + raise WorkflowException("%s in %s" % (v, obj)) for r in reversed(requirements): if r["class"] == "ExpressionEngineRequirement" and r["id"] == ex["engine"]: @@ -56,13 +54,13 @@ class DR(object): _logger.debug("Invoking expression engine %s with %s", runtime + aslist(r["engineCommand"]), - json.dumps(inp, indent=4)) + json.dumps(inp, indent=4)) sp = subprocess.Popen(runtime + aslist(r["engineCommand"]), - shell=False, - close_fds=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) + shell=False, + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) (stdoutdata, stderrdata) = sp.communicate(json.dumps(inp) + "\n\n") if sp.returncode != 0: @@ -72,6 +70,7 @@ class DR(object): raise WorkflowException("Unknown expression engine '%s'" % ex["engine"]) + def do_eval(ex, jobinput, requirements, outdir, tmpdir, context=None, pull_image=True): if isinstance(ex, dict) and "engine" in ex and "script" in ex: return exeval(ex, jobinput, requirements, outdir, tmpdir, context, pull_image) diff --git a/cwltool/job.py b/cwltool/job.py index d6d7109c7..0f02ff639 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -1,12 +1,8 @@ import subprocess import os -import tempfile -import glob import json -import yaml import logging import sys -import requests import docker from process import get_feature, empty_subtree from errors import WorkflowException @@ -20,6 +16,7 @@ needs_shell_quoting = re.compile(r"""(^$|[\s|&;()<>\'"$@])""").search + def deref_links(outputs): if isinstance(outputs, dict): if outputs.get("class") == "File": @@ -33,12 +30,13 @@ def deref_links(outputs): for v in outputs: deref_links(v) + class CommandLineJob(object): def run(self, dry_run=False, pull_image=True, rm_container=True, rm_tmpdir=True, move_outputs=True, **kwargs): if not os.path.exists(self.outdir): os.makedirs(self.outdir) - #with open(os.path.join(outdir, "cwl.input.json"), "w") as fp: + # with open(os.path.join(outdir, "cwl.input.json"), "w") as fp: # json.dump(self.joborder, fp) runtime = [] @@ -74,7 +72,7 @@ def run(self, dry_run=False, pull_image=True, rm_container=True, rm_tmpdir=True, runtime.append("--env=TMPDIR=/tmp/job_tmp") - for t,v in self.environment.items(): + for t, v in self.environment.items(): runtime.append("--env=%s=%s" % (t, v)) runtime.append(img_id) diff --git a/cwltool/main.py b/cwltool/main.py index 4461399cc..16f6cf659 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import draft2tool import argparse from schema_salad.ref_resolver import Loader import json @@ -15,7 +14,6 @@ import yaml import urlparse import process -import job from cwlrdf import printrdf, printdot import pkg_resources # part of setuptools import update @@ -27,6 +25,7 @@ _logger.addHandler(defaultStreamHandler) _logger.setLevel(logging.INFO) + def arg_parser(): parser = argparse.ArgumentParser() parser.add_argument("--conformance-test", action="store_true") @@ -40,17 +39,17 @@ def arg_parser(): parser.add_argument("--preserve-environment", type=str, nargs='+', help="Preserve specified environment variables when running CommandLineTools", - metavar=("VAR1","VAR2"), + metavar=("VAR1", "VAR2"), dest="preserve_environment") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--rm-container", action="store_true", default=True, - help="Delete Docker container used by jobs after they exit (default)", - dest="rm_container") + help="Delete Docker container used by jobs after they exit (default)", + dest="rm_container") exgroup.add_argument("--leave-container", action="store_false", - default=True, help="Do not delete Docker container used by jobs after they exit", - dest="rm_container") + default=True, help="Do not delete Docker container used by jobs after they exit", + dest="rm_container") parser.add_argument("--tmpdir-prefix", type=str, help="Path prefix for temporary directories", @@ -62,28 +61,28 @@ def arg_parser(): exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--rm-tmpdir", action="store_true", default=True, - help="Delete intermediate temporary directories (default)", - dest="rm_tmpdir") + help="Delete intermediate temporary directories (default)", + dest="rm_tmpdir") exgroup.add_argument("--leave-tmpdir", action="store_false", - default=True, help="Do not delete intermediate temporary directories", - dest="rm_tmpdir") + default=True, help="Do not delete intermediate temporary directories", + dest="rm_tmpdir") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--move-outputs", action="store_true", default=True, - help="Move output files to the workflow output directory and delete intermediate output directories (default).", - dest="move_outputs") + help="Move output files to the workflow output directory and delete intermediate output directories (default).", + dest="move_outputs") exgroup.add_argument("--leave-outputs", action="store_false", default=True, - help="Leave output files in intermediate output directories.", - dest="move_outputs") + help="Leave output files in intermediate output directories.", + dest="move_outputs") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--enable-pull", default=True, action="store_true", - help="Try to pull Docker images", dest="enable_pull") + help="Try to pull Docker images", dest="enable_pull") exgroup.add_argument("--disable-pull", default=True, action="store_false", - help="Do not try to pull Docker images", dest="enable_pull") + help="Do not try to pull Docker images", dest="enable_pull") parser.add_argument("--dry-run", action="store_true", help="Load and validate but do not execute") @@ -94,7 +93,7 @@ def arg_parser(): exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--print-rdf", action="store_true", - help="Print corresponding RDF graph for workflow and exit") + help="Print corresponding RDF graph for workflow and exit") exgroup.add_argument("--print-dot", action="store_true", help="Print workflow visualization in graphviz format and exit") exgroup.add_argument("--print-pre", action="store_true", help="Print CWL document after preprocessing.") exgroup.add_argument("--version", action="store_true", help="Print version and exit") @@ -114,6 +113,7 @@ def arg_parser(): return parser + def single_job_executor(t, job_order, input_basedir, args, **kwargs): final_output = [] final_status = [] @@ -166,6 +166,7 @@ def output_callback(out, processStatus): return final_output[0] + def create_loader(ctx): loader = Loader() url_fields = [] @@ -176,19 +177,24 @@ def create_loader(ctx): loader.idx["cwl:JsonPointer"] = {} return loader + class FileAction(argparse.Action): + def __init__(self, option_strings, dest, nargs=None, **kwargs): if nargs is not None: raise ValueError("nargs not allowed") super(FileAction, self).__init__(option_strings, dest, **kwargs) + def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, {"class": "File", "path": values}) + class FileAppendAction(argparse.Action): def __init__(self, option_strings, dest, nargs=None, **kwargs): if nargs is not None: raise ValueError("nargs not allowed") super(FileAppendAction, self).__init__(option_strings, dest, **kwargs) + def __call__(self, parser, namespace, values, option_string=None): g = getattr(namespace, self.dest) if not g: @@ -196,6 +202,7 @@ def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, g) g.append({"class": "File", "path": values}) + def generate_parser(toolparser, tool, namemap): toolparser.add_argument("job_order", nargs="?", help="Job input json file") namemap["job_order"] = "job_order" @@ -253,6 +260,7 @@ def generate_parser(toolparser, tool, namemap): return toolparser + def load_tool(argsworkflow, updateonly, strict, makeTool, debug, print_pre=False): (document_loader, avsc_names, schema_metadata) = process.get_schema() @@ -296,6 +304,7 @@ def load_tool(argsworkflow, updateonly, strict, makeTool, debug, print_pre=False return t + def main(args=None, executor=single_job_executor, makeTool=workflow.defaultMakeTool, @@ -410,7 +419,7 @@ def main(args=None, else: job_order_object = {} - job_order_object.update({namemap[k]: v for k,v in cmd_line.items()}) + job_order_object.update({namemap[k]: v for k, v in cmd_line.items()}) _logger.debug("Parsed job order from command line: %s", job_order_object) else: job_order_object = None diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 7fbb40c68..621bc7ae9 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -5,6 +5,7 @@ _logger = logging.getLogger("cwltool") + def abspath(src, basedir): if src.startswith("file://"): ab = src[7:] @@ -12,6 +13,7 @@ def abspath(src, basedir): ab = src if os.path.isabs(src) else os.path.join(basedir, src) return ab + class PathMapper(object): """Mapping of files from relative path provided in the file to a tuple of (absolute local path, absolute container path)""" @@ -29,10 +31,11 @@ def files(self): return self._pathmap.keys() def reversemap(self, target): - for k,v in self._pathmap.items(): + for k, v in self._pathmap.items(): if v[1] == target: return (k, v[0]) + class DockerPathMapper(PathMapper): def __init__(self, referenced_files, basedir): self._pathmap = {} @@ -44,8 +47,8 @@ def __init__(self, referenced_files, basedir): subdir = False for d in self.dirs: if dir.startswith(d): - subdir = True - break + subdir = True + break if not subdir: for d in list(self.dirs): diff --git a/cwltool/process.py b/cwltool/process.py index 993d703f8..8ebc09896 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -4,13 +4,10 @@ import schema_salad.validate as validate import copy import yaml -import copy import logging -import pprint from aslist import aslist import schema_salad.schema import urlparse -import pprint from pkg_resources import resource_stream import stat from builder import Builder @@ -31,12 +28,14 @@ "MultipleInputFeatureRequirement", "ShellCommandRequirement"] + def get_schema(): f = resource_stream(__name__, 'schemas/draft-3/cwl-avro.yml') j = yaml.load(f) j["name"] = "https://w3id.org/cwl/cwl" return schema_salad.schema.load_schema(j) + def get_feature(self, feature): for t in reversed(self.requirements): if t["class"] == feature: @@ -46,10 +45,12 @@ def get_feature(self, feature): return (t, False) return (None, None) + def shortname(inputid): (_, d) = urlparse.urldefrag(inputid) return d.split("/")[-1].split(".")[-1] + class StdFsAccess(object): def __init__(self, basedir): self.basedir = basedir @@ -66,6 +67,7 @@ def open(self, fn, mode): def exists(self, fn): return os.path.exists(self._abs(fn)) + def checkRequirements(rec, supportedProcessRequirements): if isinstance(rec, dict): if "requirements" in rec: @@ -81,6 +83,7 @@ def checkRequirements(rec, supportedProcessRequirements): for d in rec: checkRequirements(d, supportedProcessRequirements) + def adjustFiles(rec, op): """Apply a mapping function to each File path in the object `rec`.""" @@ -93,6 +96,7 @@ def adjustFiles(rec, op): for d in rec: adjustFiles(d, op) + class Process(object): def __init__(self, toolpath_object, **kwargs): (_, self.names, _) = get_schema() @@ -149,7 +153,6 @@ def __init__(self, toolpath_object, **kwargs): except avro.schema.SchemaParseException as e: raise validate.ValidationException("Got error `%s` while prcoessing outputs of %s:\n%s" % (str(e), self.tool["id"], json.dumps(self.outputs_record_schema, indent=4))) - def _init_job(self, joborder, input_basedir, **kwargs): builder = Builder() builder.job = copy.deepcopy(joborder) @@ -189,7 +192,6 @@ def _init_job(self, joborder, input_basedir, **kwargs): return builder - def validate_hints(self, hints, strict): for r in hints: try: @@ -203,6 +205,7 @@ def validate_hints(self, hints, strict): def get_requirement(self, feature): return get_feature(self, feature) + def empty_subtree(dirpath): # Test if a directory tree contains any files (does not count empty # subdirectories) diff --git a/cwltool/schemas/draft-2/draft-2/search.py b/cwltool/schemas/draft-2/draft-2/search.py index 002a627e3..20e44fc76 100755 --- a/cwltool/schemas/draft-2/draft-2/search.py +++ b/cwltool/schemas/draft-2/draft-2/search.py @@ -4,7 +4,6 @@ # appears. import sys -import os mainfile = sys.argv[1] indexfile = sys.argv[1] + ".idx" diff --git a/cwltool/schemas/site/generate.py b/cwltool/schemas/site/generate.py index ae4adc84d..a1d90bc35 100644 --- a/cwltool/schemas/site/generate.py +++ b/cwltool/schemas/site/generate.py @@ -1,11 +1,10 @@ import cwltool.avro_ld.schema import cwltool.avro_ld.makedoc -import mistune + import sys with open(sys.argv[1]) as f: cwltool.avro_ld.makedoc.avrold_doc([{"name": " ", - "type": "doc", - "doc": f.read().decode("utf-8") - }], - sys.stdout) + "type": "doc", + "doc": f.read().decode("utf-8")}], + sys.stdout) diff --git a/cwltool/update.py b/cwltool/update.py index 1f1698fb1..c68ed65d5 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -1,7 +1,7 @@ -import sys import urlparse import json + def findId(doc, frg): if isinstance(doc, dict): if "id" in doc and doc["id"] == frg: @@ -18,6 +18,7 @@ def findId(doc, frg): return f return None + def fixType(doc): if isinstance(doc, list): return [fixType(f) for f in doc] @@ -27,6 +28,7 @@ def fixType(doc): return "#" + doc return doc + def _draft2toDraft3(doc, loader, baseuri): try: if isinstance(doc, dict): @@ -61,7 +63,6 @@ def _draft2toDraft3(doc, loader, baseuri): doc["requirements"] = [] doc["requirements"].append({"class": "MultipleInputFeatureRequirement"}) - for a in doc: doc[a] = _draft2toDraft3(doc[a], loader, baseuri) @@ -77,9 +78,11 @@ def _draft2toDraft3(doc, loader, baseuri): err = doc["name"] raise Exception("Error updating '%s'\n %s" % (err, e)) + def draft2toDraft3(doc, loader, baseuri): return (_draft2toDraft3(doc, loader, baseuri), "https://w3id.org/cwl/cwl#draft-3.dev1") + def update(doc, loader, baseuri): updates = { "https://w3id.org/cwl/cwl#draft-2": draft2toDraft3, diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 9fc08ef91..fb74418b8 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -1,26 +1,22 @@ -import job import draft2tool from aslist import aslist -from process import Process, get_feature, empty_subtree, shortname +from process import Process, empty_subtree, shortname from errors import WorkflowException import copy import logging import random import os from collections import namedtuple -import pprint import functools import schema_salad.validate as validate -import urlparse -import pprint import tempfile import shutil -import json _logger = logging.getLogger("cwltool") WorkflowStateItem = namedtuple('WorkflowStateItem', ['parameter', 'value']) + def defaultMakeTool(toolpath_object, **kwargs): if not isinstance(toolpath_object, dict): raise WorkflowException("Not a dict: `%s`" % toolpath_object) @@ -34,6 +30,7 @@ def defaultMakeTool(toolpath_object, **kwargs): raise WorkflowException("Missing or invalid 'class' field in %s, expecting one of: CommandLineTool, ExpressionTool, Workflow" % toolpath_object["id"]) + def findfiles(wo, fn=None): if fn is None: fn = [] @@ -100,7 +97,7 @@ def object_from_state(state, parms, frag_only, supportsMultipleInput): for src in connections: if src in state and state[src] is not None: if not match_types(inp["type"], state[src], iid, inputobj, - inp.get("linkMerge", ("merge_nested" if len(connections) > 1 else None))): + inp.get("linkMerge", ("merge_nested" if len(connections) > 1 else None))): raise WorkflowException("Type mismatch between source '%s' (%s) and sink '%s' (%s)" % (src, state[src].parameter["type"], inp["id"], inp["type"])) elif src not in state: raise WorkflowException("Connect source '%s' on parameter '%s' does not exist" % (src, inp["id"])) @@ -126,6 +123,7 @@ def job(self, joborder, basedir, output_callback, **kwargs): for j in self.step.job(joborder, basedir, output_callback, **kwargs): yield j + class WorkflowJob(object): def __init__(self, workflow, **kwargs): self.workflow = workflow @@ -203,7 +201,7 @@ def try_make_job(self, step, basedir, **kwargs): yield j except WorkflowException: raise - except Exception as e: + except Exception: _logger.exception("Unhandled exception") self.processStatus = "permanentFail" step.completed = True @@ -264,7 +262,7 @@ def job(self, joborder, basedir, output_callback, move_outputs=True, **kwargs): for a in output_dirs: if f["path"].startswith(a): src = f["path"] - dst = os.path.join(self.outdir, src[len(a)+1:]) + dst = os.path.join(self.outdir, src[len(a) + 1:]) if dst in targets: conflicts.add(dst) else: @@ -274,7 +272,7 @@ def job(self, joborder, basedir, output_callback, move_outputs=True, **kwargs): for a in output_dirs: if f["path"].startswith(a): src = f["path"] - dst = os.path.join(self.outdir, src[len(a)+1:]) + dst = os.path.join(self.outdir, src[len(a) + 1:]) if dst in conflicts: sp = os.path.splitext(dst) dst = "%s-%s%s" % (sp[0], str(random.randint(1, 1000000000)), sp[1]) @@ -302,8 +300,8 @@ def __init__(self, toolpath_object, **kwargs): kwargs["requirements"] = self.requirements kwargs["hints"] = self.hints - makeTool = kwargs.get("makeTool") - self.steps = [WorkflowStep(step, n, **kwargs) for n,step in enumerate(self.tool.get("steps", []))] + kwargs.get("makeTool") + self.steps = [WorkflowStep(step, n, **kwargs) for n, step in enumerate(self.tool.get("steps", []))] random.shuffle(self.steps) # TODO: statically validate data links instead of doing it at runtime. @@ -386,7 +384,7 @@ def __init__(self, toolpath_object, pos, **kwargs): self.tool["outputs"] = outputparms def receive_output(self, output_callback, jobout, processStatus): - #_logger.debug("WorkflowStep output from run is %s", jobout) + # _logger.debug("WorkflowStep output from run is %s", jobout) output = {} for i in self.tool["outputs"]: field = shortname(i["id"]) @@ -422,7 +420,7 @@ def __init__(self, output_callback, dest): self.output_callback = output_callback def receive_scatter_output(self, index, jobout, processStatus): - for k,v in jobout.items(): + for k, v in jobout.items(): self.dest[k][index] = v if processStatus != "success": @@ -500,6 +498,7 @@ def crossproduct_size(joborder, scatter_keys): sum += crossproduct_size(joborder, scatter_keys[1:]) return sum + def flat_crossproduct_scatter(process, joborder, basedir, scatter_keys, output_callback, startindex, **kwargs): scatter_key = scatter_keys[0] l = len(joborder[scatter_key]) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..3363abe67 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 200 +max-complexity = 38 diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..1143a5418 --- /dev/null +++ b/tox.ini @@ -0,0 +1,7 @@ +[tox] +envlist = py27-lint + +[testenv:py27-lint] +commands = flake8 cwltool +skip_install = True +deps = flake8