diff --git a/.gitignore b/.gitignore index 3645a572a8..93e1c190ac 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ __pycache__ /test-report-*.xml /venv/ /src/toil/test/cwl/spec +/cwltool_deps/ /docs/generated_rst/ /docker/Dockerfile /docker/toil-*.tar.gz diff --git a/setup.py b/setup.py index afe82c873d..fd9e0ab25c 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ def runSetup(): 'cwl': [ 'cwltool==1.0.20170822192924', 'schema-salad >= 2.6, < 3', + 'galaxy-lib==17.9.3', 'cwltest>=1.0.20170214185319']}, package_dir={'': 'src'}, packages=find_packages(where='src', diff --git a/src/toil/cwl/cwltoil.py b/src/toil/cwl/cwltoil.py index fcf395fa4f..554747cdde 100755 --- a/src/toil/cwl/cwltoil.py +++ b/src/toil/cwl/cwltoil.py @@ -40,6 +40,7 @@ import cwltool.draft2tool from cwltool.pathmapper import PathMapper, adjustDirObjs, adjustFileObjs, get_listing, MapperEnt, visit_class, normalizeFilesDirs from cwltool.process import shortname, fillInDefaults, compute_checksums, collectFilesAndDirs, stageFiles +from cwltool.software_requirements import DependenciesConfiguration, get_container_from_software_requirements from cwltool.utils import aslist import schema_salad.validate as validate import schema_salad.ref_resolver @@ -831,6 +832,14 @@ def main(args=None, stdout=sys.stdout): metavar=("VAR1 VAR2"), default=("PATH",), dest="preserve_environment") + # help="Dependency resolver configuration file describing how to adapt 'SoftwareRequirement' packages to current system." + parser.add_argument("--beta-dependency-resolvers-configuration", default=None) + # help="Defaut root directory used by dependency resolvers configuration." + parser.add_argument("--beta-dependencies-directory", default=None) + # help="Use biocontainers for tools without an explicitly annotated Docker container." + parser.add_argument("--beta-use-biocontainers", default=None, action="store_true") + # help="Short cut to use Conda to resolve 'SoftwareRequirement' packages." + parser.add_argument("--beta-conda-dependencies", default=None, action="store_true") # mkdtemp actually creates the directory, but # toil requires that the directory not exist, @@ -853,22 +862,32 @@ def main(args=None, stdout=sys.stdout): outdir = os.path.abspath(options.outdir) fileindex = {} existing = {} + make_tool_kwargs = {} + conf_file = getattr(options, "beta_dependency_resolvers_configuration", None) # Text + use_conda_dependencies = getattr(options, "beta_conda_dependencies", None) # Text + job_script_provider = None + if conf_file or use_conda_dependencies: + dependencies_configuration = DependenciesConfiguration(options) # type: DependenciesConfiguration + job_script_provider = dependencies_configuration + + options.default_container = None + make_tool_kwargs["find_default_container"] = functools.partial(find_default_container, options) with Toil(options) as toil: if options.restart: outobj = toil.restart() else: useStrict = not options.not_strict + make_tool_kwargs["hints"] = [{ + "class": "ResourceRequirement", + "coresMin": toil.config.defaultCores, + "ramMin": toil.config.defaultMemory / (2**20), + "outdirMin": toil.config.defaultDisk / (2**20), + "tmpdirMin": 0 + }] try: t = cwltool.load_tool.load_tool(options.cwltool, toilMakeTool, - kwargs={ - "hints": [{ - "class": "ResourceRequirement", - "coresMin": toil.config.defaultCores, - "ramMin": toil.config.defaultMemory / (2**20), - "outdirMin": toil.config.defaultDisk / (2**20), - "tmpdirMin": 0 - }]}, + kwargs=make_tool_kwargs, resolver=cwltool.resolver.tool_resolver, strict=useStrict) unsupportedRequirementsCheck(t.requirements) @@ -931,7 +950,8 @@ def setSecondary(fileobj): try: (wf1, wf2) = makeJob(t, {}, use_container=use_container, preserve_environment=options.preserve_environment, - tmpdir=os.path.realpath(outdir), workdir=options.workDir) + tmpdir=os.path.realpath(outdir), workdir=options.workDir, + job_script_provider=job_script_provider) except cwltool.process.UnsupportedRequirement as e: logging.error(e) return 33 @@ -948,3 +968,13 @@ def setSecondary(fileobj): stdout.write(json.dumps(outobj, indent=4)) return 0 + + +def find_default_container(args, builder): + default_container = None + if args.default_container: + default_container = args.default_container + elif args.beta_use_biocontainers: + default_container = get_container_from_software_requirements(args, builder) + + return default_container diff --git a/src/toil/test/cwl/2.fasta b/src/toil/test/cwl/2.fasta new file mode 100644 index 0000000000..3bfe7d3d3e --- /dev/null +++ b/src/toil/test/cwl/2.fasta @@ -0,0 +1,11 @@ +>Sequence 561 BP; 135 A; 106 C; 98 G; 222 T; 0 other; +gttcgatgcc taaaatacct tcttttgtcc ctacacagac cacagttttc ctaatggctt +tacaccgact agaaattctt gtgcaagcac taattgaaag cggttggcct agagtgttac +cggtttgtat agctgagcgc gtctcttgcc ctgatcaaag gttcattttc tctactttgg +aagacgttgt ggaagaatac aacaagtacg agtctctccc ccctggtttg ctgattactg +gatacagttg taataccctt cgcaacaccg cgtaactatc tatatgaatt attttccctt +tattatatgt agtaggttcg tctttaatct tcctttagca agtcttttac tgttttcgac +ctcaatgttc atgttcttag gttgttttgg ataatatgcg gtcagtttaa tcttcgttgt +ttcttcttaa aatatttatt catggtttaa tttttggttt gtacttgttc aggggccagt +tcattattta ctctgtttgt atacagcagt tcttttattt ttagtatgat tttaatttaa +aacaattcta atggtcaaaa a \ No newline at end of file diff --git a/src/toil/test/cwl/2.fastq b/src/toil/test/cwl/2.fastq new file mode 100644 index 0000000000..436c05ff2c --- /dev/null +++ b/src/toil/test/cwl/2.fastq @@ -0,0 +1,12 @@ +@EAS54_6_R1_2_1_413_324 +CCCTTCTTGTCTTCAGCGTTTCTCC ++ +;;3;;;;;;;;;;;;7;;;;;;;88 +@EAS54_6_R1_2_1_540_792 +TTGGCAGGCCAAGGCCGATGGATCA ++ +;;;;;;;;;;;7;;;;;-;;;3;83 +@EAS54_6_R1_2_1_443_348 +GTTGCTTCTGGCGTGGGTGGGGGGG ++EAS54_6_R1_2_1_443_348 +;;;;;;;;;;;9;7;;.7;393333 \ No newline at end of file diff --git a/src/toil/test/cwl/cwlTest.py b/src/toil/test/cwl/cwlTest.py index d813a95743..75d7b44d37 100644 --- a/src/toil/test/cwl/cwlTest.py +++ b/src/toil/test/cwl/cwlTest.py @@ -27,22 +27,20 @@ from toil.test import ToilTest, needs_cwl, slow - @needs_cwl class CWLTest(ToilTest): - def _tester(self, cwlfile, jobfile, outDir, expect): + def _tester(self, cwlfile, jobfile, outDir, expect, main_args=[], out_name="output"): from toil.cwl import cwltoil rootDir = self._projectRootPath() st = StringIO() - cwltoil.main(['--outdir', outDir, - os.path.join(rootDir, cwlfile), - os.path.join(rootDir, jobfile)], - stdout=st) + main_args = main_args[:] + main_args.extend(['--outdir', outDir, os.path.join(rootDir, cwlfile), os.path.join(rootDir, jobfile)]) + cwltoil.main(main_args, stdout=st) out = json.loads(st.getvalue()) - out["output"].pop("http://commonwl.org/cwltool#generation", None) - out["output"].pop("nameext", None) - out["output"].pop("nameroot", None) + out[out_name].pop("http://commonwl.org/cwltool#generation", None) + out[out_name].pop("nameext", None) + out[out_name].pop("nameroot", None) self.assertEquals(out, expect) def _debug_worker_tester(self, cwlfile, jobfile, outDir, expect): @@ -148,3 +146,32 @@ def test_run_conformance(self): if not only_unsupported: print(e.output) raise e + + @slow + def test_bioconda(self): + outDir = self._createTempDir() + self._tester('src/toil/test/cwl/seqtk_seq.cwl', + 'src/toil/test/cwl/seqtk_seq_job.json', + outDir, + self._expected_seqtk_output(outDir), + main_args=["--beta-conda-dependencies"], + out_name="output1") + + def test_biocontainers(self): + outDir = self._createTempDir() + self._tester('src/toil/test/cwl/seqtk_seq.cwl', + 'src/toil/test/cwl/seqtk_seq_job.json', + outDir, + self._expected_seqtk_output(outDir), + main_args=["--beta-use-biocontainers"], + out_name="output1") + + def _expected_seqtk_output(self, outDir): + return { + u"output1": { + u"location": "file://" + str(os.path.join(outDir, 'out')), + u"checksum": u"sha1$322e001e5a99f19abdce9f02ad0f02a17b5066c2", + u"basename": str("out"), + u"class": u"File", + u"size": 150} + } diff --git a/src/toil/test/cwl/seqtk_seq.cwl b/src/toil/test/cwl/seqtk_seq.cwl new file mode 100644 index 0000000000..b97d6c25e6 --- /dev/null +++ b/src/toil/test/cwl/seqtk_seq.cwl @@ -0,0 +1,24 @@ +cwlVersion: v1.0 +class: CommandLineTool +id: "seqtk_seq" +doc: "Convert to FASTA (seqtk)" +inputs: + - id: input1 + type: File + inputBinding: + position: 1 + prefix: "-a" +outputs: + - id: output1 + type: File + outputBinding: + glob: out +baseCommand: ["seqtk", "seq"] +arguments: [] +stdout: out +hints: + SoftwareRequirement: + packages: + - package: seqtk + version: + - r93 diff --git a/src/toil/test/cwl/seqtk_seq_job.json b/src/toil/test/cwl/seqtk_seq_job.json new file mode 100644 index 0000000000..79ea46c378 --- /dev/null +++ b/src/toil/test/cwl/seqtk_seq_job.json @@ -0,0 +1,6 @@ +{ + "input1": { + "class": "File", + "location": "2.fastq" + } +}