run workflows with toil python API (#144)

mypy.ini

@@ -82,3 +82,6 @@ ignore_missing_imports = True
 
 [mypy-ruamel.*]
 ignore_missing_imports = True
+
+[mypy-toil.*]
+ignore_missing_imports = True

src/wic/run_local.py

@@ -11,8 +11,9 @@
 
 try:
     import cwltool.main
+    import toil.cwl.cwltoil  # transitively imports cwltool
 except ImportError as exc:
-    print('Could not import cwltool.main')
+    print('Could not import cwltool.main and/or toil.cwl.cwltoil')
     # (pwd is imported transitively in cwltool.provenance)
     print(exc)
     if exc.msg == "No module named 'pwd'":
@@ -173,6 +174,34 @@ def run_local(args: argparse.Namespace, rose_tree: RoseTree, cachedir: Optional[
                 f'autogenerated/{yaml_stem}.cwl', f'autogenerated/{yaml_stem}_inputs.yml']
 
         print('Running ' + ' '.join(cmd))
+        if use_subprocess:
+            # To run in parallel (i.e. pytest ... --workers 8 ...), we need to
+            # use separate processes. Otherwise:
+            # "signal only works in main thread or with __pypy__.thread.enable_signals()"
+            proc = sub.run(cmd, check=False)
+            retval = proc.returncode
+            return retval  # Skip copying files to outdir/ for CI
+        else:
+            print('via python API')
+            try:
+                retval = toil.cwl.cwltoil.main(cmd[1:])
+                assert retval == 0
+
+                if args.write_summary:
+                    print(f'Final output json blob is in {args.write_summary}')
+
+            except Exception as e:
+                print('Failed to execute', yaml_path)
+                print(f'See error_{yaml_stem}.txt for detailed technical information.')
+                # Do not display a nasty stack trace to the user; hide it in a file.
+                with open(f'error_{yaml_stem}.txt', mode='w', encoding='utf-8') as f:
+                    if sys.version_info >= (3, 10):
+                        traceback.print_exception(type(e), value=e, tb=None, file=f)
+                    else:
+                        traceback.print_exception(etype=type(e), value=e, tb=None, file=f)
+                if not cachedir:  # if running on CI
+                    print(e)
+
         proc = sub.run(cmd, check=False)
         retval = proc.returncode