Skip to content

Commit

Permalink
Better support for $import or "run: " references to parent directories.
Browse files Browse the repository at this point in the history
  • Loading branch information
@mr-c
mr-c committed Nov 15, 2021
1 parent db787f1 commit 61b40c7
Show file tree
Hide file tree
Showing 2 changed files with 50 additions and 41 deletions.
85 changes: 47 additions & 38 deletions cwlupgrader/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@ def parse_args(args: List[str]) -> argparse.Namespace:
"""Argument parser."""
parser = argparse.ArgumentParser(
description="Tool to upgrade CWL documents from one version to another. "
"Supports upgrading 'draft-3', 'v1.0', and 'v1.1' to 'v1.2'",
"Supports upgrading 'draft-3', 'v1.0', and 'v1.1' to 'v1.2'. For workflows "
'that include "$import" or "run:" reference that refer to parent '
"directories, start cwl-upgrader from the topmost directory of your project.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
Expand All @@ -40,11 +42,11 @@ def parse_args(args: List[str]) -> argparse.Namespace:
parser.add_argument(
"--v1.1-only",
dest="v1_1_only",
help="Don't upgrade past cwlVersion: v1.1",
help="Don't upgrade past cwlVersion: v1.1.",
action="store_true",
)
parser.add_argument(
"--dir", help="Directory in which to save converted files", default=Path.cwd()
"--dir", help="Directory in which to save converted files.", default=Path.cwd()
)
parser.add_argument(
"inputs",
Expand All @@ -64,10 +66,21 @@ def main(args: Optional[List[str]] = None) -> int:
def run(args: argparse.Namespace) -> int:
"""Main function."""
imports: Set[str] = set()
if args.dir and not os.path.exists(args.dir):
os.makedirs(args.dir)
for path in args.inputs:
if args.dir:
out_dir = Path(args.dir)
out_dir.mkdir(parents=True, exist_ok=True)
else:
out_dir = Path.cwd()
for p in args.inputs:
path = Path(os.path.normpath(p))
_logger.info("Processing %s", path)
if not p.startswith("../") and path.resolve() != path:
# common_path = os.path.commonpath([out_dir.resolve(), path.resolve()])
# current_out_dir = out_dir / os.path.relpath(path, start=common_path)
current_out_dir = out_dir / path.parent
current_out_dir.mkdir(parents=True, exist_ok=True)
else:
current_out_dir = out_dir
document = load_cwl_document(path)
if "cwlVersion" not in document:
_logger.warn("No cwlVersion found in %s, skipping it.", path)
Expand All @@ -89,17 +102,17 @@ def run(args: argparse.Namespace) -> int:
target_version = "latest"
upgraded_document = upgrade_document(
document,
args.dir,
current_out_dir,
target_version=target_version,
imports=imports,
)
write_cwl_document(upgraded_document, Path(path).name, args.dir)
write_cwl_document(upgraded_document, current_out_dir / (path.name))
return 0


def upgrade_document(
document: Any,
output_dir: str,
output_dir: Path,
target_version: Optional[str] = "latest",
imports: Optional[Set[str]] = None,
) -> Any:
Expand Down Expand Up @@ -158,7 +171,7 @@ def upgrade_document(
return main_updater(document, output_dir)


def load_cwl_document(path: str) -> Any:
def load_cwl_document(path: Union[str, Path]) -> Any:
"""
Load the given path using the Ruamel YAML round-trip loader.
Expand All @@ -171,15 +184,14 @@ def load_cwl_document(path: str) -> Any:
return document


def write_cwl_document(document: Any, name: str, dirname: str) -> None:
def write_cwl_document(document: Any, path: Path) -> None:
"""
Serialize the document using the Ruamel YAML round trip dumper.
Will also prepend "#!/usr/bin/env cwl-runner\n" and
set the executable bit if it is a CWL document.
"""
ruamel.yaml.scalarstring.walk_tree(document)
path = Path(dirname) / name
with open(path, "w") as handle:
if "cwlVersion" in document:
handle.write("#!/usr/bin/env cwl-runner\n")
Expand All @@ -189,7 +201,7 @@ def write_cwl_document(document: Any, name: str, dirname: str) -> None:


def process_imports(
document: Any, imports: Set[str], updater: Callable[[Any, str], Any], outdir: str
document: Any, imports: Set[str], updater: Callable[[Any, Path], Any], outdir: Path
) -> None:
"""Find any '$import's and process them."""
if isinstance(document, CommentedMap):
Expand All @@ -200,14 +212,9 @@ def process_imports(
import_doc = load_cwl_document(
Path(document.lc.filename).parent / value
)
write_cwl_document(
updater(
import_doc,
outdir,
),
Path(value).name,
outdir,
)
new_path = (outdir / value).resolve()
new_path.parent.mkdir(parents=True, exist_ok=True)
write_cwl_document(updater(import_doc, outdir), new_path)
imports.add(value)
else:
process_imports(value, imports, updater, outdir)
Expand All @@ -216,7 +223,7 @@ def process_imports(
process_imports(entry, imports, updater, outdir)


def v1_0_to_v1_1(document: CommentedMap, outdir: str) -> CommentedMap:
def v1_0_to_v1_1(document: CommentedMap, outdir: Path) -> CommentedMap:
"""CWL v1.0.x to v1.1 transformation loop."""
_v1_0_to_v1_1(document, outdir)
for key, value in document.items():
Expand All @@ -231,20 +238,20 @@ def v1_0_to_v1_1(document: CommentedMap, outdir: str) -> CommentedMap:
return sort_v1_0(document)


def v1_0_to_v1_2(document: CommentedMap, outdir: str) -> CommentedMap:
def v1_0_to_v1_2(document: CommentedMap, outdir: Path) -> CommentedMap:
"""CWL v1.0.x to v1.2 transformation."""
document = v1_0_to_v1_1(document, outdir)
document["cwlVersion"] = "v1.2"
return document


def v1_1_to_v1_2(document: CommentedMap, outdir: str) -> CommentedMap:
def v1_1_to_v1_2(document: CommentedMap, outdir: Path) -> CommentedMap:
"""CWL v1.1 to v1.2 transformation."""
document["cwlVersion"] = "v1.2"
return document


def draft3_to_v1_0(document: CommentedMap, outdir: str) -> CommentedMap:
def draft3_to_v1_0(document: CommentedMap, outdir: Path) -> CommentedMap:
"""Transformation loop."""
_draft3_to_v1_0(document, outdir)
if isinstance(document, MutableMapping):
Expand All @@ -260,17 +267,17 @@ def draft3_to_v1_0(document: CommentedMap, outdir: str) -> CommentedMap:
return sort_v1_0(document)


def draft3_to_v1_1(document: CommentedMap, outdir: str) -> CommentedMap:
def draft3_to_v1_1(document: CommentedMap, outdir: Path) -> CommentedMap:
"""transformation loop."""
return v1_0_to_v1_1(draft3_to_v1_0(document, outdir), outdir)


def draft3_to_v1_2(document: CommentedMap, outdir: str) -> CommentedMap:
def draft3_to_v1_2(document: CommentedMap, outdir: Path) -> CommentedMap:
"""transformation loop."""
return v1_1_to_v1_2(v1_0_to_v1_1(draft3_to_v1_0(document, outdir), outdir), outdir)


def _draft3_to_v1_0(document: CommentedMap, outdir: str) -> CommentedMap:
def _draft3_to_v1_0(document: CommentedMap, outdir: Path) -> CommentedMap:
"""Inner loop for transforming draft-3 to v1.0."""
if "class" in document:
if document["class"] == "Workflow":
Expand All @@ -295,11 +302,11 @@ def _draft3_to_v1_0(document: CommentedMap, outdir: str) -> CommentedMap:
return document


def _draft3_to_v1_1(document: CommentedMap, outdir: str) -> CommentedMap:
def _draft3_to_v1_1(document: CommentedMap, outdir: Path) -> CommentedMap:
return v1_0_to_v1_1(_draft3_to_v1_0(document, outdir), outdir)


def _draft3_to_v1_2(document: CommentedMap, outdir: str) -> CommentedMap:
def _draft3_to_v1_2(document: CommentedMap, outdir: Path) -> CommentedMap:
return _draft3_to_v1_1(document, outdir) # nothing needs doing for 1.2


Expand All @@ -318,7 +325,7 @@ def _draft3_to_v1_2(document: CommentedMap, outdir: str) -> CommentedMap:
}


def _v1_0_to_v1_1(document: CommentedMap, outdir: str) -> CommentedMap:
def _v1_0_to_v1_1(document: CommentedMap, outdir: Path) -> CommentedMap:
"""Inner loop for transforming draft-3 to v1.0."""
if "class" in document:
if document["class"] == "Workflow":
Expand Down Expand Up @@ -350,11 +357,13 @@ def _v1_0_to_v1_1(document: CommentedMap, outdir: str) -> CommentedMap:
isinstance(entry["run"], str)
and "#" not in entry["run"]
):
path = Path(document.lc.filename).parent / entry["run"]
process = v1_0_to_v1_1(
load_cwl_document(str(path)), outdir
)
write_cwl_document(process, path.name, outdir)
path = (
Path(document.lc.filename).parent / entry["run"]
).resolve()
process = v1_0_to_v1_1(load_cwl_document(path), outdir)
new_path = (outdir / entry["run"]).resolve()
new_path.parent.mkdir(parents=True, exist_ok=True)
write_cwl_document(process, new_path)
elif isinstance(entry["run"], str) and "#" in entry["run"]:
pass # reference to $graph entry
else:
Expand Down Expand Up @@ -397,11 +406,11 @@ def _v1_0_to_v1_1(document: CommentedMap, outdir: str) -> CommentedMap:
return document


def _v1_0_to_v1_2(document: CommentedMap, outdir: str) -> CommentedMap:
def _v1_0_to_v1_2(document: CommentedMap, outdir: Path) -> CommentedMap:
return _v1_0_to_v1_1(document, outdir) # nothing needs doing for v1.2


def _v1_1_to_v1_2(document: CommentedMap, outdir: str) -> CommentedMap:
def _v1_1_to_v1_2(document: CommentedMap, outdir: Path) -> CommentedMap:
return document


Expand Down
6 changes: 3 additions & 3 deletions tests/test_complete.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,21 +37,21 @@ def test_draft3_tool_long_form_arrays(tmp_path: Path) -> None:
def test_invalid_target(tmp_path: Path) -> None:
"""Test for invalid target version"""
doc = load_cwl_document(get_data("testdata/v1.0/listing_deep1.cwl"))
result = upgrade_document(doc, str(tmp_path), "invalid-version")
result = upgrade_document(doc, tmp_path, "invalid-version")
assert result is None


def test_v1_0_to_v1_1(tmp_path: Path) -> None:
"""Basic CWL v1.0 to CWL v1.1 test."""
doc = load_cwl_document(get_data("testdata/v1.0/listing_deep1.cwl"))
upgraded = upgrade_document(doc, str(tmp_path), "v1.1")
upgraded = upgrade_document(doc, tmp_path, "v1.1")
assert doc == upgraded


def test_v1_1_to_v1_2(tmp_path: Path) -> None:
"""Basic CWL v1.1 to CWL v1.2 test."""
doc = load_cwl_document(get_data("testdata/v1.1/listing_deep1.cwl"))
upgraded = upgrade_document(doc, str(tmp_path), "v1.2")
upgraded = upgrade_document(doc, tmp_path, "v1.2")
assert doc == upgraded


Expand Down

0 comments on commit 61b40c7

Please sign in to comment.