Merge pull request #124 from ywang271828/ci_inlined_runnable

Ci inlined runnable

.github/workflows/run_workflows_weekly.yml

@@ -108,7 +108,17 @@ jobs:
       # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252
       run: cd wic/ && pytest -k test_run_workflows_on_push --workers 8 --cwl_runner toil-cwl-runner # --cov
 
+    - name: PyTest Run Inlined Workflows
+      if: always()
+      # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252
+      run: cd wic/ && pytest -k test_run_inlined_workflows --workers 8 --cwl_runner toil-cwl-runner # --cov
+
     - name: PyTest Run Workflows (Weekly)
       if: always()
       # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252
       run: cd wic/ && pytest -k test_run_workflows_weekly --workers 8 --cwl_runner toil-cwl-runner # --cov
+
+    - name: PyTest Run Inlined Workflows (Weekly)
+      if: always()
+      # NOTE: Do NOT add coverage to PYPY CI runs https://github.com/tox-dev/tox/issues/2252
+      run: cd wic/ && pytest -k test_run_inlined_workflows_weekly_inlined --workers 8 --cwl_runner toil-cwl-runner # --cov

tests/test_examples.py

@@ -4,6 +4,7 @@
 import signal
 import sys
 from typing import List
+import argparse
 
 import graphviz
 import networkx as nx
@@ -16,6 +17,7 @@
 import wic.compiler
 import wic.run_local
 import wic.utils
+import wic.ast
 from wic import auto_gen_header
 from wic.wic_types import GraphData, GraphReps, NodeData, StepId, Yaml, YamlTree, Json
 
@@ -108,20 +110,39 @@ def test_run_workflows_on_push(yml_path_str: str, yml_path: Path, cwl_runner: st
        directories in yml_dirs.txt, excluding all workflows which have been
        blacklisted in the various config_ci.json files and excluding the weekly
        workflows."""
-    run_workflows(yml_path_str, yml_path, cwl_runner)
+    args = get_args(str(yml_path))
+    run_workflows(yml_path_str, yml_path, cwl_runner, args)
+
+
+@pytest.mark.slow
+@pytest.mark.parametrize("yml_path_str, yml_path", yml_paths_tuples_not_blacklist_on_push)
+def test_run_inlined_workflows(yml_path_str: str, yml_path: Path, cwl_runner: str) -> None:
+    """Inlines and runs all of the workflows auto-discovered from the various
+       directories in yml_dirs.txt, excluding all workflows which have been
+       blacklisted in the various config_ci.json files and excluding the weekly
+       workflows."""
+    args = get_args(str(yml_path), cwl_inline_subworkflows=True)
+    run_workflows(yml_path_str, yml_path, cwl_runner, args)
 
 
 @pytest.mark.slow
 @pytest.mark.parametrize("yml_path_str, yml_path", yml_paths_tuples_weekly)
 def test_run_workflows_weekly(yml_path_str: str, yml_path: Path, cwl_runner: str) -> None:
     """Runs all of the run_weekly workflows whitelisted in the various config_ci.json files."""
-    run_workflows(yml_path_str, yml_path, cwl_runner)
+    args = get_args(str(yml_path))
+    run_workflows(yml_path_str, yml_path, cwl_runner, args)
 
 
-def run_workflows(yml_path_str: str, yml_path: Path, cwl_runner: str) -> None:
-    """Runs all of the given workflows."""
+@pytest.mark.slow
+@pytest.mark.parametrize("yml_path_str, yml_path", yml_paths_tuples_weekly)
+def test_run_inlined_workflows_weekly(yml_path_str: str, yml_path: Path, cwl_runner: str) -> None:
+    """Inlines and runs all of the run_weekly workflows whitelisted in the various config_ci.json files."""
+    args = get_args(str(yml_path), cwl_inline_subworkflows=True)
+    run_workflows(yml_path_str, yml_path, cwl_runner, args)
 
-    args = get_args(str(yml_path))
+
+def run_workflows(yml_path_str: str, yml_path: Path, cwl_runner: str, args: argparse.Namespace) -> None:
+    """Runs all of the given workflows."""
 
     # First compile the workflow.
     # Load the high-level yaml workflow file.
@@ -139,6 +160,18 @@ def run_workflows(yml_path_str: str, yml_path: Path, cwl_runner: str) -> None:
     with open(f'autogenerated/{Path(yml_path).stem}_tree_merged.yml', mode='w', encoding='utf-8') as f:
         f.write(yaml.dump(yaml_tree.yml))
 
+    if args.cwl_inline_subworkflows:
+        while True:
+            # Inlineing changes the namespaces, so we have to get new namespaces after each inlineing operation.
+            namespaces_list = wic.ast.get_inlineable_subworkflows(yaml_tree, tools_cwl, False, [])
+            if namespaces_list == []:
+                break
+
+            yaml_tree, _len_substeps = wic.ast.inline_subworkflow(yaml_tree, namespaces_list[0])
+
+        with open(f'autogenerated/{Path(yml_path).stem}_tree_merged_inlined.yml', mode='w', encoding='utf-8') as f:
+            f.write(yaml.dump(yaml_tree.yml))
+
     graph = get_graph_reps(str(yml_path))
     compiler_info = wic.compiler.compile_workflow(yaml_tree, args, [], [graph], {}, {}, {}, {},
                                                   tools_cwl, True, relative_run_path=True, testing=True)

tests/test_setup.py

@@ -18,13 +18,15 @@
 from wic.wic_types import Json, Yaml
 
 
-def get_args(yaml_path: str = '') -> argparse.Namespace:
+def get_args(yaml_path: str = '', cwl_inline_subworkflows: bool = False) -> argparse.Namespace:
     """This is used to get mock command line arguments.
 
     Returns:
         argparse.Namespace: The mocked command line arguments
     """
     testargs = ['wic', '--yaml', yaml_path, '--cwl_output_intermediate_files', 'True']  # ignore --yaml
+    if cwl_inline_subworkflows:
+        testargs.append("--cwl_inline_subworkflows")
     # For now, we need to enable --cwl_output_intermediate_files. See comment in compiler.py
     with patch.object(sys, 'argv', testargs):
         args: argparse.Namespace = wic.cli.parser.parse_args()