chore: roll back temporary CI workaround and streamline global config…

… generation (#314)

* Roll back temporary CI workaround

* Update config in CI run

* Few more CI updates

* fix json key update

* updated config generator

* update basic conf generation

* minor refactor

* remove extra config

.github/update_sophios_config.py

@@ -0,0 +1,52 @@
+import json
+from pathlib import Path
+
+config_file = Path(__file__).parent.parent / 'src' / 'sophios' / 'config_basic.json'
+workdir_path = Path(__file__).parent.parent.parent
+config = {}
+# config_file can contain absolute or relative paths
+with open(config_file, 'r', encoding='utf-8') as f:
+    config = json.load(f)
+
+conf_tags = ['search_paths_cwl', 'search_paths_wic']
+
+cwl_locations = [            
+                "image-workflows/cwl_adapters",
+                "biobb_adapters/biobb_adapters",
+                "mm-workflows/cwl_adapters",
+                "sophios/cwl_adapters"
+                ]
+
+wic_locations = [
+                "sophios/docs/tutorials",
+                "image-workflows/workflows",
+                "mm-workflows/examples"
+                ]
+
+gpu_cwl_locations = [
+                    "mm-workflows/gpu"
+                    ]
+
+
+updated_cwl_locations = []
+for loc in cwl_locations:
+    updated_cwl_locations.append(str(workdir_path / loc))
+config['search_paths_cwl']['global'] = updated_cwl_locations
+
+updated_gpu_cwl_locations = []
+for loc in gpu_cwl_locations:
+    updated_gpu_cwl_locations.append(str(workdir_path / loc))
+config['search_paths_cwl']['gpu'] = updated_gpu_cwl_locations
+
+
+updated_wic_loactions = []
+for loc in wic_locations:
+    updated_wic_loactions.append(str(workdir_path / loc))
+config['search_paths_wic']['global'] = updated_wic_loactions
+
+config_path = Path.home() / 'wic'
+config_path.mkdir(parents=True, exist_ok=True)
+# global config file in ~/wic
+global_config_file = config_path / 'global_config.json'
+with global_config_file.open('w', encoding='utf-8') as f:
+    json.dump(config, f, indent=4)

.github/workflows/fuzzy_compile_weekly.yml

@@ -78,10 +78,14 @@ jobs:
           channels: conda-forge
           python-version: "3.9.*" # pypy is not yet compatible with 3.10 and 3.11
 
-      - name: Install Workflow Inference Compiler
+      - name: Install Sophios
         if: always()
         run: cd sophios/ && pip install ".[all_except_runner_src]"
 
+      - name: Update Sophios Config
+        if: always()
+        run: cd sophios/ && python .github/update_sophios_config.py
+
       - name: Install Molecular Modeling Workflows
         if: always()
         # Also run mm-workflows command to generate

.github/workflows/lint_and_test.yml

@@ -160,10 +160,14 @@ jobs:
         # "SC1017 (error): Literal carriage return. Run script through tr -d '\r' ."
         run: shellcheck -e SC1017 $(find sophios/ -name "*.sh" -and -not -path "./3/*")
 
-      - name: Install Workflow Inference Compiler
+      - name: Install Sophios
         if: always()
         run: cd sophios/ && pip install ".[all_except_runner_src]"
 
+      - name: Update Sophios Config
+        if: always()
+        run: cd sophios/ && python .github/update_sophios_config.py
+
       - name: Install Molecular Modeling Workflows
         if: always()
         # Also run mm-workflows command to generate

.github/workflows/lint_and_test_macos.yml

@@ -84,10 +84,14 @@ jobs:
         # "SC1017 (error): Literal carriage return. Run script through tr -d '\r' ."
         run: shellcheck -e SC1017 $(find sophios/ -name "*.sh" -and -not -path "./3/*")
 
-      - name: Install Workflow Inference Compiler
+      - name: Install Sophios
         if: always()
         run: cd sophios/ && pip install ".[all_except_runner_src]"
 
+      - name: Update Sophios Config
+        if: always()
+        run: cd sophios/ && python .github/update_sophios_config.py
+
       - name: Install Molecular Modeling Workflows
         if: always()
         # Also run mm-workflows command to generate

.github/workflows/run_workflows.yml

@@ -138,10 +138,14 @@ jobs:
     #   run: cd mm-workflows/ && ./dockerPull.sh
     #   # For self-hosted runners, make sure the docker cache is up-to-date.
 
-    - name: Install Workflow Inference Compiler
+    - name: Install Sophios
       if: always()
       run: cd sophios/ && pip install ".[all_except_runner_src]"
 
+    - name: Update Sophios Config
+      if: always()
+      run: cd sophios/ && python .github/update_sophios_config.py
+
     - name: Install Molecular Modeling Workflows
       if: always()
       # Also run mm-workflows command to generate

.github/workflows/run_workflows_weekly.yml

@@ -101,10 +101,14 @@ jobs:
     #   run: cd mm-workflows/ && ./dockerPull.sh
     #   # For self-hosted runners, make sure the docker cache is up-to-date.
 
-    - name: Install Workflow Inference Compiler
+    - name: Install Sophios
       if: always()
       run: cd sophios/ && pip install ".[all_except_runner_src]"
 
+    - name: Update Sophios Config
+      if: always()
+      run: cd sophios/ && python .github/update_sophios_config.py
+
     - name: Install Molecular Modeling Workflows
       if: always()
       # Also run mm-workflows command to generate

setup.py

@@ -18,27 +18,21 @@ def run(self) -> None:
         copytree(adapters_dir, adapters_target_dir, dirs_exist_ok=True)
         copytree(examples_dir, examples_target_dir, ignore=ignore_patterns(*extlist), dirs_exist_ok=True)
         # Never overwrite user config
-        if not (Path.home() / 'wic').exists():
-            # config_path
-            config_path = Path.home() / 'wic'
-            # global config file in ~/wic
-            global_config_file = config_path / 'global_config.json'
+        global_config_file = Path.home() / 'wic/global_config.json'
 
-            # setup.py always gets executed in the project_root
-            config_file = Path(__file__).parent / 'src' / 'sophios' / 'config.json'
+        if not global_config_file.exists():
+            config_path = global_config_file.parent
+            config_path.mkdir(parents=True, exist_ok=True)
+            basic_config_file = Path(__file__).parent / 'src' / 'sophios' / 'config_basic.json'
             config = {}
             # config_file can contain absolute or relative paths
-            with open(config_file, 'r', encoding='utf-8') as f:
+            with open(basic_config_file, 'r', encoding='utf-8') as f:
                 config = json.load(f)
-            conf_tags = ['search_paths_cwl', 'search_paths_wic']
-            for tag in conf_tags:
-                for ns in config[tag]:
-                    abs_paths = [str(Path(path).absolute()) for path in config[tag][ns]]
-                    config[tag][ns] = abs_paths
-            config_path.mkdir(parents=True, exist_ok=True)
+            config['search_paths_cwl']['global'] = [str(adapters_dir)]
+            config['search_paths_wic']['global'] = [str(examples_dir)]
             # write out the config file with paths
             with open(global_config_file, 'w', encoding='utf-8') as f:
-                json.dump(config, f)
+                json.dump(config, f, indent=4)
 
         # Continue with the standard build process
         super().run()

src/sophios/__init__.py

@@ -4,4 +4,4 @@
 
 # Well, not THIS file.
 auto_gen_header = f"""# This file was autogenerated using the Workflow Inference Compiler, version {__version__}
-# https://github.com/PolusAI/workflow-inference-compiler\n"""
+# https://github.com/PolusAI/sophios\n"""

src/sophios/input_output.py

@@ -141,21 +141,6 @@ def write_config_to_disk(config: Json, config_file: Path) -> None:
         json.dump(config, f, indent=4, sort_keys=True)
 
 
-def check_sister_directories(parent_dir: Path, target_names: list[str]) -> bool:
-    """Checks if sister directories with given names exist."""
-
-    parent_path = Path(parent_dir)
-    if not parent_path.is_dir():
-        return False
-
-    for target_name in target_names:
-        sister_path = parent_path.parent / target_name
-        if not sister_path.is_dir():
-            return False
-
-    return True
-
-
 def get_config(config_file: Path, default_config_file: Path) -> Json:
     """Returns the config json object from config_file with absolute paths
 
@@ -167,26 +152,13 @@ def get_config(config_file: Path, default_config_file: Path) -> Json:
         Json: The config json object with absolute filepaths
     """
     global_config: Json = {}
-    ci_sister_dirs = ['mm-workflows', 'image-workflows']
     if not config_file.exists():
-        # check if sophios is run from  'ci environment' dir
-        if check_sister_directories(Path.cwd(), ci_sister_dirs):
-            if config_file == default_config_file:
-                global_config = get_default_config()
-                # write the default config object to the 'global_config.json' file in user's ~/wic directory
-                # for user to inspect and or modify the config json file
-                write_config_to_disk(global_config, default_config_file)
-                print(f'default config file : {default_config_file} generated')
-            else:
-                print(f"Error user specified config file {config_file} doesn't exist")
-                sys.exit()
-        else:
-            global_config = get_basic_config()
-            # write the basic config object to the 'global_config.json' file in user's ~/wic directory
-            # for user to inspect and or modify the config json file
-            write_config_to_disk(global_config, default_config_file)
-            move_adapters_and_examples(global_config)
-            print(f'default config file : {default_config_file} generated')
+        global_config = get_basic_config()
+        # write the basic config object to the 'global_config.json' file in user's ~/wic directory
+        # for user to inspect and or modify the config json file
+        write_config_to_disk(global_config, default_config_file)
+        move_adapters_and_examples(global_config)
+        print(f'default config file : {default_config_file} generated')
     else:
         # reading user specified config file only if it exists
         # never overwrite user's config file or generate another file in user's non-default directory
@@ -242,19 +214,6 @@ def read_config_from_disk(config_file: Path, abspath: bool = True) -> Json:
     return config
 
 
-def get_default_config() -> Json:
-    """Returns the default config with absolute paths
-
-    Returns:
-        Json: The config json object with absolute filepaths
-    """
-    src_dir = Path(__file__).parent
-    default_config: Json = {}
-    # read_config_from_disk handles converting them to absolute paths
-    default_config = read_config_from_disk(src_dir/'config.json')
-    return default_config
-
-
 def get_basic_config() -> Json:
     """Returns the (default) basic config with absolute paths
 

src/sophios/plugins.py

@@ -481,7 +481,7 @@ def blindly_execute_python_workflows() -> None:
     # Since this is completely different test path we have to copy
     # default .txt files to default global_config.json
     config_file = Path().home()/'wic'/'global_config.json'
-    global_config = io.get_default_config()
+    global_config = io.read_config_from_disk(config_file)
     pythonapi.global_config = get_tools_cwl(global_config)  # Use path fallback in the CI
     paths = get_py_paths(global_config)
     # Above we are assuming that config is default