More improvements to handling of library searches

Refactor how dependencies are handled
Simplify libraries specification to allow relative paths

tests/drivers/test_CompiledModelDriver.py

@@ -70,7 +70,7 @@ def test_locate_library_file():
     compiler = CModelDriver.get_tool('compiler').__class__
     files = {}
     for k in ['shared', 'static']:
-        fname = CModelDriver.external_libraries['zmq'].get(k, '')
+        fname = CModelDriver.libraries['zmq'].get(k, '', compiler=compiler)
         if os.path.isfile(fname):
             files[k] = fname
     if not files:
@@ -102,7 +102,6 @@ def test_find_standard_library(lang, toolname, lib):
             and compiler.disassembler(allow_uninstalled=True).is_installed()):
         pytest.skip(f"No compiler for {toolname}")
     out = compiler.find_standard_library(dont_cache=True)
-    print(out, type(out))
     if lib:
         if out is None:
             out = compiler.find_standard_library(verbose=True)
@@ -120,7 +119,7 @@ def test_create_windows_import_gcc():
         kws = {'toolname': 'gcc'}
         if platform._is_win:
             kws['libtype'] = 'shared'
-        dll = CModelDriver.get_dependency_library('python', **kws)
+        dll = CModelDriver.libraries.getfile('python', 'library', **kws)
         CompiledModelDriver.create_windows_import(dll, for_gnu=True,
                                                   overwrite=True)
     else:
@@ -300,12 +299,13 @@ def test_build(self, python_class):
         order = ['shared', 'object', 'static']
         order.remove(CompiledModelDriver._default_libtype)
         order.append(CompiledModelDriver._default_libtype)
-        for libtype in ['shared', 'object', 'static']:
+        for libtype in order:
             python_class.compile_dependencies(
                 libtype=libtype, overwrite=True)
-            if libtype == 'shared':
+            if libtype == CompiledModelDriver._default_libtype:
                 python_class.compile_dependencies(
                     libtype=libtype, overwrite=False)
+                break
             python_class.cleanup_dependencies(libtype=libtype)
 
     def test_get_tool(self, python_class):
@@ -315,80 +315,69 @@ def test_get_tool(self, python_class):
         with pytest.raises(ValueError):
             python_class.get_tool('compiler', return_prop='invalid')
 
-    def test_get_dependency_info(self, python_class):
-        r"""Test get_dependency_info."""
-        dep_list = (
-            python_class.get_dependency_order(
-                python_class.interface_library)
-            + list(python_class.external_libraries.keys()))
+    def test_libraries_get(self, python_class):
+        r"""Test libraries.get."""
+        dep_list = python_class.libraries.keys()
         for dep in dep_list:
-            python_class.get_dependency_info(dep, default='default')
+            python_class.libraries.get(dep, default='default')
         with pytest.raises(KeyError):
-            python_class.get_dependency_info('invalid')
-        assert (python_class.get_dependency_info('invalid', default='default')
+            python_class.libraries.get('invalid')
+        assert (python_class.libraries.get('invalid', default='default')
                 == 'default')
 
-    def test_get_dependency_source(self, python_class):
-        r"""Test get_dependency_source."""
-        dep_list = (
-            python_class.get_dependency_order(
-                python_class.interface_library)
-            + list(python_class.external_libraries.keys()))
+    def test_libraries_getfile_source(self, python_class):
+        r"""Test libraries.getfile for source."""
+        dep_list = python_class.libraries.keys()
         for dep in dep_list:
-            python_class.get_dependency_source(dep, default='default')
-        with pytest.raises(ValueError):
-            python_class.get_dependency_source('invalid')
-        assert python_class.get_dependency_source(__file__) == __file__
-        assert (python_class.get_dependency_source(
-            'invalid', default='default') == 'default')
-
-    def test_get_dependency_object(self, python_class):
-        r"""Test get_dependency_object."""
-        dep_list = (
-            python_class.get_dependency_order(
-                python_class.interface_library)
-            + list(python_class.external_libraries.keys()))
+            python_class.libraries.getfile(dep, 'source',
+                                           default='default')
+        with pytest.raises(KeyError):
+            python_class.libraries.getfile('invalid', 'source')
+        assert (python_class.libraries.getfile(
+            'invalid', 'source', default='default') == 'default')
+
+    def test_libraries_getfile_object(self, python_class):
+        r"""Test libraries.getfile for object.."""
+        dep_list = python_class.libraries.keys()
         for dep in dep_list:
-            python_class.get_dependency_object(dep, default='default')
-        with pytest.raises(ValueError):
-            python_class.get_dependency_object('invalid')
-        assert python_class.get_dependency_object(__file__) == __file__
-        assert (python_class.get_dependency_object(
-            'invalid', default='default') == 'default')
+            python_class.libraries.getfile(dep, 'object',
+                                           default='default')
+        with pytest.raises(KeyError):
+            python_class.libraries.getfile('invalid', 'object')
+        assert (python_class.libraries.getfile(
+            'invalid', 'object', default='default') == 'default')
 
-    def test_get_dependency_library(self, python_class):
-        r"""Test get_dependency_library."""
-        with pytest.raises(ValueError):
-            python_class.get_dependency_library('invalid', libtype='invalid')
-        for dep, info in python_class.external_libraries.items():
+    def test_libraries_getfile_library(self, python_class):
+        r"""Test libraries.getfile for library."""
+        with pytest.raises(KeyError):
+            python_class.libraries.getfile(
+                'invalid', 'library', libtype='invalid')
+        for dep, info in python_class.libraries.external.items():
             libtype_orig = info.get('libtype', None)
             if libtype_orig not in ['static', 'shared']:
                 continue
             if libtype_orig == 'static':  # pragma: no cover
                 libtype = 'shared'
             else:
                 libtype = 'static'
-            with pytest.raises(ValueError):
-                python_class.get_dependency_library(dep, libtype=libtype)
-        with pytest.raises(ValueError):
-            python_class.get_dependency_library('invalid')
-        assert python_class.get_dependency_library(__file__) == __file__
-        assert (python_class.get_dependency_library(
-            'invalid', default='default') == 'default')
+            with pytest.raises(KeyError):
+                python_class.libraries.getfile(dep, libtype)
+        with pytest.raises(KeyError):
+            python_class.libraries.getfile('invalid', 'library')
+        assert (python_class.libraries.getfile(
+            'invalid', 'library', default='default') == 'default')
 
-    def test_get_dependency_include_dirs(self, python_class):
-        r"""Test get_dependency_include_dirs."""
-        with pytest.raises(ValueError):
-            python_class.get_dependency_include_dirs('invalid')
-        assert (python_class.get_dependency_include_dirs(__file__)
-                == [os.path.dirname(__file__)])
-        assert (python_class.get_dependency_include_dirs(
-            'invalid', default='default') == ['default'])
-
-    def test_get_dependency_order(self, python_class):
-        r"""Test get_dependency_order."""
-        deps = list(python_class.internal_libraries.keys())
-        python_class.get_dependency_order(deps)
+    def test_libraries_getfile_include_dirs(self, python_class):
+        r"""Test libraries.getfile for include_dirs."""
+        with pytest.raises(KeyError):
+            python_class.libraries.getfile('invalid', 'include_dirs')
+        assert (python_class.libraries.getfile(
+            'invalid', 'include_dirs', default=['default']) == ['default'])
+
+    def test_dependency_order(self, python_class):
+        r"""Test dependency_order."""
+        dep = python_class.libraries.get(python_class.interface_library)
+        dep.dependency_order()
 
     def test_get_flags(self, python_class):
         r"""Test get_flags."""

yggdrasil/.ygg_schema.yml

@@ -2786,6 +2786,7 @@ definitions:
           model repository or when providing the model as a service.
         type: string
       disable_python_c_api:
+        default: false
         description: If True, the Python C API will be disabled. Defaults to False.
         type: boolean
       disassembler:
@@ -3298,6 +3299,7 @@ definitions:
         description: Path to file where model weights are saved
         type: string
       with_asan:
+        default: false
         description: If True, the model will be compiled and linked with the address
           sanitizer enabled (if there is one available for the selected compiler).
         type: boolean
@@ -3373,6 +3375,7 @@ definitions:
           type: string
         type: array
       disable_python_c_api:
+        default: false
         description: If True, the Python C API will be disabled. Defaults to False.
         type: boolean
       disassembler:
@@ -3419,6 +3422,7 @@ definitions:
           type: string
         type: array
       with_asan:
+        default: false
         description: If True, the model will be compiled and linked with the address
           sanitizer enabled (if there is one available for the selected compiler).
         type: boolean
@@ -3468,6 +3472,7 @@ definitions:
         description: Build type/configuration that should be built. Defaults to 'Release'.
         type: string
       disable_python_c_api:
+        default: false
         description: If True, the Python C API will be disabled. Defaults to False.
         type: boolean
       driver:
@@ -3559,6 +3564,7 @@ definitions:
           type: string
         type: array
       with_asan:
+        default: false
         description: If True, the model will be compiled and linked with the address
           sanitizer enabled (if there is one available for the selected compiler).
         type: boolean
@@ -3605,6 +3611,7 @@ definitions:
           type: string
         type: array
       disable_python_c_api:
+        default: false
         description: If True, the Python C API will be disabled. Defaults to False.
         type: boolean
       disassembler:
@@ -3653,6 +3660,7 @@ definitions:
           type: string
         type: array
       with_asan:
+        default: false
         description: If True, the model will be compiled and linked with the address
           sanitizer enabled (if there is one available for the selected compiler).
         type: boolean
@@ -3785,6 +3793,7 @@ definitions:
           type: string
         type: array
       disable_python_c_api:
+        default: false
         description: If True, the Python C API will be disabled. Defaults to False.
         type: boolean
       disassembler:
@@ -3837,6 +3846,7 @@ definitions:
         - f2008
         type: string
       with_asan:
+        default: false
         description: If True, the model will be compiled and linked with the address
           sanitizer enabled (if there is one available for the selected compiler).
         type: boolean
@@ -3986,6 +3996,7 @@ definitions:
           type: string
         type: array
       disable_python_c_api:
+        default: false
         description: If True, the Python C API will be disabled. Defaults to False.
         type: boolean
       driver:
@@ -4082,6 +4093,7 @@ definitions:
           type: string
         type: array
       with_asan:
+        default: false
         description: If True, the model will be compiled and linked with the address
           sanitizer enabled (if there is one available for the selected compiler).
         type: boolean

yggdrasil/config.py

@@ -6,6 +6,7 @@
 
 """
 import os
+import re
 import sys
 import json
 import shutil
@@ -193,6 +194,39 @@ def backwards_str2val(self, val):  # pragma: no cover
                 out = val
         return out
 
+    def get_regex(self, section, option, default=None, return_all=False,
+                  **kwargs):
+        r"""Check for an option by regex.
+
+        Args:
+            section (str): Name of section.
+            option (str): Name of option in section.
+            default (obj, optional): Value that should be returned if the
+                section and/or option are not found or are an empty string.
+                Defaults to None.
+            return_all (bool, optional): If True, return a mapping of
+                all options that match the regex. Defaults to False.
+            **kwargs: Additional keyword arguments are passed to the parent
+                class's get.
+
+        Returns:
+            obj: String entry if the section & option exist, otherwise default.
+
+        """
+        out = default
+        if return_all:
+            out = {}
+        if self.has_section(section):
+            regex = re.compile(option)
+            for k in self[section]:
+                if regex.match(k):
+                    val = self.get(section, k, **kwargs)
+                    if return_all:
+                        out[k] = val
+                    else:
+                        return val
+        return out
+
     def get(self, section, option, default=None, **kwargs):
         r"""Return None if the section/option does not exist.
 

yggdrasil/drivers/BuildModelDriver.py

@@ -490,13 +490,12 @@ def is_source_file(cls, fname):
                 return True
         return False
 
-    def compile_dependencies_instance(self, *args, **kwargs):
+    def compile_dependencies_for_model(self, **kwargs):
         r"""Compile dependencies specifically for this instance."""
         if (((self.target_language_driver is not None)
              and (not kwargs.get('dry_run', False)))):
-            suffix_kws = self.select_suffix_kwargs(kwargs)
             self.target_language_driver.compile_dependencies(
-                toolname=self.target_compiler, **suffix_kws)
+                toolname=self.target_compiler, **kwargs)
 
     def compile_model(self, **kwargs):
         r"""Compile model executable(s).