Added debug message for tool configuration

Added flag to get config diff
Added debug message for getting CFLAGS from environment
Check configuration options for default_executable during compilation tool registration
Add gnu to set of compatible toolsets for MSVC linker & archiver
Fix bug in use of build_env for cmake

yggdrasil/command_line.py

@@ -1412,7 +1412,12 @@ class update_config(SubCommand):
           {'action': 'store_true', 'default': None,
            'help': ('Don\'t set the KMP_DUPLICATE_LIB_OK environment variable '
                     'when running models (see help for \'--allow-multiple-omp\' '
-                    'for more information).')})]
+                    'for more information).')}),
+         (('--diff', ),
+          {'action': 'store_true',
+           'help': ('Display the diff between the updated configuration '
+                    'and the existing configuration without actually '
+                    'updating the configuration file.')})]
         + [(('--%s-compiler' % k, ),
             {'help': ('Name or path to compiler that should be used to compile '
                       'models written in %s.' % k)})
@@ -1523,7 +1528,7 @@ def func(cls, args):
                     lang_kwargs[k][x] = getattr(args, '%s_%s' % (k, x))
         config.update_language_config(
             args.languages, overwrite=args.overwrite,
-            verbose=(not args.quiet),
+            verbose=(not args.quiet), diff=args.diff,
             disable_languages=args.disable_languages,
             enable_languages=args.enable_languages,
             allow_multiple_omp=args.allow_multiple_omp,

yggdrasil/config.py

@@ -14,6 +14,7 @@
 import warnings
 import configparser
 import copy
+import pprint
 import argparse
 from contextlib import contextmanager
 from collections import OrderedDict
@@ -257,6 +258,38 @@ def get(self, section, option, default=None, **kwargs):
             return self.backwards_str2val(out)
         return default
 
+    def diff(self, solf):
+        r"""Get the difference between this and another YggConfigParser.
+
+        Args:
+            solf (YggConfigParser): Parser to compare against this one.
+
+        Returns:
+            dict: Sets of added, missing and changed parameters between
+                this config parser and solf.
+
+        """
+        out = {k: [] for k in ['added', 'missing', 'changed']}
+        for s in self.sections():
+            if not solf.has_section(s):
+                out['missing'] += [(s, opt) for opt in self.options(s)]
+                continue
+            for opt in self.options(s):
+                if not solf.has_option(s, opt):
+                    out['missing'].append((s, opt))
+                elif self.get(s, opt) != solf.get(s, opt):
+                    out['changed'].append((s, opt,
+                                           self.get(s, opt),
+                                           solf.get(s, opt)))
+        for s in solf.sections():
+            if not self.has_section(s):
+                out['added'] += [(s, opt) for opt in solf.options(s)]
+                continue
+            for opt in solf.options(s):
+                if not self.has_option(s, opt):
+                    out['added'].append((s, opt))
+        return out
+
 
 def get_language_order(drivers):
     r"""Get the correct language order, including any base languages.
@@ -306,7 +339,7 @@ def get_language_order(drivers):
 def update_language_config(languages=None, skip_warnings=False,
                            disable_languages=None, enable_languages=None,
                            allow_multiple_omp=None, lang_kwargs=None,
-                           overwrite=False, verbose=False):
+                           overwrite=False, verbose=False, diff=False):
     r"""Update configuration options for a language driver.
 
     Args:
@@ -327,14 +360,17 @@ def update_language_config(languages=None, skip_warnings=False,
             Defaults to False.
         verbose (bool, optional): If True, information about the config file
             will be displayed. Defaults to False.
+        diff (bool, optional): If True, display the difference between
+            the updated configuration and the existing one, without
+            actually updating the file. Defaults to False.
         lang_kwargs (dict, optional): Dictionary containing language
             specific keyword arguments. Defaults to {}.
 
     """
     from yggdrasil.components import import_component
-    if verbose:
-        logger.info("Updating user configuration file for yggdrasil at:\n\t%s"
-                    % usr_config_file)
+    if verbose and not diff:
+        logger.info(f"Updating user configuration file for yggdrasil "
+                    f"at:\n\t{usr_config_file}")
     miss = []
     if (languages is None) or overwrite:
         all_languages = tools.get_supported_lang()
@@ -350,44 +386,65 @@ def update_language_config(languages=None, skip_warnings=False,
         enable_languages = []
     if lang_kwargs is None:
         lang_kwargs = {}
-    if overwrite:
-        shutil.copy(def_config_file, usr_config_file)
-        ygg_cfg_usr.reload()
-    if allow_multiple_omp is not None:
-        if not ygg_cfg_usr.has_section('general'):
-            ygg_cfg_usr.add_section('general')
-        ygg_cfg_usr.set('general', 'allow_multiple_omp', allow_multiple_omp)
-    drivers = OrderedDict([(lang, import_component('model', lang))
-                           for lang in languages])
-    drv = list(get_language_order(drivers).values())
-    for idrv in drv:
-        if (((idrv.language in disable_languages)
-             and (idrv.language in enable_languages))):
-            logger.info(("%s language both enabled and disabled. "
-                         "No action will be taken.") % idrv.language)
-        elif idrv.language in disable_languages:
-            if not ygg_cfg_usr.has_section(idrv.language):
-                ygg_cfg_usr.add_section(idrv.language)
-            ygg_cfg_usr.set(idrv.language, 'disable', 'True')
-        elif idrv.language in enable_languages:
-            if not ygg_cfg_usr.has_section(idrv.language):
-                ygg_cfg_usr.add_section(idrv.language)
-            ygg_cfg_usr.set(idrv.language, 'disable', 'False')
-        if ygg_cfg_usr.get(idrv.language, 'disable', 'False').lower() == 'true':
-            continue  # pragma: no cover
-        miss += idrv.configure(ygg_cfg_usr,
-                               **lang_kwargs.get(idrv.language, {}))
-    ygg_cfg_usr.update_file()
-    ygg_cfg.reload()
-    if not skip_warnings:
-        for sect, opt, desc in miss:  # pragma: windows
-            warnings.warn(
-                f"Could not set option {opt} in section {sect}. Please "
-                f"set this in {ygg_cfg_usr.file_to_update} to: {desc}",
-                RuntimeWarning)
-    if verbose:
-        with open(usr_config_file, 'r') as fd:
-            print(fd.read())
+    existing = None
+    cached = '_cached'.join(os.path.splitext(usr_config_file))
+    if diff:
+        shutil.copy2(usr_config_file, cached)
+        existing = YggConfigParser.from_files(
+            [def_config_file, cached, loc_config_file])
+    try:
+        if overwrite:
+            shutil.copy(def_config_file, usr_config_file)
+            ygg_cfg_usr.reload()
+        if allow_multiple_omp is not None:
+            if not ygg_cfg_usr.has_section('general'):
+                ygg_cfg_usr.add_section('general')
+            ygg_cfg_usr.set('general', 'allow_multiple_omp',
+                            allow_multiple_omp)
+        drivers = OrderedDict([(lang, import_component('model', lang))
+                               for lang in languages])
+        drv = list(get_language_order(drivers).values())
+        for idrv in drv:
+            if (((idrv.language in disable_languages)
+                 and (idrv.language in enable_languages))):
+                logger.info(("%s language both enabled and disabled. "
+                             "No action will be taken.") % idrv.language)
+            elif idrv.language in disable_languages:
+                if not ygg_cfg_usr.has_section(idrv.language):
+                    ygg_cfg_usr.add_section(idrv.language)
+                ygg_cfg_usr.set(idrv.language, 'disable', 'True')
+            elif idrv.language in enable_languages:
+                if not ygg_cfg_usr.has_section(idrv.language):
+                    ygg_cfg_usr.add_section(idrv.language)
+                ygg_cfg_usr.set(idrv.language, 'disable', 'False')
+            if ygg_cfg_usr.get(idrv.language, 'disable', 'False').lower() == 'true':
+                continue  # pragma: no cover
+            miss += idrv.configure(ygg_cfg_usr,
+                                   **lang_kwargs.get(idrv.language, {}))
+        ygg_cfg_usr.update_file()
+        ygg_cfg.reload()
+        if not skip_warnings:
+            for sect, opt, desc in miss:  # pragma: windows
+                warnings.warn(
+                    f"Could not set option {opt} in section {sect}. Please "
+                    f"set this in {ygg_cfg_usr.file_to_update} to: {desc}",
+                    RuntimeWarning)
+        if verbose:
+            with open(usr_config_file, 'r') as fd:
+                print(fd.read())
+        if diff:
+            diffdict = existing.diff(ygg_cfg)
+            msg = (f"Cached:  {cached}\n"
+                   f"Updated: {usr_config_file}\n")
+            for k in ['missing', 'added', 'changed']:
+                msg += f"{k.title():7}: {pprint.pformat(diffdict[k])}\n"
+            print(msg)
+    finally:
+        if diff:
+            shutil.move(cached, usr_config_file)
+            ygg_cfg_usr.reload()
+            ygg_cfg.reload()
+            assert not any(existing.diff(ygg_cfg).values())
 
 
 def get_ygg_loglevel(cfg=None, default='DEBUG'):

yggdrasil/drivers/CMakeModelDriver.py

@@ -648,10 +648,10 @@ def create_dep(cls, without_wrapper=False, **kwargs):
         if platform._is_win and out['target_dep'].tool('compiler').is_gnu:
             gcc = out['target_dep'].tool('compiler').get_executable(
                 full_path=True)
-            env = out['build_env']
+            env = out.get('builder_env', for_build=True)
             path = cls.prune_sh_gcc(env, gcc)
             env['PATH'] = path
-            out.set('build_env', env)
+            out.set('builder_env', env)
             if not shutil.which('sh', path=path):  # pragma: appveyor
                 # This will not be run on Github actions where
                 # the shell is always set

yggdrasil/drivers/CModelDriver.py

@@ -169,7 +169,7 @@ class ClangCompiler(CCompilerBase):
     compatible_toolsets = ['gnu']
     next_stage_flag_flag = '-Xlinker'
     version_regex = [
-        r'(?P<version>(?:Apple )?clang version \d+\.\d+\.\d+)']
+        r'(?P<version>(?:Apple )?(?:(?:clang)|(?:LLVM)) version \d+\.\d+\.\d+)']
     flag_options = OrderedDict(list(CCompilerBase.flag_options.items())
                                + [('sysroot', '--sysroot'),
                                   ('isysroot', {'key': '-isysroot',
@@ -371,6 +371,7 @@ class MSVCLinker(LinkerBase):
     platforms = MSVCCompiler.platforms
     languages = MSVCCompiler.languages
     toolset = MSVCCompiler.toolset
+    compatible_toolsets = ['llvm', 'gnu']
     aliases = []
     output_key = '/OUT:%s'
     output_first = True
@@ -440,7 +441,7 @@ class MSVCArchiver(ArchiverBase):
     libtype_flags = {}
     output_key = '/OUT:%s'
     toolset = 'msvc'
-    compatible_toolsets = ['llvm']
+    compatible_toolsets = ['llvm', 'gnu']
     search_path_envvar = ['LIB']
 
 

yggdrasil/drivers/CompiledModelDriver.py

@@ -4191,6 +4191,7 @@ def before_registration(cls):
         to registration including things like platform dependent properties and
         checking environment variables for default settings.
         """
+        from yggdrasil.config import ygg_cfg
         if cls.toolname is None:  # pragma: debug
             raise CompilationToolError("Registering unnamed compilation tool.")
         cls.is_gnu = (cls.toolset == 'gnu')
@@ -4201,6 +4202,9 @@ def before_registration(cls):
             # Copy so that list modification is not propagated to subclasses
             setattr(cls, k, copy.deepcopy(getattr(cls, k, [])))
         # Set attributes based on environment variables or sysconfig
+        if cls.default_executable is None and cls.languages:
+            cls.default_executable = ygg_cfg.get(
+                cls.languages[0], f'{cls.toolname}_executable', None)
         if cls.default_executable is None:
             cls.default_executable = cls.env_matches_tool()
         if cls.default_executable is None:
@@ -4670,6 +4674,10 @@ def env_matches_tool(cls, use_sysconfig=False, env=None,
             env.update(sysconfig.get_config_vars())
         else:
             env.update(os.environ)
+        # TODO: temp
+        if (('-arch arm64' in env.get('CFLAGS', '')
+             and '-arch x86_64' in env.get('CFLAGS', ''))):
+            verbose = True
         envi_full = ''
         if isinstance(cls.default_executable_env, str):
             envi_full = env.get(cls.default_executable_env, '').split(
@@ -6918,6 +6926,8 @@ def configure(cls, cfg, **kwargs):
             cfg.set(cls.language, k, vtool.toolname)
             setattr(cls, f'default_{k}', vtool.toolname)
             if os.path.isfile(v):
+                setattr(_tool_registry.tool(k, vtool.toolname),
+                        'default_executable', v)
                 cfg.set(cls.language, f'{vtool.toolname}_executable', v)
         # Clear dependencies that should be set based on tool
         if kwargs:
@@ -7004,6 +7014,7 @@ def configure_libraries(cls, cfg):
                 kws[k] = cls.get_tool(k)
             except InvalidCompilationTool:
                 pass
+        print(f"CONFIGURATION SPECIALIZATION:\n{pprint.pformat(kws)}")
         libs = cls.libraries.specialized(**kws)
         for v in libs.libraries.values():
             v.from_cache(cfg)
@@ -7084,7 +7095,7 @@ def compile_dependencies(cls, dep=None, add_all_products=False,
             dep.products(kwargs['products'],
                          include_dependencies=True,
                          skip_build_products=True)
-        dep.logInfo()
+        dep.logInfo(f'compiled: {dep}')
 
     @classmethod
     def cleanup_dependencies(cls, dep=None, products=None,

yggdrasil/tools.py

@@ -1036,7 +1036,7 @@ def get_supported_comm(dont_include_value=False):
     for k in excl_list:
         if k in out:
             out.remove(k)
-    return list(set(out))
+    return out  # list(set(out))
 
 
 def is_lang_installed(lang):