Merge pull request #2056 from hpcugent/eb302

sync develop with eb302 branch after merging #2047+#2052 & #2051

easybuild/framework/easyblock.py

@@ -1982,8 +1982,10 @@ def _sanity_check_step_dry_run(self, custom_paths=None, custom_commands=None, **
         else:
             self.dry_run_msg("  (none)")
 
-        if build_option('rpath'):
+        if self.toolchain.use_rpath:
             self.sanity_check_rpath()
+        else:
+            self.log.debug("Skiping RPATH sanity check")
 
     def _sanity_check_step(self, custom_paths=None, custom_commands=None, extension=False):
         """Real version of sanity_check_step method."""
@@ -2051,11 +2053,13 @@ def _sanity_check_step(self, custom_paths=None, custom_commands=None, extension=
         if fake_mod_data:
             self.clean_up_fake_module(fake_mod_data)
 
-        if build_option('rpath'):
+        if self.toolchain.use_rpath:
             rpath_fails = self.sanity_check_rpath()
             if rpath_fails:
                 self.log.warning("RPATH sanity check failed!")
                 self.sanity_check_fail_msgs.extend(rpath_fails)
+        else:
+            self.log.debug("Skiping RPATH sanity check")
 
         # pass or fail
         if self.sanity_check_fail_msgs:

easybuild/framework/easyconfig/easyconfig.py

@@ -1378,6 +1378,42 @@ def robot_find_easyconfig(name, version):
     return res
 
 
+def verify_easyconfig_filename(path, specs, parsed_ec=None):
+    """
+    Check whether parsed easyconfig at specified path matches expected specs;
+    this basically verifies whether the easyconfig filename corresponds to its contents
+
+    :param path: path to easyconfig file
+    :param specs: expected specs (dict with easyconfig parameter values)
+    :param parsed_ec: (list of) EasyConfig instance(s) corresponding to easyconfig file
+    """
+    if isinstance(parsed_ec, EasyConfig):
+        ecs = [parsed_ec]
+    elif isinstance(parsed_ec, (list, tuple)):
+        ecs = parsed_ec
+    elif parsed_ec is None:
+        ecs = process_easyconfig(path)
+    else:
+        raise EasyBuildError("Unexpected value type for parsed_ec: %s (%s)", type(parsed_ec), parsed_ec)
+
+    fullver = det_full_ec_version(specs)
+
+    expected_filename = '%s-%s.eb' % (specs['name'], fullver)
+    if os.path.basename(path) != expected_filename:
+        raise EasyBuildError("Easyconfig filename %s does not match provided specs %s", os.path.basename(path), specs)
+
+    for ec in ecs:
+        found_fullver = det_full_ec_version(ec['ec'])
+        if ec['ec']['name'] != specs['name'] or found_fullver != fullver:
+            subspec = dict((key, specs[key]) for key in ['name', 'toolchain', 'version', 'versionsuffix'])
+            error_msg = "Contents of %s does not match with filename" % path
+            error_msg += "; expected filename based on contents: %s-%s.eb" % (ec['ec']['name'], found_fullver)
+            error_msg += "; expected (relevant) parameters based on filename %s: %s" % (os.path.basename(path), subspec)
+            raise EasyBuildError(error_msg)
+
+    _log.info("Contents of %s verified against easyconfig filename, matches %s", path, specs)
+
+
 def robot_find_minimal_toolchain_of_dependency(dep, modtool, parent_tc=None, parent_first=False):
     """
     Find the minimal toolchain of a dependency

easybuild/tools/robot.py

@@ -39,8 +39,8 @@
 from vsc.utils import fancylogger
 from vsc.utils.missing import nub
 
-from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR
-from easybuild.framework.easyconfig.easyconfig import ActiveMNS, process_easyconfig, robot_find_easyconfig
+from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, process_easyconfig
+from easybuild.framework.easyconfig.easyconfig import robot_find_easyconfig, verify_easyconfig_filename
 from easybuild.framework.easyconfig.tools import find_resolved_modules, skip_available
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option
@@ -318,11 +318,7 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False):
                         processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden)
 
                         # ensure that selected easyconfig provides required dependency
-                        mods = [spec['ec'].full_mod_name for spec in processed_ecs]
-                        dep_mod_name = ActiveMNS().det_full_module_name(cand_dep)
-                        if not dep_mod_name in mods:
-                            raise EasyBuildError("easyconfig file %s does not contain module %s (mods: %s)",
-                                                 path, dep_mod_name, mods)
+                        verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs)
 
                         for ec in processed_ecs:
                             if not ec in easyconfigs + additional:

easybuild/tools/toolchain/compiler.py

@@ -85,6 +85,7 @@ class Compiler(Toolchain):
         '32bit': (False, "Compile 32bit target"),  # LA, FFTW
         'openmp': (False, "Enable OpenMP"),
         'packed-linker-options': (False, "Pack the linker options as comma separated list"),  # ScaLAPACK mainly
+        'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"),
     }
 
     COMPILER_UNIQUE_OPTION_MAP = None

easybuild/tools/toolchain/toolchain.py

@@ -134,6 +134,8 @@ def __init__(self, name=None, version=None, mns=None, class_constants=None, tcde
 
         self.modules_tool = modtool
 
+        self.use_rpath = False
+
         self.mns = mns
         self.mod_full_name = None
         self.mod_short_name = None
@@ -679,7 +681,11 @@ def prepare(self, onlymod=None, silent=False, loadmod=True, rpath_filter_dirs=No
                 self.prepare_compiler_cache(cache_tool)
 
         if build_option('rpath'):
-            self.prepare_rpath_wrappers()
+            if self.options.get('rpath', True):
+                self.prepare_rpath_wrappers()
+                self.use_rpath = True
+            else:
+                self.log.info("Not putting RPATH wrappers in place, disabled via 'rpath' toolchain option")
 
     def comp_cache_compilers(self, cache_tool):
         """

test/framework/easyconfig.py

@@ -45,8 +45,9 @@
 import easybuild.framework.easyconfig as easyconfig
 from easybuild.framework.easyblock import EasyBlock
 from easybuild.framework.easyconfig.constants import EXTERNAL_MODULE_MARKER
-from easybuild.framework.easyconfig.easyconfig import ActiveMNS, EasyConfig, create_paths, letter_dir_for
-from easybuild.framework.easyconfig.easyconfig import copy_easyconfigs, get_easyblock_class, resolve_template
+from easybuild.framework.easyconfig.easyconfig import ActiveMNS, EasyConfig, create_paths, copy_easyconfigs
+from easybuild.framework.easyconfig.easyconfig import letter_dir_for, get_easyblock_class, process_easyconfig
+from easybuild.framework.easyconfig.easyconfig import resolve_template, verify_easyconfig_filename
 from easybuild.framework.easyconfig.licenses import License, LicenseGPLv3
 from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig
 from easybuild.framework.easyconfig.templates import template_constant_dict, to_template_str
@@ -1968,6 +1969,44 @@ def test_resolve_template(self):
         # '%(name)' is not a correct template spec (missing trailing 's')
         self.assertEqual(resolve_template('%(name)', tmpl_dict), '%(name)')
 
+    def test_verify_easyconfig_filename(self):
+        """Test verify_easyconfig_filename function"""
+        test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+        toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-1.3.12-test.eb')
+        specs = {
+            'name': 'toy',
+            'toolchain': {'name': 'gompi', 'version': '1.3.12'},
+            'version': '0.0',
+            'versionsuffix': '-test'
+        }
+
+        # all is well
+        verify_easyconfig_filename(toy_ec, specs)
+
+        # pass parsed easyconfig
+        verify_easyconfig_filename(toy_ec, specs, parsed_ec=process_easyconfig(toy_ec))
+
+        # incorrect spec
+        specs['versionsuffix'] = ''
+        error_pattern = "filename %s does not match provided specs" % os.path.basename(toy_ec)
+        self.assertErrorRegex(EasyBuildError, error_pattern, verify_easyconfig_filename, toy_ec, specs)
+        specs['versionsuffix'] = '-test'
+
+        # incorrect file name
+        toy_txt = read_file(toy_ec)
+        toy_ec = os.path.join(self.test_prefix, 'toy.eb')
+        write_file(toy_ec, toy_txt)
+        error_pattern = "filename toy.eb does not match provided specs"
+        self.assertErrorRegex(EasyBuildError, error_pattern, verify_easyconfig_filename, toy_ec, specs)
+
+        # incorrect file contents
+        error_pattern = r"Contents of .*/%s does not match with filename" % os.path.basename(toy_ec)
+        toy_txt = toy_txt.replace("versionsuffix = '-test'", "versionsuffix = ''")
+        toy_ec = os.path.join(self.test_prefix, 'toy-0.0-gompi-1.3.12-test.eb')
+        write_file(toy_ec, toy_txt)
+        error_pattern = "Contents of .*/%s does not match with filename" % os.path.basename(toy_ec)
+        self.assertErrorRegex(EasyBuildError, error_pattern, verify_easyconfig_filename, toy_ec, specs)
+
 
 def suite():
     """ returns all the testcases in this module """

test/framework/toolchain.py

@@ -1164,6 +1164,21 @@ def test_toolchain_prepare_rpath(self):
         # preparing RPATH wrappers requires --experimental, need to bypass that here
         tc.log.experimental = lambda x: x
 
+        # 'rpath' toolchain option gives control to disable use of RPATH wrappers
+        tc.set_options({})
+        self.assertTrue(tc.options['rpath'])  # enabled by default
+
+        # setting 'rpath' toolchain option to false implies no RPATH wrappers being used
+        tc.set_options({'rpath': False})
+        tc.prepare()
+        res = which('gcc', retain_all=True)
+        self.assertTrue(len(res) >= 1)
+        self.assertFalse(tc.is_rpath_wrapper(res[0]))
+        self.assertFalse(any(tc.is_rpath_wrapper(x) for x in res[1:]))
+        self.assertTrue(os.path.samefile(res[0], fake_gcc))
+
+        # enable 'rpath' toolchain option again (equivalent to the default setting)
+        tc.set_options({'rpath': True})
         tc.prepare()
 
         # check that wrapper is indeed in place

test/framework/toy_build.py

@@ -1241,10 +1241,18 @@ def test_toy_iter(self):
 
     def test_toy_rpath(self):
         """Test toy build using --rpath."""
-        self.test_toy_build(extra_args=['--rpath', '--experimental'])
+        self.test_toy_build(extra_args=['--rpath', '--experimental'], raise_error=True)
 
         # also test use of --rpath-filter
-        self.test_toy_build(extra_args=['--rpath', '--rpath-filter=/test.*,/foo.*', '--experimental'])
+        self.test_toy_build(extra_args=['--rpath', '--rpath-filter=/test.*,/foo.*', '--experimental'], raise_error=True)
+
+        # test use of rpath toolchain option
+        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+        toy_ec_txt = read_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb'))
+        toy_ec_txt += "\ntoolchainopts = {'rpath': False}\n"
+        toy_ec = os.path.join(self.test_prefix, 'toy.eb')
+        write_file(toy_ec, toy_ec_txt)
+        self.test_toy_build(ec_file=toy_ec, extra_args=['--rpath', '--experimental'], raise_error=True)
 
 
 def suite():