From 8b6fda4df38d93f33265f57f73a3bd608ef5955f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 19 Dec 2024 08:41:32 +0100 Subject: [PATCH 1/5] be more careful when checking install_target value in PythonPackage easyblock, to avoid trouble with unresolved template values like '%(parallel)s' --- easybuild/easyblocks/generic/pythonpackage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 4b76d7e14b..921b6e3ff8 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -546,12 +546,13 @@ def determine_install_command(self): else: self.use_setup_py = True self.install_cmd = SETUP_PY_INSTALL_CMD + install_target = self.cfg.get_ref('install_target') - if self.cfg['install_target'] == EASY_INSTALL_TARGET: + if install_target == EASY_INSTALL_TARGET: self.install_cmd += " %(loc)s" self.py_installopts.append('--no-deps') if self.cfg.get('zipped_egg', False): - if self.cfg['install_target'] == EASY_INSTALL_TARGET: + if install_target == EASY_INSTALL_TARGET: self.py_installopts.append('--zip-ok') else: raise EasyBuildError("Installing zipped eggs requires using easy_install or pip") From 88d0a81f6a059a401f7d5a0a7eb619eff1a83787 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 19 Dec 2024 08:42:11 +0100 Subject: [PATCH 2/5] be more careful with determining value of 'sources' and 'source_urls' in constructor of Bundle easyblock, to avoid trouble with unresolved template values like '%(builddir)s' --- easybuild/easyblocks/generic/bundle.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 437cd707d4..c15c0bb44d 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -164,24 +164,26 @@ def __init__(self, *args, **kwargs): comp_cfg.enable_templating = True # 'sources' is strictly required - if comp_cfg['sources']: + comp_sources = comp_cfg.get_ref('sources') + if comp_sources: # If per-component source URLs are provided, attach them directly to the relevant sources - if comp_cfg['source_urls']: - for source in comp_cfg['sources']: + comp_source_urls = comp_cfg.get_ref('source_urls') + if comp_source_urls: + for source in comp_sources: if isinstance(source, str): - self.cfg.update('sources', [{'filename': source, 'source_urls': comp_cfg['source_urls']}]) + self.cfg.update('sources', [{'filename': source, 'source_urls': comp_source_urls[:]}]) elif isinstance(source, dict): # Update source_urls in the 'source' dict to use the one for the components # (if it doesn't already exist) if 'source_urls' not in source: - source['source_urls'] = comp_cfg['source_urls'] + source['source_urls'] = comp_source_urls[:] self.cfg.update('sources', [source]) else: raise EasyBuildError("Source %s for component %s is neither a string nor a dict, cannot " "process it.", source, comp_cfg['name']) else: # add component sources to list of sources - self.cfg.update('sources', comp_cfg['sources']) + self.cfg.update('sources', comp_sources) else: raise EasyBuildError("No sources specification for component %s v%s", comp_name, comp_version) From 1973cbb0fe8c7f94d6e131e66e3808cb93813ec9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 19 Dec 2024 09:43:16 +0100 Subject: [PATCH 3/5] be even more careful when determining easyconfig parameter values in Bundle constructor, to avoid trouble with unresolved templates --- easybuild/easyblocks/generic/bundle.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index c15c0bb44d..1c41ab9f82 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -79,9 +79,9 @@ def __init__(self, *args, **kwargs): check_for_sources = getattr(self, 'check_for_sources', True) # list of sources for bundle itself *must* be empty (unless overridden by subclass) if check_for_sources: - if self.cfg['sources']: + if self.cfg.get_ref('sources'): raise EasyBuildError("List of sources for bundle itself must be empty, found %s", self.cfg['sources']) - if self.cfg['patches']: + if self.cfg.get_ref('patches'): raise EasyBuildError("List of patches for bundle itself must be empty, found %s", self.cfg['patches']) # copy EasyConfig instance before we make changes to it @@ -187,29 +187,31 @@ def __init__(self, *args, **kwargs): else: raise EasyBuildError("No sources specification for component %s v%s", comp_name, comp_version) - if comp_cfg['checksums']: - src_cnt = len(comp_cfg['sources']) + comp_checksums = comp_cfg.get_ref('checksums') + if comp_checksums: + src_cnt = len(comp_sources) # add per-component checksums for sources to list of checksums - self.cfg.update('checksums', comp_cfg['checksums'][:src_cnt]) + self.cfg.update('checksums', comp_checksums[:src_cnt]) # add per-component checksums for patches to list of checksums for patches - checksums_patches.extend(comp_cfg['checksums'][src_cnt:]) + checksums_patches.extend(comp_checksums[src_cnt:]) - if comp_cfg['patches']: - self.cfg.update('patches', comp_cfg['patches']) + comp_patches = comp_cfg.get_ref('patches') + if comp_patches: + self.cfg.update('patches', comp_patches) self.comp_cfgs.append(comp_cfg) self.cfg.update('checksums', checksums_patches) - self.cfg.enable_templating = True - # restore general sanity checks if using component-specific sanity checks if self.cfg['sanity_check_components'] or self.cfg['sanity_check_all_components']: self.cfg['sanity_check_paths'] = self.backup_sanity_paths self.cfg['sanity_check_commands'] = self.backup_sanity_cmds + self.cfg.enable_templating = True + def check_checksums(self): """ Check whether a SHA256 checksum is available for all sources & patches (incl. extensions). From 46e847c8d0c2cd8b0fbf80024d76e42114557371 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 19 Dec 2024 10:33:22 +0100 Subject: [PATCH 4/5] temporarily disable expectation that all template value can be resolved in Bundle constructor --- easybuild/easyblocks/generic/bundle.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 1c41ab9f82..196b0e5f8b 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -160,14 +160,18 @@ def __init__(self, *args, **kwargs): for key in comp_specs: comp_cfg[key] = comp_specs[key] - # enable resolving of templates for component-specific EasyConfig instance + # enable resolving of templates for component-specific EasyConfig instance, + # but don't require that all template values can be resolved at this point; + # this is important to ensure that template values like %(name)s and %(version)s + # are correctly resolved with the component name/version before values are copied over to self.cfg comp_cfg.enable_templating = True + comp_cfg.expect_resolved_template_values = False # 'sources' is strictly required - comp_sources = comp_cfg.get_ref('sources') + comp_sources = comp_cfg['sources'] if comp_sources: # If per-component source URLs are provided, attach them directly to the relevant sources - comp_source_urls = comp_cfg.get_ref('source_urls') + comp_source_urls = comp_cfg['source_urls'] if comp_source_urls: for source in comp_sources: if isinstance(source, str): @@ -187,7 +191,7 @@ def __init__(self, *args, **kwargs): else: raise EasyBuildError("No sources specification for component %s v%s", comp_name, comp_version) - comp_checksums = comp_cfg.get_ref('checksums') + comp_checksums = comp_cfg['checksums'] if comp_checksums: src_cnt = len(comp_sources) @@ -197,10 +201,12 @@ def __init__(self, *args, **kwargs): # add per-component checksums for patches to list of checksums for patches checksums_patches.extend(comp_checksums[src_cnt:]) - comp_patches = comp_cfg.get_ref('patches') + comp_patches = comp_cfg['patches'] if comp_patches: self.cfg.update('patches', comp_patches) + comp_cfg.expect_resolved_template_values = True + self.comp_cfgs.append(comp_cfg) self.cfg.update('checksums', checksums_patches) @@ -274,6 +280,7 @@ def install_step(self): comp.src = [] # find match entries in self.src for this component + comp.cfg.expect_resolved_template_values = False for source in comp.cfg['sources']: if isinstance(source, str): comp_src_fn = source @@ -297,6 +304,7 @@ def install_step(self): # location of first unpacked source is used to determine where to apply patch(es) comp.src[-1]['finalpath'] = comp.cfg['start_dir'] + comp.cfg.expect_resolved_template_values = True # check if sanity checks are enabled for the component if self.cfg['sanity_check_all_components'] or comp.cfg['name'] in self.cfg['sanity_check_components']: From 346ecb46ae30e331b0c645400ac27365a607867f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 19 Dec 2024 10:34:01 +0100 Subject: [PATCH 5/5] be more careful when getting value for 'buildcmd' custom easyconfig parameter in PythonPackage easyblock, to avoid error due to '%(python)s' template not being resolvable --- easybuild/easyblocks/generic/pythonpackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 921b6e3ff8..855a0c533e 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -818,7 +818,7 @@ def configure_step(self): def build_step(self): """Build Python package using setup.py""" - build_cmd = self.cfg['buildcmd'] + build_cmd = self.cfg.get_ref('buildcmd') if self.use_setup_py: if get_software_root('CMake'):