Skip to content

Commit

Permalink
Merge pull request #1949 from Flamefire/update_r_config_guess
Browse files Browse the repository at this point in the history 
Update config.guess for all R packages
  • Loading branch information
@boegel
boegel authored May 18, 2020
2 parents c7530aa + 429c797 commit 0cb9337
Show file tree
Hide file tree
Showing 2 changed files with 125 additions and 97 deletions.
168 changes: 102 additions & 66 deletions easybuild/easyblocks/generic/configuremake.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,14 @@
import stat
from datetime import datetime

from easybuild.base import fancylogger
from easybuild.easyblocks import VERSION as EASYBLOCKS_VERSION
from easybuild.framework.easyblock import EasyBlock
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.build_log import print_warning
from easybuild.tools.config import source_paths
from easybuild.tools.config import source_paths, build_option
from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file
from easybuild.tools.filetools import read_file, remove_file, verify_checksum
from easybuild.tools.filetools import read_file, remove_file
from easybuild.tools.run import run_cmd

# string that indicates that a configure script was generated by Autoconf
Expand All @@ -55,7 +56,6 @@
AUTOCONF_GENERATED_MSG = b"Generated by GNU Autoconf"

# download location & SHA256 for config.guess script
# note: if this is updated, don't forget to trash the cached download from generic/Configure/<eb_version>/!
CONFIG_GUESS_VERSION = '2018-08-29'
CONFIG_GUESS_URL_STUB = "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb="
CONFIG_GUESS_COMMIT_ID = "59e2ce0e6b46bb47ef81b68b600ed087e14fdaad"
Expand All @@ -67,6 +67,103 @@
DEFAULT_INSTALL_CMD = 'make install'


def check_config_guess(config_guess):
"""Check timestamp & SHA256 checksum of config.guess script.
:param config_guess: Path to config.guess script to check
:return: Whether the script is valid (matches the version and checksum)
"""
log = fancylogger.getLogger('check_config_guess')

# config.guess includes a "timestamp='...'" indicating the version
config_guess_version = None
version_regex = re.compile("^timestamp='(.*)'", re.M)
res = version_regex.search(read_file(config_guess))
if res:
config_guess_version = res.group(1)

config_guess_checksum = compute_checksum(config_guess, checksum_type=CHECKSUM_TYPE_SHA256)
try:
config_guess_timestamp = datetime.fromtimestamp(os.stat(config_guess).st_mtime).isoformat()
except OSError as err:
log.warning("Failed to determine timestamp of %s: %s", config_guess, err)
config_guess_timestamp = None

log.info("config.guess version: %s (last updated: %s, SHA256 checksum: %s)",
config_guess_version, config_guess_timestamp, config_guess_checksum)

result = True

# check version & SHA256 checksum before declaring victory
if config_guess_version != CONFIG_GUESS_VERSION:
result = False
log.warning("config.guess version at %s does not match expected version: %s vs %s",
config_guess, config_guess_version, CONFIG_GUESS_VERSION)

elif config_guess_checksum != CONFIG_GUESS_SHA256:
result = False
log.warning("SHA256 checksum of config.guess at %s does not match expected checksum: %s vs %s",
config_guess, config_guess_checksum, CONFIG_GUESS_SHA256)

return result


def obtain_config_guess(download_source_path=None, search_source_paths=None):
"""
Locate or download an up-to-date config.guess
:param download_source_path: Path to download config.guess to
:param search_source_paths: Paths to search for config.guess
:return: Path to config.guess or None
"""
log = fancylogger.getLogger('obtain_config_guess')

eb_source_paths = source_paths()

if download_source_path is None:
download_source_path = eb_source_paths[0]
else:
log.deprecated("Specifying custom source path to download config.guess via 'download_source_path'", '5.0')

if search_source_paths is None:
search_source_paths = eb_source_paths
else:
log.deprecated("Specifying custom location to search for updated config.guess via 'search_source_paths'", '5.0')

config_guess = 'config.guess'
sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake')

config_guess_path = None

# check if config.guess has already been downloaded to source path
for path in search_source_paths:
cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess)
if os.path.isfile(cand_config_guess_path) and check_config_guess(cand_config_guess_path):
force_download = build_option('force_download')
if force_download:
print_warning("Found file %s at %s, but re-downloading it anyway..."
% (config_guess, cand_config_guess_path))
else:
config_guess_path = cand_config_guess_path
log.info("Found %s at %s", config_guess, config_guess_path)
break

if not config_guess_path:
cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess)
config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID
if not download_file(config_guess, config_guess_url, cand_config_guess_path):
print_warning("Failed to download recent %s to %s", config_guess, cand_config_guess_path, log=log)
elif not check_config_guess(cand_config_guess_path):
print_warning("Verification failed for file %s, not using it!", cand_config_guess_path, log=log)
remove_file(cand_config_guess_path)
else:
config_guess_path = cand_config_guess_path
adjust_permissions(config_guess_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True)
log.info("Verified %s at %s, using it if required", config_guess, config_guess_path)

return config_guess_path


class ConfigureMake(EasyBlock):
"""
Support for building and installing applications with configure/make/make install
Expand Down Expand Up @@ -106,74 +203,13 @@ def obtain_config_guess(self, download_source_path=None, search_source_paths=Non
:param search_source_paths: Paths to search for config.guess
:return: Path to config.guess or None
"""
eb_source_paths = source_paths()
if download_source_path is None:
download_source_path = eb_source_paths[0]
if search_source_paths is None:
search_source_paths = eb_source_paths

config_guess = 'config.guess'
sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake')

config_guess_path = None

# check if config.guess has already been downloaded to source path
for path in eb_source_paths:
cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess)
if os.path.isfile(cand_config_guess_path):
config_guess_path = cand_config_guess_path
self.log.info("Found recent %s at %s, using it if required", config_guess, config_guess_path)
break

# if not found, try to download it
if config_guess_path is None:
cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess)
config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID
downloaded_path = download_file(config_guess, config_guess_url, cand_config_guess_path)
if downloaded_path is not None:
# verify SHA256 checksum of download to avoid using a corrupted download
if verify_checksum(downloaded_path, CONFIG_GUESS_SHA256):
config_guess_path = downloaded_path
# add execute permissions
adjust_permissions(downloaded_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True)
self.log.info("Downloaded recent %s to %s, using it if required", config_guess, config_guess_path)
else:
self.log.warning("Checksum failed for downloaded file %s, not using it!", downloaded_path)
remove_file(downloaded_path)
else:
self.log.warning("Failed to download recent %s to %s for use with ConfigureMake easyblock (if needed)",
config_guess, cand_config_guess_path)

return config_guess_path
return obtain_config_guess(download_source_path=download_source_path, search_source_paths=search_source_paths)

def check_config_guess(self):
"""Check timestamp & SHA256 checksum of config.guess script."""
# log version, timestamp & SHA256 checksum of config.guess that was found (if any)
if self.config_guess:
# config.guess includes a "timestamp='...'" indicating the version
config_guess_version = None
version_regex = re.compile("^timestamp='(.*)'", re.M)
res = version_regex.search(read_file(self.config_guess))
if res:
config_guess_version = res.group(1)

config_guess_checksum = compute_checksum(self.config_guess, checksum_type=CHECKSUM_TYPE_SHA256)
try:
config_guess_timestamp = datetime.fromtimestamp(os.stat(self.config_guess).st_mtime).isoformat()
except OSError as err:
self.log.warning("Failed to determine timestamp of %s: %s", self.config_guess, err)
config_guess_timestamp = None

self.log.info("config.guess version: %s (last updated: %s, SHA256 checksum: %s)",
config_guess_version, config_guess_timestamp, config_guess_checksum)

if config_guess_version != CONFIG_GUESS_VERSION:
tup = (self.config_guess, config_guess_version, CONFIG_GUESS_VERSION)
print_warning("config.guess version at %s does not match expected version: %s vs %s" % tup)

if config_guess_checksum != CONFIG_GUESS_SHA256:
tup = (self.config_guess, config_guess_checksum, CONFIG_GUESS_SHA256)
print_warning("SHA256 checksum of config.guess at %s does not match expected checksum: %s vs %s" % tup)
check_config_guess(self.config_guess)

def fetch_step(self, *args, **kwargs):
"""Custom fetch step for ConfigureMake so we use an updated config.guess."""
Expand Down
54 changes: 23 additions & 31 deletions easybuild/easyblocks/generic/rpackage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,13 @@
@author: Balazs Hajgato (Vrije Universiteit Brussel)
"""
import os
import shutil

from easybuild.easyblocks.r import EXTS_FILTER_R_PACKAGES, EB_R
from easybuild.easyblocks.generic.configuremake import check_config_guess, obtain_config_guess
from easybuild.framework.easyconfig import CUSTOM
from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import mkdir
from easybuild.tools.filetools import mkdir, copy_file
from easybuild.tools.run import run_cmd, parse_log_for_error


Expand Down Expand Up @@ -132,12 +132,8 @@ def make_cmdline_cmd(self, prefix=None):
else:
prefix = ''

if self.cfg['unpack_sources']:
loc = self.start_dir
elif self.patches:
loc = self.ext_dir
else:
loc = self.ext_src
loc = self.start_dir or self.ext_dir or self.ext_src

cmd = ' '.join([
self.cfg['preinstallopts'],
"R CMD INSTALL",
Expand All @@ -152,23 +148,6 @@ def make_cmdline_cmd(self, prefix=None):
self.log.debug("make_cmdline_cmd returns %s" % cmd)
return cmd, None

def extract_step(self):
"""Source should not be extracted."""

if self.cfg['unpack_sources']:
super(RPackage, self).extract_step()
elif len(self.src) > 1:
raise EasyBuildError("Don't know how to handle R packages with multiple sources.'")
else:
try:
shutil.copy2(self.src[0]['path'], self.builddir)
except OSError as err:
raise EasyBuildError("Failed to copy source to build dir: %s", err)
self.ext_src = self.src[0]['name']

# set final path since it can't be determined from unpacked sources (used for guessing start_dir)
self.src[0]['finalpath'] = self.builddir

def configure_step(self):
"""No configuration for installing R packages."""
pass
Expand All @@ -195,9 +174,23 @@ def install_R_package(self, cmd, inp=None):
else:
self.log.debug("R package %s installed succesfully" % self.name)

def update_config_guess(self, path):
"""Update any config.guess found in specified directory"""
for config_guess_dir in (root for root, _, files in os.walk(path) if 'config.guess' in files):
config_guess = os.path.join(config_guess_dir, 'config.guess')
if not check_config_guess(config_guess):
updated_config_guess = obtain_config_guess()
if updated_config_guess:
self.log.debug("Replacing outdated %s with more recent %s", config_guess, updated_config_guess)
copy_file(updated_config_guess, config_guess)
else:
raise EasyBuildError("Failed to obtain updated config.guess")

def install_step(self):
"""Install procedure for R packages."""

# Update config.guess if the package was extracted
if self.start_dir:
self.update_config_guess(self.start_dir)
cmd, stdin = self.make_cmdline_cmd(prefix=os.path.join(self.installdir, self.cfg['exts_subdir']))
self.install_R_package(cmd, inp=stdin)

Expand All @@ -215,16 +208,15 @@ def run(self):
lib_install_prefix = os.path.join(self.installdir, self.cfg['exts_subdir'])
mkdir(lib_install_prefix, parents=True)

if self.patches:
super(RPackage, self).run(unpack_src=True)
else:
super(RPackage, self).run()

if self.src:
super(RPackage, self).run(unpack_src=True)
self.ext_src = self.src
self.update_config_guess(self.ext_dir)
self.log.debug("Installing R package %s version %s." % (self.name, self.version))
cmd, stdin = self.make_cmdline_cmd(prefix=lib_install_prefix)
else:
if self.patches:
raise EasyBuildError("Cannot patch R package %s as no explicit source is given!", self.name)
self.log.debug("Installing most recent version of R package %s (source not found)." % self.name)
cmd, stdin = self.make_r_cmd(prefix=lib_install_prefix)

Expand Down

0 comments on commit 0cb9337

Please sign in to comment.