Merge pull request #2313 from boegel/force_download_ignore_checksums

add support for --force-download and --ignore-checksums

easybuild/framework/easyblock.py

@@ -510,9 +510,11 @@ def fetch_extension_sources(self, skip_checksums=False):
  if checksums:
  fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0)
  if verify_checksum(src_fn, fn_checksum):
- self.log.info('Checksum for ext source %s verified', fn)
+ self.log.info('Checksum for extension source %s verified', fn)
+ elif build_option('ignore_checksums'):
+ print_warning("Ignoring failing checksum verification for %s" % fn)
  else:
- raise EasyBuildError('Checksum for ext source %s failed', fn)
+ raise EasyBuildError('Checksum verification for extension source %s failed', fn)
 
  ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True)
  if ext_patches:
@@ -533,6 +535,8 @@ def fetch_extension_sources(self, skip_checksums=False):
  checksum = self.get_checksum_for(checksums[1:], filename=patch, index=idx)
  if verify_checksum(patch, checksum):
  self.log.info('Checksum for extension patch %s verified', patch)
+ elif build_option('ignore_checksums'):
+ print_warning("Ignoring failing checksum verification for %s" % patch)
  else:
  raise EasyBuildError('Checksum for extension patch %s failed', patch)
  else:
@@ -562,6 +566,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
  :param urls: list of source URLs where this file may be available
  :param download_filename: filename with which the file should be downloaded, and then renamed to <filename>
  """
+ res = None
  srcpaths = source_paths()
 
  # should we download or just try and find it?
@@ -583,13 +588,15 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
 
  # only download when it's not there yet
  if os.path.exists(fullpath):
- self.log.info("Found file %s at %s, no need to download it." % (filename, filepath))
- return fullpath
-
- else:
- if download_file(filename, url, fullpath):
+ if build_option('force_download'):
+ print_warning("Found file %s at %s, but re-downloading it anyway..." % (filename, filepath))
+ else:
+ self.log.info("Found file %s at %s, no need to download it", filename, filepath)
  return fullpath
 
+ if download_file(filename, url, fullpath):
+ return fullpath
+
  except IOError, err:
  raise EasyBuildError("Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err)
 
@@ -636,13 +643,17 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
 
  for fp in fullpaths:
  if os.path.isfile(fp):
- self.log.info("Found file %s at %s" % (filename, fp))
+ self.log.info("Found file %s at %s", filename, fp)
  foundfile = os.path.abspath(fp)
  break # no need to try further
  else:
  failedpaths.append(fp)
 
  if foundfile:
+ if build_option('force_download'):
+ print_warning("Found file %s at %s, but re-downloading it anyway..." % (filename, foundfile))
+ foundfile = None
+
  break # no need to try other source paths
 
  if foundfile:
@@ -1635,10 +1646,12 @@ def checksum_step(self):
  expected_checksum = fil['checksum'] or '(none)'
  self.dry_run_msg("* expected checksum for %s: %s", filename, expected_checksum)
  else:
- if not verify_checksum(fil['path'], fil['checksum']):
- raise EasyBuildError("Checksum verification for %s using %s failed.", fil['path'], fil['checksum'])
- else:
+ if verify_checksum(fil['path'], fil['checksum']):
  self.log.info("Checksum verification for %s using %s passed." % (fil['path'], fil['checksum']))
+ elif build_option('ignore_checksums'):
+ print_warning("Ignoring failing checksum verification for %s" % fil['name'])
+ else:
+ raise EasyBuildError("Checksum verification for %s using %s failed.", fil['path'], fil['checksum'])
 
  def extract_step(self):
  """

easybuild/tools/config.py

@@ -156,8 +156,10 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
  'experimental',
  'fixed_installdir_naming_scheme',
  'force',
+ 'force_download',
  'group_writable_installdir',
  'hidden',
+ 'ignore_checksums',
  'install_latest_eb_release',
  'minimal_toolchains',
  'module_only',

easybuild/tools/filetools.py

@@ -161,14 +161,26 @@ def read_file(path, log_error=True):
  return txt
 
 
-def write_file(path, txt, append=False, forced=False):
- """Write given contents to file at given path (overwrites current file contents!)."""
-
+def write_file(path, txt, append=False, forced=False, backup=False):
+ """
+ Write given contents to file at given path;
+ overwrites current file contents without backup by default!
+
+ :param path: location of file
+ :param txt: contents to write to file
+ :param append: append to existing file rather than overwrite
+ :param forced: force actually writing file in (extended) dry run mode
+ :param backup: back up existing file before overwriting or modifying it
+ """
  # early exit in 'dry run' mode
  if not forced and build_option('extended_dry_run'):
  dry_run_msg("file written: %s" % path, silent=build_option('silent'))
  return
 
+ if backup and os.path.exists(path):
+ backup = back_up_file(path)
+ _log.info("Existing file %s backed up to %s", path, backup)
+
  # note: we can't use try-except-finally, because Python 2.4 doesn't support it as a single block
  try:
  mkdir(os.path.dirname(path), parents=True)
@@ -413,7 +425,7 @@ def download_file(filename, url, path, forced=False):
  # urllib2 does the right thing for http proxy setups, urllib does not!
  url_fd = urllib2.urlopen(url_req, timeout=timeout)
  _log.debug('response code for given url %s: %s' % (url, url_fd.getcode()))
- write_file(path, url_fd.read(), forced=forced)
+ write_file(path, url_fd.read(), forced=forced, backup=True)
  _log.info("Downloaded file %s from url %s to %s" % (filename, url, path))
  downloaded = True
  url_fd.close()
@@ -1184,12 +1196,12 @@ def find_backup_name_candidate(src_file):
 
 def back_up_file(src_file, backup_extension='bak', hidden=False):
  """
- Backs up a file appending a backup extension and a number to it (if there is already an existing backup). Returns
- the name of the backup
+ Backs up a file appending a backup extension and timestamp to it (if there is already an existing backup).
 
  :param src_file: file to be back up
  :param backup_extension: extension to use for the backup file (can be empty or None)
  :param hidden: make backup hidden (leading dot in filename)
+ :return: location of backed up file
  """
  fn_prefix, fn_suffix = '', ''
  if hidden:

easybuild/tools/options.py

@@ -361,6 +361,8 @@ def override_options(self):
  "module files generated by EasyBuild", 'strlist', 'extend', None),
  'fixed-installdir-naming-scheme': ("Use fixed naming scheme for installation directories", None,
  'store_true', False),
+ 'force-download': ("Force re-downloading of sources, even if sources are available already in source path",
+ None, 'store_true', False),
  'group': ("Group to be used for software installations (only verified, not set)", None, 'store', None),
  'group-writable-installdir': ("Enable group write permissions on installation directory after installation",
  None, 'store_true', False),
@@ -369,6 +371,7 @@ def override_options(self):
  "(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None),
  'hide-toolchains': ("Comma separated list of toolchains that you want automatically hidden, "
  "(e.g. --hide-toolchains=GCCcore)", 'strlist', 'extend', None),
+ 'ignore-checksums': ("Ignore failing checksum verification", None, 'store_true', False),
  'ignore-osdeps': ("Ignore any listed OS dependencies", None, 'store_true', False),
  'install-latest-eb-release': ("Install latest known version of easybuild", None, 'store_true', False),
  'max-fail-ratio-adjust-permissions': ("Maximum ratio for failures to allow when adjusting permissions",

test/framework/easyblock.py

@@ -44,7 +44,7 @@
 from easybuild.tools import config
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import get_module_syntax
-from easybuild.tools.filetools import mkdir, read_file, write_file
+from easybuild.tools.filetools import copy_file, mkdir, read_file, remove_file, write_file
 from easybuild.tools.modules import modules_tool
 
 
@@ -888,6 +888,31 @@ def test_obtain_file(self):
  res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir])
  self.assertEqual(res, toy_tarball_path)
 
+ # --force-download results in re-downloading, even if file is already in sourcepath
+ build_options = {
+ 'force_download': True,
+ }
+ init_config(args=["--sourcepath=%s:%s" % (tmpdir, sandbox_sources)], build_options=build_options)
+
+ # clean up toy tarballs in tmpdir, so the one in sourcepath is found
+ remove_file(os.path.join(tmpdir, toy_tarball))
+ remove_file(os.path.join(tmpdir, 't', 'toy', toy_tarball))
+
+ self.mock_stderr(True)
+ self.mock_stdout(True)
+ res = eb.obtain_file(toy_tarball, urls=['file://%s' % tmpdir_subdir])
+ stderr = self.get_stderr()
+ stdout = self.get_stdout()
+ self.mock_stderr(False)
+ self.mock_stdout(False)
+ self.assertEqual(stdout, '')
+ msg = "WARNING: Found file toy-0.0.tar.gz at %s, but re-downloading it anyway..." % toy_tarball_path
+ self.assertEqual(stderr.strip(), msg)
+
+ # toy tarball was indeed re-downloaded to tmpdir
+ self.assertEqual(res, os.path.join(tmpdir, 't', 'toy', toy_tarball))
+ self.assertTrue(os.path.exists(os.path.join(tmpdir, 't', 'toy', toy_tarball)))
+
  # obtain_file yields error for non-existing files
  fn = 'thisisclearlyanonexistingfile'
  error_regex = "Couldn't find file %s anywhere, and downloading it didn't work either" % fn
@@ -1162,6 +1187,62 @@ def test_prepare_step(self):
  self.assertEqual(len(self.modtool.list()), 1)
  self.assertEqual(self.modtool.list()[0]['mod_name'], 'GCC/4.7.2')
 
+ def test_checksum_step(self):
+ """Test checksum step"""
+ testdir = os.path.abspath(os.path.dirname(__file__))
+ toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-1.3.12-test.eb')
+
+ ec = process_easyconfig(toy_ec)[0]
+ eb = get_easyblock_instance(ec)
+ eb.fetch_sources()
+ eb.checksum_step()
+
+ # fiddle with checksum to check whether faulty checksum is catched
+ copy_file(toy_ec, self.test_prefix)
+ toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec))
+ ectxt = read_file(toy_ec)
+ # replace MD5 checksum for toy-0.0.tar.gz
+ ectxt = ectxt.replace('be662daa971a640e40be5c804d9d7d10', '00112233445566778899aabbccddeeff')
+ # replace SHA256 checksums for source of bar extension
+ ectxt = ectxt.replace('f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7', '01234567' * 8)
+ write_file(toy_ec, ectxt)
+
+ ec = process_easyconfig(toy_ec)[0]
+ eb = get_easyblock_instance(ec)
+ eb.fetch_sources()
+ error_msg ="Checksum verification for .*/toy-0.0.tar.gz using .* failed"
+ self.assertErrorRegex(EasyBuildError, error_msg, eb.checksum_step)
+
+ # also check verification of checksums for extensions, which is part of fetch_extension_sources
+ error_msg = "Checksum verification for extension source bar-0.0.tar.gz failed"
+ self.assertErrorRegex(EasyBuildError, error_msg, eb.fetch_extension_sources)
+
+ # if --ignore-checksums is enabled, faulty checksums are reported but otherwise ignored (no error)
+ build_options = {
+ 'ignore_checksums': True,
+ }
+ init_config(build_options=build_options)
+
+ self.mock_stderr(True)
+ self.mock_stdout(True)
+ eb.checksum_step()
+ stderr = self.get_stderr()
+ stdout = self.get_stdout()
+ self.mock_stderr(False)
+ self.mock_stdout(False)
+ self.assertEqual(stdout, '')
+ self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for toy-0.0.tar.gz")
+
+ self.mock_stderr(True)
+ self.mock_stdout(True)
+ eb.fetch_extension_sources()
+ stderr = self.get_stderr()
+ stdout = self.get_stdout()
+ self.mock_stderr(False)
+ self.mock_stdout(False)
+ self.assertEqual(stdout, '')
+ self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz")
+
 
 def suite():
  """ return all the tests in this file """

test/framework/filetools.py

@@ -31,6 +31,7 @@
 @author: Ward Poelmans (Ghent University)
 """
 import datetime
+import glob
 import os
 import re
 import shutil
@@ -250,9 +251,12 @@ def test_download_file(self):
  target_location = os.path.join(self.test_buildpath, 'some', 'subdir', fn)
  # provide local file path as source URL
  test_dir = os.path.abspath(os.path.dirname(__file__))
- source_url = 'file://%s/sandbox/sources/toy/%s' % (test_dir, fn)
+ toy_source_dir = os.path.join(test_dir, 'sandbox', 'sources', 'toy')
+ source_url = 'file://%s/%s' % (toy_source_dir, fn)
  res = ft.download_file(fn, source_url, target_location)
  self.assertEqual(res, target_location, "'download' of local file works")
+ downloads = glob.glob(target_location + '*')
+ self.assertEqual(len(downloads), 1)
 
  # non-existing files result in None return value
  self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None)
@@ -266,11 +270,21 @@ def test_download_file(self):
  # this tests whether proxies are taken into account by download_file
  self.assertEqual(ft.download_file(fn, source_url, target_location), None, "download over broken proxy fails")
 
+ # modify existing download so we can verify re-download
+ ft.write_file(target_location, '')
+
  # restore a working file handler, and retest download of local file
  urllib2.install_opener(urllib2.build_opener(urllib2.FileHandler()))
  res = ft.download_file(fn, source_url, target_location)
  self.assertEqual(res, target_location, "'download' of local file works after removing broken proxy")
 
+ # existing file was re-downloaded, so a backup should have been created of the existing file
+ downloads = glob.glob(target_location + '*')
+ self.assertEqual(len(downloads), 2)
+ backup = [d for d in downloads if os.path.basename(d) != fn][0]
+ self.assertEqual(ft.read_file(backup), '')
+ self.assertEqual(ft.compute_checksum(target_location), ft.compute_checksum(os.path.join(toy_source_dir, fn)))
+
  # make sure specified timeout is parsed correctly (as a float, not a string)
  opts = init_config(args=['--download-timeout=5.3'])
  init_config(build_options={'download_timeout': opts.download_timeout})
@@ -451,6 +465,24 @@ def test_read_write_file(self):
  ft.write_file(fp, txt2, append=True)
  self.assertEqual(ft.read_file(fp), txt+txt2)
 
+ # test backing up of existing file
+ ft.write_file(fp, 'foo', backup=True)
+ self.assertEqual(ft.read_file(fp), 'foo')
+
+ test_files = glob.glob(fp + '*')
+ self.assertEqual(len(test_files), 2)
+ backup1 = [x for x in test_files if os.path.basename(x) != 'test.txt'][0]
+ self.assertEqual(ft.read_file(backup1), txt + txt2)
+
+ ft.write_file(fp, 'bar', append=True, backup=True)
+ self.assertEqual(ft.read_file(fp), 'foobar')
+
+ test_files = glob.glob(fp + '*')
+ self.assertEqual(len(test_files), 3)
+ backup2 = [x for x in test_files if x != backup1 and os.path.basename(x) != 'test.txt'][0]
+ self.assertEqual(ft.read_file(backup1), txt + txt2)
+ self.assertEqual(ft.read_file(backup2), 'foo')
+
  # also test behaviour of write_file under --dry-run
  build_options = {
  'extended_dry_run': True,