From 8cd901f504425013f0624b866ccad000f492a4d6 Mon Sep 17 00:00:00 2001 From: Chris Heisterkamp Date: Fri, 8 Jun 2018 12:12:14 -0700 Subject: [PATCH] Separate the resolution cache and repository cache in Ivy --- src/python/pants/backend/jvm/ivy_utils.py | 40 ++++++++++--------- .../pants/backend/jvm/tasks/ivy_resolve.py | 4 +- .../pants/backend/jvm/tasks/ivy_task_mixin.py | 17 +++++++- src/python/pants/ivy/bootstrapper.py | 4 +- src/python/pants/ivy/ivy.py | 24 +++++------ src/python/pants/ivy/ivy_subsystem.py | 21 +++++++++- .../backend/jvm/tasks/test_ivy_utils.py | 7 +++- .../pants_test/ivy/test_bootstrapper.py | 4 +- 8 files changed, 79 insertions(+), 42 deletions(-) diff --git a/src/python/pants/backend/jvm/ivy_utils.py b/src/python/pants/backend/jvm/ivy_utils.py index d3aca655d17..b30fde61fa9 100644 --- a/src/python/pants/backend/jvm/ivy_utils.py +++ b/src/python/pants/backend/jvm/ivy_utils.py @@ -39,15 +39,16 @@ class IvyResolutionStep(object): # NB(nh): This class is the base class for the ivy resolve and fetch steps. # It also specifies the abstract methods that define the components of resolution steps. - def __init__(self, confs, hash_name, pinned_artifacts, soft_excludes, ivy_cache_dir, - global_ivy_workdir): + def __init__(self, confs, hash_name, pinned_artifacts, soft_excludes, ivy_resolution_cache_dir, + ivy_repository_cache_dir, global_ivy_workdir): """ :param confs: A tuple of string ivy confs to resolve for. :param hash_name: A unique string name for this resolve. :param pinned_artifacts: A tuple of "artifact-alikes" to force the versions of. :param soft_excludes: A flag marking whether to pass excludes to Ivy or to apply them after the fact. - :param ivy_cache_dir: The cache directory used by Ivy for this resolution step. + :param ivy_repository_cache_dir: The cache directory used by Ivy for repository cache data. + :param ivy_resolution_cache_dir: The cache directory used by Ivy for resolution cache data. :param global_ivy_workdir: The workdir that all ivy outputs live in. """ @@ -56,7 +57,8 @@ def __init__(self, confs, hash_name, pinned_artifacts, soft_excludes, ivy_cache_ self.pinned_artifacts = pinned_artifacts self.soft_excludes = soft_excludes - self.ivy_cache_dir = ivy_cache_dir + self.ivy_repository_cache_dir = ivy_repository_cache_dir + self.ivy_resolution_cache_dir = ivy_resolution_cache_dir self.global_ivy_workdir = global_ivy_workdir self.workdir_reports_by_conf = {c: self.resolve_report_path(c) for c in confs} @@ -109,7 +111,7 @@ def resolve_report_path(self, conf): def _construct_and_load_symlink_map(self): artifact_paths, symlink_map = IvyUtils.construct_and_load_symlink_map( self.symlink_dir, - self.ivy_cache_dir, + self.ivy_repository_cache_dir, self.ivy_cache_classpath_filename, self.symlink_classpath_filename) return artifact_paths, symlink_map @@ -122,7 +124,7 @@ def _call_ivy(self, executor, extra_args, ivyxml, jvm_options, hash_name_for_rep jvm_options, self.workdir_reports_by_conf, self.confs, - self.ivy_cache_dir, + self.ivy_resolution_cache_dir, self.ivy_cache_classpath_filename, hash_name_for_report, workunit_factory, @@ -726,7 +728,7 @@ def _load_classpath_from_cachepath(path): @classmethod def do_resolve(cls, executor, extra_args, ivyxml, jvm_options, workdir_report_paths_by_conf, - confs, ivy_cache_dir, ivy_cache_classpath_filename, resolve_hash_name, + confs, ivy_resolution_cache_dir, ivy_cache_classpath_filename, resolve_hash_name, workunit_factory, workunit_name): """Execute Ivy with the given ivy.xml and copies all relevant files into the workdir. @@ -764,15 +766,15 @@ def do_resolve(cls, executor, extra_args, ivyxml, jvm_options, workdir_report_pa raise cls.IvyError('Ivy failed to create classpath file at {}' .format(raw_target_classpath_file_tmp)) - cls._copy_ivy_reports(workdir_report_paths_by_conf, confs, ivy_cache_dir, resolve_hash_name) + cls._copy_ivy_reports(workdir_report_paths_by_conf, confs, ivy_resolution_cache_dir, resolve_hash_name) logger.debug('Moved ivy classfile file to {dest}' .format(dest=ivy_cache_classpath_filename)) @classmethod - def _copy_ivy_reports(cls, workdir_report_paths_by_conf, confs, ivy_cache_dir, resolve_hash_name): + def _copy_ivy_reports(cls, workdir_report_paths_by_conf, confs, ivy_resolution_cache_dir, resolve_hash_name): for conf in confs: - ivy_cache_report_path = IvyUtils.xml_report_path(ivy_cache_dir, resolve_hash_name, + ivy_cache_report_path = IvyUtils.xml_report_path(ivy_resolution_cache_dir, resolve_hash_name, conf) workdir_report_path = workdir_report_paths_by_conf[conf] try: @@ -808,7 +810,7 @@ def _exec_ivy(cls, ivy, confs, ivyxml, args, jvm_options, executor, raise IvyUtils.IvyError(e) @classmethod - def construct_and_load_symlink_map(cls, symlink_dir, ivy_cache_dir, + def construct_and_load_symlink_map(cls, symlink_dir, ivy_repository_cache_dir, ivy_cache_classpath_filename, symlink_classpath_filename): # Make our actual classpath be symlinks, so that the paths are uniform across systems. # Note that we must do this even if we read the raw_target_classpath_file from the artifact @@ -818,7 +820,7 @@ def construct_and_load_symlink_map(cls, symlink_dir, ivy_cache_dir, # in artifact-cached analysis files are consistent across systems. # Note that we have one global, well-known symlink dir, again so that paths are # consistent across builds. - symlink_map = cls._symlink_cachepath(ivy_cache_dir, + symlink_map = cls._symlink_cachepath(ivy_repository_cache_dir, ivy_cache_classpath_filename, symlink_dir, symlink_classpath_filename) @@ -826,18 +828,18 @@ def construct_and_load_symlink_map(cls, symlink_dir, ivy_cache_dir, return classpath, symlink_map @classmethod - def _symlink_cachepath(cls, ivy_cache_dir, inpath, symlink_dir, outpath): - """Symlinks all paths listed in inpath that are under ivy_cache_dir into symlink_dir. + def _symlink_cachepath(cls, ivy_repository_cache_dir, inpath, symlink_dir, outpath): + """Symlinks all paths listed in inpath that are under ivy_repository_cache_dir into symlink_dir. If there is an existing symlink for a file under inpath, it is used rather than creating a new symlink. Preserves all other paths. Writes the resulting paths to outpath. Returns a map of path -> symlink to that path. """ safe_mkdir(symlink_dir) - # The ivy_cache_dir might itself be a symlink. In this case, ivy may return paths that + # The ivy_repository_cache_dir might itself be a symlink. In this case, ivy may return paths that # reference the realpath of the .jar file after it is resolved in the cache dir. To handle # this case, add both the symlink'ed path and the realpath to the jar to the symlink map. - real_ivy_cache_dir = os.path.realpath(ivy_cache_dir) + real_ivy_cache_dir = os.path.realpath(ivy_repository_cache_dir) symlink_map = OrderedDict() inpaths = cls._load_classpath_from_cachepath(inpath) @@ -870,7 +872,7 @@ def _symlink_cachepath(cls, ivy_cache_dir, inpath, symlink_dir, outpath): return dict(symlink_map) @classmethod - def xml_report_path(cls, cache_dir, resolve_hash_name, conf): + def xml_report_path(cls, resolution_cache_dir, resolve_hash_name, conf): """The path to the xml report ivy creates after a retrieve. :API: public @@ -882,8 +884,8 @@ def xml_report_path(cls, cache_dir, resolve_hash_name, conf): :returns: The report path. :rtype: string """ - return os.path.join(cache_dir, '{}-{}-{}.xml'.format(IvyUtils.INTERNAL_ORG_NAME, - resolve_hash_name, conf)) + return os.path.join(resolution_cache_dir, '{}-{}-{}.xml'.format(IvyUtils.INTERNAL_ORG_NAME, + resolve_hash_name, conf)) @classmethod def parse_xml_report(cls, conf, path): diff --git a/src/python/pants/backend/jvm/tasks/ivy_resolve.py b/src/python/pants/backend/jvm/tasks/ivy_resolve.py index 0588393f9a9..6fa3e54759e 100644 --- a/src/python/pants/backend/jvm/tasks/ivy_resolve.py +++ b/src/python/pants/backend/jvm/tasks/ivy_resolve.py @@ -147,7 +147,7 @@ def make_empty_report(report, organisation, module, conf): report = None org = IvyUtils.INTERNAL_ORG_NAME name = result.resolve_hash_name - xsl = os.path.join(self.ivy_cache_dir, 'ivy-report.xsl') + xsl = os.path.join(self.ivy_resolution_cache_dir, 'ivy-report.xsl') # Xalan needs this dir to exist - ensure that, but do no more - we have no clue where this # points. @@ -180,7 +180,7 @@ def make_empty_report(report, organisation, module, conf): css = os.path.join(self._outdir, 'ivy-report.css') if os.path.exists(css): os.unlink(css) - shutil.copy(os.path.join(self.ivy_cache_dir, 'ivy-report.css'), self._outdir) + shutil.copy(os.path.join(self.ivy_resolution_cache_dir, 'ivy-report.css'), self._outdir) if self._open and report: try: diff --git a/src/python/pants/backend/jvm/tasks/ivy_task_mixin.py b/src/python/pants/backend/jvm/tasks/ivy_task_mixin.py index 21c38b895e8..60672c48b8b 100644 --- a/src/python/pants/backend/jvm/tasks/ivy_task_mixin.py +++ b/src/python/pants/backend/jvm/tasks/ivy_task_mixin.py @@ -13,6 +13,7 @@ from pants.backend.jvm.subsystems.jar_dependency_management import JarDependencyManagement from pants.backend.jvm.targets.jar_library import JarLibrary from pants.backend.jvm.targets.jvm_target import JvmTarget +from pants.base.deprecated import deprecated from pants.base.exceptions import TaskError from pants.base.fingerprint_strategy import FingerprintStrategy from pants.invalidation.cache_manager import VersionedTargetSet @@ -101,6 +102,8 @@ def implementation_version(cls): return super(IvyTaskMixin, cls).implementation_version() + [('IvyTaskMixin', 4)] @memoized_property + @deprecated(removal_version='1.10.0.dev0', + hint_message='Use ivy_repository_cache_dir or ivy_resolution_cache_dir instead.') def ivy_cache_dir(self): """The path of the ivy cache dir used for resolves. @@ -111,6 +114,14 @@ def ivy_cache_dir(self): # TODO(John Sirois): Fixup the IvySubsystem to encapsulate its properties. return IvySubsystem.global_instance().get_options().cache_dir + @memoized_property + def ivy_repository_cache_dir(self): + return IvySubsystem.global_instance().repository_cache_dir() + + @memoized_property + def ivy_resolution_cache_dir(self): + return IvySubsystem.global_instance().resolution_cache_dir() + def resolve(self, executor, targets, classpath_products, confs=None, extra_args=None, invalidate_dependents=False): """Resolves external classpath products (typically jars) for the given targets. @@ -237,13 +248,15 @@ def _ivy_resolve(self, resolve_hash_name, pinned_artifacts, self.get_options().soft_excludes, - self.ivy_cache_dir, + self.ivy_resolution_cache_dir, + self.ivy_repository_cache_dir, global_ivy_workdir) resolve = IvyResolveStep(confs, resolve_hash_name, pinned_artifacts, self.get_options().soft_excludes, - self.ivy_cache_dir, + self.ivy_resolution_cache_dir, + self.ivy_repository_cache_dir, global_ivy_workdir) return self._perform_resolution(fetch, resolve, executor, extra_args, invalidation_check, diff --git a/src/python/pants/ivy/bootstrapper.py b/src/python/pants/ivy/bootstrapper.py index 19a70ff74a0..fed4c4498fb 100644 --- a/src/python/pants/ivy/bootstrapper.py +++ b/src/python/pants/ivy/bootstrapper.py @@ -87,7 +87,7 @@ def ivy(self, bootstrap_workunit_factory=None): """ return Ivy(self._get_classpath(bootstrap_workunit_factory), ivy_settings=self._ivy_subsystem.get_options().ivy_settings, - ivy_cache_dir=self._ivy_subsystem.get_options().cache_dir, + ivy_resolution_cache_dir=self._ivy_subsystem.resolution_cache_dir(), extra_jvm_options=self._ivy_subsystem.extra_jvm_options()) def _get_classpath(self, workunit_factory): @@ -159,5 +159,5 @@ def _bootstrap_ivy(self, bootstrap_jar_path): return Ivy(bootstrap_jar_path, ivy_settings=options.bootstrap_ivy_settings or options.ivy_settings, - ivy_cache_dir=options.cache_dir, + ivy_resolution_cache_dir=self._ivy_subsystem.resolution_cache_dir(), extra_jvm_options=self._ivy_subsystem.extra_jvm_options()) diff --git a/src/python/pants/ivy/ivy.py b/src/python/pants/ivy/ivy.py index a71b1c6f8d4..e5a4805892c 100644 --- a/src/python/pants/ivy/ivy.py +++ b/src/python/pants/ivy/ivy.py @@ -28,11 +28,11 @@ class Ivy(object): class Error(Exception): """Indicates an error executing an ivy command.""" - def __init__(self, classpath, ivy_settings=None, ivy_cache_dir=None, extra_jvm_options=None): + def __init__(self, classpath, ivy_settings=None, ivy_resolution_cache_dir=None, extra_jvm_options=None): """Configures an ivy wrapper for the ivy distribution at the given classpath. :param ivy_settings: path to find settings.xml file - :param ivy_cache_dir: path to store downloaded ivy artifacts + :param ivy_resolution_cache_dir: path to store downloaded ivy artifacts :param extra_jvm_options: list of strings to add to command line when invoking Ivy """ self._classpath = maybe_list(classpath) @@ -41,14 +41,14 @@ def __init__(self, classpath, ivy_settings=None, ivy_cache_dir=None, extra_jvm_o raise ValueError('ivy_settings must be a string, given {} of type {}'.format( self._ivy_settings, type(self._ivy_settings))) - self._ivy_cache_dir = ivy_cache_dir - if not isinstance(self._ivy_cache_dir, string_types): - raise ValueError('ivy_cache_dir must be a string, given {} of type {}'.format( - self._ivy_cache_dir, type(self._ivy_cache_dir))) + self._ivy_resolution_cache_dir = ivy_resolution_cache_dir + if not isinstance(self._ivy_resolution_cache_dir, string_types): + raise ValueError('ivy_resolution_cache_dir must be a string, given {} of type {}'.format( + self._ivy_resolution_cache_dir, type(self._ivy_resolution_cache_dir))) self._extra_jvm_options = extra_jvm_options or [] self._lock = OwnerPrintingInterProcessFileLock( - os.path.join(self._ivy_cache_dir, 'pants_ivy.file_lock')) + os.path.join(self._ivy_resolution_cache_dir, 'pants_ivy.file_lock')) @property def ivy_settings(self): @@ -60,14 +60,14 @@ def ivy_settings(self): return self._ivy_settings @property - def ivy_cache_dir(self): + def ivy_resolution_cache_dir(self): """Returns the ivy cache dir used by this `Ivy` instance.""" - return self._ivy_cache_dir + return self._ivy_resolution_cache_dir @property @contextmanager def resolution_lock(self): - safe_mkdir(self._ivy_cache_dir) + safe_mkdir(self._ivy_resolution_cache_dir) with self._lock: yield @@ -100,12 +100,12 @@ def runner(self, jvm_options=None, args=None, executor=None): args = ['-settings', self._ivy_settings] + args options = list(jvm_options) if jvm_options else [] - if self._ivy_cache_dir and '-cache' not in args: + if self._ivy_resolution_cache_dir and '-cache' not in args: # TODO(John Sirois): Currently this is a magic property to support hand-crafted in # ivysettings.xml. Ideally we'd support either simple -caches or these hand-crafted cases # instead of just hand-crafted. Clean this up by taking over ivysettings.xml and generating # it from BUILD constructs. - options += ['-Divy.cache.dir={}'.format(self._ivy_cache_dir)] + options += ['-Divy.cache.dir={}'.format(self._ivy_resolution_cache_dir)] options += self._extra_jvm_options executor = executor or SubprocessExecutor(DistributionLocator.cached()) diff --git a/src/python/pants/ivy/ivy_subsystem.py b/src/python/pants/ivy/ivy_subsystem.py index 3b466269b79..cbd2d216bcd 100644 --- a/src/python/pants/ivy/ivy_subsystem.py +++ b/src/python/pants/ivy/ivy_subsystem.py @@ -39,7 +39,14 @@ def register_options(cls, register): register('--ivy-profile', advanced=True, default=cls._DEFAULT_VERSION, help='The version of ivy to fetch.') register('--cache-dir', advanced=True, default=os.path.expanduser('~/.ivy2/pants'), - help='Directory to store artifacts retrieved by Ivy.') + help='The default directory used for both the Ivy resolution and repository caches.' + 'If you want to isolate the resolution cache from the repository cache, we ' + 'recommend setting both the --resolution-cache-dir and --repository-cache-dir ' + 'instead of using --cache-dir') + register('--resolution-cache-dir', advanced=True, + help='Directory to store Ivy resolution artifacts.') + register('--repository-cache-dir', advanced=True, + help='Directory to store Ivy repository artifacts.') register('--ivy-settings', advanced=True, help='Location of XML configuration file for Ivy settings.') register('--bootstrap-ivy-settings', advanced=True, @@ -93,3 +100,15 @@ def extra_jvm_options(self): def _parse_proxy_string(self, proxy_string): parse_result = urllib.parse.urlparse(proxy_string) return parse_result.hostname, parse_result.port + + def resolution_cache_dir(self): + if self.get_options().resolution_cache_dir: + return self.get_options().resolution_cache_dir + else: + return self.get_options().cache_dir + + def repository_cache_dir(self): + if self.get_options().repository_cache_dir: + return self.get_options().repository_cache_dir + else: + return self.get_options().cache_dir diff --git a/tests/python/pants_test/backend/jvm/tasks/test_ivy_utils.py b/tests/python/pants_test/backend/jvm/tasks/test_ivy_utils.py index ad31a4b86ff..e4d6df8c2bb 100644 --- a/tests/python/pants_test/backend/jvm/tasks/test_ivy_utils.py +++ b/tests/python/pants_test/backend/jvm/tasks/test_ivy_utils.py @@ -593,7 +593,8 @@ def test_if_not_all_symlinked_files_exist_after_successful_resolve_fail(self): 'hash_name', None, False, - 'cache_dir', + 'resolution_cache_dir', + 'repository_cache_dir', 'workdir') # Stub resolving and creating the result, returning one missing artifacts. @@ -608,7 +609,9 @@ def test_if_not_all_symlinked_files_exist_after_successful_fetch_fail(self): 'hash_name', False, None, - 'ivy_cache_dir', 'global_ivy_workdir') + 'ivy_resolution_cache_dir', + 'ivy_repository_cache_dir', + 'global_ivy_workdir') # Stub resolving and creating the result, returning one missing artifacts. fetch._do_fetch = do_nothing diff --git a/tests/python/pants_test/ivy/test_bootstrapper.py b/tests/python/pants_test/ivy/test_bootstrapper.py index 46bad436575..d3bada08d16 100644 --- a/tests/python/pants_test/ivy/test_bootstrapper.py +++ b/tests/python/pants_test/ivy/test_bootstrapper.py @@ -22,7 +22,7 @@ def test_simple(self): ivy_subsystem = IvySubsystem.global_instance() bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem) ivy = bootstrapper.ivy() - self.assertIsNotNone(ivy.ivy_cache_dir) + self.assertIsNotNone(ivy.ivy_resolution_cache_dir) self.assertIsNone(ivy.ivy_settings) bootstrap_jar_path = os.path.join(ivy_subsystem.get_options().pants_bootstrapdir, 'tools', 'jvm', 'ivy', 'bootstrap.jar') @@ -36,5 +36,5 @@ def test_reset(self): def test_default_ivy(self): ivy = Bootstrapper.default_ivy() - self.assertIsNotNone(ivy.ivy_cache_dir) + self.assertIsNotNone(ivy.ivy_resolution_cache_dir) self.assertIsNone(ivy.ivy_settings)