added test cases and rez env flag

Signed-off-by: Ben Andersen <ben@isohedron.com.au>

src/rez/cli/env.py

@@ -116,6 +116,10 @@ def setup_parser(parser, completions=False):
  parser.add_argument(
  "--no-pkg-cache", action="store_true",
  help="Disable package caching")
+ parser.add_argument(
+ "--pkg-cache-sync", action="store_true",
+ help="Disable asynchronous package caching. "
+ "Process will block until packages are cached.")
  parser.add_argument(
  "--pre-command", type=str, help=SUPPRESS)
  PKG_action = parser.add_argument(
@@ -212,7 +216,8 @@ def command(opts, parser, extra_arg_groups=None):
  caching=(not opts.no_cache),
  suppress_passive=opts.no_passive,
  print_stats=opts.stats,
- package_caching=(not opts.no_pkg_cache)
+ package_caching=(not opts.no_pkg_cache),
+ package_cache_async=(not opts.pkg_cache_sync),
  )
 
  success = (context.status == ResolverStatus.solved)

src/rez/package_cache.py

@@ -366,7 +366,7 @@ def remove_variant(self, variant):
 
  return self.VARIANT_REMOVED
 
- def add_variants(self, variants, _async=True):
+ def add_variants(self, variants, package_cache_async=True):
  """Update the package cache by adding some or all of the given variants.
 
  This method is called when a context is created or sourced. Variants
@@ -460,18 +460,14 @@ def add_variants(self, variants, _async=True):
  else:
  out_target = devnull
 
- func = subprocess.Popen
-
- # use subprocess.call blocks where subprocess.Popen doesn't
- if not _async:
- func = subprocess.call
-
- func(
+ process = subprocess.Popen(
  args,
  stdout=out_target,
  stderr=out_target,
  **kwargs
  )
+ if not package_cache_async:
+ process.wait()
 
  except Exception as e:
  print_warning(

src/rez/resolved_context.py

@@ -167,7 +167,7 @@ def __init__(self, package_requests, verbosity=0, timestamp=None,
  package_filter=None, package_orderers=None, max_fails=-1,
  add_implicit_packages=True, time_limit=-1, callback=None,
  package_load_callback=None, buf=None, suppress_passive=False,
- print_stats=False, package_caching=None):
+ print_stats=False, package_caching=None, package_cache_async=None):
  """Perform a package resolve, and store the result.
 
  Args:
@@ -205,6 +205,8 @@ def __init__(self, package_requests, verbosity=0, timestamp=None,
  package_caching (bool|None): If True, apply package caching settings
  as per the config. If None, enable as determined by config
  setting :data:`package_cache_during_build`.
+ package_cache_async (Optional[bool]): If True, cache packages asynchronously.
+ If None, use the config setting :data:`package_cache_async`
  """
  self.load_path = None
 
@@ -249,6 +251,10 @@ def __init__(self, package_requests, verbosity=0, timestamp=None,
 
  self.package_caching = package_caching
 
+ if package_cache_async is None:
+ package_cache_async = config.package_cache_async
+ self.package_cache_async = package_cache_async
+
  # patch settings
  self.default_patch_lock = PatchLock.no_lock
  self.patch_locks = {}
@@ -1846,7 +1852,7 @@ def _update_package_cache(self):
  if pkgcache:
  pkgcache.add_variants(
  self.resolved_packages,
- config.package_cache_async
+ self.package_cache_async
  )
 
  @classmethod

src/rez/rezconfig.py

@@ -279,7 +279,8 @@
 package_cache_during_build = False
 
 # Enable package caching to run asynchronously during a resolve.
-package_cache_async = True 
+# If this is false, a resolve will block until all packages are cached.
+package_cache_async = True
 
 # Allow caching of local packages. You would only want to set this True for
 # testing purposes.
@@ -316,7 +317,7 @@
 # This is useful as Platform.os might show different
 # values depending on the availability of ``lsb-release`` on the system.
 # The map supports regular expression, e.g. to keep versions.
-# 
+#
 # .. note::
 # The following examples are not necessarily recommendations.
 #
@@ -1122,7 +1123,7 @@
 
 # Enables/disables colorization globally.
 #
-# .. warning:: 
+# .. warning::
 # Turned off for Windows currently as there seems to be a problem with the colorama module.
 #
 # May also set to the string ``force``, which will make rez output color styling

src/rez/tests/test_package_cache.py

@@ -134,7 +134,6 @@ def test_caching_on_resolve(self):
 
  # Creating the context will asynchronously add variants to the cache
  # in a separate proc.
- #
  c = ResolvedContext([
  "timestamped-1.2.0",
  "pyfoo-3.1.0" # won't cache, see earlier test
@@ -144,14 +143,64 @@ def test_caching_on_resolve(self):
 
  # Retry 50 times with 0.1 sec interval, 5 secs is more than enough for
  # the very small variant to be copied to cache.
- #
  cached_root = None
+ resolve_not_always_cached = False
  for _ in range(50):
- time.sleep(0.1)
  cached_root = pkgcache.get_cached_root(variant)
  if cached_root:
  break
 
+ resolve_not_always_cached = True
+ time.sleep(0.1)
+
+ self.assertNotEqual(cached_root, None)
+
+ # Test that the package is not immediately cached, since it is asynchronous
+ # WARNING: This is dangerous since it does open the test to a race condition and
+ # will fail if the cache happens faster than the resolve.
+ self.assertNotEqual(resolve_not_always_cached, False)
+
+ expected_payload_file = os.path.join(cached_root, "stuff.txt")
+ self.assertTrue(os.path.exists(expected_payload_file))
+
+ # check that refs to root point to cache location in rex code
+ for ref in ("resolve.timestamped.root", "'{resolve.timestamped.root}'"):
+ proc = c.execute_rex_code(
+ code="info(%s)" % ref,
+ stdout=subprocess.PIPE,
+ universal_newlines=True
+ )
+
+ out, _ = proc.communicate()
+ root = out.strip()
+
+ self.assertEqual(
+ root, cached_root,
+ "Reference %r should resolve to %s, but resolves to %s"
+ % (ref, cached_root, root)
+ )
+
+ @install_dependent()
+ def test_caching_on_resolve_synchronous(self):
+ """Test that cache is updated as expected on
+ resolved env using syncrhonous package caching."""
+ pkgcache = self._pkgcache()
+
+ with restore_os_environ():
+ # set config settings into env so rez-pkg-cache proc sees them
+ os.environ.update(self.get_settings_env())
+
+ # Creating the context will synchronously add variants to the cache
+ c = ResolvedContext([
+ "timestamped-1.2.0",
+ "pyfoo-3.1.0",
+ ],
+ package_cache_async=False,
+ )
+
+ variant = c.get_resolved_package("timestamped")
+ # The first time we try to access it will be cached, because the cache is blocking
+ cached_root = pkgcache.get_cached_root(variant)
  self.assertNotEqual(cached_root, None)
 
  expected_payload_file = os.path.join(cached_root, "stuff.txt")