From 9d827066cfaf9409a1c57388cc154449ec998afc Mon Sep 17 00:00:00 2001 From: Peter Kriens Date: Thu, 30 Jun 2022 15:09:57 +0200 Subject: [PATCH 1/2] [resolve] Add cache mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cache – Will use a cache file in the workspace cache. If that file is stale relative to the workspace or project or it does not exist, then the bnd(run) file will be resolved and the result is stored in the cache file. Signed-off-by: Peter Kriens --- .../instructions/ResolutionInstructions.java | 5 + .../bnd/help/instructions/package-info.java | 2 +- .../src/biz/aQute/resolve/Bndrun.java | 105 ++++++++++++++++++ .../src/biz/aQute/resolve/package-info.java | 2 +- .../biz/aQute/resolve/RunResolutionTest.java | 44 ++++++++ docs/_instructions/resolve.md | 4 +- 6 files changed, 159 insertions(+), 3 deletions(-) diff --git a/biz.aQute.bndlib/src/aQute/bnd/help/instructions/ResolutionInstructions.java b/biz.aQute.bndlib/src/aQute/bnd/help/instructions/ResolutionInstructions.java index 6a44734d7e..52cd3eed15 100644 --- a/biz.aQute.bndlib/src/aQute/bnd/help/instructions/ResolutionInstructions.java +++ b/biz.aQute.bndlib/src/aQute/bnd/help/instructions/ResolutionInstructions.java @@ -78,6 +78,11 @@ enum ResolveMode { @SyntaxAnnotation(lead = "A resolve will take place before launching when in batch mode (e.g. Gradle) but not in IDE mode (e.g. Eclipse)") batch, + /** + * Run the resolver before launching during batch mode + */ + @SyntaxAnnotation(lead = "Resolve when the runbundles are needed unless there is a cache file that is newer than the bndrun/project & workspace. The cache file has the same name as the project/bndrun file but starts with a '.'") + cache, } @SyntaxAnnotation(lead = "Resolve mode defines when resolving takes place. The default, manual, requires a manual step in bndtools. Auto will resolve on save, and beforelaunch runs the resolver before being launched, batchlaunch is like beforelaunch but only in batch mode", example = "'-resolve manual", pattern = "(manual|auto|beforelaunch|batch)") diff --git a/biz.aQute.bndlib/src/aQute/bnd/help/instructions/package-info.java b/biz.aQute.bndlib/src/aQute/bnd/help/instructions/package-info.java index ae16697ce3..7fb18f56f0 100644 --- a/biz.aQute.bndlib/src/aQute/bnd/help/instructions/package-info.java +++ b/biz.aQute.bndlib/src/aQute/bnd/help/instructions/package-info.java @@ -1,2 +1,2 @@ -@org.osgi.annotation.versioning.Version("1.4.0") +@org.osgi.annotation.versioning.Version("1.5.0") package aQute.bnd.help.instructions; diff --git a/biz.aQute.resolve/src/biz/aQute/resolve/Bndrun.java b/biz.aQute.resolve/src/biz/aQute/resolve/Bndrun.java index c848a53ee0..cf0e5b1820 100644 --- a/biz.aQute.resolve/src/biz/aQute/resolve/Bndrun.java +++ b/biz.aQute.resolve/src/biz/aQute/resolve/Bndrun.java @@ -1,8 +1,10 @@ package biz.aQute.resolve; import java.io.File; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import org.osgi.resource.Requirement; @@ -19,6 +21,7 @@ import aQute.bnd.build.model.conversions.CollectionFormatter; import aQute.bnd.build.model.conversions.Converter; import aQute.bnd.build.model.conversions.HeaderClauseFormatter; +import aQute.bnd.exceptions.Exceptions; import aQute.bnd.header.Parameters; import aQute.bnd.help.Syntax; import aQute.bnd.help.instructions.ResolutionInstructions; @@ -27,6 +30,9 @@ import aQute.bnd.osgi.Processor; import aQute.bnd.osgi.resource.FilterParser; import aQute.bnd.osgi.resource.FilterParser.Expression; +import aQute.lib.io.IO; +import aQute.lib.utf8properties.UTF8Properties; +import aQute.libg.cryptography.SHA1; /** * This is a resolving version of the Run class. The name of this class is known @@ -171,6 +177,9 @@ public Collection getRunbundles() throws Exception { default : break; + case cache : + return cache(); + case batch : if (!gestalt.containsKey(Constants.GESTALT_BATCH) && !super.getRunbundles().isEmpty()) break; @@ -185,4 +194,100 @@ public Collection getRunbundles() throws Exception { return super.getRunbundles(); } + private Collection cache() throws Exception { + File ours = getPropertiesFile(); + File cache = getCacheFile(ours); + File workspace = getWorkspace().getPropertiesFile(); + if (workspace == null) + workspace = ours; + + String force; + if (!cache.isFile()) + force = "no cache file"; + else if (cache.lastModified() <= ours.lastModified()) + force = "cache file is older than our properties"; + else if (cache.lastModified() <= workspace.lastModified()) + force = "cache file is older than our workspace"; + else + force = null; + trace("force = %s", force); + + boolean tryCache = force == null; + + try (Processor p = new Processor()) { + + if (cache.isFile()) + p.setProperties(cache); + + if (tryCache) { + trace("attempting to use cache"); + String runbundles = p.getProperty(Constants.RUNBUNDLES); + Collection containers = parseRunbundles(runbundles); + if (isAllOk(containers)) { + return containers; + } else { + trace("the cached bundles were not ok, will resolve"); + } + } + trace("resolving"); + + IO.delete(cache); + if (cache.isFile()) { + throw new IllegalStateException("cannot delete cache file " + cache); + } + + RunResolution resolved = RunResolution.resolve(this, Collections.emptyList()); + if (!resolved.isOK()) { + throw new IllegalStateException(resolved.report(false)); + } + + String spec = resolved.getRunBundlesAsString(); + + List containers = parseRunbundles(spec); + if (isAllOk(containers)) { + UTF8Properties props = new UTF8Properties(p.getProperties()); + props.setProperty(Constants.RUNBUNDLES, spec); + cache.getParentFile() + .mkdirs(); + props.store(cache); + } + return containers; + } + } + + /** + * Return the file used to cache the resolved solution for the given file + * + * @param file the file to find the cached file for + * @return the cached file + */ + public File getCacheFile(File file) { + try { + String path = file.getAbsolutePath(); + String digest = SHA1.digest(path.getBytes(StandardCharsets.UTF_8)) + .asHex(); + return getWorkspace().getCache("resolved-cache/".concat(digest) + .concat(".resolved")); + } catch (Exception e) { + // not gonna happen + throw Exceptions.duck(e); + } + } + + boolean testIgnoreDownloadErrors = false; + + private boolean isAllOk(Collection containers) { + for (Container c : containers) { + if (c.getError() != null) { + if (testIgnoreDownloadErrors) { + if (c.getError() + .contains("FileNotFoundException")) + continue; + } + return false; + } + } + return true; + } + } diff --git a/biz.aQute.resolve/src/biz/aQute/resolve/package-info.java b/biz.aQute.resolve/src/biz/aQute/resolve/package-info.java index 828c8ce2ad..a0b212e44e 100644 --- a/biz.aQute.resolve/src/biz/aQute/resolve/package-info.java +++ b/biz.aQute.resolve/src/biz/aQute/resolve/package-info.java @@ -1,4 +1,4 @@ -@Version("8.1.0") +@Version("8.2.0") package biz.aQute.resolve; import org.osgi.annotation.versioning.Version; diff --git a/biz.aQute.resolve/test/biz/aQute/resolve/RunResolutionTest.java b/biz.aQute.resolve/test/biz/aQute/resolve/RunResolutionTest.java index eed9436f27..9761ff6188 100644 --- a/biz.aQute.resolve/test/biz/aQute/resolve/RunResolutionTest.java +++ b/biz.aQute.resolve/test/biz/aQute/resolve/RunResolutionTest.java @@ -1,7 +1,9 @@ package biz.aQute.resolve; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertTrue; +import java.io.File; import java.nio.file.Path; import java.util.Collection; import java.util.Collections; @@ -118,6 +120,48 @@ public void testCachingOfResult() throws Exception { // assertThat(runbundles).hasSize(22); } + @Test + public void testResolveCached() throws Exception { + + Bndrun bndrun = Bndrun.createBndrun(workspace, IO.getFile(ws.toFile(), "test.simple/resolve.bndrun")); + File file = bndrun.getPropertiesFile(); + bndrun.testIgnoreDownloadErrors = true; + assertTrue(bndrun.check()); + File cache = bndrun.getCacheFile(file); + + System.out.println("get the embedded list of runbundles"); + bndrun.setProperty("-resolve", "manual"); + Collection manual = bndrun.getRunbundles(); + + System.out.println("remove the embedded list and set mode to 'cache'"); + bndrun.setProperty("-resolve", "cache"); + bndrun.unsetProperty("-runbundles"); + + assertThat(cache).doesNotExist(); + + System.out.println("First time we should resolve & create a cache file"); + Collection cached = bndrun.getRunbundles(); + assertThat(cache).isFile(); + assertThat(cached).containsAll(manual); + assertThat(cached).hasSize(manual.size()); + assertThat(cache.lastModified()).isGreaterThan(file.lastModified()); + + System.out + .println("Second time, the cache file should used, so make it valid but empty "); + long lastModified = cache.lastModified(); + IO.store("-runbundles ", cache); + cache.setLastModified(file.lastModified() + 1000); + cached = bndrun.getRunbundles(); + assertThat(cached).isEmpty(); + + System.out.println("Now make cache invalid, which be ignored"); + IO.store("-runbundles is not a valid file", cache); + cache.setLastModified(file.lastModified() + 1000); + cached = bndrun.getRunbundles(); + assertThat(manual).containsAll(cached); + + } + @Test public void testNotCachingOfResultForOtherResolveOption() throws Exception { Bndrun bndrun = Bndrun.createBndrun(workspace, IO.getFile(tmp.toFile(), "resolver.bndrun")); diff --git a/docs/_instructions/resolve.md b/docs/_instructions/resolve.md index 9e14ea861b..f8d17d024e 100644 --- a/docs/_instructions/resolve.md +++ b/docs/_instructions/resolve.md @@ -1,7 +1,7 @@ --- layout: default class: Workspace -title: -resolve (manual|auto|beforelaunch|batch) +title: -resolve (manual|auto|beforelaunch|batch|cache) summary: Defines when/how resolving is done to calculate the -runbundles --- @@ -13,6 +13,8 @@ The values are: * `auto` – Whenever the initial requirements are saved, the resolver will be used to set new `-runbundles` * `beforelaunch` – Calculate the `-runbundles` on demand. This ignores the value of the `-runbundles` and runs the resolver. The results of the resolver are cached. This cache works by creating a checksum over all the properties of the project. * `batch` – When running in batch mode, the run bundles will be resolved. In all other modes this will only resolve when the `-runbundles` are empty. +* `cache` – Will use a cache file in the workspace cache. If that file is stale relative to the workspace or project or it does not exist, then the bnd(run) file will be resolved and the result is stored in the cache file. + ## Example From fc37052d5c71beae8dc05e36ee5a63425419bb59 Mon Sep 17 00:00:00 2001 From: Peter Kriens Date: Fri, 1 Jul 2022 11:23:06 +0200 Subject: [PATCH 2/2] [resolve] Improvements after BJ's comments - Uses the project/workspace lastmodifiedtime, which includes include files - Removed removal of container errors in test, uses a resolve file that has proper dependencies - Added a testReason so the caching choice could be tested from outside - Increased test coverage Signed-off-by: Peter Kriens --- .../src/biz/aQute/resolve/Bndrun.java | 56 ++++++++------ .../biz/aQute/resolve/RunResolutionTest.java | 75 ++++++++++++++++--- .../test.simple/empty-included-in-resolve.bnd | 0 .../test.simple/resolve.bndrun | 1 + 4 files changed, 101 insertions(+), 31 deletions(-) create mode 100644 biz.aQute.resolve/testdata/pre-buildworkspace/test.simple/empty-included-in-resolve.bnd diff --git a/biz.aQute.resolve/src/biz/aQute/resolve/Bndrun.java b/biz.aQute.resolve/src/biz/aQute/resolve/Bndrun.java index cf0e5b1820..82ea5be7e1 100644 --- a/biz.aQute.resolve/src/biz/aQute/resolve/Bndrun.java +++ b/biz.aQute.resolve/src/biz/aQute/resolve/Bndrun.java @@ -15,6 +15,7 @@ import aQute.bnd.build.Container; import aQute.bnd.build.Run; import aQute.bnd.build.Workspace; +import aQute.bnd.build.WorkspaceLayout; import aQute.bnd.build.model.BndEditModel; import aQute.bnd.build.model.clauses.HeaderClause; import aQute.bnd.build.model.clauses.VersionedClause; @@ -194,38 +195,55 @@ public Collection getRunbundles() throws Exception { return super.getRunbundles(); } + enum CacheReason { + NO_CACHE_FILE, + NOT_A_BND_LAYOUT, + CACHE_STALE_PROJECT, + CACHE_STALE_WORKSPACE, + USE_CACHE, + INVALID_CACHE; + + } + + CacheReason testReason; + private Collection cache() throws Exception { + Workspace ws = getWorkspace(); File ours = getPropertiesFile(); File cache = getCacheFile(ours); - File workspace = getWorkspace().getPropertiesFile(); - if (workspace == null) - workspace = ours; - - String force; - if (!cache.isFile()) - force = "no cache file"; - else if (cache.lastModified() <= ours.lastModified()) - force = "cache file is older than our properties"; - else if (cache.lastModified() <= workspace.lastModified()) - force = "cache file is older than our workspace"; + + long cacheLastModified = cache.lastModified(); + + CacheReason reason; + if (ws.getLayout() != WorkspaceLayout.BND) + reason = CacheReason.NOT_A_BND_LAYOUT; + else if (!cache.isFile()) + reason = CacheReason.NO_CACHE_FILE; + else if (cacheLastModified <= ws.lastModified()) + reason = CacheReason.CACHE_STALE_WORKSPACE; + else if (cacheLastModified <= lastModified()) + reason = CacheReason.CACHE_STALE_PROJECT; else - force = null; - trace("force = %s", force); + reason = CacheReason.USE_CACHE; - boolean tryCache = force == null; + testReason = reason; + + trace("force = %s", reason); try (Processor p = new Processor()) { if (cache.isFile()) p.setProperties(cache); - if (tryCache) { + if (reason == CacheReason.USE_CACHE) { trace("attempting to use cache"); String runbundles = p.getProperty(Constants.RUNBUNDLES); Collection containers = parseRunbundles(runbundles); if (isAllOk(containers)) { + trace("from cache %s", containers); return containers; } else { + testReason = CacheReason.INVALID_CACHE; trace("the cached bundles were not ok, will resolve"); } } @@ -255,6 +273,7 @@ else if (cache.lastModified() <= workspace.lastModified()) } } + /** * Return the file used to cache the resolved solution for the given file * @@ -274,16 +293,9 @@ public File getCacheFile(File file) { } } - boolean testIgnoreDownloadErrors = false; - private boolean isAllOk(Collection containers) { for (Container c : containers) { if (c.getError() != null) { - if (testIgnoreDownloadErrors) { - if (c.getError() - .contains("FileNotFoundException")) - continue; - } return false; } } diff --git a/biz.aQute.resolve/test/biz/aQute/resolve/RunResolutionTest.java b/biz.aQute.resolve/test/biz/aQute/resolve/RunResolutionTest.java index 9761ff6188..02d405be24 100644 --- a/biz.aQute.resolve/test/biz/aQute/resolve/RunResolutionTest.java +++ b/biz.aQute.resolve/test/biz/aQute/resolve/RunResolutionTest.java @@ -28,6 +28,7 @@ import aQute.bnd.result.Result; import aQute.bnd.test.jupiter.InjectTemporaryDirectory; import aQute.lib.io.IO; +import biz.aQute.resolve.Bndrun.CacheReason; public class RunResolutionTest { @@ -120,18 +121,27 @@ public void testCachingOfResult() throws Exception { // assertThat(runbundles).hasSize(22); } + @Test + public void testResolveCachedWithStandalone() throws Exception { + Bndrun bndrun = Bndrun.createBndrun(workspace, IO.getFile(tmp.toFile(), "resolver.bndrun")); + bndrun.setProperty("-resolve", "cache"); + Collection runbundles = bndrun.getRunbundles(); + assertThat(bndrun.testReason).isEqualTo(CacheReason.NOT_A_BND_LAYOUT); + } + @Test public void testResolveCached() throws Exception { Bndrun bndrun = Bndrun.createBndrun(workspace, IO.getFile(ws.toFile(), "test.simple/resolve.bndrun")); + bndrun.setTrace(true); File file = bndrun.getPropertiesFile(); - bndrun.testIgnoreDownloadErrors = true; assertTrue(bndrun.check()); File cache = bndrun.getCacheFile(file); - System.out.println("get the embedded list of runbundles"); + System.out.println("get the embedded list of runbundles, this is out benchmark"); bndrun.setProperty("-resolve", "manual"); Collection manual = bndrun.getRunbundles(); + assertThat(manual).hasSize(2); System.out.println("remove the embedded list and set mode to 'cache'"); bndrun.setProperty("-resolve", "cache"); @@ -141,25 +151,72 @@ public void testResolveCached() throws Exception { System.out.println("First time we should resolve & create a cache file"); Collection cached = bndrun.getRunbundles(); + assertTrue(bndrun.check()); assertThat(cache).isFile(); - assertThat(cached).containsAll(manual); - assertThat(cached).hasSize(manual.size()); - assertThat(cache.lastModified()).isGreaterThan(file.lastModified()); + assertThat(cached).containsExactlyElementsOf(manual); + assertThat(cache.lastModified()).isGreaterThan(bndrun.lastModified()); + assertThat(bndrun.testReason).isEqualTo(CacheReason.NO_CACHE_FILE); System.out .println("Second time, the cache file should used, so make it valid but empty "); long lastModified = cache.lastModified(); IO.store("-runbundles ", cache); - cache.setLastModified(file.lastModified() + 1000); cached = bndrun.getRunbundles(); + assertTrue(bndrun.check()); assertThat(cached).isEmpty(); + assertThat(bndrun.testReason).isEqualTo(CacheReason.USE_CACHE); - System.out.println("Now make cache invalid, which be ignored"); + System.out.println("Now make cache invalid, should be ignored"); IO.store("-runbundles is not a valid file", cache); - cache.setLastModified(file.lastModified() + 1000); cached = bndrun.getRunbundles(); - assertThat(manual).containsAll(cached); + assertTrue(bndrun.check()); + assertThat(cached).containsExactlyElementsOf(manual); + assertThat(bndrun.testReason).isEqualTo(CacheReason.INVALID_CACHE); + System.out.println("Now empty cache, but still use it"); + IO.store("-runbundles ", cache); + cached = bndrun.getRunbundles(); + assertTrue(bndrun.check()); + assertThat(cached).isEmpty(); + assertThat(bndrun.testReason).isEqualTo(CacheReason.USE_CACHE); + + System.out.println("Refresh and check we still use the cache"); + bndrun.refresh(); + bndrun.setProperty("-resolve", "cache"); + bndrun.unsetProperty("-runbundles"); + cached = bndrun.getRunbundles(); + assertThat(cached).isEmpty(); + assertThat(bndrun.testReason).isEqualTo(CacheReason.USE_CACHE); + + System.out.println("Update an include file, refresh and check we still sue the cache"); + File empty = IO.getFile(ws.toFile(), "test.simple/empty-included-in-resolve.bnd"); + long now = System.currentTimeMillis(); + empty.setLastModified(now); + assertThat(bndrun.lastModified()).isLessThan(now); + bndrun.refresh(); + bndrun.setProperty("-resolve", "cache"); + bndrun.unsetProperty("-runbundles"); + cached = bndrun.getRunbundles(); + assertTrue(bndrun.check()); + assertThat(cached).containsExactlyElementsOf(manual); + assertThat(bndrun.testReason).isEqualTo(CacheReason.CACHE_STALE_PROJECT); + + System.out.println("Next we use the cache"); + cached = bndrun.getRunbundles(); + assertThat(cached).containsExactlyElementsOf(manual); + assertThat(bndrun.testReason).isEqualTo(CacheReason.USE_CACHE); + + Thread.sleep(100); + + System.out.println("Update the cnf/build file"); + File build = IO.getFile(ws.toFile(), "cnf/build.bnd"); + now = System.currentTimeMillis(); + build.setLastModified(now); + workspace.refresh(); + cached = bndrun.getRunbundles(); + assertTrue(bndrun.check()); + assertThat(cached).containsExactlyElementsOf(manual); + assertThat(bndrun.testReason).isEqualTo(CacheReason.CACHE_STALE_WORKSPACE); } @Test diff --git a/biz.aQute.resolve/testdata/pre-buildworkspace/test.simple/empty-included-in-resolve.bnd b/biz.aQute.resolve/testdata/pre-buildworkspace/test.simple/empty-included-in-resolve.bnd new file mode 100644 index 0000000000..e69de29bb2 diff --git a/biz.aQute.resolve/testdata/pre-buildworkspace/test.simple/resolve.bndrun b/biz.aQute.resolve/testdata/pre-buildworkspace/test.simple/resolve.bndrun index c671e45e93..03ea7fb2ac 100644 --- a/biz.aQute.resolve/testdata/pre-buildworkspace/test.simple/resolve.bndrun +++ b/biz.aQute.resolve/testdata/pre-buildworkspace/test.simple/resolve.bndrun @@ -1,3 +1,4 @@ +-include empty-included-in-resolve.bnd -runfw: org.eclipse.osgi;version='[3.13.0,3.13.1)' -runee: JavaSE-1.8 -runrequires: osgi.identity;filter:='(osgi.identity=test.simple)'