Features/237 cache all the things (#241)

Fixes #237

base/src/main/java/org/ndx/aadarchi/base/enhancers/ModelElementKeys.java

@@ -38,6 +38,10 @@ public static interface WorkspaceDsl {
 			public String NAME = ModelElementKeys.PREFIX + "dsl";
 			public String VALUE = "${project.basedir}/src/architecture/resources/workspace.dsl";
 		}
+		public static interface CacheDir {
+			public String NAME = ModelElementKeys.PREFIX + "cache.dir";
+			public String VALUE = "${project.basedir}/.cache";
+		}
 		public static interface AsciidocSourceDir {
 			public String NAME = "asciidoc.source.docs.directory";
 			public String VALUE = "${project.basedir}/src/docs/asciidoc";

base/src/main/java/org/ndx/aadarchi/base/enhancers/scm/SCMReadmeReader.java

@@ -19,6 +19,7 @@
 import org.ndx.aadarchi.base.OutputBuilder.Format;
 import org.ndx.aadarchi.base.enhancers.ModelElementAdapter;
 import org.ndx.aadarchi.base.enhancers.ModelElementKeys.Scm;
+import org.ndx.aadarchi.base.utils.FileContentCache;
 import org.ndx.aadarchi.base.utils.StructurizrUtils;
 
 import com.structurizr.annotation.Component;
@@ -38,6 +39,8 @@
 public class SCMReadmeReader extends ModelElementAdapter {
 	@Inject @ConfigProperty(name="force", defaultValue="false") boolean force;
 
+	@Inject FileContentCache cache;
+
 	@Inject Logger logger;
 
 	@Inject @UsesComponent(description = "Get SCM infos") Instance<SCMHandler> scmHandlers;
@@ -108,7 +111,7 @@ void writeReadmeFor(SCMHandler handler, Element element, String elementProject,
 				}
 				try {
 					// Now we have content as asciidoc, so let's write it to the conventional location
-					String readmeText = IOUtils.toString(readme.content(), "UTF-8");
+					String readmeText = IOUtils.toString(cache.openStreamFor(readme), "UTF-8");
 					if(readme.name().toLowerCase().endsWith(".md")) {
 						readmeText = Converter.convertMarkdownToAsciiDoc(readmeText);
 					}

base/src/main/java/org/ndx/aadarchi/base/utils/FileContentCache.java

@@ -0,0 +1,82 @@
+package org.ndx.aadarchi.base.utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.util.function.Function;
+
+import javax.enterprise.context.ApplicationScoped;
+import javax.inject.Inject;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.deltaspike.core.api.config.ConfigProperty;
+import org.ndx.aadarchi.base.enhancers.ModelElementKeys;
+import org.ndx.aadarchi.base.enhancers.ModelElementKeys.ConfigProperties.Force;
+import org.ndx.aadarchi.base.enhancers.scm.SCMFile;
+
+/**
+ * A specific file cache, usable to avoid downloading multiple times the same file content.
+ * @author Nicolas
+ *
+ */
+@com.structurizr.annotation.Component(technology = "Java, CDI")
+@ApplicationScoped
+public class FileContentCache {
+
+	@Inject
+	@ConfigProperty(name = Force.NAME, defaultValue=Force.VALUE)
+	boolean force;
+	@Inject @ConfigProperty(
+			name = ModelElementKeys.ConfigProperties.CacheDir.NAME, 
+			defaultValue = ModelElementKeys.ConfigProperties.CacheDir.VALUE) File cacheDir;
+
+	public InputStream openStreamFor(URL url, Function<URL, InputStream> cacheLoader) throws IOException {
+		File file = toCacheFile(url);
+		if(force || !file.exists() || shouldRefresh(file)) {
+			refreshCache(file, url, cacheLoader);
+		}
+		// Now it's time to load file in cache
+		return FileUtils.openInputStream(file);
+	}
+	public InputStream openStreamFor(String string, Function<URL, InputStream> cacheLoader) throws IOException {
+		return openStreamFor(new URL(string), cacheLoader);
+	}
+
+	/**
+	 * Get cached version of remote file
+	 * @param file, remote scm file
+	 * @return input stream to locally cached version of that file
+	 * @throws IOException thrown if remote file can't be read
+	 */
+	public InputStream openStreamFor(SCMFile file) throws IOException {
+		return openStreamFor(file.url(), _url -> file.content());
+	}
+
+	private void refreshCache(File file, URL url, Function<URL, InputStream> cacheLoader) throws IOException {
+		file.getParentFile().mkdirs();
+		Files.copy(cacheLoader.apply(url), file.toPath());
+	}
+
+	/**
+	 * If file is more than 12 hours old, refresh it
+	 * @param file
+	 * @return
+	 */
+	private boolean shouldRefresh(File file) {
+		return file.lastModified()<(System.currentTimeMillis()-1000*60*60*12);
+	}
+
+	/**
+	 * Convert url to file by removing protocol then concatenating all segments below cache dir
+	 * @param url url for which we want a cache key
+	 * @return a cache file path
+	 */
+	private File toCacheFile(URL url) {
+		File domain = new File(cacheDir, url.getHost());
+		File path = new File(domain, url.getFile().replace('?', '_'));
+		return path;
+	}
+}

github-scm-handler/src/main/java/org/ndx/aadarchi/github/GitHubFile.java

@@ -42,7 +42,7 @@ public String name() {
 	@Override
 	public long lastModified() {
 		try {
-			PagedIterable<GHCommit> commits = repository.queryCommits().path(source.getPath()).list();
+			PagedIterable<GHCommit> commits = repository.queryCommits().pageSize(1).path(source.getPath()).list();
 			return commits.toList().stream()
 				.findFirst()
 				.map(ThrowingFunction.unchecked(commit -> commit.getCommitDate().getTime()))

github-scm-handler/src/main/java/org/ndx/aadarchi/github/GithubSCMHandler.java

@@ -19,14 +19,17 @@
 import org.kohsuke.github.GitHub;
 import org.ndx.aadarchi.base.enhancers.scm.SCMFile;
 import org.ndx.aadarchi.base.enhancers.scm.SCMHandler;
+import org.ndx.aadarchi.base.utils.FileContentCache;
 
+import com.pivovarit.function.ThrowingFunction;
 import com.structurizr.annotation.Component;
 
 @Component
 @ApplicationScoped
 public class GithubSCMHandler implements SCMHandler {
 	@Inject Logger logger;
 	@Inject GitHub github;
+	@Inject FileContentCache fileCache;
 	@Override
 	public boolean canHandle(String project) {
 		return Constants.isGitHubProject(project);
@@ -68,6 +71,10 @@ public String asciidocText() {
 	 */
 	@Override
 	public InputStream openStream(URL url) throws IOException {
+		return fileCache.openStreamFor(url, ThrowingFunction.unchecked(this::doOpenStream));
+	}
+
+	private InputStream doOpenStream(URL url) throws IOException {
 		String urlText = url.toString();
 		if(Constants.isGitHubProject(urlText)) {
 			String project = Constants.getGitHubProjectName(urlText);

maven-metadata-inferer/src/main/java/org/ndx/aadarchi/inferer/maven/MavenDetailsInfererEnhancer.java

@@ -46,6 +46,7 @@
 import org.ndx.aadarchi.base.enhancers.ModelElementKeys.ConfigProperties.BasePath;
 import org.ndx.aadarchi.base.enhancers.scm.SCMFile;
 import org.ndx.aadarchi.base.enhancers.scm.SCMHandler;
+import org.ndx.aadarchi.base.utils.FileContentCache;
 import org.ndx.aadarchi.base.utils.FileResolver;
 
 import com.pivovarit.function.ThrowingFunction;
@@ -303,6 +304,7 @@ protected void endEnhanceWithMavenProject(MavenProject mavenProject) {
 	Logger logger;
 	@Inject @ConfigProperty(name=BasePath.NAME, defaultValue = BasePath.VALUE) File basePath;
 
+	@Inject FileContentCache cache;
 	@Inject FileResolver fileResolver;
 	@Inject Instance<SCMHandler> scmHandler;
 	/**
@@ -597,7 +599,9 @@ protected Optional<MavenProject> processModelElement(Element element) {
 				try {
 					Collection<SCMFile> pomSCMFile = handler.find(project, "/", file -> "pom.xml".equals(file.name()));
 					for(SCMFile pom : pomSCMFile) {
-						return Optional.ofNullable(readMavenProject(pom.url(), new URL(pom.url()), pom.content()));
+						URL url = new URL(pom.url());
+						return Optional.ofNullable(readMavenProject(pom.url(), url, 
+								cache.openStreamFor(pom)));
 					}
 				} catch (IOException | XmlPullParserException e) {
 					logger.log(Level.FINER, String.format("There is no pom.xml in %s, maybe it's normal", project), e);