Download java sources lazily for Maven projects

Signed-off-by: Snjezana Peco <snjezana.peco@redhat.com>

org.eclipse.jdt.ls.core/META-INF/MANIFEST.MF

@@ -15,6 +15,7 @@ Require-Bundle: org.eclipse.core.runtime;bundle-version="3.12.0",
  org.eclipse.ltk.core.refactoring,
  org.eclipse.text;bundle-version="3.6.0",
  org.eclipse.m2e.core,
+ org.eclipse.m2e.jdt,
  com.ibm.icu.base,
  org.eclipse.core.filebuffers;bundle-version="3.6.0",
  org.eclipse.m2e.maven.runtime,

org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/HoverInfoProvider.java

@@ -14,13 +14,16 @@
 import java.io.Reader;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Stream;
 
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.OperationCanceledException;
 import org.eclipse.core.runtime.Status;
+import org.eclipse.jdt.core.IBuffer;
+import org.eclipse.jdt.core.IClassFile;
 import org.eclipse.jdt.core.ICompilationUnit;
 import org.eclipse.jdt.core.IJavaElement;
 import org.eclipse.jdt.core.ILocalVariable;
@@ -35,8 +38,10 @@
 import org.eclipse.jdt.core.search.SearchParticipant;
 import org.eclipse.jdt.core.search.SearchPattern;
 import org.eclipse.jdt.core.search.SearchRequestor;
+import org.eclipse.jdt.internal.core.BinaryMember;
 import org.eclipse.jdt.ls.core.internal.hover.JavaElementLabels;
 import org.eclipse.jdt.ls.core.internal.javadoc.JavadocContentAccess2;
+import org.eclipse.jdt.ls.core.internal.managers.IBuildSupport;
 import org.eclipse.jdt.ls.core.internal.preferences.PreferenceManager;
 import org.eclipse.lsp4j.MarkedString;
 import org.eclipse.lsp4j.jsonrpc.messages.Either;
@@ -103,6 +108,16 @@ public List<Either<String, MarkedString>> computeHover(int line, int column, IPr
 			}
 			boolean resolved = isResolved(curr, monitor);
 			if (resolved) {
+				IBuffer buffer = curr.getOpenable().getBuffer();
+				if (buffer == null && curr instanceof BinaryMember) {
+					IClassFile classFile = ((BinaryMember) curr).getClassFile();
+					if (classFile != null) {
+						Optional<IBuildSupport> bs = JavaLanguageServerPlugin.getProjectsManager().getBuildSupport(curr.getJavaProject().getProject());
+						if (bs.isPresent()) {
+							bs.get().discoverSource(classFile, monitor);
+						}
+					}
+				}
 				MarkedString signature = computeSignature(curr);
 				if (signature != null) {
 					res.add(Either.forRight(signature));

org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/JobHelpers.java

@@ -40,6 +40,7 @@ private JobHelpers() {
 	}
 
 	private static final int POLLING_DELAY = 10;
+	public static final int MAX_TIME_MILLIS = 300000;
 
 	public static void waitForJobsToComplete() {
 		try {
@@ -135,11 +136,15 @@ private static List<IBackgroundProcessingQueue> getProcessingQueues(IJobManager
 	}
 
 	private static void waitForBuildJobs() {
-		waitForJobs(BuildJobMatcher.INSTANCE, 300000);
+		waitForJobs(BuildJobMatcher.INSTANCE, MAX_TIME_MILLIS);
 	}
 
 	public static void waitForInitializeJobs() {
-		waitForJobs(InitializeJobMatcher.INSTANCE, 300000);
+		waitForJobs(InitializeJobMatcher.INSTANCE, MAX_TIME_MILLIS);
+	}
+
+	public static void waitForDownloadSourcesJobs(int maxTimeMillis) {
+		waitForJobs(DownloadSourcesJobMatcher.INSTANCE, maxTimeMillis);
 	}
 
 	public static void waitForJobs(IJobMatcher matcher, int maxWaitMillis) {
@@ -202,4 +207,15 @@ public boolean matches(Job job) {
 
 	}
 
+	static class DownloadSourcesJobMatcher implements IJobMatcher {
+
+		public static final IJobMatcher INSTANCE = new DownloadSourcesJobMatcher();
+
+		@Override
+		public boolean matches(Job job) {
+			return ("org.eclipse.m2e.jdt.internal.DownloadSourcesJob".equals(job.getClass().getName()));
+		}
+
+	}
+
 }

org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/SourceContentProvider.java

@@ -12,12 +12,15 @@
 package org.eclipse.jdt.ls.core.internal;
 
 import java.net.URI;
+import java.util.Optional;
 
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.jdt.core.IBuffer;
 import org.eclipse.jdt.core.IClassFile;
 import org.eclipse.jdt.core.JavaModelException;
+import org.eclipse.jdt.ls.core.internal.managers.IBuildSupport;
+import org.eclipse.jdt.ls.core.internal.managers.ProjectsManager;
 
 public class SourceContentProvider implements IDecompiler {
 
@@ -35,10 +38,17 @@ public String getSource(IClassFile classFile, IProgressMonitor monitor) throws C
 		String source = null;
 		try {
 			IBuffer buffer = classFile.getBuffer();
-			if (buffer != null) {
-				if (monitor.isCanceled()) {
-					return null;
+			if (buffer == null) {
+				ProjectsManager projectsManager = JavaLanguageServerPlugin.getProjectsManager();
+				if (projectsManager != null) {
+					Optional<IBuildSupport> bs = projectsManager.getBuildSupport(classFile.getJavaProject().getProject());
+					if (bs.isPresent()) {
+						bs.get().discoverSource(classFile, monitor);
+					}
 				}
+				buffer = classFile.getBuffer();
+			}
+			if (buffer != null) {
 				source = buffer.getContents();
 				JavaLanguageServerPlugin.logInfo("ClassFile contents request completed");
 			}

org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/managers/IBuildSupport.java

@@ -14,6 +14,7 @@
 import org.eclipse.core.resources.IResource;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.jdt.core.IClassFile;
 import org.eclipse.jdt.ls.core.internal.managers.ProjectsManager.CHANGE_TYPE;
 
 /**
@@ -76,4 +77,16 @@ default void refresh(IResource resource, CHANGE_TYPE changeType, IProgressMonito
 		}
 	}
 
+	/**
+	 * Discover the source for classFile and attach it to the project it belongs to.
+	 *
+	 * @param classFile
+	 *            - a class file
+	 * @param monitor
+	 *            - a progress monitor
+	 * @throws CoreException
+	 */
+	default void discoverSource(IClassFile classFile, IProgressMonitor monitor) throws CoreException {
+	}
+
 }

org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/managers/MavenBuildSupport.java

@@ -15,6 +15,7 @@
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IProject;
@@ -23,7 +24,11 @@
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.jdt.core.IClassFile;
+import org.eclipse.jdt.core.IJavaElement;
+import org.eclipse.jdt.core.IPackageFragmentRoot;
 import org.eclipse.jdt.ls.core.internal.JavaLanguageServerPlugin;
+import org.eclipse.jdt.ls.core.internal.JobHelpers;
 import org.eclipse.jdt.ls.core.internal.ProjectUtils;
 import org.eclipse.jdt.ls.core.internal.managers.ProjectsManager.CHANGE_TYPE;
 import org.eclipse.m2e.core.MavenPlugin;
@@ -34,12 +39,21 @@
 import org.eclipse.m2e.core.project.IMavenProjectRegistry;
 import org.eclipse.m2e.core.project.IProjectConfigurationManager;
 import org.eclipse.m2e.core.project.MavenUpdateRequest;
+import org.eclipse.m2e.jdt.IClasspathManager;
+import org.eclipse.m2e.jdt.MavenJdtPlugin;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
 
 /**
  * @author Fred Bricon
  *
  */
 public class MavenBuildSupport implements IBuildSupport {
+
+	private static final int MAX_TIME_MILLIS = 3000;
+	private static Cache<String, Boolean> downloadRequestsCache = CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite(1, TimeUnit.HOURS).build();
+
 	private IProjectConfigurationManager configurationManager;
 	private ProjectRegistryManager projectManager;
 	private DigestStore digestStore;
@@ -123,4 +137,37 @@ public boolean fileChanged(IResource resource, CHANGE_TYPE changeType, IProgress
 		}
 		return IBuildSupport.super.fileChanged(resource, changeType, monitor) || isBuildFile(resource);
 	}
+
+	/* (non-Javadoc)
+	 * @see org.eclipse.jdt.ls.core.internal.managers.IBuildSupport#getSource(org.eclipse.jdt.core.IClassFile, org.eclipse.core.runtime.IProgressMonitor)
+	 */
+	@Override
+	public void discoverSource(IClassFile classFile, IProgressMonitor monitor) throws CoreException {
+		if (classFile == null) {
+			return;
+		}
+		IJavaElement element = classFile;
+		while (element.getParent() != null) {
+			element = element.getParent();
+			if (element instanceof IPackageFragmentRoot) {
+				final IPackageFragmentRoot fragment = (IPackageFragmentRoot) element;
+				IPath attachmentPath = fragment.getSourceAttachmentPath();
+				if (attachmentPath != null && !attachmentPath.isEmpty() && attachmentPath.toFile().exists()) {
+					break;
+				}
+				if (fragment.isArchive()) {
+					IPath path = fragment.getPath();
+					Boolean downloaded = downloadRequestsCache.getIfPresent(path.toString());
+					if (downloaded == null) {
+						downloadRequestsCache.put(path.toString(), true);
+						IClasspathManager buildpathManager = MavenJdtPlugin.getDefault().getBuildpathManager();
+						buildpathManager.scheduleDownload(fragment, true, true);
+						JobHelpers.waitForDownloadSourcesJobs(MAX_TIME_MILLIS);
+					}
+					break;
+				}
+			}
+		}
+	}
+
 }

org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/managers/MavenProjectImporter.java

@@ -119,7 +119,7 @@ public void reset() {
 	public void importToWorkspace(IProgressMonitor monitor) throws CoreException, OperationCanceledException {
 		JavaLanguageServerPlugin.logInfo(IMPORTING_MAVEN_PROJECTS);
 		MavenConfigurationImpl configurationImpl = (MavenConfigurationImpl)MavenPlugin.getMavenConfiguration();
-		configurationImpl.setDownloadSources(true);
+		configurationImpl.setDownloadSources(JavaLanguageServerPlugin.getPreferencesManager().getPreferences().isMavenDownloadSources());
 		configurationImpl.setNotCoveredMojoExecutionSeverity(ProblemSeverity.ignore.toString());
 		SubMonitor subMonitor = SubMonitor.convert(monitor, 105);
 		subMonitor.setTaskName(IMPORTING_MAVEN_PROJECTS);

org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/managers/ProjectsManager.java

@@ -452,7 +452,7 @@ public IStatus runInWorkspace(IProgressMonitor monitor) {
 		return job;
 	}
 
-	private Optional<IBuildSupport> getBuildSupport(IProject project) {
+	public Optional<IBuildSupport> getBuildSupport(IProject project) {
 		return buildSupports().filter(bs -> bs.applies(project)).findFirst();
 	}
 

org.eclipse.jdt.ls.core/src/org/eclipse/jdt/ls/core/internal/preferences/Preferences.java

@@ -64,6 +64,10 @@ public class Preferences {
 	 * Preference key to enable/disable maven importer.
 	 */
 	public static final String IMPORT_MAVEN_ENABLED = "java.import.maven.enabled";
+	/**
+	 * Preference key to enable/disable downloading Maven source artifacts.
+	 */
+	public static final String MAVEN_DOWNLOAD_SOURCES = "java.maven.downloadSources";
 	/**
 	 * Preference key to enable/disable reference code lenses.
 	 */
@@ -265,6 +269,7 @@ public class Preferences {
 	private boolean referencesCodeLensEnabled;
 	private boolean importGradleEnabled;
 	private boolean importMavenEnabled;
+	private boolean mavenDownloadSources;
 	private boolean implementationsCodeLensEnabled;
 	private boolean javaFormatEnabled;
 	private boolean javaFormatOnTypeEnabled;
@@ -366,6 +371,7 @@ public Preferences() {
 		updateBuildConfigurationStatus = FeatureStatus.interactive;
 		importGradleEnabled = true;
 		importMavenEnabled = true;
+		mavenDownloadSources = false;
 		referencesCodeLensEnabled = true;
 		implementationsCodeLensEnabled = false;
 		javaFormatEnabled = true;
@@ -415,6 +421,8 @@ public static Preferences createFrom(Map<String, Object> configuration) {
 		prefs.setImportGradleEnabled(importGradleEnabled);
 		boolean importMavenEnabled = getBoolean(configuration, IMPORT_MAVEN_ENABLED, true);
 		prefs.setImportMavenEnabled(importMavenEnabled);
+		boolean downloadSources = getBoolean(configuration, MAVEN_DOWNLOAD_SOURCES, false);
+		prefs.setMavenDownloadSources(downloadSources);
 		boolean referenceCodelensEnabled = getBoolean(configuration, REFERENCES_CODE_LENS_ENABLED_KEY, true);
 		prefs.setReferencesCodelensEnabled(referenceCodelensEnabled);
 		boolean implementationCodeLensEnabled = getBoolean(configuration, IMPLEMENTATIONS_CODE_LENS_ENABLED_KEY, false);
@@ -560,6 +568,11 @@ public Preferences setImportMavenEnabled(boolean enabled) {
 		return this;
 	}
 
+	public Preferences setMavenDownloadSources(boolean enabled) {
+		this.mavenDownloadSources = enabled;
+		return this;
+	}
+
 	private Preferences setSignatureHelpEnabled(boolean enabled) {
 		this.signatureHelpEnabled = enabled;
 		return this;
@@ -701,6 +714,10 @@ public boolean isImportMavenEnabled() {
 		return importMavenEnabled;
 	}
 
+	public boolean isMavenDownloadSources() {
+		return mavenDownloadSources;
+	}
+
 	public boolean isImplementationsCodeLensEnabled() {
 		return implementationsCodeLensEnabled;
 	}

org.eclipse.jdt.ls.tests/src/org/eclipse/jdt/ls/core/internal/managers/MavenBuildSupportTest.java

@@ -15,12 +15,17 @@
 import static org.eclipse.jdt.ls.core.internal.ResourceUtils.setContent;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.io.File;
 import java.net.URI;
 import java.util.LinkedHashSet;
 import java.util.Set;
 
+import org.apache.commons.io.FileUtils;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IProject;
 import org.eclipse.core.resources.IncrementalProjectBuilder;
@@ -29,6 +34,14 @@
 import org.eclipse.core.runtime.jobs.IJobChangeListener;
 import org.eclipse.core.runtime.jobs.Job;
 import org.eclipse.core.runtime.jobs.JobChangeAdapter;
+import org.eclipse.jdt.core.IClassFile;
+import org.eclipse.jdt.core.IJavaProject;
+import org.eclipse.jdt.core.IType;
+import org.eclipse.jdt.core.JavaCore;
+import org.eclipse.jdt.internal.core.BinaryType;
+import org.eclipse.jdt.ls.core.internal.DependencyUtil;
+import org.eclipse.jdt.ls.core.internal.JobHelpers;
+import org.eclipse.jdt.ls.core.internal.SourceContentProvider;
 import org.eclipse.jdt.ls.core.internal.WorkspaceHelper;
 import org.eclipse.jdt.ls.core.internal.managers.ProjectsManager.CHANGE_TYPE;
 import org.eclipse.jdt.ls.core.internal.preferences.Preferences.FeatureStatus;
@@ -142,6 +155,25 @@ public void testBuildHelperSupport() throws Exception {
 		assertNoErrors(project);
 	}
 
+	@Test
+	public void testDownloadSources() throws Exception {
+		File file = DependencyUtil.getSources("org.apache.commons", "commons-lang3", "3.5");
+		FileUtils.deleteDirectory(file.getParentFile());
+		IProject project = importMavenProject("salut");
+		waitForBackgroundJobs();
+		assertTrue(!file.exists());
+		IJavaProject javaProject = JavaCore.create(project);
+		IType type = javaProject.findType("org.apache.commons.lang3.StringUtils");
+		IClassFile classFile = ((BinaryType) type).getClassFile();
+		assertNull(classFile.getBuffer());
+		String source = new SourceContentProvider().getSource(classFile, new NullProgressMonitor());
+		if (source == null) {
+			JobHelpers.waitForDownloadSourcesJobs(JobHelpers.MAX_TIME_MILLIS);
+			source = new SourceContentProvider().getSource(classFile, new NullProgressMonitor());
+		}
+		assertNotNull(source);
+	}
+
 	protected void testNonStandardCompilerId(String projectName) throws Exception {
 		IProject project = importMavenProject(projectName);
 		assertIsJavaProject(project);