Improve Intellij Idea support (#351)

* Improve Intellij Idea support

Improves the Intellij Idea support in various ways :

* Cherrypicks the idea conf that needs deleting rather than deleting
the whole .idea directory. That directory contains elements of
configuration like VCS reference that were annoying to set again
every time mill regenerated idea config.
* Attempts to retrieve libraries that the build depends on by inspecting
the classloader of the top module
* Attempts at grouping jars and sources together in order to have both
in the same idea files, which appears to give better jump to definition
* Hacks the library names for the libraries the build depends on, in
order to match Intellij's ammonite support and not show red to the user
about the library that has successfuly been resolved. Also allows to
jump to the library sources from the magic import.

* Remove un-necessary filters

* Avoid Agg throwing because of duplicated build libraries

* Removing hardcoded version from SBT idea module names

scalalib/src/mill/scalalib/GenIdeaImpl.scala

@@ -1,13 +1,14 @@
 package mill.scalalib
 
 import ammonite.ops._
-import coursier.Repository
+import ammonite.runtime.SpecialClassLoader
+import coursier.{Cache, CoursierPaths, Repository}
 import mill.define._
 import mill.eval.{Evaluator, PathRef, Result}
-import mill.{T, scalalib}
 import mill.util.Ctx.{Home, Log}
-import mill.util.{Loose, PrintLogger, Strict}
 import mill.util.Strict.Agg
+import mill.util.{Loose, Strict}
+import mill.{T, scalalib}
 
 import scala.util.Try
 
@@ -34,7 +35,8 @@ object GenIdeaImpl {
 
     val jdkInfo = extractCurrentJdk(pwd / ".idea" / "misc.xml").getOrElse(("JDK_1_8", "1.8 (1)"))
 
-    rm! pwd/".idea"
+    rm! pwd/".idea"/"libraries"
+    rm! pwd/".idea"/"scala_compiler.xml"
     rm! pwd/".idea_modules"
 
 
@@ -81,6 +83,17 @@ object GenIdeaImpl {
           res.items.toList.map(_.path)
       }
 
+    val buildDepsPaths = Try(evaluator
+      .rootModule
+      .getClass
+      .getClassLoader
+      .asInstanceOf[SpecialClassLoader]
+    ).map {
+       _.allJars
+        .map(url => Path(url.getFile))
+        .filter(_.toIO.exists)
+    }.getOrElse(Seq())
+
     val resolved = for((path, mod) <- modules) yield {
       val scalaLibraryIvyDeps = mod match{
         case x: ScalaModule => x.scalaLibraryIvyDeps
@@ -119,8 +132,8 @@ object GenIdeaImpl {
     }
     val moduleLabels = modules.map(_.swap).toMap
 
+    val allResolved = resolved.flatMap(_._2) ++ buildLibraryPaths ++ buildDepsPaths
 
-    val allResolved = resolved.flatMap(_._2) ++ buildLibraryPaths
     val commonPrefix =
       if (allResolved.isEmpty) 0
       else {
@@ -148,13 +161,64 @@ object GenIdeaImpl {
       }
       .toMap
 
+    sealed trait ResolvedLibrary { def path : Path }
+    case class CoursierResolved(path : Path, pom : Path, sources : Option[Path])
+      extends ResolvedLibrary
+    case class OtherResolved(path : Path) extends ResolvedLibrary
+
+    // Tries to group jars with their poms and sources.
+    def toResolvedJar(path : Path) : Option[ResolvedLibrary] = {
+      val inCoursierCache = path.startsWith(Path(CoursierPaths.cacheDirectory()))
+      val isSource = path.segments.last.endsWith("sources.jar")
+      val isPom = path.ext == "pom"
+      if (inCoursierCache && (isSource || isPom)) {
+        // Remove sources and pom as they'll be recovered from the jar path
+        None
+      } else if (inCoursierCache && path.ext == "jar") {
+        val withoutExt = path.segments.last.dropRight(path.ext.length + 1)
+        val pom = path / up / s"$withoutExt.pom"
+        val sources = Some(path / up / s"$withoutExt-sources.jar")
+          .filter(_.toIO.exists())
+        Some(CoursierResolved(path, pom, sources))
+      } else Some(OtherResolved(path))
+    }
+
+    // Hack so that Intellij does not complain about unresolved magic
+    // imports in build.sc when in fact they are resolved
+    def sbtLibraryNameFromPom(pom : Path) : String = {
+      val xml = scala.xml.XML.loadFile(pom.toIO)
+
+      val groupId = (xml \ "groupId").text
+      val artifactId = (xml \ "artifactId").text
+      val version = (xml \ "version").text
+
+      // The scala version here is non incidental
+      s"SBT: $groupId:$artifactId:$version:jar"
+    }
+
+    def libraryName(resolvedJar: ResolvedLibrary) : String = resolvedJar match {
+      case CoursierResolved(path, pom, _) if buildDepsPaths.contains(path) =>
+        sbtLibraryNameFromPom(pom)
+      case CoursierResolved(path, _, _) =>
+        pathToLibName(path)
+      case OtherResolved(path) =>
+        pathToLibName(path)
+    }
+
+    def resolvedLibraries(resolved : Seq[Path]) : Seq[ResolvedLibrary] = resolved
+      .map(toResolvedJar)
+      .collect { case Some(r) => r}
+
     val compilerSettings = resolved
       .foldLeft(Map[(Loose.Agg[Path], Seq[String]), Vector[JavaModule]]()) {
         (r, q) =>
           val key = (q._4, q._5)
           r + (key -> (r.getOrElse(key, Vector()) :+ q._3))
       }
 
+    val allBuildLibraries : Set[ResolvedLibrary] =
+      resolvedLibraries(buildLibraryPaths ++ buildDepsPaths).toSet
+
     val fixedFiles = Seq(
       Tuple2(".idea"/"misc.xml", miscXmlTemplate(jdkInfo)),
       Tuple2(".idea"/"scala_settings.xml", scalaSettingsTemplate()),
@@ -168,8 +232,8 @@ object GenIdeaImpl {
       Tuple2(
         ".idea_modules"/"mill-build.iml",
         rootXmlTemplate(
-          for(path <- buildLibraryPaths)
-          yield pathToLibName(path)
+          for(lib <- allBuildLibraries)
+          yield libraryName(lib)
         )
       ),
       Tuple2(
@@ -178,16 +242,15 @@ object GenIdeaImpl {
       )
     )
 
-    val libraries = allResolved.map{path =>
+    val libraries = resolvedLibraries(allResolved).map{ resolved =>
+      import resolved.path
       val url = "jar://" + path + "!/"
-      val name = pathToLibName(path)
-      Tuple2(".idea"/'libraries/s"$name.xml", libraryXmlTemplate(name, url))
-    }
-
-    val buildLibraries = buildLibraryPaths.map{path =>
-      val url = "jar://" + path + "!/"
-      val name = pathToLibName(path)
-      Tuple2(".idea"/'libraries/s"$name.xml", libraryXmlTemplate(name, url))
+      val name = libraryName(resolved)
+      val sources = resolved match {
+        case CoursierResolved(_, _, s) => s.map(p => "jar://" + p + "!/")
+        case OtherResolved(_) => None
+      }
+      Tuple2(".idea"/'libraries/s"$name.xml", libraryXmlTemplate(name, url, sources))
     }
 
     val moduleFiles = resolved.map{ case (path, resolvedDeps, mod, _, _) =>
@@ -231,7 +294,7 @@ object GenIdeaImpl {
       Tuple2(".idea_modules"/s"${moduleName(path)}.iml", elem)
     }
 
-    fixedFiles ++ libraries ++ moduleFiles ++ buildLibraries
+    fixedFiles ++ libraries ++ moduleFiles
   }
 
   def evalOrElse[T](evaluator: Evaluator[_], e: Task[T], default: => T): T = {
@@ -307,12 +370,18 @@ object GenIdeaImpl {
       </component>
     </module>
   }
-  def libraryXmlTemplate(name: String, url: String) = {
+  def libraryXmlTemplate(name: String, url: String, sources: Option[String]) = {
     <component name="libraryTable">
       <library name={name} type={if(name.contains("scala-library-")) "Scala" else null}>
         <CLASSES>
           <root url={url}/>
         </CLASSES>
+        { if (sources.isDefined) {
+          <SOURCES>
+            <root url={sources.get}/>
+          </SOURCES>
+          }
+        }
       </library>
     </component>
   }

scalalib/test/resources/gen-idea/idea/libraries/scala-library-2.12.4.jar.xml

@@ -3,5 +3,8 @@
         <CLASSES>
             <root url="jar://COURSIER_HOME/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.12.4/scala-library-2.12.4.jar!/"/>
         </CLASSES>
+        <SOURCES>
+            <root url="jar://COURSIER_HOME/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.12.4/scala-library-2.12.4-sources.jar!/"/>
+        </SOURCES>
     </library>
 </component>

scalalib/test/src/mill/scalalib/GenIdeaTests.scala

@@ -43,8 +43,6 @@ object GenIdeaTests extends TestSuite {
           millSourcePath / "generated" / ".idea_modules" /"mill-build.iml",
         "gen-idea/idea/libraries/scala-library-2.12.4.jar.xml" ->
           millSourcePath / "generated" / ".idea" / "libraries" / "scala-library-2.12.4.jar.xml",
-        "gen-idea/idea/libraries/scala-library-2.12.4-sources.jar.xml" ->
-          millSourcePath / "generated" / ".idea" / "libraries" / "scala-library-2.12.4-sources.jar.xml",
         "gen-idea/idea/modules.xml" ->
           millSourcePath / "generated" / ".idea" / "modules.xml",
         "gen-idea/idea/misc.xml" ->