[WIP] split out ZincWorkerImpl into a separate subprocess

kotlinlib/src/mill/kotlinlib/KotlinModule.scala

@@ -347,7 +347,7 @@ trait KotlinModule extends JavaModule { outer =>
       javacOptions: Seq[String],
       compileProblemReporter: Option[CompileProblemReporter],
       reportOldProblems: Boolean
-  )(implicit ctx: ZincWorkerApi.Ctx): Result[CompilationResult] = {
+  ): Result[CompilationResult] = {
     worker.compileJava(
       upstreamCompileOutput = upstreamCompileOutput,
       sources = javaSourceFiles,

runner/src/mill/runner/MillBuildRootModule.scala

@@ -301,7 +301,8 @@ abstract class MillBuildRootModule()(implicit
         reporter = T.reporter.apply(hashCode),
         reportCachedProblems = zincReportCachedProblems(),
         incrementalCompilation = zincIncrementalCompilation(),
-        auxiliaryClassFileExtensions = zincAuxiliaryClassFileExtensions()
+        auxiliaryClassFileExtensions = zincAuxiliaryClassFileExtensions(),
+        compilerBridge = bridgeJarTask().path
       )
   }
 

scalalib/api/src/mill/scalalib/api/ZincWorkerApi.scala

@@ -6,7 +6,6 @@ import mill.api.Loose.Agg
 import scala.annotation.nowarn
 
 object ZincWorkerApi {
-  type Ctx = mill.api.Ctx.Dest with mill.api.Ctx.Log with mill.api.Ctx.Home
 }
 trait ZincWorkerApi {
 
@@ -19,7 +18,7 @@ trait ZincWorkerApi {
       reporter: Option[CompileProblemReporter],
       reportCachedProblems: Boolean,
       incrementalCompilation: Boolean
-  )(implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] =
+  ): mill.api.Result[CompilationResult] =
     compileJava(
       upstreamCompileOutput = upstreamCompileOutput,
       sources = sources,
@@ -38,7 +37,7 @@ trait ZincWorkerApi {
       javacOptions: Seq[String],
       reporter: Option[CompileProblemReporter],
       reportCachedProblems: Boolean
-  )(implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] =
+  ): mill.api.Result[CompilationResult] =
     compileJava(
       upstreamCompileOutput = upstreamCompileOutput,
       sources = sources,
@@ -63,8 +62,9 @@ trait ZincWorkerApi {
       reporter: Option[CompileProblemReporter],
       reportCachedProblems: Boolean,
       incrementalCompilation: Boolean,
-      auxiliaryClassFileExtensions: Seq[String]
-  )(implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] =
+      auxiliaryClassFileExtensions: Seq[String],
+      compilerBridge: os.Path
+  ): mill.api.Result[CompilationResult] =
     compileMixed(
       upstreamCompileOutput = upstreamCompileOutput,
       sources = sources,
@@ -78,70 +78,8 @@ trait ZincWorkerApi {
       reporter = reporter,
       reportCachedProblems = reportCachedProblems,
       incrementalCompilation = incrementalCompilation,
-      auxiliaryClassFileExtensions = auxiliaryClassFileExtensions
-    )
-
-  /** Compile a mixed Scala/Java or Scala-only project */
-  @deprecated("Use override with `incrementalCompilation` parameter", "Mill 0.11.6")
-  def compileMixed(
-      upstreamCompileOutput: Seq[CompilationResult],
-      sources: Agg[os.Path],
-      compileClasspath: Agg[os.Path],
-      javacOptions: Seq[String],
-      scalaVersion: String,
-      scalaOrganization: String,
-      scalacOptions: Seq[String],
-      compilerClasspath: Agg[PathRef],
-      scalacPluginClasspath: Agg[PathRef],
-      reporter: Option[CompileProblemReporter],
-      reportCachedProblems: Boolean
-  )(implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] =
-    compileMixed(
-      upstreamCompileOutput = upstreamCompileOutput,
-      sources = sources,
-      compileClasspath = compileClasspath,
-      javacOptions = javacOptions,
-      scalaVersion = scalaVersion,
-      scalaOrganization = scalaOrganization,
-      scalacOptions = scalacOptions,
-      compilerClasspath = compilerClasspath,
-      scalacPluginClasspath = scalacPluginClasspath,
-      reporter = reporter,
-      reportCachedProblems = reportCachedProblems,
-      incrementalCompilation = true,
-      auxiliaryClassFileExtensions = Seq.empty[String]
-    )
-
-  /** Compile a mixed Scala/Java or Scala-only project */
-  @deprecated("Use override with `auxiliaryClassFileExtensions` parameter", "Mill 0.11.8")
-  def compileMixed(
-      upstreamCompileOutput: Seq[CompilationResult],
-      sources: Agg[os.Path],
-      compileClasspath: Agg[os.Path],
-      javacOptions: Seq[String],
-      scalaVersion: String,
-      scalaOrganization: String,
-      scalacOptions: Seq[String],
-      compilerClasspath: Agg[PathRef],
-      scalacPluginClasspath: Agg[PathRef],
-      reporter: Option[CompileProblemReporter],
-      reportCachedProblems: Boolean,
-      incrementalCompilation: Boolean
-  )(implicit ctx: ZincWorkerApi.Ctx): mill.api.Result[CompilationResult] =
-    compileMixed(
-      upstreamCompileOutput = upstreamCompileOutput,
-      sources = sources,
-      compileClasspath = compileClasspath,
-      javacOptions = javacOptions,
-      scalaVersion = scalaVersion,
-      scalaOrganization = scalaOrganization,
-      scalacOptions = scalacOptions,
-      compilerClasspath = compilerClasspath,
-      scalacPluginClasspath = scalacPluginClasspath,
-      reporter = reporter,
-      reportCachedProblems = reportCachedProblems,
-      incrementalCompilation = incrementalCompilation,
-      auxiliaryClassFileExtensions = Seq.empty[String]
+      auxiliaryClassFileExtensions = auxiliaryClassFileExtensions,
+      compilerBridge = compilerBridge
     )
 
   /**
@@ -154,8 +92,9 @@ trait ZincWorkerApi {
       scalaOrganization: String,
       compilerClasspath: Agg[PathRef],
       scalacPluginClasspath: Agg[PathRef],
-      args: Seq[String]
-  )(implicit ctx: ZincWorkerApi.Ctx): Boolean
+      args: Seq[String],
+      compilerBridge: os.Path
+  ): Boolean
 
   /**
    * Discover main classes by inspecting the classpath.

scalalib/src/mill/scalalib/ScalaModule.scala

@@ -285,7 +285,8 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer =>
         reporter = T.reporter.apply(hashCode),
         reportCachedProblems = zincReportCachedProblems(),
         incrementalCompilation = zincIncrementalCompilation(),
-        auxiliaryClassFileExtensions = zincAuxiliaryClassFileExtensions()
+        auxiliaryClassFileExtensions = zincAuxiliaryClassFileExtensions(),
+        compilerBridge = bridgeJarTask().path
       )
   }
 
@@ -315,6 +316,21 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer =>
     else allSources()
   }
 
+  def bridgeJarTask = Task.Anon{
+
+    val (cp, bridgeJar) = zincWorker()
+      .scalaCompilerBridgeJar(scalaVersion(), scalaOrganization(), zincWorker().repositoriesTask())
+      .getOrThrow
+    val path = zincWorker().compileBridgeIfNeeded(
+      scalaVersion(),
+      scalaOrganization(),
+      compileClasspath(),
+      Task.dest,
+      cp.map(_.toSeq),
+      bridgeJar
+    )
+    PathRef(path)
+  }
   override def docJar: T[PathRef] = Task {
     val compileCp = Seq(
       "-classpath",
@@ -336,7 +352,8 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer =>
             scalaDocClasspath(),
             scalacPluginClasspath(),
             options ++ compileCp ++ scalaDocOptions() ++
-              files.map(_.toString())
+              files.map(_.toString()),
+            compilerBridge = bridgeJarTask().path
           ) match {
           case true => Result.Success(createJar(Agg(javadocDir))(T.dest))
           case false => Result.Failure("docJar generation failed")
@@ -644,7 +661,8 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer =>
         reporter = None,
         reportCachedProblems = zincReportCachedProblems(),
         incrementalCompilation = zincIncrementalCompilation(),
-        auxiliaryClassFileExtensions = zincAuxiliaryClassFileExtensions()
+        auxiliaryClassFileExtensions = zincAuxiliaryClassFileExtensions(),
+        compilerBridge = bridgeJarTask().path
       )
       .map(compileRes =>
         SemanticDbJavaModule.copySemanticdbFiles(

scalalib/src/mill/scalalib/UnidocModule.scala

@@ -50,7 +50,8 @@ trait UnidocModule extends ScalaModule {
       scalaOrganization(),
       scalaDocClasspath(),
       scalacPluginClasspath(),
-      options ++ unidocSourceFiles.map(_.path.toString)
+      options ++ unidocSourceFiles.map(_.path.toString),
+      compilerBridge = bridgeJarTask().path
     ) match {
       case true => mill.api.Result.Success(PathRef(T.dest))
       case false => mill.api.Result.Failure("unidoc generation failed")

scalalib/src/mill/scalalib/ZincWorkerModule.scala

@@ -78,36 +78,22 @@ trait ZincWorkerModule extends mill.Module with OfflineSupportModule with Coursi
 
     val cls = cl.loadClass("mill.scalalib.worker.ZincWorkerImpl")
     val instance = cls.getConstructor(
-      classOf[
-        Either[
-          (ZincWorkerApi.Ctx, (String, String) => (Option[Agg[PathRef]], PathRef)),
-          String => PathRef
-        ]
-      ], // compilerBridge
-      classOf[(Agg[PathRef], String) => PathRef], // libraryJarNameGrep
-      classOf[(Agg[PathRef], String) => PathRef], // compilerJarNameGrep
-      classOf[KeyedLockedCache[_]], // compilerCache
+      classOf[(String, String) => (Option[Agg[PathRef]], PathRef)], // compilerBridge
+      classOf[Int], // jobs
       classOf[Boolean], // compileToJar
       classOf[Boolean], // zincLogDebug
-      classOf[Option[PathRef]] // javaHome
+      classOf[Option[os.Path]], // javaHome
+      classOf[os.Path], // dest
+      classOf[Boolean] // colored
     )
       .newInstance(
-        Left((
-          T.ctx(),
-          (x: String, y: String) =>
-            scalaCompilerBridgeJar(x, y, repositoriesTask())
-              .asSuccess
-              .getOrElse(
-                throw new Exception(s"Failed to load compiler bridge for $x $y")
-              )
-              .value
-        )),
-        ZincWorkerUtil.grepJar(_, "scala-library", _, sources = false),
-        ZincWorkerUtil.grepJar(_, "scala-compiler", _, sources = false),
-        new FixSizedCache(jobs),
+//        (x: String, y: String) => scalaCompilerBridgeJar(x, y, repositoriesTask()).getOrThrow,
+        jobs,
         java.lang.Boolean.FALSE,
         java.lang.Boolean.valueOf(zincLogDebug()),
-        javaHome()
+        javaHome().map(_.path),
+        Task.dest,
+        Task.log.colored
       )
     instance.asInstanceOf[ZincWorkerApi]
   }
@@ -166,6 +152,113 @@ trait ZincWorkerModule extends mill.Module with OfflineSupportModule with Coursi
     }
   }
 
+  /**
+   * If needed, compile (for Scala 2) or download (for Dotty) the compiler bridge.
+   * @return a path to the directory containing the compiled classes, or to the downloaded jar file
+   */
+  def compileBridgeIfNeeded(
+      scalaVersion: String,
+      scalaOrganization: String,
+      compilerClasspath: Agg[PathRef],
+      dest: os.Path,
+      cp: Option[Seq[PathRef]],
+      bridgeJar: PathRef
+  ): os.Path = {
+    val workingDir = dest / s"zinc-${Versions.zinc}" / scalaVersion
+    val compiledDest = workingDir / "compiled"
+    if (os.exists(compiledDest / "DONE")) compiledDest
+    else {
+      cp match {
+        case None => bridgeJar.path
+        case Some(bridgeClasspath) =>
+          compileZincBridge(
+            workingDir,
+            compiledDest,
+            scalaVersion,
+            compilerClasspath,
+            bridgeClasspath,
+            bridgeJar.path
+          )
+          os.write(compiledDest / "DONE", "")
+          compiledDest
+      }
+    }
+  }
+
+  /** Compile the SBT/Zinc compiler bridge in the `compileDest` directory */
+  def compileZincBridge(
+                         workingDir: os.Path,
+                         compileDest: os.Path,
+                         scalaVersion: String,
+                         compilerClasspath: Agg[PathRef],
+                         compilerBridgeClasspath: Agg[PathRef],
+                         compilerBridgeSourcesJar: os.Path
+                       ): Unit = {
+    if (scalaVersion == "2.12.0") {
+      // The Scala 2.10.0 compiler fails on compiling the compiler bridge
+      throw new IllegalArgumentException(
+        "The current version of Zinc is incompatible with Scala 2.12.0.\n" +
+          "Use Scala 2.12.1 or greater (2.12.12 is recommended)."
+      )
+    }
+
+    System.err.println("Compiling compiler interface...")
+
+    os.makeDir.all(workingDir)
+    os.makeDir.all(compileDest)
+
+    val sourceFolder = os.unzip(compilerBridgeSourcesJar, workingDir / "unpacked")
+    val classloader = mill.api.ClassLoader.create(
+      compilerClasspath.iterator.map(_.path.toIO.toURI.toURL).toSeq,
+      null
+    )(new mill.api.Ctx.Home{ def home: os.Path = os.home})
+
+    val (sources, resources) =
+      os.walk(sourceFolder).filter(os.isFile)
+        .partition(a => a.ext == "scala" || a.ext == "java")
+
+    resources.foreach { res =>
+      val dest = compileDest / res.relativeTo(sourceFolder)
+      os.move(res, dest, replaceExisting = true, createFolders = true)
+    }
+
+    val argsArray = Array[String](
+      "-d",
+      compileDest.toString,
+      "-classpath",
+      (compilerClasspath.iterator ++ compilerBridgeClasspath).map(_.path).mkString(
+        java.io.File.pathSeparator
+      )
+    ) ++ sources.map(_.toString)
+
+    val allScala = sources.forall(_.ext == "scala")
+    val allJava = sources.forall(_.ext == "java")
+    if (allJava) {
+      val javacExe: String =
+        sys.props
+          .get("java.home")
+          .map(h =>
+            if (scala.util.Properties.isWin) new java.io.File(h, "bin\\javac.exe")
+            else new java.io.File(h, "bin/javac")
+          )
+          .filter(f => f.exists())
+          .fold("javac")(_.getAbsolutePath())
+      import scala.sys.process._
+      (Seq(javacExe) ++ argsArray).!
+    } else if (allScala) {
+      val compilerMain = classloader.loadClass(
+        if (ZincWorkerUtil.isDottyOrScala3(scalaVersion)) "dotty.tools.dotc.Main"
+        else "scala.tools.nsc.Main"
+      )
+      compilerMain
+        .getMethod("process", classOf[Array[String]])
+        .invoke(null, argsArray ++ Array("-nowarn"))
+    } else {
+      throw new IllegalArgumentException("Currently not implemented case.")
+    }
+  }
+
+
   def compilerInterfaceClasspath(
       scalaVersion: String,
       scalaOrganization: String,