Merge pull request #162 from scala-cli/update

Update fork to all the newest deps and code

Author: tgodzik

.github/workflows/ci.yml

@@ -42,7 +42,11 @@ jobs:
         run: |
           .github/setup-test-projects.sh &&\
           ./mill -i "bridges.scalajs-1[_].publishLocal" &&\
-          ./mill -i "bridges.scalajs-1[_].test"
+          ./mill -i "bridges.scala-native-04[_].publishLocal" &&\
+          ./mill -i "bridges.scala-native-05[_].publishLocal" &&\
+          ./mill -i "bridges.scalajs-1[_].test" &&\
+          ./mill -i "bridges.scala-native-04[_].test" &&\
+          ./mill -i "bridges.scala-native-05[_].test"
         shell: bash
 
   test:
@@ -67,8 +71,8 @@ jobs:
           .github/setup-test-projects.sh &&\
           ./mill -i 'backend[_].test.compile' &&\
           ./mill -i 'frontend[_].test.compile' &&\
-          ./mill -i 'backend[2.12.18].test' &&\
-          ./mill -i 'frontend[2.12.18].test'
+          ./mill -i 'backend[2.12.19].test' &&\
+          ./mill -i 'frontend[2.12.19].test'
         shell: bash
 
   jvm-tests:

.scalafmt.conf

@@ -1,4 +1,4 @@
-version = "3.7.15"
+version = "3.8.0"
 
 align.preset = more
 maxColumn = 100

backend/src/main/scala/bloop/ClientClassesObserver.scala

@@ -0,0 +1,65 @@
+package bloop
+
+import java.io.File
+import java.util.concurrent.atomic.AtomicReference
+
+import scala.jdk.CollectionConverters._
+
+import bloop.io.AbsolutePath
+import bloop.task.Task
+
+import monix.reactive.Observable
+import monix.reactive.subjects.PublishSubject
+import sbt.internal.inc.PlainVirtualFileConverter
+import xsbti.VirtualFileRef
+import xsbti.compile.CompileAnalysis
+import xsbti.compile.analysis.Stamp
+
+/**
+ * Each time a new compile analysis is produced for a given client, it is given to
+ * the [[ClientClassObserver]] which computes the list of classes that changed or got created.
+ *
+ * A client can subscribe to the observer to get notified of classes to update.
+ * It is used by DAP to hot reload classes in the debuggee process.
+ *
+ * @param clientClassesDir the class directory for the client
+ */
+private[bloop] class ClientClassesObserver(val classesDir: AbsolutePath) {
+  private val converter = PlainVirtualFileConverter.converter
+  private val previousAnalysis: AtomicReference[CompileAnalysis] = new AtomicReference()
+  private val classesSubject: PublishSubject[Seq[String]] = PublishSubject()
+
+  def observable: Observable[Seq[String]] = classesSubject
+
+  def nextAnalysis(analysis: CompileAnalysis): Task[Unit] = {
+    val prev = previousAnalysis.getAndSet(analysis)
+    if (prev != null && classesSubject.size > 0) {
+      Task {
+        val previousStamps = prev.readStamps.getAllProductStamps
+        analysis.readStamps.getAllProductStamps.asScala.iterator.collect {
+          case (vf, stamp) if isClassFile(vf) && isNewer(stamp, previousStamps.get(vf)) =>
+            getFullyQualifiedClassName(vf)
+        }.toSeq
+      }
+        .flatMap { classesToUpdate =>
+          Task.fromFuture(classesSubject.onNext(classesToUpdate)).map(_ => ())
+        }
+    } else Task.unit
+  }
+
+  private def isClassFile(vf: VirtualFileRef): Boolean = vf.id.endsWith(".class")
+
+  private def isNewer(current: Stamp, previous: Stamp): Boolean =
+    previous == null || {
+      val currentHash = current.getHash
+      val previousHash = previous.getHash
+      currentHash.isPresent &&
+      (!previousHash.isPresent || currentHash.get != previousHash.get)
+    }
+
+  private def getFullyQualifiedClassName(vf: VirtualFileRef): String = {
+    val path = converter.toPath(vf)
+    val relativePath = classesDir.underlying.relativize(path)
+    relativePath.toString.replace(File.separator, ".").stripSuffix(".class")
+  }
+}

backend/src/main/scala/bloop/CompileBackgroundTasks.scala

@@ -8,7 +8,7 @@ import bloop.tracing.BraveTracer
 
 abstract class CompileBackgroundTasks {
   def trigger(
-      clientClassesDir: AbsolutePath,
+      clientClassesObserver: ClientClassesObserver,
       clientReporter: Reporter,
       clientTracer: BraveTracer,
       clientLogger: Logger
@@ -20,7 +20,7 @@ object CompileBackgroundTasks {
   val empty: CompileBackgroundTasks = {
     new CompileBackgroundTasks {
       def trigger(
-          clientClassesDir: AbsolutePath,
+          clientClassesObserver: ClientClassesObserver,
           clientReporter: Reporter,
           clientTracer: BraveTracer,
           clientLogger: Logger

backend/src/main/scala/bloop/Compiler.scala

@@ -40,6 +40,7 @@ import sbt.util.InterfaceUtil
 import xsbti.T2
 import xsbti.VirtualFileRef
 import xsbti.compile.{CompilerCache => _, ScalaInstance => _, _}
+import scala.util.Try
 
 case class CompileInputs(
     scalaInstance: ScalaInstance,
@@ -319,6 +320,7 @@ object Compiler {
     val classpathOptions = compileInputs.classpathOptions
     val compilers = compileInputs.compilerCache.get(
       scalaInstance,
+      classpathOptions,
       compileInputs.javacBin,
       compileInputs.javacOptions.toList
     )
@@ -451,11 +453,12 @@ object Compiler {
 
             val backgroundTasks = new CompileBackgroundTasks {
               def trigger(
-                  clientClassesDir: AbsolutePath,
+                  clientClassesObserver: ClientClassesObserver,
                   clientReporter: Reporter,
                   clientTracer: BraveTracer,
                   clientLogger: Logger
               ): Task[Unit] = Task.defer {
+                val clientClassesDir = clientClassesObserver.classesDir
                 clientLogger.debug(s"Triggering background tasks for $clientClassesDir")
                 val updateClientState =
                   updateExternalClassesDirWithReadOnly(clientClassesDir, clientTracer, clientLogger)
@@ -471,10 +474,20 @@ object Compiler {
                 }
 
                 val deleteNewClassesDir = Task(BloopPaths.delete(AbsolutePath(newClassesDir)))
-                val allTasks = List(deleteNewClassesDir, updateClientState, writeAnalysisIfMissing)
+                val publishClientAnalysis = Task {
+                  rebaseAnalysisClassFiles(
+                    analysis,
+                    readOnlyClassesDir,
+                    clientClassesDir.underlying,
+                    sourcesWithFatal
+                  )
+                }
+                  .flatMap(clientClassesObserver.nextAnalysis)
                 Task
-                  .gatherUnordered(allTasks)
-                  .map(_ => ())
+                  .gatherUnordered(
+                    List(deleteNewClassesDir, updateClientState, writeAnalysisIfMissing)
+                  )
+                  .flatMap(_ => publishClientAnalysis)
                   .onErrorHandleWith(err => {
                     clientLogger.debug("Caught error in background tasks"); clientLogger.trace(err);
                     Task.raiseError(err)
@@ -494,14 +507,12 @@ object Compiler {
             )
           } else {
             val allGeneratedProducts = allGeneratedRelativeClassFilePaths.toMap
-            val analysisForFutureCompilationRuns = {
-              rebaseAnalysisClassFiles(
-                analysis,
-                readOnlyClassesDir,
-                newClassesDir,
-                sourcesWithFatal
-              )
-            }
+            val analysisForFutureCompilationRuns = rebaseAnalysisClassFiles(
+              analysis,
+              readOnlyClassesDir,
+              newClassesDir,
+              sourcesWithFatal
+            )
 
             val resultForFutureCompilationRuns = {
               resultForDependentCompilationsInSameRun.withAnalysis(
@@ -516,12 +527,12 @@ object Compiler {
             // Schedule the tasks to run concurrently after the compilation end
             val backgroundTasksExecution = new CompileBackgroundTasks {
               def trigger(
-                  clientClassesDir: AbsolutePath,
+                  clientClassesObserver: ClientClassesObserver,
                   clientReporter: Reporter,
                   clientTracer: BraveTracer,
                   clientLogger: Logger
               ): Task[Unit] = {
-                val clientClassesDirPath = clientClassesDir.toString
+                val clientClassesDir = clientClassesObserver.classesDir
                 val successBackgroundTasks =
                   backgroundTasksWhenNewSuccessfulAnalysis
                     .map(f => f(clientClassesDir, clientReporter, clientTracer))
@@ -542,15 +553,26 @@ object Compiler {
                       val syntax = path.syntax
                       if (syntax.startsWith(readOnlyClassesDirPath)) {
                         val rebasedFile = AbsolutePath(
-                          syntax.replace(readOnlyClassesDirPath, clientClassesDirPath)
+                          syntax.replace(readOnlyClassesDirPath, clientClassesDir.toString)
                         )
                         if (rebasedFile.exists) {
                           Files.delete(rebasedFile.underlying)
                         }
                       }
                     }
                   }
-                  Task.gatherUnordered(List(firstTask, secondTask)).map(_ => ())
+
+                  val publishClientAnalysis = Task {
+                    rebaseAnalysisClassFiles(
+                      analysis,
+                      newClassesDir,
+                      clientClassesDir.underlying,
+                      sourcesWithFatal
+                    )
+                  }.flatMap(clientClassesObserver.nextAnalysis)
+                  Task
+                    .gatherUnordered(List(firstTask, secondTask))
+                    .flatMap(_ => publishClientAnalysis)
                 }
 
                 allClientSyncTasks.doOnFinish(_ => Task(clientReporter.reportEndCompilation()))
@@ -632,25 +654,33 @@ object Compiler {
       case None => scalacOptions
       case Some(_) if existsReleaseSetting || sameHome => scalacOptions
       case Some(version) =>
-        try {
-          val numVer = if (version.startsWith("1.8")) 8 else version.takeWhile(_.isDigit).toInt
-          val bloopNumVer = JavaRuntime.version.takeWhile(_.isDigit).toInt
-          if (bloopNumVer > numVer) {
-            scalacOptions ++ List("-release", numVer.toString())
-          } else {
-            logger.warn(
-              s"Bloop is runing with ${JavaRuntime.version} but your code requires $version to compile, " +
-                "this might cause some compilation issues when using JDK API unsupported by the Bloop's current JVM version"
-            )
-            scalacOptions
+        val options: Option[Array[String]] =
+          for {
+            numVer <- parseJavaVersion(version)
+            bloopNumVer <- parseJavaVersion(JavaRuntime.version)
+            if (bloopNumVer >= 9 && numVer != bloopNumVer)
+          } yield {
+            if (bloopNumVer > numVer) {
+              scalacOptions ++ List("-release", numVer.toString())
+            } else {
+              logger.warn(
+                s"Bloop is running with ${JavaRuntime.version} but your code requires $version to compile, " +
+                  "this might cause some compilation issues when using JDK API unsupported by the Bloop's current JVM version"
+              )
+              scalacOptions
+            }
           }
-        } catch {
-          case NonFatal(_) =>
-            scalacOptions
-        }
+        options.getOrElse(scalacOptions)
     }
   }
 
+  private def parseJavaVersion(version: String): Option[Int] =
+    version.split('-').head.split('.').toList match {
+      case "1" :: minor :: _ => Try(minor.toInt).toOption
+      case single :: _ => Try(single.toInt).toOption
+      case _ => None
+    }
+
   private def getCompilationOptions(
       inputs: CompileInputs,
       logger: Logger,
@@ -688,11 +718,12 @@ object Compiler {
   ): CompileBackgroundTasks = {
     new CompileBackgroundTasks {
       def trigger(
-          clientClassesDir: AbsolutePath,
+          clientClassesObserver: ClientClassesObserver,
           clientReporter: Reporter,
           tracer: BraveTracer,
           clientLogger: Logger
       ): Task[Unit] = {
+        val clientClassesDir = clientClassesObserver.classesDir
         val backgroundTasks = tasks.map(f => f(clientClassesDir, clientReporter, tracer))
         Task.gatherUnordered(backgroundTasks).memoize.map(_ => ())
       }
@@ -780,19 +811,19 @@ object Compiler {
    */
   def rebaseAnalysisClassFiles(
       analysis0: CompileAnalysis,
-      readOnlyClassesDir: Path,
-      newClassesDir: Path,
+      origin: Path,
+      target: Path,
       sourceFilesWithFatalWarnings: scala.collection.Set[File]
   ): Analysis = {
     // Cast to the only internal analysis that we support
     val analysis = analysis0.asInstanceOf[Analysis]
     def rebase(file: VirtualFileRef): VirtualFileRef = {
 
       val filePath = converter.toPath(file).toAbsolutePath()
-      if (!filePath.startsWith(readOnlyClassesDir)) file
+      if (!filePath.startsWith(origin)) file
       else {
         // Hash for class file is the same because the copy duplicates metadata
-        val path = newClassesDir.resolve(readOnlyClassesDir.relativize(filePath))
+        val path = target.resolve(origin.relativize(filePath))
         converter.toVirtualFile(path)
       }
     }

backend/src/main/scala/bloop/CompilerCache.scala

@@ -35,10 +35,10 @@ import sbt.internal.util.LoggerWriter
 import xsbti.ComponentProvider
 import xsbti.VirtualFile
 import xsbti.compile.ClassFileManager
+import xsbti.compile.ClasspathOptions
 import xsbti.compile.Compilers
 import xsbti.compile.JavaCompiler
 import xsbti.compile.Output
-import xsbti.compile.ScalaCompiler
 import xsbti.{Logger => XLogger}
 import xsbti.{Reporter => XReporter}
 
@@ -50,19 +50,21 @@ final class CompilerCache(
     logger: Logger
 ) {
 
-  private val scalaCompilerCache = new ConcurrentHashMap[ScalaInstance, ScalaCompiler]()
-
+  private val scalaInstanceCache = new ConcurrentHashMap[ScalaInstance, ScalaInstance]()
   private val javaCompilerCache = new ConcurrentHashMap[JavacKey, JavaCompiler]()
 
   def get(
       scalaInstance: ScalaInstance,
+      classpathOptions: ClasspathOptions,
       javacBin: Option[AbsolutePath],
       javacOptions: List[String]
   ): Compilers = {
-    val scalaCompiler = scalaCompilerCache.computeIfAbsent(
+    val scalaInstanceFromCache = scalaInstanceCache.computeIfAbsent(
       scalaInstance,
-      getScalaCompiler(_, componentProvider)
+      _ => scalaInstance
     )
+    val scalaCompiler =
+      getScalaCompiler(scalaInstanceFromCache, classpathOptions, componentProvider)
 
     val allowLocal = !hasRuntimeJavacOptions(javacOptions)
     val javaCompiler =
@@ -108,19 +110,21 @@ final class CompilerCache(
 
   def getScalaCompiler(
       scalaInstance: ScalaInstance,
+      classpathOptions: ClasspathOptions,
       componentProvider: ComponentProvider
   ): AnalyzingCompiler = {
     val bridgeSources = BloopComponentCompiler.getModuleForBridgeSources(scalaInstance)
     val bridgeId = BloopComponentCompiler.getBridgeComponentId(bridgeSources, scalaInstance)
     componentProvider.component(bridgeId) match {
-      case Array(jar) => ZincUtil.scalaCompiler(scalaInstance, jar)
+      case Array(jar) => ZincUtil.scalaCompiler(scalaInstance, jar, classpathOptions)
       case _ =>
         BloopZincLibraryManagement.scalaCompiler(
           scalaInstance,
           BloopComponentsLock,
           componentProvider,
           Some(Paths.getCacheDirectory("bridge-cache").toFile),
           bridgeSources,
+          classpathOptions,
           logger
         )
     }

backend/src/main/scala/bloop/io/ParallelOps.scala

@@ -173,7 +173,7 @@ object ParallelOps {
           ()
         } catch {
           case NonFatal(t) =>
-            logger.report(
+            logger.error(
               s"Unexpected error when copying $originFile to $targetFile, you might need to restart the build server.",
               t
             )