Merge pull request #118 from alexarchambault/merge-upstream

Merge upstream changes

Author: alexarchambault

backend/src/main/scala/bloop/task/Task.scala

@@ -164,11 +164,14 @@ sealed trait Task[+A] { self =>
     Task
       .chooseFirstOf(
         self,
-        Task.sleep(duration).flatMap(_ => backup)
+        Task.sleep(duration)
       )
-      .map {
-        case Left((a, _)) => a
-        case Right((_, b)) => b
+      .flatMap {
+        case Left((a, _)) =>
+          Task.now(a)
+        case Right((a, _)) =>
+          a.cancel()
+          backup
       }
   }
 

backend/src/test/scala/bloop/task/TaskSpec.scala

@@ -183,4 +183,15 @@ class TaskSpec {
 
   }
 
+  @Test
+  def timeoutTo2: Unit = {
+    val ref = new AtomicBoolean(true)
+    val t1 = Task.unit.delayExecution(500.milli)
+    val t2 = Task(ref.set(false))
+    val withTimeout = t1.timeoutTo(1.second, t2)
+
+    Await.result((withTimeout *> Task.sleep(2.seconds)).runAsync, 3.second)
+    assertEquals(true, ref.get())
+  }
+
 }

config/src/main/scala/bloop/config/Config.scala

@@ -236,6 +236,12 @@ object Config {
       excludes: List[String]
   )
 
+  case class SourceGenerator(
+      sourcesGlobs: List[SourcesGlobs],
+      outputDirectory: Path,
+      command: List[String]
+  )
+
   case class Project(
       name: String,
       directory: Path,
@@ -254,12 +260,13 @@ object Config {
       test: Option[Test],
       platform: Option[Platform],
       resolution: Option[Resolution],
-      tags: Option[List[String]]
+      tags: Option[List[String]],
+      sourceGenerators: Option[List[SourceGenerator]]
   )
 
   object Project {
     // FORMAT: OFF
-    private[bloop] val empty: Project = Project("", emptyPath, None, List(), None, None, List(), List(),  emptyPath, emptyPath, None, None, None, None, None, None, None, None)
+    private[bloop] val empty: Project = Project("", emptyPath, None, List(), None, None, List(), List(),  emptyPath, emptyPath, None, None, None, None, None, None, None, None, None)
     // FORMAT: ON
 
     def analysisFileName(projectName: String): String = s"$projectName-analysis.bin"
@@ -328,6 +335,7 @@ object Config {
         Some(Test(List(), TestOptions(Nil, Nil))),
         Some(platform),
         Some(Resolution(Nil)),
+        None,
         None
       )
 

frontend/src/it/scala/bloop/CommunityBuild.scala

@@ -21,7 +21,7 @@ import bloop.engine.{
   Run,
   State
 }
-import bloop.engine.caches.ResultsCache
+import bloop.engine.caches.{ResultsCache, SourceGeneratorCache}
 import bloop.io.AbsolutePath
 import bloop.logging.{BloopLogger, Logger, NoopLogger}
 import bloop.task.Task
@@ -95,7 +95,7 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) {
     val loadedProjects = BuildLoader.loadSynchronously(configDirectory, logger)
     val workspaceSettings = WorkspaceSettings.readFromFile(configDirectory, logger)
     val build = Build(configDirectory, loadedProjects, workspaceSettings)
-    val state = State.forTests(build, compilerCache, logger)
+    val state = State.forTests(build, compilerCache, SourceGeneratorCache.empty, logger)
     state.copy(results = ResultsCache.emptyForTests)
   }
 
@@ -139,6 +139,7 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) {
         sources = Nil,
         sourcesGlobs = Nil,
         sourceRoots = None,
+        sourceGenerators = Nil,
         testFrameworks = Nil,
         testOptions = Config.TestOptions.empty,
         out = dummyClassesDir,

frontend/src/main/scala/bloop/Cli.scala

@@ -88,8 +88,16 @@ object Cli {
       else parse(args, nailgunOptions)
     }
 
-    val exitStatus = run(cmd, NailgunPool(ngContext))
-    ngContext.exit(exitStatus.code)
+    try {
+      val exitStatus = run(cmd, NailgunPool(ngContext))
+      ngContext.exit(exitStatus.code)
+    } catch {
+      case x: java.util.concurrent.ExecutionException =>
+        // print stack trace of fatal errors thrown in asynchronous code, see https://stackoverflow.com/questions/17265022/what-is-a-boxed-error-in-scala
+        // the stack trace is somehow propagated all the way to the client when printing this
+        x.getCause.printStackTrace(ngContext.out)
+        ngContext.exit(ExitStatus.UnexpectedError.code)
+    }
   }
 
   val commands: Seq[String] = Commands.RawCommand.help.messages.flatMap(_._1.headOption.toSeq)

frontend/src/main/scala/bloop/bsp/BloopBspServices.scala

@@ -20,6 +20,14 @@ import ch.epfl.scala.bsp
 import ch.epfl.scala.bsp.BuildTargetIdentifier
 import ch.epfl.scala.bsp.MessageType
 import ch.epfl.scala.bsp.ShowMessageParams
+import ch.epfl.scala.bsp.CompileResult
+import ch.epfl.scala.bsp.StatusCode
+import ch.epfl.scala.bsp.Uri
+import ch.epfl.scala.bsp.endpoints
+import ch.epfl.scala.bsp.CompileResult
+import ch.epfl.scala.bsp.MessageType
+import ch.epfl.scala.bsp.ShowMessageParams
+import ch.epfl.scala.bsp.StatusCode
 import ch.epfl.scala.bsp.Uri
 import ch.epfl.scala.bsp.endpoints
 import ch.epfl.scala.debugadapter.DebugServer
@@ -302,7 +310,7 @@ final class BloopBspServices(
               compileProvider = Some(BloopBspServices.DefaultCompileProvider),
               testProvider = Some(BloopBspServices.DefaultTestProvider),
               runProvider = Some(BloopBspServices.DefaultRunProvider),
-              debugProvider = None, // todo kpodsiad
+              debugProvider = Some(BloopBspServices.DefaultDebugProvider),
               inverseSourcesProvider = Some(true),
               dependencySourcesProvider = Some(true),
               dependencyModulesProvider = None,
@@ -689,10 +697,7 @@ final class BloopBspServices(
   }
 
   def test(params: bsp.TestParams): BspEndpointResponse[bsp.TestResult] = {
-    def test(
-        project: Project,
-        state: State
-    ): Task[State] = {
+    def test(project: Project, state: State): Task[Tasks.TestRuns] = {
       val testFilter = TestInternals.parseFilters(Nil) // Don't support test only for now
       val handler = new LoggingEventHandler(state.logger)
       Tasks.test(
@@ -717,25 +722,35 @@ final class BloopBspServices(
           val args = params.arguments.getOrElse(Nil)
           val logger = logger0.asBspServerVerbose
           compileProjects(mappings, state, args, originId, logger).flatMap {
-            case (newState, compileResult) =>
-              compileResult match {
-                case Right(_) =>
-                  val sequentialTestExecution = mappings.foldLeft(Task.now(newState)) {
-                    case (taskState, (_, p)) => taskState.flatMap(state => test(p, state))
-                  }
-
-                  sequentialTestExecution.materialize.map(_.toEither).map {
-                    case Right(newState) =>
+            case (newState, Right(CompileResult(_, StatusCode.Ok, _, _))) =>
+              val sequentialTestExecution: Task[Seq[Tasks.TestRuns]] =
+                Task.sequence(mappings.map { case (_, p) => test(p, state) })
+
+              sequentialTestExecution.materialize.map {
+                case Success(testRunsSeq) =>
+                  testRunsSeq.reduceOption(_ ++ _) match {
+                    case None =>
                       (newState, Right(bsp.TestResult(originId, bsp.StatusCode.Ok, None, None)))
-                    case Left(e) =>
-                      //(newState, Right(bsp.TestResult(None, bsp.StatusCode.Error, None)))
-                      val errorMessage =
-                        Response.internalError(s"Failed test execution: ${e.getMessage}")
-                      (newState, Left(errorMessage))
+                    case Some(testRuns) =>
+                      val status = testRuns.status
+                      val bspStatus =
+                        if (status == ExitStatus.Ok) bsp.StatusCode.Ok else bsp.StatusCode.Error
+                      (
+                        newState.mergeStatus(status),
+                        Right(bsp.TestResult(originId, bspStatus, None, None))
+                      )
                   }
-
-                case Left(error) => Task.now((newState, Left(error)))
+                case Failure(e) =>
+                  val errorMessage =
+                    Response.internalError(s"Failed test execution: ${e.getMessage}")
+                  (newState, Left(errorMessage))
               }
+
+            case (newState, Right(CompileResult(_, errorCode, _, _))) =>
+              Task.now((newState, Right(bsp.TestResult(originId, errorCode, None, None))))
+
+            case (newState, Left(error)) =>
+              Task.now((newState, Left(error)))
           }
       }
     }
@@ -1026,7 +1041,8 @@ final class BloopBspServices(
               val capabilities = bsp.BuildTargetCapabilities(
                 canCompile = true,
                 canTest = true,
-                canRun = true
+                canRun = true,
+                canDebug = true
               )
               val isJavaOnly = p.scalaInstance.isEmpty
               val languageIds =
@@ -1064,29 +1080,49 @@ final class BloopBspServices(
         projects: Seq[ProjectMapping],
         state: State
     ): BspResult[bsp.SourcesResult] = {
-      val sourcesItems = projects.iterator.map {
-        case (target, project) =>
-          project.allSourceFilesAndDirectories.map { sources =>
-            val items = sources.map { s =>
-              import bsp.SourceItemKind._
-              val uri = s.underlying.toUri()
-              val (bspUri, kind) = if (s.exists) {
-                (uri, if (s.isFile) File else Directory)
-              } else {
-                val fileMatcher = FileSystems.getDefault.getPathMatcher("glob:*.{scala, java}")
-                if (fileMatcher.matches(s.underlying.getFileName)) (uri, File)
-                // If path doesn't exist and its name doesn't look like a file, assume it's a dir
-                else (new URI(uri.toString + "/"), Directory)
-              }
-              // TODO(jvican): Don't default on false for generated, add this info to JSON fields
-              bsp.SourceItem(bsp.Uri(bspUri), kind, false)
-            }
+      def sourceItem(s: AbsolutePath, isGenerated: Boolean): bsp.SourceItem = {
+        import bsp.SourceItemKind._
+        val uri = s.underlying.toUri()
+        val (bspUri, kind) = if (s.exists) {
+          (uri, if (s.isFile) File else Directory)
+        } else {
+          val fileMatcher = FileSystems.getDefault.getPathMatcher("glob:*.{scala, java}")
+          if (fileMatcher.matches(s.underlying.getFileName)) (uri, File)
+          // If path doesn't exist and its name doesn't look like a file, assume it's a dir
+          else (new URI(uri.toString + "/"), Directory)
+        }
+        bsp.SourceItem(bsp.Uri(bspUri), kind, isGenerated)
+      }
+
+      val dag = Aggregate(projects.map(p => state.build.getDagFor(p._2)).toList)
+
+      // Collect the projects' sources following the projects topological sorting, so that
+      // source generators that depend on other source generators' outputs can run correctly.
+      val collectSourcesTasks = Dag.topologicalSort(dag).map { project =>
+        val unmanagedSources = project.allUnmanagedSourceFilesAndDirectories.map { sources =>
+          sources.map(sourceItem(_, isGenerated = false))
+        }
+
+        val managedSources = {
+          val tasks = project.sourceGenerators
+            .map(state.sourceGeneratorCache.update(_, state.logger, state.commonOptions))
+          Task.gatherUnordered(tasks).map(_.flatten.map(sourceItem(_, isGenerated = true)))
+        }
+
+        for {
+          unmanaged <- unmanagedSources
+          managed <- managedSources
+        } yield (project, unmanaged ++ managed)
+      }
+
+      val projectToTarget = projects.map { case (target, project) => project -> target }.toMap
+      Task.sequence(collectSourcesTasks).map { results =>
+        val items = results.flatMap {
+          case (project, items) =>
             val roots = project.sourceRoots.map(_.map(p => bsp.Uri(p.toBspUri)))
-            bsp.SourcesItem(target, items, roots)
-          }
-      }.toList
+            projectToTarget.get(project).map(bsp.SourcesItem(_, items, roots))
+        }
 
-      Task.sequence(sourcesItems).map { items =>
         (state, Right(bsp.SourcesResult(items)))
       }
     }
@@ -1309,4 +1345,5 @@ object BloopBspServices {
   private[bloop] val DefaultCompileProvider = bsp.CompileProvider(DefaultLanguages)
   private[bloop] val DefaultTestProvider = bsp.TestProvider(DefaultLanguages)
   private[bloop] val DefaultRunProvider = bsp.RunProvider(DefaultLanguages)
+  private[bloop] val DefaultDebugProvider = bsp.DebugProvider(DefaultLanguages)
 }

frontend/src/main/scala/bloop/bsp/BloopRpcServices.scala

@@ -41,6 +41,18 @@ object BloopRpcServices {
 
   object BloopEndpoint {
 
+    private def extractJsonParams(params: Option[RawJson]): RawJson = {
+      // if params are empty, bsp4s endpoint's codec might decode it as just a 'null'
+      // and then parsing it using `readFromArray` fails
+      // to avoid this issue replace 'null' by empty object '{}'
+      params
+        .map(json =>
+          if (java.util.Arrays.equals(json.value, RawJson.nullValue.value)) RawJson.emptyObj
+          else json
+        )
+        .getOrElse(RawJson.emptyObj)
+    }
+
     def request[A, B](
         endpoint: Endpoint[A, B]
     )(f: A => Task[Either[Response.Error, B]]): BloopEndpoint =
@@ -51,7 +63,7 @@ object BloopRpcServices {
           val method = endpoint.method
           message match {
             case Request(`method`, params, id, jsonrpc, headers) =>
-              val paramsJson = params.flatMap(v => Option(v)).getOrElse(RawJson.emptyObj)
+              val paramsJson = extractJsonParams(params)
               Try(readFromArray[A](paramsJson.value)) match {
                 case Success(value) =>
                   f(value).materialize.map { v =>
@@ -87,10 +99,11 @@ object BloopRpcServices {
           val method = endpoint.method
           message match {
             case Notification(`method`, params, _, headers) =>
-              val paramsJson = params.flatMap(v => Option(v)).getOrElse(RawJson.emptyObj)
+              val paramsJson = extractJsonParams(params)
               Try(readFromArray[A](paramsJson.value)) match {
                 case Success(value) => f(value).map(a => Response.None)
-                case Failure(err) => fail(s"Failed to parse notification $message. Errors: $err")
+                case Failure(err) =>
+                  fail(s"Failed to parse notification $message. Params: $paramsJson. Errors: $err")
               }
             case Notification(invalidMethod, _, _, headers) =>
               fail(s"Expected method '$method', obtained '$invalidMethod'")

frontend/src/main/scala/bloop/data/Project.scala

@@ -14,6 +14,7 @@ import bloop.bsp.ProjectUris
 import bloop.config.Config
 import bloop.config.ConfigCodecs
 import bloop.engine.Dag
+import bloop.engine.SourceGenerator
 import bloop.engine.tasks.toolchains.JvmToolchain
 import bloop.engine.tasks.toolchains.ScalaJsToolchain
 import bloop.engine.tasks.toolchains.ScalaNativeToolchain
@@ -45,6 +46,7 @@ final case class Project(
     sources: List[AbsolutePath],
     sourcesGlobs: List[SourcesGlobs],
     sourceRoots: Option[List[AbsolutePath]],
+    sourceGenerators: List[SourceGenerator],
     testFrameworks: List[Config.TestFramework],
     testOptions: Config.TestOptions,
     out: AbsolutePath,
@@ -88,8 +90,11 @@ final case class Project(
     customWorkingDirectory.getOrElse(baseDirectory)
   }
 
+  def allGeneratorInputs: Task[List[AbsolutePath]] =
+    Task.sequence(sourceGenerators.map(_.getSources)).map(_.flatten)
+
   /** Returns concatenated list of "sources" and expanded "sourcesGlobs". */
-  def allSourceFilesAndDirectories: Task[List[AbsolutePath]] = Task {
+  def allUnmanagedSourceFilesAndDirectories: Task[List[AbsolutePath]] = Task {
     val buf = mutable.ListBuffer.empty[AbsolutePath]
     buf ++= sources
     for (glob <- sourcesGlobs) glob.walkThrough(buf += _)
@@ -354,10 +359,11 @@ object Project {
     val sourceRoots = project.sourceRoots.map(_.map(AbsolutePath.apply))
 
     val tags = project.tags.getOrElse(Nil)
+    val projectDirectory = AbsolutePath(project.directory)
 
     Project(
       project.name,
-      AbsolutePath(project.directory),
+      projectDirectory,
       project.workspaceDir.map(AbsolutePath.apply),
       project.dependencies,
       instance,
@@ -370,6 +376,7 @@ object Project {
       project.sources.map(AbsolutePath.apply),
       SourcesGlobs.fromConfig(project, logger),
       sourceRoots,
+      project.sourceGenerators.getOrElse(Nil).map(SourceGenerator.fromConfig(projectDirectory, _)),
       project.test.map(_.frameworks).getOrElse(Nil),
       project.test.map(_.options).getOrElse(Config.TestOptions.empty),
       AbsolutePath(project.out),