From 47134ddfcb5f77b5cfa2cfff6261d6460374f6c2 Mon Sep 17 00:00:00 2001 From: tgodzik Date: Tue, 25 Jun 2019 17:19:26 +0200 Subject: [PATCH 01/22] Add an additional mechanism to save information about the samenticDB plugin version for Metals. That information is then used to add scala options to compiler instances for the SemanticDB plugin. --- .gitignore | 3 +- .../scala/bloop/BloopComponentsLock.scala | 4 +- .../src/main/scala/bloop/CompilerCache.scala | 3 +- .../internal/inc/ZincComponentManager.scala | 4 +- .../scala/sbt/internal/inc/ZincLmUtil.scala | 5 +- .../src/it/scala/bloop/CommunityBuild.scala | 4 +- frontend/src/main/scala/bloop/Bloop.scala | 4 +- .../scala/bloop/bsp/BloopBspDefinitions.scala | 3 +- .../scala/bloop/bsp/BloopBspServices.scala | 65 +++++++--- .../main/scala/bloop/data/LoadedBuild.scala | 3 + .../scala/bloop/data/WorkspaceSettings.scala | 71 +++++++++++ .../src/main/scala/bloop/engine/Build.scala | 12 +- .../main/scala/bloop/engine/BuildLoader.scala | 119 ++++++++++++++++-- .../scala/bloop/engine/SemanticDBCache.scala | 75 +++++++++++ .../src/main/scala/bloop/engine/State.scala | 12 +- .../bloop/engine/caches/StateCache.scala | 28 +++-- .../test/scala/bloop/bsp/BspBaseSuite.scala | 22 ++-- .../scala/bloop/bsp/BspMetalsClientSpec.scala | 109 ++++++++++++++++ .../scala/bloop/bsp/BspProtocolSpec.scala | 14 ++- .../src/test/scala/bloop/util/TestUtil.scala | 11 +- .../gradle/ConfigGenerationSuite.scala | 4 +- .../main/scala/bloop/io/AbsolutePath.scala | 5 + 22 files changed, 494 insertions(+), 86 deletions(-) create mode 100644 frontend/src/main/scala/bloop/data/LoadedBuild.scala create mode 100644 frontend/src/main/scala/bloop/data/WorkspaceSettings.scala create mode 100644 frontend/src/main/scala/bloop/engine/SemanticDBCache.scala create mode 100644 frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala diff --git a/.gitignore b/.gitignore index e2dd65dcf8..b960890510 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .idea/ bin/.coursier bin/.scalafmt* +.vscode/ # Required because these are the proxies for the sourcedeps .bridge/ @@ -33,4 +34,4 @@ node_modules/ package-lock.json .metals/ *.lock -benchmark-bridge/corpus/ \ No newline at end of file +benchmark-bridge/corpus/ diff --git a/backend/src/main/scala/bloop/BloopComponentsLock.scala b/backend/src/main/scala/bloop/BloopComponentsLock.scala index 8d0356a590..4db2a2d65d 100644 --- a/backend/src/main/scala/bloop/BloopComponentsLock.scala +++ b/backend/src/main/scala/bloop/BloopComponentsLock.scala @@ -5,6 +5,8 @@ import java.util.concurrent.Callable import xsbti.GlobalLock -object BloopComponentsLock extends GlobalLock { +trait ComponentLock extends GlobalLock { override def apply[T](file: File, callable: Callable[T]): T = synchronized { callable.call() } } + +object BloopComponentsLock extends ComponentLock diff --git a/backend/src/main/scala/bloop/CompilerCache.scala b/backend/src/main/scala/bloop/CompilerCache.scala index 7a680a48dc..a79d7a999f 100644 --- a/backend/src/main/scala/bloop/CompilerCache.scala +++ b/backend/src/main/scala/bloop/CompilerCache.scala @@ -85,8 +85,7 @@ final class CompilerCache( Some(Paths.getCacheDirectory("bridge-cache").toFile), DependencyResolution.getEngine(userResolvers), bridgeSources, - retrieveDir.toFile, - logger + retrieveDir.toFile ) } } diff --git a/backend/src/main/scala/sbt/internal/inc/ZincComponentManager.scala b/backend/src/main/scala/sbt/internal/inc/ZincComponentManager.scala index c444224830..1ffab5254d 100644 --- a/backend/src/main/scala/sbt/internal/inc/ZincComponentManager.scala +++ b/backend/src/main/scala/sbt/internal/inc/ZincComponentManager.scala @@ -29,10 +29,8 @@ import xsbti.ArtifactInfo.SbtOrganization class ZincComponentManager( globalLock: GlobalLock, provider: ComponentProvider, - secondaryCacheDir: Option[File], - log0: Logger, + secondaryCacheDir: Option[File] ) { - val log = new FullLogger(log0) /** Get all of the files for component 'id', throwing an exception if no files exist for the component. */ def files(id: String)(ifMissing: IfMissing): Iterable[File] = { diff --git a/backend/src/main/scala/sbt/internal/inc/ZincLmUtil.scala b/backend/src/main/scala/sbt/internal/inc/ZincLmUtil.scala index fe2e96659a..e862e9d33c 100644 --- a/backend/src/main/scala/sbt/internal/inc/ZincLmUtil.scala +++ b/backend/src/main/scala/sbt/internal/inc/ZincLmUtil.scala @@ -30,12 +30,11 @@ object ZincLmUtil { secondaryCacheDir: Option[File], dependencyResolution: DependencyResolution, compilerBridgeSource: ModuleID, - scalaJarsTarget: File, - log: Logger + scalaJarsTarget: File ): AnalyzingCompiler = { val compilerBridgeProvider = ZincComponentCompiler.interfaceProvider( compilerBridgeSource, - new ZincComponentManager(globalLock, componentProvider, secondaryCacheDir, log), + new ZincComponentManager(globalLock, componentProvider, secondaryCacheDir), dependencyResolution, scalaJarsTarget ) diff --git a/frontend/src/it/scala/bloop/CommunityBuild.scala b/frontend/src/it/scala/bloop/CommunityBuild.scala index 89ac78981e..230de99961 100644 --- a/frontend/src/it/scala/bloop/CommunityBuild.scala +++ b/frontend/src/it/scala/bloop/CommunityBuild.scala @@ -88,8 +88,8 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) { def loadStateForBuild(configDirectory: AbsolutePath, logger: Logger): State = { assert(configDirectory.exists, "Does not exist: " + configDirectory) - val loadedProjects = BuildLoader.loadSynchronously(configDirectory, logger) - val build = Build(configDirectory, loadedProjects) + val loadedBuild = BuildLoader.loadSynchronously(configDirectory, logger) + val build = Build(configDirectory, loadedBuild.workspaceSettings, loadedBuild.projects) val state = State.forTests(build, compilerCache, logger) state.copy(results = ResultsCache.emptyForTests) } diff --git a/frontend/src/main/scala/bloop/Bloop.scala b/frontend/src/main/scala/bloop/Bloop.scala index de9a43c8d8..054764dbcc 100644 --- a/frontend/src/main/scala/bloop/Bloop.scala +++ b/frontend/src/main/scala/bloop/Bloop.scala @@ -35,8 +35,8 @@ object Bloop extends CaseApp[CliOptions] { ) logger.warn("Please refer to our documentation for more information.") val client = ClientInfo.CliClientInfo("bloop-single-app", () => true) - val projects = BuildLoader.loadSynchronously(configDirectory, logger) - val build = Build(configDirectory, projects) + val loadedBuild = BuildLoader.loadSynchronously(configDirectory, logger) + val build = Build(configDirectory, loadedBuild.workspaceSettings, loadedBuild.projects) val state = State(build, client, NoPool, options.common, logger) run(state, options) } diff --git a/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala b/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala index 97cf8cebb0..b790776d39 100644 --- a/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala +++ b/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala @@ -6,7 +6,8 @@ import ch.epfl.scala.bsp.Uri object BloopBspDefinitions { final case class BloopExtraBuildParams( - clientClassesRootDir: Option[Uri] + clientClassesRootDir: Option[Uri], + semanticDBVersion : Option[String] ) object BloopExtraBuildParams { diff --git a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala index 4529ba7a18..3d82d4e932 100644 --- a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala +++ b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala @@ -40,6 +40,7 @@ import scala.util.Success import scala.util.Failure import monix.execution.Cancelable import io.circe.Json +import bloop.data.WorkspaceSettings final class BloopBspServices( callSiteState: State, @@ -108,25 +109,31 @@ final class BloopBspServices( } private val previouslyFailedCompilations = new TrieMap[Project, Compiler.Result.Failed]() - private def reloadState(config: AbsolutePath, clientInfo: ClientInfo): Task[State] = { + private def reloadState( + config: AbsolutePath, + clientInfo: ClientInfo, + workspaceSettings: Option[WorkspaceSettings] = None + ): Task[State] = { val pool = currentState.pool val defaultOpts = currentState.commonOptions bspLogger.debug(s"Reloading bsp state for ${config.syntax}") - State.loadActiveStateFor(config, clientInfo, pool, defaultOpts, bspLogger).map { state0 => - /* Create a new state that has the previously compiled results in this BSP - * client as the last compiled result available for a project. This is required - * because in diagnostics reporting in BSP is stateful. When compilations - * happen in other clients, the previous result does not contain the list of - * previous problems (that tracks where we reported diagnostics) that this client - * had and therefore we can fail to reset diagnostics. */ - val newState = { - val previous = previouslyFailedCompilations.toMap - state0.copy(results = state0.results.replacePreviousResults(previous)) - } + State + .loadActiveStateFor(config, clientInfo, pool, defaultOpts, bspLogger, workspaceSettings) + .map { state0 => + /* Create a new state that has the previously compiled results in this BSP + * client as the last compiled result available for a project. This is required + * because in diagnostics reporting in BSP is stateful. When compilations + * happen in other clients, the previous result does not contain the list of + * previous problems (that tracks where we reported diagnostics) that this client + * had and therefore we can fail to reset diagnostics. */ + val newState = { + val previous = previouslyFailedCompilations.toMap + state0.copy(results = state0.results.replacePreviousResults(previous)) + } - currentState = newState - newState - } + currentState = newState + newState + } } private def saveState(state: State): Task[Unit] = { @@ -175,7 +182,10 @@ final class BloopBspServices( ): BspEndpointResponse[bsp.InitializeBuildResult] = { val uri = new java.net.URI(params.rootUri.value) val configDir = AbsolutePath(uri).resolve(relativeConfigPath) - val clientClassesRootDir = parseClientClassesRootDir(params.data) + val extraBuildParams = parseClientClassesRootDir(params.data) + val clientClassesRootDir = extraBuildParams.flatMap( + extra => extra.clientClassesRootDir.map(dir => AbsolutePath(dir.toPath)) + ) val client = ClientInfo.BspClientInfo( params.displayName, params.version, @@ -184,8 +194,23 @@ final class BloopBspServices( () => isClientConnected.get ) - reloadState(configDir, client).map { state => - callSiteState.logger.info("request received: build/initialize") + /* Metals specific settings that are used to store the + * SemanticDB version that will later be applied to all + * projects in the workspace. If the client is Metals but + * the version is not specified we use `latest.release` + */ + val metalsSettings = + if (params.displayName.contains("Metals")) { + val semanticDBVersion = extraBuildParams + .flatMap(extra => extra.semanticDBVersion) + .getOrElse(SemanticDBCache.latestRelease) + Some(WorkspaceSettings(semanticDBVersion)) + } else { + None + } + + reloadState(configDir, client, metalsSettings).map { state => + callSiteState.logger.info(s"request received: build/initialize") clientInfo.success(client) connectedBspClients.put(client, configDir) observer.foreach(_.onNext(state.copy(client = client))) @@ -209,11 +234,11 @@ final class BloopBspServices( } } - private def parseClientClassesRootDir(data: Option[Json]): Option[AbsolutePath] = { + private def parseClientClassesRootDir(data: Option[Json]): Option[BloopExtraBuildParams] = { data.flatMap { json => BloopExtraBuildParams.decoder.decodeJson(json) match { case Right(bloopParams) => - bloopParams.clientClassesRootDir.map(dir => AbsolutePath(dir.toPath)) + Some(bloopParams) case Left(failure) => callSiteState.logger.warn( s"Unexpected error decoding bloop-specific initialize params: ${failure.message}" diff --git a/frontend/src/main/scala/bloop/data/LoadedBuild.scala b/frontend/src/main/scala/bloop/data/LoadedBuild.scala new file mode 100644 index 0000000000..7cb41c485b --- /dev/null +++ b/frontend/src/main/scala/bloop/data/LoadedBuild.scala @@ -0,0 +1,3 @@ +package bloop.data + +case class LoadedBuild(projects : List[Project], workspaceSettings : Option[WorkspaceSettings]) \ No newline at end of file diff --git a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala new file mode 100644 index 0000000000..2d91122674 --- /dev/null +++ b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala @@ -0,0 +1,71 @@ +package bloop.data + +import bloop.engine.BuildLoader +import bloop.logging.Logger +import bloop.logging.DebugFilter +import java.nio.charset.StandardCharsets +import bloop.config.ConfigEncoderDecoders +import io.circe.Printer +import io.circe.Json +import io.circe.parser +import bloop.io.AbsolutePath +import java.nio.file.Files +import io.circe.JsonObject +import bloop.DependencyResolution +import scala.util.Try +import io.circe.ObjectEncoder +import io.circe.Decoder +import io.circe.derivation._ +import java.nio.file.Path +import scala.util.Failure +import scala.util.Success + +case class WorkspaceSettings(semanticDBVersion: String) + +object WorkspaceSettings { + + /** File to store Metals specific settings*/ + val settingsFileName = "bloop.settings.json" + private val settingsEncoder: ObjectEncoder[WorkspaceSettings] = deriveEncoder + private val settingsDecoder: Decoder[WorkspaceSettings] = deriveDecoder + + def fromFile(configPath: AbsolutePath, logger: Logger): Option[WorkspaceSettings] = { + val settingsPath = configPath.resolve(settingsFileName) + if (settingsPath.isFile) { + val bytes = Files.readAllBytes(settingsPath.underlying) + logger.debug(s"Loading workspace settings from $settingsFileName")( + DebugFilter.All + ) + val contents = new String(bytes, StandardCharsets.UTF_8) + parser.parse(contents) match { + case Left(failure) => throw failure + case Right(json) => Option(fromJson(json)) + } + } else { + None + } + } + + def write(configDir: AbsolutePath, settings: WorkspaceSettings): Either[Throwable, Path] = { + Try { + val jsonObject = settingsEncoder(settings) + val output = Printer.spaces4.copy(dropNullValues = true).pretty(jsonObject) + Files.write( + configDir.resolve(settingsFileName).underlying, + output.getBytes(StandardCharsets.UTF_8) + ) + } match { + case Failure(exception) => Left(exception) + case Success(value) => Right(value) + } + } + + def fromJson(json: Json): WorkspaceSettings = { + settingsDecoder.decodeJson(json) match { + case Right(settings) => settings + case Left(failure) => throw failure + } + } + + +} diff --git a/frontend/src/main/scala/bloop/engine/Build.scala b/frontend/src/main/scala/bloop/engine/Build.scala index 85ba113403..d9041b4a11 100644 --- a/frontend/src/main/scala/bloop/engine/Build.scala +++ b/frontend/src/main/scala/bloop/engine/Build.scala @@ -7,9 +7,11 @@ import bloop.logging.Logger import bloop.util.CacheHashCode import bloop.io.ByteHasher import monix.eval.Task +import bloop.data.WorkspaceSettings final case class Build private ( origin: AbsolutePath, + settings: Option[WorkspaceSettings], projects: List[Project] ) extends CacheHashCode { private val stringToProjects: Map[String, Project] = projects.map(p => p.name -> p).toMap @@ -32,8 +34,11 @@ final case class Build private ( val files = projects.iterator.map(p => p.origin.toAttributedPath).toSet val newFiles = BuildLoader.readConfigurationFilesInBase(origin, logger).toSet + val loadedSettings = WorkspaceSettings.fromFile(origin, logger) + val sameSettings = loadedSettings == settings + // This is the fast path to short circuit quickly if they are the same - if (newFiles == files) Task.now(Build.ReturnPreviousState) + if (newFiles == files && sameSettings) Task.now(Build.ReturnPreviousState) else { val filesToAttributed = projects.iterator.map(p => p.origin.path -> p).toMap // There has been at least either one addition, one removal or one change in a file time @@ -58,7 +63,7 @@ final case class Build private ( val deleted = files.toList.collect { case f if !newToAttributed.contains(f.path) => f.path } (newOrModified, deleted) match { case (Nil, Nil) => Build.ReturnPreviousState - case _ => Build.UpdateState(newOrModified, deleted) + case _ => Build.UpdateState(newOrModified, deleted, loadedSettings) } } } @@ -70,7 +75,8 @@ object Build { case object ReturnPreviousState extends ReloadAction case class UpdateState( createdOrModified: List[ReadConfiguration], - deleted: List[AbsolutePath] + deleted: List[AbsolutePath], + settingsChanged: Option[WorkspaceSettings] ) extends ReloadAction /** A configuration file is a combination of an absolute path and a file time. */ diff --git a/frontend/src/main/scala/bloop/engine/BuildLoader.scala b/frontend/src/main/scala/bloop/engine/BuildLoader.scala index d23e94596a..4d3224449f 100644 --- a/frontend/src/main/scala/bloop/engine/BuildLoader.scala +++ b/frontend/src/main/scala/bloop/engine/BuildLoader.scala @@ -6,6 +6,8 @@ import bloop.io.AbsolutePath import bloop.logging.{DebugFilter, Logger} import bloop.io.ByteHasher import monix.eval.Task +import bloop.data.WorkspaceSettings +import bloop.data.LoadedBuild object BuildLoader { @@ -22,7 +24,9 @@ object BuildLoader { * @return A map associating each tracked file with its last modification time. */ def readConfigurationFilesInBase(base: AbsolutePath, logger: Logger): List[AttributedPath] = { - bloop.io.Paths.attributedPathFilesUnder(base, JsonFilePattern, logger, 1) + bloop.io.Paths + .attributedPathFilesUnder(base, JsonFilePattern, logger, 1) + .filterNot(_.path.toFile.getName() == WorkspaceSettings.settingsFileName) } /** @@ -35,15 +39,19 @@ object BuildLoader { def loadBuildFromConfigurationFiles( configDir: AbsolutePath, configFiles: List[Build.ReadConfiguration], + settingsFile: Option[WorkspaceSettings], logger: Logger - ): Task[List[Project]] = { + ): Task[LoadedBuild] = { + logger.debug(s"Loading ${configFiles.length} projects from '${configDir.syntax}'...")( DebugFilter.Compilation ) - - val all = configFiles.map(f => Task(Project.fromBytesAndOrigin(f.bytes, f.origin, logger))) + val all = configFiles.map(f => Task(loadProject(f.bytes, f.origin, logger, settingsFile))) val groupTasks = all.grouped(10).map(group => Task.gatherUnordered(group)).toList - Task.sequence(groupTasks).map(_.flatten).executeOn(ExecutionContext.ioScheduler) + Task + .sequence(groupTasks) + .map(fp => LoadedBuild(fp.flatten, settingsFile)) + .executeOn(ExecutionContext.ioScheduler) } /** @@ -55,8 +63,10 @@ object BuildLoader { */ def load( configDir: AbsolutePath, + incomingSettings: Option[WorkspaceSettings], logger: Logger - ): Task[List[Project]] = { + ): Task[LoadedBuild] = { + val workspaceSettings = Task(updateWorkspaceSettings(configDir, logger, incomingSettings)) val configFiles = readConfigurationFilesInBase(configDir, logger).map { ap => Task { val bytes = ap.path.readAllBytes @@ -65,9 +75,13 @@ object BuildLoader { } } - Task - .gatherUnordered(configFiles) - .flatMap(fs => loadBuildFromConfigurationFiles(configDir, fs, logger)) + workspaceSettings.flatMap { settings => + Task + .gatherUnordered(configFiles) + .flatMap { fs => + loadBuildFromConfigurationFiles(configDir, fs, settings, logger) + } + } } /** @@ -80,7 +94,8 @@ object BuildLoader { def loadSynchronously( configDir: AbsolutePath, logger: Logger - ): List[Project] = { + ): LoadedBuild = { + val settings = WorkspaceSettings.fromFile(configDir, logger) val configFiles = readConfigurationFilesInBase(configDir, logger).map { ap => val bytes = ap.path.readAllBytes val hash = ByteHasher.hashBytes(bytes) @@ -90,6 +105,88 @@ object BuildLoader { logger.debug(s"Loading ${configFiles.length} projects from '${configDir.syntax}'...")( DebugFilter.Compilation ) - configFiles.map(f => Project.fromBytesAndOrigin(f.bytes, f.origin, logger)) + LoadedBuild(configFiles.map(f => loadProject(f.bytes, f.origin, logger, settings)), settings) + } + + private def loadProject( + bytes: Array[Byte], + origin: Origin, + logger: Logger, + settings: Option[WorkspaceSettings] + ): Project = { + val project = Project.fromBytesAndOrigin(bytes, origin, logger) + settings.map(applySettings(_, project, logger)).getOrElse(project) + } + + private def updateWorkspaceSettings( + configDir: AbsolutePath, + logger: Logger, + incomingSettings: Option[WorkspaceSettings] + ): Option[WorkspaceSettings] = { + val savedSettings = WorkspaceSettings.fromFile(configDir, logger) + incomingSettings match { + case Some(incoming) => + if (savedSettings.isEmpty || savedSettings.exists(_ != incoming)) { + WorkspaceSettings.write(configDir, incoming) + Some(incoming) + } else { + savedSettings + } + case None => + savedSettings + } + } + + /** + * Applies workspace settings from bloop.settings.json file to a project. This includes: + * - SemanticDB plugin version to resolve and include in Scala compiler options + */ + private def applySettings( + settings: WorkspaceSettings, + project: Project, + logger: Logger + ): Project = { + + def addSemanticDBOptions(pluginPath: AbsolutePath) = { + { + val optionsSet = project.scalacOptions.toSet + val containsSemanticDB = optionsSet.find( + setting => setting.contains("-Xplugin") && setting.contains("semanticdb-scalac") + ) + val containsYrangepos = optionsSet.find(_.contains("-Yrangepos")) + val semanticDBAdded = if (containsSemanticDB.isDefined) { + logger.info(s"SemanticDB plugin already added: ${containsSemanticDB.get}") + optionsSet + } else { + optionsSet ++ Set( + "-P:semanticdb:failures:warning", + s"-P:semanticdb:sourceroot:${project.baseDirectory}", + "-P:semanticdb:synthetics:on", + "-Xplugin-require:semanticdb", + s"-Xplugin:$pluginPath" + ) + } + if (containsYrangepos.isDefined) { + semanticDBAdded + } else { + semanticDBAdded + "-Yrangepos" + } + } + } + + val mappedProject = for { + scalaInstance <- project.scalaInstance + pluginPath <- SemanticDBCache.findSemanticDBPlugin( + scalaInstance.version, + settings.semanticDBVersion, + logger + ) + } yield { + val scalacOptions = addSemanticDBOptions(pluginPath) + project.copy(scalacOptions = scalacOptions.toList) + } + mappedProject.getOrElse(project) + } + } diff --git a/frontend/src/main/scala/bloop/engine/SemanticDBCache.scala b/frontend/src/main/scala/bloop/engine/SemanticDBCache.scala new file mode 100644 index 0000000000..e830155bc2 --- /dev/null +++ b/frontend/src/main/scala/bloop/engine/SemanticDBCache.scala @@ -0,0 +1,75 @@ +package bloop.engine +import bloop.logging.Logger +import coursier.core.Repository +import bloop.io.AbsolutePath +import bloop.io.Paths +import bloop.DependencyResolution +import scala.util.Try +import java.nio.file.Files +import xsbti.ComponentProvider +import xsbti.GlobalLock +import java.io.File +import java.util.concurrent.Callable +import sbt.internal.inc.bloop.ZincInternals +import sbt.internal.inc.ZincComponentManager +import sbt.internal.inc.IfMissing +import scala.util.Failure +import scala.util.Success +import bloop.ComponentLock + +object SemanticDBCache { + + val latestRelease = "latest.release" + + private object SemanticDBCacheLock extends ComponentLock + private val provider = ZincInternals.getComponentProvider(Paths.getCacheDirectory("semanticdb")) + private val zincComponentManager = + new ZincComponentManager(SemanticDBCacheLock, provider, secondaryCacheDir = None) + + def findSemanticDBPlugin( + scalaVersion: String, + semanticDBVersion: String, + logger: Logger + ): Option[AbsolutePath] = { + Try { + resolveCache( + "org.scalameta", + s"semanticdb-scalac_$scalaVersion", + semanticDBVersion, + logger + )(bloop.engine.ExecutionContext.ioScheduler) + }.toOption.flatten + } + + private def resolveCache( + organization: String, + module: String, + version: String, + logger: Logger, + additionalRepositories: Seq[Repository] = Nil + )(implicit ec: scala.concurrent.ExecutionContext): Option[AbsolutePath] = { + + def getFromResolution: Option[AbsolutePath] = { + DependencyResolution + .resolve(organization, module, version, logger, additionalRepositories) + .find(_.toString().contains("semanticdb-scalac")) + } + if (version == latestRelease) { + getFromResolution + } else { + val semanticDBId = s"$organization.$module.$version" + Try(zincComponentManager.file(semanticDBId)(IfMissing.Fail)) match { + case Failure(exception) => + val resolved = getFromResolution + resolved match { + case None => + logger.warn("Could not find semanticDB version:\n" + exception.getMessage()) + case Some(value) => + zincComponentManager.define(semanticDBId, Seq(value.toFile)) + } + resolved + case Success(value) => Some(AbsolutePath(value)) + } + } + } +} diff --git a/frontend/src/main/scala/bloop/engine/State.scala b/frontend/src/main/scala/bloop/engine/State.scala index a84dabe032..5881b6abb8 100644 --- a/frontend/src/main/scala/bloop/engine/State.scala +++ b/frontend/src/main/scala/bloop/engine/State.scala @@ -7,6 +7,8 @@ import bloop.engine.caches.{ResultsCache, StateCache} import bloop.io.Paths import bloop.logging.{DebugFilter, Logger} import monix.eval.Task +import bloop.data.LoadedBuild +import bloop.data.WorkspaceSettings /** * Represents the state for a given build. @@ -86,12 +88,14 @@ object State { client: ClientInfo, pool: ClientPool, opts: CommonOptions, - logger: Logger + logger: Logger, + incomingSettings: Option[WorkspaceSettings] = None ): Task[State] = { def loadState(path: bloop.io.AbsolutePath): Task[State] = { - BuildLoader.load(configDir, logger).map { projects => - val build: Build = Build(configDir, projects) - State(build, client, pool, opts, logger) + BuildLoader.load(configDir, incomingSettings, logger).map { + case LoadedBuild(projects, buildSettings) => + val build: Build = Build(configDir, buildSettings, projects) + State(build, client, pool, opts, logger) } } diff --git a/frontend/src/main/scala/bloop/engine/caches/StateCache.scala b/frontend/src/main/scala/bloop/engine/caches/StateCache.scala index fdc832ecc0..810e83bbc1 100644 --- a/frontend/src/main/scala/bloop/engine/caches/StateCache.scala +++ b/frontend/src/main/scala/bloop/engine/caches/StateCache.scala @@ -10,6 +10,7 @@ import bloop.io.AbsolutePath import bloop.cli.ExitStatus import monix.eval.Task +import bloop.data.LoadedBuild /** Cache that holds the state associated to each loaded build. */ final class StateCache(cache: ConcurrentHashMap[AbsolutePath, StateCache.CachedState]) { @@ -89,18 +90,21 @@ final class StateCache(cache: ConcurrentHashMap[AbsolutePath, StateCache.CachedS case Some(state) => state.build.checkForChange(logger).flatMap { case Build.ReturnPreviousState => Task.now(state) - case Build.UpdateState(createdOrModified, deleted) => - BuildLoader.loadBuildFromConfigurationFiles(from, createdOrModified, logger).map { - newProjects => - val currentProjects = state.build.projects - val toRemove = deleted.toSet ++ newProjects.map(_.origin.path) - val untouched = - currentProjects.collect { case p if !toRemove.contains(p.origin.path) => p } - val newBuild = state.build.copy(projects = untouched ++ newProjects) - val newState = state.copy(build = newBuild) - cache.put(from, StateCache.CachedState.fromState(newState)) - newState - } + case Build.UpdateState(createdOrModified, deleted, settingsChanged) => + BuildLoader + .loadBuildFromConfigurationFiles(from, createdOrModified, settingsChanged, logger) + .map { + case LoadedBuild(newProjects, settings) => + val currentProjects = state.build.projects + val toRemove = deleted.toSet ++ newProjects.map(_.origin.path) + val untouched = + currentProjects.collect { case p if !toRemove.contains(p.origin.path) => p } + val newBuild = + state.build.copy(projects = untouched ++ newProjects, settings = settings) + val newState = state.copy(build = newBuild) + cache.put(from, StateCache.CachedState.fromState(newState)) + newState + } } case None => computeBuild(from).map { state => diff --git a/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala b/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala index 87ab8f36be..0232cc126d 100644 --- a/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala +++ b/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala @@ -49,7 +49,6 @@ abstract class BspBaseSuite extends BaseSuite with BspClientTest { private val serverStates: Observable[State] ) { val status = state.status - def toUnsafeManagedState: ManagedBspTestState = { new ManagedBspTestState( state, @@ -363,7 +362,8 @@ abstract class BspBaseSuite extends BaseSuite with BspClientTest { workspace: AbsolutePath, projects: List[TestProject], logger: RecordingLogger, - clientClassesRootDir: Option[AbsolutePath] = None + bspClientName: String = "test-bloop-client", + additionalData: Option[Json] = None )(runTest: ManagedBspTestState => Unit): Unit = { val bspLogger = new BspClientLogger(logger) val configDir = TestProject.populateWorkspace(workspace, projects) @@ -374,7 +374,8 @@ abstract class BspBaseSuite extends BaseSuite with BspClientTest { bspCommand, configDir, bspLogger, - clientClassesRootDir = clientClassesRootDir + clientName = bspClientName, + additionalData = additionalData ).withinSession(runTest(_)) } @@ -386,7 +387,9 @@ abstract class BspBaseSuite extends BaseSuite with BspClientTest { allowError: Boolean = false, userIOScheduler: Option[Scheduler] = None, userComputationScheduler: Option[Scheduler] = None, - clientClassesRootDir: Option[AbsolutePath] = None + clientClassesRootDir: Option[AbsolutePath] = None, + clientName: String = "test-bloop-client", + additionalData: Option[Json] = None ): UnmanagedBspTestState = { val compileIteration = AtomicInt(0) val readyToConnect = Promise[Unit]() @@ -428,22 +431,15 @@ abstract class BspBaseSuite extends BaseSuite with BspClientTest { val lsServer = new BloopLanguageServer(messages, lsClient, services, ioScheduler, logger) val runningClientServer = lsServer.startTask.runAsync(ioScheduler) - - val initializeData: Option[Json] = { - clientClassesRootDir - .map(d => Uri(d.toBspUri)) - .map(uri => BloopExtraBuildParams.encoder(BloopExtraBuildParams(Some(uri)))) - } - val cwd = configDirectory.underlying.getParent val initializeServer = endpoints.Build.initialize.request( bsp.InitializeBuildParams( - "test-bloop-client", + clientName, "1.0.0", BuildInfo.bspVersion, rootUri = bsp.Uri(cwd.toAbsolutePath.toUri), capabilities = bsp.BuildClientCapabilities(List("scala", "java")), - initializeData + additionalData ) ) diff --git a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala new file mode 100644 index 0000000000..27305fd7a0 --- /dev/null +++ b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala @@ -0,0 +1,109 @@ +package bloop.bsp +import bloop.cli.BspProtocol +import bloop.util.TestUtil +import bloop.util.TestProject +import bloop.logging.RecordingLogger +import bloop.logging.BspClientLogger +import bloop.cli.ExitStatus +import java.nio.file.Files +import bloop.data.WorkspaceSettings +import io.circe.JsonObject +import io.circe.Json +import bloop.engine.SemanticDBCache + +object LocalBspMetalsClientSpec extends BspMetalsClientSpec(BspProtocol.Local) + +class BspMetalsClientSpec( + override val protocol: BspProtocol +) extends BspBaseSuite { + + test("initialize metals client and save settings") { + TestUtil.withinWorkspace { workspace => + val metalsProject = TestProject(workspace, "metals-project", Nil) + val projects = List(metalsProject) + val configDir = TestProject.populateWorkspace(workspace, projects) + val logger = new RecordingLogger(ansiCodesSupported = false) + val additionalData = Some(Json.fromFields(Nil)) + val bspState = loadBspState(workspace, projects, logger, "Metals", additionalData) { state => + assert(configDir.resolve(WorkspaceSettings.settingsFileName).exists) + } + } + } + + test("compile with semanticDB") { + TestUtil.withinWorkspace { workspace => + val sources = List( + """/main/scala/Foo.scala + |class Foo + """.stripMargin + ) + val projectName = "metals-project" + val metalsProject = TestProject(workspace, projectName, sources) + val projects = List(metalsProject) + val configDir = TestProject.populateWorkspace(workspace, projects) + WorkspaceSettings.write(configDir, WorkspaceSettings(SemanticDBCache.latestRelease)) + val logger = new RecordingLogger(ansiCodesSupported = false) + val bspState = loadBspState(workspace, projects, logger) { state => + val compiledState = state.compile(metalsProject).toTestState + assert(compiledState.status == ExitStatus.Ok) + val classpath = compiledState.client.getUniqueClassesDirFor( + compiledState.build.getProjectFor(projectName).get + ) + val semanticDBFile = + classpath.resolve("META-INF/semanticdb/src/main/scala/Foo.scala.semanticdb") + assert(semanticDBFile.exists) + } + } + } + + test("compile with semanticDB using cached plugin") { + TestUtil.withinWorkspace { workspace => + val sources = List( + """/main/scala/Foo.scala + |class Foo + """.stripMargin + ) + val projectName = "metals-project" + val metalsProject = TestProject(workspace, projectName, sources) + val projects = List(metalsProject) + val configDir = TestProject.populateWorkspace(workspace, projects) + WorkspaceSettings.write(configDir, WorkspaceSettings("4.1.11")) + val logger = new RecordingLogger(ansiCodesSupported = false) + val bspState = loadBspState(workspace, projects, logger) { state => + val compiledState = state.compile(metalsProject).toTestState + assert(compiledState.status == ExitStatus.Ok) + val classpath = compiledState.client.getUniqueClassesDirFor( + compiledState.build.getProjectFor(projectName).get + ) + val semanticDBFile = + classpath.resolve("META-INF/semanticdb/src/main/scala/Foo.scala.semanticdb") + assert(semanticDBFile.exists) + } + } + } + + test("save settings and compile with semanticDB") { + TestUtil.withinWorkspace { workspace => + val sources = List( + """/main/scala/Foo.scala + |class Foo + """.stripMargin + ) + val projectName = "metals-project" + val metalsProject = TestProject(workspace, projectName, sources) + val projects = List(metalsProject) + val configDir = TestProject.populateWorkspace(workspace, projects) + val logger = new RecordingLogger(ansiCodesSupported = false) + val bspState = loadBspState(workspace, projects, logger, "Metals") { state => + val compiledState = state.compile(metalsProject).toTestState + assert(compiledState.status == ExitStatus.Ok) + val classpath = compiledState.client.getUniqueClassesDirFor( + compiledState.build.getProjectFor(projectName).get + ) + val semanticDBFile = + classpath.resolve("META-INF/semanticdb/src/main/scala/Foo.scala.semanticdb") + assert(semanticDBFile.exists) + } + } + } +} diff --git a/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala b/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala index 97a61ff236..4591ebff87 100644 --- a/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala @@ -13,6 +13,9 @@ import java.util.stream.Collectors import scala.collection.JavaConverters._ import ch.epfl.scala.bsp.ScalacOptionsItem +import bloop.bsp.BloopBspDefinitions.BloopExtraBuildParams +import io.circe.Json +import ch.epfl.scala.bsp.Uri object TcpBspProtocolSpec extends BspProtocolSpec(BspProtocol.Tcp) object LocalBspProtocolSpec extends BspProtocolSpec(BspProtocol.Local) @@ -100,15 +103,22 @@ class BspProtocolSpec( var firstScalacOptions: List[ScalacOptionsItem] = Nil var secondScalacOptions: List[ScalacOptionsItem] = Nil + + val extraBloopParams = BloopExtraBuildParams( + Some(Uri(userClientClassesRootDir.toBspUri)), + semanticDBVersion = None + ) + val initializeData = BloopExtraBuildParams.encoder(extraBloopParams) + // Start first client and query for scalac options which creates client classes dirs - loadBspState(workspace, projects, logger, Some(userClientClassesRootDir)) { bspState => + loadBspState(workspace, projects, logger, additionalData = Some(initializeData)) { bspState => val (_, options) = bspState.scalaOptions(`A`) firstScalacOptions = options.items firstScalacOptions.foreach(d => assertIsDirectory(AbsolutePath(d.classDirectory.toPath))) } // Start second client and query for scalac options which should use same dirs as before - loadBspState(workspace, projects, logger, Some(userClientClassesRootDir)) { bspState => + loadBspState(workspace, projects, logger, additionalData = Some(initializeData)) { bspState => val (_, options) = bspState.scalaOptions(`A`) secondScalacOptions = options.items secondScalacOptions.foreach(d => assertIsDirectory(AbsolutePath(d.classDirectory.toPath))) diff --git a/frontend/src/test/scala/bloop/util/TestUtil.scala b/frontend/src/test/scala/bloop/util/TestUtil.scala index 5b795164bf..c1a5ce4934 100644 --- a/frontend/src/test/scala/bloop/util/TestUtil.scala +++ b/frontend/src/test/scala/bloop/util/TestUtil.scala @@ -37,6 +37,7 @@ import scala.concurrent.{Await, ExecutionException} import scala.meta.jsonrpc.Services import scala.tools.nsc.Properties import scala.util.control.NonFatal +import bloop.data.WorkspaceSettings object TestUtil { def projectDir(base: Path, name: String) = base.resolve(name) @@ -72,6 +73,7 @@ object TestUtil { def checkAfterCleanCompilation( structures: Map[String, Map[String, String]], dependencies: Map[String, Set[String]], + buildSettings: Option[WorkspaceSettings] = None, rootProjects: List[String] = List(RootProject), scalaInstance: ScalaInstance = TestUtil.scalaInstance, javaEnv: JavaEnv = JavaEnv.default, @@ -80,7 +82,7 @@ object TestUtil { useSiteLogger: Option[Logger] = None, order: CompileOrder = Config.Mixed )(afterCompile: State => Unit = (_ => ())) = { - testState(structures, dependencies, scalaInstance, javaEnv, order) { (state: State) => + testState(structures, dependencies, buildSettings, scalaInstance, javaEnv, order) { (state: State) => def action(state0: State): Unit = { val state = useSiteLogger.map(logger => state0.copy(logger = logger)).getOrElse(state0) // Check that this is a clean compile! @@ -197,8 +199,8 @@ object TestUtil { assert(Files.exists(configDir), "Does not exist: " + configDir) val configDirectory = AbsolutePath(configDir) - val loadedProjects = transformProjects(BuildLoader.loadSynchronously(configDirectory, logger)) - val build = Build(configDirectory, loadedProjects) + val loadedBuild = BuildLoader.loadSynchronously(configDirectory, logger) + val build = Build(configDirectory, loadedBuild.workspaceSettings, transformProjects(loadedBuild.projects)) val state = State.forTests(build, TestUtil.getCompilerCache(logger), logger) val state1 = state.copy(commonOptions = state.commonOptions.copy(env = runAndTestProperties)) if (!emptyResults) state1 else state1.copy(results = ResultsCache.emptyForTests) @@ -260,6 +262,7 @@ object TestUtil { def testState[T]( projectStructures: Map[String, Map[String, String]], dependenciesMap: Map[String, Set[String]], + buildSettings: Option[WorkspaceSettings] = None, instance: ScalaInstance = TestUtil.scalaInstance, env: JavaEnv = JavaEnv.default, order: CompileOrder = Config.Mixed, @@ -273,7 +276,7 @@ object TestUtil { val deps = dependenciesMap.getOrElse(name, Set.empty) makeProject(temp, name, sources, deps, Some(instance), env, logger, order, extraJars) } - val build = Build(temp, projects.toList) + val build = Build(temp, buildSettings, projects.toList) val state = State.forTests(build, TestUtil.getCompilerCache(logger), logger) try op(state) catch { diff --git a/integrations/gradle-bloop/src/test/scala/bloop/integrations/gradle/ConfigGenerationSuite.scala b/integrations/gradle-bloop/src/test/scala/bloop/integrations/gradle/ConfigGenerationSuite.scala index d4f767ddff..5abd74141e 100644 --- a/integrations/gradle-bloop/src/test/scala/bloop/integrations/gradle/ConfigGenerationSuite.scala +++ b/integrations/gradle-bloop/src/test/scala/bloop/integrations/gradle/ConfigGenerationSuite.scala @@ -1096,8 +1096,8 @@ abstract class ConfigGenerationSuite { val logger = BloopLogger.default(configDir.toString) assert(Files.exists(configDir.toPath), "Does not exist: " + configDir) val configDirectory = AbsolutePath(configDir) - val loadedProjects = BuildLoader.loadSynchronously(configDirectory, logger) - val build = Build(configDirectory, loadedProjects) + val loadedBuild = BuildLoader.loadSynchronously(configDirectory, logger) + val build = Build(configDirectory, loadedBuild.workspaceSettings, loadedBuild.projects) State.forTests(build, TestUtil.getCompilerCache(logger), logger) } diff --git a/shared/src/main/scala/bloop/io/AbsolutePath.scala b/shared/src/main/scala/bloop/io/AbsolutePath.scala index bb43cb3ba5..1ca7718857 100644 --- a/shared/src/main/scala/bloop/io/AbsolutePath.scala +++ b/shared/src/main/scala/bloop/io/AbsolutePath.scala @@ -4,6 +4,7 @@ package bloop.io import java.io.File import java.net.URI import java.nio.file.{Files, Path, Paths => NioPaths} +import scala.collection.JavaConverters._ final class AbsolutePath private (val underlying: Path) extends AnyVal { def syntax: String = toString @@ -18,6 +19,10 @@ final class AbsolutePath private (val underlying: Path) extends AnyVal { def resolve(other: String): AbsolutePath = AbsolutePath(underlying.resolve(other))(this) def getParent: AbsolutePath = AbsolutePath(underlying.getParent) + def list: List[AbsolutePath] = + Files.list(underlying).iterator().asScala.toList.map(AbsolutePath.apply) + + def name: String = underlying.getFileName().toString() def exists: Boolean = Files.exists(underlying) def isFile: Boolean = Files.isRegularFile(underlying) def isDirectory: Boolean = Files.isDirectory(underlying) From 9dc0729bfe5b1040979c6bff9c8a72b801c51eb8 Mon Sep 17 00:00:00 2001 From: tgodzik Date: Fri, 19 Jul 2019 08:38:44 +0200 Subject: [PATCH 02/22] Address review feedback: - add logging to notify user about problems when dowloading SemanticDB plugin - add test suites - add supportedScalaVersions parameter to make sure we don't try to download non existing plugin - add reapplySettings parameter to be able to force redownload of the plugin even if the build didn't change - other fixes --- .../scala/bloop/BloopComponentsLock.scala | 4 +- .../scala/bloop/DependencyResolution.scala | 13 +- .../scala/bloop/bsp/BloopBspDefinitions.scala | 10 +- .../scala/bloop/bsp/BloopBspServices.scala | 34 +- .../scala/bloop/data/WorkspaceSettings.scala | 9 +- .../src/main/scala/bloop/engine/Build.scala | 31 +- .../main/scala/bloop/engine/BuildLoader.scala | 43 +-- .../scala/bloop/engine/SemanticDBCache.scala | 65 ++-- .../src/main/scala/bloop/engine/State.scala | 14 +- .../bloop/engine/caches/StateCache.scala | 7 +- .../test/scala/bloop/BuildLoaderSpec.scala | 69 ++++- .../test/scala/bloop/bsp/BspBaseSuite.scala | 8 +- .../scala/bloop/bsp/BspMetalsClientSpec.scala | 292 ++++++++++++++++-- .../scala/bloop/bsp/BspProtocolSpec.scala | 9 +- .../scala/bloop/testing/BloopHelpers.scala | 5 +- .../src/test/scala/bloop/util/TestUtil.scala | 8 +- .../main/scala/bloop/io/AbsolutePath.scala | 5 - 17 files changed, 508 insertions(+), 118 deletions(-) diff --git a/backend/src/main/scala/bloop/BloopComponentsLock.scala b/backend/src/main/scala/bloop/BloopComponentsLock.scala index 4db2a2d65d..494d6885b1 100644 --- a/backend/src/main/scala/bloop/BloopComponentsLock.scala +++ b/backend/src/main/scala/bloop/BloopComponentsLock.scala @@ -5,8 +5,10 @@ import java.util.concurrent.Callable import xsbti.GlobalLock -trait ComponentLock extends GlobalLock { +sealed trait ComponentLock extends GlobalLock { override def apply[T](file: File, callable: Callable[T]): T = synchronized { callable.call() } } object BloopComponentsLock extends ComponentLock + +object SemanticDBCacheLock extends ComponentLock \ No newline at end of file diff --git a/backend/src/main/scala/bloop/DependencyResolution.scala b/backend/src/main/scala/bloop/DependencyResolution.scala index a62f28c6df..868647002e 100644 --- a/backend/src/main/scala/bloop/DependencyResolution.scala +++ b/backend/src/main/scala/bloop/DependencyResolution.scala @@ -45,7 +45,8 @@ object DependencyResolution { module: String, version: String, logger: Logger, - additionalRepositories: Seq[Repository] = Nil + additionalRepositories: Seq[Repository] = Nil, + shouldReportErrors: Boolean = false )(implicit ec: scala.concurrent.ExecutionContext): Array[AbsolutePath] = { logger.debug(s"Resolving $organization:$module:$version")(DebugFilter.Compilation) val org = coursier.Organization(organization) @@ -62,6 +63,7 @@ object DependencyResolution { } val fetch = ResolutionProcess.fetch(repositories, Cache.default.fetch) val resolution = start.process.run(fetch).unsafeRun() + if (shouldReportErrors) reportErrors(resolution, logger) val localArtifacts: Seq[(Boolean, Either[ArtifactError, File])] = { Gather[Task] .gather(resolution.artifacts().map { artifact => @@ -83,4 +85,13 @@ object DependencyResolution { ) } } + + private def reportErrors(resolution: Resolution, logger: Logger): Unit = { + resolution.errorCache.foreach { + case ((module, version), errors) => + logger.displayWarningToUser( + s"There were issues resolving $module:$version - ${errors.mkString("; ")}." + ) + } + } } diff --git a/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala b/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala index b790776d39..59a099a3c1 100644 --- a/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala +++ b/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala @@ -7,10 +7,18 @@ import ch.epfl.scala.bsp.Uri object BloopBspDefinitions { final case class BloopExtraBuildParams( clientClassesRootDir: Option[Uri], - semanticDBVersion : Option[String] + semanticdbVersion: Option[String], + supportedScalaVersions: List[String], + reapplySettings: Boolean ) object BloopExtraBuildParams { + val empty = BloopExtraBuildParams( + clientClassesRootDir = None, + semanticdbVersion = None, + supportedScalaVersions = Nil, + reapplySettings = false + ) val encoder: RootEncoder[BloopExtraBuildParams] = deriveEncoder val decoder: Decoder[BloopExtraBuildParams] = deriveDecoder } diff --git a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala index 3d82d4e932..9eae3c246d 100644 --- a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala +++ b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala @@ -112,13 +112,22 @@ final class BloopBspServices( private def reloadState( config: AbsolutePath, clientInfo: ClientInfo, - workspaceSettings: Option[WorkspaceSettings] = None + workspaceSettings: Option[WorkspaceSettings] = None, + reapplySettings: Boolean = false ): Task[State] = { val pool = currentState.pool val defaultOpts = currentState.commonOptions bspLogger.debug(s"Reloading bsp state for ${config.syntax}") State - .loadActiveStateFor(config, clientInfo, pool, defaultOpts, bspLogger, workspaceSettings) + .loadActiveStateFor( + config, + clientInfo, + pool, + defaultOpts, + bspLogger, + workspaceSettings, + reapplySettings + ) .map { state0 => /* Create a new state that has the previously compiled results in this BSP * client as the last compiled result available for a project. This is required @@ -196,20 +205,25 @@ final class BloopBspServices( /* Metals specific settings that are used to store the * SemanticDB version that will later be applied to all - * projects in the workspace. If the client is Metals but + * projects in the workspace. If the client is Metals but * the version is not specified we use `latest.release` */ + val reapplySettings = extraBuildParams.map(_.reapplySettings).getOrElse(false) val metalsSettings = - if (params.displayName.contains("Metals")) { - val semanticDBVersion = extraBuildParams - .flatMap(extra => extra.semanticDBVersion) - .getOrElse(SemanticDBCache.latestRelease) - Some(WorkspaceSettings(semanticDBVersion)) - } else { + if (!params.displayName.contains("Metals")) { None + } else { + extraBuildParams + .flatMap(extra => extra.semanticdbVersion) + .map { semanticDBVersion => + WorkspaceSettings( + semanticDBVersion, + extraBuildParams.toList.flatMap(_.supportedScalaVersions) + ) + } } - reloadState(configDir, client, metalsSettings).map { state => + reloadState(configDir, client, metalsSettings, reapplySettings).map { state => callSiteState.logger.info(s"request received: build/initialize") clientInfo.success(client) connectedBspClients.put(client, configDir) diff --git a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala index 2d91122674..c697cce1dc 100644 --- a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala +++ b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala @@ -20,7 +20,7 @@ import java.nio.file.Path import scala.util.Failure import scala.util.Success -case class WorkspaceSettings(semanticDBVersion: String) +case class WorkspaceSettings(semanticDBVersion: String, supportedScalaVersions: List[String]) object WorkspaceSettings { @@ -31,7 +31,9 @@ object WorkspaceSettings { def fromFile(configPath: AbsolutePath, logger: Logger): Option[WorkspaceSettings] = { val settingsPath = configPath.resolve(settingsFileName) - if (settingsPath.isFile) { + if (!settingsPath.isFile) { + None + } else { val bytes = Files.readAllBytes(settingsPath.underlying) logger.debug(s"Loading workspace settings from $settingsFileName")( DebugFilter.All @@ -41,8 +43,6 @@ object WorkspaceSettings { case Left(failure) => throw failure case Right(json) => Option(fromJson(json)) } - } else { - None } } @@ -67,5 +67,4 @@ object WorkspaceSettings { } } - } diff --git a/frontend/src/main/scala/bloop/engine/Build.scala b/frontend/src/main/scala/bloop/engine/Build.scala index d9041b4a11..b0d2c4f0e0 100644 --- a/frontend/src/main/scala/bloop/engine/Build.scala +++ b/frontend/src/main/scala/bloop/engine/Build.scala @@ -30,16 +30,26 @@ final case class Build private ( * @param logger A logger that receives errors, if any. * @return The status of the directory from which the build was loaded. */ - def checkForChange(logger: Logger): Task[Build.ReloadAction] = { + def checkForChange( + logger: Logger, + incomingSettings: Option[WorkspaceSettings] = None, + reapplySettings: Boolean = false + ): Task[Build.ReloadAction] = { val files = projects.iterator.map(p => p.origin.toAttributedPath).toSet val newFiles = BuildLoader.readConfigurationFilesInBase(origin, logger).toSet - val loadedSettings = WorkspaceSettings.fromFile(origin, logger) - val sameSettings = loadedSettings == settings - + def relevantChange(workspaceSettings: WorkspaceSettings) = + (workspaceSettings.semanticDBVersion, workspaceSettings.supportedScalaVersions) + val changedSettings = reapplySettings || + (incomingSettings.nonEmpty && + settings.map(relevantChange) != incomingSettings.map(relevantChange)) + if (reapplySettings) { + logger.info(s"Forcing reload of all projects") + } // This is the fast path to short circuit quickly if they are the same - if (newFiles == files && sameSettings) Task.now(Build.ReturnPreviousState) - else { + if (newFiles == files && !changedSettings) { + Task.now(Build.ReturnPreviousState) + } else { val filesToAttributed = projects.iterator.map(p => p.origin.path -> p).toMap // There has been at least either one addition, one removal or one change in a file time val newOrModifiedConfigurations = newFiles.map { f => @@ -51,6 +61,7 @@ final case class Build private ( } filesToAttributed.get(f.path) match { + case _ if changedSettings => List(configuration) case Some(p) if p.origin.hash == configuration.origin.hash => Nil case _ => List(configuration) } @@ -61,9 +72,9 @@ final case class Build private ( Task.gatherUnordered(newOrModifiedConfigurations).map(_.flatten).map { newOrModified => val newToAttributed = newFiles.iterator.map(ap => ap.path -> ap).toMap val deleted = files.toList.collect { case f if !newToAttributed.contains(f.path) => f.path } - (newOrModified, deleted) match { - case (Nil, Nil) => Build.ReturnPreviousState - case _ => Build.UpdateState(newOrModified, deleted, loadedSettings) + (newOrModified, deleted, changedSettings) match { + case (Nil, Nil, false) => Build.ReturnPreviousState + case _ => Build.UpdateState(newOrModified, deleted, incomingSettings) } } } @@ -76,7 +87,7 @@ object Build { case class UpdateState( createdOrModified: List[ReadConfiguration], deleted: List[AbsolutePath], - settingsChanged: Option[WorkspaceSettings] + changedSetings: Option[WorkspaceSettings] ) extends ReloadAction /** A configuration file is a combination of an absolute path and a file time. */ diff --git a/frontend/src/main/scala/bloop/engine/BuildLoader.scala b/frontend/src/main/scala/bloop/engine/BuildLoader.scala index 4d3224449f..3a1e1dcabe 100644 --- a/frontend/src/main/scala/bloop/engine/BuildLoader.scala +++ b/frontend/src/main/scala/bloop/engine/BuildLoader.scala @@ -39,18 +39,21 @@ object BuildLoader { def loadBuildFromConfigurationFiles( configDir: AbsolutePath, configFiles: List[Build.ReadConfiguration], - settingsFile: Option[WorkspaceSettings], + incomingSettings: Option[WorkspaceSettings], logger: Logger ): Task[LoadedBuild] = { - + val workspaceSettings = Task(updateWorkspaceSettings(configDir, logger, incomingSettings)) logger.debug(s"Loading ${configFiles.length} projects from '${configDir.syntax}'...")( DebugFilter.Compilation ) - val all = configFiles.map(f => Task(loadProject(f.bytes, f.origin, logger, settingsFile))) - val groupTasks = all.grouped(10).map(group => Task.gatherUnordered(group)).toList - Task - .sequence(groupTasks) - .map(fp => LoadedBuild(fp.flatten, settingsFile)) + workspaceSettings + .flatMap { settings => + val all = configFiles.map(f => Task(loadProject(f.bytes, f.origin, logger, settings))) + val groupTasks = all.grouped(10).map(group => Task.gatherUnordered(group)).toList + Task + .sequence(groupTasks) + .map(fp => LoadedBuild(fp.flatten, settings)) + } .executeOn(ExecutionContext.ioScheduler) } @@ -66,7 +69,6 @@ object BuildLoader { incomingSettings: Option[WorkspaceSettings], logger: Logger ): Task[LoadedBuild] = { - val workspaceSettings = Task(updateWorkspaceSettings(configDir, logger, incomingSettings)) val configFiles = readConfigurationFilesInBase(configDir, logger).map { ap => Task { val bytes = ap.path.readAllBytes @@ -75,13 +77,11 @@ object BuildLoader { } } - workspaceSettings.flatMap { settings => - Task - .gatherUnordered(configFiles) - .flatMap { fs => - loadBuildFromConfigurationFiles(configDir, fs, settings, logger) - } - } + Task + .gatherUnordered(configFiles) + .flatMap { fs => + loadBuildFromConfigurationFiles(configDir, fs, incomingSettings, logger) + } } /** @@ -147,10 +147,9 @@ object BuildLoader { project: Project, logger: Logger ): Project = { - def addSemanticDBOptions(pluginPath: AbsolutePath) = { { - val optionsSet = project.scalacOptions.toSet + val optionsSet = project.scalacOptions val containsSemanticDB = optionsSet.find( setting => setting.contains("-Xplugin") && setting.contains("semanticdb-scalac") ) @@ -159,9 +158,10 @@ object BuildLoader { logger.info(s"SemanticDB plugin already added: ${containsSemanticDB.get}") optionsSet } else { + val workspaceDir = project.origin.path.getParent.getParent optionsSet ++ Set( "-P:semanticdb:failures:warning", - s"-P:semanticdb:sourceroot:${project.baseDirectory}", + s"-P:semanticdb:sourceroot:$workspaceDir", "-P:semanticdb:synthetics:on", "-Xplugin-require:semanticdb", s"-Xplugin:$pluginPath" @@ -170,8 +170,8 @@ object BuildLoader { if (containsYrangepos.isDefined) { semanticDBAdded } else { - semanticDBAdded + "-Yrangepos" - } + semanticDBAdded :+ "-Yrangepos" + }.distinct } } @@ -179,12 +179,13 @@ object BuildLoader { scalaInstance <- project.scalaInstance pluginPath <- SemanticDBCache.findSemanticDBPlugin( scalaInstance.version, + settings.supportedScalaVersions, settings.semanticDBVersion, logger ) } yield { val scalacOptions = addSemanticDBOptions(pluginPath) - project.copy(scalacOptions = scalacOptions.toList) + project.copy(scalacOptions = scalacOptions) } mappedProject.getOrElse(project) } diff --git a/frontend/src/main/scala/bloop/engine/SemanticDBCache.scala b/frontend/src/main/scala/bloop/engine/SemanticDBCache.scala index e830155bc2..8963542b9c 100644 --- a/frontend/src/main/scala/bloop/engine/SemanticDBCache.scala +++ b/frontend/src/main/scala/bloop/engine/SemanticDBCache.scala @@ -16,56 +16,73 @@ import sbt.internal.inc.IfMissing import scala.util.Failure import scala.util.Success import bloop.ComponentLock +import bloop.SemanticDBCacheLock object SemanticDBCache { - val latestRelease = "latest.release" - - private object SemanticDBCacheLock extends ComponentLock - private val provider = ZincInternals.getComponentProvider(Paths.getCacheDirectory("semanticdb")) - private val zincComponentManager = - new ZincComponentManager(SemanticDBCacheLock, provider, secondaryCacheDir = None) + private val latestRelease = "latest.release" def findSemanticDBPlugin( scalaVersion: String, + supportedScalaVersion: List[String], semanticDBVersion: String, logger: Logger ): Option[AbsolutePath] = { - Try { - resolveCache( - "org.scalameta", - s"semanticdb-scalac_$scalaVersion", - semanticDBVersion, - logger - )(bloop.engine.ExecutionContext.ioScheduler) - }.toOption.flatten + if (!supportedScalaVersion.contains(scalaVersion)) { + logger.displayWarningToUser( + s"$scalaVersion is not supported for semanticDB version $semanticDBVersion" + ) + None + } else + Try { + resolveFromCache( + "org.scalameta", + s"semanticdb-scalac_$scalaVersion", + semanticDBVersion, + logger + ) + }.toOption.flatten + } - private def resolveCache( + private def resolveFromCache( organization: String, module: String, version: String, logger: Logger, additionalRepositories: Seq[Repository] = Nil - )(implicit ec: scala.concurrent.ExecutionContext): Option[AbsolutePath] = { - + ): Option[AbsolutePath] = { + val provider = ZincInternals.getComponentProvider(Paths.getCacheDirectory("semanticdb")) + val manager = + new ZincComponentManager(SemanticDBCacheLock, provider, secondaryCacheDir = None) def getFromResolution: Option[AbsolutePath] = { - DependencyResolution - .resolve(organization, module, version, logger, additionalRepositories) - .find(_.toString().contains("semanticdb-scalac")) + val all = DependencyResolution + .resolve( + organization, + module, + version, + logger, + additionalRepositories, + shouldReportErrors = true + )( + bloop.engine.ExecutionContext.ioScheduler + ) + all.find(_.toString().contains("semanticdb-scalac")) } if (version == latestRelease) { getFromResolution } else { val semanticDBId = s"$organization.$module.$version" - Try(zincComponentManager.file(semanticDBId)(IfMissing.Fail)) match { + Try(manager.file(semanticDBId)(IfMissing.Fail)) match { case Failure(exception) => val resolved = getFromResolution resolved match { - case None => - logger.warn("Could not find semanticDB version:\n" + exception.getMessage()) case Some(value) => - zincComponentManager.define(semanticDBId, Seq(value.toFile)) + manager.define(semanticDBId, Seq(value.toFile)) + case None => + logger.warn( + s"Could not resolve semanticDB version $version" + ) } resolved case Success(value) => Some(AbsolutePath(value)) diff --git a/frontend/src/main/scala/bloop/engine/State.scala b/frontend/src/main/scala/bloop/engine/State.scala index 5881b6abb8..41efd75c31 100644 --- a/frontend/src/main/scala/bloop/engine/State.scala +++ b/frontend/src/main/scala/bloop/engine/State.scala @@ -89,7 +89,8 @@ object State { pool: ClientPool, opts: CommonOptions, logger: Logger, - incomingSettings: Option[WorkspaceSettings] = None + incomingSettings: Option[WorkspaceSettings] = None, + reapplySettings: Boolean = false ): Task[State] = { def loadState(path: bloop.io.AbsolutePath): Task[State] = { BuildLoader.load(configDir, incomingSettings, logger).map { @@ -99,7 +100,16 @@ object State { } } - val cached = State.stateCache.addIfMissing(configDir, client, pool, opts, logger, loadState(_)) + val cached = State.stateCache.addIfMissing( + configDir, + client, + pool, + opts, + logger, + loadState(_), + incomingSettings, + reapplySettings + ) cached.map(_.copy(pool = pool, client = client, commonOptions = opts, logger = logger)) } diff --git a/frontend/src/main/scala/bloop/engine/caches/StateCache.scala b/frontend/src/main/scala/bloop/engine/caches/StateCache.scala index 810e83bbc1..e6c89a7b64 100644 --- a/frontend/src/main/scala/bloop/engine/caches/StateCache.scala +++ b/frontend/src/main/scala/bloop/engine/caches/StateCache.scala @@ -3,6 +3,7 @@ package bloop.engine.caches import java.util.concurrent.ConcurrentHashMap import bloop.data.ClientInfo +import bloop.data.WorkspaceSettings import bloop.logging.Logger import bloop.cli.CommonOptions import bloop.engine.{Build, BuildLoader, State, ClientPool} @@ -84,11 +85,13 @@ final class StateCache(cache: ConcurrentHashMap[AbsolutePath, StateCache.CachedS pool: ClientPool, commonOptions: CommonOptions, logger: Logger, - computeBuild: AbsolutePath => Task[State] + computeBuild: AbsolutePath => Task[State], + incomingSettings: Option[WorkspaceSettings], + reapplySettings: Boolean ): Task[State] = { getStateFor(from, client, pool, commonOptions, logger) match { case Some(state) => - state.build.checkForChange(logger).flatMap { + state.build.checkForChange(logger, incomingSettings, reapplySettings).flatMap { case Build.ReturnPreviousState => Task.now(state) case Build.UpdateState(createdOrModified, deleted, settingsChanged) => BuildLoader diff --git a/frontend/src/test/scala/bloop/BuildLoaderSpec.scala b/frontend/src/test/scala/bloop/BuildLoaderSpec.scala index 9d9d85ceae..3ed690c6f7 100644 --- a/frontend/src/test/scala/bloop/BuildLoaderSpec.scala +++ b/frontend/src/test/scala/bloop/BuildLoaderSpec.scala @@ -10,6 +10,8 @@ import bloop.util.TestUtil import monix.eval.Task import bloop.testing.BaseSuite +import bloop.data.WorkspaceSettings +import bloop.internal.build.BuildInfo object BuildLoaderSpec extends BaseSuite { testLoad("don't reload if nothing changes") { (testBuild, logger) => testBuild.state.build.checkForChange(logger).map { @@ -18,6 +20,60 @@ object BuildLoaderSpec extends BaseSuite { } } + testLoad("reload if forced") { (testBuild, logger) => + testBuild.state.build.checkForChange(logger, reapplySettings = true).map { + case Build.ReturnPreviousState => + sys.error(s"Expected return updated state, got previous state") + case action: Build.UpdateState => + assert(action.createdOrModified.size == 4) + } + } + + testLoad("reload if settings are added") { (testBuild, logger) => + val settings = WorkspaceSettings("4.2.0", List(BuildInfo.scalaVersion)) + testBuild.state.build + .checkForChange(logger, incomingSettings = Some(settings)) + .map { + case Build.ReturnPreviousState => + sys.error(s"Expected return updated state, got previous state") + case action: Build.UpdateState => + assert(action.createdOrModified.size == 4) + } + } + + val sameSettings = WorkspaceSettings("4.2.0", List(BuildInfo.scalaVersion)) + testLoad("do not reload if same settings are added", Some(sameSettings)) { (testBuild, logger) => + testBuild.state.build + .checkForChange(logger, incomingSettings = Some(sameSettings)) + .map { + case Build.ReturnPreviousState => () + case action: Build.UpdateState => + sys.error(s"Expected return previous state, got updated state") + } + } + + testLoad("reload if new settings are added", Some(sameSettings)) { (testBuild, logger) => + val newSettings = WorkspaceSettings("4.1.11", List(BuildInfo.scalaVersion)) + testBuild.state.build + .checkForChange(logger, incomingSettings = Some(newSettings)) + .map { + case Build.ReturnPreviousState => + sys.error(s"Expected return updated state, got previous state") + case action: Build.UpdateState => + assert(action.createdOrModified.size == 4) + } + } + + testLoad("do not reload on empty settings") { (testBuild, logger) => + val configDir = testBuild.configFileFor(testBuild.projects.head).getParent + testBuild.state.build + .checkForChange(logger, incomingSettings = None) + .map { + case Build.ReturnPreviousState => () + case action: Build.UpdateState => sys.error(s"Expected return previous state, got $action") + } + } + private def configurationFiles(build: TestBuild): List[AbsolutePath] = { build.projects.map(p => build.configFileFor(p)) } @@ -122,13 +178,18 @@ object BuildLoaderSpec extends BaseSuite { } } - def testLoad[T](name: String)(fun: (TestBuild, RecordingLogger) => Task[T]): Unit = { + def testLoad[T](name: String, settings: Option[WorkspaceSettings] = None)( + fun: (TestBuild, RecordingLogger) => Task[T] + ): Unit = { test(name) { - loadBuildState(fun) + loadBuildState(fun, settings) } } - def loadBuildState[T](f: (TestBuild, RecordingLogger) => Task[T]): T = { + def loadBuildState[T]( + f: (TestBuild, RecordingLogger) => Task[T], + settings: Option[WorkspaceSettings] = None + ): T = { TestUtil.withinWorkspace { workspace => import bloop.util.TestProject val logger = new RecordingLogger(ansiCodesSupported = false) @@ -137,7 +198,7 @@ object BuildLoaderSpec extends BaseSuite { val c = TestProject(workspace, "c", Nil) val d = TestProject(workspace, "d", Nil) val projects = List(a, b, c, d) - val state = loadState(workspace, projects, logger) + val state = loadState(workspace, projects, logger, settings) val configDir = state.build.origin val build = TestBuild(state, projects) TestUtil.blockOnTask(f(build, logger), 5) diff --git a/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala b/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala index 0232cc126d..6f708c6917 100644 --- a/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala +++ b/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala @@ -37,6 +37,7 @@ import scala.meta.jsonrpc.{BaseProtocolMessage, LanguageClient, LanguageServer, import monix.execution.Scheduler import ch.epfl.scala.bsp.Uri import io.circe.Json +import scala.util.Try abstract class BspBaseSuite extends BaseSuite with BspClientTest { final class UnmanagedBspTestState( @@ -363,7 +364,7 @@ abstract class BspBaseSuite extends BaseSuite with BspClientTest { projects: List[TestProject], logger: RecordingLogger, bspClientName: String = "test-bloop-client", - additionalData: Option[Json] = None + bloopExtraParams: BloopExtraBuildParams = BloopExtraBuildParams.empty )(runTest: ManagedBspTestState => Unit): Unit = { val bspLogger = new BspClientLogger(logger) val configDir = TestProject.populateWorkspace(workspace, projects) @@ -375,7 +376,7 @@ abstract class BspBaseSuite extends BaseSuite with BspClientTest { configDir, bspLogger, clientName = bspClientName, - additionalData = additionalData + bloopExtraParams = bloopExtraParams ).withinSession(runTest(_)) } @@ -389,7 +390,7 @@ abstract class BspBaseSuite extends BaseSuite with BspClientTest { userComputationScheduler: Option[Scheduler] = None, clientClassesRootDir: Option[AbsolutePath] = None, clientName: String = "test-bloop-client", - additionalData: Option[Json] = None + bloopExtraParams: BloopExtraBuildParams = BloopExtraBuildParams.empty ): UnmanagedBspTestState = { val compileIteration = AtomicInt(0) val readyToConnect = Promise[Unit]() @@ -432,6 +433,7 @@ abstract class BspBaseSuite extends BaseSuite with BspClientTest { val lsServer = new BloopLanguageServer(messages, lsClient, services, ioScheduler, logger) val runningClientServer = lsServer.startTask.runAsync(ioScheduler) val cwd = configDirectory.underlying.getParent + val additionalData = Try(BloopExtraBuildParams.encoder(bloopExtraParams)).toOption val initializeServer = endpoints.Build.initialize.request( bsp.InitializeBuildParams( clientName, diff --git a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala index 27305fd7a0..d539aa1074 100644 --- a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala @@ -10,26 +10,270 @@ import bloop.data.WorkspaceSettings import io.circe.JsonObject import io.circe.Json import bloop.engine.SemanticDBCache +import bloop.internal.build.BuildInfo +import bloop.bsp.BloopBspDefinitions.BloopExtraBuildParams +import monix.execution.Scheduler +import monix.execution.ExecutionModel +import monix.eval.Task +import scala.concurrent.duration.FiniteDuration +import bloop.io.AbsolutePath +import ch.epfl.scala.bsp.endpoints.BuildTarget.scalacOptions object LocalBspMetalsClientSpec extends BspMetalsClientSpec(BspProtocol.Local) +object TcpBspMetalsClientSpec extends BspMetalsClientSpec(BspProtocol.Tcp) class BspMetalsClientSpec( override val protocol: BspProtocol ) extends BspBaseSuite { + val testedScalaVersion = "2.12.8" + val projectName = "metals-project" + test("initialize metals client and save settings") { TestUtil.withinWorkspace { workspace => - val metalsProject = TestProject(workspace, "metals-project", Nil) + val metalsProject = + TestProject(workspace, projectName, Nil, scalaVersion = Some(testedScalaVersion)) + val projects = List(metalsProject) + val configDir = TestProject.populateWorkspace(workspace, projects) + val logger = new RecordingLogger(ansiCodesSupported = false) + val semanticdbVersion = "4.2.0" + val extraParams = BloopExtraBuildParams( + clientClassesRootDir = None, + semanticdbVersion = Some(semanticdbVersion), + supportedScalaVersions = List(testedScalaVersion), + reapplySettings = false + ) + val bspState = + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { + state => + assert(configDir.resolve(WorkspaceSettings.settingsFileName).exists) + val settings = WorkspaceSettings.fromFile(configDir, logger) + assert(settings.isDefined && settings.get.semanticDBVersion == semanticdbVersion) + val scalacOptions = state.scalaOptions(metalsProject)._2.items.head.options + assert( + List( + "-P:semanticdb:failures:warning", + s"-P:semanticdb:sourceroot:$workspace", + "-P:semanticdb:synthetics:on", + "-Xplugin-require:semanticdb", + "semanticdb-scalac" + ).forall(opt => scalacOptions.find(_.contains(opt)).isDefined) + ) + } + } + } + + test("do not initialize metals client and save settings with unsupported scala version") { + TestUtil.withinWorkspace { workspace => + val metalsProject = + TestProject(workspace, projectName, Nil, scalaVersion = Some("2.12.4")) + val projects = List(metalsProject) + val configDir = TestProject.populateWorkspace(workspace, projects) + val logger = new RecordingLogger(ansiCodesSupported = false) + val semanticdbVersion = "4.2.0" + val extraParams = BloopExtraBuildParams( + clientClassesRootDir = None, + semanticdbVersion = Some(semanticdbVersion), + supportedScalaVersions = List("2.12.8"), + reapplySettings = false + ) + val bspState = + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { + state => + val scalacOptions = state.scalaOptions(metalsProject)._2.items.head.options + assert(scalacOptions.isEmpty) + } + } + } + + test("initialize metals client with existing plugin in workspace") { + TestUtil.withinWorkspace { workspace => + val defaultScalacOptions = List( + "-P:semanticdb:failures:warning", + s"-P:semanticdb:sourceroot:$workspace", + "-P:semanticdb:synthetics:on", + "-Xplugin-require:semanticdb", + s"-Xplugin:path-to-plugin/semanticdb-scalac_2.12.8-4.2.0.jar.jar" + ) + val metalsProject = + TestProject( + workspace, + projectName, + Nil, + scalaVersion = Some(testedScalaVersion), + scalacOptions = defaultScalacOptions //${getClass().getResource("semanticdb_2.12.8-4.1.11.jar")} + ) + val projects = List(metalsProject) + val configDir = TestProject.populateWorkspace(workspace, projects) + val logger = new RecordingLogger(ansiCodesSupported = false) + val semanticdbVersion = "4.2.0" + val extraParams = BloopExtraBuildParams( + clientClassesRootDir = None, + semanticdbVersion = Some(semanticdbVersion), + supportedScalaVersions = List(testedScalaVersion), + reapplySettings = false + ) + val bspState = + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { + state => + val scalacOptions = state.scalaOptions(metalsProject)._2.items.head.options + val expected = defaultScalacOptions :+ "-Yrangepos" + assert(scalacOptions == expected) + } + } + } + + test("initialize metals client with existing plugin and -Yrangepos in workspace") { + TestUtil.withinWorkspace { workspace => + val defaultScalacOptions = List( + "-P:semanticdb:failures:warning", + s"-P:semanticdb:sourceroot:$workspace", + "-P:semanticdb:synthetics:on", + "-Xplugin-require:semanticdb", + s"-Xplugin:path-to-plugin/semanticdb-scalac_2.12.8-4.2.0.jar.jar", + "-Yrangepos" + ) + val metalsProject = + TestProject( + workspace, + projectName, + Nil, + scalaVersion = Some(testedScalaVersion), + scalacOptions = defaultScalacOptions //${getClass().getResource("semanticdb_2.12.8-4.1.11.jar")} + ) + val projects = List(metalsProject) + val configDir = TestProject.populateWorkspace(workspace, projects) + val logger = new RecordingLogger(ansiCodesSupported = false) + val semanticdbVersion = "4.2.0" + val extraParams = BloopExtraBuildParams( + clientClassesRootDir = None, + semanticdbVersion = Some(semanticdbVersion), + supportedScalaVersions = List(testedScalaVersion), + reapplySettings = false + ) + val bspState = + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { + state => + val scalacOptions = state.scalaOptions(metalsProject)._2.items.head.options + assert(scalacOptions == defaultScalacOptions) + } + } + } + + test("force reload of all projects if reapplySettings is set to true") { + TestUtil.withinWorkspace { workspace => + val semanticdbVersion = "4.2.0" + val extraParams = BloopExtraBuildParams( + clientClassesRootDir = None, + semanticdbVersion = Some(semanticdbVersion), + supportedScalaVersions = List(testedScalaVersion), + reapplySettings = true + ) + val metalsProject = TestProject(workspace, projectName, Nil) val projects = List(metalsProject) val configDir = TestProject.populateWorkspace(workspace, projects) + WorkspaceSettings.write( + configDir, + WorkspaceSettings(semanticdbVersion, List(testedScalaVersion)) + ) val logger = new RecordingLogger(ansiCodesSupported = false) - val additionalData = Some(Json.fromFields(Nil)) - val bspState = loadBspState(workspace, projects, logger, "Metals", additionalData) { state => + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams)(_ => ()) + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => + assert(logger.infos.contains("Forcing reload of all projects")) + } + } + } + + test("should save workspace settings with cached build") { + TestUtil.withinWorkspace { workspace => + val semanticdbVersion = "4.2.0" + val extraParams = BloopExtraBuildParams( + clientClassesRootDir = None, + semanticdbVersion = Some(semanticdbVersion), + supportedScalaVersions = List(testedScalaVersion), + reapplySettings = false + ) + val metalsProject = TestProject(workspace, projectName, Nil) + val projects = List(metalsProject) + val configDir = TestProject.populateWorkspace(workspace, projects) + WorkspaceSettings.write( + configDir, + WorkspaceSettings(semanticdbVersion, List(testedScalaVersion)) + ) + val logger = new RecordingLogger(ansiCodesSupported = false) + loadBspState(workspace, projects, logger, "Metals")(_ => ()) + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => assert(configDir.resolve(WorkspaceSettings.settingsFileName).exists) + val settings = WorkspaceSettings.fromFile(configDir, logger) + assert(settings.isDefined && settings.get.semanticDBVersion == semanticdbVersion) } } } + test("initialize multiple metals clients and save settings") { + TestUtil.withinWorkspace { workspace => + val poolFor6Clients: Scheduler = Scheduler( + java.util.concurrent.Executors.newFixedThreadPool(20), + ExecutionModel.Default + ) + val metalsProject = + TestProject(workspace, projectName, Nil, scalaVersion = Some(testedScalaVersion)) + val projects = List(metalsProject) + val configDir = TestProject.populateWorkspace(workspace, projects) + val logger = new RecordingLogger(ansiCodesSupported = false) + + def createClient( + semanticdbVersion: String, + clientName: String = "normalClient" + ): Task[UnmanagedBspTestState] = { + Task { + val extraParams = BloopExtraBuildParams( + clientClassesRootDir = None, + semanticdbVersion = Some(semanticdbVersion), + supportedScalaVersions = List(testedScalaVersion), + reapplySettings = false + ) + val bspLogger = new BspClientLogger(logger) + val bspCommand = createBspCommand(configDir) + val state = TestUtil.loadTestProject(configDir.underlying, logger) + val scheduler = Some(poolFor6Clients) + val bspState = openBspConnection( + state, + bspCommand, + configDir, + bspLogger, + userIOScheduler = scheduler, + clientName = clientName, + bloopExtraParams = extraParams + ) + + assert(bspState.status == ExitStatus.Ok) + // wait for all clients to connect + Thread.sleep(500) + bspState + } + } + + val normalClientsVersion = "4.2.0" + val metalsClientVersion = "4.1.11" + val client1 = createClient(normalClientsVersion) + val client2 = createClient(normalClientsVersion) + val client3 = createClient(normalClientsVersion) + val client4 = createClient(normalClientsVersion) + val client5 = createClient(normalClientsVersion) + val metalsClient = createClient(metalsClientVersion, "Metals") + + val allClients = List(client1, client2, client3, client4, client5, metalsClient) + TestUtil.await(FiniteDuration(5, "s"), poolFor6Clients) { + Task.gatherUnordered(allClients).map(_ => ()) + } + + assert(configDir.resolve(WorkspaceSettings.settingsFileName).exists) + val settings = WorkspaceSettings.fromFile(configDir, logger) + assert(settings.isDefined && settings.get.semanticDBVersion == metalsClientVersion) + } + } + test("compile with semanticDB") { TestUtil.withinWorkspace { workspace => val sources = List( @@ -37,20 +281,20 @@ class BspMetalsClientSpec( |class Foo """.stripMargin ) - val projectName = "metals-project" + val metalsProject = TestProject(workspace, projectName, sources) val projects = List(metalsProject) val configDir = TestProject.populateWorkspace(workspace, projects) - WorkspaceSettings.write(configDir, WorkspaceSettings(SemanticDBCache.latestRelease)) + WorkspaceSettings.write(configDir, WorkspaceSettings("4.2.0", List(testedScalaVersion))) val logger = new RecordingLogger(ansiCodesSupported = false) val bspState = loadBspState(workspace, projects, logger) { state => val compiledState = state.compile(metalsProject).toTestState assert(compiledState.status == ExitStatus.Ok) - val classpath = compiledState.client.getUniqueClassesDirFor( + val classesDir = compiledState.client.getUniqueClassesDirFor( compiledState.build.getProjectFor(projectName).get ) val semanticDBFile = - classpath.resolve("META-INF/semanticdb/src/main/scala/Foo.scala.semanticdb") + classesDir.resolve(s"META-INF/semanticdb/$projectName/src/main/scala/Foo.scala.semanticdb") assert(semanticDBFile.exists) } } @@ -67,16 +311,16 @@ class BspMetalsClientSpec( val metalsProject = TestProject(workspace, projectName, sources) val projects = List(metalsProject) val configDir = TestProject.populateWorkspace(workspace, projects) - WorkspaceSettings.write(configDir, WorkspaceSettings("4.1.11")) + WorkspaceSettings.write(configDir, WorkspaceSettings("4.1.11", List(testedScalaVersion))) val logger = new RecordingLogger(ansiCodesSupported = false) val bspState = loadBspState(workspace, projects, logger) { state => val compiledState = state.compile(metalsProject).toTestState assert(compiledState.status == ExitStatus.Ok) - val classpath = compiledState.client.getUniqueClassesDirFor( + val classesDir = compiledState.client.getUniqueClassesDirFor( compiledState.build.getProjectFor(projectName).get ) val semanticDBFile = - classpath.resolve("META-INF/semanticdb/src/main/scala/Foo.scala.semanticdb") + classesDir.resolve(s"META-INF/semanticdb/$projectName/src/main/scala/Foo.scala.semanticdb") assert(semanticDBFile.exists) } } @@ -94,16 +338,24 @@ class BspMetalsClientSpec( val projects = List(metalsProject) val configDir = TestProject.populateWorkspace(workspace, projects) val logger = new RecordingLogger(ansiCodesSupported = false) - val bspState = loadBspState(workspace, projects, logger, "Metals") { state => - val compiledState = state.compile(metalsProject).toTestState - assert(compiledState.status == ExitStatus.Ok) - val classpath = compiledState.client.getUniqueClassesDirFor( - compiledState.build.getProjectFor(projectName).get - ) - val semanticDBFile = - classpath.resolve("META-INF/semanticdb/src/main/scala/Foo.scala.semanticdb") - assert(semanticDBFile.exists) - } + val extraParams = BloopExtraBuildParams( + clientClassesRootDir = None, + semanticdbVersion = Some("4.2.0"), + supportedScalaVersions = List(testedScalaVersion), + reapplySettings = false + ) + val bspState = + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { + state => + val compiledState = state.compile(metalsProject).toTestState + assert(compiledState.status == ExitStatus.Ok) + val classesDir = compiledState.client.getUniqueClassesDirFor( + compiledState.build.getProjectFor(projectName).get + ) + val semanticDBFile = + classesDir.resolve(s"META-INF/semanticdb/$projectName/src/main/scala/Foo.scala.semanticdb") + assert(semanticDBFile.exists) + } } } } diff --git a/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala b/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala index 4591ebff87..6a93fbd8ee 100644 --- a/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala @@ -106,19 +106,20 @@ class BspProtocolSpec( val extraBloopParams = BloopExtraBuildParams( Some(Uri(userClientClassesRootDir.toBspUri)), - semanticDBVersion = None + semanticdbVersion = None, + supportedScalaVersions = Nil, + reapplySettings = false ) - val initializeData = BloopExtraBuildParams.encoder(extraBloopParams) // Start first client and query for scalac options which creates client classes dirs - loadBspState(workspace, projects, logger, additionalData = Some(initializeData)) { bspState => + loadBspState(workspace, projects, logger, bloopExtraParams = extraBloopParams) { bspState => val (_, options) = bspState.scalaOptions(`A`) firstScalacOptions = options.items firstScalacOptions.foreach(d => assertIsDirectory(AbsolutePath(d.classDirectory.toPath))) } // Start second client and query for scalac options which should use same dirs as before - loadBspState(workspace, projects, logger, additionalData = Some(initializeData)) { bspState => + loadBspState(workspace, projects, logger, bloopExtraParams = extraBloopParams) { bspState => val (_, options) = bspState.scalaOptions(`A`) secondScalacOptions = options.items secondScalacOptions.foreach(d => assertIsDirectory(AbsolutePath(d.classDirectory.toPath))) diff --git a/frontend/src/test/scala/bloop/testing/BloopHelpers.scala b/frontend/src/test/scala/bloop/testing/BloopHelpers.scala index d2cb91aa95..4b708c5eef 100644 --- a/frontend/src/test/scala/bloop/testing/BloopHelpers.scala +++ b/frontend/src/test/scala/bloop/testing/BloopHelpers.scala @@ -19,15 +19,18 @@ import monix.execution.CancelableFuture import java.nio.file.{Files, Path} import java.nio.charset.StandardCharsets +import bloop.data.WorkspaceSettings trait BloopHelpers { self: BaseSuite => def loadState( workspace: AbsolutePath, projects: List[TestProject], - logger: RecordingLogger + logger: RecordingLogger, + settings: Option[WorkspaceSettings] = None ): TestState = { val configDir = TestProject.populateWorkspace(workspace, projects) + settings.foreach(WorkspaceSettings.write(configDir, _)) new TestState(TestUtil.loadTestProject(configDir.underlying, logger)) } diff --git a/frontend/src/test/scala/bloop/util/TestUtil.scala b/frontend/src/test/scala/bloop/util/TestUtil.scala index c1a5ce4934..31d6e15be5 100644 --- a/frontend/src/test/scala/bloop/util/TestUtil.scala +++ b/frontend/src/test/scala/bloop/util/TestUtil.scala @@ -73,7 +73,7 @@ object TestUtil { def checkAfterCleanCompilation( structures: Map[String, Map[String, String]], dependencies: Map[String, Set[String]], - buildSettings: Option[WorkspaceSettings] = None, + workspaceSettings: Option[WorkspaceSettings] = None, rootProjects: List[String] = List(RootProject), scalaInstance: ScalaInstance = TestUtil.scalaInstance, javaEnv: JavaEnv = JavaEnv.default, @@ -82,7 +82,7 @@ object TestUtil { useSiteLogger: Option[Logger] = None, order: CompileOrder = Config.Mixed )(afterCompile: State => Unit = (_ => ())) = { - testState(structures, dependencies, buildSettings, scalaInstance, javaEnv, order) { (state: State) => + testState(structures, dependencies, workspaceSettings, scalaInstance, javaEnv, order) { (state: State) => def action(state0: State): Unit = { val state = useSiteLogger.map(logger => state0.copy(logger = logger)).getOrElse(state0) // Check that this is a clean compile! @@ -262,7 +262,7 @@ object TestUtil { def testState[T]( projectStructures: Map[String, Map[String, String]], dependenciesMap: Map[String, Set[String]], - buildSettings: Option[WorkspaceSettings] = None, + workspaceSettings: Option[WorkspaceSettings] = None, instance: ScalaInstance = TestUtil.scalaInstance, env: JavaEnv = JavaEnv.default, order: CompileOrder = Config.Mixed, @@ -276,7 +276,7 @@ object TestUtil { val deps = dependenciesMap.getOrElse(name, Set.empty) makeProject(temp, name, sources, deps, Some(instance), env, logger, order, extraJars) } - val build = Build(temp, buildSettings, projects.toList) + val build = Build(temp, workspaceSettings, projects.toList) val state = State.forTests(build, TestUtil.getCompilerCache(logger), logger) try op(state) catch { diff --git a/shared/src/main/scala/bloop/io/AbsolutePath.scala b/shared/src/main/scala/bloop/io/AbsolutePath.scala index 1ca7718857..564f350776 100644 --- a/shared/src/main/scala/bloop/io/AbsolutePath.scala +++ b/shared/src/main/scala/bloop/io/AbsolutePath.scala @@ -18,11 +18,6 @@ final class AbsolutePath private (val underlying: Path) extends AnyVal { AbsolutePath(underlying.resolve(other.underlying))(this) def resolve(other: String): AbsolutePath = AbsolutePath(underlying.resolve(other))(this) def getParent: AbsolutePath = AbsolutePath(underlying.getParent) - - def list: List[AbsolutePath] = - Files.list(underlying).iterator().asScala.toList.map(AbsolutePath.apply) - - def name: String = underlying.getFileName().toString() def exists: Boolean = Files.exists(underlying) def isFile: Boolean = Files.isRegularFile(underlying) def isDirectory: Boolean = Files.isDirectory(underlying) From f8bcfb2f06d70c7bd77603f166c45e21320284a6 Mon Sep 17 00:00:00 2001 From: Jorge Vicente Cantero Date: Wed, 24 Jul 2019 12:25:58 +0200 Subject: [PATCH 03/22] Remove unnecessary import --- shared/src/main/scala/bloop/io/AbsolutePath.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/shared/src/main/scala/bloop/io/AbsolutePath.scala b/shared/src/main/scala/bloop/io/AbsolutePath.scala index 564f350776..72adac7ce2 100644 --- a/shared/src/main/scala/bloop/io/AbsolutePath.scala +++ b/shared/src/main/scala/bloop/io/AbsolutePath.scala @@ -4,7 +4,6 @@ package bloop.io import java.io.File import java.net.URI import java.nio.file.{Files, Path, Paths => NioPaths} -import scala.collection.JavaConverters._ final class AbsolutePath private (val underlying: Path) extends AnyVal { def syntax: String = toString From 184e306580d465bc2c5bc04e792193d8622af2eb Mon Sep 17 00:00:00 2001 From: Jorge Vicente Cantero Date: Wed, 24 Jul 2019 22:49:15 +0200 Subject: [PATCH 04/22] Make mostly stylistic changes to the bsp metals test --- .../scala/bloop/bsp/BspMetalsClientSpec.scala | 232 ++++++++---------- 1 file changed, 103 insertions(+), 129 deletions(-) diff --git a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala index d539aa1074..69cda11674 100644 --- a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala @@ -1,22 +1,28 @@ package bloop.bsp + +import bloop.io.AbsolutePath import bloop.cli.BspProtocol import bloop.util.TestUtil import bloop.util.TestProject import bloop.logging.RecordingLogger import bloop.logging.BspClientLogger import bloop.cli.ExitStatus -import java.nio.file.Files import bloop.data.WorkspaceSettings -import io.circe.JsonObject -import io.circe.Json import bloop.engine.SemanticDBCache import bloop.internal.build.BuildInfo import bloop.bsp.BloopBspDefinitions.BloopExtraBuildParams + +import java.nio.file.Files + +import io.circe.JsonObject +import io.circe.Json + import monix.execution.Scheduler import monix.execution.ExecutionModel import monix.eval.Task + import scala.concurrent.duration.FiniteDuration -import bloop.io.AbsolutePath + import ch.epfl.scala.bsp.endpoints.BuildTarget.scalacOptions object LocalBspMetalsClientSpec extends BspMetalsClientSpec(BspProtocol.Local) @@ -25,15 +31,12 @@ object TcpBspMetalsClientSpec extends BspMetalsClientSpec(BspProtocol.Tcp) class BspMetalsClientSpec( override val protocol: BspProtocol ) extends BspBaseSuite { - - val testedScalaVersion = "2.12.8" - val projectName = "metals-project" + private val testedScalaVersion = "2.12.8" test("initialize metals client and save settings") { TestUtil.withinWorkspace { workspace => - val metalsProject = - TestProject(workspace, projectName, Nil, scalaVersion = Some(testedScalaVersion)) - val projects = List(metalsProject) + val `A` = TestProject(workspace, "A", Nil, scalaVersion = Some(testedScalaVersion)) + val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) val logger = new RecordingLogger(ansiCodesSupported = false) val semanticdbVersion = "4.2.0" @@ -43,50 +46,46 @@ class BspMetalsClientSpec( supportedScalaVersions = List(testedScalaVersion), reapplySettings = false ) - val bspState = - loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { - state => - assert(configDir.resolve(WorkspaceSettings.settingsFileName).exists) - val settings = WorkspaceSettings.fromFile(configDir, logger) - assert(settings.isDefined && settings.get.semanticDBVersion == semanticdbVersion) - val scalacOptions = state.scalaOptions(metalsProject)._2.items.head.options - assert( - List( - "-P:semanticdb:failures:warning", - s"-P:semanticdb:sourceroot:$workspace", - "-P:semanticdb:synthetics:on", - "-Xplugin-require:semanticdb", - "semanticdb-scalac" - ).forall(opt => scalacOptions.find(_.contains(opt)).isDefined) - ) - } + + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => + assert(configDir.resolve(WorkspaceSettings.settingsFileName).exists) + val settings = WorkspaceSettings.fromFile(configDir, logger) + assert(settings.isDefined && settings.get.semanticDBVersion == semanticdbVersion) + val scalacOptions = state.scalaOptions(`A`)._2.items.head.options + assert( + List( + "-P:semanticdb:failures:warning", + s"-P:semanticdb:sourceroot:$workspace", + "-P:semanticdb:synthetics:on", + "-Xplugin-require:semanticdb", + "semanticdb-scalac" + ).forall(opt => scalacOptions.find(_.contains(opt)).isDefined) + ) + } } } test("do not initialize metals client and save settings with unsupported scala version") { TestUtil.withinWorkspace { workspace => - val metalsProject = - TestProject(workspace, projectName, Nil, scalaVersion = Some("2.12.4")) - val projects = List(metalsProject) + val semanticdbVersion = "4.2.0" // Doesn't support 2.12.4 + val `A` = TestProject(workspace, "A", Nil, scalaVersion = Some("2.12.4")) + val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) val logger = new RecordingLogger(ansiCodesSupported = false) - val semanticdbVersion = "4.2.0" val extraParams = BloopExtraBuildParams( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), supportedScalaVersions = List("2.12.8"), reapplySettings = false ) - val bspState = - loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { - state => - val scalacOptions = state.scalaOptions(metalsProject)._2.items.head.options - assert(scalacOptions.isEmpty) - } + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => + val scalacOptions = state.scalaOptions(`A`)._2.items.head.options + assert(scalacOptions.isEmpty) + } } } - test("initialize metals client with existing plugin in workspace") { + test("initialize metals client in workspace with already enabled semanticdb") { TestUtil.withinWorkspace { workspace => val defaultScalacOptions = List( "-P:semanticdb:failures:warning", @@ -95,15 +94,15 @@ class BspMetalsClientSpec( "-Xplugin-require:semanticdb", s"-Xplugin:path-to-plugin/semanticdb-scalac_2.12.8-4.2.0.jar.jar" ) - val metalsProject = - TestProject( - workspace, - projectName, - Nil, - scalaVersion = Some(testedScalaVersion), - scalacOptions = defaultScalacOptions //${getClass().getResource("semanticdb_2.12.8-4.1.11.jar")} - ) - val projects = List(metalsProject) + + val `A` = TestProject( + workspace, + "A", + Nil, + scalaVersion = Some(testedScalaVersion), + scalacOptions = defaultScalacOptions + ) + val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) val logger = new RecordingLogger(ansiCodesSupported = false) val semanticdbVersion = "4.2.0" @@ -113,17 +112,16 @@ class BspMetalsClientSpec( supportedScalaVersions = List(testedScalaVersion), reapplySettings = false ) - val bspState = - loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { - state => - val scalacOptions = state.scalaOptions(metalsProject)._2.items.head.options - val expected = defaultScalacOptions :+ "-Yrangepos" - assert(scalacOptions == expected) - } + + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => + val scalacOptions = state.scalaOptions(`A`)._2.items.head.options + val expected = defaultScalacOptions :+ "-Yrangepos" + assert(scalacOptions == expected) + } } } - test("initialize metals client with existing plugin and -Yrangepos in workspace") { + test("initialize metals client in workspace with already enabled semanticdb and -Yrangepos") { TestUtil.withinWorkspace { workspace => val defaultScalacOptions = List( "-P:semanticdb:failures:warning", @@ -133,15 +131,14 @@ class BspMetalsClientSpec( s"-Xplugin:path-to-plugin/semanticdb-scalac_2.12.8-4.2.0.jar.jar", "-Yrangepos" ) - val metalsProject = - TestProject( - workspace, - projectName, - Nil, - scalaVersion = Some(testedScalaVersion), - scalacOptions = defaultScalacOptions //${getClass().getResource("semanticdb_2.12.8-4.1.11.jar")} - ) - val projects = List(metalsProject) + val `A` = TestProject( + workspace, + "A", + Nil, + scalaVersion = Some(testedScalaVersion), + scalacOptions = defaultScalacOptions + ) + val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) val logger = new RecordingLogger(ansiCodesSupported = false) val semanticdbVersion = "4.2.0" @@ -151,12 +148,10 @@ class BspMetalsClientSpec( supportedScalaVersions = List(testedScalaVersion), reapplySettings = false ) - val bspState = - loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { - state => - val scalacOptions = state.scalaOptions(metalsProject)._2.items.head.options - assert(scalacOptions == defaultScalacOptions) - } + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => + val scalacOptions = state.scalaOptions(`A`)._2.items.head.options + assert(scalacOptions == defaultScalacOptions) + } } } @@ -169,8 +164,8 @@ class BspMetalsClientSpec( supportedScalaVersions = List(testedScalaVersion), reapplySettings = true ) - val metalsProject = TestProject(workspace, projectName, Nil) - val projects = List(metalsProject) + val `A` = TestProject(workspace, "A", Nil) + val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) WorkspaceSettings.write( configDir, @@ -193,8 +188,8 @@ class BspMetalsClientSpec( supportedScalaVersions = List(testedScalaVersion), reapplySettings = false ) - val metalsProject = TestProject(workspace, projectName, Nil) - val projects = List(metalsProject) + val `A` = TestProject(workspace, "A", Nil) + val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) WorkspaceSettings.write( configDir, @@ -216,9 +211,8 @@ class BspMetalsClientSpec( java.util.concurrent.Executors.newFixedThreadPool(20), ExecutionModel.Default ) - val metalsProject = - TestProject(workspace, projectName, Nil, scalaVersion = Some(testedScalaVersion)) - val projects = List(metalsProject) + val `A` = TestProject(workspace, "A", Nil, scalaVersion = Some(testedScalaVersion)) + val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) val logger = new RecordingLogger(ansiCodesSupported = false) @@ -264,7 +258,7 @@ class BspMetalsClientSpec( val metalsClient = createClient(metalsClientVersion, "Metals") val allClients = List(client1, client2, client3, client4, client5, metalsClient) - TestUtil.await(FiniteDuration(5, "s"), poolFor6Clients) { + TestUtil.await(FiniteDuration(10, "s"), poolFor6Clients) { Task.gatherUnordered(allClients).map(_ => ()) } @@ -276,66 +270,38 @@ class BspMetalsClientSpec( test("compile with semanticDB") { TestUtil.withinWorkspace { workspace => - val sources = List( - """/main/scala/Foo.scala - |class Foo - """.stripMargin - ) - - val metalsProject = TestProject(workspace, projectName, sources) - val projects = List(metalsProject) + val `A` = TestProject(workspace, "A", dummyFooSources) + val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) WorkspaceSettings.write(configDir, WorkspaceSettings("4.2.0", List(testedScalaVersion))) val logger = new RecordingLogger(ansiCodesSupported = false) val bspState = loadBspState(workspace, projects, logger) { state => - val compiledState = state.compile(metalsProject).toTestState + val compiledState = state.compile(`A`).toTestState assert(compiledState.status == ExitStatus.Ok) - val classesDir = compiledState.client.getUniqueClassesDirFor( - compiledState.build.getProjectFor(projectName).get - ) - val semanticDBFile = - classesDir.resolve(s"META-INF/semanticdb/$projectName/src/main/scala/Foo.scala.semanticdb") - assert(semanticDBFile.exists) + assertSemanticdbFileFor("Foo.scala", compiledState) } } } test("compile with semanticDB using cached plugin") { TestUtil.withinWorkspace { workspace => - val sources = List( - """/main/scala/Foo.scala - |class Foo - """.stripMargin - ) - val projectName = "metals-project" - val metalsProject = TestProject(workspace, projectName, sources) - val projects = List(metalsProject) + val `A` = TestProject(workspace, "A", dummyFooSources) + val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) WorkspaceSettings.write(configDir, WorkspaceSettings("4.1.11", List(testedScalaVersion))) val logger = new RecordingLogger(ansiCodesSupported = false) - val bspState = loadBspState(workspace, projects, logger) { state => - val compiledState = state.compile(metalsProject).toTestState + loadBspState(workspace, projects, logger) { state => + val compiledState = state.compile(`A`).toTestState assert(compiledState.status == ExitStatus.Ok) - val classesDir = compiledState.client.getUniqueClassesDirFor( - compiledState.build.getProjectFor(projectName).get - ) - val semanticDBFile = - classesDir.resolve(s"META-INF/semanticdb/$projectName/src/main/scala/Foo.scala.semanticdb") - assert(semanticDBFile.exists) + assertSemanticdbFileFor("Foo.scala", compiledState) } } } test("save settings and compile with semanticDB") { TestUtil.withinWorkspace { workspace => - val sources = List( - """/main/scala/Foo.scala - |class Foo - """.stripMargin - ) - val projectName = "metals-project" - val metalsProject = TestProject(workspace, projectName, sources) - val projects = List(metalsProject) + val `A` = TestProject(workspace, "A", dummyFooSources) + val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) val logger = new RecordingLogger(ansiCodesSupported = false) val extraParams = BloopExtraBuildParams( @@ -344,18 +310,26 @@ class BspMetalsClientSpec( supportedScalaVersions = List(testedScalaVersion), reapplySettings = false ) - val bspState = - loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { - state => - val compiledState = state.compile(metalsProject).toTestState - assert(compiledState.status == ExitStatus.Ok) - val classesDir = compiledState.client.getUniqueClassesDirFor( - compiledState.build.getProjectFor(projectName).get - ) - val semanticDBFile = - classesDir.resolve(s"META-INF/semanticdb/$projectName/src/main/scala/Foo.scala.semanticdb") - assert(semanticDBFile.exists) - } + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => + val compiledState = state.compile(`A`).toTestState + assert(compiledState.status == ExitStatus.Ok) + assertSemanticdbFileFor("Foo.scala", compiledState) + } } } + + private val dummyFooSources = List( + """/Foo.scala + |class Foo + """.stripMargin + ) + + private def assertSemanticdbFileFor(sourceFileName: String, state: TestState): Unit = { + val projectA = state.build.getProjectFor("A").get + val classesDir = state.client.getUniqueClassesDirFor(projectA) + val sourcePath = if (sourceFileName.startsWith("/")) sourceFileName else s"/$sourceFileName" + assertIsFile( + classesDir.resolve(s"META-INF/semanticdb/A/src/$sourcePath.semanticdb") + ) + } } From 380a028fe64b4d544c97866c6423bed4a78d2366 Mon Sep 17 00:00:00 2001 From: Jorge Vicente Cantero Date: Thu, 25 Jul 2019 00:03:23 +0200 Subject: [PATCH 05/22] Make test suite less verbose --- .../scala/bloop/bsp/BspMetalsClientSpec.scala | 82 +++++++++++++++---- 1 file changed, 68 insertions(+), 14 deletions(-) diff --git a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala index 69cda11674..cc7c8b49b4 100644 --- a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala @@ -48,18 +48,26 @@ class BspMetalsClientSpec( ) loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => - assert(configDir.resolve(WorkspaceSettings.settingsFileName).exists) - val settings = WorkspaceSettings.fromFile(configDir, logger) - assert(settings.isDefined && settings.get.semanticDBVersion == semanticdbVersion) - val scalacOptions = state.scalaOptions(`A`)._2.items.head.options - assert( - List( - "-P:semanticdb:failures:warning", - s"-P:semanticdb:sourceroot:$workspace", - "-P:semanticdb:synthetics:on", - "-Xplugin-require:semanticdb", - "semanticdb-scalac" - ).forall(opt => scalacOptions.find(_.contains(opt)).isDefined) + assertNoDiffInSettingsFile( + configDir, + """|{ + | "semanticDBVersion" : "4.2.0", + | "supportedScalaVersions" : [ + | "2.12.8" + | ] + |} + |""".stripMargin + ) + assertScalacOptions( + state, + `A`, + """|-Xplugin-require:semanticdb + |-P:semanticdb:failures:warning + |-P:semanticdb:sourceroot:$workspace + |-P:semanticdb:synthetics:on + |-Xplugin:semanticdb-scalac_2.12.8-4.2.0.jar + |-Yrangepos + |""".stripMargin ) } } @@ -79,8 +87,18 @@ class BspMetalsClientSpec( reapplySettings = false ) loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => - val scalacOptions = state.scalaOptions(`A`)._2.items.head.options - assert(scalacOptions.isEmpty) + assertNoDiffInSettingsFile( + configDir, + """|{ + | "semanticDBVersion" : "4.2.0", + | "supportedScalaVersions" : [ + | "2.12.8" + | ] + |} + |""".stripMargin + ) + assertScalacOptions(state, `A`, "") + assertNoDiff(logger.warnings.mkString(System.lineSeparator), "") } } } @@ -332,4 +350,40 @@ class BspMetalsClientSpec( classesDir.resolve(s"META-INF/semanticdb/A/src/$sourcePath.semanticdb") ) } + + private def assertNoDiffInSettingsFile(configDir: AbsolutePath, expected: String): Unit = { + val settingsFile = configDir.resolve(WorkspaceSettings.settingsFileName) + assertIsFile(settingsFile) + assertNoDiff( + readFile(settingsFile), + expected + ) + } + + private def assertScalacOptions( + state: ManagedBspTestState, + project: TestProject, + unorderedExpectedOptions: String + ): Unit = { + // Not the best way to obtain workspace but valid for tests + val workspaceDir = state.underlying.build.origin.getParent.syntax + val scalacOptions = state.scalaOptions(project)._2.items.flatMap(_.options).map { opt => + if (!opt.startsWith("-Xplugin:")) opt + else { + opt.split(":") match { + case Array(key, value) => s"$key:${value.split(java.io.File.separator).last}" + } + } + } + + val expectedOptions = unorderedExpectedOptions + .replace("$workspace", workspaceDir) + .split(System.lineSeparator) + .sorted + .mkString(System.lineSeparator) + assertNoDiff( + scalacOptions.sorted.mkString(System.lineSeparator), + expectedOptions + ) + } } From 01f760dae88e35175d750c4ce075cac97ff2f631 Mon Sep 17 00:00:00 2001 From: Jorge Vicente Cantero Date: Thu, 25 Jul 2019 00:05:38 +0200 Subject: [PATCH 06/22] Remove displayWarningToUser when version is unsupported The intent of this piece of code was good but after thinking about it and trying it out locally I've realized this is too verbose of a message and this information should be shown in the Metals doctor instead. Some of this code will be added back in a next commit in a different place without the `displayWarningToUser`, which IMO should only be used when the semanticdb plugin could **not** be resolved for a version we know it's compatible for. This is truly an scenario that is unlikely to happen and where good communication with users is key. --- .../main/scala/bloop/engine/BuildLoader.scala | 1 - .../scala/bloop/engine/SemanticDBCache.scala | 39 +++++++------------ 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/frontend/src/main/scala/bloop/engine/BuildLoader.scala b/frontend/src/main/scala/bloop/engine/BuildLoader.scala index 3a1e1dcabe..f22867e08c 100644 --- a/frontend/src/main/scala/bloop/engine/BuildLoader.scala +++ b/frontend/src/main/scala/bloop/engine/BuildLoader.scala @@ -179,7 +179,6 @@ object BuildLoader { scalaInstance <- project.scalaInstance pluginPath <- SemanticDBCache.findSemanticDBPlugin( scalaInstance.version, - settings.supportedScalaVersions, settings.semanticDBVersion, logger ) diff --git a/frontend/src/main/scala/bloop/engine/SemanticDBCache.scala b/frontend/src/main/scala/bloop/engine/SemanticDBCache.scala index 8963542b9c..54e28e22e3 100644 --- a/frontend/src/main/scala/bloop/engine/SemanticDBCache.scala +++ b/frontend/src/main/scala/bloop/engine/SemanticDBCache.scala @@ -24,25 +24,17 @@ object SemanticDBCache { def findSemanticDBPlugin( scalaVersion: String, - supportedScalaVersion: List[String], semanticDBVersion: String, logger: Logger ): Option[AbsolutePath] = { - if (!supportedScalaVersion.contains(scalaVersion)) { - logger.displayWarningToUser( - s"$scalaVersion is not supported for semanticDB version $semanticDBVersion" + Try { + resolveFromCache( + "org.scalameta", + s"semanticdb-scalac_$scalaVersion", + semanticDBVersion, + logger ) - None - } else - Try { - resolveFromCache( - "org.scalameta", - s"semanticdb-scalac_$scalaVersion", - semanticDBVersion, - logger - ) - }.toOption.flatten - + }.toOption.flatten } private def resolveFromCache( @@ -53,8 +45,7 @@ object SemanticDBCache { additionalRepositories: Seq[Repository] = Nil ): Option[AbsolutePath] = { val provider = ZincInternals.getComponentProvider(Paths.getCacheDirectory("semanticdb")) - val manager = - new ZincComponentManager(SemanticDBCacheLock, provider, secondaryCacheDir = None) + val manager = new ZincComponentManager(SemanticDBCacheLock, provider, secondaryCacheDir = None) def getFromResolution: Option[AbsolutePath] = { val all = DependencyResolution .resolve( @@ -75,16 +66,14 @@ object SemanticDBCache { val semanticDBId = s"$organization.$module.$version" Try(manager.file(semanticDBId)(IfMissing.Fail)) match { case Failure(exception) => - val resolved = getFromResolution - resolved match { - case Some(value) => - manager.define(semanticDBId, Seq(value.toFile)) + val resolvedPlugin = getFromResolution + resolvedPlugin match { + case Some(resolvedPlugin) => + manager.define(semanticDBId, Seq(resolvedPlugin.toFile)) case None => - logger.warn( - s"Could not resolve semanticDB version $version" - ) + logger.warn(s"Could not resolve SemanticDB version $version") } - resolved + resolvedPlugin case Success(value) => Some(AbsolutePath(value)) } } From d1d63bb30ad52356a6571183c89afc6e25ff8d9a Mon Sep 17 00:00:00 2001 From: Jorge Vicente Cantero Date: Thu, 25 Jul 2019 14:03:50 +0200 Subject: [PATCH 07/22] Add changes to resolution and transformation logic - Use new coursier API at the request of Alex, simplifies error handling - Add documentation to `enableMetalsSettings` and `detectWorkspaceDirectory`. - Cache `latest.release` resolution so that we only resolve the plugin once in the whole server session, otherwise there's too much overhead of resolving a `latest.release` artifact per project. - Refine the logic transforming the project so that we unconditionally apply range positions and we only emit warnings to users when a resolution error that we didn't expect happens. --- .../scala/bloop/DependencyResolution.scala | 98 ++++++++----------- .../src/main/scala/bloop/data/Project.scala | 79 +++++++++++++++ .../scala/bloop/data/WorkspaceSettings.scala | 16 +++ .../main/scala/bloop/engine/BuildLoader.scala | 54 +--------- .../scala/bloop/engine/SemanticDBCache.scala | 81 --------------- .../bloop/engine/caches/SemanticDBCache.scala | 72 ++++++++++++++ .../scala/bloop/bsp/BspMetalsClientSpec.scala | 5 +- 7 files changed, 212 insertions(+), 193 deletions(-) delete mode 100644 frontend/src/main/scala/bloop/engine/SemanticDBCache.scala create mode 100644 frontend/src/main/scala/bloop/engine/caches/SemanticDBCache.scala diff --git a/backend/src/main/scala/bloop/DependencyResolution.scala b/backend/src/main/scala/bloop/DependencyResolution.scala index 868647002e..c5c4359c99 100644 --- a/backend/src/main/scala/bloop/DependencyResolution.scala +++ b/backend/src/main/scala/bloop/DependencyResolution.scala @@ -5,6 +5,8 @@ import bloop.io.AbsolutePath import sbt.librarymanagement._ import sbt.librarymanagement.ivy._ +import coursier.core.Repository +import coursier.error.CoursierError object DependencyResolution { private final val BloopResolvers = @@ -15,83 +17,65 @@ object DependencyResolution { IvyDependencyResolution(configuration) } - import java.io.File - import coursier.util.{Gather, Task} - import coursier.cache.{Cache, ArtifactError} - import coursier.{ - Dependency, - Fetch, - MavenRepository, - Module, - Repository, - Resolution, - LocalRepositories, - ResolutionProcess - } - /** * Resolve the specified module and get all the files. By default, the local ivy - * repository and Maven Central are included in resolution. + * repository and Maven Central are included in resolution. This resolution throws + * in case there is an error. * * @param organization The module's organization. * @param module The module's name. * @param version The module's version. * @param logger A logger that receives messages about resolution. * @param additionalRepositories Additional repositories to include in resolition. - * @return All the files that compose the module and that could be found. + * @return All the resolved files. */ def resolve( organization: String, module: String, version: String, logger: Logger, - additionalRepositories: Seq[Repository] = Nil, - shouldReportErrors: Boolean = false + additionalRepos: Seq[Repository] = Nil )(implicit ec: scala.concurrent.ExecutionContext): Array[AbsolutePath] = { - logger.debug(s"Resolving $organization:$module:$version")(DebugFilter.Compilation) + resolveWithErrors(organization, module, version, logger, additionalRepos) match { + case Right(paths) => paths + case Left(error) => throw error + } + } + + /** + * Resolve the specified module and get all the files. By default, the local ivy + * repository and Maven Central are included in resolution. This resolution is + * pure and returns either some errors or some resolved jars. + * + * @param organization The module's organization. + * @param module The module's name. + * @param version The module's version. + * @param logger A logger that receives messages about resolution. + * @param additionalRepositories Additional repositories to include in resolition. + * @return Either a coursier error or all the resolved files. + */ + def resolveWithErrors( + organization: String, + module: String, + version: String, + logger: Logger, + additionalRepositories: Seq[Repository] = Nil + )(implicit ec: scala.concurrent.ExecutionContext): Either[CoursierError, Array[AbsolutePath]] = { + import coursier._ + logger.debug(s"Resolving $organization:$module:$version")(DebugFilter.All) val org = coursier.Organization(organization) val moduleName = coursier.ModuleName(module) val dependency = Dependency(Module(org, moduleName), version) - val start = Resolution(List(dependency)) - val repositories = { - val baseRepositories = Seq( - LocalRepositories.ivy2Local, - MavenRepository("https://repo1.maven.org/maven2"), - MavenRepository("https://dl.bintray.com/scalacenter/releases") - ) - baseRepositories ++ additionalRepositories + var fetch = Fetch() + .addDependencies(dependency) + .addRepositories(Repositories.bintray("scalacenter", "releases")) + for (repository <- additionalRepositories) { + fetch.addRepositories(repository) } - val fetch = ResolutionProcess.fetch(repositories, Cache.default.fetch) - val resolution = start.process.run(fetch).unsafeRun() - if (shouldReportErrors) reportErrors(resolution, logger) - val localArtifacts: Seq[(Boolean, Either[ArtifactError, File])] = { - Gather[Task] - .gather(resolution.artifacts().map { artifact => - Cache.default.file(artifact).run.map(artifact.optional -> _) - }) - .unsafeRun()(ec) - } - - val fileErrors = localArtifacts.collect { - case (isOptional, Left(error)) if !isOptional || !error.notFound => error - } - if (fileErrors.isEmpty) { - localArtifacts.collect { case (_, Right(f)) => AbsolutePath(f.toPath) }.toArray - } else { - val moduleInfo = s"$organization:$module:$version" - val prettyFileErrors = fileErrors.map(_.describe).mkString(System.lineSeparator) - sys.error( - s"Resolution of module $moduleInfo failed with:${System.lineSeparator}${prettyFileErrors}" - ) - } - } - private def reportErrors(resolution: Resolution, logger: Logger): Unit = { - resolution.errorCache.foreach { - case ((module, version), errors) => - logger.displayWarningToUser( - s"There were issues resolving $module:$version - ${errors.mkString("; ")}." - ) + try Right(fetch.run().map(f => AbsolutePath(f.toPath)).toArray) + catch { + case error: CoursierError => Left(error) } } } diff --git a/frontend/src/main/scala/bloop/data/Project.scala b/frontend/src/main/scala/bloop/data/Project.scala index 546a682c5e..236bea8c89 100644 --- a/frontend/src/main/scala/bloop/data/Project.scala +++ b/frontend/src/main/scala/bloop/data/Project.scala @@ -7,6 +7,7 @@ import bloop.ScalaInstance import bloop.bsp.ProjectUris import bloop.config.{Config, ConfigEncoderDecoders} import bloop.engine.Dag +import bloop.engine.caches.SemanticDBCache import bloop.engine.tasks.toolchains.{JvmToolchain, ScalaJsToolchain, ScalaNativeToolchain} import bloop.io.ByteHasher @@ -199,4 +200,82 @@ object Project { } } } + + /** + * Enable any Metals-specific setting in a project by applying an in-memory + * project transformation. A setting is Metals-specific if it's required for + * Metals to provide a complete IDE experience to users. + * + * A side-effect of this transformation is that we force the resolution of the + * semanticdb plugin. This is an expensive operation that is heavily cached + * inside [[bloop.engine.caches.SemanticDBCache]] and which can be retried in + * case the resolution for a version hasn't been successful yet and the + * workspace settings passed as a parameter asks for another attempt. + * + * @param project The project that we want to transform. + * @param settings The settings that contain Metals-specific information such + * as the expected semanticdb version or supported Scala versions. + * @param logger The logger responsible of tracking any transformation-related event. + * + */ + def enableMetalsSettings( + project: Project, + settings: WorkspaceSettings, + logger: Logger + ): Project = { + val workspaceDir = WorkspaceSettings.detectWorkspaceDirectory(project, settings) + def enableSemanticDB(options: List[String], pluginPath: AbsolutePath): List[String] = { + val hasSemanticDB = + options.exists(opt => opt.contains("-Xplugin") && opt.contains("semanticdb-scalac")) + + if (hasSemanticDB) options + else { + // TODO: Handle user-configured `targetroot`s inside Bloop's compilation + // engine so that semanticdb files are replicated in those directories + val semanticdbScalacOptions = List( + "-P:semanticdb:failures:warning", + s"-P:semanticdb:sourceroot:$workspaceDir", + "-P:semanticdb:synthetics:on", + "-Xplugin-require:semanticdb", + s"-Xplugin:$pluginPath" + ) + + (options ++ semanticdbScalacOptions.toList).distinct + } + } + + def enableRangePositions(options: List[String]): List[String] = { + val hasYrangepos = options.exists(_.contains("-Yrangepos")) + if (hasYrangepos) options else options :+ "-Yrangepos" + } + + project.scalaInstance match { + case None => project + case Some(instance) => + val projectWithRangePositions = + project.copy(scalacOptions = enableRangePositions(project.scalacOptions)) + + // Recognize 2.12.8-abdcddd as supported if 2.12.8 exists in supported versions + val isUnsupportedVersion = + !settings.supportedScalaVersions.exists(instance.version.startsWith(_)) + if (isUnsupportedVersion) { + logger.debug( + s"Skipping configuration of SemanticDB for '${project.name}': unsupported Scala v${instance.version}" + )(DebugFilter.All) + projectWithRangePositions + } else { + SemanticDBCache.fetchPlugin(instance.version, settings.semanticDBVersion, logger) match { + case Right(pluginPath) => + val options = projectWithRangePositions.scalacOptions + val optionsWithSemanticDB = enableSemanticDB(options, pluginPath) + projectWithRangePositions.copy(scalacOptions = optionsWithSemanticDB) + case Left(error) => + logger.displayWarningToUser( + s"Skipping configuration of SemanticDB for '${project.name}': $error" + ) + projectWithRangePositions + } + } + } + } } diff --git a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala index c697cce1dc..7fa095b779 100644 --- a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala +++ b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala @@ -67,4 +67,20 @@ object WorkspaceSettings { } } + /** + * Detects the workspace directory of a project. + * + * Bloop doesn't have the notion of workspace directory yet so this is just an + * approximation. We assume that the parent of `.bloop` is the workspace. This + * assumption is broken when source dependencies are used because we inline the + * configuration files of the projects in source dependencies into a single + * .bloop configuration directory. To fix this well-known limitation, we need + * to introduce a new field to the bloop configuration file so that we can map + * a project with a workspace irrevocably. + */ + def detectWorkspaceDirectory(project: Project, settings: WorkspaceSettings): AbsolutePath = { + val configFile = project.origin.path + val configDir = configFile.getParent + configDir.getParent + } } diff --git a/frontend/src/main/scala/bloop/engine/BuildLoader.scala b/frontend/src/main/scala/bloop/engine/BuildLoader.scala index f22867e08c..b2ab19fa34 100644 --- a/frontend/src/main/scala/bloop/engine/BuildLoader.scala +++ b/frontend/src/main/scala/bloop/engine/BuildLoader.scala @@ -115,7 +115,7 @@ object BuildLoader { settings: Option[WorkspaceSettings] ): Project = { val project = Project.fromBytesAndOrigin(bytes, origin, logger) - settings.map(applySettings(_, project, logger)).getOrElse(project) + settings.map(Project.enableMetalsSettings(project, _, logger)).getOrElse(project) } private def updateWorkspaceSettings( @@ -137,56 +137,4 @@ object BuildLoader { } } - - /** - * Applies workspace settings from bloop.settings.json file to a project. This includes: - * - SemanticDB plugin version to resolve and include in Scala compiler options - */ - private def applySettings( - settings: WorkspaceSettings, - project: Project, - logger: Logger - ): Project = { - def addSemanticDBOptions(pluginPath: AbsolutePath) = { - { - val optionsSet = project.scalacOptions - val containsSemanticDB = optionsSet.find( - setting => setting.contains("-Xplugin") && setting.contains("semanticdb-scalac") - ) - val containsYrangepos = optionsSet.find(_.contains("-Yrangepos")) - val semanticDBAdded = if (containsSemanticDB.isDefined) { - logger.info(s"SemanticDB plugin already added: ${containsSemanticDB.get}") - optionsSet - } else { - val workspaceDir = project.origin.path.getParent.getParent - optionsSet ++ Set( - "-P:semanticdb:failures:warning", - s"-P:semanticdb:sourceroot:$workspaceDir", - "-P:semanticdb:synthetics:on", - "-Xplugin-require:semanticdb", - s"-Xplugin:$pluginPath" - ) - } - if (containsYrangepos.isDefined) { - semanticDBAdded - } else { - semanticDBAdded :+ "-Yrangepos" - }.distinct - } - } - - val mappedProject = for { - scalaInstance <- project.scalaInstance - pluginPath <- SemanticDBCache.findSemanticDBPlugin( - scalaInstance.version, - settings.semanticDBVersion, - logger - ) - } yield { - val scalacOptions = addSemanticDBOptions(pluginPath) - project.copy(scalacOptions = scalacOptions) - } - mappedProject.getOrElse(project) - } - } diff --git a/frontend/src/main/scala/bloop/engine/SemanticDBCache.scala b/frontend/src/main/scala/bloop/engine/SemanticDBCache.scala deleted file mode 100644 index 54e28e22e3..0000000000 --- a/frontend/src/main/scala/bloop/engine/SemanticDBCache.scala +++ /dev/null @@ -1,81 +0,0 @@ -package bloop.engine -import bloop.logging.Logger -import coursier.core.Repository -import bloop.io.AbsolutePath -import bloop.io.Paths -import bloop.DependencyResolution -import scala.util.Try -import java.nio.file.Files -import xsbti.ComponentProvider -import xsbti.GlobalLock -import java.io.File -import java.util.concurrent.Callable -import sbt.internal.inc.bloop.ZincInternals -import sbt.internal.inc.ZincComponentManager -import sbt.internal.inc.IfMissing -import scala.util.Failure -import scala.util.Success -import bloop.ComponentLock -import bloop.SemanticDBCacheLock - -object SemanticDBCache { - - private val latestRelease = "latest.release" - - def findSemanticDBPlugin( - scalaVersion: String, - semanticDBVersion: String, - logger: Logger - ): Option[AbsolutePath] = { - Try { - resolveFromCache( - "org.scalameta", - s"semanticdb-scalac_$scalaVersion", - semanticDBVersion, - logger - ) - }.toOption.flatten - } - - private def resolveFromCache( - organization: String, - module: String, - version: String, - logger: Logger, - additionalRepositories: Seq[Repository] = Nil - ): Option[AbsolutePath] = { - val provider = ZincInternals.getComponentProvider(Paths.getCacheDirectory("semanticdb")) - val manager = new ZincComponentManager(SemanticDBCacheLock, provider, secondaryCacheDir = None) - def getFromResolution: Option[AbsolutePath] = { - val all = DependencyResolution - .resolve( - organization, - module, - version, - logger, - additionalRepositories, - shouldReportErrors = true - )( - bloop.engine.ExecutionContext.ioScheduler - ) - all.find(_.toString().contains("semanticdb-scalac")) - } - if (version == latestRelease) { - getFromResolution - } else { - val semanticDBId = s"$organization.$module.$version" - Try(manager.file(semanticDBId)(IfMissing.Fail)) match { - case Failure(exception) => - val resolvedPlugin = getFromResolution - resolvedPlugin match { - case Some(resolvedPlugin) => - manager.define(semanticDBId, Seq(resolvedPlugin.toFile)) - case None => - logger.warn(s"Could not resolve SemanticDB version $version") - } - resolvedPlugin - case Success(value) => Some(AbsolutePath(value)) - } - } - } -} diff --git a/frontend/src/main/scala/bloop/engine/caches/SemanticDBCache.scala b/frontend/src/main/scala/bloop/engine/caches/SemanticDBCache.scala new file mode 100644 index 0000000000..205ac8279a --- /dev/null +++ b/frontend/src/main/scala/bloop/engine/caches/SemanticDBCache.scala @@ -0,0 +1,72 @@ +package bloop.engine.caches + +import bloop.logging.Logger +import coursier.core.Repository +import bloop.io.AbsolutePath +import bloop.io.Paths +import bloop.DependencyResolution +import scala.util.Try +import java.nio.file.Files +import xsbti.ComponentProvider +import xsbti.GlobalLock +import java.io.File +import java.util.concurrent.Callable +import sbt.internal.inc.bloop.ZincInternals +import sbt.internal.inc.ZincComponentManager +import sbt.internal.inc.IfMissing +import scala.util.Failure +import scala.util.Success +import bloop.ComponentLock +import bloop.SemanticDBCacheLock +import java.nio.file.Path +import coursier.error.CoursierError + +object SemanticDBCache { + @volatile private var latestResolvedSemanticDB: Path = null + def fetchPlugin( + scalaVersion: String, + version: String, + logger: Logger + ): Either[String, AbsolutePath] = { + val organization = "org.scalameta" + val module = s"semanticdb-scalac_$scalaVersion" + val semanticDBId = s"$organization.$module.$version" + val provider = ZincInternals.getComponentProvider(Paths.getCacheDirectory("semanticdb")) + val manager = new ZincComponentManager(SemanticDBCacheLock, provider, secondaryCacheDir = None) + + def attemptResolution: Either[String, AbsolutePath] = { + import bloop.engine.ExecutionContext.ioScheduler + DependencyResolution.resolveWithErrors(organization, module, version, logger)(ioScheduler) match { + case Left(error) => Left(error.getMessage()) + case Right(paths) => + paths.find(_.syntax.contains("semanticdb-scalac")) match { + case Some(pluginPath) => Right(pluginPath) + case None => + Left( + s"Missing semanticdb plugin in resolved jars ${paths.map(_.syntax).mkString(",")}" + ) + } + } + } + + if (version == "latest.release") { + // Only resolve once per bloop server invocation to avoid excessive overhead + latestResolvedSemanticDB.synchronized { + if (latestResolvedSemanticDB != null) Right(AbsolutePath(latestResolvedSemanticDB)) + else { + val latestResolvedPlugin = attemptResolution + latestResolvedPlugin.foreach(plugin => latestResolvedSemanticDB = plugin.underlying) + latestResolvedPlugin + } + } + } else { + Try(manager.file(semanticDBId)(IfMissing.Fail)) match { + case Success(pluginPath) => Right(AbsolutePath(pluginPath)) + case Failure(exception) => + val resolvedPlugin = attemptResolution + resolvedPlugin.foreach(plugin => manager.define(semanticDBId, Seq(plugin.toFile))) + resolvedPlugin + } + } + } +} diff --git a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala index cc7c8b49b4..3e22802cb1 100644 --- a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala @@ -8,7 +8,6 @@ import bloop.logging.RecordingLogger import bloop.logging.BspClientLogger import bloop.cli.ExitStatus import bloop.data.WorkspaceSettings -import bloop.engine.SemanticDBCache import bloop.internal.build.BuildInfo import bloop.bsp.BloopBspDefinitions.BloopExtraBuildParams @@ -97,7 +96,8 @@ class BspMetalsClientSpec( |} |""".stripMargin ) - assertScalacOptions(state, `A`, "") + // Expect only range positions to be added, semanticdb is missing + assertScalacOptions(state, `A`, "-Yrangepos") assertNoDiff(logger.warnings.mkString(System.lineSeparator), "") } } @@ -379,6 +379,7 @@ class BspMetalsClientSpec( val expectedOptions = unorderedExpectedOptions .replace("$workspace", workspaceDir) .split(System.lineSeparator) + .filterNot(_.isEmpty) .sorted .mkString(System.lineSeparator) assertNoDiff( From 32351b4a67c82744a65f328e5e446717fa079b68 Mon Sep 17 00:00:00 2001 From: Jorge Vicente Cantero Date: Sun, 28 Jul 2019 13:38:57 +0200 Subject: [PATCH 08/22] Change read and write of workspace settings These changes are not really important. They will be refined in an upcoming commit. --- .../scala/bloop/data/WorkspaceSettings.scala | 20 ++++++---------- .../src/main/scala/bloop/engine/Build.scala | 14 +++++++---- .../main/scala/bloop/engine/BuildLoader.scala | 24 +++++++++---------- .../scala/bloop/bsp/BspMetalsClientSpec.scala | 3 ++- 4 files changed, 30 insertions(+), 31 deletions(-) diff --git a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala index 7fa095b779..9d03a35650 100644 --- a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala +++ b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala @@ -29,24 +29,21 @@ object WorkspaceSettings { private val settingsEncoder: ObjectEncoder[WorkspaceSettings] = deriveEncoder private val settingsDecoder: Decoder[WorkspaceSettings] = deriveDecoder - def fromFile(configPath: AbsolutePath, logger: Logger): Option[WorkspaceSettings] = { + def readFromFile(configPath: AbsolutePath, logger: Logger): Option[WorkspaceSettings] = { val settingsPath = configPath.resolve(settingsFileName) - if (!settingsPath.isFile) { - None - } else { + if (!settingsPath.isFile) None + else { val bytes = Files.readAllBytes(settingsPath.underlying) - logger.debug(s"Loading workspace settings from $settingsFileName")( - DebugFilter.All - ) + logger.debug(s"Loading workspace settings from $settingsFileName")(DebugFilter.All) val contents = new String(bytes, StandardCharsets.UTF_8) parser.parse(contents) match { - case Left(failure) => throw failure + case Left(e) => throw e case Right(json) => Option(fromJson(json)) } } } - def write(configDir: AbsolutePath, settings: WorkspaceSettings): Either[Throwable, Path] = { + def writeToFile(configDir: AbsolutePath, settings: WorkspaceSettings): Either[Throwable, Path] = { Try { val jsonObject = settingsEncoder(settings) val output = Printer.spaces4.copy(dropNullValues = true).pretty(jsonObject) @@ -54,10 +51,7 @@ object WorkspaceSettings { configDir.resolve(settingsFileName).underlying, output.getBytes(StandardCharsets.UTF_8) ) - } match { - case Failure(exception) => Left(exception) - case Success(value) => Right(value) - } + }.toEither } def fromJson(json: Json): WorkspaceSettings = { diff --git a/frontend/src/main/scala/bloop/engine/Build.scala b/frontend/src/main/scala/bloop/engine/Build.scala index b0d2c4f0e0..72fe4b7b36 100644 --- a/frontend/src/main/scala/bloop/engine/Build.scala +++ b/frontend/src/main/scala/bloop/engine/Build.scala @@ -8,6 +8,7 @@ import bloop.util.CacheHashCode import bloop.io.ByteHasher import monix.eval.Task import bloop.data.WorkspaceSettings +import bloop.logging.DebugFilter final case class Build private ( origin: AbsolutePath, @@ -38,14 +39,19 @@ final case class Build private ( val files = projects.iterator.map(p => p.origin.toAttributedPath).toSet val newFiles = BuildLoader.readConfigurationFilesInBase(origin, logger).toSet - def relevantChange(workspaceSettings: WorkspaceSettings) = + def settingsForProjectReload(workspaceSettings: WorkspaceSettings) = (workspaceSettings.semanticDBVersion, workspaceSettings.supportedScalaVersions) - val changedSettings = reapplySettings || + val changedSettings = reapplySettings || ( (incomingSettings.nonEmpty && - settings.map(relevantChange) != incomingSettings.map(relevantChange)) + settings.map(settingsForProjectReload) != incomingSettings.map(settingsForProjectReload)) + ) + if (reapplySettings) { - logger.info(s"Forcing reload of all projects") + logger.debug(s"Incoming BSP workspace settings require reloading all projects")( + DebugFilter.All + ) } + // This is the fast path to short circuit quickly if they are the same if (newFiles == files && !changedSettings) { Task.now(Build.ReturnPreviousState) diff --git a/frontend/src/main/scala/bloop/engine/BuildLoader.scala b/frontend/src/main/scala/bloop/engine/BuildLoader.scala index b2ab19fa34..149c72ce02 100644 --- a/frontend/src/main/scala/bloop/engine/BuildLoader.scala +++ b/frontend/src/main/scala/bloop/engine/BuildLoader.scala @@ -30,7 +30,7 @@ object BuildLoader { } /** - * Load only the projects passed as arguments. + * Loads only the projects passed as arguments. * * @param configRoot The base directory from which to load the projects. * @param logger The logger that collects messages about project loading. @@ -44,7 +44,7 @@ object BuildLoader { ): Task[LoadedBuild] = { val workspaceSettings = Task(updateWorkspaceSettings(configDir, logger, incomingSettings)) logger.debug(s"Loading ${configFiles.length} projects from '${configDir.syntax}'...")( - DebugFilter.Compilation + DebugFilter.All ) workspaceSettings .flatMap { settings => @@ -95,7 +95,7 @@ object BuildLoader { configDir: AbsolutePath, logger: Logger ): LoadedBuild = { - val settings = WorkspaceSettings.fromFile(configDir, logger) + val settings = WorkspaceSettings.readFromFile(configDir, logger) val configFiles = readConfigurationFilesInBase(configDir, logger).map { ap => val bytes = ap.path.readAllBytes val hash = ByteHasher.hashBytes(bytes) @@ -123,18 +123,16 @@ object BuildLoader { logger: Logger, incomingSettings: Option[WorkspaceSettings] ): Option[WorkspaceSettings] = { - val savedSettings = WorkspaceSettings.fromFile(configDir, logger) + val currentSettings = WorkspaceSettings.readFromFile(configDir, logger) incomingSettings match { - case Some(incoming) => - if (savedSettings.isEmpty || savedSettings.exists(_ != incoming)) { - WorkspaceSettings.write(configDir, incoming) - Some(incoming) - } else { - savedSettings + case Some(newSettings) + if currentSettings.isEmpty || currentSettings.exists(_ != newSettings) => + WorkspaceSettings.writeToFile(configDir, newSettings).left.foreach { t => + logger.debug(s"Unexpected failure when writing workspace settings: $t")(DebugFilter.All) + logger.trace(t) } - case None => - savedSettings + Some(newSettings) + case _ => currentSettings } - } } diff --git a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala index 3e22802cb1..c9d610d7fb 100644 --- a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala @@ -47,6 +47,7 @@ class BspMetalsClientSpec( ) loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => + assertNoDiff(logger.warnings.mkString(System.lineSeparator), "") assertNoDiffInSettingsFile( configDir, """|{ @@ -217,7 +218,7 @@ class BspMetalsClientSpec( loadBspState(workspace, projects, logger, "Metals")(_ => ()) loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => assert(configDir.resolve(WorkspaceSettings.settingsFileName).exists) - val settings = WorkspaceSettings.fromFile(configDir, logger) + val settings = WorkspaceSettings.readFromFile(configDir, logger) assert(settings.isDefined && settings.get.semanticDBVersion == semanticdbVersion) } } From 5020ef6414a3fe928cf3329eb9b369a2a6386043 Mon Sep 17 00:00:00 2001 From: Jorge Vicente Cantero Date: Mon, 29 Jul 2019 09:17:49 +0200 Subject: [PATCH 09/22] Add `LoadedProject` abstraction And lay the foundation for the big changes coming in the next commit. --- .../main/scala/bloop/data/LoadedBuild.scala | 8 ++- .../main/scala/bloop/data/LoadedProject.scala | 15 +++++ .../src/main/scala/bloop/data/Project.scala | 12 ++-- .../scala/bloop/data/WorkspaceSettings.scala | 37 ++++++++---- .../src/main/scala/bloop/engine/Build.scala | 56 ++++++++++++++++- .../main/scala/bloop/engine/BuildLoader.scala | 60 ++++++++++++------- .../bloop/engine/caches/StateCache.scala | 4 +- .../scala/bloop/bsp/BspMetalsClientSpec.scala | 13 ++-- .../scala/bloop/testing/BloopHelpers.scala | 2 +- 9 files changed, 153 insertions(+), 54 deletions(-) create mode 100644 frontend/src/main/scala/bloop/data/LoadedProject.scala diff --git a/frontend/src/main/scala/bloop/data/LoadedBuild.scala b/frontend/src/main/scala/bloop/data/LoadedBuild.scala index 7cb41c485b..915af6a588 100644 --- a/frontend/src/main/scala/bloop/data/LoadedBuild.scala +++ b/frontend/src/main/scala/bloop/data/LoadedBuild.scala @@ -1,3 +1,9 @@ package bloop.data -case class LoadedBuild(projects : List[Project], workspaceSettings : Option[WorkspaceSettings]) \ No newline at end of file +/** + * A partial loaded build is the incremental result of loading a certain amount + * of configuration files from disk and post-processing them in-memory. + */ +case class PartialLoadedBuild( + projects: List[LoadedProject] +) diff --git a/frontend/src/main/scala/bloop/data/LoadedProject.scala b/frontend/src/main/scala/bloop/data/LoadedProject.scala new file mode 100644 index 0000000000..59922d3f7e --- /dev/null +++ b/frontend/src/main/scala/bloop/data/LoadedProject.scala @@ -0,0 +1,15 @@ +package bloop.data + +sealed trait LoadedProject + +object LoadedProject { + final case class RawProject( + project: Project + ) extends LoadedProject + + final case class ConfiguredProject( + project: Project, + originalProject: Project, + settings: WorkspaceSettings + ) extends LoadedProject +} diff --git a/frontend/src/main/scala/bloop/data/Project.scala b/frontend/src/main/scala/bloop/data/Project.scala index 236bea8c89..fa8ff1d16c 100644 --- a/frontend/src/main/scala/bloop/data/Project.scala +++ b/frontend/src/main/scala/bloop/data/Project.scala @@ -216,13 +216,13 @@ object Project { * @param settings The settings that contain Metals-specific information such * as the expected semanticdb version or supported Scala versions. * @param logger The logger responsible of tracking any transformation-related event. - * + * @return Either the same project as before or the transformed project. */ def enableMetalsSettings( project: Project, settings: WorkspaceSettings, logger: Logger - ): Project = { + ): Either[Project, Project] = { val workspaceDir = WorkspaceSettings.detectWorkspaceDirectory(project, settings) def enableSemanticDB(options: List[String], pluginPath: AbsolutePath): List[String] = { val hasSemanticDB = @@ -250,7 +250,7 @@ object Project { } project.scalaInstance match { - case None => project + case None => Left(project) case Some(instance) => val projectWithRangePositions = project.copy(scalacOptions = enableRangePositions(project.scalacOptions)) @@ -262,18 +262,18 @@ object Project { logger.debug( s"Skipping configuration of SemanticDB for '${project.name}': unsupported Scala v${instance.version}" )(DebugFilter.All) - projectWithRangePositions + Right(projectWithRangePositions) } else { SemanticDBCache.fetchPlugin(instance.version, settings.semanticDBVersion, logger) match { case Right(pluginPath) => val options = projectWithRangePositions.scalacOptions val optionsWithSemanticDB = enableSemanticDB(options, pluginPath) - projectWithRangePositions.copy(scalacOptions = optionsWithSemanticDB) + Right(projectWithRangePositions.copy(scalacOptions = optionsWithSemanticDB)) case Left(error) => logger.displayWarningToUser( s"Skipping configuration of SemanticDB for '${project.name}': $error" ) - projectWithRangePositions + Right(projectWithRangePositions) } } } diff --git a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala index 9d03a35650..0abfae032d 100644 --- a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala +++ b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala @@ -3,29 +3,40 @@ package bloop.data import bloop.engine.BuildLoader import bloop.logging.Logger import bloop.logging.DebugFilter -import java.nio.charset.StandardCharsets import bloop.config.ConfigEncoderDecoders -import io.circe.Printer -import io.circe.Json -import io.circe.parser import bloop.io.AbsolutePath -import java.nio.file.Files -import io.circe.JsonObject import bloop.DependencyResolution +import bloop.io.RelativePath + import scala.util.Try -import io.circe.ObjectEncoder -import io.circe.Decoder -import io.circe.derivation._ -import java.nio.file.Path import scala.util.Failure import scala.util.Success -case class WorkspaceSettings(semanticDBVersion: String, supportedScalaVersions: List[String]) +import java.nio.file.Path +import java.nio.file.Files +import java.nio.charset.StandardCharsets + +import io.circe.Json +import io.circe.parser +import io.circe.Printer +import io.circe.Decoder +import io.circe.ObjectEncoder +import io.circe.JsonObject + +case class WorkspaceSettings( + semanticDBVersion: String, + supportedScalaVersions: List[String] +) object WorkspaceSettings { - /** File to store Metals specific settings*/ - val settingsFileName = "bloop.settings.json" + sealed trait DetectedChange + final case class SemanticdbVersionChange(newVersion: String) extends DetectedChange + + /** File name to store Metals specific settings*/ + private[bloop] val settingsFileName = RelativePath("bloop.settings.json") + + import io.circe.derivation._ private val settingsEncoder: ObjectEncoder[WorkspaceSettings] = deriveEncoder private val settingsDecoder: Decoder[WorkspaceSettings] = deriveDecoder diff --git a/frontend/src/main/scala/bloop/engine/Build.scala b/frontend/src/main/scala/bloop/engine/Build.scala index 72fe4b7b36..7f0f5b623a 100644 --- a/frontend/src/main/scala/bloop/engine/Build.scala +++ b/frontend/src/main/scala/bloop/engine/Build.scala @@ -26,19 +26,32 @@ final case class Build private ( def hasMissingDependencies(project: Project): Option[List[String]] = missingDeps.get(project) /** - * Detect changes in the build definition since the last time it was loaded. + * Detect changes in the build definition since the last time it was loaded + * and tell the compiler which action should be applied to update the build. * + * There are two major kinds of changes: + * + * 1. A change in a configuration file backing up the metadata of a project. + * 2. A change in a workspace setting that requires transforming a project + * after it has been loaded from the configuration file. + * + * @param newSettings The new settings that should be applied to detect changes. + * These settings are passed by certain clients such as Metals + * to apply in-memory transformations on projects. They can + * differ from the settings written to disk. * @param logger A logger that receives errors, if any. + * @param retryFailedProjects * @return The status of the directory from which the build was loaded. */ def checkForChange( + newSettings: Option[WorkspaceSettings], logger: Logger, - incomingSettings: Option[WorkspaceSettings] = None, - reapplySettings: Boolean = false + retryFailedProjects: Boolean ): Task[Build.ReloadAction] = { val files = projects.iterator.map(p => p.origin.toAttributedPath).toSet val newFiles = BuildLoader.readConfigurationFilesInBase(origin, logger).toSet + /* def settingsForProjectReload(workspaceSettings: WorkspaceSettings) = (workspaceSettings.semanticDBVersion, workspaceSettings.supportedScalaVersions) val changedSettings = reapplySettings || ( @@ -51,6 +64,7 @@ final case class Build private ( DebugFilter.All ) } + */ // This is the fast path to short circuit quickly if they are the same if (newFiles == files && !changedSettings) { @@ -85,6 +99,42 @@ final case class Build private ( } } } + + sealed trait UpdateSettingsAction + + final case class ForceReload( + settings: WorkspaceSettings, + changes: List[WorkspaceSettings.DetectedChange] + ) extends UpdateSettingsAction + + final case class WriteSettings( + settings: WorkspaceSettings + ) extends UpdateSettingsAction + + /** + * Picks the settings that have to be used to reload the build. + * @return Either the current settings when nothing has changed or the new + * settingsb + */ + def findUpdateSettingsAction( + newSettings: Option[WorkspaceSettings], + logger: Logger + ): Either[Option[WorkspaceSettings], UpdateSettingsAction] = { + val currentSettings = WorkspaceSettings.readFromFile(origin, logger) + (currentSettings, newSettings) match { + case (Some(currentSettings), Some(newSettings)) + if currentSettings.semanticDBVersion != newSettings.semanticDBVersion => + // Write the only supported change affecting build load semantics so far + val changes = List( + WorkspaceSettings.SemanticdbVersionChange(newSettings.semanticDBVersion) + ) + Right(ForceReload(newSettings, changes)) + case (Some(_), Some(newSettings)) => Left(Some(newSettings)) + case (None, Some(newSettings)) => Right(WriteSettings(newSettings)) + case (Some(currentSettings), None) => Left(Some(currentSettings)) + case (None, None) => Left(None) + } + } } object Build { diff --git a/frontend/src/main/scala/bloop/engine/BuildLoader.scala b/frontend/src/main/scala/bloop/engine/BuildLoader.scala index 149c72ce02..5332008408 100644 --- a/frontend/src/main/scala/bloop/engine/BuildLoader.scala +++ b/frontend/src/main/scala/bloop/engine/BuildLoader.scala @@ -5,9 +5,11 @@ import bloop.io.Paths.AttributedPath import bloop.io.AbsolutePath import bloop.logging.{DebugFilter, Logger} import bloop.io.ByteHasher -import monix.eval.Task import bloop.data.WorkspaceSettings -import bloop.data.LoadedBuild +import bloop.data.PartialLoadedBuild +import bloop.data.LoadedProject + +import monix.eval.Task object BuildLoader { @@ -39,36 +41,36 @@ object BuildLoader { def loadBuildFromConfigurationFiles( configDir: AbsolutePath, configFiles: List[Build.ReadConfiguration], - incomingSettings: Option[WorkspaceSettings], + newSettings: Option[WorkspaceSettings], logger: Logger - ): Task[LoadedBuild] = { - val workspaceSettings = Task(updateWorkspaceSettings(configDir, logger, incomingSettings)) + ): Task[PartialLoadedBuild] = { + val workspaceSettings = Task(updateWorkspaceSettings(configDir, logger, newSettings)) logger.debug(s"Loading ${configFiles.length} projects from '${configDir.syntax}'...")( DebugFilter.All ) - workspaceSettings - .flatMap { settings => - val all = configFiles.map(f => Task(loadProject(f.bytes, f.origin, logger, settings))) - val groupTasks = all.grouped(10).map(group => Task.gatherUnordered(group)).toList - Task - .sequence(groupTasks) - .map(fp => LoadedBuild(fp.flatten, settings)) - } - .executeOn(ExecutionContext.ioScheduler) + + val loadSettingsAndBuild = workspaceSettings.flatMap { settings => + val all = configFiles.map(f => Task(loadProject(f.bytes, f.origin, logger, settings))) + val groupTasks = all.grouped(10).map(group => Task.gatherUnordered(group)).toList + Task.sequence(groupTasks).map(fp => PartialLoadedBuild(fp.flatten)) + } + + loadSettingsAndBuild.executeOn(ExecutionContext.ioScheduler) } /** * Load all the projects from `configDir` in a parallel, lazy fashion via monix Task. * * @param configDir The base directory from which to load the projects. + * @param newSettings The settings that we should use to load this build. * @param logger The logger that collects messages about project loading. * @return The list of loaded projects. */ def load( configDir: AbsolutePath, - incomingSettings: Option[WorkspaceSettings], + newSettings: Option[WorkspaceSettings], logger: Logger - ): Task[LoadedBuild] = { + ): Task[PartialLoadedBuild] = { val configFiles = readConfigurationFilesInBase(configDir, logger).map { ap => Task { val bytes = ap.path.readAllBytes @@ -80,12 +82,16 @@ object BuildLoader { Task .gatherUnordered(configFiles) .flatMap { fs => - loadBuildFromConfigurationFiles(configDir, fs, incomingSettings, logger) + loadBuildFromConfigurationFiles(configDir, fs, newSettings, logger) } } /** - * Load all the projects from `configDir` synchronously. + * Loads all the projects from `configDir` synchronously. + * + * This method does not take any new settings because its call-sites are + * not used in the CLI/bloop server, instead this is an entrypoint used + * mostly for our testing and community build infrastructure. * * @param configDir The base directory from which to load the projects. * @param logger The logger that collects messages about project loading. @@ -94,7 +100,7 @@ object BuildLoader { def loadSynchronously( configDir: AbsolutePath, logger: Logger - ): LoadedBuild = { + ): PartialLoadedBuild = { val settings = WorkspaceSettings.readFromFile(configDir, logger) val configFiles = readConfigurationFilesInBase(configDir, logger).map { ap => val bytes = ap.path.readAllBytes @@ -103,9 +109,10 @@ object BuildLoader { } logger.debug(s"Loading ${configFiles.length} projects from '${configDir.syntax}'...")( - DebugFilter.Compilation + DebugFilter.All ) - LoadedBuild(configFiles.map(f => loadProject(f.bytes, f.origin, logger, settings)), settings) + + PartialLoadedBuild(configFiles.map(f => loadProject(f.bytes, f.origin, logger, settings))) } private def loadProject( @@ -113,9 +120,16 @@ object BuildLoader { origin: Origin, logger: Logger, settings: Option[WorkspaceSettings] - ): Project = { + ): LoadedProject = { val project = Project.fromBytesAndOrigin(bytes, origin, logger) - settings.map(Project.enableMetalsSettings(project, _, logger)).getOrElse(project) + settings match { + case None => LoadedProject.RawProject(project) + case Some(settings) => + Project.enableMetalsSettings(project, settings, logger) match { + case Left(project) => LoadedProject.RawProject(project) + case Right(transformed) => LoadedProject.ConfiguredProject(transformed, project, settings) + } + } } private def updateWorkspaceSettings( diff --git a/frontend/src/main/scala/bloop/engine/caches/StateCache.scala b/frontend/src/main/scala/bloop/engine/caches/StateCache.scala index e6c89a7b64..0d7e89a69f 100644 --- a/frontend/src/main/scala/bloop/engine/caches/StateCache.scala +++ b/frontend/src/main/scala/bloop/engine/caches/StateCache.scala @@ -91,13 +91,13 @@ final class StateCache(cache: ConcurrentHashMap[AbsolutePath, StateCache.CachedS ): Task[State] = { getStateFor(from, client, pool, commonOptions, logger) match { case Some(state) => - state.build.checkForChange(logger, incomingSettings, reapplySettings).flatMap { + state.build.checkForChange(incomingSettings, logger, reapplySettings).flatMap { case Build.ReturnPreviousState => Task.now(state) case Build.UpdateState(createdOrModified, deleted, settingsChanged) => BuildLoader .loadBuildFromConfigurationFiles(from, createdOrModified, settingsChanged, logger) .map { - case LoadedBuild(newProjects, settings) => + case PartialLoadedBuild(newProjects, settings) => val currentProjects = state.build.projects val toRemove = deleted.toSet ++ newProjects.map(_.origin.path) val untouched = diff --git a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala index c9d610d7fb..dda57d83ea 100644 --- a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala @@ -186,7 +186,7 @@ class BspMetalsClientSpec( val `A` = TestProject(workspace, "A", Nil) val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) - WorkspaceSettings.write( + WorkspaceSettings.writeToFile( configDir, WorkspaceSettings(semanticdbVersion, List(testedScalaVersion)) ) @@ -210,7 +210,7 @@ class BspMetalsClientSpec( val `A` = TestProject(workspace, "A", Nil) val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) - WorkspaceSettings.write( + WorkspaceSettings.writeToFile( configDir, WorkspaceSettings(semanticdbVersion, List(testedScalaVersion)) ) @@ -282,7 +282,7 @@ class BspMetalsClientSpec( } assert(configDir.resolve(WorkspaceSettings.settingsFileName).exists) - val settings = WorkspaceSettings.fromFile(configDir, logger) + val settings = WorkspaceSettings.readFromFile(configDir, logger) assert(settings.isDefined && settings.get.semanticDBVersion == metalsClientVersion) } } @@ -292,7 +292,7 @@ class BspMetalsClientSpec( val `A` = TestProject(workspace, "A", dummyFooSources) val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) - WorkspaceSettings.write(configDir, WorkspaceSettings("4.2.0", List(testedScalaVersion))) + WorkspaceSettings.writeToFile(configDir, WorkspaceSettings("4.2.0", List(testedScalaVersion))) val logger = new RecordingLogger(ansiCodesSupported = false) val bspState = loadBspState(workspace, projects, logger) { state => val compiledState = state.compile(`A`).toTestState @@ -307,7 +307,10 @@ class BspMetalsClientSpec( val `A` = TestProject(workspace, "A", dummyFooSources) val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) - WorkspaceSettings.write(configDir, WorkspaceSettings("4.1.11", List(testedScalaVersion))) + WorkspaceSettings.writeToFile( + configDir, + WorkspaceSettings("4.1.11", List(testedScalaVersion)) + ) val logger = new RecordingLogger(ansiCodesSupported = false) loadBspState(workspace, projects, logger) { state => val compiledState = state.compile(`A`).toTestState diff --git a/frontend/src/test/scala/bloop/testing/BloopHelpers.scala b/frontend/src/test/scala/bloop/testing/BloopHelpers.scala index 4b708c5eef..8c128c3ff9 100644 --- a/frontend/src/test/scala/bloop/testing/BloopHelpers.scala +++ b/frontend/src/test/scala/bloop/testing/BloopHelpers.scala @@ -30,7 +30,7 @@ trait BloopHelpers { settings: Option[WorkspaceSettings] = None ): TestState = { val configDir = TestProject.populateWorkspace(workspace, projects) - settings.foreach(WorkspaceSettings.write(configDir, _)) + settings.foreach(WorkspaceSettings.writeToFile(configDir, _)) new TestState(TestUtil.loadTestProject(configDir.underlying, logger)) } From f1fa503041b2f6fb467db73461f2c2a43821afd8 Mon Sep 17 00:00:00 2001 From: jvican Date: Tue, 30 Jul 2019 17:37:11 +0200 Subject: [PATCH 10/22] Add incremental build load mechanism This big commit rethinks how build load works to make it incremental. It adds more tests and documentation to the appropiate internal APIs to make this area easier to browse in the future. The previous logic was working perfectly fine but could incur some performance overhead because every time there was a change in the server the build load process would restart and that implies also resolving semanticdb plugins for all scala versions in a project. That is not a big problem because most of the times they are cached but the overhead of that operation is nevertheless too expensive, so the incremental build load process mitigates that. --- .../src/it/scala/bloop/CommunityBuild.scala | 20 +- frontend/src/main/scala/bloop/Bloop.scala | 6 +- .../scala/bloop/bsp/BloopBspDefinitions.scala | 11 +- .../scala/bloop/bsp/BloopBspServices.scala | 32 +-- .../src/main/scala/bloop/bsp/BspServer.scala | 3 +- .../main/scala/bloop/data/LoadedProject.scala | 19 +- .../src/main/scala/bloop/data/Origin.scala | 7 +- .../src/main/scala/bloop/data/Project.scala | 52 ++-- .../scala/bloop/data/WorkspaceSettings.scala | 45 ++- .../src/main/scala/bloop/engine/Build.scala | 259 ++++++++++++------ .../main/scala/bloop/engine/BuildLoader.scala | 207 +++++++++----- .../main/scala/bloop/engine/Feedback.scala | 7 + .../main/scala/bloop/engine/Interpreter.scala | 15 +- .../src/main/scala/bloop/engine/State.scala | 20 +- .../bloop/engine/caches/ResultsCache.scala | 3 +- .../bloop/engine/caches/StateCache.scala | 69 +++-- .../test/scala/bloop/BuildLoaderSpec.scala | 146 ++++++---- frontend/src/test/scala/bloop/RunSpec.scala | 4 +- .../scala/bloop/bsp/BspMetalsClientSpec.scala | 97 +++---- .../scala/bloop/bsp/BspProtocolSpec.scala | 3 +- .../scala/bloop/testing/BloopHelpers.scala | 2 +- .../src/test/scala/bloop/util/TestUtil.scala | 29 +- .../gradle/ConfigGenerationSuite.scala | 85 +++--- shared/src/main/scala/bloop/io/Paths.scala | 2 +- 24 files changed, 704 insertions(+), 439 deletions(-) diff --git a/frontend/src/it/scala/bloop/CommunityBuild.scala b/frontend/src/it/scala/bloop/CommunityBuild.scala index 230de99961..64044fed36 100644 --- a/frontend/src/it/scala/bloop/CommunityBuild.scala +++ b/frontend/src/it/scala/bloop/CommunityBuild.scala @@ -7,7 +7,7 @@ import scala.concurrent.Await import scala.concurrent.duration.Duration import bloop.cli.{Commands, ExitStatus} import bloop.config.Config -import bloop.data.{Origin, Project} +import bloop.data.{Origin, Project, LoadedProject} import bloop.engine.{ Action, Build, @@ -88,8 +88,8 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) { def loadStateForBuild(configDirectory: AbsolutePath, logger: Logger): State = { assert(configDirectory.exists, "Does not exist: " + configDirectory) - val loadedBuild = BuildLoader.loadSynchronously(configDirectory, logger) - val build = Build(configDirectory, loadedBuild.workspaceSettings, loadedBuild.projects) + val loadedProjects = BuildLoader.loadSynchronously(configDirectory, logger) + val build = Build(configDirectory, loadedProjects) val state = State.forTests(build, compilerCache, logger) state.copy(results = ResultsCache.emptyForTests) } @@ -111,19 +111,19 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) { val logger = BloopLogger.default("community-build-logger") val initialState = loadStateForBuild(buildBaseDir.resolve(".bloop"), logger) val blacklistedProjects = readBlacklistFile(buildBaseDir.resolve("blacklist.buildpress")) - val allProjectsInBuild = - initialState.build.projects.filterNot(p => blacklistedProjects.contains(p.name)) + val allProjectsInBuild = initialState.build.loadedProjects + .filterNot(lp => blacklistedProjects.contains(lp.project.name)) val rootProjectName = "bloop-test-root" val dummyExistingBaseDir = buildBaseDir.resolve("project") val dummyClassesDir = dummyExistingBaseDir.resolve("target") - val origin = Origin(buildBaseDir, FileTime.fromMillis(0), scala.util.Random.nextInt()) + val origin = Origin(buildBaseDir, FileTime.fromMillis(0), 0L, scala.util.Random.nextInt()) val analysisOut = dummyClassesDir.resolve(Config.Project.analysisFileName(rootProjectName)) val rootProject = Project( name = rootProjectName, baseDirectory = dummyExistingBaseDir, - dependencies = allProjectsInBuild.map(_.name), - scalaInstance = allProjectsInBuild.head.scalaInstance, + dependencies = allProjectsInBuild.map(_.project.name), + scalaInstance = allProjectsInBuild.head.project.scalaInstance, rawClasspath = Nil, resources = Nil, compileSetup = Config.CompileSetup.empty, @@ -141,8 +141,8 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) { origin = origin ) - val newProjects = rootProject :: allProjectsInBuild - val state = initialState.copy(build = initialState.build.copy(projects = newProjects)) + val newLoaded = LoadedProject.RawProject(rootProject) :: allProjectsInBuild + val state = initialState.copy(build = initialState.build.copy(loadedProjects = newLoaded)) val allReachable = Dag.dfs(state.build.getDagFor(rootProject)) val reachable = allReachable.filter(_ != rootProject) val cleanAction = Run(Commands.Clean(reachable.map(_.name)), Exit(ExitStatus.Ok)) diff --git a/frontend/src/main/scala/bloop/Bloop.scala b/frontend/src/main/scala/bloop/Bloop.scala index 054764dbcc..da5dbb3b64 100644 --- a/frontend/src/main/scala/bloop/Bloop.scala +++ b/frontend/src/main/scala/bloop/Bloop.scala @@ -35,8 +35,8 @@ object Bloop extends CaseApp[CliOptions] { ) logger.warn("Please refer to our documentation for more information.") val client = ClientInfo.CliClientInfo("bloop-single-app", () => true) - val loadedBuild = BuildLoader.loadSynchronously(configDirectory, logger) - val build = Build(configDirectory, loadedBuild.workspaceSettings, loadedBuild.projects) + val loadedProjects = BuildLoader.loadSynchronously(configDirectory, logger) + val build = Build(configDirectory, loadedProjects) val state = State(build, client, NoPool, options.common, logger) run(state, options) } @@ -71,7 +71,7 @@ object Bloop extends CaseApp[CliOptions] { run(waitForState(action, Interpreter.execute(action, Task.now(state))), options) case Array("clean") => - val allProjects = state.build.projects.map(_.name) + val allProjects = state.build.loadedProjects.map(_.project.name) val action = Run(Commands.Clean(allProjects), Exit(ExitStatus.Ok)) run(waitForState(action, Interpreter.execute(action, Task.now(state))), options) diff --git a/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala b/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala index 59a099a3c1..c693106d1a 100644 --- a/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala +++ b/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala @@ -1,24 +1,23 @@ package bloop.bsp -import io.circe._ -import io.circe.derivation._ import ch.epfl.scala.bsp.Uri object BloopBspDefinitions { final case class BloopExtraBuildParams( clientClassesRootDir: Option[Uri], semanticdbVersion: Option[String], - supportedScalaVersions: List[String], - reapplySettings: Boolean + supportedScalaVersions: List[String] ) object BloopExtraBuildParams { val empty = BloopExtraBuildParams( clientClassesRootDir = None, semanticdbVersion = None, - supportedScalaVersions = Nil, - reapplySettings = false + supportedScalaVersions = Nil ) + + import io.circe.{RootEncoder, Decoder} + import io.circe.derivation._ val encoder: RootEncoder[BloopExtraBuildParams] = deriveEncoder val decoder: Decoder[BloopExtraBuildParams] = deriveDecoder } diff --git a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala index 9eae3c246d..78cf79c628 100644 --- a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala +++ b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala @@ -112,22 +112,13 @@ final class BloopBspServices( private def reloadState( config: AbsolutePath, clientInfo: ClientInfo, - workspaceSettings: Option[WorkspaceSettings] = None, - reapplySettings: Boolean = false + clientSettings: Option[WorkspaceSettings] = None ): Task[State] = { val pool = currentState.pool val defaultOpts = currentState.commonOptions bspLogger.debug(s"Reloading bsp state for ${config.syntax}") State - .loadActiveStateFor( - config, - clientInfo, - pool, - defaultOpts, - bspLogger, - workspaceSettings, - reapplySettings - ) + .loadActiveStateFor(config, clientInfo, pool, defaultOpts, bspLogger, clientSettings) .map { state0 => /* Create a new state that has the previously compiled results in this BSP * client as the last compiled result available for a project. This is required @@ -203,13 +194,14 @@ final class BloopBspServices( () => isClientConnected.get ) - /* Metals specific settings that are used to store the - * SemanticDB version that will later be applied to all - * projects in the workspace. If the client is Metals but - * the version is not specified we use `latest.release` + /** + * A Metals BSP client enables a special transformation of a build via the + * workspace settings. These workspace settings contains all of the + * information required by bloop to enable Metals-specific settings in + * every project of a build so that users from different build tools don't + * need to manually enable these in their build. */ - val reapplySettings = extraBuildParams.map(_.reapplySettings).getOrElse(false) - val metalsSettings = + val metalsSettings = { if (!params.displayName.contains("Metals")) { None } else { @@ -222,8 +214,9 @@ final class BloopBspServices( ) } } + } - reloadState(configDir, client, metalsSettings, reapplySettings).map { state => + reloadState(configDir, client, metalsSettings).map { state => callSiteState.logger.info(s"request received: build/initialize") clientInfo.success(client) connectedBspClients.put(client, configDir) @@ -640,8 +633,9 @@ final class BloopBspServices( Task.now((state, Right(bsp.WorkspaceBuildTargetsResult(Nil)))) else { val build = state.build + val projects = build.loadedProjects.map(_.project) val targets = bsp.WorkspaceBuildTargetsResult( - build.projects.map { p => + projects.map { p => val id = toBuildTargetId(p) val tag = { if (p.name.endsWith("-test") && build.getProjectFor(s"${p.name}-test").isEmpty) diff --git a/frontend/src/main/scala/bloop/bsp/BspServer.scala b/frontend/src/main/scala/bloop/bsp/BspServer.scala index fb2f1608c2..959e10c7b4 100644 --- a/frontend/src/main/scala/bloop/bsp/BspServer.scala +++ b/frontend/src/main/scala/bloop/bsp/BspServer.scala @@ -305,8 +305,9 @@ object BspServer { } } finally { // Guarantee that we always schedule the external classes directories deletion - val deleteExternalDirsTasks = latestState.build.projects.map { project => + val deleteExternalDirsTasks = latestState.build.loadedProjects.map { loadedProject => import bloop.io.Paths + val project = loadedProject.project try { val externalClientClassesDir = latestState.client.getUniqueClassesDirFor(project) val skipDirectoryManagement = diff --git a/frontend/src/main/scala/bloop/data/LoadedProject.scala b/frontend/src/main/scala/bloop/data/LoadedProject.scala index 59922d3f7e..e2edfb21cb 100644 --- a/frontend/src/main/scala/bloop/data/LoadedProject.scala +++ b/frontend/src/main/scala/bloop/data/LoadedProject.scala @@ -1,15 +1,30 @@ package bloop.data -sealed trait LoadedProject +sealed trait LoadedProject { + def project: Project +} object LoadedProject { + + /** + * Represents a project that doesn't have any configuration applied to it. + * For example, if the Metals client asks to enable the SemanticDB plugin for + * all Scala projects, Bloop will skip configuring Java projects and wrap + * them in raw projects. + */ final case class RawProject( project: Project ) extends LoadedProject + /** + * Represents a project that has been transformed in-memory. It also contains + * the original project before the transformation took place and the settings + * that were used for the transformation so that the build logic can detect + * whether this configured project deserves to be reconfigured or not. + */ final case class ConfiguredProject( project: Project, - originalProject: Project, + original: Project, settings: WorkspaceSettings ) extends LoadedProject } diff --git a/frontend/src/main/scala/bloop/data/Origin.scala b/frontend/src/main/scala/bloop/data/Origin.scala index 51f6efa9a6..e3cd73a3b3 100644 --- a/frontend/src/main/scala/bloop/data/Origin.scala +++ b/frontend/src/main/scala/bloop/data/Origin.scala @@ -6,12 +6,13 @@ import bloop.io.AbsolutePath import bloop.io.Paths.AttributedPath import bloop.util.CacheHashCode -case class Origin(path: AbsolutePath, lastModifiedtime: FileTime, hash: Int) extends CacheHashCode { - def toAttributedPath: AttributedPath = AttributedPath(path, lastModifiedtime) +case class Origin(path: AbsolutePath, lastModifiedtime: FileTime, size: Long, hash: Int) + extends CacheHashCode { + def toAttributedPath: AttributedPath = AttributedPath(path, lastModifiedtime, size) } object Origin { def apply(path: AttributedPath, hash: Int): Origin = { - Origin(path.path, path.lastModifiedTime, hash) + Origin(path.path, path.lastModifiedTime, path.size, hash) } } diff --git a/frontend/src/main/scala/bloop/data/Project.scala b/frontend/src/main/scala/bloop/data/Project.scala index fa8ff1d16c..c10d68d40e 100644 --- a/frontend/src/main/scala/bloop/data/Project.scala +++ b/frontend/src/main/scala/bloop/data/Project.scala @@ -201,6 +201,10 @@ object Project { } } + def pprint(projects: Traversable[Project]): String = { + projects.map(p => s"'${p.name}'").mkString(", ") + } + /** * Enable any Metals-specific setting in a project by applying an in-memory * project transformation. A setting is Metals-specific if it's required for @@ -220,14 +224,12 @@ object Project { */ def enableMetalsSettings( project: Project, - settings: WorkspaceSettings, + semanticDBPlugin: Option[AbsolutePath], + workspaceDir: AbsolutePath, logger: Logger - ): Either[Project, Project] = { - val workspaceDir = WorkspaceSettings.detectWorkspaceDirectory(project, settings) + ): Project = { def enableSemanticDB(options: List[String], pluginPath: AbsolutePath): List[String] = { - val hasSemanticDB = - options.exists(opt => opt.contains("-Xplugin") && opt.contains("semanticdb-scalac")) - + val hasSemanticDB = hasSemanticDBEnabledInCompilerOptions(options) if (hasSemanticDB) options else { // TODO: Handle user-configured `targetroot`s inside Bloop's compilation @@ -249,33 +251,19 @@ object Project { if (hasYrangepos) options else options :+ "-Yrangepos" } - project.scalaInstance match { - case None => Left(project) - case Some(instance) => - val projectWithRangePositions = - project.copy(scalacOptions = enableRangePositions(project.scalacOptions)) - + val projectWithRangePositions = + project.copy(scalacOptions = enableRangePositions(project.scalacOptions)) + semanticDBPlugin match { + case None => projectWithRangePositions + case Some(pluginPath) => // Recognize 2.12.8-abdcddd as supported if 2.12.8 exists in supported versions - val isUnsupportedVersion = - !settings.supportedScalaVersions.exists(instance.version.startsWith(_)) - if (isUnsupportedVersion) { - logger.debug( - s"Skipping configuration of SemanticDB for '${project.name}': unsupported Scala v${instance.version}" - )(DebugFilter.All) - Right(projectWithRangePositions) - } else { - SemanticDBCache.fetchPlugin(instance.version, settings.semanticDBVersion, logger) match { - case Right(pluginPath) => - val options = projectWithRangePositions.scalacOptions - val optionsWithSemanticDB = enableSemanticDB(options, pluginPath) - Right(projectWithRangePositions.copy(scalacOptions = optionsWithSemanticDB)) - case Left(error) => - logger.displayWarningToUser( - s"Skipping configuration of SemanticDB for '${project.name}': $error" - ) - Right(projectWithRangePositions) - } - } + val options = projectWithRangePositions.scalacOptions + val optionsWithSemanticDB = enableSemanticDB(options, pluginPath) + projectWithRangePositions.copy(scalacOptions = optionsWithSemanticDB) } } + + def hasSemanticDBEnabledInCompilerOptions(options: List[String]): Boolean = { + options.exists(opt => opt.contains("-Xplugin") && opt.contains("semanticdb-scalac")) + } } diff --git a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala index 0abfae032d..088cd43a1f 100644 --- a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala +++ b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala @@ -23,6 +23,27 @@ import io.circe.Decoder import io.circe.ObjectEncoder import io.circe.JsonObject +/** + * Defines the settings of a given workspace. A workspace is a URI that has N + * configuration files associated with it. Typically the workspace is the root + * directory where all of the projects in the configuration files are defined. + * + * Workspace settings have a special status in bloop as they change the build + * load semantics. Only bloop's build server has permission to write workspace + * settings and the existence of the workspace settings file is an internal + * detail. + * + * Workspace settings can be written to disk when, for example, Metals asks to + * import a build and Bloop needs to cache the fact that a build needs to + * enable Metals specific settings based on some inputs from the BSP clients. + * These keys are usually the fields of the workspace settings. + * + * @param semanticDBVersion The version that should be used to enable the + * Semanticdb compiler plugin in a project. + * @param semanticDBScalaVersions The sequence of Scala versions for which the + * SemanticDB plugin can be resolved for. Important to know for which projects + * we should skip the resolution of the plugin. + */ case class WorkspaceSettings( semanticDBVersion: String, supportedScalaVersions: List[String] @@ -30,8 +51,9 @@ case class WorkspaceSettings( object WorkspaceSettings { + /** Represents the supported changes in the workspace. */ sealed trait DetectedChange - final case class SemanticdbVersionChange(newVersion: String) extends DetectedChange + final case object SemanticDBVersionChange extends DetectedChange /** File name to store Metals specific settings*/ private[bloop] val settingsFileName = RelativePath("bloop.settings.json") @@ -54,14 +76,17 @@ object WorkspaceSettings { } } - def writeToFile(configDir: AbsolutePath, settings: WorkspaceSettings): Either[Throwable, Path] = { + def writeToFile( + configDir: AbsolutePath, + settings: WorkspaceSettings, + logger: Logger + ): Either[Throwable, Path] = { Try { + val settingsFile = configDir.resolve(settingsFileName) + logger.debug(s"Writing workspace settings to $settingsFile")(DebugFilter.All) val jsonObject = settingsEncoder(settings) val output = Printer.spaces4.copy(dropNullValues = true).pretty(jsonObject) - Files.write( - configDir.resolve(settingsFileName).underlying, - output.getBytes(StandardCharsets.UTF_8) - ) + Files.write(settingsFile.underlying, output.getBytes(StandardCharsets.UTF_8)) }.toEither } @@ -73,7 +98,7 @@ object WorkspaceSettings { } /** - * Detects the workspace directory of a project. + * Detects the workspace directory from the config dir. * * Bloop doesn't have the notion of workspace directory yet so this is just an * approximation. We assume that the parent of `.bloop` is the workspace. This @@ -83,9 +108,9 @@ object WorkspaceSettings { * to introduce a new field to the bloop configuration file so that we can map * a project with a workspace irrevocably. */ - def detectWorkspaceDirectory(project: Project, settings: WorkspaceSettings): AbsolutePath = { - val configFile = project.origin.path - val configDir = configFile.getParent + def detectWorkspaceDirectory( + configDir: AbsolutePath + ): AbsolutePath = { configDir.getParent } } diff --git a/frontend/src/main/scala/bloop/engine/Build.scala b/frontend/src/main/scala/bloop/engine/Build.scala index 7f0f5b623a..14bc682c76 100644 --- a/frontend/src/main/scala/bloop/engine/Build.scala +++ b/frontend/src/main/scala/bloop/engine/Build.scala @@ -1,150 +1,237 @@ package bloop.engine -import bloop.data.{Origin, Project} +import bloop.data.{Origin, Project, LoadedProject, WorkspaceSettings} import bloop.engine.Dag.DagResult import bloop.io.AbsolutePath import bloop.logging.Logger import bloop.util.CacheHashCode import bloop.io.ByteHasher -import monix.eval.Task -import bloop.data.WorkspaceSettings import bloop.logging.DebugFilter +import scala.collection.mutable + +import monix.eval.Task + final case class Build private ( origin: AbsolutePath, - settings: Option[WorkspaceSettings], - projects: List[Project] + loadedProjects: List[LoadedProject] ) extends CacheHashCode { - private val stringToProjects: Map[String, Project] = projects.map(p => p.name -> p).toMap + private val stringToProjects: Map[String, Project] = + loadedProjects.map(lp => lp.project.name -> lp.project).toMap private[bloop] val DagResult(dags, missingDeps, traces) = Dag.fromMap(stringToProjects) - def getProjectFor(name: String): Option[Project] = stringToProjects.get(name) + def getProjectFor(name: String): Option[Project] = + stringToProjects.get(name) def getDagFor(project: Project): Dag[Project] = Dag.dagFor(dags, project).getOrElse(sys.error(s"Project $project does not have a DAG!")) - def hasMissingDependencies(project: Project): Option[List[String]] = missingDeps.get(project) + def hasMissingDependencies(project: Project): Option[List[String]] = + missingDeps.get(project) /** * Detect changes in the build definition since the last time it was loaded * and tell the compiler which action should be applied to update the build. * - * There are two major kinds of changes: - * - * 1. A change in a configuration file backing up the metadata of a project. - * 2. A change in a workspace setting that requires transforming a project - * after it has been loaded from the configuration file. + * The logic to incrementally update the build is complex due to the need of + * transforming projects in-memory after they have been loaded. These + * transformations depend on values of the workspace settings and + * `checkForChange` defines many of the semantics of these settings and what + * should be the implicitations that a change has in the whole build. * * @param newSettings The new settings that should be applied to detect changes. * These settings are passed by certain clients such as Metals * to apply in-memory transformations on projects. They can * differ from the settings written to disk. * @param logger A logger that receives errors, if any. - * @param retryFailedProjects * @return The status of the directory from which the build was loaded. */ def checkForChange( newSettings: Option[WorkspaceSettings], - logger: Logger, - retryFailedProjects: Boolean + logger: Logger ): Task[Build.ReloadAction] = { - val files = projects.iterator.map(p => p.origin.toAttributedPath).toSet + val oldFilesMap = loadedProjects.iterator.map { lp => + val origin = lp.project.origin + origin.path -> origin.toAttributedPath + }.toMap + val newFiles = BuildLoader.readConfigurationFilesInBase(origin, logger).toSet + val newToAttributed = newFiles.iterator.map(ap => ap.path -> ap).toMap - /* - def settingsForProjectReload(workspaceSettings: WorkspaceSettings) = - (workspaceSettings.semanticDBVersion, workspaceSettings.supportedScalaVersions) - val changedSettings = reapplySettings || ( - (incomingSettings.nonEmpty && - settings.map(settingsForProjectReload) != incomingSettings.map(settingsForProjectReload)) - ) - - if (reapplySettings) { - logger.debug(s"Incoming BSP workspace settings require reloading all projects")( - DebugFilter.All - ) - } - */ - - // This is the fast path to short circuit quickly if they are the same - if (newFiles == files && !changedSettings) { - Task.now(Build.ReturnPreviousState) - } else { - val filesToAttributed = projects.iterator.map(p => p.origin.path -> p).toMap - // There has been at least either one addition, one removal or one change in a file time - val newOrModifiedConfigurations = newFiles.map { f => - Task { - val configuration = { - val bytes = f.path.readAllBytes - val hash = ByteHasher.hashBytes(bytes) - Build.ReadConfiguration(Origin(f, hash), bytes) - } + val currentSettings = WorkspaceSettings.readFromFile(origin, logger) + val settingsForReload = pickAndPersistSettingsForReload(currentSettings, newSettings, logger) + val changedSettings = currentSettings != settingsForReload + val filesToProjects = loadedProjects.iterator.map(lp => lp.project.origin.path -> lp).toMap + + val detectedChanges = newFiles.map { f => + def hasSameMetadata: Boolean = { + oldFilesMap.get(f.path) match { + case Some(oldFile) if oldFile == f => true + case _ => false + } + } - filesToAttributed.get(f.path) match { - case _ if changedSettings => List(configuration) - case Some(p) if p.origin.hash == configuration.origin.hash => Nil - case _ => List(configuration) - } + def readConfiguration = { + val bytes = f.path.readAllBytes + val hash = ByteHasher.hashBytes(bytes) + Build.ReadConfiguration(Origin(f, hash), bytes) + } + + def invalidateProject( + project: Project, + externalChanges: Option[List[WorkspaceSettings.DetectedChange]] + ) = { + val changes = externalChanges.getOrElse { + // Add all changes here so that projects with no changes get fully invalidated + List(WorkspaceSettings.SemanticDBVersionChange) } + List(Build.InvalidatedInMemoryProject(project, changes)) } - // Recompute all the build -- this step could be incremental but its cost is negligible - Task.gatherUnordered(newOrModifiedConfigurations).map(_.flatten).map { newOrModified => - val newToAttributed = newFiles.iterator.map(ap => ap.path -> ap).toMap - val deleted = files.toList.collect { case f if !newToAttributed.contains(f.path) => f.path } - (newOrModified, deleted, changedSettings) match { - case (Nil, Nil, false) => Build.ReturnPreviousState - case _ => Build.UpdateState(newOrModified, deleted, incomingSettings) + Task { + filesToProjects.get(f.path) match { + case None => List(Build.NewProject(readConfiguration)) + + case Some(LoadedProject.RawProject(project)) => + def invalidateIfSettings = { + // Attempt to configure project when settings exist + if (newSettings.isEmpty || !changedSettings) Nil + else invalidateProject(project, None) + } + + if (hasSameMetadata) invalidateIfSettings + else { + val configuration = readConfiguration + val hasSameHash = project.origin.hash == configuration.origin.hash + if (!hasSameHash) List(Build.ModifiedProject(configuration)) + else invalidateIfSettings + } + + case Some(LoadedProject.ConfiguredProject(project, original, settings)) => + findUpdateSettingsAction(Some(settings), settingsForReload) match { + case Build.AvoidReload(_) => + val options = project.scalacOptions + val reattemptConfiguration = newSettings.nonEmpty && changedSettings && { + Project.hasSemanticDBEnabledInCompilerOptions(project.scalacOptions) + } + + if (reattemptConfiguration) { + invalidateProject(project, None) + } else if (hasSameMetadata) { + Nil + } else { + val configuration = readConfiguration + val hasSameHash = original.origin.hash == configuration.origin.hash + if (hasSameHash) Nil + else List(Build.ModifiedProject(configuration)) + } + + case f: Build.ForceReload => + if (hasSameMetadata) { + invalidateProject(original, Some(f.changes)) + } else { + val configuration = readConfiguration + val hasSameHash = original.origin.hash == configuration.origin.hash + if (hasSameHash) invalidateProject(original, Some(f.changes)) + else List(Build.ModifiedProject(configuration)) + } + } } } } - } - sealed trait UpdateSettingsAction + Task.gatherUnordered(detectedChanges).map(_.flatten).map { changes => + val deleted = oldFilesMap.values.collect { + case f if !newToAttributed.contains(f.path) => f.path + } - final case class ForceReload( - settings: WorkspaceSettings, - changes: List[WorkspaceSettings.DetectedChange] - ) extends UpdateSettingsAction + (changes, deleted) match { + case (Nil, Nil) => Build.ReturnPreviousState + case (changes, deleted) => + val added = new mutable.ListBuffer[Build.ReadConfiguration]() + val modified = new mutable.ListBuffer[Build.ReadConfiguration]() + val inMemoryChanged = new mutable.ListBuffer[Build.InvalidatedInMemoryProject]() + changes.foreach { + case Build.NewProject(project) => added.+=(project) + case Build.ModifiedProject(project) => modified.+=(project) + case change: Build.InvalidatedInMemoryProject => inMemoryChanged.+=(change) + } - final case class WriteSettings( - settings: WorkspaceSettings - ) extends UpdateSettingsAction + Build.UpdateState( + added.toList, + modified.toList, + deleted.toList, + inMemoryChanged.toList, + settingsForReload, + settingsForReload.nonEmpty && changedSettings + ) + } + } + } + + def pickAndPersistSettingsForReload( + currentSettings: Option[WorkspaceSettings], + newSettings: Option[WorkspaceSettings], + logger: Logger + ): Option[WorkspaceSettings] = { + findUpdateSettingsAction(currentSettings, newSettings) match { + case Build.AvoidReload(settings) => settings + case Build.ForceReload(settings, _) => Some(settings) + } + } /** - * Picks the settings that have to be used to reload the build. - * @return Either the current settings when nothing has changed or the new - * settingsb + * Produces the action to update the build based on changes in the settings. + * + * The order in which settings are compared matters because if current and + * new settings exist and don't have any conflict regarding the semantics + * of the build process, the new settings are returned so that they are + * mapped with the projects that have changed and have been reloaded. */ def findUpdateSettingsAction( - newSettings: Option[WorkspaceSettings], - logger: Logger - ): Either[Option[WorkspaceSettings], UpdateSettingsAction] = { - val currentSettings = WorkspaceSettings.readFromFile(origin, logger) + currentSettings: Option[WorkspaceSettings], + newSettings: Option[WorkspaceSettings] + ): Build.UpdateSettingsAction = { (currentSettings, newSettings) match { case (Some(currentSettings), Some(newSettings)) if currentSettings.semanticDBVersion != newSettings.semanticDBVersion => - // Write the only supported change affecting build load semantics so far - val changes = List( - WorkspaceSettings.SemanticdbVersionChange(newSettings.semanticDBVersion) - ) - Right(ForceReload(newSettings, changes)) - case (Some(_), Some(newSettings)) => Left(Some(newSettings)) - case (None, Some(newSettings)) => Right(WriteSettings(newSettings)) - case (Some(currentSettings), None) => Left(Some(currentSettings)) - case (None, None) => Left(None) + Build.ForceReload(newSettings, List(WorkspaceSettings.SemanticDBVersionChange)) + case (Some(_), Some(newSettings)) => Build.AvoidReload(Some(newSettings)) + case (None, Some(newSettings)) => Build.AvoidReload(Some(newSettings)) + case (Some(currentSettings), None) => Build.AvoidReload(Some(currentSettings)) + case (None, None) => Build.AvoidReload(None) } } } object Build { sealed trait ReloadAction - case object ReturnPreviousState extends ReloadAction + final case object ReturnPreviousState extends ReloadAction case class UpdateState( - createdOrModified: List[ReadConfiguration], + created: List[ReadConfiguration], + modified: List[ReadConfiguration], deleted: List[AbsolutePath], - changedSetings: Option[WorkspaceSettings] - ) extends ReloadAction + invalidated: List[InvalidatedInMemoryProject], + settingsForReload: Option[WorkspaceSettings], + writeSettingsToDisk: Boolean + ) extends ReloadAction { + def createdOrModified = created ++ modified + } + + sealed trait UpdateSettingsAction + final case class AvoidReload(settings: Option[WorkspaceSettings]) extends UpdateSettingsAction + final case class ForceReload( + settings: WorkspaceSettings, + changes: List[WorkspaceSettings.DetectedChange] + ) extends UpdateSettingsAction + + sealed trait ProjectChange + final case class NewProject(configuration: Build.ReadConfiguration) extends ProjectChange + final case class ModifiedProject(configuration: Build.ReadConfiguration) extends ProjectChange + final case class InvalidatedInMemoryProject( + project: Project, + changes: List[WorkspaceSettings.DetectedChange] + ) extends ProjectChange /** A configuration file is a combination of an absolute path and a file time. */ case class ReadConfiguration(origin: Origin, bytes: Array[Byte]) extends CacheHashCode diff --git a/frontend/src/main/scala/bloop/engine/BuildLoader.scala b/frontend/src/main/scala/bloop/engine/BuildLoader.scala index 5332008408..79e44c639c 100644 --- a/frontend/src/main/scala/bloop/engine/BuildLoader.scala +++ b/frontend/src/main/scala/bloop/engine/BuildLoader.scala @@ -8,8 +8,10 @@ import bloop.io.ByteHasher import bloop.data.WorkspaceSettings import bloop.data.PartialLoadedBuild import bloop.data.LoadedProject +import bloop.engine.caches.SemanticDBCache import monix.eval.Task +import scala.collection.mutable object BuildLoader { @@ -26,64 +28,129 @@ object BuildLoader { * @return A map associating each tracked file with its last modification time. */ def readConfigurationFilesInBase(base: AbsolutePath, logger: Logger): List[AttributedPath] = { + val workspaceFileName = WorkspaceSettings.settingsFileName.toFile.getName() bloop.io.Paths .attributedPathFilesUnder(base, JsonFilePattern, logger, 1) - .filterNot(_.path.toFile.getName() == WorkspaceSettings.settingsFileName) + .filterNot(_.path.toFile.getName() == workspaceFileName) } /** - * Loads only the projects passed as arguments. + * Loads the build incrementally based on the inputs. * * @param configRoot The base directory from which to load the projects. + * @param configs The read configurations for added/modified projects. + * @param inMemoryChanged The projects that require a re-transformation based on settings. + * @param settingsForLoad The settings to be used to reload the build. * @param logger The logger that collects messages about project loading. * @return The list of loaded projects. */ - def loadBuildFromConfigurationFiles( + def loadBuildIncrementally( configDir: AbsolutePath, - configFiles: List[Build.ReadConfiguration], - newSettings: Option[WorkspaceSettings], + configs: List[Build.ReadConfiguration], + inMemoryChanged: List[Build.InvalidatedInMemoryProject], + settingsForLoad: Option[WorkspaceSettings], logger: Logger - ): Task[PartialLoadedBuild] = { - val workspaceSettings = Task(updateWorkspaceSettings(configDir, logger, newSettings)) - logger.debug(s"Loading ${configFiles.length} projects from '${configDir.syntax}'...")( - DebugFilter.All - ) + ): Task[List[LoadedProject]] = { + val incrementalLoadTask = Task { + val loadMsg = s"Loading ${configs.length} projects from '${configDir.syntax}'..." + logger.debug(loadMsg)(DebugFilter.All) + val rawProjects = configs.map(f => Task(loadProject(f.bytes, f.origin, logger))) + val groupTasks = rawProjects.grouped(10).map(group => Task.gatherUnordered(group)).toList + val newOrModifiedRawProjects = Task.sequence(groupTasks).map(fp => fp.flatten) - val loadSettingsAndBuild = workspaceSettings.flatMap { settings => - val all = configFiles.map(f => Task(loadProject(f.bytes, f.origin, logger, settings))) - val groupTasks = all.grouped(10).map(group => Task.gatherUnordered(group)).toList - Task.sequence(groupTasks).map(fp => PartialLoadedBuild(fp.flatten)) + newOrModifiedRawProjects.flatMap { projects => + val projectsRequiringMetalsTransformation = projects ++ inMemoryChanged.collect { + case Build.InvalidatedInMemoryProject(project, changes) + if changes.contains(WorkspaceSettings.SemanticDBVersionChange) => + project + } + + settingsForLoad match { + case None => + Task.now(projectsRequiringMetalsTransformation.map(LoadedProject.RawProject(_))) + case Some(settings) => + resolveSemanticDBForProjects( + projectsRequiringMetalsTransformation, + configDir, + settings.semanticDBVersion, + settings.supportedScalaVersions, + logger + ).map { transformedProjects => + transformedProjects.map { + case (project, None) => LoadedProject.RawProject(project) + case (project, Some(original)) => + LoadedProject.ConfiguredProject(project, original, settings) + } + } + } + } } - loadSettingsAndBuild.executeOn(ExecutionContext.ioScheduler) + incrementalLoadTask.flatten.executeOn(ExecutionContext.ioScheduler) } /** - * Load all the projects from `configDir` in a parallel, lazy fashion via monix Task. + * Resolves the semanticdb plugin for every project with a different Scala + * version and then enables the Metals settings on those projects that have + * changed in this incremental build load. * - * @param configDir The base directory from which to load the projects. - * @param newSettings The settings that we should use to load this build. - * @param logger The logger that collects messages about project loading. - * @return The list of loaded projects. + * Some of the logic of this method has a duplicate version in + * `loadSynchronously` that doesn't use `Task` to resolve SemanticDB plugins + * in parallel and apply the Metals projects. Any change here **must** also + * be propagated there. */ - def load( + private def resolveSemanticDBForProjects( + rawProjects: List[Project], configDir: AbsolutePath, - newSettings: Option[WorkspaceSettings], + semanticDBVersion: String, + supportedScalaVersions: List[String], logger: Logger - ): Task[PartialLoadedBuild] = { - val configFiles = readConfigurationFilesInBase(configDir, logger).map { ap => - Task { - val bytes = ap.path.readAllBytes - val hash = ByteHasher.hashBytes(bytes) - Build.ReadConfiguration(Origin(ap, hash), bytes) + ): Task[List[(Project, Option[Project])]] = { + val projectsWithNoScalaConfig = new mutable.ListBuffer[Project]() + val groupedProjectsPerScalaVersion = new mutable.HashMap[String, List[Project]]() + rawProjects.foreach { project => + project.scalaInstance match { + case None => projectsWithNoScalaConfig.+=(project) + case Some(instance) => + val scalaVersion = instance.version + groupedProjectsPerScalaVersion.get(scalaVersion) match { + case Some(projects) => + groupedProjectsPerScalaVersion.put(scalaVersion, project :: projects) + case None => groupedProjectsPerScalaVersion.put(scalaVersion, project :: Nil) + } } } - Task - .gatherUnordered(configFiles) - .flatMap { fs => - loadBuildFromConfigurationFiles(configDir, fs, newSettings, logger) - } + val workspace = WorkspaceSettings.detectWorkspaceDirectory(configDir) + val enableMetalsInProjectsTask = groupedProjectsPerScalaVersion.toList.map { + case (scalaVersion, projects) => + def enableMetalsTask(plugin: Option[AbsolutePath]) = { + Task( + projects.map(p => Project.enableMetalsSettings(p, plugin, workspace, logger) -> Some(p)) + ) + } + + // Recognize 2.12.8-abdcddd as supported if 2.12.8 exists in supported versions + val isUnsupportedVersion = !supportedScalaVersions.exists(scalaVersion.startsWith(_)) + if (isUnsupportedVersion) { + logger.debug(Feedback.skippedUnsupportedScalaMetals(scalaVersion))(DebugFilter.All) + enableMetalsTask(None) + } else { + SemanticDBCache.fetchPlugin(scalaVersion, semanticDBVersion, logger) match { + case Right(path) => + logger.debug(Feedback.configuredMetalsProjects(projects))(DebugFilter.All) + enableMetalsTask(Some(path)) + case Left(cause) => + logger.displayWarningToUser(Feedback.failedMetalsConfiguration(scalaVersion, cause)) + enableMetalsTask(None) + } + } + } + + Task.gatherUnordered(enableMetalsInProjectsTask).map { pps => + // Add projects with Metals settings enabled + projects with no scala config at all + pps.flatten ++ projectsWithNoScalaConfig.toList.map(_ -> None) + } } /** @@ -100,7 +167,7 @@ object BuildLoader { def loadSynchronously( configDir: AbsolutePath, logger: Logger - ): PartialLoadedBuild = { + ): List[LoadedProject] = { val settings = WorkspaceSettings.readFromFile(configDir, logger) val configFiles = readConfigurationFilesInBase(configDir, logger).map { ap => val bytes = ap.path.readAllBytes @@ -112,41 +179,49 @@ object BuildLoader { DebugFilter.All ) - PartialLoadedBuild(configFiles.map(f => loadProject(f.bytes, f.origin, logger, settings))) - } + configFiles.map { f => + val project = loadProject(f.bytes, f.origin, logger) + settings match { + case None => LoadedProject.RawProject(project) + case Some(settings) => + project.scalaInstance match { + case None => LoadedProject.RawProject(project) + case Some(instance) => + val workspace = WorkspaceSettings.detectWorkspaceDirectory(configDir) + def enableMetals(plugin: Option[AbsolutePath]) = { + LoadedProject.ConfiguredProject( + Project.enableMetalsSettings(project, plugin, workspace, logger), + project, + settings + ) + } - private def loadProject( - bytes: Array[Byte], - origin: Origin, - logger: Logger, - settings: Option[WorkspaceSettings] - ): LoadedProject = { - val project = Project.fromBytesAndOrigin(bytes, origin, logger) - settings match { - case None => LoadedProject.RawProject(project) - case Some(settings) => - Project.enableMetalsSettings(project, settings, logger) match { - case Left(project) => LoadedProject.RawProject(project) - case Right(transformed) => LoadedProject.ConfiguredProject(transformed, project, settings) - } + val scalaVersion = instance.version + import settings.{supportedScalaVersions, semanticDBVersion} + + // Recognize 2.12.8-abdcddd as supported if 2.12.8 exists in supported versions + val isUnsupportedVersion = !supportedScalaVersions.exists(scalaVersion.startsWith(_)) + if (isUnsupportedVersion) { + logger.debug(Feedback.skippedUnsupportedScalaMetals(scalaVersion))(DebugFilter.All) + enableMetals(None) + } else { + SemanticDBCache.fetchPlugin(scalaVersion, semanticDBVersion, logger) match { + case Right(path) => + logger.debug(Feedback.configuredMetalsProjects(List(project)))(DebugFilter.All) + enableMetals(Some(path)) + case Left(cause) => + logger.displayWarningToUser( + Feedback.failedMetalsConfiguration(scalaVersion, cause) + ) + enableMetals(None) + } + } + } + } } } - private def updateWorkspaceSettings( - configDir: AbsolutePath, - logger: Logger, - incomingSettings: Option[WorkspaceSettings] - ): Option[WorkspaceSettings] = { - val currentSettings = WorkspaceSettings.readFromFile(configDir, logger) - incomingSettings match { - case Some(newSettings) - if currentSettings.isEmpty || currentSettings.exists(_ != newSettings) => - WorkspaceSettings.writeToFile(configDir, newSettings).left.foreach { t => - logger.debug(s"Unexpected failure when writing workspace settings: $t")(DebugFilter.All) - logger.trace(t) - } - Some(newSettings) - case _ => currentSettings - } + private def loadProject(bytes: Array[Byte], origin: Origin, logger: Logger): Project = { + Project.fromBytesAndOrigin(bytes, origin, logger) } } diff --git a/frontend/src/main/scala/bloop/engine/Feedback.scala b/frontend/src/main/scala/bloop/engine/Feedback.scala index b472991533..76f7016dbd 100644 --- a/frontend/src/main/scala/bloop/engine/Feedback.scala +++ b/frontend/src/main/scala/bloop/engine/Feedback.scala @@ -93,6 +93,13 @@ object Feedback { def unknownHostName(host: String): String = s"Host name '$host' could not be either parsed or resolved" + def skippedUnsupportedScalaMetals(scalaVersion: String): String = + s"Skipped configuration of SemanticDB in unsupported $scalaVersion projects" + def configuredMetalsProjects(projects: Traversable[Project]): String = + s"Configured SemanticDB in projects ${Project.pprint(projects)}" + def failedMetalsConfiguration(version: String, cause: String): String = + s"Stopped configuration of SemanticDB in Scala $version projects: $cause" + implicit class XMessageString(msg: String) { def suggest(suggestion: String): String = s"$msg\n$suggestion" } diff --git a/frontend/src/main/scala/bloop/engine/Interpreter.scala b/frontend/src/main/scala/bloop/engine/Interpreter.scala index c87989d6e9..302918eb13 100644 --- a/frontend/src/main/scala/bloop/engine/Interpreter.scala +++ b/frontend/src/main/scala/bloop/engine/Interpreter.scala @@ -151,7 +151,10 @@ object Interpreter { // Make new state cleaned of all compilation products if compilation is not incremental val state: Task[State] = { if (cmd.incremental) Task.now(state0) - else Tasks.clean(state0, state0.build.projects, true) + else { + val projects = state0.build.loadedProjects.map(_.project) + Tasks.clean(state0, projects, true) + } } val compileTask = state.flatMap { state => @@ -195,7 +198,7 @@ object Interpreter { } else { val configDirectory = state.build.origin.syntax logger.debug(s"Projects loaded from '$configDirectory':")(DebugFilter.All) - state.build.projects.map(_.name).sorted.foreach(logger.info) + state.build.loadedProjects.map(_.project.name).sorted.foreach(logger.info) } state.mergeStatus(ExitStatus.Ok) @@ -397,7 +400,8 @@ object Interpreter { case Mode.Projects => Task { for { - project <- state.build.projects + loadedProject <- state.build.loadedProjects + project = loadedProject.project completion <- cmd.format.showProject(project) } state.logger.info(completion) state @@ -468,9 +472,8 @@ object Interpreter { private def clean(cmd: Commands.Clean, state: State): Task[State] = { if (cmd.projects.isEmpty) { - Tasks - .clean(state, state.build.projects, cmd.includeDependencies) - .map(_.mergeStatus(ExitStatus.Ok)) + val projects = state.build.loadedProjects.map(_.project) + Tasks.clean(state, projects, cmd.includeDependencies).map(_.mergeStatus(ExitStatus.Ok)) } else { val lookup = lookupProjects(cmd.projects, state, state.build.getProjectFor(_)) if (!lookup.missing.isEmpty) diff --git a/frontend/src/main/scala/bloop/engine/State.scala b/frontend/src/main/scala/bloop/engine/State.scala index 41efd75c31..cf0afb53c6 100644 --- a/frontend/src/main/scala/bloop/engine/State.scala +++ b/frontend/src/main/scala/bloop/engine/State.scala @@ -7,7 +7,6 @@ import bloop.engine.caches.{ResultsCache, StateCache} import bloop.io.Paths import bloop.logging.{DebugFilter, Logger} import monix.eval.Task -import bloop.data.LoadedBuild import bloop.data.WorkspaceSettings /** @@ -89,27 +88,18 @@ object State { pool: ClientPool, opts: CommonOptions, logger: Logger, - incomingSettings: Option[WorkspaceSettings] = None, - reapplySettings: Boolean = false + clientSettings: Option[WorkspaceSettings] = None ): Task[State] = { - def loadState(path: bloop.io.AbsolutePath): Task[State] = { - BuildLoader.load(configDir, incomingSettings, logger).map { - case LoadedBuild(projects, buildSettings) => - val build: Build = Build(configDir, buildSettings, projects) - State(build, client, pool, opts, logger) - } - } - - val cached = State.stateCache.addIfMissing( + val cached = State.stateCache.loadState( configDir, client, pool, opts, logger, - loadState(_), - incomingSettings, - reapplySettings + State(_, client, pool, opts, logger), + clientSettings ) + cached.map(_.copy(pool = pool, client = client, commonOptions = opts, logger = logger)) } diff --git a/frontend/src/main/scala/bloop/engine/caches/ResultsCache.scala b/frontend/src/main/scala/bloop/engine/caches/ResultsCache.scala index 31662045b1..ed7b9cb9eb 100644 --- a/frontend/src/main/scala/bloop/engine/caches/ResultsCache.scala +++ b/frontend/src/main/scala/bloop/engine/caches/ResultsCache.scala @@ -257,7 +257,8 @@ object ResultsCache { } } - val all = build.projects.map(p => fetchPreviousResult(p).map(r => p -> r)) + val projects = build.loadedProjects.map(_.project) + val all = projects.map(p => fetchPreviousResult(p).map(r => p -> r)) Task.gatherUnordered(all).executeOn(ExecutionContext.ioScheduler).map { projectResults => val newCache = new ResultsCache(Map.empty, Map.empty) val cleanupTasks = new mutable.ListBuffer[Task[Unit]]() diff --git a/frontend/src/main/scala/bloop/engine/caches/StateCache.scala b/frontend/src/main/scala/bloop/engine/caches/StateCache.scala index 0d7e89a69f..fe3ccc6fa4 100644 --- a/frontend/src/main/scala/bloop/engine/caches/StateCache.scala +++ b/frontend/src/main/scala/bloop/engine/caches/StateCache.scala @@ -11,7 +11,6 @@ import bloop.io.AbsolutePath import bloop.cli.ExitStatus import monix.eval.Task -import bloop.data.LoadedBuild /** Cache that holds the state associated to each loaded build. */ final class StateCache(cache: ConcurrentHashMap[AbsolutePath, StateCache.CachedState]) { @@ -79,41 +78,53 @@ final class StateCache(cache: ConcurrentHashMap[AbsolutePath, StateCache.CachedS * @param computeBuild A function that computes the state from a location. * @return The state associated with `from`, or the newly computed state. */ - def addIfMissing( + def loadState( from: AbsolutePath, client: ClientInfo, pool: ClientPool, commonOptions: CommonOptions, logger: Logger, - computeBuild: AbsolutePath => Task[State], - incomingSettings: Option[WorkspaceSettings], - reapplySettings: Boolean + createState: Build => State, + clientSettings: Option[WorkspaceSettings] ): Task[State] = { - getStateFor(from, client, pool, commonOptions, logger) match { - case Some(state) => - state.build.checkForChange(incomingSettings, logger, reapplySettings).flatMap { - case Build.ReturnPreviousState => Task.now(state) - case Build.UpdateState(createdOrModified, deleted, settingsChanged) => - BuildLoader - .loadBuildFromConfigurationFiles(from, createdOrModified, settingsChanged, logger) - .map { - case PartialLoadedBuild(newProjects, settings) => - val currentProjects = state.build.projects - val toRemove = deleted.toSet ++ newProjects.map(_.origin.path) - val untouched = - currentProjects.collect { case p if !toRemove.contains(p.origin.path) => p } - val newBuild = - state.build.copy(projects = untouched ++ newProjects, settings = settings) - val newState = state.copy(build = newBuild) - cache.put(from, StateCache.CachedState.fromState(newState)) - newState - } - } - case None => - computeBuild(from).map { state => - cache.put(from, StateCache.CachedState.fromState(state)) - state + val empty = Build(from, Nil) + val previousState = getStateFor(from, client, pool, commonOptions, logger) + val build = previousState.map(_.build).getOrElse(empty) + + build.checkForChange(clientSettings, logger).flatMap { + case Build.ReturnPreviousState => Task.now(previousState.getOrElse(createState(empty))) + case Build.UpdateState(created, modified, deleted, changed, settings, writeSettings) => + if (writeSettings) { + settings.foreach { settings => + // Write settings, swallow any error and report it to the user instead of propagating it + WorkspaceSettings.writeToFile(from, settings, logger).left.foreach { e => + logger.displayWarningToUser(s"Failed to write workspace settings: ${e.getMessage}") + logger.trace(e) + } + } } + + val createdOrModified = created ++ modified + BuildLoader + .loadBuildIncrementally(from, createdOrModified, changed, settings, logger) + .map { newProjects => + val newState = previousState match { + case Some(state) => + // Update the build incrementally and then create a new updated state + val currentProjects = state.build.loadedProjects + val toRemove = deleted.toSet ++ newProjects.map(_.project.origin.path) + val untouched = currentProjects.collect { + case p if !toRemove.contains(p.project.origin.path) => p + } + + val newBuild = state.build.copy(loadedProjects = untouched ++ newProjects) + state.copy(build = newBuild) + // Create a new state since there was no previous one + case None => createState(Build(from, newProjects)) + } + cache.put(from, StateCache.CachedState.fromState(newState)) + newState + } } } } diff --git a/frontend/src/test/scala/bloop/BuildLoaderSpec.scala b/frontend/src/test/scala/bloop/BuildLoaderSpec.scala index 3ed690c6f7..5c4289885d 100644 --- a/frontend/src/test/scala/bloop/BuildLoaderSpec.scala +++ b/frontend/src/test/scala/bloop/BuildLoaderSpec.scala @@ -7,71 +7,127 @@ import bloop.engine.{Build, BuildLoader, State} import bloop.io.{AbsolutePath, Paths} import bloop.logging.{Logger, RecordingLogger} import bloop.util.TestUtil -import monix.eval.Task - import bloop.testing.BaseSuite import bloop.data.WorkspaceSettings import bloop.internal.build.BuildInfo + +import monix.eval.Task + object BuildLoaderSpec extends BaseSuite { testLoad("don't reload if nothing changes") { (testBuild, logger) => - testBuild.state.build.checkForChange(logger).map { + testBuild.state.build.checkForChange(None, logger).map { case Build.ReturnPreviousState => () case action: Build.UpdateState => sys.error(s"Expected return previous state, got ${action}") } } - testLoad("reload if forced") { (testBuild, logger) => - testBuild.state.build.checkForChange(logger, reapplySettings = true).map { + testLoad("reload if settings are added") { (testBuild, logger) => + val settings = WorkspaceSettings("4.2.0", List(BuildInfo.scalaVersion)) + testBuild.state.build.checkForChange(Some(settings), logger).map { case Build.ReturnPreviousState => sys.error(s"Expected return updated state, got previous state") case action: Build.UpdateState => - assert(action.createdOrModified.size == 4) + assert(action.createdOrModified.size == 0) + assert(action.deleted.size == 0) + assert(action.invalidated.size == 4) + assert(action.settingsForReload == Some(settings)) } } - testLoad("reload if settings are added") { (testBuild, logger) => - val settings = WorkspaceSettings("4.2.0", List(BuildInfo.scalaVersion)) - testBuild.state.build - .checkForChange(logger, incomingSettings = Some(settings)) - .map { + val sameSettings = WorkspaceSettings("4.2.0", List(BuildInfo.scalaVersion)) + testLoad("do not reload if same settings are added", Some(sameSettings)) { (testBuild, logger) => + testBuild.state.build.checkForChange(Some(sameSettings), logger).map { + case Build.ReturnPreviousState => () + case action: Build.UpdateState => + pprint.log(action.invalidated) + sys.error(s"Expected return previous state, got updated state") + } + } + + testLoad("reload if new settings are added", Some(sameSettings)) { (testBuild, logger) => + val newSettings = WorkspaceSettings("4.1.11", List(BuildInfo.scalaVersion)) + testBuild.state.build.checkForChange(Some(newSettings), logger).map { + case Build.ReturnPreviousState => + sys.error(s"Expected return updated state, got previous state") + case action: Build.UpdateState => + assert(action.createdOrModified.size == 0) + assert(action.deleted.size == 0) + assert(action.invalidated.size == 4) + assert(action.settingsForReload == Some(newSettings)) + } + } + + private def changeHashOfRandomFiles(build: TestBuild, n: Int): Unit = { + val randomFiles = scala.util.Random.shuffle(configurationFiles(build)).take(n) + randomFiles.foreach { f => + // Add whitespace at the end to modify hash + val bytes = Files.readAllBytes(f.underlying) + val contents = new String(bytes, StandardCharsets.UTF_8) + " " + Files.write(f.underlying, contents.getBytes(StandardCharsets.UTF_8)) + } + } + + testLoad("reload if two file contents changed in build with previous", Some(sameSettings)) { + (testBuild, logger) => + changeHashOfRandomFiles(testBuild, 2) + // Don't pass in any settings so that the previous ones are used instead + testBuild.state.build.checkForChange(None, logger).map { case Build.ReturnPreviousState => sys.error(s"Expected return updated state, got previous state") case action: Build.UpdateState => - assert(action.createdOrModified.size == 4) + assert(action.createdOrModified.size == 2) + assert(action.deleted.size == 0) + assert(action.invalidated.size == 0) + assert(action.settingsForReload == Some(sameSettings)) } } - val sameSettings = WorkspaceSettings("4.2.0", List(BuildInfo.scalaVersion)) - testLoad("do not reload if same settings are added", Some(sameSettings)) { (testBuild, logger) => - testBuild.state.build - .checkForChange(logger, incomingSettings = Some(sameSettings)) - .map { - case Build.ReturnPreviousState => () + testLoad("reload if two file contents changed with same settings", Some(sameSettings)) { + (testBuild, logger) => + changeHashOfRandomFiles(testBuild, 2) + testBuild.state.build.checkForChange(Some(sameSettings), logger).map { + case Build.ReturnPreviousState => + sys.error(s"Expected return updated state, got previous state") case action: Build.UpdateState => - sys.error(s"Expected return previous state, got updated state") + assert(action.createdOrModified.size == 2) + assert(action.deleted.size == 0) + assert(action.invalidated.size == 0) + assert(action.settingsForReload == Some(sameSettings)) } } - testLoad("reload if new settings are added", Some(sameSettings)) { (testBuild, logger) => - val newSettings = WorkspaceSettings("4.1.11", List(BuildInfo.scalaVersion)) - testBuild.state.build - .checkForChange(logger, incomingSettings = Some(newSettings)) - .map { + testLoad("reload if new settings are added and two file contents changed", Some(sameSettings)) { + (testBuild, logger) => + changeHashOfRandomFiles(testBuild, 2) + val newSettings = WorkspaceSettings("4.1.11", List(BuildInfo.scalaVersion)) + testBuild.state.build.checkForChange(Some(newSettings), logger).map { case Build.ReturnPreviousState => sys.error(s"Expected return updated state, got previous state") case action: Build.UpdateState => - assert(action.createdOrModified.size == 4) + assert(action.deleted.size == 0) + assert(action.createdOrModified.size == 2) + assert(action.invalidated.size == 2) + assert(action.settingsForReload == Some(newSettings)) } } + testLoad( + "do not reload if no settings are passed to build configured with previous settings", + Some(sameSettings) + ) { (testBuild, logger) => + testBuild.state.build.checkForChange(None, logger).map { + case Build.ReturnPreviousState => () + case action: Build.UpdateState => + sys.error(s"Expected return previous state, got updated state") + } + } + testLoad("do not reload on empty settings") { (testBuild, logger) => val configDir = testBuild.configFileFor(testBuild.projects.head).getParent - testBuild.state.build - .checkForChange(logger, incomingSettings = None) - .map { - case Build.ReturnPreviousState => () - case action: Build.UpdateState => sys.error(s"Expected return previous state, got $action") - } + testBuild.state.build.checkForChange(None, logger).map { + case Build.ReturnPreviousState => () + case action: Build.UpdateState => sys.error(s"Expected return previous state, got $action") + } } private def configurationFiles(build: TestBuild): List[AbsolutePath] = { @@ -82,7 +138,7 @@ object BuildLoaderSpec extends BaseSuite { val randomConfigFiles = scala.util.Random.shuffle(configurationFiles(testBuild)).take(5) // Update the timestamps of the configuration files to trigger a reload randomConfigFiles.foreach(f => Files.write(f.underlying, Files.readAllBytes(f.underlying))) - testBuild.state.build.checkForChange(logger).map { + testBuild.state.build.checkForChange(None, logger).map { case Build.ReturnPreviousState => () case action: Build.UpdateState => sys.error(s"Expected return previous state, got ${action}") } @@ -109,21 +165,19 @@ object BuildLoaderSpec extends BaseSuite { val pathOfDummyFile = testBuild.state.build.origin.resolve("dummy.json").underlying Files.write(pathOfDummyFile, ContentsNewConfigurationFile.getBytes(StandardCharsets.UTF_8)) - testBuild.state.build - .checkForChange(logger) - .map { - case action: Build.UpdateState => - val hasDummyPath = - action.createdOrModified.exists(_.origin.path.underlying == pathOfDummyFile) - if (action.deleted.isEmpty && hasDummyPath) () - else sys.error(s"Expected state with new project addition, got ${action}") - case Build.ReturnPreviousState => - sys.error(s"Expected state with new project addition, got ReturnPreviousState") - } + testBuild.state.build.checkForChange(None, logger).map { + case action: Build.UpdateState => + val hasDummyPath = + action.createdOrModified.exists(_.origin.path.underlying == pathOfDummyFile) + if (action.deleted.isEmpty && hasDummyPath) () + else sys.error(s"Expected state with new project addition, got ${action}") + case Build.ReturnPreviousState => + sys.error(s"Expected state with new project addition, got ReturnPreviousState") + } } testLoad("reload when existing configuration files change") { (testBuild, logger) => - val projectsToModify = testBuild.state.build.projects.take(2) + val projectsToModify = testBuild.state.build.loadedProjects.map(_.project).take(2) val backups = projectsToModify.map(p => p -> p.origin.path.readAllBytes) val changes = backups.map { @@ -137,7 +191,7 @@ object BuildLoaderSpec extends BaseSuite { Task .gatherUnordered(changes) .flatMap { _ => - testBuild.state.build.checkForChange(logger).map { + testBuild.state.build.checkForChange(None, logger).map { case action: Build.UpdateState => val hasAllProjects = { val originProjects = projectsToModify.map(_.origin.path).toSet @@ -159,7 +213,7 @@ object BuildLoaderSpec extends BaseSuite { } change.flatMap { _ => - testBuild.state.build.checkForChange(logger).map { + testBuild.state.build.checkForChange(None, logger).map { case action: Build.UpdateState => val hasProjectDeleted = { action.deleted match { diff --git a/frontend/src/test/scala/bloop/RunSpec.scala b/frontend/src/test/scala/bloop/RunSpec.scala index d3b23674ae..f88acd82a3 100644 --- a/frontend/src/test/scala/bloop/RunSpec.scala +++ b/frontend/src/test/scala/bloop/RunSpec.scala @@ -252,7 +252,7 @@ class RunSpec { val ourInputStream = new ByteArrayInputStream("Hello!\n".getBytes(StandardCharsets.UTF_8)) val hijackedCommonOptions = state0.commonOptions.copy(in = ourInputStream) val state = state0.copy(logger = logger).copy(commonOptions = hijackedCommonOptions) - val projects = state.build.projects + val projects = state.build.loadedProjects.map(_.project) val projectA = getProject("A", state) val action = Run(Commands.Run(List("A"))) val duration = Duration.apply(15, TimeUnit.SECONDS) @@ -284,7 +284,7 @@ class RunSpec { val ourInputStream = new ByteArrayInputStream("Hello!\n".getBytes(StandardCharsets.UTF_8)) val hijackedCommonOptions = state0.commonOptions.copy(in = ourInputStream) val state = state0.copy(logger = logger).copy(commonOptions = hijackedCommonOptions) - val projects = state.build.projects + val projects = state.build.loadedProjects.map(_.project) val projectA = getProject("A", state) val action = Run(Commands.Run(List("A"))) val duration = Duration.apply(13, TimeUnit.SECONDS) diff --git a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala index dda57d83ea..c4bad26356 100644 --- a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala @@ -42,8 +42,7 @@ class BspMetalsClientSpec( val extraParams = BloopExtraBuildParams( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), - supportedScalaVersions = List(testedScalaVersion), - reapplySettings = false + supportedScalaVersions = List(testedScalaVersion) ) loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => @@ -83,9 +82,9 @@ class BspMetalsClientSpec( val extraParams = BloopExtraBuildParams( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), - supportedScalaVersions = List("2.12.8"), - reapplySettings = false + supportedScalaVersions = List("2.12.8") ) + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => assertNoDiffInSettingsFile( configDir, @@ -97,7 +96,7 @@ class BspMetalsClientSpec( |} |""".stripMargin ) - // Expect only range positions to be added, semanticdb is missing + // Expect only range positions to be added, semanticdb is not supported assertScalacOptions(state, `A`, "-Yrangepos") assertNoDiff(logger.warnings.mkString(System.lineSeparator), "") } @@ -128,11 +127,21 @@ class BspMetalsClientSpec( val extraParams = BloopExtraBuildParams( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), - supportedScalaVersions = List(testedScalaVersion), - reapplySettings = false + supportedScalaVersions = List(testedScalaVersion) ) loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => + assertNoDiffInSettingsFile( + configDir, + """|{ + | "semanticDBVersion" : "4.2.0", + | "supportedScalaVersions" : [ + | "2.12.8" + | ] + |} + |""".stripMargin + ) + val scalacOptions = state.scalaOptions(`A`)._2.items.head.options val expected = defaultScalacOptions :+ "-Yrangepos" assert(scalacOptions == expected) @@ -164,63 +173,56 @@ class BspMetalsClientSpec( val extraParams = BloopExtraBuildParams( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), - supportedScalaVersions = List(testedScalaVersion), - reapplySettings = false + supportedScalaVersions = List(testedScalaVersion) ) + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => + assertNoDiffInSettingsFile( + configDir, + """|{ + | "semanticDBVersion" : "4.2.0", + | "supportedScalaVersions" : [ + | "2.12.8" + | ] + |} + |""".stripMargin + ) + val scalacOptions = state.scalaOptions(`A`)._2.items.head.options assert(scalacOptions == defaultScalacOptions) } } } - test("force reload of all projects if reapplySettings is set to true") { - TestUtil.withinWorkspace { workspace => - val semanticdbVersion = "4.2.0" - val extraParams = BloopExtraBuildParams( - clientClassesRootDir = None, - semanticdbVersion = Some(semanticdbVersion), - supportedScalaVersions = List(testedScalaVersion), - reapplySettings = true - ) - val `A` = TestProject(workspace, "A", Nil) - val projects = List(`A`) - val configDir = TestProject.populateWorkspace(workspace, projects) - WorkspaceSettings.writeToFile( - configDir, - WorkspaceSettings(semanticdbVersion, List(testedScalaVersion)) - ) - val logger = new RecordingLogger(ansiCodesSupported = false) - loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams)(_ => ()) - loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => - assert(logger.infos.contains("Forcing reload of all projects")) - } - } - } - test("should save workspace settings with cached build") { TestUtil.withinWorkspace { workspace => val semanticdbVersion = "4.2.0" val extraParams = BloopExtraBuildParams( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), - supportedScalaVersions = List(testedScalaVersion), - reapplySettings = false + supportedScalaVersions = List(testedScalaVersion) ) val `A` = TestProject(workspace, "A", Nil) val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) + val logger = new RecordingLogger(ansiCodesSupported = false) WorkspaceSettings.writeToFile( configDir, - WorkspaceSettings(semanticdbVersion, List(testedScalaVersion)) + WorkspaceSettings(semanticdbVersion, List(testedScalaVersion)), + logger ) - val logger = new RecordingLogger(ansiCodesSupported = false) - loadBspState(workspace, projects, logger, "Metals")(_ => ()) - loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => + + def checkSettings: Unit = { assert(configDir.resolve(WorkspaceSettings.settingsFileName).exists) val settings = WorkspaceSettings.readFromFile(configDir, logger) assert(settings.isDefined && settings.get.semanticDBVersion == semanticdbVersion) } + + loadBspState(workspace, projects, logger, "Metals")(_ => checkSettings) + loadBspState(workspace, projects, logger, "unrecognized")(_ => checkSettings) + loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { _ => + checkSettings + } } } @@ -243,8 +245,7 @@ class BspMetalsClientSpec( val extraParams = BloopExtraBuildParams( clientClassesRootDir = None, semanticdbVersion = Some(semanticdbVersion), - supportedScalaVersions = List(testedScalaVersion), - reapplySettings = false + supportedScalaVersions = List(testedScalaVersion) ) val bspLogger = new BspClientLogger(logger) val bspCommand = createBspCommand(configDir) @@ -292,8 +293,12 @@ class BspMetalsClientSpec( val `A` = TestProject(workspace, "A", dummyFooSources) val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) - WorkspaceSettings.writeToFile(configDir, WorkspaceSettings("4.2.0", List(testedScalaVersion))) val logger = new RecordingLogger(ansiCodesSupported = false) + WorkspaceSettings.writeToFile( + configDir, + WorkspaceSettings("4.2.0", List(testedScalaVersion)), + logger + ) val bspState = loadBspState(workspace, projects, logger) { state => val compiledState = state.compile(`A`).toTestState assert(compiledState.status == ExitStatus.Ok) @@ -307,11 +312,12 @@ class BspMetalsClientSpec( val `A` = TestProject(workspace, "A", dummyFooSources) val projects = List(`A`) val configDir = TestProject.populateWorkspace(workspace, projects) + val logger = new RecordingLogger(ansiCodesSupported = false) WorkspaceSettings.writeToFile( configDir, - WorkspaceSettings("4.1.11", List(testedScalaVersion)) + WorkspaceSettings("4.1.11", List(testedScalaVersion)), + logger ) - val logger = new RecordingLogger(ansiCodesSupported = false) loadBspState(workspace, projects, logger) { state => val compiledState = state.compile(`A`).toTestState assert(compiledState.status == ExitStatus.Ok) @@ -329,8 +335,7 @@ class BspMetalsClientSpec( val extraParams = BloopExtraBuildParams( clientClassesRootDir = None, semanticdbVersion = Some("4.2.0"), - supportedScalaVersions = List(testedScalaVersion), - reapplySettings = false + supportedScalaVersions = List(testedScalaVersion) ) loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => val compiledState = state.compile(`A`).toTestState diff --git a/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala b/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala index 6a93fbd8ee..2fbca7d667 100644 --- a/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspProtocolSpec.scala @@ -107,8 +107,7 @@ class BspProtocolSpec( val extraBloopParams = BloopExtraBuildParams( Some(Uri(userClientClassesRootDir.toBspUri)), semanticdbVersion = None, - supportedScalaVersions = Nil, - reapplySettings = false + supportedScalaVersions = Nil ) // Start first client and query for scalac options which creates client classes dirs diff --git a/frontend/src/test/scala/bloop/testing/BloopHelpers.scala b/frontend/src/test/scala/bloop/testing/BloopHelpers.scala index 8c128c3ff9..eee3e50bd0 100644 --- a/frontend/src/test/scala/bloop/testing/BloopHelpers.scala +++ b/frontend/src/test/scala/bloop/testing/BloopHelpers.scala @@ -30,7 +30,7 @@ trait BloopHelpers { settings: Option[WorkspaceSettings] = None ): TestState = { val configDir = TestProject.populateWorkspace(workspace, projects) - settings.foreach(WorkspaceSettings.writeToFile(configDir, _)) + settings.foreach(WorkspaceSettings.writeToFile(configDir, _, logger)) new TestState(TestUtil.loadTestProject(configDir.underlying, logger)) } diff --git a/frontend/src/test/scala/bloop/util/TestUtil.scala b/frontend/src/test/scala/bloop/util/TestUtil.scala index 31d6e15be5..baab3c7f15 100644 --- a/frontend/src/test/scala/bloop/util/TestUtil.scala +++ b/frontend/src/test/scala/bloop/util/TestUtil.scala @@ -38,6 +38,7 @@ import scala.meta.jsonrpc.Services import scala.tools.nsc.Properties import scala.util.control.NonFatal import bloop.data.WorkspaceSettings +import bloop.data.LoadedProject object TestUtil { def projectDir(base: Path, name: String) = base.resolve(name) @@ -73,7 +74,6 @@ object TestUtil { def checkAfterCleanCompilation( structures: Map[String, Map[String, String]], dependencies: Map[String, Set[String]], - workspaceSettings: Option[WorkspaceSettings] = None, rootProjects: List[String] = List(RootProject), scalaInstance: ScalaInstance = TestUtil.scalaInstance, javaEnv: JavaEnv = JavaEnv.default, @@ -82,11 +82,11 @@ object TestUtil { useSiteLogger: Option[Logger] = None, order: CompileOrder = Config.Mixed )(afterCompile: State => Unit = (_ => ())) = { - testState(structures, dependencies, workspaceSettings, scalaInstance, javaEnv, order) { (state: State) => + testState(structures, dependencies, scalaInstance, javaEnv, order) { (state: State) => def action(state0: State): Unit = { val state = useSiteLogger.map(logger => state0.copy(logger = logger)).getOrElse(state0) // Check that this is a clean compile! - val projects = state.build.projects + val projects = state.build.loadedProjects.map(_.project) assert(projects.forall(p => noPreviousAnalysis(p, state))) val action = Run(Commands.Compile(rootProjects, incremental = true)) val compiledState = TestUtil.blockingExecute(action, state) @@ -199,8 +199,18 @@ object TestUtil { assert(Files.exists(configDir), "Does not exist: " + configDir) val configDirectory = AbsolutePath(configDir) - val loadedBuild = BuildLoader.loadSynchronously(configDirectory, logger) - val build = Build(configDirectory, loadedBuild.workspaceSettings, transformProjects(loadedBuild.projects)) + val loadedProjects = BuildLoader.loadSynchronously(configDirectory, logger) + val transformedProjects = loadedProjects.map { + case LoadedProject.RawProject(project) => + LoadedProject.RawProject(transformProjects(List(project)).head) + case LoadedProject.ConfiguredProject(project, original, settings) => + LoadedProject.ConfiguredProject( + transformProjects(List(project)).head, + original, + settings + ) + } + val build = Build(configDirectory, transformedProjects) val state = State.forTests(build, TestUtil.getCompilerCache(logger), logger) val state1 = state.copy(commonOptions = state.commonOptions.copy(env = runAndTestProperties)) if (!emptyResults) state1 else state1.copy(results = ResultsCache.emptyForTests) @@ -262,7 +272,6 @@ object TestUtil { def testState[T]( projectStructures: Map[String, Map[String, String]], dependenciesMap: Map[String, Set[String]], - workspaceSettings: Option[WorkspaceSettings] = None, instance: ScalaInstance = TestUtil.scalaInstance, env: JavaEnv = JavaEnv.default, order: CompileOrder = Config.Mixed, @@ -276,7 +285,8 @@ object TestUtil { val deps = dependenciesMap.getOrElse(name, Set.empty) makeProject(temp, name, sources, deps, Some(instance), env, logger, order, extraJars) } - val build = Build(temp, workspaceSettings, projects.toList) + val loaded = projects.map(p => LoadedProject.RawProject(p)) + val build = Build(temp, loaded.toList) val state = State.forTests(build, TestUtil.getCompilerCache(logger), logger) try op(state) catch { @@ -300,7 +310,7 @@ object TestUtil { state.results.lastSuccessfulResult(project).isDefined private[bloop] def syntheticOriginFor(path: AbsolutePath): Origin = - Origin(path, FileTime.fromMillis(0), scala.util.Random.nextInt()) + Origin(path, FileTime.fromMillis(0), 0L, scala.util.Random.nextInt()) def makeProject( baseDir: AbsolutePath, @@ -379,7 +389,8 @@ object TestUtil { } def ensureCompilationInAllTheBuild(state: State): Unit = { - state.build.projects.foreach { p => + state.build.loadedProjects.foreach { loadedProject => + val p = loadedProject.project Assert.assertTrue(s"${p.name} was not compiled", hasPreviousResult(p, state)) } } diff --git a/integrations/gradle-bloop/src/test/scala/bloop/integrations/gradle/ConfigGenerationSuite.scala b/integrations/gradle-bloop/src/test/scala/bloop/integrations/gradle/ConfigGenerationSuite.scala index 5abd74141e..3d8b8b5ad5 100644 --- a/integrations/gradle-bloop/src/test/scala/bloop/integrations/gradle/ConfigGenerationSuite.scala +++ b/integrations/gradle-bloop/src/test/scala/bloop/integrations/gradle/ConfigGenerationSuite.scala @@ -22,11 +22,11 @@ import bloop.engine.BuildLoader import scala.collection.JavaConverters._ -class ConfigGenerationSuite481 extends ConfigGenerationSuite{ +class ConfigGenerationSuite481 extends ConfigGenerationSuite { protected val gradleVersion: String = "4.8.1" } -class ConfigGenerationSuite531 extends ConfigGenerationSuite{ +class ConfigGenerationSuite531 extends ConfigGenerationSuite { protected val gradleVersion: String = "5.3.1" } @@ -214,7 +214,8 @@ abstract class ConfigGenerationSuite { createHelloWorldScalaSource(buildDirC, "package z { class C }") createHelloWorldScalaSource( buildDirD, - "package zz { class D extends x.A { println(new z.C) } }") + "package zz { class D extends x.A { println(new z.C) } }" + ) GradleRunner .create() @@ -265,16 +266,19 @@ abstract class ConfigGenerationSuite { def assertSources(config: Config.File, entryName: String): Unit = { assertTrue( s"Resolution field for ${config.project.name} does not exist", - config.project.resolution.isDefined) + config.project.resolution.isDefined + ) config.project.resolution.foreach { resolution => val sources = resolution.modules.find( module => - module.name.contains(entryName) && module.artifacts.exists( - _.classifier.contains("sources"))) + module.name.contains(entryName) && module.artifacts + .exists(_.classifier.contains("sources")) + ) assertTrue(s"Sources for $entryName do not exist", sources.isDefined) assertTrue( s"There are more sources than one for $entryName:\n${sources.get.artifacts.mkString("\n")}", - sources.exists(_.artifacts.size == 2)) + sources.exists(_.artifacts.size == 2) + ) } } @@ -290,33 +294,24 @@ abstract class ConfigGenerationSuite { assertSources(configBTest, "scala-library") assert(hasClasspathEntryName(configCTest, "scala-library")) assertSources(configCTest, "scala-library") - assert( - hasClasspathEntryName(configATest, "/a/build/classes".replace('/', File.separatorChar))) - assert( - hasClasspathEntryName(configCTest, "/c/build/classes".replace('/', File.separatorChar))) + assert(hasClasspathEntryName(configATest, "/a/build/classes".replace('/', File.separatorChar))) + assert(hasClasspathEntryName(configCTest, "/c/build/classes".replace('/', File.separatorChar))) assert(hasClasspathEntryName(configB, "cats-core")) assertSources(configB, "cats-core") assert(hasClasspathEntryName(configB, "/a/build/classes".replace('/', File.separatorChar))) assert(hasClasspathEntryName(configB, "/c/build/classes".replace('/', File.separatorChar))) assert(hasClasspathEntryName(configBTest, "cats-core")) assertSources(configBTest, "cats-core") - assert( - hasClasspathEntryName(configBTest, "/a/build/classes".replace('/', File.separatorChar))) - assert( - hasClasspathEntryName(configBTest, "/b/build/classes".replace('/', File.separatorChar))) - assert( - hasClasspathEntryName(configBTest, "/c/build/classes".replace('/', File.separatorChar))) + assert(hasClasspathEntryName(configBTest, "/a/build/classes".replace('/', File.separatorChar))) + assert(hasClasspathEntryName(configBTest, "/b/build/classes".replace('/', File.separatorChar))) + assert(hasClasspathEntryName(configBTest, "/c/build/classes".replace('/', File.separatorChar))) assert(hasClasspathEntryName(configD, "/a/build/classes".replace('/', File.separatorChar))) assert(hasClasspathEntryName(configD, "/b/build/classes".replace('/', File.separatorChar))) assert(hasClasspathEntryName(configD, "/c/build/classes".replace('/', File.separatorChar))) - assert( - hasClasspathEntryName(configDTest, "/a/build/classes".replace('/', File.separatorChar))) - assert( - hasClasspathEntryName(configDTest, "/b/build/classes".replace('/', File.separatorChar))) - assert( - hasClasspathEntryName(configDTest, "/c/build/classes".replace('/', File.separatorChar))) - assert( - hasClasspathEntryName(configDTest, "/d/build/classes".replace('/', File.separatorChar))) + assert(hasClasspathEntryName(configDTest, "/a/build/classes".replace('/', File.separatorChar))) + assert(hasClasspathEntryName(configDTest, "/b/build/classes".replace('/', File.separatorChar))) + assert(hasClasspathEntryName(configDTest, "/c/build/classes".replace('/', File.separatorChar))) + assert(hasClasspathEntryName(configDTest, "/d/build/classes".replace('/', File.separatorChar))) assert(compileBloopProject("b", bloopDir).status.isOk) assert(compileBloopProject("d", bloopDir).status.isOk) @@ -421,13 +416,13 @@ abstract class ConfigGenerationSuite { assert(!hasClasspathEntryName(configB, "/a/build/classes".replace('/', File.separatorChar))) assert( - !hasClasspathEntryName(configB, "/a-test/build/classes".replace('/', File.separatorChar))) - assert( - !hasClasspathEntryName(configBTest, "/a/build/classes".replace('/', File.separatorChar))) - assert( - hasClasspathEntryName(configBTest, "/b/build/classes".replace('/', File.separatorChar))) + !hasClasspathEntryName(configB, "/a-test/build/classes".replace('/', File.separatorChar)) + ) + assert(!hasClasspathEntryName(configBTest, "/a/build/classes".replace('/', File.separatorChar))) + assert(hasClasspathEntryName(configBTest, "/b/build/classes".replace('/', File.separatorChar))) assert( - hasClasspathEntryName(configBTest, "/a-test/build/classes".replace('/', File.separatorChar))) + hasClasspathEntryName(configBTest, "/a-test/build/classes".replace('/', File.separatorChar)) + ) assert(compileBloopProject("b", bloopDir).status.isOk) } @@ -542,11 +537,12 @@ abstract class ConfigGenerationSuite { config.project.classpath.exists(_.toString.contains(entryName)) assert(!hasClasspathEntryName(configB, "/a/build/classes".replace('/', File.separatorChar))) assert( - !hasClasspathEntryName(configB, "/a-test/build/classes".replace('/', File.separatorChar))) - assert( - hasClasspathEntryName(configBTest, "/a-test/build/classes".replace('/', File.separatorChar))) + !hasClasspathEntryName(configB, "/a-test/build/classes".replace('/', File.separatorChar)) + ) assert( - hasClasspathEntryName(configBTest, "/b/build/classes".replace('/', File.separatorChar))) + hasClasspathEntryName(configBTest, "/a-test/build/classes".replace('/', File.separatorChar)) + ) + assert(hasClasspathEntryName(configBTest, "/b/build/classes".replace('/', File.separatorChar))) assert(compileBloopProject("b-test", bloopDir).status.isOk) } @@ -596,7 +592,8 @@ abstract class ConfigGenerationSuite { assertEquals( List("-deprecation", "-encoding", "utf8", "-unchecked"), - resultConfig.project.`scala`.get.options) + resultConfig.project.`scala`.get.options + ) } @Test def flagsWithArgsGeneratedCorrectly(): Unit = { @@ -656,7 +653,8 @@ abstract class ConfigGenerationSuite { "-deprecation", "-encoding", "utf8", - "-unchecked"), + "-unchecked" + ), resultConfig.project.`scala`.get.options ) } @@ -771,8 +769,7 @@ abstract class ConfigGenerationSuite { .withArguments("bloopInstall", "-Si") .build() - assert( - result.getOutput.contains("Ignoring 'bloopInstall' on non-Scala and non-Java project")) + assert(result.getOutput.contains("Ignoring 'bloopInstall' on non-Scala and non-Java project")) val projectName = testProjectDir.getRoot.getName val bloopFile = new File(new File(testProjectDir.getRoot, ".bloop"), projectName + ".json") assert(!bloopFile.exists()) @@ -1084,11 +1081,13 @@ abstract class ConfigGenerationSuite { assert(resultConfig.project.resolution.nonEmpty) assert( - resultConfig.project.resolution.get.modules.exists(p => p.name == "semanticdb-scalac_2.12.8")) + resultConfig.project.resolution.get.modules.exists(p => p.name == "semanticdb-scalac_2.12.8") + ) assert( resultConfig.project.`scala`.get.options - .contains(s"-P:semanticdb:sourceroot:${testProjectDir.getRoot.getCanonicalPath()}")) + .contains(s"-P:semanticdb:sourceroot:${testProjectDir.getRoot.getCanonicalPath()}") + ) assert(resultConfig.project.`scala`.get.options.exists(p => p.startsWith("-Xplugin:"))) } @@ -1096,8 +1095,8 @@ abstract class ConfigGenerationSuite { val logger = BloopLogger.default(configDir.toString) assert(Files.exists(configDir.toPath), "Does not exist: " + configDir) val configDirectory = AbsolutePath(configDir) - val loadedBuild = BuildLoader.loadSynchronously(configDirectory, logger) - val build = Build(configDirectory, loadedBuild.workspaceSettings, loadedBuild.projects) + val loadedProjects = BuildLoader.loadSynchronously(configDirectory, logger) + val build = Build(configDirectory, loadedProjects) State.forTests(build, TestUtil.getCompilerCache(logger), logger) } diff --git a/shared/src/main/scala/bloop/io/Paths.scala b/shared/src/main/scala/bloop/io/Paths.scala index 3863b3459e..177b563cad 100644 --- a/shared/src/main/scala/bloop/io/Paths.scala +++ b/shared/src/main/scala/bloop/io/Paths.scala @@ -79,7 +79,7 @@ object Paths { out.toList } - case class AttributedPath(path: AbsolutePath, lastModifiedTime: FileTime, size: Long = 0L) + case class AttributedPath(path: AbsolutePath, lastModifiedTime: FileTime, size: Long) /** * Get all files under `base` that match the pattern `pattern` up to depth `maxDepth`. From 7ff9cb3b933926394d06cd5ba4613c61a105f85b Mon Sep 17 00:00:00 2001 From: jvican Date: Tue, 30 Jul 2019 17:57:37 +0200 Subject: [PATCH 11/22] Remove loaded build abstraction --- frontend/src/main/scala/bloop/data/LoadedBuild.scala | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 frontend/src/main/scala/bloop/data/LoadedBuild.scala diff --git a/frontend/src/main/scala/bloop/data/LoadedBuild.scala b/frontend/src/main/scala/bloop/data/LoadedBuild.scala deleted file mode 100644 index 915af6a588..0000000000 --- a/frontend/src/main/scala/bloop/data/LoadedBuild.scala +++ /dev/null @@ -1,9 +0,0 @@ -package bloop.data - -/** - * A partial loaded build is the incremental result of loading a certain amount - * of configuration files from disk and post-processing them in-memory. - */ -case class PartialLoadedBuild( - projects: List[LoadedProject] -) From 0fca102e7effe010241497e248cf7907c9b9f899 Mon Sep 17 00:00:00 2001 From: jvican Date: Tue, 30 Jul 2019 18:07:19 +0200 Subject: [PATCH 12/22] Remove `changedSettings` from `reattemptConfiguration` The semantics to retry adding the semanticdb plugin to a project depend uniquely on whether the project has semanticdb disabled and whether the `newSettings` passed by the client are not empty. Whether there is a change in the settings or not should not affect these semantics. --- frontend/src/main/scala/bloop/engine/Build.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/main/scala/bloop/engine/Build.scala b/frontend/src/main/scala/bloop/engine/Build.scala index 14bc682c76..ddbeafee7b 100644 --- a/frontend/src/main/scala/bloop/engine/Build.scala +++ b/frontend/src/main/scala/bloop/engine/Build.scala @@ -111,7 +111,7 @@ final case class Build private ( findUpdateSettingsAction(Some(settings), settingsForReload) match { case Build.AvoidReload(_) => val options = project.scalacOptions - val reattemptConfiguration = newSettings.nonEmpty && changedSettings && { + val reattemptConfiguration = newSettings.nonEmpty && { Project.hasSemanticDBEnabledInCompilerOptions(project.scalacOptions) } From 8acbe60e9874a44e61179bbf70972ffdebfb32f9 Mon Sep 17 00:00:00 2001 From: jvican Date: Tue, 30 Jul 2019 18:11:43 +0200 Subject: [PATCH 13/22] Throw exceptions when writing workspace settings file Because `writeToFile` returned `Either`, there were places in our tests where we were completely swallowing any exception that could be happening when writing the workspace settings file. Now we will throw the exception instead. In the case of the internal build logic, we do swallow any exception thrown by this logic intentionally. --- .../main/scala/bloop/data/WorkspaceSettings.scala | 15 +++++++-------- .../scala/bloop/engine/caches/StateCache.scala | 8 +++++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala index 088cd43a1f..42bd5ef186 100644 --- a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala +++ b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala @@ -80,14 +80,13 @@ object WorkspaceSettings { configDir: AbsolutePath, settings: WorkspaceSettings, logger: Logger - ): Either[Throwable, Path] = { - Try { - val settingsFile = configDir.resolve(settingsFileName) - logger.debug(s"Writing workspace settings to $settingsFile")(DebugFilter.All) - val jsonObject = settingsEncoder(settings) - val output = Printer.spaces4.copy(dropNullValues = true).pretty(jsonObject) - Files.write(settingsFile.underlying, output.getBytes(StandardCharsets.UTF_8)) - }.toEither + ): AbsolutePath = { + val settingsFile = configDir.resolve(settingsFileName) + logger.debug(s"Writing workspace settings to $settingsFile")(DebugFilter.All) + val jsonObject = settingsEncoder(settings) + val output = Printer.spaces4.copy(dropNullValues = true).pretty(jsonObject) + Files.write(settingsFile.underlying, output.getBytes(StandardCharsets.UTF_8)) + settingsFile } def fromJson(json: Json): WorkspaceSettings = { diff --git a/frontend/src/main/scala/bloop/engine/caches/StateCache.scala b/frontend/src/main/scala/bloop/engine/caches/StateCache.scala index fe3ccc6fa4..fcd15adcf0 100644 --- a/frontend/src/main/scala/bloop/engine/caches/StateCache.scala +++ b/frontend/src/main/scala/bloop/engine/caches/StateCache.scala @@ -97,9 +97,11 @@ final class StateCache(cache: ConcurrentHashMap[AbsolutePath, StateCache.CachedS if (writeSettings) { settings.foreach { settings => // Write settings, swallow any error and report it to the user instead of propagating it - WorkspaceSettings.writeToFile(from, settings, logger).left.foreach { e => - logger.displayWarningToUser(s"Failed to write workspace settings: ${e.getMessage}") - logger.trace(e) + try WorkspaceSettings.writeToFile(from, settings, logger) + catch { + case e: Throwable => + logger.displayWarningToUser(s"Failed to write workspace settings: ${e.getMessage}") + logger.trace(e) } } } From e3241d1ed4081490ac1495c7d238bccc5fdfd642 Mon Sep 17 00:00:00 2001 From: Tomasz Godzik Date: Thu, 25 Jul 2019 16:29:45 +0200 Subject: [PATCH 14/22] Add an additional setting for workspace root --- config/src/main/scala/bloop/config/Config.scala | 4 +++- frontend/src/it/scala/bloop/CommunityBuild.scala | 1 + frontend/src/main/scala/bloop/data/Project.scala | 2 ++ frontend/src/test/scala/bloop/DagSpec.scala | 2 +- frontend/src/test/scala/bloop/util/TestProject.scala | 1 + frontend/src/test/scala/bloop/util/TestUtil.scala | 3 ++- .../bloop/integrations/gradle/model/BloopConverter.scala | 1 + .../scala/bloop/integrations/maven/MojoImplementation.scala | 2 +- .../src/main/scala/bloop/integrations/mill/MillBloop.scala | 1 + .../src/main/scala/bloop/integrations/sbt/SbtBloop.scala | 2 +- 10 files changed, 14 insertions(+), 5 deletions(-) diff --git a/config/src/main/scala/bloop/config/Config.scala b/config/src/main/scala/bloop/config/Config.scala index e7a742c6e6..95e377d508 100644 --- a/config/src/main/scala/bloop/config/Config.scala +++ b/config/src/main/scala/bloop/config/Config.scala @@ -216,6 +216,7 @@ object Config { case class Project( name: String, directory: Path, + workspaceRoot: Option[Path], sources: List[Path], dependencies: List[String], classpath: List[Path], @@ -232,7 +233,7 @@ object Config { object Project { // FORMAT: OFF - private[bloop] val empty: Project = Project("", emptyPath, List(), List(), List(), emptyPath, emptyPath, None, None, None, None, None, None, None) + private[bloop] val empty: Project = Project("", emptyPath, None, List(), List(), List(), emptyPath, emptyPath, None, None, None, None, None, None, None) // FORMAT: ON def analysisFileName(projectName: String) = s"$projectName-analysis.bin" @@ -271,6 +272,7 @@ object Config { val project = Project( "dummy-project", workingDirectory, + Some(workingDirectory), List(sourceFile), List("dummy-2"), List(scalaLibraryJar), diff --git a/frontend/src/it/scala/bloop/CommunityBuild.scala b/frontend/src/it/scala/bloop/CommunityBuild.scala index 64044fed36..a116576360 100644 --- a/frontend/src/it/scala/bloop/CommunityBuild.scala +++ b/frontend/src/it/scala/bloop/CommunityBuild.scala @@ -122,6 +122,7 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) { val rootProject = Project( name = rootProjectName, baseDirectory = dummyExistingBaseDir, + workspaceRootDirectory = Some(buildBaseDir), dependencies = allProjectsInBuild.map(_.project.name), scalaInstance = allProjectsInBuild.head.project.scalaInstance, rawClasspath = Nil, diff --git a/frontend/src/main/scala/bloop/data/Project.scala b/frontend/src/main/scala/bloop/data/Project.scala index c10d68d40e..41de6f7064 100644 --- a/frontend/src/main/scala/bloop/data/Project.scala +++ b/frontend/src/main/scala/bloop/data/Project.scala @@ -23,6 +23,7 @@ import xsbti.compile.{ClasspathOptions, CompileOrder} final case class Project( name: String, baseDirectory: AbsolutePath, + workspaceRootDirectory: Option[AbsolutePath], dependencies: List[String], scalaInstance: Option[ScalaInstance], rawClasspath: List[AbsolutePath], @@ -167,6 +168,7 @@ object Project { Project( project.name, AbsolutePath(project.directory), + project.workspaceRoot.map(AbsolutePath.apply), project.dependencies, instance, project.classpath.map(AbsolutePath.apply), diff --git a/frontend/src/test/scala/bloop/DagSpec.scala b/frontend/src/test/scala/bloop/DagSpec.scala index 617fbe2400..5ba4e38d81 100644 --- a/frontend/src/test/scala/bloop/DagSpec.scala +++ b/frontend/src/test/scala/bloop/DagSpec.scala @@ -21,7 +21,7 @@ class DagSpec { // format: OFF def dummyOrigin = TestUtil.syntheticOriginFor(dummyPath) def dummyProject(name: String, dependencies: List[String]): Project = - Project(name, dummyPath, dependencies, Some(dummyInstance), Nil, Nil, compileOptions, + Project(name, dummyPath, None, dependencies, Some(dummyInstance), Nil, Nil, compileOptions, dummyPath, Nil, Nil, Nil, Nil, Config.TestOptions.empty, dummyPath, dummyPath, Project.defaultPlatform(logger), None, None, dummyOrigin) // format: ON diff --git a/frontend/src/test/scala/bloop/util/TestProject.scala b/frontend/src/test/scala/bloop/util/TestProject.scala index 47d0f0331c..0e9788230b 100644 --- a/frontend/src/test/scala/bloop/util/TestProject.scala +++ b/frontend/src/test/scala/bloop/util/TestProject.scala @@ -120,6 +120,7 @@ object TestProject { val config = Config.Project( name, projectBaseDir, + Option(baseDir.underlying), List(sourceDir.underlying), directDependencies.map(_.config.name), classpath, diff --git a/frontend/src/test/scala/bloop/util/TestUtil.scala b/frontend/src/test/scala/bloop/util/TestUtil.scala index baab3c7f15..80a32e67d1 100644 --- a/frontend/src/test/scala/bloop/util/TestUtil.scala +++ b/frontend/src/test/scala/bloop/util/TestUtil.scala @@ -343,6 +343,7 @@ object TestUtil { Project( name = name, baseDirectory = AbsolutePath(baseDirectory), + workspaceRootDirectory = Option(baseDir), dependencies = dependencies.toList, scalaInstance = scalaInstance, rawClasspath = classpath, @@ -465,7 +466,7 @@ object TestUtil { val classesDir = Files.createDirectory(outDir.resolve("classes")) // format: OFF - val configFileG = bloop.config.Config.File(Config.File.LatestVersion, Config.Project("g", baseDir, Nil, List("g"), Nil, outDir, classesDir, None, None, None, None, None, None, None)) + val configFileG = bloop.config.Config.File(Config.File.LatestVersion, Config.Project("g", baseDir, Option(baseDir), Nil, List("g"), Nil, outDir, classesDir, None, None, None, None, None, None, None)) bloop.config.write(configFileG, jsonTargetG) // format: ON diff --git a/integrations/gradle-bloop/src/main/scala/bloop/integrations/gradle/model/BloopConverter.scala b/integrations/gradle-bloop/src/main/scala/bloop/integrations/gradle/model/BloopConverter.scala index 30f3317aa8..fbb496abb8 100644 --- a/integrations/gradle-bloop/src/main/scala/bloop/integrations/gradle/model/BloopConverter.scala +++ b/integrations/gradle-bloop/src/main/scala/bloop/integrations/gradle/model/BloopConverter.scala @@ -197,6 +197,7 @@ final class BloopConverter(parameters: BloopParameters) { bloopProject = Config.Project( name = getProjectName(project, sourceSet), directory = project.getProjectDir.toPath, + workspaceRoot = Option(project.getRootProject().getProjectDir().toPath()), sources = sources, dependencies = allDependencies, classpath = classpath, diff --git a/integrations/maven-bloop/src/main/scala/bloop/integrations/maven/MojoImplementation.scala b/integrations/maven-bloop/src/main/scala/bloop/integrations/maven/MojoImplementation.scala index af24d9555f..c152f722a6 100644 --- a/integrations/maven-bloop/src/main/scala/bloop/integrations/maven/MojoImplementation.scala +++ b/integrations/maven-bloop/src/main/scala/bloop/integrations/maven/MojoImplementation.scala @@ -131,7 +131,7 @@ object MojoImplementation { val platform = Some(Config.Platform.Jvm(Config.JvmConfig(javaHome, launcher.getJvmArgs().toList), mainClass)) // Resources in Maven require val resources = Some(resources0.asScala.toList.flatMap(a => Option(a.getTargetPath).toList).map(classesDir.resolve)) - val project = Config.Project(name, baseDirectory, sourceDirs, dependencyNames, classpath, out, classesDir, resources, `scala`, java, sbt, test, platform, resolution) + val project = Config.Project(name, baseDirectory, Some(root.toPath), sourceDirs, dependencyNames, classpath, out, classesDir, resources, `scala`, java, sbt, test, platform, resolution) Config.File(Config.File.LatestVersion, project) } // FORMAT: ON diff --git a/integrations/mill-bloop/src/main/scala/bloop/integrations/mill/MillBloop.scala b/integrations/mill-bloop/src/main/scala/bloop/integrations/mill/MillBloop.scala index 32b1733d74..8c05157d0d 100644 --- a/integrations/mill-bloop/src/main/scala/bloop/integrations/mill/MillBloop.scala +++ b/integrations/mill-bloop/src/main/scala/bloop/integrations/mill/MillBloop.scala @@ -99,6 +99,7 @@ object Bloop extends ExternalModule { Config.Project( name = name(module), directory = module.millSourcePath.toNIO, + workspaceRoot = Option(pwd.wrapped), sources = module.allSources().map(_.path.toNIO).toList, dependencies = module.moduleDeps.map(name).toList, classpath = classpath().map(_.toNIO).toList, diff --git a/integrations/sbt-bloop/src/main/scala/bloop/integrations/sbt/SbtBloop.scala b/integrations/sbt-bloop/src/main/scala/bloop/integrations/sbt/SbtBloop.scala index 31e1ca90ed..70609ed12d 100644 --- a/integrations/sbt-bloop/src/main/scala/bloop/integrations/sbt/SbtBloop.scala +++ b/integrations/sbt-bloop/src/main/scala/bloop/integrations/sbt/SbtBloop.scala @@ -868,7 +868,7 @@ object BloopDefaults { val resources = Some(bloopResourcesTask.value) val sbt = computeSbtMetadata.value.map(_.config) - val project = Config.Project(projectName, baseDirectory, sources, dependenciesAndAggregates, + val project = Config.Project(projectName, baseDirectory, Option(buildBaseDirectory.toPath), sources, dependenciesAndAggregates, classpath, out, classesDir, resources, Some(`scala`), Some(java), sbt, Some(testOptions), Some(platform), resolution) Config.File(Config.File.LatestVersion, project) } From 194cc1679316b5ba4e8a0469623f67196bcecefe Mon Sep 17 00:00:00 2001 From: jvican Date: Tue, 30 Jul 2019 18:21:56 +0200 Subject: [PATCH 15/22] Rename `workspaceRoot` to `workspaceDir` The workspace already implies that it's the root directory and we already use `Dir` as the suffix of many fields in the configuration, so this is a more idiomatic choice. --- config/src/main/scala/bloop/config/Config.scala | 5 +++-- frontend/src/it/scala/bloop/CommunityBuild.scala | 2 +- frontend/src/main/scala/bloop/data/Project.scala | 4 ++-- frontend/src/main/scala/bloop/engine/BuildLoader.scala | 1 - frontend/src/test/scala/bloop/util/TestUtil.scala | 2 +- .../bloop/integrations/gradle/model/BloopConverter.scala | 2 +- .../src/main/scala/bloop/integrations/mill/MillBloop.scala | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/config/src/main/scala/bloop/config/Config.scala b/config/src/main/scala/bloop/config/Config.scala index 95e377d508..552221ba77 100644 --- a/config/src/main/scala/bloop/config/Config.scala +++ b/config/src/main/scala/bloop/config/Config.scala @@ -216,7 +216,7 @@ object Config { case class Project( name: String, directory: Path, - workspaceRoot: Option[Path], + workspaceDir: Option[Path], sources: List[Path], dependencies: List[String], classpath: List[Path], @@ -288,7 +288,8 @@ object Config { List(), Some(outAnalysisFile), Some(CompileSetup.empty) - )), + ) + ), Some(Java(List("-version"))), Some(Sbt("1.1.0", Nil)), Some(Test(List(), TestOptions(Nil, Nil))), diff --git a/frontend/src/it/scala/bloop/CommunityBuild.scala b/frontend/src/it/scala/bloop/CommunityBuild.scala index a116576360..d64166ffc6 100644 --- a/frontend/src/it/scala/bloop/CommunityBuild.scala +++ b/frontend/src/it/scala/bloop/CommunityBuild.scala @@ -122,7 +122,7 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) { val rootProject = Project( name = rootProjectName, baseDirectory = dummyExistingBaseDir, - workspaceRootDirectory = Some(buildBaseDir), + workspaceDirectory = Some(buildBaseDir), dependencies = allProjectsInBuild.map(_.project.name), scalaInstance = allProjectsInBuild.head.project.scalaInstance, rawClasspath = Nil, diff --git a/frontend/src/main/scala/bloop/data/Project.scala b/frontend/src/main/scala/bloop/data/Project.scala index 41de6f7064..9f91037f6f 100644 --- a/frontend/src/main/scala/bloop/data/Project.scala +++ b/frontend/src/main/scala/bloop/data/Project.scala @@ -23,7 +23,7 @@ import xsbti.compile.{ClasspathOptions, CompileOrder} final case class Project( name: String, baseDirectory: AbsolutePath, - workspaceRootDirectory: Option[AbsolutePath], + workspaceDirectory: Option[AbsolutePath], dependencies: List[String], scalaInstance: Option[ScalaInstance], rawClasspath: List[AbsolutePath], @@ -168,7 +168,7 @@ object Project { Project( project.name, AbsolutePath(project.directory), - project.workspaceRoot.map(AbsolutePath.apply), + project.workspaceDir.map(AbsolutePath.apply), project.dependencies, instance, project.classpath.map(AbsolutePath.apply), diff --git a/frontend/src/main/scala/bloop/engine/BuildLoader.scala b/frontend/src/main/scala/bloop/engine/BuildLoader.scala index 79e44c639c..d556052b18 100644 --- a/frontend/src/main/scala/bloop/engine/BuildLoader.scala +++ b/frontend/src/main/scala/bloop/engine/BuildLoader.scala @@ -6,7 +6,6 @@ import bloop.io.AbsolutePath import bloop.logging.{DebugFilter, Logger} import bloop.io.ByteHasher import bloop.data.WorkspaceSettings -import bloop.data.PartialLoadedBuild import bloop.data.LoadedProject import bloop.engine.caches.SemanticDBCache diff --git a/frontend/src/test/scala/bloop/util/TestUtil.scala b/frontend/src/test/scala/bloop/util/TestUtil.scala index 80a32e67d1..38ca9ed38e 100644 --- a/frontend/src/test/scala/bloop/util/TestUtil.scala +++ b/frontend/src/test/scala/bloop/util/TestUtil.scala @@ -343,7 +343,7 @@ object TestUtil { Project( name = name, baseDirectory = AbsolutePath(baseDirectory), - workspaceRootDirectory = Option(baseDir), + workspaceDirectory = Option(baseDir), dependencies = dependencies.toList, scalaInstance = scalaInstance, rawClasspath = classpath, diff --git a/integrations/gradle-bloop/src/main/scala/bloop/integrations/gradle/model/BloopConverter.scala b/integrations/gradle-bloop/src/main/scala/bloop/integrations/gradle/model/BloopConverter.scala index fbb496abb8..eb3f4ecee0 100644 --- a/integrations/gradle-bloop/src/main/scala/bloop/integrations/gradle/model/BloopConverter.scala +++ b/integrations/gradle-bloop/src/main/scala/bloop/integrations/gradle/model/BloopConverter.scala @@ -197,7 +197,7 @@ final class BloopConverter(parameters: BloopParameters) { bloopProject = Config.Project( name = getProjectName(project, sourceSet), directory = project.getProjectDir.toPath, - workspaceRoot = Option(project.getRootProject().getProjectDir().toPath()), + workspaceDir = Option(project.getRootProject().getProjectDir().toPath()), sources = sources, dependencies = allDependencies, classpath = classpath, diff --git a/integrations/mill-bloop/src/main/scala/bloop/integrations/mill/MillBloop.scala b/integrations/mill-bloop/src/main/scala/bloop/integrations/mill/MillBloop.scala index 8c05157d0d..f12e9fd769 100644 --- a/integrations/mill-bloop/src/main/scala/bloop/integrations/mill/MillBloop.scala +++ b/integrations/mill-bloop/src/main/scala/bloop/integrations/mill/MillBloop.scala @@ -99,7 +99,7 @@ object Bloop extends ExternalModule { Config.Project( name = name(module), directory = module.millSourcePath.toNIO, - workspaceRoot = Option(pwd.wrapped), + workspaceDir = Option(pwd.wrapped), sources = module.allSources().map(_.path.toNIO).toList, dependencies = module.moduleDeps.map(name).toList, classpath = classpath().map(_.toNIO).toList, From 27ef85a68e752c96d21c338fd0861a8a0a3392eb Mon Sep 17 00:00:00 2001 From: jvican Date: Tue, 30 Jul 2019 18:27:58 +0200 Subject: [PATCH 16/22] Use real workspace directory to enable semanticdb --- .../src/main/scala/bloop/data/Project.scala | 3 ++- .../scala/bloop/data/WorkspaceSettings.scala | 17 ----------------- .../main/scala/bloop/engine/BuildLoader.scala | 6 ++---- 3 files changed, 4 insertions(+), 22 deletions(-) diff --git a/frontend/src/main/scala/bloop/data/Project.scala b/frontend/src/main/scala/bloop/data/Project.scala index 9f91037f6f..537b9fffb9 100644 --- a/frontend/src/main/scala/bloop/data/Project.scala +++ b/frontend/src/main/scala/bloop/data/Project.scala @@ -226,14 +226,15 @@ object Project { */ def enableMetalsSettings( project: Project, + configDir: AbsolutePath, semanticDBPlugin: Option[AbsolutePath], - workspaceDir: AbsolutePath, logger: Logger ): Project = { def enableSemanticDB(options: List[String], pluginPath: AbsolutePath): List[String] = { val hasSemanticDB = hasSemanticDBEnabledInCompilerOptions(options) if (hasSemanticDB) options else { + val workspaceDir = project.workspaceDirectory.getOrElse(configDir.getParent) // TODO: Handle user-configured `targetroot`s inside Bloop's compilation // engine so that semanticdb files are replicated in those directories val semanticdbScalacOptions = List( diff --git a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala index 42bd5ef186..1991b6a7f2 100644 --- a/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala +++ b/frontend/src/main/scala/bloop/data/WorkspaceSettings.scala @@ -95,21 +95,4 @@ object WorkspaceSettings { case Left(failure) => throw failure } } - - /** - * Detects the workspace directory from the config dir. - * - * Bloop doesn't have the notion of workspace directory yet so this is just an - * approximation. We assume that the parent of `.bloop` is the workspace. This - * assumption is broken when source dependencies are used because we inline the - * configuration files of the projects in source dependencies into a single - * .bloop configuration directory. To fix this well-known limitation, we need - * to introduce a new field to the bloop configuration file so that we can map - * a project with a workspace irrevocably. - */ - def detectWorkspaceDirectory( - configDir: AbsolutePath - ): AbsolutePath = { - configDir.getParent - } } diff --git a/frontend/src/main/scala/bloop/engine/BuildLoader.scala b/frontend/src/main/scala/bloop/engine/BuildLoader.scala index d556052b18..51e96b1bca 100644 --- a/frontend/src/main/scala/bloop/engine/BuildLoader.scala +++ b/frontend/src/main/scala/bloop/engine/BuildLoader.scala @@ -120,12 +120,11 @@ object BuildLoader { } } - val workspace = WorkspaceSettings.detectWorkspaceDirectory(configDir) val enableMetalsInProjectsTask = groupedProjectsPerScalaVersion.toList.map { case (scalaVersion, projects) => def enableMetalsTask(plugin: Option[AbsolutePath]) = { Task( - projects.map(p => Project.enableMetalsSettings(p, plugin, workspace, logger) -> Some(p)) + projects.map(p => Project.enableMetalsSettings(p, configDir, plugin, logger) -> Some(p)) ) } @@ -186,10 +185,9 @@ object BuildLoader { project.scalaInstance match { case None => LoadedProject.RawProject(project) case Some(instance) => - val workspace = WorkspaceSettings.detectWorkspaceDirectory(configDir) def enableMetals(plugin: Option[AbsolutePath]) = { LoadedProject.ConfiguredProject( - Project.enableMetalsSettings(project, plugin, workspace, logger), + Project.enableMetalsSettings(project, configDir, plugin, logger), project, settings ) From 745b15408606d8c68f846eb146f9233fc15c8668 Mon Sep 17 00:00:00 2001 From: jvican Date: Tue, 30 Jul 2019 18:33:07 +0200 Subject: [PATCH 17/22] Remove unnecessary `Project.pprint` --- frontend/src/main/scala/bloop/data/Project.scala | 4 ---- frontend/src/main/scala/bloop/engine/Feedback.scala | 4 +++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/frontend/src/main/scala/bloop/data/Project.scala b/frontend/src/main/scala/bloop/data/Project.scala index 537b9fffb9..7de22c2ac8 100644 --- a/frontend/src/main/scala/bloop/data/Project.scala +++ b/frontend/src/main/scala/bloop/data/Project.scala @@ -203,10 +203,6 @@ object Project { } } - def pprint(projects: Traversable[Project]): String = { - projects.map(p => s"'${p.name}'").mkString(", ") - } - /** * Enable any Metals-specific setting in a project by applying an in-memory * project transformation. A setting is Metals-specific if it's required for diff --git a/frontend/src/main/scala/bloop/engine/Feedback.scala b/frontend/src/main/scala/bloop/engine/Feedback.scala index 76f7016dbd..dc68596ea0 100644 --- a/frontend/src/main/scala/bloop/engine/Feedback.scala +++ b/frontend/src/main/scala/bloop/engine/Feedback.scala @@ -93,10 +93,12 @@ object Feedback { def unknownHostName(host: String): String = s"Host name '$host' could not be either parsed or resolved" + def pprint(projects: Traversable[Project]): String = + projects.map(p => s"'${p.name}'").mkString(", ") def skippedUnsupportedScalaMetals(scalaVersion: String): String = s"Skipped configuration of SemanticDB in unsupported $scalaVersion projects" def configuredMetalsProjects(projects: Traversable[Project]): String = - s"Configured SemanticDB in projects ${Project.pprint(projects)}" + s"Configured SemanticDB in projects ${pprint(projects)}" def failedMetalsConfiguration(version: String, cause: String): String = s"Stopped configuration of SemanticDB in Scala $version projects: $cause" From 2972653abed9080369401a9aae0b442bd6f50467 Mon Sep 17 00:00:00 2001 From: jvican Date: Tue, 30 Jul 2019 18:35:48 +0200 Subject: [PATCH 18/22] Rename misnomer in `pickSettingsForReload` method --- frontend/src/main/scala/bloop/engine/Build.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/main/scala/bloop/engine/Build.scala b/frontend/src/main/scala/bloop/engine/Build.scala index ddbeafee7b..42de0e826a 100644 --- a/frontend/src/main/scala/bloop/engine/Build.scala +++ b/frontend/src/main/scala/bloop/engine/Build.scala @@ -59,7 +59,7 @@ final case class Build private ( val newToAttributed = newFiles.iterator.map(ap => ap.path -> ap).toMap val currentSettings = WorkspaceSettings.readFromFile(origin, logger) - val settingsForReload = pickAndPersistSettingsForReload(currentSettings, newSettings, logger) + val settingsForReload = pickSettingsForReload(currentSettings, newSettings, logger) val changedSettings = currentSettings != settingsForReload val filesToProjects = loadedProjects.iterator.map(lp => lp.project.origin.path -> lp).toMap @@ -169,7 +169,7 @@ final case class Build private ( } } - def pickAndPersistSettingsForReload( + def pickSettingsForReload( currentSettings: Option[WorkspaceSettings], newSettings: Option[WorkspaceSettings], logger: Logger From eba47e19e6959cf75474afa64ce3c29241fcf5cc Mon Sep 17 00:00:00 2001 From: jvican Date: Tue, 30 Jul 2019 18:44:01 +0200 Subject: [PATCH 19/22] Use workspace dir from project in tests too --- frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala index c4bad26356..e379934d2a 100644 --- a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala @@ -375,7 +375,10 @@ class BspMetalsClientSpec( unorderedExpectedOptions: String ): Unit = { // Not the best way to obtain workspace but valid for tests - val workspaceDir = state.underlying.build.origin.getParent.syntax + val workspaceDir = project.config.workspaceDir + .map(AbsolutePath(_)) + .getOrElse(state.underlying.build.origin.getParent) + .syntax val scalacOptions = state.scalaOptions(project)._2.items.flatMap(_.options).map { opt => if (!opt.startsWith("-Xplugin:")) opt else { From eb9338937e1e64324e4cadabee55dd897b92d126 Mon Sep 17 00:00:00 2001 From: jvican Date: Tue, 30 Jul 2019 20:17:40 +0200 Subject: [PATCH 20/22] Fix error computing `reattemptConfiguration` --- frontend/src/main/scala/bloop/engine/Build.scala | 2 +- frontend/src/test/scala/bloop/BuildLoaderSpec.scala | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/frontend/src/main/scala/bloop/engine/Build.scala b/frontend/src/main/scala/bloop/engine/Build.scala index 42de0e826a..e9b0461030 100644 --- a/frontend/src/main/scala/bloop/engine/Build.scala +++ b/frontend/src/main/scala/bloop/engine/Build.scala @@ -112,7 +112,7 @@ final case class Build private ( case Build.AvoidReload(_) => val options = project.scalacOptions val reattemptConfiguration = newSettings.nonEmpty && { - Project.hasSemanticDBEnabledInCompilerOptions(project.scalacOptions) + !Project.hasSemanticDBEnabledInCompilerOptions(project.scalacOptions) } if (reattemptConfiguration) { diff --git a/frontend/src/test/scala/bloop/BuildLoaderSpec.scala b/frontend/src/test/scala/bloop/BuildLoaderSpec.scala index 5c4289885d..2c5414ce1f 100644 --- a/frontend/src/test/scala/bloop/BuildLoaderSpec.scala +++ b/frontend/src/test/scala/bloop/BuildLoaderSpec.scala @@ -39,7 +39,6 @@ object BuildLoaderSpec extends BaseSuite { testBuild.state.build.checkForChange(Some(sameSettings), logger).map { case Build.ReturnPreviousState => () case action: Build.UpdateState => - pprint.log(action.invalidated) sys.error(s"Expected return previous state, got updated state") } } From 30927d78a9bfe564c1840f8656e36c2d4d982df7 Mon Sep 17 00:00:00 2001 From: Jorge Vicente Cantero Date: Wed, 31 Jul 2019 12:03:11 +0200 Subject: [PATCH 21/22] Return `ForceReload` when new settings are added --- frontend/src/main/scala/bloop/engine/Build.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/main/scala/bloop/engine/Build.scala b/frontend/src/main/scala/bloop/engine/Build.scala index e9b0461030..e1caeb87ce 100644 --- a/frontend/src/main/scala/bloop/engine/Build.scala +++ b/frontend/src/main/scala/bloop/engine/Build.scala @@ -197,7 +197,8 @@ final case class Build private ( if currentSettings.semanticDBVersion != newSettings.semanticDBVersion => Build.ForceReload(newSettings, List(WorkspaceSettings.SemanticDBVersionChange)) case (Some(_), Some(newSettings)) => Build.AvoidReload(Some(newSettings)) - case (None, Some(newSettings)) => Build.AvoidReload(Some(newSettings)) + case (None, Some(newSettings)) => + Build.ForceReload(newSettings, List(WorkspaceSettings.SemanticDBVersionChange)) case (Some(currentSettings), None) => Build.AvoidReload(Some(currentSettings)) case (None, None) => Build.AvoidReload(None) } From 3f0a295621703a14bd522ec611ca65550a20c679 Mon Sep 17 00:00:00 2001 From: Jorge Vicente Cantero Date: Wed, 31 Jul 2019 12:30:35 +0200 Subject: [PATCH 22/22] Use `Coeval` to abstract over logic applying semanticdb --- .../src/main/scala/bloop/data/Project.scala | 1 - .../main/scala/bloop/engine/BuildLoader.scala | 95 +++++++++++-------- 2 files changed, 55 insertions(+), 41 deletions(-) diff --git a/frontend/src/main/scala/bloop/data/Project.scala b/frontend/src/main/scala/bloop/data/Project.scala index 7de22c2ac8..da79be846c 100644 --- a/frontend/src/main/scala/bloop/data/Project.scala +++ b/frontend/src/main/scala/bloop/data/Project.scala @@ -255,7 +255,6 @@ object Project { semanticDBPlugin match { case None => projectWithRangePositions case Some(pluginPath) => - // Recognize 2.12.8-abdcddd as supported if 2.12.8 exists in supported versions val options = projectWithRangePositions.scalacOptions val optionsWithSemanticDB = enableSemanticDB(options, pluginPath) projectWithRangePositions.copy(scalacOptions = optionsWithSemanticDB) diff --git a/frontend/src/main/scala/bloop/engine/BuildLoader.scala b/frontend/src/main/scala/bloop/engine/BuildLoader.scala index 51e96b1bca..26738257ee 100644 --- a/frontend/src/main/scala/bloop/engine/BuildLoader.scala +++ b/frontend/src/main/scala/bloop/engine/BuildLoader.scala @@ -9,8 +9,9 @@ import bloop.data.WorkspaceSettings import bloop.data.LoadedProject import bloop.engine.caches.SemanticDBCache -import monix.eval.Task +import monix.eval.{Task, Coeval} import scala.collection.mutable +import bloop.ScalaInstance object BuildLoader { @@ -122,27 +123,19 @@ object BuildLoader { val enableMetalsInProjectsTask = groupedProjectsPerScalaVersion.toList.map { case (scalaVersion, projects) => - def enableMetalsTask(plugin: Option[AbsolutePath]) = { - Task( - projects.map(p => Project.enableMetalsSettings(p, configDir, plugin, logger) -> Some(p)) + val coeval = tryEnablingSemanticDB( + projects, + configDir, + scalaVersion, + semanticDBVersion, + supportedScalaVersions, + logger + ) { (plugin: Option[AbsolutePath]) => + projects.map( + p => Project.enableMetalsSettings(p, configDir, plugin, logger) -> Some(p) ) } - - // Recognize 2.12.8-abdcddd as supported if 2.12.8 exists in supported versions - val isUnsupportedVersion = !supportedScalaVersions.exists(scalaVersion.startsWith(_)) - if (isUnsupportedVersion) { - logger.debug(Feedback.skippedUnsupportedScalaMetals(scalaVersion))(DebugFilter.All) - enableMetalsTask(None) - } else { - SemanticDBCache.fetchPlugin(scalaVersion, semanticDBVersion, logger) match { - case Right(path) => - logger.debug(Feedback.configuredMetalsProjects(projects))(DebugFilter.All) - enableMetalsTask(Some(path)) - case Left(cause) => - logger.displayWarningToUser(Feedback.failedMetalsConfiguration(scalaVersion, cause)) - enableMetalsTask(None) - } - } + coeval.task } Task.gatherUnordered(enableMetalsInProjectsTask).map { pps => @@ -185,7 +178,15 @@ object BuildLoader { project.scalaInstance match { case None => LoadedProject.RawProject(project) case Some(instance) => - def enableMetals(plugin: Option[AbsolutePath]) = { + val scalaVersion = instance.version + val coeval = tryEnablingSemanticDB( + List(project), + configDir, + scalaVersion, + settings.semanticDBVersion, + settings.supportedScalaVersions, + logger + ) { (plugin: Option[AbsolutePath]) => LoadedProject.ConfiguredProject( Project.enableMetalsSettings(project, configDir, plugin, logger), project, @@ -193,31 +194,45 @@ object BuildLoader { ) } - val scalaVersion = instance.version - import settings.{supportedScalaVersions, semanticDBVersion} - - // Recognize 2.12.8-abdcddd as supported if 2.12.8 exists in supported versions - val isUnsupportedVersion = !supportedScalaVersions.exists(scalaVersion.startsWith(_)) - if (isUnsupportedVersion) { - logger.debug(Feedback.skippedUnsupportedScalaMetals(scalaVersion))(DebugFilter.All) - enableMetals(None) - } else { - SemanticDBCache.fetchPlugin(scalaVersion, semanticDBVersion, logger) match { - case Right(path) => - logger.debug(Feedback.configuredMetalsProjects(List(project)))(DebugFilter.All) - enableMetals(Some(path)) - case Left(cause) => - logger.displayWarningToUser( - Feedback.failedMetalsConfiguration(scalaVersion, cause) - ) - enableMetals(None) - } + // Run coeval, we rethrow but note that `tryEnablingSemanticDB` handles errors + coeval.run match { + case Left(value) => throw value + case Right(value) => value } } } } } + private def tryEnablingSemanticDB[T]( + projects: List[Project], + configDir: AbsolutePath, + scalaVersion: String, + semanticDBVersion: String, + supportedScalaVersions: List[String], + logger: Logger + )( + enableMetals: Option[AbsolutePath] => T + ): Coeval[T] = { + // Recognize 2.12.8-abdcddd as supported if 2.12.8 exists in supported versions + val isUnsupportedVersion = !supportedScalaVersions.exists(scalaVersion.startsWith(_)) + if (isUnsupportedVersion) { + logger.debug(Feedback.skippedUnsupportedScalaMetals(scalaVersion))(DebugFilter.All) + Coeval.now(enableMetals(None)) + } else { + Coeval.eval { + SemanticDBCache.fetchPlugin(scalaVersion, semanticDBVersion, logger) match { + case Right(path) => + logger.debug(Feedback.configuredMetalsProjects(projects))(DebugFilter.All) + enableMetals(Some(path)) + case Left(cause) => + logger.displayWarningToUser(Feedback.failedMetalsConfiguration(scalaVersion, cause)) + enableMetals(None) + } + } + } + } + private def loadProject(bytes: Array[Byte], origin: Origin, logger: Logger): Project = { Project.fromBytesAndOrigin(bytes, origin, logger) }