Merge pull request #1650 from scala-steward-org/topic/load-latest-sca…

…lafix-migrations

Load Scalafix migrations from this repository

docs/scalafix-migrations.md

@@ -31,21 +31,21 @@ which resolves those rules and edits the source code accordingly.
 See the [Scalafix Developer Guide][scalafix-dev-guide] for more information
 about writing rewrite rules or have a look at the existing
 [migration rules][migrations] for inspiration. Rules in Scala Steward must be
-accessible via the [github:][using-github], the [http:][using-http] or the [dependency:][using-dependency] schemes.
+accessible via the [github:][using-github], the [http:][using-http] or the
+[dependency:][using-dependency] schemes.
 
 ## Adding migration rules to Scala Steward
 
 After you have written a new migration rule for a new version of your project,
 Scala Steward needs to be made aware of it. Creating a pull request that adds
 the new rule to the list of [migrations][migrations] is enough for that. Once
 that pull request is merged, Scala Steward will start using this migration.
-When running Scala Steward you can specify a file containing extra migrations
-that might not be present in the [default list][migrations].
-You can also specify if you want the default list to be disabled.
 
-The file is in [HOCON][HOCON] and should look like this:
+When running Scala Steward you can also specify files or URLs (via the
+`--scalafix-migrations` command-line option) that contain additional
+migrations which are not present in the [default list][migrations].
+These files are in [HOCON][HOCON] format and should look like this:
 ```hocon
-disableDefaults = true
 migrations = [
   {
     groupId: "org.http4s",

modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala

@@ -43,7 +43,8 @@ object Cli {
       ignoreOptsFiles: Boolean = false,
       envVar: List[EnvVar] = Nil,
       processTimeout: FiniteDuration = 10.minutes,
-      scalafixMigrations: Option[String] = None,
+      scalafixMigrations: List[Uri] = Nil,
+      disableDefaultScalafixMigrations: Boolean = false,
       groupMigrations: Option[String] = None,
       cacheTtl: FiniteDuration = 2.hours,
       cacheMissDelay: FiniteDuration = 0.milliseconds,

modules/core/src/main/scala/org/scalasteward/core/application/Config.scala

@@ -21,6 +21,7 @@ import cats.effect.Sync
 import org.http4s.Uri
 import org.http4s.Uri.UserInfo
 import org.scalasteward.core.application.Cli.EnvVar
+import org.scalasteward.core.application.Config.Scalafix
 import org.scalasteward.core.git.Author
 import org.scalasteward.core.util
 import org.scalasteward.core.vcs.data.AuthenticatedUser
@@ -68,7 +69,7 @@ final case class Config(
     ignoreOptsFiles: Boolean,
     envVars: List[EnvVar],
     processTimeout: FiniteDuration,
-    scalafixMigrations: Option[File],
+    scalafix: Scalafix,
     groupMigrations: Option[File],
     cacheTtl: FiniteDuration,
     cacheMissDelay: FiniteDuration,
@@ -86,6 +87,11 @@ final case class Config(
 }
 
 object Config {
+  final case class Scalafix(
+      migrations: List[Uri],
+      disableDefaults: Boolean
+  )
+
   def create[F[_]](args: Cli.Args)(implicit F: Sync[F]): F[Config] =
     F.delay {
       Config(
@@ -105,7 +111,7 @@ object Config {
         ignoreOptsFiles = args.ignoreOptsFiles,
         envVars = args.envVar,
         processTimeout = args.processTimeout,
-        scalafixMigrations = args.scalafixMigrations.map(_.toFile),
+        scalafix = Scalafix(args.scalafixMigrations, args.disableDefaultScalafixMigrations),
         groupMigrations = args.groupMigrations.map(_.toFile),
         cacheTtl = args.cacheTtl,
         cacheMissDelay = args.cacheMissDelay,

modules/core/src/main/scala/org/scalasteward/core/application/Context.scala

@@ -18,6 +18,7 @@ package org.scalasteward.core.application
 
 import cats.Parallel
 import cats.effect._
+import cats.syntax.all._
 import io.chrisdavenport.log4cats.Logger
 import io.chrisdavenport.log4cats.slf4j.Slf4jLogger
 import org.http4s.client.Client
@@ -34,7 +35,7 @@ import org.scalasteward.core.nurture.{NurtureAlg, PullRequestRepository}
 import org.scalasteward.core.persistence.JsonKeyValueStore
 import org.scalasteward.core.repocache.{RefreshErrorAlg, RepoCacheAlg, RepoCacheRepository}
 import org.scalasteward.core.repoconfig.RepoConfigAlg
-import org.scalasteward.core.scalafix.MigrationAlg
+import org.scalasteward.core.scalafix.{MigrationAlg, MigrationsLoader}
 import org.scalasteward.core.scalafmt.ScalafmtAlg
 import org.scalasteward.core.update.{FilterAlg, GroupMigrations, PruningAlg, UpdateAlg}
 import org.scalasteward.core.util._
@@ -48,15 +49,15 @@ object Context {
   ): Resource[F, StewardAlg[F]] =
     for {
       blocker <- Blocker[F]
+      implicit0(logger: Logger[F]) <- Resource.liftF(Slf4jLogger.create[F])
+      _ <- Resource.liftF(printBanner[F])
       implicit0(config: Config) <- Resource.liftF(Config.create[F](args))
       implicit0(client: Client[F]) <- AsyncHttpClient.resource[F]()
-      implicit0(logger: Logger[F]) <- Resource.liftF(Slf4jLogger.create[F])
       implicit0(httpExistenceClient: HttpExistenceClient[F]) <- HttpExistenceClient.create[F]
       implicit0(user: AuthenticatedUser) <- Resource.liftF(config.vcsUser[F])
       implicit0(fileAlg: FileAlg[F]) = FileAlg.create[F]
-      implicit0(migrationAlg: MigrationAlg) <- Resource.liftF(
-        MigrationAlg.create[F](config.scalafixMigrations)
-      )
+      implicit0(migrationAlg: MigrationAlg) <-
+        Resource.liftF(new MigrationsLoader[F].loadAll(config.scalafix).map(new MigrationAlg(_)))
       implicit0(groupMigration: GroupMigrations) <- Resource.liftF(GroupMigrations.create[F])
     } yield {
       val kvsPrefix = Some(config.vcsType.asString)
@@ -93,4 +94,16 @@ object Context {
       implicit val pruningAlg: PruningAlg[F] = new PruningAlg[F]
       new StewardAlg[F]
     }
+
+  private def printBanner[F[_]](implicit logger: Logger[F]): F[Unit] = {
+    val banner =
+      """|  ____            _         ____  _                             _
+         | / ___|  ___ __ _| | __ _  / ___|| |_ _____      ____ _ _ __ __| |
+         | \___ \ / __/ _` | |/ _` | \___ \| __/ _ \ \ /\ / / _` | '__/ _` |
+         |  ___) | (_| (_| | | (_| |  ___) | ||  __/\ V  V / (_| | | | (_| |
+         | |____/ \___\__,_|_|\__,_| |____/ \__\___| \_/\_/ \__,_|_|  \__,_|""".stripMargin
+    val msg = List(" ", banner, s" v${org.scalasteward.core.BuildInfo.version}", " ")
+      .mkString(System.lineSeparator())
+    logger.info(msg)
+  }
 }

modules/core/src/main/scala/org/scalasteward/core/application/StewardAlg.scala

@@ -47,18 +47,6 @@ final class StewardAlg[F[_]](implicit
     workspaceAlg: WorkspaceAlg[F],
     F: BracketThrowable[F]
 ) {
-  private def printBanner: F[Unit] = {
-    val banner =
-      """|  ____            _         ____  _                             _
-         | / ___|  ___ __ _| | __ _  / ___|| |_ _____      ____ _ _ __ __| |
-         | \___ \ / __/ _` | |/ _` | \___ \| __/ _ \ \ /\ / / _` | '__/ _` |
-         |  ___) | (_| (_| | | (_| |  ___) | ||  __/\ V  V / (_| | | | (_| |
-         | |____/ \___\__,_|_|\__,_| |____/ \__\___| \_/\_/ \__,_|_|  \__,_|""".stripMargin
-    val msg = List(" ", banner, s" v${org.scalasteward.core.BuildInfo.version}", " ")
-      .mkString(System.lineSeparator())
-    logger.info(msg)
-  }
-
   private def readRepos(reposFile: File): Stream[F, Repo] =
     Stream.evals {
       fileAlg.readFile(reposFile).map { maybeContent =>
@@ -88,7 +76,6 @@ final class StewardAlg[F[_]](implicit
   def runF: F[ExitCode] =
     logger.infoTotalTime("run") {
       for {
-        _ <- printBanner
         _ <- selfCheckAlg.checkAll
         _ <- workspaceAlg.cleanWorkspace
         exitCode <- sbtAlg.addGlobalPlugins {

modules/core/src/main/scala/org/scalasteward/core/io/FileAlg.scala

@@ -23,6 +23,8 @@ import cats.{Functor, Traverse}
 import fs2.Stream
 import io.chrisdavenport.log4cats.Logger
 import org.apache.commons.io.FileUtils
+import org.http4s.Uri
+import org.http4s.implicits.http4sLiteralsSyntax
 import org.scalasteward.core.util.MonadThrowable
 import scala.io.Source
 
@@ -43,6 +45,8 @@ trait FileAlg[F[_]] {
 
   def readResource(resource: String): F[String]
 
+  def readUri(uri: Uri): F[String]
+
   def walk(dir: File): Stream[F, File]
 
   def writeFile(file: File, content: String): F[Unit]
@@ -125,9 +129,16 @@ object FileAlg {
         F.delay(if (file.exists) Some(file.contentAsString) else None)
 
       override def readResource(resource: String): F[String] =
-        Resource
-          .fromAutoCloseable(F.delay(Source.fromResource(resource)))
-          .use(src => F.delay(src.mkString))
+        readSource(Source.fromResource(resource))
+
+      override def readUri(uri: Uri): F[String] = {
+        val scheme = uri.scheme.getOrElse(scheme"file")
+        val withScheme = uri.copy(scheme = Some(scheme))
+        readSource(Source.fromURL(withScheme.renderString))
+      }
+
+      private def readSource(source: => Source): F[String] =
+        Resource.fromAutoCloseable(F.delay(source)).use(src => F.delay(src.mkString))
 
       override def walk(dir: File): Stream[F, File] =
         Stream.eval(F.delay(dir.walk())).flatMap(Stream.fromIterator(_))

modules/core/src/main/scala/org/scalasteward/core/scalafix/MigrationAlg.scala

@@ -16,64 +16,16 @@
 
 package org.scalasteward.core.scalafix
 
-import better.files.File
-import cats.data.OptionT
-import cats.effect.Sync
 import cats.syntax.all._
-import io.circe.config.parser.decode
 import org.scalasteward.core.data.{Update, Version}
-import org.scalasteward.core.io.FileAlg
-import org.scalasteward.core.util.{ApplicativeThrowable, MonadThrowable}
 
-trait MigrationAlg {
-  def findMigrations(update: Update): List[Migration]
-}
-
-object MigrationAlg {
-  def create[F[_]](extraMigrations: Option[File])(implicit
-      fileAlg: FileAlg[F],
-      F: Sync[F]
-  ): F[MigrationAlg] =
-    loadMigrations(extraMigrations).map { migrations =>
-      new MigrationAlg {
-        override def findMigrations(update: Update): List[Migration] =
-          findMigrationsImpl(migrations, update)
-      }
-    }
-
-  def loadMigrations[F[_]](
-      extraMigrations: Option[File]
-  )(implicit fileAlg: FileAlg[F], F: MonadThrowable[F]): F[List[Migration]] =
-    for {
-      default <-
-        fileAlg
-          .readResource("scalafix-migrations.conf")
-          .flatMap(decodeMigrations[F](_, "default"))
-      maybeExtra <- OptionT(extraMigrations.flatTraverse(fileAlg.readFile))
-        .semiflatMap(decodeMigrations[F](_, "extra"))
-        .value
-      migrations = maybeExtra match {
-        case Some(extra) if extra.disableDefaults => extra.migrations
-        case Some(extra)                          => default.migrations ++ extra.migrations
-        case None                                 => default.migrations
-      }
-    } yield migrations
-
-  private def decodeMigrations[F[_]](content: String, tpe: String)(implicit
-      F: ApplicativeThrowable[F]
-  ): F[ScalafixMigrations] =
-    F.fromEither(decode[ScalafixMigrations](content))
-      .adaptErr(new Throwable(s"Failed to load $tpe Scalafix migrations", _))
-
-  private def findMigrationsImpl(
-      givenMigrations: List[Migration],
-      update: Update
-  ): List[Migration] =
-    givenMigrations.filter { migration =>
+final class MigrationAlg(migrations: List[Migration]) {
+  def findMigrations(update: Update): List[Migration] =
+    migrations.filter { migration =>
       update.groupId === migration.groupId &&
-      migration.artifactIds.exists(re =>
+      migration.artifactIds.exists { re =>
         update.artifactIds.exists(artifactId => re.r.findFirstIn(artifactId.name).isDefined)
-      ) &&
+      } &&
       Version(update.currentVersion) < migration.newVersion &&
       Version(update.newerVersions.head) >= migration.newVersion
     }

modules/core/src/main/scala/org/scalasteward/core/scalafix/MigrationsLoader.scala

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2018-2020 Scala Steward contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.scalasteward.core.scalafix
+
+import cats.syntax.all._
+import io.chrisdavenport.log4cats.Logger
+import io.circe.config.parser.decode
+import org.http4s.Uri
+import org.http4s.implicits.http4sLiteralsSyntax
+import org.scalasteward.core.application.Config
+import org.scalasteward.core.io.FileAlg
+import org.scalasteward.core.scalafix.MigrationsLoader._
+import org.scalasteward.core.util.MonadThrowable
+
+final class MigrationsLoader[F[_]](implicit
+    fileAlg: FileAlg[F],
+    logger: Logger[F],
+    F: MonadThrowable[F]
+) {
+  def loadAll(config: Config.Scalafix): F[List[Migration]] = {
+    val maybeDefaultMigrationsUrl =
+      Option.unless(config.disableDefaults)(defaultScalafixMigrationsUrl)
+    (maybeDefaultMigrationsUrl.toList ++ config.migrations)
+      .flatTraverse(loadMigrations)
+      .flatTap(migrations => logger.info(s"Loaded ${migrations.size} Scalafix migrations"))
+  }
+
+  private def loadMigrations(uri: Uri): F[List[Migration]] =
+    logger.debug(s"Loading Scalafix migrations from $uri") >>
+      fileAlg.readUri(uri).flatMap(decodeMigrations(_, uri)).map(_.migrations)
+
+  private def decodeMigrations(content: String, uri: Uri): F[ScalafixMigrations] =
+    F.fromEither(decode[ScalafixMigrations](content))
+      .adaptErr(new Throwable(s"Failed to load Scalafix migrations from ${uri.renderString}", _))
+}
+
+object MigrationsLoader {
+  val defaultScalafixMigrationsUrl: Uri =
+    uri"https://raw.githubusercontent.com/scala-steward-org/scala-steward/master/modules/core/src/main/resources/scalafix-migrations.conf"
+}

modules/core/src/main/scala/org/scalasteward/core/scalafix/ScalafixMigrations.scala

@@ -21,7 +21,6 @@ import io.circe.generic.extras.Configuration
 import io.circe.generic.extras.semiauto._
 
 final case class ScalafixMigrations(
-    disableDefaults: Boolean = false,
     migrations: List[Migration] = List.empty
 )
 

modules/core/src/test/scala/org/scalasteward/core/buildtool/maven/MavenAlgTest.scala

@@ -19,7 +19,7 @@ class MavenAlgTest extends AnyFunSuite with Matchers {
     val state =
       mavenAlg.getDependencies(repo).runS(MockState.empty.copy(files = files)).unsafeRunSync()
 
-    state shouldBe MockState(
+    state shouldBe MockState.empty.copy(
       files = files,
       logs = Vector.empty,
       commands = Vector(

modules/core/src/test/scala/org/scalasteward/core/io/FileAlgTest.scala

@@ -2,6 +2,7 @@ package org.scalasteward.core.io
 
 import better.files.File
 import cats.effect.IO
+import org.http4s.Uri
 import org.scalacheck.Arbitrary
 import org.scalasteward.core.TestInstances.ioLogger
 import org.scalasteward.core.io.FileAlgTest.ioFileAlg
@@ -101,6 +102,16 @@ class FileAlgTest extends AnyFunSuite with Matchers {
     } yield symlinkExists
     p.unsafeRunSync() shouldBe false
   }
+
+  test("readUri: local file without scheme") {
+    val file = File.temp / "steward" / "readUri.txt"
+    val content = "42"
+    val p = for {
+      _ <- ioFileAlg.writeFile(file, content)
+      read <- ioFileAlg.readUri(Uri.unsafeFromString(file.toString))
+    } yield read
+    p.unsafeRunSync() shouldBe content
+  }
 }
 
 object FileAlgTest {

modules/core/src/test/scala/org/scalasteward/core/io/MockFileAlg.scala

@@ -4,6 +4,7 @@ import better.files.File
 import cats.data.StateT
 import cats.effect.IO
 import fs2.Stream
+import org.http4s.Uri
 import org.scalasteward.core.mock.{applyPure, MockEff, MockState}
 
 class MockFileAlg extends FileAlg[MockEff] {
@@ -42,6 +43,16 @@ class MockFileAlg extends FileAlg[MockEff] {
       content <- StateT.liftF(FileAlgTest.ioFileAlg.readResource(resource))
     } yield content
 
+  override def readUri(uri: Uri): MockEff[String] =
+    for {
+      _ <- StateT.modify[IO, MockState](_.exec(List("read", uri.renderString)))
+      s <- StateT.get[IO, MockState]
+      content <- StateT.liftF[IO, MockState, String](s.uris.get(uri) match {
+        case Some(content) => IO.pure(content)
+        case None          => IO.raiseError(new Throwable("URL not found"))
+      })
+    } yield content
+
   override def walk(dir: File): Stream[MockEff, File] = {
     val dirAsString = dir.pathAsString
     val state: MockEff[List[File]] = StateT.inspect {