Implement new patch infrastructure.

This commit is too big, I'm sorry. Main changes:

- Patch(from, to, replace) replaced with TokenPatch(token, newToken)
  with helpful combinators such as Add{Left,Right}
- New high-level TreePatch datatype with can be converted to TokenPatch.
- Implemented two tree patches, AddGlobalImport and RemoveGlobalImport.
- Implemented OrganizeImport, which is a prerequisite to use import tree
  patches.
- OrganizeImports
  - orders imports by configurable groups
  - removes unused imports using scalac -Ywarn-unused-import
    infrastructure. The implementation is requires hijacking a few
    private fields in g.analyzer using evil reflection hackery.
  - option to expand relative imports
  - handles renames
  - configurable "always used" to force keeping an import such as
    acyclic.file

Known bugs:

- introduces false import on non-existing cats.data.UUID in circe
- makes an import unused by expanding relative imports to
  fully-qualified imports, requiring you to run scalafix
  twice to fully remove unused imports (e.g. in cats with
  -Ywarn-unused-import)
- crashes on scala.meta parser bug (in akka), not scalafix problem
  really.

Expand relative

Test that patch is unchanged

.scalafix.conf

@@ -0,0 +1 @@
+imports.optimize = true

bin/scalacfix

@@ -0,0 +1 @@
+scalac -Ywarn-unused-import -Yrangepos -Xplugin:/Users/ollie/.ivy2/local/ch.epfl.scala/scalafix-nsc_2.11/0.2.0-SNAPSHOT/jars/scalafix-nsc_2.11.jar $1

build.sbt

@@ -48,6 +48,7 @@ lazy val commonSettings = Seq(
  triggeredMessage in ThisBuild := Watched.clearWhenTriggered,
  scalacOptions := compilerOptions,
  scalacOptions in (Compile, console) := compilerOptions :+ "-Yrepl-class-based",
+ test in assembly := {},
  testOptions in Test += Tests.Argument("-oD")
 )
 
@@ -146,10 +147,7 @@ lazy val core = project
  "com.typesafe" % "config" % "1.3.1",
  "com.lihaoyi" %% "sourcecode" % "0.1.3",
  "org.scalameta" %% "scalameta" % Build.metaV,
- "org.scala-lang" % "scala-reflect" % scalaVersion.value,
- // Test dependencies
- "org.scalatest" %% "scalatest" % Build.testV % "test",
- "com.googlecode.java-diff-utils" % "diffutils" % "1.3.0" % "test"
+ "org.scala-lang" % "scala-reflect" % scalaVersion.value
  )
  )
  .enablePlugins(BuildInfoPlugin)
@@ -160,9 +158,17 @@ lazy val `scalafix-nsc` = project
  scalaVersion := "2.11.8",
  crossScalaVersions := crossVersions,
  libraryDependencies ++= Seq(
- "org.scala-lang" % "scala-compiler" % scalaVersion.value,
- "org.scalameta" %% "scalameta" % Build.metaV % "provided",
- "org.scalatest" %% "scalatest" % Build.testV % Test
+ "org.scala-lang" % "scala-compiler" % scalaVersion.value,
+ "org.scalameta" %% "scalameta" % Build.metaV % "provided",
+ "com.googlecode.java-diff-utils" % "diffutils" % "1.3.0" % "test",
+ "org.scalatest" %% "scalatest" % Build.testV % Test,
+ "com.lihaoyi" %% "ammonite-ops" % Build.ammoniteV % Test,
+ // integration property tests
+ "org.typelevel" %% "catalysts-platform" % "0.0.5" % Test,
+ "com.typesafe.slick" %% "slick" % "3.2.0-M2" % Test,
+ "io.circe" %% "circe-core" % "0.6.0" % Test,
+ "org.typelevel" %% "cats-core" % "0.7.2" % Test,
+ "org.scalacheck" %% "scalacheck" % "1.13.4" % Test
  ),
  // sbt does not fetch transitive dependencies of compiler plugins.
  // to overcome this issue, all transitive dependencies are included
@@ -256,7 +262,7 @@ lazy val `scalafix-tests` = project
  )
  .value,
  libraryDependencies ++= Seq(
- "com.lihaoyi" %% "ammonite-ops" % "0.8.0",
+ "com.lihaoyi" %% "ammonite-ops" % Build.ammoniteV,
  "org.scalatest" %% "scalatest" % Build.testV % "test"
  )
  )

core/src/main/scala/scalafix/FilterMatcher.scala

@@ -0,0 +1,27 @@
+package scalafix
+
+import scala.util.matching.Regex
+import scalafix.util.AbsoluteFile
+
+case class FilterMatcher(include: Regex, exclude: Regex) {
+ def matches(file: AbsoluteFile): Boolean = matches(file.path)
+ def matches(input: String): Boolean =
+ include.findFirstIn(input).isDefined &&
+ exclude.findFirstIn(input).isEmpty
+}
+
+object FilterMatcher {
+ val matchEverything = new FilterMatcher(".*".r, mkRegexp(Nil))
+
+ def mkRegexp(filters: Seq[String]): Regex =
+ filters match {
+ case Nil => "$a".r // will never match anything
+ case head :: Nil => head.r
+ case _ => filters.mkString("(", "|", ")").r
+ }
+
+ def apply(includes: Seq[String], excludes: Seq[String]): FilterMatcher =
+ new FilterMatcher(mkRegexp(includes), mkRegexp(excludes))
+ def apply(include: String): FilterMatcher =
+ new FilterMatcher(mkRegexp(Seq(include)), mkRegexp(Nil))
+}

core/src/main/scala/scalafix/Scalafix.scala

@@ -1,23 +1,30 @@
 package scalafix
 
-import scala.meta.parsers.Parsed
+import scala.collection.immutable.Seq
 import scala.meta._
 import scala.meta.inputs.Input
+import scala.meta.parsers.Parsed
 import scalafix.rewrite.RewriteCtx
 import scalafix.rewrite.SemanticApi
+import scalafix.util.AssociatedComments
 import scalafix.util.Patch
 import scalafix.util.TokenList
-import scalafix.util.logger
 
 object Scalafix {
  def fix(code: Input,
  config: ScalafixConfig,
  semanticApi: Option[SemanticApi]): Fixed = {
  config.parser.apply(code, config.dialect) match {
  case Parsed.Success(ast) =>
- val ctx = RewriteCtx(config, new TokenList(ast.tokens), semanticApi)
- val patches: Seq[Patch] = config.rewrites.flatMap(_.rewrite(ast, ctx))
- Fixed.Success(Patch.apply(ast.tokens, patches))
+ val tokens = ast.tokens
+ implicit val ctx = RewriteCtx(
+ config,
+ new TokenList(ast.tokens),
+ AssociatedComments(tokens),
+ semanticApi
+ )
+ val patches = config.rewrites.flatMap(_.rewrite(ast, ctx))
+ Fixed.Success(Patch.apply(ast, patches))
  case Parsed.Error(pos, msg, e) =>
  Fixed.Failed(Failure.ParseError(pos, msg, e))
  }

core/src/main/scala/scalafix/ScalafixConfig.scala

@@ -1,20 +1,40 @@
 package scalafix
 
-import scala.meta.Dialect
-import scala.meta.Tree
+import scala.collection.immutable.Seq
+import scala.meta._
 import scala.meta.dialects.Scala211
 import scala.meta.parsers.Parse
 import scala.util.control.NonFatal
 import scalafix.rewrite.Rewrite
+import scalafix.syntax._
 
 import java.io.File
 
 import com.typesafe.config.Config
 import com.typesafe.config.ConfigFactory
 
+case class ImportsConfig(
+ expandRelative: Boolean = true,
+ spaceAroundCurlyBrace: Boolean = false,
+ organize: Boolean = true,
+ removeUnused: Boolean = true,
+ alwaysUsed: Seq[Ref] = Seq(),
+ groups: Seq[FilterMatcher] = Seq(
+ FilterMatcher("scala.language.*"),
+ FilterMatcher("(scala|scala\\..*)$"),
+ FilterMatcher("(java|java\\..*)$"),
+ FilterMatcher(".*")
+ ),
+ groupByPrefix: Boolean = false
+)
+object ImportsConfig {
+ def default: ImportsConfig = ImportsConfig()
+}
 case class ScalafixConfig(
  rewrites: Seq[Rewrite] = Rewrite.defaultRewrites,
  parser: Parse[_ <: Tree] = Parse.parseSource,
+ imports: ImportsConfig = ImportsConfig(),
+ fatalWarning: Boolean = true,
  dialect: Dialect = Scala211
 )
 
@@ -25,6 +45,7 @@ object ScalafixConfig {
  catch {
  case NonFatal(e) => Left(e.getMessage)
  }
+ val default = ScalafixConfig()
 
  def fromFile(file: File): Either[String, ScalafixConfig] =
  saferThanTypesafe(() => ConfigFactory.parseFile(file))
@@ -34,15 +55,35 @@ object ScalafixConfig {
 
  def fromConfig(config: Config): Either[String, ScalafixConfig] = {
  import scala.collection.JavaConverters._
+ val base = ScalafixConfig(
+ fatalWarning = config.getBoolOrElse("fatalWarnings",
+ ScalafixConfig.default.fatalWarning),
+ imports = ImportsConfig(
+ expandRelative =
+ config.getBoolOrElse("imports.expandRelative",
+ ImportsConfig.default.expandRelative),
+ spaceAroundCurlyBrace =
+ config.getBoolOrElse("imports.spaceAroundCurlyBrace",
+ ImportsConfig.default.spaceAroundCurlyBrace),
+ organize = config.getBoolOrElse("imports.organize",
+ ImportsConfig.default.organize),
+ removeUnused =
+ config.getBoolOrElse("imports.removeUnused",
+ ImportsConfig.default.removeUnused),
+ groupByPrefix =
+ config.getBoolOrElse("imports.groupByPrefix",
+ ImportsConfig.default.groupByPrefix)
+ )
+ )
  if (config.hasPath("rewrites"))
- fromNames(config.getStringList("rewrites").asScala.toList)
+ fromNames(config.getStringList("rewrites").asScala.toList).right
+ .map(rewrites => base.copy(rewrites = rewrites))
  else Right(ScalafixConfig())
  }
 
- def fromNames(names: List[String]): Either[String, ScalafixConfig] =
+ def fromNames(names: List[String]): Either[String, Seq[Rewrite]] =
  names match {
- case "all" :: Nil =>
- Right(ScalafixConfig(rewrites = Rewrite.allRewrites))
+ case "all" :: Nil => Right(Rewrite.allRewrites)
  case _ =>
  val invalidNames =
  names.filterNot(Rewrite.name2rewrite.contains)
@@ -51,8 +92,7 @@ object ScalafixConfig {
  s"Invalid rewrite rule: ${invalidNames.mkString(",")}. " +
  s"Valid rules are: ${Rewrite.name2rewrite.keys.mkString(",")}")
  } else {
- val rewrites = names.map(Rewrite.name2rewrite)
- Right(ScalafixConfig(rewrites = rewrites))
+ Right(names.map(Rewrite.name2rewrite))
  }
  }
 }

core/src/main/scala/scalafix/rewrite/ExplicitImplicit.scala

@@ -3,6 +3,8 @@ package scalafix.rewrite
 import scala.{meta => m}
 import scalafix.util.Patch
 import scalafix.util.Whitespace
+import scala.collection.immutable.Seq
+import scalafix.util.TokenPatch
 
 case object ExplicitImplicit extends Rewrite {
  // Don't explicitly annotate vals when the right-hand body is a single call
@@ -27,8 +29,8 @@ case object ExplicitImplicit extends Rewrite {
  replace <- lhsTokens.reverseIterator.find(x =>
  !x.is[Token.Equals] && !x.is[Whitespace])
  typ <- semantic.typeSignature(defn)
- } yield Patch(replace, replace, s"$replace: ${typ.syntax}")
- }.toSeq
+ } yield TokenPatch.AddRight(replace, s": ${typ.syntax}")
+ }.to[Seq]
  ast.collect {
  case t @ m.Defn.Val(mods, _, None, body)
  if mods.exists(_.syntax == "implicit") &&

core/src/main/scala/scalafix/rewrite/ProcedureSyntax.scala

@@ -7,6 +7,8 @@ import scala.meta.tokens.Token.RightBrace
 import scala.meta.tokens.Token.RightParen
 import scala.meta.tokens.Token.Space
 import scalafix.util.Patch
+import scala.collection.immutable.Seq
+import scalafix.util.TokenPatch
 
 case object ProcedureSyntax extends Rewrite {
  override def rewrite(ast: Tree, ctx: RewriteCtx): Seq[Patch] = {
@@ -27,7 +29,7 @@ case object ProcedureSyntax extends Rewrite {
  if (between.nonEmpty) " " + between
  else ""
  }
- Patch(next(closingParen), bodyStart, s": Unit = {$comment")
+ TokenPatch.AddRight(closingParen, s": Unit =")
  }
  patches
  }

core/src/main/scala/scalafix/rewrite/Rewrite.scala

@@ -2,7 +2,9 @@ package scalafix.rewrite
 
 import scala.meta._
 import scalafix.Failure.MissingSemanticApi
+import scalafix.util.TreePatch
 import scalafix.util.Patch
+import scala.collection.immutable.Seq
 
 abstract class Rewrite {
  def getSemanticApi(ctx: RewriteCtx): SemanticApi = ctx.semantic.getOrElse {
@@ -16,7 +18,10 @@ object Rewrite {
  t.map(x => x.source -> x.value).toMap
  }
 
- val syntaxRewrites: Seq[Rewrite] = Seq(ProcedureSyntax, VolatileLazyVal)
+ val syntaxRewrites: Seq[Rewrite] = Seq(
+ ProcedureSyntax,
+ VolatileLazyVal
+ )
  val semanticRewrites: Seq[Rewrite] = Seq(ExplicitImplicit)
  val allRewrites: Seq[Rewrite] = syntaxRewrites ++ semanticRewrites
  val defaultRewrites: Seq[Rewrite] =

core/src/main/scala/scalafix/rewrite/RewriteCtx.scala

@@ -1,9 +1,11 @@
 package scalafix.rewrite
 import scalafix.ScalafixConfig
+import scalafix.util.AssociatedComments
 import scalafix.util.TokenList
 
 case class RewriteCtx(
  config: ScalafixConfig,
  tokenList: TokenList,
+ comments: AssociatedComments,
  semantic: Option[SemanticApi]
 )

core/src/main/scala/scalafix/rewrite/SemanticApi.scala

@@ -1,7 +1,6 @@
 package scalafix.rewrite
 
-import scala.meta.Defn
-import scala.meta.Type
+import scala.meta._
 
 /** A custom semantic api for scalafix rewrites.
  *
@@ -18,4 +17,13 @@ trait SemanticApi {
 
  /** Returns the type annotation for given val/def. */
  def typeSignature(defn: Defn): Option[Type]
+
+ /** Returns the fully qualified name of this name, or none if unable to find it*/
+ def fqn(name: Ref): Option[Ref]
+
+ /** Returns all used refs in this compilation unit */
+ def usedFqns: Seq[Ref]
+
+ /** Returns true if importee is not used in this compilation unit, false otherwise */
+ def isUnusedImport(importee: Importee): Boolean
 }

core/src/main/scala/scalafix/rewrite/VolatileLazyVal.scala

@@ -2,6 +2,8 @@ package scalafix.rewrite
 
 import scala.meta._
 import scalafix.util.Patch
+import scala.collection.immutable.Seq
+import scalafix.util.TokenPatch
 
 case object VolatileLazyVal extends Rewrite {
  private object NonVolatileLazyVal {
@@ -17,7 +19,7 @@ case object VolatileLazyVal extends Rewrite {
  override def rewrite(ast: Tree, ctx: RewriteCtx): Seq[Patch] = {
  ast.collect {
  case NonVolatileLazyVal(tok) =>
- Patch(tok, tok, s"@volatile ${tok.syntax}")
+ TokenPatch.AddLeft(tok, s"@volatile ")
  }
  }
 }

core/src/main/scala/scalafix/syntax/package.scala

@@ -0,0 +1,31 @@
+package scalafix
+
+import scala.meta.Importee
+import scala.meta.tokens.Token
+import scalafix.util.CanonicalImport
+import scalafix.util.ImportPatch
+import scalafix.util.logger
+
+import com.typesafe.config.Config
+
+package object syntax {
+ implicit class XtensionImporter(i: CanonicalImport) {
+ def supersedes(patch: ImportPatch): Boolean =
+ i.ref.structure == patch.importer.ref.structure &&
+ (i.importee.is[Importee.Wildcard] ||
+ i.importee.structure == patch.importer.importee.structure)
+ }
+
+ implicit class XtensionToken(token: Token) {
+ def posTuple: (Int, Int) = token.start -> token.end
+ }
+
+ implicit class XtensionConfig(config: Config) {
+ def getBoolOrElse(key: String, els: Boolean): Boolean =
+ if (config.hasPath(key)) config.getBoolean(key)
+ else els
+ }
+ implicit class XtensionString(str: String) {
+ def reveal: String = logger.reveal(str)
+ }
+}

core/src/main/scala/scalafix/util/AbsoluteFile.scala

@@ -0,0 +1,31 @@
+package scalafix.util
+
+import java.io.File
+
+/** Wrapper around java.io.File with an absolute path. */
+sealed abstract case class AbsoluteFile(jfile: File) {
+ def path: String = jfile.getAbsolutePath
+ def /(other: String) = new AbsoluteFile(new File(jfile, other)) {}
+}
+
+object AbsoluteFile {
+ def fromFiles(files: Seq[File],
+ workingDir: AbsoluteFile): Seq[AbsoluteFile] = {
+ files.map(x => fromFile(x, workingDir))
+ }
+ private def makeAbsolute(workingDir: File)(file: File): File =
+ if (file.isAbsolute) file
+ else new File(workingDir, file.getPath)
+
+ // If file is already absolute, then workingDir is not used.
+ def fromFile(file: File, workingDir: AbsoluteFile): AbsoluteFile = {
+ new AbsoluteFile(makeAbsolute(workingDir.jfile)(file)) {}
+ }
+ def fromPath(path: String): Option[AbsoluteFile] = {
+ val file = new File(path)
+ if (file.isAbsolute) Some(new AbsoluteFile(file) {})
+ else None
+ }
+ def userDir = new AbsoluteFile(new File(System.getProperty("user.dir"))) {}
+ def homeDir = new AbsoluteFile(new File(System.getProperty("user.home"))) {}
+}