CommunityBuild: test community-code formatting

Initially, add only the munit tests. We'll add additional repositories
as scalameta/scalafmt bugs revealed there are fixed.

.github/workflows/ci.yml

@@ -37,6 +37,24 @@ jobs:
         shell: bash
       - run: TEST="2.13" sbt ci-test
         shell: bash
+  community-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        java: [ '11' ]
+        os: [windows-latest, ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up JVM
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java }}
+          distribution: 'temurin'
+          cache: 'sbt'
+      - run: sbt communityTests/test
   formatting:
     runs-on: ubuntu-latest
     steps:

build.sbt

@@ -202,6 +202,19 @@ lazy val tests = project.in(file("scalafmt-tests")).settings(
     Seq[BuildInfoKey]("resourceDirectory" -> (Test / resourceDirectory).value),
 ).enablePlugins(BuildInfoPlugin).dependsOn(coreJVM, dynamic, cli)
 
+lazy val communityTests = project.in(file("scalafmt-tests-community")).settings(
+  publish / skip := true,
+  libraryDependencies ++= Seq(
+    // Test dependencies
+    "com.lihaoyi" %% "scalatags" % "0.13.1",
+    scalametaTestkit,
+    munit.value,
+  ),
+  scalacOptions ++= scalacJvmOptions.value,
+  javaOptions += "-Dfile.encoding=UTF8",
+  buildInfoPackage := "org.scalafmt.tests",
+).enablePlugins(BuildInfoPlugin).dependsOn(coreJVM)
+
 lazy val benchmarks = project.in(file("scalafmt-benchmarks")).settings(
   publish / skip := true,
   moduleName := "scalafmt-benchmarks",

scalafmt-tests-community/src/test/scala/org/scalafmt/community/CommunityBuild.scala

@@ -0,0 +1,12 @@
+package org.scalafmt.community
+
+import scala.meta._
+
+case class CommunityBuild(
+    giturl: String,
+    commit: String,
+    name: String,
+    excluded: List[String],
+    checkedFiles: Int,
+    dialect: sourcecode.Text[Dialect],
+)

scalafmt-tests-community/src/test/scala/org/scalafmt/community/CommunityMunitSuite.scala

@@ -0,0 +1,26 @@
+package org.scalafmt.community
+
+import scala.meta._
+
+class CommunityMunitSuite extends CommunitySuite {
+
+  override protected def builds = Seq(
+    getBuild("v1.0.1", dialects.Scala213, 109),
+    // latest commit from 30.03.2021
+    getBuild("06346adfe3519c384201eec531762dad2f4843dc", dialects.Scala213, 102),
+  )
+
+  private def getBuild(
+      ref: String,
+      dialect: sourcecode.Text[Dialect],
+      files: Int,
+  ) = CommunityBuild(
+    "https://github.com/scalameta/munit.git",
+    ref,
+    "munit",
+    Nil,
+    files,
+    dialect,
+  )
+
+}

scalafmt-tests-community/src/test/scala/org/scalafmt/community/CommunitySuite.scala

@@ -0,0 +1,95 @@
+package org.scalafmt.community
+
+import org.scalafmt.config._
+
+import java.nio.file._
+
+import scala.concurrent.duration
+import scala.sys.process._
+
+import munit.FunSuite
+
+abstract class CommunitySuite extends FunSuite {
+
+  import TestHelpers._
+
+  override val munitTimeout = new duration.FiniteDuration(5, duration.MINUTES)
+
+  protected def builds: Seq[CommunityBuild]
+
+  for {
+    build <- builds
+    (k, v) <- TestStyles.styles
+  } {
+    val prefix = s"[ref ${build.commit}, style $k]"
+    val style: ScalafmtConfig = v.withDialect(NamedDialect(build.dialect))
+    test(s"community-build: ${build.name} $prefix")(check(k)(build, style))
+  }
+
+  private def check(
+      styleName: String,
+  )(implicit build: CommunityBuild, style: ScalafmtConfig): Unit = {
+    val folder = fetchCommunityBuild
+
+    val stats = checkFilesRecursive(styleName, folder.toAbsolutePath)
+      .getOrElse(TestStats.init)
+    val timePer1KLines = Math
+      .round(stats.timeTaken / (stats.linesParsed / 1000.0))
+
+    println("--------------------------")
+    println(build.name)
+    println(s"Files parsed correctly ${stats.checkedFiles - stats.errors}")
+    println(s"Files errored: ${stats.errors}")
+    println(s"Time taken: ${stats.timeTaken}ms")
+    if (stats.linesParsed < 1000) println(s"Lines parsed: ${stats.linesParsed}")
+    else println(s"Lines parsed: ~${stats.linesParsed / 1000}k")
+    println(s"Parsing speed per 1k lines ===> $timePer1KLines ms/1klines")
+    println("--------------------------")
+    stats.lastError.foreach(e => throw e)
+
+    assertEquals(stats.errors, 0)
+    assertEquals(
+      stats.checkedFiles,
+      build.checkedFiles * 2,
+      s"expected ${stats.checkedFiles / 2} per run",
+    )
+  }
+
+  private def fetchCommunityBuild(implicit build: CommunityBuild): Path = {
+    if (!Files.exists(communityDirectory)) Files
+      .createDirectory(communityDirectory)
+
+    val log = new StringBuilder
+    val logger = ProcessLogger(s => log.append(s))
+
+    def runCmd(cmd: String, what: => String): Unit = {
+      val result: Int = cmd.!(logger)
+      assertEquals(
+        clue(result),
+        0,
+        s"Community build ${build.name}: $what failed:\n$log",
+      )
+      log.clear()
+    }
+
+    val folderPath = communityDirectory.resolve(build.name)
+    val folder = folderPath.toString
+
+    if (!Files.exists(folderPath)) runCmd(
+      s"git clone --depth=1 --no-single-branch ${build.giturl} $folder",
+      "cloning",
+    )
+
+    val ref = build.commit
+
+    runCmd(s"git fetch -C $folder --depth=1 origin $ref", s"fetching [ref=$ref]")
+
+    runCmd(
+      s"git checkout -C $folder -f -B ref-$ref $ref",
+      s"checking out [ref=$ref]",
+    )
+
+    folderPath
+  }
+
+}

scalafmt-tests-community/src/test/scala/org/scalafmt/community/TestHelpers.scala

@@ -0,0 +1,118 @@
+package org.scalafmt.community
+
+import org.scalafmt.CompatCollections.JavaConverters._
+import org.scalafmt.CompatCollections.ParConverters._
+import org.scalafmt.Formatted
+import org.scalafmt.Scalafmt
+import org.scalafmt.config._
+
+import scala.meta._
+
+import java.io._
+import java.nio.file._
+
+import munit.ComparisonFailException
+import munit.diff.Diff
+import munit.diff.console.AnsiColors
+
+object TestHelpers {
+
+  val communityDirectory = Paths
+    .get("scalafmt-tests-community/target/community-projects")
+
+  private val ignoreParts = List(
+    ".git/",
+    "tests/",
+    "test/",
+    "test-resources/scripting/",
+    "test-resources/repl/",
+    "sbt-test/",
+    "out/",
+  ).map(Paths.get(_))
+
+  private def timeIt[A](block: => A): (Long, A) = {
+    val t0 = System.currentTimeMillis()
+    val res = block
+    val t1 = System.currentTimeMillis()
+    (t1 - t0, res)
+  }
+
+  private def runFile(styleName: String, path: Path, absPathString: String)(
+      implicit style: ScalafmtConfig,
+  ): TestStats = {
+    val input = Input.File(path).chars
+    val lines1 = input.count(_ == '\n')
+    val (duration1, result1) = timeIt(
+      Scalafmt.formatCode(new String(input), style, filename = absPathString),
+    )
+    result1.formatted match {
+      case x1: Formatted.Failure =>
+        println(s"Failed for original file $absPathString")
+        val trace = new StringWriter()
+        trace.append(x1.e.getMessage).append('\n')
+        x1.e.printStackTrace(new PrintWriter(trace))
+        println(s"Error: " + trace.toString)
+        TestStats(1, 1, Some(x1.e), duration1, lines1)
+      case x1: Formatted.Success =>
+        val stats1 = TestStats(1, 0, None, duration1, lines1)
+        val out1 = x1.formattedCode
+        val lines2 = x1.formattedCode.count(_ == '\n')
+        val (duration2, result2) =
+          timeIt(Scalafmt.formatCode(out1, style, filename = absPathString))
+        def saveFormatted(): Unit = Files.writeString(
+          Paths.get(absPathString + s".formatted.$styleName"),
+          out1,
+          StandardOpenOption.CREATE,
+          StandardOpenOption.TRUNCATE_EXISTING,
+        )
+        val stats2 = result2.formatted match {
+          case x2: Formatted.Failure =>
+            println(s"Failed for formatted file $absPathString")
+            println(s"Error: " + x2.e.getMessage)
+            saveFormatted()
+            TestStats(1, 1, Some(x2.e), duration2, lines2)
+          case x2: Formatted.Success =>
+            val out2 = x2.formattedCode
+            val diff = new Diff(out2, out1)
+            if (diff.isEmpty) TestStats(1, 0, None, duration2, lines2)
+            else {
+              val msg = AnsiColors.filterAnsi(diff.createDiffOnlyReport())
+              val loc = new munit.Location(absPathString, 0)
+              val exc = new ComparisonFailException(msg, out2, out1, loc, false)
+              println(s"Failed idempotency for file $absPathString")
+              println(msg)
+              saveFormatted()
+              TestStats(1, 1, Some(exc), duration2, lines2)
+            }
+        }
+        TestStats.merge(stats1, stats2)
+    }
+  }
+
+  def checkFilesRecursive(styleName: String, path: Path)(implicit
+      build: CommunityBuild,
+      style: ScalafmtConfig,
+  ): Option[TestStats] =
+    if (ignoreParts.exists(path.endsWith)) None
+    else {
+      val ds = Files.newDirectoryStream(path)
+      val (dirs, files) =
+        try ds.iterator().asScala.toList.partition(Files.isDirectory(_))
+        finally ds.close()
+      val fileStats = files.par.flatMap { x =>
+        val fileStr = x.toString
+        if (fileStr.endsWith(".scala") && !excluded(fileStr))
+          Some(runFile(styleName, x, fileStr))
+        else None
+      }.reduceLeftOption(TestStats.merge)
+      val dirStats = dirs.par.flatMap(checkFilesRecursive(styleName, _))
+        .reduceLeftOption(TestStats.merge)
+      fileStats.fold(dirStats)(x =>
+        dirStats.map(TestStats.merge(_, x)).orElse(fileStats),
+      )
+    }
+
+  def excluded(path: String)(implicit build: CommunityBuild): Boolean = build
+    .excluded.exists(el => path.endsWith(el))
+
+}

scalafmt-tests-community/src/test/scala/org/scalafmt/community/TestStats.scala

@@ -0,0 +1,22 @@
+package org.scalafmt.community
+
+case class TestStats(
+    checkedFiles: Int,
+    errors: Int,
+    lastError: Option[Throwable],
+    timeTaken: Long,
+    linesParsed: Int,
+)
+
+object TestStats {
+  final val init = TestStats(0, 0, None, 0, 0)
+
+  def merge(s1: TestStats, s2: TestStats): TestStats = TestStats(
+    s1.checkedFiles + s2.checkedFiles,
+    s1.errors + s2.errors,
+    s1.lastError.orElse(s2.lastError),
+    s1.timeTaken + s2.timeTaken,
+    s1.linesParsed + s2.linesParsed,
+  )
+
+}

scalafmt-tests-community/src/test/scala/org/scalafmt/community/TestStyles.scala

@@ -0,0 +1,55 @@
+package org.scalafmt.community
+
+import org.scalafmt.config._
+import org.scalafmt.rewrite._
+
+import scala.collection.immutable.SortedMap
+
+object TestStyles {
+
+  private val baseClassicStyle = {
+    val base = ScalafmtConfig.default
+    val isWin = System.lineSeparator() == "\r\n"
+    base.copy(
+      docstrings = base.docstrings.copy(wrap = Docstrings.Wrap.keep),
+      project = base.project
+        .copy(git = true, layout = Some(ProjectFiles.Layout.StandardConvention)),
+      lineEndings = Some(if (isWin) LineEndings.windows else LineEndings.unix),
+      runner = base.runner.copy(
+        maxStateVisits = 10000000,
+        optimizer = base.runner.optimizer.copy(escapeInPathologicalCases = false),
+      ),
+    )
+  }
+
+  private def withSource(source: Newlines.SourceHints) = baseClassicStyle
+    .copy(newlines = baseClassicStyle.newlines.copy(source = source))
+
+  private def withRewrites(style: ScalafmtConfig) = style.copy(rewrite =
+    style.rewrite.copy(
+      rules = Seq(RedundantParens, RedundantBraces, SortModifiers, AvoidInfix),
+      scala3 = style.rewrite.scala3.copy(
+        convertToNewSyntax = true,
+        removeOptionalBraces = RewriteScala3Settings.RemoveOptionalBraces.yes,
+        insertEndMarkerMinLines = 5,
+      ),
+      redundantBraces = RedundantBracesSettings.all,
+      redundantParens = RedundantParensSettings.all,
+    ),
+  )
+
+  private val baseKeepStyle = withSource(Newlines.keep)
+
+  val styles = SortedMap(
+    "classic" -> baseClassicStyle,
+    "classicWithRewrites" -> withRewrites(baseClassicStyle),
+    "classicWithAlign" -> baseClassicStyle.withAlign(Align.most),
+    "keep" -> baseKeepStyle,
+    "keepWithRewrites" -> withRewrites(baseKeepStyle),
+    "keepWithAlign" -> baseKeepStyle.withAlign(Align.most),
+    "keepWithScalaJS" -> baseKeepStyle.forScalaJs,
+    "fold" -> withSource(Newlines.fold),
+    "unfold" -> withSource(Newlines.unfold),
+  )
+
+}