Generalize handling of source folders in cross-platform/version scena…

…rios (#2531)

Originally bumped into these issues trying to upgrade
fansi/os-lib/upickle to 0.11.0-M9

1. Generalize `CrossScalaModuleTests#sources` to
`JavaModuleTests#sources`.
1. Rather than explicitly duplicating the behavior of `CrossScalaModule`
in `CrossScalaModuleTests`, we instead define that all `Tests` module
within a `JavaModule` of any kind most follow the parent module's source
layout, relativized to the test module's `millSourcePath`.
2. This should allow the "test module follows parent module source"
approach to generalize for any combination of `CrossScalaModule`,
`PlatformScalaModule`, etc. without needing it to be aware of every
combination of such cross-modules

2. Generalize `PlatformScalaModule#sources` to work for cross modules
where the cross segment is the last one, i.e. `foo.jvm[2.13.10]` vs
`foo[2.13.10].jvm`


Tested both changes with an additional example test included in the
documentation. This demonstrates an alternative approach to that in
`example/web/6-cross-version-platform-publishing/`, one commonly used in
the com-lihaoyi libraries, so I think it's worth making a full example
v.s. just testing this via unit tests

docs/modules/ROOT/pages/Web_Build_Examples.adoc

@@ -27,4 +27,8 @@ include::example/web/5-webapp-scalajs-shared.adoc[]
 
 == Publishing Cross-Platform Scala Modules
 
-include::example/web/6-cross-platform-publishing.adoc[]
+include::example/web/6-cross-version-platform-publishing.adoc[]
+
+== Publishing Cross-Platform Scala Modules Alternative
+
+include::example/web/7-cross-platform-version-publishing.adoc[]

example/web/6-cross-platform-publishing/build.sc → example/web/6-cross-version-platform-publishing/build.sc

@@ -16,7 +16,7 @@ trait FooModule extends Cross.Module[String] {
 
     def ivyDeps = Agg(ivy"com.lihaoyi::scalatags::0.12.0")
 
-    object test extends Tests {
+    trait FooTestModule extends Tests {
       def ivyDeps = Agg(ivy"com.lihaoyi::utest::0.7.11")
       def testFramework = "utest.runner.Framework"
     }
@@ -27,18 +27,26 @@ trait FooModule extends Cross.Module[String] {
   }
 
   object bar extends Module {
-    object jvm extends Shared
-    object js extends SharedJS
+    object jvm extends Shared{
+      object test extends Tests with FooTestModule
+    }
+    object js extends SharedJS{
+      object test extends Tests with FooTestModule
+    }
   }
 
   object qux extends Module{
     object jvm extends Shared{
       def moduleDeps = Seq(bar.jvm)
       def ivyDeps = super.ivyDeps() ++ Agg(ivy"com.lihaoyi::upickle::3.0.0")
+
+      object test extends Tests with FooTestModule
     }
 
     object js extends SharedJS {
       def moduleDeps = Seq(bar.js)
+
+      object test extends Tests with FooTestModule
     }
   }
 }
@@ -83,6 +91,10 @@ Bar.value: <p>world Specific code for Scala 3.x</p>
 Parsing JSON with js.JSON.parse
 Qux.main: Set(<p>i</p>, <p>cow</p>, <p>me</p>)
 
+> ./mill foo[3.2.2].__.js.test
++ bar.BarTests.test ...  <p>world Specific code for Scala 3.x</p>
++ qux.QuxTests.parseJsonGetKeys ...  Set(i, cow, me)
+
 > ./mill __.publishLocal
 Publishing Artifact(com.lihaoyi,foo-bar_sjs1_2.13,0.0.1) to ivy repo...
 Publishing Artifact(com.lihaoyi,foo-bar_2.13,0.0.1) to ivy repo...
@@ -92,4 +104,5 @@ Publishing Artifact(com.lihaoyi,foo-bar_sjs1_3,0.0.1) to ivy repo...
 Publishing Artifact(com.lihaoyi,foo-bar_3,0.0.1) to ivy repo...
 Publishing Artifact(com.lihaoyi,foo-qux_sjs1_3,0.0.1) to ivy repo...
 Publishing Artifact(com.lihaoyi,foo-qux_3,0.0.1) to ivy repo...
+
 */

example/web/6-cross-platform-publishing/foo/qux/test/src/QuxTests.scala → example/web/6-cross-version-platform-publishing/foo/qux/test/src/QuxTests.scala

@@ -3,7 +3,7 @@ import utest._
 object QuxTests extends TestSuite {
   def tests = Tests {
     test("parseJsonGetKeys") {
-      val string = """{"i": "am", "cow": "hear", "me": "moo}"""
+      val string = """{"i": "am", "cow": "hear", "me": "moo"}"""
       val keys = QuxPlatformSpecific.parseJsonGetKeys(string)
       assert(keys == Set("i", "cow", "me"))
       keys

example/web/7-cross-platform-version-publishing/bar/src-2/BarVersionSpecific.scala

@@ -0,0 +1,4 @@
+package bar
+object BarVersionSpecific {
+  def text(): String = "Specific code for Scala 2.x"
+}

example/web/7-cross-platform-version-publishing/bar/src-3/BarVersionSpecific.scala

@@ -0,0 +1,4 @@
+package bar
+object BarVersionSpecific {
+  def text(): String = "Specific code for Scala 3.x"
+}

example/web/7-cross-platform-version-publishing/bar/src/Bar.scala

@@ -0,0 +1,5 @@
+package bar
+import scalatags.Text.all._
+object Bar {
+  val value = p("world", " ", BarVersionSpecific.text())
+}

example/web/7-cross-platform-version-publishing/bar/test/src/BarTests.scala

@@ -0,0 +1,12 @@
+package bar
+import utest._
+object BarTests extends TestSuite {
+  def tests = Tests {
+    test("test") {
+      val result = Bar.value.toString
+      val matcher = "<p>world Specific code for Scala [23].x</p>".r
+      assert(matcher.matches(result))
+      result
+    }
+  }
+}

example/web/7-cross-platform-version-publishing/build.sc

@@ -0,0 +1,112 @@
+import mill._, scalalib._, scalajslib._, publish._
+
+trait Shared extends CrossScalaModule with PlatformScalaModule with PublishModule {
+  def publishVersion = "0.0.1"
+
+  def pomSettings = PomSettings(
+    description = "Hello",
+    organization = "com.lihaoyi",
+    url = "https://github.com/lihaoyi/example",
+    licenses = Seq(License.MIT),
+    versionControl = VersionControl.github("lihaoyi", "example"),
+    developers = Seq(Developer("lihaoyi", "Li Haoyi", "https://github.com/lihaoyi"))
+  )
+
+  def ivyDeps = Agg(ivy"com.lihaoyi::scalatags::0.12.0")
+
+  trait SharedTestModule extends Tests {
+    def ivyDeps = Agg(ivy"com.lihaoyi::utest::0.7.11")
+    def testFramework = "utest.runner.Framework"
+  }
+}
+
+trait SharedJS extends Shared with ScalaJSModule {
+  def scalaJSVersion = "1.13.0"
+}
+
+val scalaVersions = Seq("2.13.8", "3.2.2")
+
+object bar extends Module {
+  object jvm extends Cross[JvmModule](scalaVersions)
+  trait JvmModule extends Shared{
+    object test extends Tests with SharedTestModule
+  }
+
+  object js extends Cross[JsModule](scalaVersions)
+  trait JsModule extends SharedJS{
+    object test extends Tests with SharedTestModule
+  }
+}
+
+object qux extends Module{
+  object jvm extends Cross[JvmModule](scalaVersions)
+  trait JvmModule extends Shared{
+    def moduleDeps = Seq(bar.jvm())
+    def ivyDeps = super.ivyDeps() ++ Agg(ivy"com.lihaoyi::upickle::3.0.0")
+
+    object test extends Tests with SharedTestModule
+  }
+
+  object js extends Cross[JsModule](scalaVersions)
+  trait JsModule extends SharedJS {
+    def moduleDeps = Seq(bar.js())
+
+    object test extends Tests with SharedTestModule
+  }
+}
+
+// This example demonstrates an alternative way of defining your cross-platform
+// cross-version modules: rather than wrapping them all in a `foo`
+// cross-module to provide the different versions, we instead give each module
+// `bar.jvm`, `bar.js`, `qux.jvm`, `qux.js` its own `Cross` module. This
+// approach can be useful if the different cross modules need to support
+// different sets of Scala versions, as it allows you to specify the
+// `scalaVersions` passed to each individual cross module separately.
+
+/** Usage
+
+> ./mill show qux.js[3.2.2].sources
+[
+  ".../qux/src",
+  ".../qux/src-js",
+  ".../qux/src-3.2.2",
+  ".../qux/src-3.2.2-js",
+  ".../qux/src-3.2",
+  ".../qux/src-3.2-js",
+  ".../qux/src-3",
+  ".../qux/src-3-js"
+]
+
+> ./mill show qux.js[3.2.2].test.sources
+[
+  ".../qux/test/src",
+  ".../qux/test/src-js",
+  ".../qux/test/src-3.2.2",
+  ".../qux/test/src-3.2.2-js",
+  ".../qux/test/src-3.2",
+  ".../qux/test/src-3.2-js",
+  ".../qux/test/src-3",
+  ".../qux/test/src-3-js"
+]
+
+> ./mill qux.jvm[2.13.8].run
+Bar.value: <p>world Specific code for Scala 2.x</p>
+Parsing JSON with ujson.read
+Qux.main: Set(<p>i</p>, <p>cow</p>, <p>me</p>)
+
+> ./mill __.js[3.2.2].test
++ bar.BarTests.test ...  <p>world Specific code for Scala 3.x</p>
++ qux.QuxTests.parseJsonGetKeys ...  Set(i, cow, me)
+
+> ./mill __.publishLocal
+...
+Publishing Artifact(com.lihaoyi,bar_sjs1_2.13,0.0.1) to ivy repo...
+Publishing Artifact(com.lihaoyi,bar_2.13,0.0.1) to ivy repo...
+Publishing Artifact(com.lihaoyi,qux_sjs1_2.13,0.0.1) to ivy repo...
+Publishing Artifact(com.lihaoyi,qux_2.13,0.0.1) to ivy repo...
+Publishing Artifact(com.lihaoyi,bar_sjs1_3,0.0.1) to ivy repo...
+Publishing Artifact(com.lihaoyi,bar_3,0.0.1) to ivy repo...
+Publishing Artifact(com.lihaoyi,qux_sjs1_3,0.0.1) to ivy repo...
+Publishing Artifact(com.lihaoyi,qux_3,0.0.1) to ivy repo...
+
+*/

example/web/7-cross-platform-version-publishing/qux/src-js/QuxPlatformSpecific.scala

@@ -0,0 +1,8 @@
+package qux
+import scala.scalajs.js
+object QuxPlatformSpecific {
+  def parseJsonGetKeys(s: String): Set[String] = {
+    println("Parsing JSON with js.JSON.parse")
+    js.JSON.parse(s).asInstanceOf[js.Dictionary[_]].keys.toSet
+  }
+}

example/web/7-cross-platform-version-publishing/qux/src-jvm/QuxPlatformSpecific.scala

@@ -0,0 +1,7 @@
+package qux
+object QuxPlatformSpecific {
+  def parseJsonGetKeys(s: String): Set[String] = {
+    println("Parsing JSON with ujson.read")
+    ujson.read(s).obj.keys.toSet
+  }
+}

example/web/7-cross-platform-version-publishing/qux/src/Qux.scala

@@ -0,0 +1,9 @@
+package qux
+import scalatags.Text.all._
+object Qux {
+  def main(args: Array[String]): Unit = {
+    println("Bar.value: " + bar.Bar.value)
+    val string = """{"i": "am", "cow": "hear", "me": "moo"}"""
+    println("Qux.main: " + QuxPlatformSpecific.parseJsonGetKeys(string).map(p(_)))
+  }
+}

example/web/7-cross-platform-version-publishing/qux/test/src/QuxTests.scala

@@ -0,0 +1,12 @@
+package qux
+import utest._
+object QuxTests extends TestSuite {
+  def tests = Tests {
+    test("parseJsonGetKeys") {
+      val string = """{"i": "am", "cow": "hear", "me": "moo"}"""
+      val keys = QuxPlatformSpecific.parseJsonGetKeys(string)
+      assert(keys == Set("i", "cow", "me"))
+      keys
+    }
+  }
+}

integration/feature/watch-source-input/test/src/WatchSourceInputTests.scala

@@ -104,8 +104,9 @@ object WatchSourceInputTests extends IntegrationTestSuite {
 
     test("sources") {
 
-      test("noshow") - retry(3) { testWatchSource(false) }
-      test("show") - retry(3) { testWatchSource(true) }
+      // Make sure we clean up the workspace between retries
+      test("noshow") - retry(3) { initWorkspace(); testWatchSource(false) }
+      test("show") - retry(3) { initWorkspace(); testWatchSource(true) }
     }
 
     def testWatchInput(show: Boolean) = {
@@ -146,8 +147,9 @@ object WatchSourceInputTests extends IntegrationTestSuite {
 
     test("input") {
 
-      test("noshow") - retry(3) { testWatchInput(false) }
-      test("show") - retry(3) { testWatchInput(true) }
+      // Make sure we clean up the workspace between retries
+      test("noshow") - retry(3) { initWorkspace(); testWatchInput(false) }
+      test("show") - retry(3) { initWorkspace(); testWatchInput(true) }
     }
   }
 }

scalajslib/src/mill/scalajslib/ScalaJSModule.scala

@@ -188,10 +188,16 @@ trait ScalaJSModule extends scalalib.ScalaModule { outer =>
   }
 
   override def mandatoryScalacOptions = T {
-    super.mandatoryScalacOptions() ++ {
-      if (ZincWorkerUtil.isScala3(scalaVersion())) Seq("-scalajs")
+    // Don't add flag twice, e.g. if a test suite inherits it both directly
+    // ScalaJSModule as well as from the enclosing non-test ScalaJSModule
+    val scalajsFlag =
+      if (
+        ZincWorkerUtil.isScala3(scalaVersion()) &&
+        !super.mandatoryScalacOptions().contains("-scalajs")
+      ) Seq("-scalajs")
       else Seq.empty
-    }
+
+    super.mandatoryScalacOptions() ++ scalajsFlag
   }
 
   override def scalacPluginIvyDeps = T {

scalalib/src/mill/scalalib/CrossScalaModule.scala

@@ -17,12 +17,4 @@ trait CrossScalaModule extends ScalaModule with CrossModuleBase {
     super.sources() ++
       scalaVersionDirectoryNames.map(s => PathRef(millSourcePath / s"src-$s"))
   }
-
-  trait CrossScalaModuleTests extends ScalaModuleTests {
-    override def sources = T.sources {
-      super.sources() ++
-        scalaVersionDirectoryNames.map(s => PathRef(millSourcePath / s"src-$s"))
-    }
-  }
-  trait Tests extends CrossScalaModuleTests
 }

scalalib/src/mill/scalalib/JavaModule.scala

@@ -41,7 +41,13 @@ trait JavaModule
     override def zincWorker: ZincWorkerModule = outer.zincWorker
     override def skipIdea: Boolean = outer.skipIdea
     override def runUseArgsFile: Target[Boolean] = T { outer.runUseArgsFile() }
+    override def sources = T.sources {
+      for (src <- outer.sources()) yield {
+        PathRef(this.millSourcePath / src.path.relativeTo(outer.millSourcePath))
+      }
+    }
   }
+
   trait Tests extends JavaModuleTests
 
   def defaultCommandName(): String = "run"

scalalib/src/mill/scalalib/PlatformScalaModule.scala

@@ -16,13 +16,15 @@ trait PlatformScalaModule extends ScalaModule {
   override def millSourcePath = super.millSourcePath / os.up
 
   override def sources = T.sources {
-    val platform = millModuleSegments.parts.last
-    super.sources().flatMap(source =>
-      Seq(
-        source,
-        PathRef(source.path / os.up / s"${source.path.last}-${platform}")
-      )
-    )
+    val platform = millModuleSegments
+      .value
+      .collect { case l: mill.define.Segment.Label => l.value }
+      .last
+
+    super.sources().flatMap { source =>
+      val platformPath = PathRef(source.path / _root_.os.up / s"${source.path.last}-${platform}")
+      Seq(source, platformPath)
+    }
   }
 
   override def artifactNameParts = super.artifactNameParts().dropRight(1)