diff --git a/integration/feature/reproducibility/resources/build.mill b/integration/feature/reproducibility/resources/build.mill new file mode 100644 index 00000000000..c4929e6ad21 --- /dev/null +++ b/integration/feature/reproducibility/resources/build.mill @@ -0,0 +1,84 @@ +package build +import mill._, scalalib._, scalajslib._ + +trait AppScalaModule extends ScalaModule { + def scalaVersion = "3.3.3" +} + +trait AppScalaJSModule extends AppScalaModule with ScalaJSModule { + def scalaJSVersion = "1.16.0" +} + +object `package` extends RootModule with AppScalaModule { + def moduleDeps = Seq(shared.jvm) + def ivyDeps = Agg(ivy"com.lihaoyi::cask:0.9.1") + + def resources = Task { + os.makeDir(Task.dest / "webapp") + val jsPath = client.fastLinkJS().dest.path + os.copy(jsPath / "main.js", Task.dest / "webapp/main.js") + os.copy(jsPath / "main.js.map", Task.dest / "webapp/main.js.map") + super.resources() ++ Seq(PathRef(Task.dest)) + } + + object test extends ScalaTests with TestModule.Utest { + + def ivyDeps = Agg( + ivy"com.lihaoyi::utest::0.8.4", + ivy"com.lihaoyi::requests::0.6.9" + ) + } + + object shared extends Module { + trait SharedModule extends AppScalaModule with PlatformScalaModule { + def ivyDeps = Agg( + ivy"com.lihaoyi::scalatags::0.12.0", + ivy"com.lihaoyi::upickle::3.0.0" + ) + } + + object jvm extends SharedModule + object js extends SharedModule with AppScalaJSModule + } + + object client extends AppScalaJSModule { + def moduleDeps = Seq(shared.js) + def ivyDeps = Agg(ivy"org.scala-js::scalajs-dom::2.2.0") + } +} + +// A Scala-JVM backend server wired up with a Scala.js front-end, with a +// `shared` module containing code that is used in both client and server. +// Rather than the server sending HTML for the initial page load and HTML for +// page updates, it sends HTML for the initial load and JSON for page updates +// which is then rendered into HTML on the client. +// +// The JSON serialization logic and HTML generation logic in the `shared` module +// is shared between client and server, and uses libraries like uPickle and +// Scalatags which work on both ScalaJVM and Scala.js. This allows us to freely +// move code between the client and server, without worrying about what +// platform or language the code was originally implemented in. +// +// This is a minimal example of shared code compiled to ScalaJVM and Scala.js, +// running on both client and server, meant for illustrating the build +// configuration. A full exploration of client-server code sharing techniques +// is beyond the scope of this example. + +/** Usage + +> ./mill test ++ webapp.WebAppTests.simpleRequest ... + +> ./mill runBackground + +> curl http://localhost:8083 +...What needs to be done... +... + +> curl http://localhost:8083/static/main.js +...Scala.js... +... + +> ./mill clean runBackground + +*/ diff --git a/integration/feature/reproducibility/resources/client/src/ClientApp.scala b/integration/feature/reproducibility/resources/client/src/ClientApp.scala new file mode 100644 index 00000000000..20e268655d8 --- /dev/null +++ b/integration/feature/reproducibility/resources/client/src/ClientApp.scala @@ -0,0 +1,80 @@ +package client +import org.scalajs.dom +import shared.{Todo, Shared} +object ClientApp { + var state = "all" + var todoApp = dom.document.getElementsByClassName("todoapp")(0) + + def postFetchUpdate(url: String) = { + dom.fetch( + url, + new dom.RequestInit { + method = dom.HttpMethod.POST + } + ).`then`[String](response => response.text()) + .`then`[Unit] { text => + todoApp.innerHTML = Shared + .renderBody(upickle.default.read[Seq[Todo]](text), state) + .render + + initListeners() + } + } + + def bindEvent(cls: String, url: String, endState: Option[String]) = { + dom.document.getElementsByClassName(cls)(0).addEventListener( + "mousedown", + (evt: dom.Event) => { + postFetchUpdate(url) + endState.foreach(state = _) + } + ) + } + + def bindIndexedEvent(cls: String, func: String => String) = { + for (elem <- dom.document.getElementsByClassName(cls)) { + elem.addEventListener( + "mousedown", + (evt: dom.Event) => postFetchUpdate(func(elem.getAttribute("data-todo-index"))) + ) + } + } + + def initListeners(): Unit = { + bindIndexedEvent("destroy", index => s"/delete/$state/$index") + bindIndexedEvent("toggle", index => s"/toggle/$state/$index") + bindEvent("toggle-all", s"/toggle-all/$state", None) + bindEvent("todo-all", s"/list/all", Some("all")) + bindEvent("todo-active", s"/list/all", Some("active")) + bindEvent("todo-completed", s"/list/completed", Some("completed")) + bindEvent("clear-completed", s"/clear-completed/$state", None) + + val newTodoInput = + dom.document.getElementsByClassName("new-todo")(0).asInstanceOf[dom.HTMLInputElement] + newTodoInput.addEventListener( + "keydown", + (evt: dom.KeyboardEvent) => { + if (evt.keyCode == 13) { + dom.fetch( + s"/add/$state", + new dom.RequestInit { + method = dom.HttpMethod.POST + body = newTodoInput.value + } + ).`then`[String](response => response.text()) + .`then`[Unit] { text => + newTodoInput.value = "" + + todoApp.innerHTML = Shared + .renderBody(upickle.default.read[Seq[Todo]](text), state) + .render + + initListeners() + } + } + } + ) + } + + def main(args: Array[String]): Unit = initListeners() +} diff --git a/integration/feature/reproducibility/resources/resources/webapp/index.css b/integration/feature/reproducibility/resources/resources/webapp/index.css new file mode 100644 index 00000000000..07ef4a160e3 --- /dev/null +++ b/integration/feature/reproducibility/resources/resources/webapp/index.css @@ -0,0 +1,378 @@ +html, +body { + margin: 0; + padding: 0; +} + +button { + margin: 0; + padding: 0; + border: 0; + background: none; + font-size: 100%; + vertical-align: baseline; + font-family: inherit; + font-weight: inherit; + color: inherit; + -webkit-appearance: none; + appearance: none; + -webkit-font-smoothing: antialiased; + -moz-font-smoothing: antialiased; + font-smoothing: antialiased; +} + +body { + font: 14px 'Helvetica Neue', Helvetica, Arial, sans-serif; + line-height: 1.4em; + background: #f5f5f5; + color: #4d4d4d; + min-width: 230px; + max-width: 550px; + margin: 0 auto; + -webkit-font-smoothing: antialiased; + -moz-font-smoothing: antialiased; + font-smoothing: antialiased; + font-weight: 300; +} + +button, +input[type="checkbox"] { + outline: none; +} + +.hidden { + display: none; +} + +.todoapp { + background: #fff; + margin: 130px 0 40px 0; + position: relative; + box-shadow: 0 2px 4px 0 rgba(0, 0, 0, 0.2), + 0 25px 50px 0 rgba(0, 0, 0, 0.1); +} + +.todoapp input::-webkit-input-placeholder { + font-style: italic; + font-weight: 300; + color: #e6e6e6; +} + +.todoapp input::-moz-placeholder { + font-style: italic; + font-weight: 300; + color: #e6e6e6; +} + +.todoapp input::input-placeholder { + font-style: italic; + font-weight: 300; + color: #e6e6e6; +} + +.todoapp h1 { + position: absolute; + top: -155px; + width: 100%; + font-size: 100px; + font-weight: 100; + text-align: center; + color: rgba(175, 47, 47, 0.15); + -webkit-text-rendering: optimizeLegibility; + -moz-text-rendering: optimizeLegibility; + text-rendering: optimizeLegibility; +} + +.new-todo, +.edit { + position: relative; + margin: 0; + width: 100%; + font-size: 24px; + font-family: inherit; + font-weight: inherit; + line-height: 1.4em; + border: 0; + outline: none; + color: inherit; + padding: 6px; + border: 1px solid #999; + box-shadow: inset 0 -1px 5px 0 rgba(0, 0, 0, 0.2); + box-sizing: border-box; + -webkit-font-smoothing: antialiased; + -moz-font-smoothing: antialiased; + font-smoothing: antialiased; +} + +.new-todo { + padding: 16px 16px 16px 60px; + border: none; + background: rgba(0, 0, 0, 0.003); + box-shadow: inset 0 -2px 1px rgba(0,0,0,0.03); +} + +.main { + position: relative; + z-index: 2; + border-top: 1px solid #e6e6e6; +} + +label[for='toggle-all'] { + display: none; +} + +.toggle-all { + position: absolute; + top: -55px; + left: -12px; + width: 60px; + height: 34px; + text-align: center; + border: none; /* Mobile Safari */ +} + +.toggle-all:before { + content: '❯'; + font-size: 22px; + color: #e6e6e6; + padding: 10px 27px 10px 27px; +} + +.toggle-all:checked:before { + color: #737373; +} + +.todo-list { + margin: 0; + padding: 0; + list-style: none; +} + +.todo-list li { + position: relative; + font-size: 24px; + border-bottom: 1px solid #ededed; +} + +.todo-list li:last-child { + border-bottom: none; +} + +.todo-list li.editing { + border-bottom: none; + padding: 0; +} + +.todo-list li.editing .edit { + display: block; + width: 506px; + padding: 13px 17px 12px 17px; + margin: 0 0 0 43px; +} + +.todo-list li.editing .view { + display: none; +} + +.todo-list li .toggle { + text-align: center; + width: 40px; + /* auto, since non-WebKit browsers doesn't support input styling */ + height: auto; + position: absolute; + top: 0; + bottom: 0; + margin: auto 0; + border: none; /* Mobile Safari */ + -webkit-appearance: none; + appearance: none; +} + +.todo-list li .toggle:after { + content: url('data:image/svg+xml;utf8,'); +} + +.todo-list li .toggle:checked:after { + content: url('data:image/svg+xml;utf8,'); +} + +.todo-list li label { + white-space: pre-line; + word-break: break-all; + padding: 15px 60px 15px 15px; + margin-left: 45px; + display: block; + line-height: 1.2; + transition: color 0.4s; +} + +.todo-list li.completed label { + color: #d9d9d9; + text-decoration: line-through; +} + +.todo-list li .destroy { + display: none; + position: absolute; + top: 0; + right: 10px; + bottom: 0; + width: 40px; + height: 40px; + margin: auto 0; + font-size: 30px; + color: #cc9a9a; + margin-bottom: 11px; + transition: color 0.2s ease-out; +} + +.todo-list li .destroy:hover { + color: #af5b5e; +} + +.todo-list li .destroy:after { + content: '×'; +} + +.todo-list li:hover .destroy { + display: block; +} + +.todo-list li .edit { + display: none; +} + +.todo-list li.editing:last-child { + margin-bottom: -1px; +} + +.footer { + color: #777; + padding: 10px 15px; + height: 20px; + text-align: center; + border-top: 1px solid #e6e6e6; +} + +.footer:before { + content: ''; + position: absolute; + right: 0; + bottom: 0; + left: 0; + height: 50px; + overflow: hidden; + box-shadow: 0 1px 1px rgba(0, 0, 0, 0.2), + 0 8px 0 -3px #f6f6f6, + 0 9px 1px -3px rgba(0, 0, 0, 0.2), + 0 16px 0 -6px #f6f6f6, + 0 17px 2px -6px rgba(0, 0, 0, 0.2); +} + +.todo-count { + float: left; + text-align: left; +} + +.todo-count strong { + font-weight: 300; +} + +.filters { + margin: 0; + padding: 0; + list-style: none; + position: absolute; + right: 0; + left: 0; +} + +.filters li { + display: inline; +} + +.filters li a { + color: inherit; + margin: 3px; + padding: 3px 7px; + text-decoration: none; + border: 1px solid transparent; + border-radius: 3px; +} + +.filters li a.selected, +.filters li a:hover { + border-color: rgba(175, 47, 47, 0.1); +} + +.filters li a.selected { + border-color: rgba(175, 47, 47, 0.2); +} + +.clear-completed, +html .clear-completed:active { + float: right; + position: relative; + line-height: 20px; + text-decoration: none; + cursor: pointer; + position: relative; +} + +.clear-completed:hover { + text-decoration: underline; +} + +.info { + margin: 65px auto 0; + color: #bfbfbf; + font-size: 10px; + text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5); + text-align: center; +} + +.info p { + line-height: 1; +} + +.info a { + color: inherit; + text-decoration: none; + font-weight: 400; +} + +.info a:hover { + text-decoration: underline; +} + +/* + Hack to remove background from Mobile Safari. + Can't use it globally since it destroys checkboxes in Firefox +*/ +@media screen and (-webkit-min-device-pixel-ratio:0) { + .toggle-all, + .todo-list li .toggle { + background: none; + } + + .todo-list li .toggle { + height: 40px; + } + + .toggle-all { + -webkit-transform: rotate(90deg); + transform: rotate(90deg); + -webkit-appearance: none; + appearance: none; + } +} + +@media (max-width: 430px) { + .footer { + height: 50px; + } + + .filters { + bottom: 10px; + } +} diff --git a/integration/feature/reproducibility/resources/shared/src/Shared.scala b/integration/feature/reproducibility/resources/shared/src/Shared.scala new file mode 100644 index 00000000000..6597725430f --- /dev/null +++ b/integration/feature/reproducibility/resources/shared/src/Shared.scala @@ -0,0 +1,69 @@ +package shared +import scalatags.Text.all._ +import scalatags.Text.tags2 + +case class Todo(checked: Boolean, text: String) + +object Todo { + implicit def todoRW: upickle.default.ReadWriter[Todo] = upickle.default.macroRW[Todo] +} + +object Shared { + def renderBody(todos: Seq[Todo], state: String) = { + val filteredTodos = state match { + case "all" => todos.zipWithIndex + case "active" => todos.zipWithIndex.filter(!_._1.checked) + case "completed" => todos.zipWithIndex.filter(_._1.checked) + } + div( + header(cls := "header")( + h1("todos"), + input(cls := "new-todo", placeholder := "What needs to be done?", autofocus := "") + ), + tags2.section(cls := "main")( + input( + id := "toggle-all", + cls := "toggle-all", + `type` := "checkbox", + if (todos.filter(_.checked).size != 0) checked else () + ), + label(`for` := "toggle-all")("Mark all as complete"), + ul(cls := "todo-list")( + for ((todo, index) <- filteredTodos) yield li( + if (todo.checked) cls := "completed" else (), + div(cls := "view")( + input( + cls := "toggle", + `type` := "checkbox", + if (todo.checked) checked else (), + data("todo-index") := index + ), + label(todo.text), + button(cls := "destroy", data("todo-index") := index) + ), + input(cls := "edit", value := todo.text) + ) + ) + ), + footer(cls := "footer")( + span(cls := "todo-count")( + strong(todos.filter(!_.checked).size), + " items left" + ), + ul(cls := "filters")( + li(cls := "todo-all")( + a(if (state == "all") cls := "selected" else ())("All") + ), + li(cls := "todo-active")( + a(if (state == "active") cls := "selected" else ())("Active") + ), + li(cls := "todo-completed")( + a(if (state == "completed") cls := "selected" else ())("Completed") + ) + ), + button(cls := "clear-completed")("Clear completed") + ) + ) + } + +} diff --git a/integration/feature/reproducibility/resources/src/WebApp.scala b/integration/feature/reproducibility/resources/src/WebApp.scala new file mode 100644 index 00000000000..bc93c6a4e94 --- /dev/null +++ b/integration/feature/reproducibility/resources/src/WebApp.scala @@ -0,0 +1,78 @@ +package webapp +import scalatags.Text.all._ +import scalatags.Text.tags2 +import shared.{Shared, Todo} + +object WebApp extends cask.MainRoutes { + override def port = 8083 + var todos = Seq( + Todo(true, "Get started with Cask"), + Todo(false, "Profit!") + ) + + @cask.post("/list/:state") + def list(state: String) = upickle.default.write(todos) + + @cask.post("/add/:state") + def add(state: String, request: cask.Request) = { + todos = Seq(Todo(false, request.text())) ++ todos + upickle.default.write(todos) + } + + @cask.post("/delete/:state/:index") + def delete(state: String, index: Int) = { + todos = todos.patch(index, Nil, 1) + upickle.default.write(todos) + } + + @cask.post("/toggle/:state/:index") + def toggle(state: String, index: Int) = { + todos = todos.updated(index, todos(index).copy(checked = !todos(index).checked)) + upickle.default.write(todos) + } + + @cask.post("/clear-completed/:state") + def clearCompleted(state: String) = { + todos = todos.filter(!_.checked) + upickle.default.write(todos) + } + + @cask.post("/toggle-all/:state") + def toggleAll(state: String) = { + val next = todos.filter(_.checked).size != 0 + todos = todos.map(_.copy(checked = next)) + upickle.default.write(todos) + } + + @cask.get("/") + def index() = { + doctype("html")( + html(lang := "en")( + head( + meta(charset := "utf-8"), + meta(name := "viewport", content := "width=device-width, initial-scale=1"), + tags2.title("Template • TodoMVC"), + link(rel := "stylesheet", href := "/static/index.css") + ), + body( + tags2.section(cls := "todoapp")(Shared.renderBody(todos, "all")), + footer(cls := "info")( + p("Double-click to edit a todo"), + p("Created by ")( + a(href := "http://todomvc.com")("Li Haoyi") + ), + p("Part of ")( + a(href := "http://todomvc.com")("TodoMVC") + ) + ), + script(src := "/static/main.js") + ) + ) + ) + } + + @cask.staticResources("/static") + def static() = "webapp" + + initialize() +} diff --git a/integration/feature/reproducibility/resources/test/src/WebAppTests.scala b/integration/feature/reproducibility/resources/test/src/WebAppTests.scala new file mode 100644 index 00000000000..459fd56bec0 --- /dev/null +++ b/integration/feature/reproducibility/resources/test/src/WebAppTests.scala @@ -0,0 +1,24 @@ +package webapp + +import utest._ + +object WebAppTests extends TestSuite { + def withServer[T](example: cask.main.Main)(f: String => T): T = { + val server = io.undertow.Undertow.builder + .addHttpListener(8185, "localhost") + .setHandler(example.defaultHandler) + .build + server.start() + val res = + try f("http://localhost:8185") + finally server.stop() + res + } + + val tests = Tests { + test("simpleRequest") - withServer(WebApp) { host => + val page = requests.get(host).text() + assert(page.contains("What needs to be done?")) + } + } +} diff --git a/integration/feature/reproducibility/src/ReproducibilityTests.scala b/integration/feature/reproducibility/src/ReproducibilityTests.scala new file mode 100644 index 00000000000..9452d249b0d --- /dev/null +++ b/integration/feature/reproducibility/src/ReproducibilityTests.scala @@ -0,0 +1,73 @@ +package mill.integration + +import mill.testkit.UtestIntegrationTestSuite +import utest._ + +import java.util.zip.GZIPInputStream + +object ReproducibilityTests extends UtestIntegrationTestSuite { + + def normalize(workspacePath: os.Path): Unit = { + for (p <- os.walk(workspacePath / "out")) { + val sub = p.subRelativeTo(workspacePath).toString() + + val cacheable = + (sub.contains(".dest") || sub.contains(".json") || os.isDir(p)) && + !sub.replace("out/mill-build", "").contains("mill-") && + !(p.ext == "json" && ujson.read( + os.read(p) + ).objOpt.flatMap(_.get("value")).flatMap(_.objOpt).flatMap(_.get("worker")).nonEmpty) + + if (!cacheable) { + os.remove.all(p) + } + } + } + + val tests: Tests = Tests { + test("diff") - { + def run() = integrationTest { tester => + tester.eval(("--meta-level", "1", "runClasspath")) + tester.workspacePath + } + + val workspacePath1 = run() + val workspacePath2 = run() + assert(workspacePath1 != workspacePath2) + normalize(workspacePath1) + normalize(workspacePath2) + val diff = os.call(("git", "diff", "--no-index", workspacePath1, workspacePath2)).out.text() + pprint.log(diff) + assert(diff.isEmpty) + } + + test("inspection") - { + def run() = integrationTest { tester => + tester.eval( + ("--meta-level", "1", "runClasspath"), + env = Map("MILL_TEST_TEXT_ANALYSIS_STORE" -> "1"), + check = true + ) + tester.workspacePath + } + + val workspacePath = run() + val dest = workspacePath / "out/mill-build/compile.dest/zinc.txt" + val src = workspacePath / "out/mill-build/compile.dest/zinc" + os.write(dest, new GZIPInputStream(os.read.inputStream(src))) + normalize(workspacePath) + for (p <- os.walk(workspacePath)) { + if ( + (p.ext == "json" || p.ext == "txt") + && !p.segments.contains("enablePluginScalacOptions.super") + && !p.segments.contains("allScalacOptions.json") + && !p.segments.contains("scalacOptions.json") + ) { + val txt = os.read(p) + Predef.assert(!txt.contains(mill.api.WorkspaceRoot.workspaceRoot.toString), p) + Predef.assert(!txt.contains(os.home.toString), p) + } + } + } + } +} diff --git a/main/api/src/mill/api/JsonFormatters.scala b/main/api/src/mill/api/JsonFormatters.scala index b708070fcda..2974fee1cfa 100644 --- a/main/api/src/mill/api/JsonFormatters.scala +++ b/main/api/src/mill/api/JsonFormatters.scala @@ -32,8 +32,8 @@ trait JsonFormatters { implicit val pathReadWrite: RW[os.Path] = upickle.default.readwriter[String] .bimap[os.Path]( - _.toString, - os.Path(_) + p => PathRef.normalizePath(p).toString, + s => PathRef.denormalizePath(os.Path(s)) ) implicit val regexReadWrite: RW[Regex] = upickle.default.readwriter[String] diff --git a/main/api/src/mill/api/PathRef.scala b/main/api/src/mill/api/PathRef.scala index ec9c4d516e4..48431c369fe 100644 --- a/main/api/src/mill/api/PathRef.scala +++ b/main/api/src/mill/api/PathRef.scala @@ -8,6 +8,7 @@ import java.util.concurrent.ConcurrentHashMap import scala.util.{DynamicVariable, Using} import upickle.default.{ReadWriter => RW} import scala.annotation.nowarn +import os.Path /** * A wrapper around `os.Path` that calculates it's hashcode based @@ -43,11 +44,25 @@ case class PathRef private ( case PathRef.Revalidate.Always => "vn:" } val sig = String.format("%08x", this.sig: Integer) - quick + valid + sig + ":" + path.toString() + quick + valid + sig + ":" + PathRef.normalizePath(path).toString() } } object PathRef { + def mapPathPrefixes(path: os.Path, mapping: Seq[(os.Path, os.Path)]): os.Path = { + mapping + .collectFirst { case (from, to) if path.startsWith(from) => to / path.subRelativeTo(from) } + .getOrElse(path) + } + + val defaultMapping: Seq[(Path, Path)] = Seq( + mill.api.WorkspaceRoot.workspaceRoot -> os.root / "$WORKSPACE", + os.home -> os.root / "$HOME" + ) + def normalizePath(path: os.Path): os.Path = mapPathPrefixes(path, defaultMapping) + + def denormalizePath(path: os.Path): os.Path = mapPathPrefixes(path, defaultMapping.map(_.swap)) + implicit def shellable(p: PathRef): os.Shellable = p.path /** @@ -92,7 +107,7 @@ object PathRef { } def apply(path: os.Path, quick: Boolean, sig: Int, revalidate: Revalidate): PathRef = - new PathRef(path, quick, sig, revalidate) + new PathRef(PathRef.denormalizePath(path), quick, sig, revalidate) /** * Create a [[PathRef]] by recursively digesting the content of a given `path`. @@ -107,10 +122,10 @@ object PathRef { quick: Boolean = false, revalidate: Revalidate = Revalidate.Never ): PathRef = { - val basePath = path + val basePath = PathRef.denormalizePath(path) val sig = { - val isPosix = path.wrapped.getFileSystem.supportedFileAttributeViews().contains("posix") + val isPosix = basePath.wrapped.getFileSystem.supportedFileAttributeViews().contains("posix") val digest = MessageDigest.getInstance("MD5") val digestOut = new DigestOutputStream(DummyOutputStream, digest) @@ -121,10 +136,10 @@ object PathRef { digest.update(value.toByte) } - if (os.exists(path)) { + if (os.exists(basePath)) { for ( (path, attrs) <- - os.walk.attrs(path, includeTarget = true, followLinks = true).sortBy(_._1.toString) + os.walk.attrs(basePath, includeTarget = true, followLinks = true).sortBy(_._1.toString) ) { val sub = path.subRelativeTo(basePath) digest.update(sub.toString().getBytes()) @@ -169,7 +184,7 @@ object PathRef { java.util.Arrays.hashCode(digest.digest()) } - new PathRef(path, quick, sig, revalidate) + new PathRef(basePath, quick, sig, revalidate) } /** diff --git a/main/define/src/mill/define/Task.scala b/main/define/src/mill/define/Task.scala index 21f3faa7412..fb9d0052e72 100644 --- a/main/define/src/mill/define/Task.scala +++ b/main/define/src/mill/define/Task.scala @@ -802,7 +802,7 @@ class InputImpl[T]( val writer: upickle.default.Writer[_], val isPrivate: Option[Boolean] ) extends Target[T] { - override def sideHash: Int = util.Random.nextInt() + override def sideHash: Int = 31337 // FIXME: deprecated return type: Change to Option override def writerOpt: Some[W[_]] = Some(writer) } diff --git a/main/eval/src/mill/eval/GroupEvaluator.scala b/main/eval/src/mill/eval/GroupEvaluator.scala index 4f62dcd4c1e..d05ee4a6be3 100644 --- a/main/eval/src/mill/eval/GroupEvaluator.scala +++ b/main/eval/src/mill/eval/GroupEvaluator.scala @@ -43,6 +43,14 @@ private[mill] trait GroupEvaluator { val effectiveThreadCount: Int = this.threadCount.getOrElse(Runtime.getRuntime().availableProcessors()) + def getValueHash(t: Task[_], inputsHash: Int, v: Val, jsonOpt: Option[ujson.Value]): Int = { + if (t.isInstanceOf[Worker[_]]) inputsHash + else jsonOpt match { + case Some(json) => json.hashCode() + case None => v.## + } + } + // those result which are inputs but not contained in this terminal group def evaluateGroupCached( terminal: Terminal, @@ -70,7 +78,7 @@ private[mill] trait GroupEvaluator { .flatMap(results(_).result.asSuccess.map(_.value._2)) ) - val sideHashes = MurmurHash3.orderedHash(group.iterator.map(_.sideHash)) + val sideHashes = group.iterator.map(_.sideHash).sum val scriptsHash = if (disableCallgraph) 0 @@ -129,6 +137,11 @@ private[mill] trait GroupEvaluator { val inputsHash = externalInputsHash + sideHashes + classLoaderSigHash + scriptsHash + def transformResults(results: Map[Task[_], TaskResult[(Val, Option[ujson.Value])]]) = { + for ((k, v0) <- results) + yield k -> v0.map { case (v, jsonOpt) => (v, getValueHash(k, inputsHash, v, jsonOpt)) } + } + terminal match { case Terminal.Task(task) => val (newResults, newEvaluated) = evaluateGroup( @@ -145,7 +158,13 @@ private[mill] trait GroupEvaluator { executionContext, exclusive ) - GroupEvaluator.Results(newResults, newEvaluated.toSeq, null, inputsHash, -1) + GroupEvaluator.Results( + transformResults(newResults), + newEvaluated.toSeq, + null, + inputsHash, + -1 + ) case labelled: Terminal.Labelled[_] => val out = @@ -157,7 +176,9 @@ private[mill] trait GroupEvaluator { Terminal.destSegments(labelled) ) - val cached = loadCachedJson(logger, inputsHash, labelled, paths) + val cached = Option + .when(sideHashes == 0) { loadCachedJson(logger, inputsHash, labelled, paths) } + .flatten val upToDateWorker = loadUpToDateWorker( logger, @@ -202,12 +223,13 @@ private[mill] trait GroupEvaluator { exclusive ) + val transformedResults = transformResults(newResults) newResults(labelled.task) match { - case TaskResult(Result.Failure(_, Some((v, _))), _) => - handleTaskResult(v, v.##, paths.meta, inputsHash, labelled) + case TaskResult(Result.Failure(_, Some((v, jsonOpt))), _) => + handleTaskResult(v, jsonOpt, paths.meta, inputsHash, labelled) - case TaskResult(Result.Success((v, _)), _) => - handleTaskResult(v, v.##, paths.meta, inputsHash, labelled) + case TaskResult(Result.Success((v, jsonOpt)), _) => + handleTaskResult(v, jsonOpt, paths.meta, inputsHash, labelled) case _ => // Wipe out any cached meta.json file that exists, so @@ -218,7 +240,7 @@ private[mill] trait GroupEvaluator { } GroupEvaluator.Results( - newResults, + transformedResults, newEvaluated.toSeq, cached = if (labelled.task.isInstanceOf[InputImpl[_]]) null else false, inputsHash, @@ -242,11 +264,11 @@ private[mill] trait GroupEvaluator { logger: mill.api.Logger, executionContext: mill.api.Ctx.Fork.Api, exclusive: Boolean - ): (Map[Task[_], TaskResult[(Val, Int)]], mutable.Buffer[Task[_]]) = { + ): (Map[Task[_], TaskResult[(Val, Option[ujson.Value])]], mutable.Buffer[Task[_]]) = { def computeAll() = { val newEvaluated = mutable.Buffer.empty[Task[_]] - val newResults = mutable.Map.empty[Task[_], Result[(Val, Int)]] + val newResults = mutable.Map.empty[Task[_], Result[(Val, Option[ujson.Value])]] val nonEvaluatedTargets = group.indexed.filterNot(results.contains) @@ -319,11 +341,15 @@ private[mill] trait GroupEvaluator { } newResults(task) = for (v <- res) yield { - ( - v, - if (task.isInstanceOf[Worker[_]]) inputsHash - else v.## - ) + val jsonOpt = task match { + case n: NamedTask[_] => + for (w <- n.writerOpt) yield { + upickle.default.writeJs(v.value)(using w.asInstanceOf[upickle.default.Writer[Any]]) + } + case _ => None + } + + (v, jsonOpt) } } @@ -369,39 +395,33 @@ private[mill] trait GroupEvaluator { private def handleTaskResult( v: Val, - hashCode: Int, + jsonOpt: Option[ujson.Value], metaPath: os.Path, inputsHash: Int, labelled: Terminal.Labelled[_] ): Unit = { + for (w <- labelled.task.asWorker) workerCache.synchronized { workerCache.update(w.ctx.segments, (workerCacheHash(inputsHash), v)) } - val terminalResult = labelled - .task - .writerOpt - .map { w => - upickle.default.writeJs(v.value)(using w.asInstanceOf[upickle.default.Writer[Any]]) - } - .orElse { - labelled.task.asWorker.map { w => - ujson.Obj( - "worker" -> ujson.Str(labelled.segments.render), - "toString" -> ujson.Str(v.value.toString), - "inputsHash" -> ujson.Num(inputsHash) - ) - } + val valueHash = getValueHash(labelled.task, inputsHash, v, jsonOpt) + + val terminalResult = jsonOpt.orElse { + labelled.task.asWorker.map { w => + ujson.Obj( + "worker" -> ujson.Str(labelled.segments.render), + "toString" -> ujson.Str(v.value.toString), + "inputsHash" -> ujson.Num(inputsHash) + ) } + } for (json <- terminalResult) { os.write.over( metaPath, - upickle.default.stream( - Evaluator.Cached(json, hashCode, inputsHash), - indent = 4 - ), + upickle.default.stream(Evaluator.Cached(json, valueHash, inputsHash), indent = 4), createFolders = true ) } diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala index 34c1ec3e15a..4125fbe30ea 100644 --- a/main/src/mill/main/MainModule.scala +++ b/main/src/mill/main/MainModule.scala @@ -216,10 +216,10 @@ trait MainModule extends BaseModule0 { def renderFileName(t: NamedTask[_]) = { // handle both Windows or Unix separators val fullFileName = t.ctx.fileName.replaceAll(raw"\\", "/") - val basePath = WorkspaceRoot.workspaceRoot.toString().replaceAll(raw"\\", "/") + "/" + WorkspaceRoot.workspaceRoot.toString().replaceAll(raw"\\", "/") + "/" val name = - if (fullFileName.startsWith(basePath)) { - fullFileName.drop(basePath.length) + if (fullFileName.startsWith("/$WORKSPACE/")) { + fullFileName.drop("/$WORKSPACE/".length) } else { fullFileName.split('/').last } diff --git a/main/src/mill/main/RootModule.scala b/main/src/mill/main/RootModule.scala index 33e63f1699b..c391d99c011 100644 --- a/main/src/mill/main/RootModule.scala +++ b/main/src/mill/main/RootModule.scala @@ -1,6 +1,7 @@ package mill.main import mill.api.internal +import mill.api.PathRef import mill.define.{Caller, Discover} import scala.annotation.compileTimeOnly @@ -45,10 +46,10 @@ object RootModule { output0: String, topLevelProjectRoot0: String ) = this( - enclosingClasspath0.map(os.Path(_)), - os.Path(projectRoot0), - os.Path(output0), - os.Path(topLevelProjectRoot0) + enclosingClasspath0.map(s => PathRef.denormalizePath(os.Path(s))), + PathRef.denormalizePath(os.Path(projectRoot0)), + PathRef.denormalizePath(os.Path(output0)), + PathRef.denormalizePath(os.Path(topLevelProjectRoot0)) ) implicit val millMiscInfo: Info = this diff --git a/runner/src/mill/runner/CodeGen.scala b/runner/src/mill/runner/CodeGen.scala index 5e81f7e7568..3f3f411c1fd 100644 --- a/runner/src/mill/runner/CodeGen.scala +++ b/runner/src/mill/runner/CodeGen.scala @@ -80,7 +80,7 @@ object CodeGen { val scriptCode = allScriptCode(scriptPath) val markerComment = - s"""//MILL_ORIGINAL_FILE_PATH=$scriptPath + s"""//MILL_ORIGINAL_FILE_PATH=${PathRef.normalizePath(scriptPath)} |//MILL_USER_CODE_START_MARKER""".stripMargin val parts = @@ -200,10 +200,10 @@ object CodeGen { s"""import _root_.mill.runner.MillBuildRootModule |@_root_.scala.annotation.nowarn |object MillMiscInfo extends mill.main.RootModule.Info( - | ${enclosingClasspath.map(p => literalize(p.toString))}, - | ${literalize(scriptFolderPath.toString)}, - | ${literalize(output.toString)}, - | ${literalize(millTopLevelProjectRoot.toString)} + | ${enclosingClasspath.map(p => literalize(PathRef.normalizePath(p).toString))}, + | ${literalize(PathRef.normalizePath(scriptFolderPath).toString)}, + | ${literalize(PathRef.normalizePath(output).toString)}, + | ${literalize(PathRef.normalizePath(millTopLevelProjectRoot).toString)} |) |import MillMiscInfo._ |""".stripMargin diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index dc078a7b8e5..8ddc80ca0db 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -167,7 +167,7 @@ class MillBuildBootstrap( .headOption .map(_.runClasspath) .getOrElse(millBootClasspathPathRefs) - .map(p => (p.path, p.sig)) + .map(p => (PathRef.normalizePath(p.path), p.sig)) .hashCode(), nestedState .frames diff --git a/scalalib/src/mill/scalalib/TestModule.scala b/scalalib/src/mill/scalalib/TestModule.scala index 8d9c4435dd0..34921ec0ae7 100644 --- a/scalalib/src/mill/scalalib/TestModule.scala +++ b/scalalib/src/mill/scalalib/TestModule.scala @@ -148,12 +148,12 @@ trait TestModule val testArgs = TestArgs( framework = testFramework(), - classpath = runClasspath().map(_.path), + classpath = runClasspath().map(_.path.toString), arguments = args(), sysProps = Map.empty, - outputPath = outputPath, + outputPath = outputPath.toString, colored = T.log.colored, - testCp = testClasspath().map(_.path), + testCp = testClasspath().map(_.path.toString), home = T.home, globSelectors = selectors ) diff --git a/scalalib/src/mill/scalalib/TestModuleUtil.scala b/scalalib/src/mill/scalalib/TestModuleUtil.scala index 55a6595f326..6141eaf0327 100644 --- a/scalalib/src/mill/scalalib/TestModuleUtil.scala +++ b/scalalib/src/mill/scalalib/TestModuleUtil.scala @@ -47,12 +47,12 @@ private[scalalib] object TestModuleUtil { val outputPath = base / "out.json" val testArgs = TestArgs( framework = testFramework, - classpath = runClasspath.map(_.path), + classpath = runClasspath.map(_.path.toString), arguments = args, sysProps = props, - outputPath = outputPath, + outputPath = outputPath.toString, colored = T.log.colored, - testCp = testClasspath.map(_.path), + testCp = testClasspath.map(_.path.toString), home = T.home, globSelectors = selectors2 ) diff --git a/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala index d70dbc2a03c..e328ce95285 100644 --- a/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala +++ b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala @@ -10,6 +10,7 @@ import sbt.internal.inc.{ FreshCompilerCache, ManagedLoggedReporter, MappedFileConverter, + MappedVirtualFile, ScalaInstance, Stamps, ZincUtil, @@ -19,6 +20,8 @@ import sbt.internal.inc.classpath.ClasspathUtil import sbt.internal.inc.consistent.ConsistentFileAnalysisStore import sbt.internal.util.{ConsoleAppender, ConsoleOut} import sbt.mill.SbtLoggerUtils +import xsbti.compile.analysis.{ReadMapper, Stamp, WriteMapper} +import xsbti.VirtualFileRef import xsbti.compile.analysis.ReadWriteMappers import xsbti.compile.{ AnalysisContents, @@ -38,6 +41,7 @@ import xsbti.compile.CompileProgress import java.io.File import java.net.URLClassLoader +import java.nio.file import java.util.Optional import scala.collection.mutable import scala.util.Properties.isWin @@ -483,14 +487,80 @@ class ZincWorkerImpl( } } - private def fileAnalysisStore(path: os.Path): AnalysisStore = - ConsistentFileAnalysisStore.binary( - file = path.toIO, - mappers = ReadWriteMappers.getEmptyMappers(), - sort = true, - // No need to utilize more than 8 cores to serialize a small file - parallelism = math.min(Runtime.getRuntime.availableProcessors(), 8) + private def fileAnalysisStore(path: os.Path): AnalysisStore = { + def denormalizeVirtualFile(file: VirtualFileRef) = file match { + case b: MappedVirtualFile => + MappedVirtualFile(PathRef.denormalizePath(os.Path(b.toPath())).toString(), Map()) + case b => b + } + def normalizeVirtualFile(file: VirtualFileRef) = file match { + case b: MappedVirtualFile => + MappedVirtualFile(PathRef.normalizePath(os.Path(b.toPath())).toString(), Map()) + case b => b + } + def denormalizeNioPath(file: java.nio.file.Path) = PathRef.denormalizePath(os.Path(file)).toNIO + def normalizeNioPath(file: java.nio.file.Path) = PathRef.normalizePath(os.Path(file)).toNIO + def normalizeString(str: String) = { + PathRef.defaultMapping.foldLeft(str) { case (s, (long, short)) => + s.replace(long.toString(), short.toString()) + } + } + def denormalizeString(str: String) = { + PathRef.defaultMapping.foldLeft(str) { case (s, (long, short)) => + s.replace(short.toString(), long.toString()) + } + } + val mappers = new ReadWriteMappers( + new ReadMapper { + override def mapSourceFile(sourceFile: VirtualFileRef): VirtualFileRef = + denormalizeVirtualFile(sourceFile) + override def mapBinaryFile(binaryFile: VirtualFileRef): VirtualFileRef = + denormalizeVirtualFile(binaryFile) + override def mapProductFile(productFile: VirtualFileRef): VirtualFileRef = + denormalizeVirtualFile(productFile) + override def mapOutputDir(outputDir: file.Path): file.Path = denormalizeNioPath(outputDir) + override def mapSourceDir(sourceDir: file.Path): file.Path = denormalizeNioPath(sourceDir) + override def mapClasspathEntry(classpathEntry: file.Path): file.Path = + denormalizeNioPath(classpathEntry) + override def mapJavacOption(javacOption: String): String = denormalizeString(javacOption) + override def mapScalacOption(scalacOption: String): String = denormalizeString(scalacOption) + override def mapBinaryStamp(file: VirtualFileRef, binaryStamp: Stamp): Stamp = binaryStamp + override def mapSourceStamp(file: VirtualFileRef, sourceStamp: Stamp): Stamp = sourceStamp + override def mapProductStamp(file: VirtualFileRef, productStamp: Stamp): Stamp = + productStamp + override def mapMiniSetup(miniSetup: MiniSetup): MiniSetup = miniSetup + }, + new WriteMapper { + override def mapSourceFile(sourceFile: VirtualFileRef): VirtualFileRef = + normalizeVirtualFile(sourceFile) + override def mapBinaryFile(binaryFile: VirtualFileRef): VirtualFileRef = + normalizeVirtualFile(binaryFile) + override def mapProductFile(productFile: VirtualFileRef): VirtualFileRef = + normalizeVirtualFile(productFile) + override def mapOutputDir(outputDir: file.Path): file.Path = normalizeNioPath(outputDir) + override def mapSourceDir(sourceDir: file.Path): file.Path = normalizeNioPath(sourceDir) + override def mapClasspathEntry(classpathEntry: file.Path): file.Path = + normalizeNioPath(classpathEntry) + override def mapJavacOption(javacOption: String): String = normalizeString(javacOption) + override def mapScalacOption(scalacOption: String): String = normalizeString(scalacOption) + override def mapBinaryStamp(file: VirtualFileRef, binaryStamp: Stamp): Stamp = binaryStamp + override def mapSourceStamp(file: VirtualFileRef, sourceStamp: Stamp): Stamp = sourceStamp + override def mapProductStamp(file: VirtualFileRef, productStamp: Stamp): Stamp = + productStamp + override def mapMiniSetup(miniSetup: MiniSetup): MiniSetup = miniSetup + } ) + // No need to utilize more than 8 cores to serialize a small file + val parallelism = math.min(Runtime.getRuntime.availableProcessors(), 8) + + if (System.getenv("MILL_TEST_TEXT_ANALYSIS_STORE") != null) { + ConsistentFileAnalysisStore + .text(file = path.toIO, mappers = mappers, sort = true, parallelism = parallelism) + } else { + ConsistentFileAnalysisStore + .binary(file = path.toIO, mappers = mappers, sort = true, parallelism = parallelism) + } + } private def compileInternal( upstreamCompileOutput: Seq[CompilationResult], diff --git a/testrunner/src/mill/testrunner/Model.scala b/testrunner/src/mill/testrunner/Model.scala index 20fb5472b8d..f1873e3a483 100644 --- a/testrunner/src/mill/testrunner/Model.scala +++ b/testrunner/src/mill/testrunner/Model.scala @@ -5,12 +5,12 @@ import mill.api.internal @internal case class TestArgs( framework: String, - classpath: Seq[os.Path], + classpath: Seq[String], arguments: Seq[String], sysProps: Map[String, String], - outputPath: os.Path, + outputPath: String, colored: Boolean, - testCp: Seq[os.Path], + testCp: Seq[String], home: os.Path, globSelectors: Seq[String] ) diff --git a/testrunner/src/mill/testrunner/TestRunnerMain0.scala b/testrunner/src/mill/testrunner/TestRunnerMain0.scala index a0656cefb6d..75291e9c4e6 100644 --- a/testrunner/src/mill/testrunner/TestRunnerMain0.scala +++ b/testrunner/src/mill/testrunner/TestRunnerMain0.scala @@ -30,7 +30,7 @@ import mill.util.PrintLogger val result = TestRunnerUtils.runTestFramework0( frameworkInstances = Framework.framework(testArgs.framework), - testClassfilePath = Agg.from(testArgs.testCp), + testClassfilePath = Agg.from(testArgs.testCp.map(os.Path(_))), args = testArgs.arguments, classFilter = cls => filter(cls.getName), cl = classLoader, @@ -41,7 +41,7 @@ import mill.util.PrintLogger // dirtied the thread-interrupted flag and forgot to clean up. Otherwise, // that flag causes writing the results to disk to fail Thread.interrupted() - os.write(testArgs.outputPath, upickle.default.stream(result)) + os.write(os.Path(testArgs.outputPath), upickle.default.stream(result)) } catch { case e: Throwable => println(e)