Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Reproducible portable out folder contents #4065

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions integration/feature/reproducibility/resources/build.mill
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package build
import mill._, scalalib._, scalajslib._

trait AppScalaModule extends ScalaModule {
def scalaVersion = "3.3.3"
}

trait AppScalaJSModule extends AppScalaModule with ScalaJSModule {
def scalaJSVersion = "1.16.0"
}

object `package` extends RootModule with AppScalaModule {
def moduleDeps = Seq(shared.jvm)
def ivyDeps = Agg(ivy"com.lihaoyi::cask:0.9.1")

def resources = Task {
os.makeDir(Task.dest / "webapp")
val jsPath = client.fastLinkJS().dest.path
os.copy(jsPath / "main.js", Task.dest / "webapp/main.js")
os.copy(jsPath / "main.js.map", Task.dest / "webapp/main.js.map")
super.resources() ++ Seq(PathRef(Task.dest))
}

object test extends ScalaTests with TestModule.Utest {

def ivyDeps = Agg(
ivy"com.lihaoyi::utest::0.8.4",
ivy"com.lihaoyi::requests::0.6.9"
)
}

object shared extends Module {
trait SharedModule extends AppScalaModule with PlatformScalaModule {
def ivyDeps = Agg(
ivy"com.lihaoyi::scalatags::0.12.0",
ivy"com.lihaoyi::upickle::3.0.0"
)
}

object jvm extends SharedModule
object js extends SharedModule with AppScalaJSModule
}

object client extends AppScalaJSModule {
def moduleDeps = Seq(shared.js)
def ivyDeps = Agg(ivy"org.scala-js::scalajs-dom::2.2.0")
}
}

// A Scala-JVM backend server wired up with a Scala.js front-end, with a
// `shared` module containing code that is used in both client and server.
// Rather than the server sending HTML for the initial page load and HTML for
// page updates, it sends HTML for the initial load and JSON for page updates
// which is then rendered into HTML on the client.
//
// The JSON serialization logic and HTML generation logic in the `shared` module
// is shared between client and server, and uses libraries like uPickle and
// Scalatags which work on both ScalaJVM and Scala.js. This allows us to freely
// move code between the client and server, without worrying about what
// platform or language the code was originally implemented in.
//
// This is a minimal example of shared code compiled to ScalaJVM and Scala.js,
// running on both client and server, meant for illustrating the build
// configuration. A full exploration of client-server code sharing techniques
// is beyond the scope of this example.

/** Usage

> ./mill test
+ webapp.WebAppTests.simpleRequest ...

> ./mill runBackground

> curl http://localhost:8083
...What needs to be done...
...

> curl http://localhost:8083/static/main.js
...Scala.js...
...

> ./mill clean runBackground

*/
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package client
import org.scalajs.dom
import shared.{Todo, Shared}
object ClientApp {
var state = "all"
var todoApp = dom.document.getElementsByClassName("todoapp")(0)

def postFetchUpdate(url: String) = {
dom.fetch(
url,
new dom.RequestInit {
method = dom.HttpMethod.POST
}
).`then`[String](response => response.text())
.`then`[Unit] { text =>
todoApp.innerHTML = Shared
.renderBody(upickle.default.read[Seq[Todo]](text), state)
.render

initListeners()
}
}

def bindEvent(cls: String, url: String, endState: Option[String]) = {
dom.document.getElementsByClassName(cls)(0).addEventListener(
"mousedown",
(evt: dom.Event) => {
postFetchUpdate(url)
endState.foreach(state = _)
}
)
}

def bindIndexedEvent(cls: String, func: String => String) = {
for (elem <- dom.document.getElementsByClassName(cls)) {
elem.addEventListener(
"mousedown",
(evt: dom.Event) => postFetchUpdate(func(elem.getAttribute("data-todo-index")))
)
}
}

def initListeners(): Unit = {
bindIndexedEvent("destroy", index => s"/delete/$state/$index")
bindIndexedEvent("toggle", index => s"/toggle/$state/$index")
bindEvent("toggle-all", s"/toggle-all/$state", None)
bindEvent("todo-all", s"/list/all", Some("all"))
bindEvent("todo-active", s"/list/all", Some("active"))
bindEvent("todo-completed", s"/list/completed", Some("completed"))
bindEvent("clear-completed", s"/clear-completed/$state", None)

val newTodoInput =
dom.document.getElementsByClassName("new-todo")(0).asInstanceOf[dom.HTMLInputElement]
newTodoInput.addEventListener(
"keydown",
(evt: dom.KeyboardEvent) => {
if (evt.keyCode == 13) {
dom.fetch(
s"/add/$state",
new dom.RequestInit {
method = dom.HttpMethod.POST
body = newTodoInput.value
}
).`then`[String](response => response.text())
.`then`[Unit] { text =>
newTodoInput.value = ""

todoApp.innerHTML = Shared
.renderBody(upickle.default.read[Seq[Todo]](text), state)
.render

initListeners()
}
}
}
)
}

def main(args: Array[String]): Unit = initListeners()
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,378 @@
html,
body {
margin: 0;
padding: 0;
}

button {
margin: 0;
padding: 0;
border: 0;
background: none;
font-size: 100%;
vertical-align: baseline;
font-family: inherit;
font-weight: inherit;
color: inherit;
-webkit-appearance: none;
appearance: none;
-webkit-font-smoothing: antialiased;
-moz-font-smoothing: antialiased;
font-smoothing: antialiased;
}

body {
font: 14px 'Helvetica Neue', Helvetica, Arial, sans-serif;
line-height: 1.4em;
background: #f5f5f5;
color: #4d4d4d;
min-width: 230px;
max-width: 550px;
margin: 0 auto;
-webkit-font-smoothing: antialiased;
-moz-font-smoothing: antialiased;
font-smoothing: antialiased;
font-weight: 300;
}

button,
input[type="checkbox"] {
outline: none;
}

.hidden {
display: none;
}

.todoapp {
background: #fff;
margin: 130px 0 40px 0;
position: relative;
box-shadow: 0 2px 4px 0 rgba(0, 0, 0, 0.2),
0 25px 50px 0 rgba(0, 0, 0, 0.1);
}

.todoapp input::-webkit-input-placeholder {
font-style: italic;
font-weight: 300;
color: #e6e6e6;
}

.todoapp input::-moz-placeholder {
font-style: italic;
font-weight: 300;
color: #e6e6e6;
}

.todoapp input::input-placeholder {
font-style: italic;
font-weight: 300;
color: #e6e6e6;
}

.todoapp h1 {
position: absolute;
top: -155px;
width: 100%;
font-size: 100px;
font-weight: 100;
text-align: center;
color: rgba(175, 47, 47, 0.15);
-webkit-text-rendering: optimizeLegibility;
-moz-text-rendering: optimizeLegibility;
text-rendering: optimizeLegibility;
}

.new-todo,
.edit {
position: relative;
margin: 0;
width: 100%;
font-size: 24px;
font-family: inherit;
font-weight: inherit;
line-height: 1.4em;
border: 0;
outline: none;
color: inherit;
padding: 6px;
border: 1px solid #999;
box-shadow: inset 0 -1px 5px 0 rgba(0, 0, 0, 0.2);
box-sizing: border-box;
-webkit-font-smoothing: antialiased;
-moz-font-smoothing: antialiased;
font-smoothing: antialiased;
}

.new-todo {
padding: 16px 16px 16px 60px;
border: none;
background: rgba(0, 0, 0, 0.003);
box-shadow: inset 0 -2px 1px rgba(0,0,0,0.03);
}

.main {
position: relative;
z-index: 2;
border-top: 1px solid #e6e6e6;
}

label[for='toggle-all'] {
display: none;
}

.toggle-all {
position: absolute;
top: -55px;
left: -12px;
width: 60px;
height: 34px;
text-align: center;
border: none; /* Mobile Safari */
}

.toggle-all:before {
content: '❯';
font-size: 22px;
color: #e6e6e6;
padding: 10px 27px 10px 27px;
}

.toggle-all:checked:before {
color: #737373;
}

.todo-list {
margin: 0;
padding: 0;
list-style: none;
}

.todo-list li {
position: relative;
font-size: 24px;
border-bottom: 1px solid #ededed;
}

.todo-list li:last-child {
border-bottom: none;
}

.todo-list li.editing {
border-bottom: none;
padding: 0;
}

.todo-list li.editing .edit {
display: block;
width: 506px;
padding: 13px 17px 12px 17px;
margin: 0 0 0 43px;
}

.todo-list li.editing .view {
display: none;
}

.todo-list li .toggle {
text-align: center;
width: 40px;
/* auto, since non-WebKit browsers doesn't support input styling */
height: auto;
position: absolute;
top: 0;
bottom: 0;
margin: auto 0;
border: none; /* Mobile Safari */
-webkit-appearance: none;
appearance: none;
}

.todo-list li .toggle:after {
content: url('data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" width="40" height="40" viewBox="-10 -18 100 135"><circle cx="50" cy="50" r="50" fill="none" stroke="#ededed" stroke-width="3"/></svg>');
}

.todo-list li .toggle:checked:after {
content: url('data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" width="40" height="40" viewBox="-10 -18 100 135"><circle cx="50" cy="50" r="50" fill="none" stroke="#bddad5" stroke-width="3"/><path fill="#5dc2af" d="M72 25L42 71 27 56l-4 4 20 20 34-52z"/></svg>');
}

.todo-list li label {
white-space: pre-line;
word-break: break-all;
padding: 15px 60px 15px 15px;
margin-left: 45px;
display: block;
line-height: 1.2;
transition: color 0.4s;
}

.todo-list li.completed label {
color: #d9d9d9;
text-decoration: line-through;
}

.todo-list li .destroy {
display: none;
position: absolute;
top: 0;
right: 10px;
bottom: 0;
width: 40px;
height: 40px;
margin: auto 0;
font-size: 30px;
color: #cc9a9a;
margin-bottom: 11px;
transition: color 0.2s ease-out;
}

.todo-list li .destroy:hover {
color: #af5b5e;
}

.todo-list li .destroy:after {
content: '×';
}

.todo-list li:hover .destroy {
display: block;
}

.todo-list li .edit {
display: none;
}

.todo-list li.editing:last-child {
margin-bottom: -1px;
}

.footer {
color: #777;
padding: 10px 15px;
height: 20px;
text-align: center;
border-top: 1px solid #e6e6e6;
}

.footer:before {
content: '';
position: absolute;
right: 0;
bottom: 0;
left: 0;
height: 50px;
overflow: hidden;
box-shadow: 0 1px 1px rgba(0, 0, 0, 0.2),
0 8px 0 -3px #f6f6f6,
0 9px 1px -3px rgba(0, 0, 0, 0.2),
0 16px 0 -6px #f6f6f6,
0 17px 2px -6px rgba(0, 0, 0, 0.2);
}

.todo-count {
float: left;
text-align: left;
}

.todo-count strong {
font-weight: 300;
}

.filters {
margin: 0;
padding: 0;
list-style: none;
position: absolute;
right: 0;
left: 0;
}

.filters li {
display: inline;
}

.filters li a {
color: inherit;
margin: 3px;
padding: 3px 7px;
text-decoration: none;
border: 1px solid transparent;
border-radius: 3px;
}

.filters li a.selected,
.filters li a:hover {
border-color: rgba(175, 47, 47, 0.1);
}

.filters li a.selected {
border-color: rgba(175, 47, 47, 0.2);
}

.clear-completed,
html .clear-completed:active {
float: right;
position: relative;
line-height: 20px;
text-decoration: none;
cursor: pointer;
position: relative;
}

.clear-completed:hover {
text-decoration: underline;
}

.info {
margin: 65px auto 0;
color: #bfbfbf;
font-size: 10px;
text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
text-align: center;
}

.info p {
line-height: 1;
}

.info a {
color: inherit;
text-decoration: none;
font-weight: 400;
}

.info a:hover {
text-decoration: underline;
}

/*
Hack to remove background from Mobile Safari.
Can't use it globally since it destroys checkboxes in Firefox
*/
@media screen and (-webkit-min-device-pixel-ratio:0) {
.toggle-all,
.todo-list li .toggle {
background: none;
}

.todo-list li .toggle {
height: 40px;
}

.toggle-all {
-webkit-transform: rotate(90deg);
transform: rotate(90deg);
-webkit-appearance: none;
appearance: none;
}
}

@media (max-width: 430px) {
.footer {
height: 50px;
}

.filters {
bottom: 10px;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package shared
import scalatags.Text.all._
import scalatags.Text.tags2

case class Todo(checked: Boolean, text: String)

object Todo {
implicit def todoRW: upickle.default.ReadWriter[Todo] = upickle.default.macroRW[Todo]
}

object Shared {
def renderBody(todos: Seq[Todo], state: String) = {
val filteredTodos = state match {
case "all" => todos.zipWithIndex
case "active" => todos.zipWithIndex.filter(!_._1.checked)
case "completed" => todos.zipWithIndex.filter(_._1.checked)
}
div(
header(cls := "header")(
h1("todos"),
input(cls := "new-todo", placeholder := "What needs to be done?", autofocus := "")
),
tags2.section(cls := "main")(
input(
id := "toggle-all",
cls := "toggle-all",
`type` := "checkbox",
if (todos.filter(_.checked).size != 0) checked else ()
),
label(`for` := "toggle-all")("Mark all as complete"),
ul(cls := "todo-list")(
for ((todo, index) <- filteredTodos) yield li(
if (todo.checked) cls := "completed" else (),
div(cls := "view")(
input(
cls := "toggle",
`type` := "checkbox",
if (todo.checked) checked else (),
data("todo-index") := index
),
label(todo.text),
button(cls := "destroy", data("todo-index") := index)
),
input(cls := "edit", value := todo.text)
)
)
),
footer(cls := "footer")(
span(cls := "todo-count")(
strong(todos.filter(!_.checked).size),
" items left"
),
ul(cls := "filters")(
li(cls := "todo-all")(
a(if (state == "all") cls := "selected" else ())("All")
),
li(cls := "todo-active")(
a(if (state == "active") cls := "selected" else ())("Active")
),
li(cls := "todo-completed")(
a(if (state == "completed") cls := "selected" else ())("Completed")
)
),
button(cls := "clear-completed")("Clear completed")
)
)
}

}
78 changes: 78 additions & 0 deletions integration/feature/reproducibility/resources/src/WebApp.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package webapp
import scalatags.Text.all._
import scalatags.Text.tags2
import shared.{Shared, Todo}

object WebApp extends cask.MainRoutes {
override def port = 8083
var todos = Seq(
Todo(true, "Get started with Cask"),
Todo(false, "Profit!")
)

@cask.post("/list/:state")
def list(state: String) = upickle.default.write(todos)

@cask.post("/add/:state")
def add(state: String, request: cask.Request) = {
todos = Seq(Todo(false, request.text())) ++ todos
upickle.default.write(todos)
}

@cask.post("/delete/:state/:index")
def delete(state: String, index: Int) = {
todos = todos.patch(index, Nil, 1)
upickle.default.write(todos)
}

@cask.post("/toggle/:state/:index")
def toggle(state: String, index: Int) = {
todos = todos.updated(index, todos(index).copy(checked = !todos(index).checked))
upickle.default.write(todos)
}

@cask.post("/clear-completed/:state")
def clearCompleted(state: String) = {
todos = todos.filter(!_.checked)
upickle.default.write(todos)
}

@cask.post("/toggle-all/:state")
def toggleAll(state: String) = {
val next = todos.filter(_.checked).size != 0
todos = todos.map(_.copy(checked = next))
upickle.default.write(todos)
}

@cask.get("/")
def index() = {
doctype("html")(
html(lang := "en")(
head(
meta(charset := "utf-8"),
meta(name := "viewport", content := "width=device-width, initial-scale=1"),
tags2.title("Template • TodoMVC"),
link(rel := "stylesheet", href := "/static/index.css")
),
body(
tags2.section(cls := "todoapp")(Shared.renderBody(todos, "all")),
footer(cls := "info")(
p("Double-click to edit a todo"),
p("Created by ")(
a(href := "http://todomvc.com")("Li Haoyi")
),
p("Part of ")(
a(href := "http://todomvc.com")("TodoMVC")
)
),
script(src := "/static/main.js")
)
)
)
}

@cask.staticResources("/static")
def static() = "webapp"

initialize()
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package webapp

import utest._

object WebAppTests extends TestSuite {
def withServer[T](example: cask.main.Main)(f: String => T): T = {
val server = io.undertow.Undertow.builder
.addHttpListener(8185, "localhost")
.setHandler(example.defaultHandler)
.build
server.start()
val res =
try f("http://localhost:8185")
finally server.stop()
res
}

val tests = Tests {
test("simpleRequest") - withServer(WebApp) { host =>
val page = requests.get(host).text()
assert(page.contains("What needs to be done?"))
}
}
}
73 changes: 73 additions & 0 deletions integration/feature/reproducibility/src/ReproducibilityTests.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package mill.integration

import mill.testkit.UtestIntegrationTestSuite
import utest._

import java.util.zip.GZIPInputStream

object ReproducibilityTests extends UtestIntegrationTestSuite {

def normalize(workspacePath: os.Path): Unit = {
for (p <- os.walk(workspacePath / "out")) {
val sub = p.subRelativeTo(workspacePath).toString()

val cacheable =
(sub.contains(".dest") || sub.contains(".json") || os.isDir(p)) &&
!sub.replace("out/mill-build", "").contains("mill-") &&
!(p.ext == "json" && ujson.read(
os.read(p)
).objOpt.flatMap(_.get("value")).flatMap(_.objOpt).flatMap(_.get("worker")).nonEmpty)

if (!cacheable) {
os.remove.all(p)
}
}
}

val tests: Tests = Tests {
test("diff") - {
def run() = integrationTest { tester =>
tester.eval(("--meta-level", "1", "runClasspath"))
tester.workspacePath
}

val workspacePath1 = run()
val workspacePath2 = run()
assert(workspacePath1 != workspacePath2)
normalize(workspacePath1)
normalize(workspacePath2)
val diff = os.call(("git", "diff", "--no-index", workspacePath1, workspacePath2)).out.text()
pprint.log(diff)
assert(diff.isEmpty)
}

test("inspection") - {
def run() = integrationTest { tester =>
tester.eval(
("--meta-level", "1", "runClasspath"),
env = Map("MILL_TEST_TEXT_ANALYSIS_STORE" -> "1"),
check = true
)
tester.workspacePath
}

val workspacePath = run()
val dest = workspacePath / "out/mill-build/compile.dest/zinc.txt"
val src = workspacePath / "out/mill-build/compile.dest/zinc"
os.write(dest, new GZIPInputStream(os.read.inputStream(src)))
normalize(workspacePath)
for (p <- os.walk(workspacePath)) {
if (
(p.ext == "json" || p.ext == "txt")
&& !p.segments.contains("enablePluginScalacOptions.super")
&& !p.segments.contains("allScalacOptions.json")
&& !p.segments.contains("scalacOptions.json")
) {
val txt = os.read(p)
Predef.assert(!txt.contains(mill.api.WorkspaceRoot.workspaceRoot.toString), p)
Predef.assert(!txt.contains(os.home.toString), p)
}
}
}
}
}
4 changes: 2 additions & 2 deletions main/api/src/mill/api/JsonFormatters.scala
Original file line number Diff line number Diff line change
@@ -32,8 +32,8 @@ trait JsonFormatters {

implicit val pathReadWrite: RW[os.Path] = upickle.default.readwriter[String]
.bimap[os.Path](
_.toString,
os.Path(_)
p => PathRef.normalizePath(p).toString,
s => PathRef.denormalizePath(os.Path(s))
)

implicit val regexReadWrite: RW[Regex] = upickle.default.readwriter[String]
29 changes: 22 additions & 7 deletions main/api/src/mill/api/PathRef.scala
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@ import java.util.concurrent.ConcurrentHashMap
import scala.util.{DynamicVariable, Using}
import upickle.default.{ReadWriter => RW}
import scala.annotation.nowarn
import os.Path

/**
* A wrapper around `os.Path` that calculates it's hashcode based
@@ -43,11 +44,25 @@ case class PathRef private (
case PathRef.Revalidate.Always => "vn:"
}
val sig = String.format("%08x", this.sig: Integer)
quick + valid + sig + ":" + path.toString()
quick + valid + sig + ":" + PathRef.normalizePath(path).toString()
}
}

object PathRef {
def mapPathPrefixes(path: os.Path, mapping: Seq[(os.Path, os.Path)]): os.Path = {
mapping
.collectFirst { case (from, to) if path.startsWith(from) => to / path.subRelativeTo(from) }
.getOrElse(path)
}

val defaultMapping: Seq[(Path, Path)] = Seq(
mill.api.WorkspaceRoot.workspaceRoot -> os.root / "$WORKSPACE",
os.home -> os.root / "$HOME"
)
def normalizePath(path: os.Path): os.Path = mapPathPrefixes(path, defaultMapping)

def denormalizePath(path: os.Path): os.Path = mapPathPrefixes(path, defaultMapping.map(_.swap))

implicit def shellable(p: PathRef): os.Shellable = p.path

/**
@@ -92,7 +107,7 @@ object PathRef {
}

def apply(path: os.Path, quick: Boolean, sig: Int, revalidate: Revalidate): PathRef =
new PathRef(path, quick, sig, revalidate)
new PathRef(PathRef.denormalizePath(path), quick, sig, revalidate)

/**
* Create a [[PathRef]] by recursively digesting the content of a given `path`.
@@ -107,10 +122,10 @@ object PathRef {
quick: Boolean = false,
revalidate: Revalidate = Revalidate.Never
): PathRef = {
val basePath = path
val basePath = PathRef.denormalizePath(path)

val sig = {
val isPosix = path.wrapped.getFileSystem.supportedFileAttributeViews().contains("posix")
val isPosix = basePath.wrapped.getFileSystem.supportedFileAttributeViews().contains("posix")
val digest = MessageDigest.getInstance("MD5")
val digestOut = new DigestOutputStream(DummyOutputStream, digest)

@@ -121,10 +136,10 @@ object PathRef {
digest.update(value.toByte)
}

if (os.exists(path)) {
if (os.exists(basePath)) {
for (
(path, attrs) <-
os.walk.attrs(path, includeTarget = true, followLinks = true).sortBy(_._1.toString)
os.walk.attrs(basePath, includeTarget = true, followLinks = true).sortBy(_._1.toString)
) {
val sub = path.subRelativeTo(basePath)
digest.update(sub.toString().getBytes())
@@ -169,7 +184,7 @@ object PathRef {
java.util.Arrays.hashCode(digest.digest())
}

new PathRef(path, quick, sig, revalidate)
new PathRef(basePath, quick, sig, revalidate)
}

/**
2 changes: 1 addition & 1 deletion main/define/src/mill/define/Task.scala
Original file line number Diff line number Diff line change
@@ -802,7 +802,7 @@ class InputImpl[T](
val writer: upickle.default.Writer[_],
val isPrivate: Option[Boolean]
) extends Target[T] {
override def sideHash: Int = util.Random.nextInt()
override def sideHash: Int = 31337
// FIXME: deprecated return type: Change to Option
override def writerOpt: Some[W[_]] = Some(writer)
}
88 changes: 54 additions & 34 deletions main/eval/src/mill/eval/GroupEvaluator.scala
Original file line number Diff line number Diff line change
@@ -43,6 +43,14 @@ private[mill] trait GroupEvaluator {
val effectiveThreadCount: Int =
this.threadCount.getOrElse(Runtime.getRuntime().availableProcessors())

def getValueHash(t: Task[_], inputsHash: Int, v: Val, jsonOpt: Option[ujson.Value]): Int = {
if (t.isInstanceOf[Worker[_]]) inputsHash
else jsonOpt match {
case Some(json) => json.hashCode()
case None => v.##
}
}

// those result which are inputs but not contained in this terminal group
def evaluateGroupCached(
terminal: Terminal,
@@ -70,7 +78,7 @@ private[mill] trait GroupEvaluator {
.flatMap(results(_).result.asSuccess.map(_.value._2))
)

val sideHashes = MurmurHash3.orderedHash(group.iterator.map(_.sideHash))
val sideHashes = group.iterator.map(_.sideHash).sum

val scriptsHash =
if (disableCallgraph) 0
@@ -129,6 +137,11 @@ private[mill] trait GroupEvaluator {

val inputsHash = externalInputsHash + sideHashes + classLoaderSigHash + scriptsHash

def transformResults(results: Map[Task[_], TaskResult[(Val, Option[ujson.Value])]]) = {
for ((k, v0) <- results)
yield k -> v0.map { case (v, jsonOpt) => (v, getValueHash(k, inputsHash, v, jsonOpt)) }
}

terminal match {
case Terminal.Task(task) =>
val (newResults, newEvaluated) = evaluateGroup(
@@ -145,7 +158,13 @@ private[mill] trait GroupEvaluator {
executionContext,
exclusive
)
GroupEvaluator.Results(newResults, newEvaluated.toSeq, null, inputsHash, -1)
GroupEvaluator.Results(
transformResults(newResults),
newEvaluated.toSeq,
null,
inputsHash,
-1
)

case labelled: Terminal.Labelled[_] =>
val out =
@@ -157,7 +176,9 @@ private[mill] trait GroupEvaluator {
Terminal.destSegments(labelled)
)

val cached = loadCachedJson(logger, inputsHash, labelled, paths)
val cached = Option
.when(sideHashes == 0) { loadCachedJson(logger, inputsHash, labelled, paths) }
.flatten

val upToDateWorker = loadUpToDateWorker(
logger,
@@ -202,12 +223,13 @@ private[mill] trait GroupEvaluator {
exclusive
)

val transformedResults = transformResults(newResults)
newResults(labelled.task) match {
case TaskResult(Result.Failure(_, Some((v, _))), _) =>
handleTaskResult(v, v.##, paths.meta, inputsHash, labelled)
case TaskResult(Result.Failure(_, Some((v, jsonOpt))), _) =>
handleTaskResult(v, jsonOpt, paths.meta, inputsHash, labelled)

case TaskResult(Result.Success((v, _)), _) =>
handleTaskResult(v, v.##, paths.meta, inputsHash, labelled)
case TaskResult(Result.Success((v, jsonOpt)), _) =>
handleTaskResult(v, jsonOpt, paths.meta, inputsHash, labelled)

case _ =>
// Wipe out any cached meta.json file that exists, so
@@ -218,7 +240,7 @@ private[mill] trait GroupEvaluator {
}

GroupEvaluator.Results(
newResults,
transformedResults,
newEvaluated.toSeq,
cached = if (labelled.task.isInstanceOf[InputImpl[_]]) null else false,
inputsHash,
@@ -242,11 +264,11 @@ private[mill] trait GroupEvaluator {
logger: mill.api.Logger,
executionContext: mill.api.Ctx.Fork.Api,
exclusive: Boolean
): (Map[Task[_], TaskResult[(Val, Int)]], mutable.Buffer[Task[_]]) = {
): (Map[Task[_], TaskResult[(Val, Option[ujson.Value])]], mutable.Buffer[Task[_]]) = {

def computeAll() = {
val newEvaluated = mutable.Buffer.empty[Task[_]]
val newResults = mutable.Map.empty[Task[_], Result[(Val, Int)]]
val newResults = mutable.Map.empty[Task[_], Result[(Val, Option[ujson.Value])]]

val nonEvaluatedTargets = group.indexed.filterNot(results.contains)

@@ -319,11 +341,15 @@ private[mill] trait GroupEvaluator {
}

newResults(task) = for (v <- res) yield {
(
v,
if (task.isInstanceOf[Worker[_]]) inputsHash
else v.##
)
val jsonOpt = task match {
case n: NamedTask[_] =>
for (w <- n.writerOpt) yield {
upickle.default.writeJs(v.value)(using w.asInstanceOf[upickle.default.Writer[Any]])
}
case _ => None
}

(v, jsonOpt)
}
}

@@ -369,39 +395,33 @@ private[mill] trait GroupEvaluator {

private def handleTaskResult(
v: Val,
hashCode: Int,
jsonOpt: Option[ujson.Value],
metaPath: os.Path,
inputsHash: Int,
labelled: Terminal.Labelled[_]
): Unit = {

for (w <- labelled.task.asWorker)
workerCache.synchronized {
workerCache.update(w.ctx.segments, (workerCacheHash(inputsHash), v))
}

val terminalResult = labelled
.task
.writerOpt
.map { w =>
upickle.default.writeJs(v.value)(using w.asInstanceOf[upickle.default.Writer[Any]])
}
.orElse {
labelled.task.asWorker.map { w =>
ujson.Obj(
"worker" -> ujson.Str(labelled.segments.render),
"toString" -> ujson.Str(v.value.toString),
"inputsHash" -> ujson.Num(inputsHash)
)
}
val valueHash = getValueHash(labelled.task, inputsHash, v, jsonOpt)

val terminalResult = jsonOpt.orElse {
labelled.task.asWorker.map { w =>
ujson.Obj(
"worker" -> ujson.Str(labelled.segments.render),
"toString" -> ujson.Str(v.value.toString),
"inputsHash" -> ujson.Num(inputsHash)
)
}
}

for (json <- terminalResult) {
os.write.over(
metaPath,
upickle.default.stream(
Evaluator.Cached(json, hashCode, inputsHash),
indent = 4
),
upickle.default.stream(Evaluator.Cached(json, valueHash, inputsHash), indent = 4),
createFolders = true
)
}
6 changes: 3 additions & 3 deletions main/src/mill/main/MainModule.scala
Original file line number Diff line number Diff line change
@@ -216,10 +216,10 @@ trait MainModule extends BaseModule0 {
def renderFileName(t: NamedTask[_]) = {
// handle both Windows or Unix separators
val fullFileName = t.ctx.fileName.replaceAll(raw"\\", "/")
val basePath = WorkspaceRoot.workspaceRoot.toString().replaceAll(raw"\\", "/") + "/"
WorkspaceRoot.workspaceRoot.toString().replaceAll(raw"\\", "/") + "/"
val name =
if (fullFileName.startsWith(basePath)) {
fullFileName.drop(basePath.length)
if (fullFileName.startsWith("/$WORKSPACE/")) {
fullFileName.drop("/$WORKSPACE/".length)
} else {
fullFileName.split('/').last
}
9 changes: 5 additions & 4 deletions main/src/mill/main/RootModule.scala
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package mill.main

import mill.api.internal
import mill.api.PathRef
import mill.define.{Caller, Discover}
import scala.annotation.compileTimeOnly

@@ -45,10 +46,10 @@ object RootModule {
output0: String,
topLevelProjectRoot0: String
) = this(
enclosingClasspath0.map(os.Path(_)),
os.Path(projectRoot0),
os.Path(output0),
os.Path(topLevelProjectRoot0)
enclosingClasspath0.map(s => PathRef.denormalizePath(os.Path(s))),
PathRef.denormalizePath(os.Path(projectRoot0)),
PathRef.denormalizePath(os.Path(output0)),
PathRef.denormalizePath(os.Path(topLevelProjectRoot0))
)

implicit val millMiscInfo: Info = this
10 changes: 5 additions & 5 deletions runner/src/mill/runner/CodeGen.scala
Original file line number Diff line number Diff line change
@@ -80,7 +80,7 @@ object CodeGen {
val scriptCode = allScriptCode(scriptPath)

val markerComment =
s"""//MILL_ORIGINAL_FILE_PATH=$scriptPath
s"""//MILL_ORIGINAL_FILE_PATH=${PathRef.normalizePath(scriptPath)}
|//MILL_USER_CODE_START_MARKER""".stripMargin

val parts =
@@ -200,10 +200,10 @@ object CodeGen {
s"""import _root_.mill.runner.MillBuildRootModule
|@_root_.scala.annotation.nowarn
|object MillMiscInfo extends mill.main.RootModule.Info(
| ${enclosingClasspath.map(p => literalize(p.toString))},
| ${literalize(scriptFolderPath.toString)},
| ${literalize(output.toString)},
| ${literalize(millTopLevelProjectRoot.toString)}
| ${enclosingClasspath.map(p => literalize(PathRef.normalizePath(p).toString))},
| ${literalize(PathRef.normalizePath(scriptFolderPath).toString)},
| ${literalize(PathRef.normalizePath(output).toString)},
| ${literalize(PathRef.normalizePath(millTopLevelProjectRoot).toString)}
|)
|import MillMiscInfo._
|""".stripMargin
2 changes: 1 addition & 1 deletion runner/src/mill/runner/MillBuildBootstrap.scala
Original file line number Diff line number Diff line change
@@ -167,7 +167,7 @@ class MillBuildBootstrap(
.headOption
.map(_.runClasspath)
.getOrElse(millBootClasspathPathRefs)
.map(p => (p.path, p.sig))
.map(p => (PathRef.normalizePath(p.path), p.sig))
.hashCode(),
nestedState
.frames
6 changes: 3 additions & 3 deletions scalalib/src/mill/scalalib/TestModule.scala
Original file line number Diff line number Diff line change
@@ -148,12 +148,12 @@ trait TestModule

val testArgs = TestArgs(
framework = testFramework(),
classpath = runClasspath().map(_.path),
classpath = runClasspath().map(_.path.toString),
arguments = args(),
sysProps = Map.empty,
outputPath = outputPath,
outputPath = outputPath.toString,
colored = T.log.colored,
testCp = testClasspath().map(_.path),
testCp = testClasspath().map(_.path.toString),
home = T.home,
globSelectors = selectors
)
6 changes: 3 additions & 3 deletions scalalib/src/mill/scalalib/TestModuleUtil.scala
Original file line number Diff line number Diff line change
@@ -47,12 +47,12 @@ private[scalalib] object TestModuleUtil {
val outputPath = base / "out.json"
val testArgs = TestArgs(
framework = testFramework,
classpath = runClasspath.map(_.path),
classpath = runClasspath.map(_.path.toString),
arguments = args,
sysProps = props,
outputPath = outputPath,
outputPath = outputPath.toString,
colored = T.log.colored,
testCp = testClasspath.map(_.path),
testCp = testClasspath.map(_.path.toString),
home = T.home,
globSelectors = selectors2
)
84 changes: 77 additions & 7 deletions scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@ import sbt.internal.inc.{
FreshCompilerCache,
ManagedLoggedReporter,
MappedFileConverter,
MappedVirtualFile,
ScalaInstance,
Stamps,
ZincUtil,
@@ -19,6 +20,8 @@ import sbt.internal.inc.classpath.ClasspathUtil
import sbt.internal.inc.consistent.ConsistentFileAnalysisStore
import sbt.internal.util.{ConsoleAppender, ConsoleOut}
import sbt.mill.SbtLoggerUtils
import xsbti.compile.analysis.{ReadMapper, Stamp, WriteMapper}
import xsbti.VirtualFileRef
import xsbti.compile.analysis.ReadWriteMappers
import xsbti.compile.{
AnalysisContents,
@@ -38,6 +41,7 @@ import xsbti.compile.CompileProgress

import java.io.File
import java.net.URLClassLoader
import java.nio.file
import java.util.Optional
import scala.collection.mutable
import scala.util.Properties.isWin
@@ -483,14 +487,80 @@ class ZincWorkerImpl(
}
}

private def fileAnalysisStore(path: os.Path): AnalysisStore =
ConsistentFileAnalysisStore.binary(
file = path.toIO,
mappers = ReadWriteMappers.getEmptyMappers(),
sort = true,
// No need to utilize more than 8 cores to serialize a small file
parallelism = math.min(Runtime.getRuntime.availableProcessors(), 8)
private def fileAnalysisStore(path: os.Path): AnalysisStore = {
def denormalizeVirtualFile(file: VirtualFileRef) = file match {
case b: MappedVirtualFile =>
MappedVirtualFile(PathRef.denormalizePath(os.Path(b.toPath())).toString(), Map())
case b => b
}
def normalizeVirtualFile(file: VirtualFileRef) = file match {
case b: MappedVirtualFile =>
MappedVirtualFile(PathRef.normalizePath(os.Path(b.toPath())).toString(), Map())
case b => b
}
def denormalizeNioPath(file: java.nio.file.Path) = PathRef.denormalizePath(os.Path(file)).toNIO
def normalizeNioPath(file: java.nio.file.Path) = PathRef.normalizePath(os.Path(file)).toNIO
def normalizeString(str: String) = {
PathRef.defaultMapping.foldLeft(str) { case (s, (long, short)) =>
s.replace(long.toString(), short.toString())
}
}
def denormalizeString(str: String) = {
PathRef.defaultMapping.foldLeft(str) { case (s, (long, short)) =>
s.replace(short.toString(), long.toString())
}
}
val mappers = new ReadWriteMappers(
new ReadMapper {
override def mapSourceFile(sourceFile: VirtualFileRef): VirtualFileRef =
denormalizeVirtualFile(sourceFile)
override def mapBinaryFile(binaryFile: VirtualFileRef): VirtualFileRef =
denormalizeVirtualFile(binaryFile)
override def mapProductFile(productFile: VirtualFileRef): VirtualFileRef =
denormalizeVirtualFile(productFile)
override def mapOutputDir(outputDir: file.Path): file.Path = denormalizeNioPath(outputDir)
override def mapSourceDir(sourceDir: file.Path): file.Path = denormalizeNioPath(sourceDir)
override def mapClasspathEntry(classpathEntry: file.Path): file.Path =
denormalizeNioPath(classpathEntry)
override def mapJavacOption(javacOption: String): String = denormalizeString(javacOption)
override def mapScalacOption(scalacOption: String): String = denormalizeString(scalacOption)
override def mapBinaryStamp(file: VirtualFileRef, binaryStamp: Stamp): Stamp = binaryStamp
override def mapSourceStamp(file: VirtualFileRef, sourceStamp: Stamp): Stamp = sourceStamp
override def mapProductStamp(file: VirtualFileRef, productStamp: Stamp): Stamp =
productStamp
override def mapMiniSetup(miniSetup: MiniSetup): MiniSetup = miniSetup
},
new WriteMapper {
override def mapSourceFile(sourceFile: VirtualFileRef): VirtualFileRef =
normalizeVirtualFile(sourceFile)
override def mapBinaryFile(binaryFile: VirtualFileRef): VirtualFileRef =
normalizeVirtualFile(binaryFile)
override def mapProductFile(productFile: VirtualFileRef): VirtualFileRef =
normalizeVirtualFile(productFile)
override def mapOutputDir(outputDir: file.Path): file.Path = normalizeNioPath(outputDir)
override def mapSourceDir(sourceDir: file.Path): file.Path = normalizeNioPath(sourceDir)
override def mapClasspathEntry(classpathEntry: file.Path): file.Path =
normalizeNioPath(classpathEntry)
override def mapJavacOption(javacOption: String): String = normalizeString(javacOption)
override def mapScalacOption(scalacOption: String): String = normalizeString(scalacOption)
override def mapBinaryStamp(file: VirtualFileRef, binaryStamp: Stamp): Stamp = binaryStamp
override def mapSourceStamp(file: VirtualFileRef, sourceStamp: Stamp): Stamp = sourceStamp
override def mapProductStamp(file: VirtualFileRef, productStamp: Stamp): Stamp =
productStamp
override def mapMiniSetup(miniSetup: MiniSetup): MiniSetup = miniSetup
}
)
// No need to utilize more than 8 cores to serialize a small file
val parallelism = math.min(Runtime.getRuntime.availableProcessors(), 8)

if (System.getenv("MILL_TEST_TEXT_ANALYSIS_STORE") != null) {
ConsistentFileAnalysisStore
.text(file = path.toIO, mappers = mappers, sort = true, parallelism = parallelism)
} else {
ConsistentFileAnalysisStore
.binary(file = path.toIO, mappers = mappers, sort = true, parallelism = parallelism)
}
}

private def compileInternal(
upstreamCompileOutput: Seq[CompilationResult],
6 changes: 3 additions & 3 deletions testrunner/src/mill/testrunner/Model.scala
Original file line number Diff line number Diff line change
@@ -5,12 +5,12 @@ import mill.api.internal

@internal case class TestArgs(
framework: String,
classpath: Seq[os.Path],
classpath: Seq[String],
arguments: Seq[String],
sysProps: Map[String, String],
outputPath: os.Path,
outputPath: String,
colored: Boolean,
testCp: Seq[os.Path],
testCp: Seq[String],
home: os.Path,
globSelectors: Seq[String]
)
4 changes: 2 additions & 2 deletions testrunner/src/mill/testrunner/TestRunnerMain0.scala
Original file line number Diff line number Diff line change
@@ -30,7 +30,7 @@ import mill.util.PrintLogger

val result = TestRunnerUtils.runTestFramework0(
frameworkInstances = Framework.framework(testArgs.framework),
testClassfilePath = Agg.from(testArgs.testCp),
testClassfilePath = Agg.from(testArgs.testCp.map(os.Path(_))),
args = testArgs.arguments,
classFilter = cls => filter(cls.getName),
cl = classLoader,
@@ -41,7 +41,7 @@ import mill.util.PrintLogger
// dirtied the thread-interrupted flag and forgot to clean up. Otherwise,
// that flag causes writing the results to disk to fail
Thread.interrupted()
os.write(testArgs.outputPath, upickle.default.stream(result))
os.write(os.Path(testArgs.outputPath), upickle.default.stream(result))
} catch {
case e: Throwable =>
println(e)