From a1c9a222892e08f1ebdfebc3090d8f187c8903e7 Mon Sep 17 00:00:00 2001 From: Olivier Melois Date: Thu, 29 Mar 2018 21:03:27 +0100 Subject: [PATCH] Adds envVars propagation client -> server Since Mill now executes in a long-lived JVM, the builds do not have a chance to use environment variables as inputs. This propagates the environment variables from the client all the way down to the context available to the tasks as a `Map[String, String]` so that they can be used as inputs should the user choose to do so. https://github.com/lihaoyi/mill/issues/257 --- .../src/mill/clientserver/Client.java | 1 + .../src/mill/clientserver/ClientServer.java | 84 +++++++++++++++---- .../src/mill/clientserver/Server.scala | 9 +- .../mill/clientserver/ClientServerTests.scala | 3 +- core/src/mill/eval/Evaluator.scala | 10 ++- core/src/mill/util/Ctx.scala | 3 +- main/src/mill/Main.scala | 78 ++++++++++------- main/src/mill/main/MainRunner.scala | 6 +- main/src/mill/main/RunScript.scala | 6 +- main/test/src/mill/util/ScriptTestSuite.scala | 2 +- 10 files changed, 146 insertions(+), 56 deletions(-) diff --git a/clientserver/src/mill/clientserver/Client.java b/clientserver/src/mill/clientserver/Client.java index e2a92f7fb55..48d8434cace 100644 --- a/clientserver/src/mill/clientserver/Client.java +++ b/clientserver/src/mill/clientserver/Client.java @@ -101,6 +101,7 @@ public static int run(String lockBase, FileOutputStream f = new FileOutputStream(lockBase + "/run"); ClientServer.writeArgs(System.console() != null, args, f); + ClientServer.writeEnv(f); f.close(); boolean serverInit = false; diff --git a/clientserver/src/mill/clientserver/ClientServer.java b/clientserver/src/mill/clientserver/ClientServer.java index 7af5845b607..684a13eb13c 100644 --- a/clientserver/src/mill/clientserver/ClientServer.java +++ b/clientserver/src/mill/clientserver/ClientServer.java @@ -4,6 +4,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.HashMap; +import java.util.Map; class ClientServer { public static boolean isWindows = System.getProperty("os.name").toLowerCase().startsWith("windows"); @@ -19,23 +21,77 @@ public static String[] parseArgs(InputStream argStream) throws IOException { int argsLength = argStream.read(); String[] args = new String[argsLength]; for (int i = 0; i < args.length; i++) { - int n = argStream.read(); - byte[] arr = new byte[n]; - argStream.read(arr); - args[i] = new String(arr); + args[i] = readString(argStream); } return args; } public static void writeArgs(Boolean interactive, String[] args, - OutputStream argStream) throws IOException{ - argStream.write(interactive ? 1 : 0); - argStream.write(args.length); - int i = 0; - while (i < args.length){ - argStream.write(args[i].length()); - argStream.write(args[i].getBytes()); - i += 1; + OutputStream argStream) throws IOException { + argStream.write(interactive ? 1 : 0); + argStream.write(args.length); + int i = 0; + while (i < args.length) { + writeString(argStream, args[i]); + i += 1; + } + } + + /** + * This allows the mill client to pass the environment as he sees it to the + * server (as the server remains alive over the course of several runs and + * does not see the environment changes the client would) + */ + public static void writeEnv(OutputStream argStream) throws IOException { + Map env = System.getenv(); + argStream.write(env.size()); + for (Map.Entry kv : env.entrySet()) { + writeString(argStream, kv.getKey()); + writeString(argStream, kv.getValue()); + } } - } -} \ No newline at end of file + + public static Map parseEnv(InputStream argStream) throws IOException { + Map env = new HashMap<>(); + int mapLength = argStream.read(); + for (int i = 0; i < mapLength; i++) { + String key = readString(argStream); + String value = readString(argStream); + env.put(key, value); + } + return env; + } + + private static String readString(InputStream inputStream) throws IOException { + // Result is between 0 and 255, hence the loop. + int read = inputStream.read(); + int bytesToRead = read; + while(read == 255){ + read = inputStream.read(); + bytesToRead += read; + } + byte[] arr = new byte[bytesToRead]; + int readTotal = 0; + while (readTotal < bytesToRead) { + read = inputStream.read(arr, readTotal, bytesToRead - readTotal); + readTotal += read; + } + return new String(arr); + } + + private static void writeString(OutputStream outputStream, String string) throws IOException { + // When written, an int > 255 gets splitted. This logic performs the + // split beforehand so that the reading side knows that there is still + // more metadata to come before it's able to read the actual data. + // Could do with rewriting using logical masks / shifts. + byte[] bytes = string.getBytes(); + int toWrite = bytes.length; + while(toWrite >= 255){ + outputStream.write(255); + toWrite = toWrite - 255; + } + outputStream.write(toWrite); + outputStream.write(bytes); + } + +} diff --git a/clientserver/src/mill/clientserver/Server.scala b/clientserver/src/mill/clientserver/Server.scala index 24827ac24d7..d9e67489aa4 100644 --- a/clientserver/src/mill/clientserver/Server.scala +++ b/clientserver/src/mill/clientserver/Server.scala @@ -3,6 +3,7 @@ package mill.clientserver import java.io._ import java.net.Socket +import scala.collection.JavaConverters._ import org.scalasbt.ipcsocket._ trait ServerMain[T]{ @@ -21,7 +22,8 @@ trait ServerMain[T]{ mainInteractive: Boolean, stdin: InputStream, stdout: PrintStream, - stderr: PrintStream): (Boolean, Option[T]) + stderr: PrintStream, + env : Map[String, String]): (Boolean, Option[T]) } @@ -76,6 +78,7 @@ class Server[T](lockBase: String, val argStream = new FileInputStream(lockBase + "/run") val interactive = argStream.read() != 0; val args = ClientServer.parseArgs(argStream) + val env = ClientServer.parseEnv(argStream) argStream.close() var done = false @@ -89,7 +92,9 @@ class Server[T](lockBase: String, sm.stateCache, interactive, socketIn, - stdout, stderr + stdout, + stderr, + env.asScala.toMap ) sm.stateCache = newStateCache diff --git a/clientserver/test/src/mill/clientserver/ClientServerTests.scala b/clientserver/test/src/mill/clientserver/ClientServerTests.scala index 2c9a57b0a44..0b55d4bb8fa 100644 --- a/clientserver/test/src/mill/clientserver/ClientServerTests.scala +++ b/clientserver/test/src/mill/clientserver/ClientServerTests.scala @@ -9,7 +9,8 @@ class EchoServer extends ServerMain[Int]{ mainInteractive: Boolean, stdin: InputStream, stdout: PrintStream, - stderr: PrintStream) = { + stderr: PrintStream, + env: Map[String, String]) = { val reader = new BufferedReader(new InputStreamReader(stdin)) val str = reader.readLine() diff --git a/core/src/mill/eval/Evaluator.scala b/core/src/mill/eval/Evaluator.scala index 33141c0a24f..7b3634ad4a0 100644 --- a/core/src/mill/eval/Evaluator.scala +++ b/core/src/mill/eval/Evaluator.scala @@ -2,6 +2,8 @@ package mill.eval import java.net.URLClassLoader +import scala.collection.JavaConverters._ + import mill.util.Router.EntryPoint import ammonite.ops._ import ammonite.runtime.SpecialClassLoader @@ -32,7 +34,8 @@ case class Evaluator[T](home: Path, rootModule: mill.define.BaseModule, log: Logger, classLoaderSig: Seq[(Either[String, Path], Long)] = Evaluator.classLoaderSig, - workerCache: mutable.Map[Segments, (Int, Any)] = mutable.Map.empty){ + workerCache: mutable.Map[Segments, (Int, Any)] = mutable.Map.empty, + env : Map[String, String] = Evaluator.defaultEnv){ val classLoaderSignHash = classLoaderSig.hashCode() def evaluate(goals: Agg[Task[_]]): Evaluator.Results = { mkdir(outPath) @@ -271,7 +274,8 @@ case class Evaluator[T](home: Path, } }, multiLogger, - home + home, + env ) val out = System.out @@ -335,6 +339,8 @@ object Evaluator{ // in directly) we are forced to pass it in via a ThreadLocal val currentEvaluator = new ThreadLocal[mill.eval.Evaluator[_]] + val defaultEnv: Map[String, String] = System.getenv().asScala.toMap + case class Paths(out: Path, dest: Path, meta: Path, diff --git a/core/src/mill/util/Ctx.scala b/core/src/mill/util/Ctx.scala index 998181941d4..88a8baece43 100644 --- a/core/src/mill/util/Ctx.scala +++ b/core/src/mill/util/Ctx.scala @@ -36,7 +36,8 @@ object Ctx{ class Ctx(val args: IndexedSeq[_], dest0: () => Path, val log: Logger, - val home: Path) + val home: Path, + val env : Map[String, String]) extends Ctx.Dest with Ctx.Log with Ctx.Args diff --git a/main/src/mill/Main.scala b/main/src/mill/Main.scala index 0844e485f13..2fe945ba5ba 100644 --- a/main/src/mill/Main.scala +++ b/main/src/mill/Main.scala @@ -2,26 +2,29 @@ package mill import java.io.{InputStream, PrintStream} +import scala.collection.JavaConverters._ + import ammonite.main.Cli._ import ammonite.ops._ import ammonite.util.Util import mill.eval.Evaluator import mill.util.DummyInputStream - -object ServerMain extends mill.clientserver.ServerMain[Evaluator.State]{ +object ServerMain extends mill.clientserver.ServerMain[Evaluator.State] { def main0(args: Array[String], stateCache: Option[Evaluator.State], mainInteractive: Boolean, stdin: InputStream, stdout: PrintStream, - stderr: PrintStream) = Main.main0( + stderr: PrintStream, + env: Map[String, String]) = Main.main0( args, stateCache, mainInteractive, DummyInputStream, stdout, - stderr + stderr, + env ) } object Main { @@ -29,7 +32,7 @@ object Main { def main(args: Array[String]): Unit = { val as = args match { case Array(s, _*) if s == "-i" || s == "--interactive" => args.tail - case _ => args + case _ => args } val (result, _) = main0( as, @@ -37,9 +40,10 @@ object Main { ammonite.Main.isInteractive(), System.in, System.out, - System.err + System.err, + System.getenv().asScala.toMap ) - System.exit(if(result) 0 else 1) + System.exit(if (result) 0 else 1) } def main0(args: Array[String], @@ -47,15 +51,17 @@ object Main { mainInteractive: Boolean, stdin: InputStream, stdout: PrintStream, - stderr: PrintStream): (Boolean, Option[Evaluator.State]) = { + stderr: PrintStream, + env : Map[String, String]): (Boolean, Option[Evaluator.State]) = { import ammonite.main.Cli val removed = Set("predef-code", "no-home-predef") var interactive = false val interactiveSignature = Arg[Config, Unit]( - "interactive", Some('i'), + "interactive", + Some('i'), "Run Mill in interactive mode, suitable for opening REPLs and taking user input", - (c, v) =>{ + (c, v) => { interactive = true c } @@ -69,37 +75,42 @@ object Main { args.toList, millArgSignature, Cli.Config(home = millHome, remoteLogging = false) - ) match{ + ) match { case _ if interactive => - stderr.println("-i/--interactive must be passed in as the first argument") + stderr.println( + "-i/--interactive must be passed in as the first argument") (false, None) case Left(msg) => stderr.println(msg) (false, None) case Right((cliConfig, _)) if cliConfig.help => - val leftMargin = millArgSignature.map(ammonite.main.Cli.showArg(_).length).max + 2 + val leftMargin = millArgSignature + .map(ammonite.main.Cli.showArg(_).length) + .max + 2 stdout.println( - s"""Mill Build Tool + s"""Mill Build Tool |usage: mill [mill-options] [target [target-options]] | - |${formatBlock(millArgSignature, leftMargin).mkString(Util.newLine)}""".stripMargin + |${formatBlock(millArgSignature, leftMargin) + .mkString(Util.newLine)}""".stripMargin ) (true, None) case Right((cliConfig, leftoverArgs)) => - val repl = leftoverArgs.isEmpty if (repl && stdin == DummyInputStream) { - stderr.println("Build repl needs to be run with the -i/--interactive flag") + stderr.println( + "Build repl needs to be run with the -i/--interactive flag") (false, stateCache) - }else{ + } else { val tqs = "\"\"\"" val config = - if(!repl) cliConfig - else cliConfig.copy( - predefCode = - s"""import $$file.build, build._ + if (!repl) cliConfig + else + cliConfig.copy( + predefCode = s"""import $$file.build, build._ |implicit val replApplyHandler = mill.main.ReplApplyHandler( - | ammonite.ops.Path($tqs${cliConfig.home.toIO.getCanonicalPath.replaceAllLiterally("$", "$$")}$tqs), + | ammonite.ops.Path($tqs${cliConfig.home.toIO.getCanonicalPath + .replaceAllLiterally("$", "$$")}$tqs), | interp.colors(), | repl.pprinter(), | build.millSelf.get, @@ -109,22 +120,27 @@ object Main { |import replApplyHandler.generatedEval._ | """.stripMargin, - welcomeBanner = None - ) + welcomeBanner = None + ) val runner = new mill.main.MainRunner( config.copy(colored = Some(mainInteractive)), - stdout, stderr, stdin, - stateCache + stdout, + stderr, + stdin, + stateCache, + env ) - if (repl){ + if (repl) { runner.printInfo("Loading...") - (runner.watchLoop(isRepl = true, printing = false, _.run()), runner.stateCache) + (runner.watchLoop(isRepl = true, printing = false, _.run()), + runner.stateCache) } else { - (runner.runScript(pwd / "build.sc", leftoverArgs), runner.stateCache) + (runner.runScript(pwd / "build.sc", leftoverArgs), + runner.stateCache) } - } + } } } diff --git a/main/src/mill/main/MainRunner.scala b/main/src/mill/main/MainRunner.scala index efebd5a5067..fed664fd139 100644 --- a/main/src/mill/main/MainRunner.scala +++ b/main/src/mill/main/MainRunner.scala @@ -20,7 +20,8 @@ class MainRunner(val config: ammonite.main.Cli.Config, outprintStream: PrintStream, errPrintStream: PrintStream, stdIn: InputStream, - stateCache0: Option[Evaluator.State] = None) + stateCache0: Option[Evaluator.State] = None, + env : Map[String, String]) extends ammonite.MainRunner( config, outprintStream, errPrintStream, stdIn, outprintStream, errPrintStream @@ -75,7 +76,8 @@ class MainRunner(val config: ammonite.main.Cli.Config, errPrintStream, errPrintStream, stdIn - ) + ), + env ) result match{ diff --git a/main/src/mill/main/RunScript.scala b/main/src/mill/main/RunScript.scala index 77930cc8ab1..75042dea0f3 100644 --- a/main/src/mill/main/RunScript.scala +++ b/main/src/mill/main/RunScript.scala @@ -29,7 +29,8 @@ object RunScript{ instantiateInterpreter: => Either[(Res.Failing, Seq[(Path, Long)]), ammonite.interp.Interpreter], scriptArgs: Seq[String], stateCache: Option[Evaluator.State], - log: Logger) + log: Logger, + env : Map[String, String]) : (Res[(Evaluator[Any], Seq[PathRef], Either[String, Seq[Js.Value]])], Seq[(Path, Long)]) = { val (evalState, interpWatched) = stateCache match{ @@ -53,7 +54,8 @@ object RunScript{ val evalRes = for(s <- evalState) - yield new Evaluator[Any](home, wd / 'out, wd / 'out, s.rootModule, log, s.classLoaderSig, s.workerCache) + yield new Evaluator[Any](home, wd / 'out, wd / 'out, s.rootModule, log, + s.classLoaderSig, s.workerCache, env) val evaluated = for{ evaluator <- evalRes diff --git a/main/test/src/mill/util/ScriptTestSuite.scala b/main/test/src/mill/util/ScriptTestSuite.scala index a2f2676a20e..f88007c5060 100644 --- a/main/test/src/mill/util/ScriptTestSuite.scala +++ b/main/test/src/mill/util/ScriptTestSuite.scala @@ -15,7 +15,7 @@ abstract class ScriptTestSuite(fork: Boolean) extends TestSuite{ val stdIn = new ByteArrayInputStream(Array()) lazy val runner = new mill.main.MainRunner( ammonite.main.Cli.Config(wd = workspacePath), - stdOutErr, stdOutErr, stdIn + stdOutErr, stdOutErr, stdIn, None, Map.empty ) def eval(s: String*) = { if (!fork) runner.runScript(workspacePath / "build.sc", s.toList)