From 231979590581aceb104b1469a0385199d925ed01 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 10 May 2023 08:32:30 +0800 Subject: [PATCH 1/7] compiles --- bsp/src/mill/bsp/BSP.scala | 2 +- .../bsp/worker/MillScalaBuildServer.scala | 2 +- build.sc | 17 +- .../jmh/src/mill/contrib/jmh/JmhModule.scala | 3 +- .../src/mill/playlib/RouterModule.scala | 2 +- .../src/mill/contrib/proguard/Proguard.scala | 2 +- .../mill/contrib/proguard/ProguardTests.scala | 2 +- .../contrib/scoverage/ScoverageModule.scala | 2 +- .../test/src/mill/testng/TestNGTests.scala | 2 +- .../scalamodule/10-assembly-config/build.sc | 2 +- integration/thirdparty/ammonite/repo/build.sc | 4 +- integration/thirdparty/caffeine/repo/build.sc | 2 +- .../src/mill/eval/JavaCompileJarTests.scala | 2 +- main/src/mill/main/MainModule.scala | 2 +- main/src/mill/main/VisualizeModule.scala | 2 +- main/src/mill/modules/Jvm.scala | 452 ------------------ main/src/mill/modules/Util.scala | 93 ---- .../src/mill/util}/CoursierSupport.scala | 10 +- main/util/src/mill/util/Jvm.scala | 364 +++++++++++++- main/util/src/mill/util/Util.scala | 91 +++- .../test/src/mill/util}/JvmTests.scala | 6 +- .../src/mill/runner/MillBuildRootModule.scala | 4 +- .../src/mill/scalajslib/ScalaJSModule.scala | 2 +- .../src/mill/scalalib}/Assembly.scala | 108 ++++- scalalib/src/mill/scalalib/GenIdeaImpl.scala | 2 +- scalalib/src/mill/scalalib/JavaModule.scala | 9 +- scalalib/src/mill/scalalib/Lib.scala | 4 +- .../src/mill/scalalib/PublishModule.scala | 2 +- scalalib/src/mill/scalalib/ScalaModule.scala | 4 +- scalalib/src/mill/scalalib/TestModule.scala | 2 +- .../src/mill/scalalib/ZincWorkerModule.scala | 2 +- .../mill/scalalib/giter8/Giter8Module.scala | 2 +- .../scalalib/publish/SonatypePublisher.scala | 2 +- .../src/mill/scalalib/HelloWorldTests.scala | 2 +- .../scalanativelib/ScalaNativeModule.scala | 4 +- 35 files changed, 601 insertions(+), 612 deletions(-) delete mode 100644 main/src/mill/modules/Jvm.scala delete mode 100644 main/src/mill/modules/Util.scala rename main/{src/mill/modules => util/src/mill/util}/CoursierSupport.scala (98%) rename main/{test/src/mill/modules => util/test/src/mill/util}/JvmTests.scala (97%) rename {main/src/mill/modules => scalalib/src/mill/scalalib}/Assembly.scala (62%) diff --git a/bsp/src/mill/bsp/BSP.scala b/bsp/src/mill/bsp/BSP.scala index a71c6844f30..79392b9903d 100644 --- a/bsp/src/mill/bsp/BSP.scala +++ b/bsp/src/mill/bsp/BSP.scala @@ -8,7 +8,7 @@ import mill.{Agg, T, BuildInfo => MillBuildInfo} import mill.define.{Command, Discover, ExternalModule, Task} import mill.eval.Evaluator import mill.main.{BspServerHandle, BspServerResult, BspServerStarter} -import mill.modules.Util.millProjectModule +import mill.util.Util.millProjectModule import mill.scalalib.{CoursierModule, Dep} import mill.util.PrintLogger import os.Path diff --git a/bsp/worker/src/mill/bsp/worker/MillScalaBuildServer.scala b/bsp/worker/src/mill/bsp/worker/MillScalaBuildServer.scala index d06ccfcd0c8..ad69114c8ce 100644 --- a/bsp/worker/src/mill/bsp/worker/MillScalaBuildServer.scala +++ b/bsp/worker/src/mill/bsp/worker/MillScalaBuildServer.scala @@ -15,7 +15,7 @@ import ch.epfl.scala.bsp4j.{ } import mill.{Agg, T} import mill.api.internal -import mill.modules.Jvm +import mill.util.Jvm import mill.scalalib.{JavaModule, ScalaModule, SemanticDbJavaModule, TestModule} import mill.testrunner.TestRunner import sbt.testing.Fingerprint diff --git a/build.sc b/build.sc index 53f411ebeac..0311fdf8b54 100644 --- a/build.sc +++ b/build.sc @@ -120,6 +120,7 @@ object Deps { val jgraphtCore = ivy"org.jgrapht:jgrapht-core:1.4.0" // 1.5.0+ dont support JDK8 + val jline = ivy"org.jline:jline:3.21.0" val jna = ivy"net.java.dev.jna:jna:5.13.0" val jnaPlatform = ivy"net.java.dev.jna:jna-platform:5.13.0" @@ -665,7 +666,10 @@ object main extends MillModule { object api extends MillApiModule with BuildInfo with MillAutoTestSetup { def buildInfoPackageName = "mill.api" - def buildInfoMembers = Seq(BuildInfo.Value("millVersion", millVersion(), "Mill version.")) + def buildInfoMembers = Seq( + BuildInfo.Value("millVersion", millVersion(), "Mill version."), + BuildInfo.Value("millDocUrl", Settings.docUrl, "Mill documentation url.") + ) override def ivyDeps = Agg( Deps.osLib, Deps.upickle, @@ -675,9 +679,10 @@ object main extends MillModule { ) } object util extends MillApiModule with MillAutoTestSetup { - override def moduleDeps = Seq(api) + override def moduleDeps = Seq(api, client) override def ivyDeps = Agg( - Deps.fansi + Deps.coursier, + Deps.jline ) } object define extends MillModule with BuildInfo { @@ -687,9 +692,8 @@ object main extends MillModule { ) override def ivyDeps = Agg( Deps.millModuledefs, - Deps.millModuledefsPlugin, +// Deps.millModuledefsPlugin, Deps.scalametaTrees, - Deps.coursier, // Necessary so we can share the JNA classes throughout the build process Deps.jna, Deps.jnaPlatform, @@ -719,8 +723,7 @@ object main extends MillModule { "millScalacPluginDeps", Deps.millModuledefsString, "Scalac compiler plugin dependencies to compile the build script." - ), - BuildInfo.Value("millDocUrl", Settings.docUrl, "Mill documentation url.") + ) ) } diff --git a/contrib/jmh/src/mill/contrib/jmh/JmhModule.scala b/contrib/jmh/src/mill/contrib/jmh/JmhModule.scala index 52d595f6821..fdceb97d002 100644 --- a/contrib/jmh/src/mill/contrib/jmh/JmhModule.scala +++ b/contrib/jmh/src/mill/contrib/jmh/JmhModule.scala @@ -1,6 +1,7 @@ package mill.contrib.jmh -import mill._, scalalib._, modules._ +import mill._, scalalib._ +import mill.util.Jvm /** * This module provides an easy way to integrate JMH benchmarking with Mill. diff --git a/contrib/playlib/src/mill/playlib/RouterModule.scala b/contrib/playlib/src/mill/playlib/RouterModule.scala index 661add80198..3b9ee6cc03c 100644 --- a/contrib/playlib/src/mill/playlib/RouterModule.scala +++ b/contrib/playlib/src/mill/playlib/RouterModule.scala @@ -1,7 +1,7 @@ package mill.playlib import mill.api.PathRef -import mill.modules.Util.millProjectModule +import mill.util.Util.millProjectModule import mill.playlib.api.RouteCompilerType import mill.scalalib._ import mill.scalalib.api._ diff --git a/contrib/proguard/src/mill/contrib/proguard/Proguard.scala b/contrib/proguard/src/mill/contrib/proguard/Proguard.scala index 5c655e4ea95..4cc958c3b68 100644 --- a/contrib/proguard/src/mill/contrib/proguard/Proguard.scala +++ b/contrib/proguard/src/mill/contrib/proguard/Proguard.scala @@ -4,7 +4,7 @@ import mill.java9rtexport.Export import mill.T import mill.Agg import mill.api.{Loose, PathRef} -import mill.modules.Jvm +import mill.util.Jvm import mill.scalalib.{DepSyntax, ScalaModule} import os.{Path, Shellable} diff --git a/contrib/proguard/test/src/mill/contrib/proguard/ProguardTests.scala b/contrib/proguard/test/src/mill/contrib/proguard/ProguardTests.scala index ca0adbdcac6..a79c985f583 100644 --- a/contrib/proguard/test/src/mill/contrib/proguard/ProguardTests.scala +++ b/contrib/proguard/test/src/mill/contrib/proguard/ProguardTests.scala @@ -3,7 +3,7 @@ package mill.contrib.proguard import scala.util.control.NonFatal import mill._ import mill.define.Target -import mill.modules.Util.millProjectModule +import mill.util.Util.millProjectModule import mill.scalalib.ScalaModule import mill.util.TestEvaluator import mill.util.TestUtil diff --git a/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala b/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala index a3961659a44..19010ada4ca 100644 --- a/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala +++ b/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala @@ -7,7 +7,7 @@ import mill.contrib.scoverage.api.ScoverageReportWorkerApi.ReportType import mill.scalalib.api.ZincWorkerUtil import mill.scalalib.{Dep, DepSyntax, JavaModule, ScalaModule} import mill.api.Result -import mill.modules.Util.millProjectModule +import mill.util.Util.millProjectModule import scala.util.Try diff --git a/contrib/testng/test/src/mill/testng/TestNGTests.scala b/contrib/testng/test/src/mill/testng/TestNGTests.scala index 7c2e6e034e1..90aae8e5b24 100644 --- a/contrib/testng/test/src/mill/testng/TestNGTests.scala +++ b/contrib/testng/test/src/mill/testng/TestNGTests.scala @@ -3,7 +3,7 @@ package testng import mill.api.Result.Exception import mill.define.Target -import mill.modules.Util.millProjectModule +import mill.util.Util.millProjectModule import mill.scalalib._ import mill.util.{TestEvaluator, TestUtil} import utest.framework.TestPath diff --git a/example/scalamodule/10-assembly-config/build.sc b/example/scalamodule/10-assembly-config/build.sc index b51e1031016..2ac91b0f4a5 100644 --- a/example/scalamodule/10-assembly-config/build.sc +++ b/example/scalamodule/10-assembly-config/build.sc @@ -1,5 +1,5 @@ import mill._, scalalib._ -import mill.modules.Assembly._ +import mill.scalalib.Assembly._ object foo extends ScalaModule { def moduleDeps = Seq(bar) diff --git a/integration/thirdparty/ammonite/repo/build.sc b/integration/thirdparty/ammonite/repo/build.sc index aa5f9514754..f013b46fc9f 100644 --- a/integration/thirdparty/ammonite/repo/build.sc +++ b/integration/thirdparty/ammonite/repo/build.sc @@ -58,7 +58,7 @@ trait AmmDependenciesResourceFileModule extends JavaModule { val deps0 = T.task { compileIvyDeps().map(bindDependency()) ++ transitiveIvyDeps() }() - val (_, res) = mill.modules.Jvm.resolveDependenciesMetadata( + val (_, res) = mill.util.Jvm.resolveDependenciesMetadata( repositoriesTask(), deps0.map(_.dep), deps0.filter(_.force).map(_.dep), @@ -206,7 +206,7 @@ class MainModule(val crossScalaVersion: String) extends AmmModule externalSources() def prependShellScript = T { - mill.modules.Jvm.launcherUniversalScript( + mill.scalalib.Jvm.launcherUniversalScript( mainClass().get, Agg("$0"), Agg("%~dpnx0"), diff --git a/integration/thirdparty/caffeine/repo/build.sc b/integration/thirdparty/caffeine/repo/build.sc index 516762e285b..7211b1880a9 100644 --- a/integration/thirdparty/caffeine/repo/build.sc +++ b/integration/thirdparty/caffeine/repo/build.sc @@ -1,7 +1,7 @@ import mill._ import mill.scalalib._ import coursier.MavenRepository -import mill.modules.Jvm +import mill.util.Jvm import $file.deps import deps.{ benchmarkLibraries, diff --git a/main/eval/test/src/mill/eval/JavaCompileJarTests.scala b/main/eval/test/src/mill/eval/JavaCompileJarTests.scala index 6dcbe8e19f5..ea4662145fa 100644 --- a/main/eval/test/src/mill/eval/JavaCompileJarTests.scala +++ b/main/eval/test/src/mill/eval/JavaCompileJarTests.scala @@ -1,6 +1,6 @@ package mill.eval -import mill.modules.Jvm +import mill.util.Jvm import mill.api.Ctx.Dest import mill.{Module, T} import mill.util.{DummyLogger, TestEvaluator, TestUtil} diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala index e7af7632606..21faa80a149 100644 --- a/main/src/mill/main/MainModule.scala +++ b/main/src/mill/main/MainModule.scala @@ -232,7 +232,7 @@ trait MainModule extends mill.Module { val allDocs = for (a <- annots.distinct) - yield mill.modules.Util.cleanupScaladoc(a.value).map("\n " + _).mkString + yield mill.util.Util.cleanupScaladoc(a.value).map("\n " + _).mkString pprint.Tree.Lazy(ctx => Iterator( diff --git a/main/src/mill/main/VisualizeModule.scala b/main/src/mill/main/VisualizeModule.scala index e4309949db5..2ad1b128f40 100644 --- a/main/src/mill/main/VisualizeModule.scala +++ b/main/src/mill/main/VisualizeModule.scala @@ -7,7 +7,7 @@ import coursier.maven.MavenRepository import mill.T import mill.define.{Discover, ExternalModule} import mill.api.{PathRef, Result} -import mill.modules.Util.millProjectModule +import mill.util.Util.millProjectModule object VisualizeModule extends ExternalModule with VisualizeModule { def repositories = Seq( diff --git a/main/src/mill/modules/Jvm.scala b/main/src/mill/modules/Jvm.scala deleted file mode 100644 index e2e30a673b5..00000000000 --- a/main/src/mill/modules/Jvm.scala +++ /dev/null @@ -1,452 +0,0 @@ -package mill.modules - -import mill.BuildInfo -import mill.api.Loose.Agg -import mill.api._ -import mill.main.client.InputPumper -import mill.modules.Assembly.{AppendEntry, WriteOnceEntry} -import os.SubProcess -import upickle.default.{ReadWriter => RW} - -import java.io._ -import java.lang.reflect.Modifier -import java.net.URI -import java.nio.file.attribute.PosixFilePermission -import java.nio.file.{FileSystems, Files, StandardOpenOption} -import java.util.Collections -import java.util.jar.{Attributes, JarFile, Manifest} -import scala.jdk.CollectionConverters._ -import scala.util.Properties.isWin -import scala.util.Using - -object Jvm extends CoursierSupport { - - /** - * Runs a JVM subprocess with the given configuration and returns a - * [[os.CommandResult]] with it's aggregated output and error streams - */ - def callSubprocess( - mainClass: String, - classPath: Agg[os.Path], - jvmArgs: Seq[String] = Seq.empty, - envArgs: Map[String, String] = Map.empty, - mainArgs: Seq[String] = Seq.empty, - workingDir: os.Path = null, - streamOut: Boolean = true - )(implicit ctx: Ctx) = { - - val commandArgs = - Vector(javaExe) ++ - jvmArgs ++ - Vector("-cp", classPath.iterator.mkString(java.io.File.pathSeparator), mainClass) ++ - mainArgs - - val workingDir1 = Option(workingDir).getOrElse(ctx.dest) - os.makeDir.all(workingDir1) - - os.proc(commandArgs).call(cwd = workingDir1, env = envArgs) - } - - /** - * Resolves a tool to a path under the currently used JDK (if known). - */ - def jdkTool(toolName: String): String = { - sys.props - .get("java.home") - .map(h => - if (isWin) new File(h, s"bin\\${toolName}.exe") - else new File(h, s"bin/${toolName}") - ) - .filter(f => f.exists()) - .fold(toolName)(_.getAbsolutePath()) - - } - - def javaExe: String = jdkTool("java") - - /** - * Runs a JVM subprocess with the given configuration and streams - * it's stdout and stderr to the console. - * @param mainClass The main class to run - * @param classPath The classpath - * @param JvmArgs Arguments given to the forked JVM - * @param envArgs Environment variables used when starting the forked JVM - * @param workingDir The working directory to be used by the forked JVM - * @param background `true` if the forked JVM should be spawned in background - * @param useCpPassingJar When `false`, the `-cp` parameter is used to pass the classpath - * to the forked JVM. - * When `true`, a temporary empty JAR is created - * which contains a `Class-Path` manifest entry containing the actual classpath. - * This might help with long classpaths on OS'es (like Windows) - * which only supports limited command-line length - */ - def runSubprocess( - mainClass: String, - classPath: Agg[os.Path], - jvmArgs: Seq[String] = Seq.empty, - envArgs: Map[String, String] = Map.empty, - mainArgs: Seq[String] = Seq.empty, - workingDir: os.Path = null, - background: Boolean = false, - useCpPassingJar: Boolean = false - )(implicit ctx: Ctx): Unit = { - - val cp = - if (useCpPassingJar && !classPath.iterator.isEmpty) { - val passingJar = os.temp(prefix = "run-", suffix = ".jar", deleteOnExit = false) - ctx.log.debug( - s"Creating classpath passing jar '${passingJar}' with Class-Path: ${classPath.iterator.map( - _.toNIO.toUri().toURL().toExternalForm() - ).mkString(" ")}" - ) - createClasspathPassingJar(passingJar, classPath) - Agg(passingJar) - } else { - classPath - } - - val args = - Vector(javaExe) ++ - jvmArgs ++ - Vector("-cp", cp.iterator.mkString(java.io.File.pathSeparator), mainClass) ++ - mainArgs - - ctx.log.debug(s"Run subprocess with args: ${args.map(a => s"'${a}'").mkString(" ")}") - - if (background) spawnSubprocess(args, envArgs, workingDir, background = true) - else runSubprocess(args, envArgs, workingDir) - } - - /** - * Runs a generic subprocess and waits for it to terminate. - */ - def runSubprocess(commandArgs: Seq[String], envArgs: Map[String, String], workingDir: os.Path) = { - val process = spawnSubprocess(commandArgs, envArgs, workingDir, background = false) - val shutdownHook = new Thread("subprocess-shutdown") { - override def run(): Unit = { - System.err.println("Host JVM shutdown. Forcefully destroying subprocess ...") - process.destroy() - } - } - Runtime.getRuntime().addShutdownHook(shutdownHook) - try { - process.waitFor() - } catch { - case e: InterruptedException => - System.err.println("Interrupted. Forcefully destroying subprocess ...") - process.destroy() - // rethrow - throw e - } finally { - Runtime.getRuntime().removeShutdownHook(shutdownHook) - } - if (process.exitCode() == 0) () - else throw new Exception("Interactive Subprocess Failed (exit code " + process.exitCode() + ")") - } - - /** - * Spawns a generic subprocess, streaming the stdout and stderr to the - * console. If the System.out/System.err have been substituted, makes sure - * that the subprocess's stdout and stderr streams go to the subtituted - * streams - */ - def spawnSubprocess( - commandArgs: Seq[String], - envArgs: Map[String, String], - workingDir: os.Path, - background: Boolean = false - ): SubProcess = { - // If System.in is fake, then we pump output manually rather than relying - // on `os.Inherit`. That is because `os.Inherit` does not follow changes - // to System.in/System.out/System.err, so the subprocess's streams get sent - // to the parent process's origin outputs even if we want to direct them - // elsewhere - - if (!SystemStreams.isOriginal()) { - val process = os.proc(commandArgs).spawn( - cwd = workingDir, - env = envArgs, - stdin = if (!background) os.Pipe else "", - stdout = if (!background) os.Pipe else workingDir / "stdout.log", - stderr = if (!background) os.Pipe else workingDir / "stderr.log" - ) - - val sources = Seq( - (process.stdout, System.out, "spawnSubprocess.stdout", false, () => true), - (process.stderr, System.err, "spawnSubprocess.stderr", false, () => true), - (System.in, process.stdin, "spawnSubprocess.stdin", true, () => process.isAlive()) - ) - - for ((std, dest, name, checkAvailable, runningCheck) <- sources) { - val t = new Thread( - new InputPumper(std, dest, checkAvailable, () => runningCheck()), - name - ) - t.setDaemon(true) - t.start() - } - - process - } else { - os.proc(commandArgs).spawn( - cwd = workingDir, - env = envArgs, - stdin = if (!background) os.Inherit else "", - stdout = if (!background) os.Inherit else workingDir / "stdout.log", - stderr = if (!background) os.Inherit else workingDir / "stderr.log" - ) - } - } - - def runLocal( - mainClass: String, - classPath: Agg[os.Path], - mainArgs: Seq[String] = Seq.empty - )(implicit ctx: Ctx): Unit = { - inprocess( - classPath, - classLoaderOverrideSbtTesting = false, - isolated = true, - closeContextClassLoaderWhenDone = true, - cl => { - getMainMethod(mainClass, cl).invoke(null, mainArgs.toArray) - } - ) - } - - private def getMainMethod(mainClassName: String, cl: ClassLoader) = { - val mainClass = cl.loadClass(mainClassName) - val method = mainClass.getMethod("main", classOf[Array[String]]) - // jvm allows the actual main class to be non-public and to run a method in the non-public class, - // we need to make it accessible - method.setAccessible(true) - val modifiers = method.getModifiers - if (!Modifier.isPublic(modifiers)) - throw new NoSuchMethodException(mainClassName + ".main is not public") - if (!Modifier.isStatic(modifiers)) - throw new NoSuchMethodException(mainClassName + ".main is not static") - method - } - - def inprocess[T]( - classPath: Agg[os.Path], - classLoaderOverrideSbtTesting: Boolean, - isolated: Boolean, - closeContextClassLoaderWhenDone: Boolean, - body: ClassLoader => T - )(implicit ctx: Ctx.Home): T = { - mill.util.Jvm.inprocess( - classPath, - classLoaderOverrideSbtTesting, - isolated, - closeContextClassLoaderWhenDone, - body - ) - } - - def createManifest(mainClass: Option[String]): mill.api.JarManifest = { - mainClass.foldLeft(mill.api.JarManifest.MillDefault)((m, c) => - m.add((java.util.jar.Attributes.Name.MAIN_CLASS.toString, c)) - ) - } - - /** - * Create a jar file containing all files from the specified input Paths, - * called out.jar in the implicit ctx.dest folder. An optional main class may - * be provided for the jar. An optional filter function may also be provided to - * selectively include/exclude specific files. - * @param inputPaths - `Agg` of `os.Path`s containing files to be included in the jar - * @param fileFilter - optional file filter to select files to be included. - * Given a `os.Path` (from inputPaths) and a `os.RelPath` for the individual file, - * return true if the file is to be included in the jar. - * @param ctx - implicit `Ctx.Dest` used to determine the output directory for the jar. - * @return - a `PathRef` for the created jar. - */ - def createJar( - inputPaths: Agg[os.Path], - manifest: mill.api.JarManifest = mill.api.JarManifest.MillDefault, - fileFilter: (os.Path, os.RelPath) => Boolean = (_, _) => true - )(implicit ctx: Ctx.Dest): PathRef = { - val outputPath = ctx.dest / "out.jar" - createJar( - jar = outputPath, - inputPaths = inputPaths, - manifest = manifest, - fileFilter = fileFilter - ) - PathRef(outputPath) - } - - def createJar( - jar: os.Path, - inputPaths: Agg[os.Path], - manifest: mill.api.JarManifest, - fileFilter: (os.Path, os.RelPath) => Boolean - ): Unit = - JarOps.jar(jar, inputPaths, manifest, fileFilter, includeDirs = true, timestamp = None) - - def createClasspathPassingJar(jar: os.Path, classpath: Agg[os.Path]): Unit = { - createJar( - jar = jar, - inputPaths = Agg(), - manifest = mill.api.JarManifest.MillDefault.add( - "Class-Path" -> classpath.iterator.map(_.toNIO.toUri().toURL().toExternalForm()).mkString( - " " - ) - ), - fileFilter = (_, _) => true - ) - } - - def createAssembly( - inputPaths: Agg[os.Path], - manifest: mill.api.JarManifest = mill.api.JarManifest.MillDefault, - prependShellScript: String = "", - base: Option[os.Path] = None, - assemblyRules: Seq[Assembly.Rule] = Assembly.defaultRules - )(implicit ctx: Ctx.Dest with Ctx.Log): PathRef = { - val tmp = ctx.dest / "out-tmp.jar" - - val baseUri = "jar:" + tmp.toIO.getCanonicalFile.toURI.toASCIIString - val hm = base.fold(Map("create" -> "true")) { b => - os.copy(b, tmp) - Map.empty - } - Using.resource(FileSystems.newFileSystem(URI.create(baseUri), hm.asJava)) { zipFs => - val manifestPath = zipFs.getPath(JarFile.MANIFEST_NAME) - Files.createDirectories(manifestPath.getParent) - val manifestOut = Files.newOutputStream( - manifestPath, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.CREATE - ) - manifest.build.write(manifestOut) - manifestOut.close() - - val (mappings, resourceCleaner) = Assembly.loadShadedClasspath(inputPaths, assemblyRules) - try { - Assembly.groupAssemblyEntries(mappings, assemblyRules).foreach { - case (mapping, entry) => - val path = zipFs.getPath(mapping).toAbsolutePath - entry match { - case entry: AppendEntry => - val separated = entry.inputStreams - .flatMap(inputStream => - Seq(new ByteArrayInputStream(entry.separator.getBytes), inputStream()) - ) - val cleaned = if (Files.exists(path)) separated else separated.drop(1) - val concatenated = - new SequenceInputStream(Collections.enumeration(cleaned.asJava)) - writeEntry(path, concatenated, append = true) - case entry: WriteOnceEntry => writeEntry(path, entry.inputStream(), append = false) - } - } - } finally { - resourceCleaner() - } - } - - val output = ctx.dest / "out.jar" - // Prepend shell script and make it executable - if (prependShellScript.isEmpty) os.move(tmp, output) - else { - val lineSep = if (!prependShellScript.endsWith("\n")) "\n\r\n" else "" - os.write(output, prependShellScript + lineSep) - os.write.append(output, os.read.inputStream(tmp)) - - if (!scala.util.Properties.isWin) { - os.perms.set( - output, - os.perms(output) - + PosixFilePermission.GROUP_EXECUTE - + PosixFilePermission.OWNER_EXECUTE - + PosixFilePermission.OTHERS_EXECUTE - ) - } - } - - PathRef(output) - } - - private def writeEntry(p: java.nio.file.Path, inputStream: InputStream, append: Boolean): Unit = { - if (p.getParent != null) Files.createDirectories(p.getParent) - val options = - if (append) Seq(StandardOpenOption.APPEND, StandardOpenOption.CREATE) - else Seq(StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE) - - val outputStream = java.nio.file.Files.newOutputStream(p, options: _*) - IO.stream(inputStream, outputStream) - outputStream.close() - inputStream.close() - } - - def universalScript( - shellCommands: String, - cmdCommands: String, - shebang: Boolean = false - ): String = { - Seq( - if (shebang) "#!/usr/bin/env sh" else "", - "@ 2>/dev/null # 2>nul & echo off & goto BOF\r", - ":", - shellCommands.replaceAll("\r\n|\n", "\n"), - "exit", - Seq( - "", - ":BOF", - "setlocal", - "@echo off", - cmdCommands.replaceAll("\r\n|\n", "\r\n"), - "endlocal", - "exit /B %errorlevel%", - "" - ).mkString("\r\n") - ).filterNot(_.isEmpty).mkString("\n") - } - - def launcherUniversalScript( - mainClass: String, - shellClassPath: Agg[String], - cmdClassPath: Agg[String], - jvmArgs: Seq[String], - shebang: Boolean = false - ): String = { - universalScript( - shellCommands = - s"""exec java ${jvmArgs.mkString(" ")} $$JAVA_OPTS -cp "${shellClassPath.iterator.mkString( - ":" - )}" '$mainClass' "$$@"""", - cmdCommands = - s"""java ${jvmArgs.mkString(" ")} %JAVA_OPTS% -cp "${cmdClassPath.iterator.mkString( - ";" - )}" $mainClass %*""", - shebang = shebang - ) - } - def createLauncher(mainClass: String, classPath: Agg[os.Path], jvmArgs: Seq[String])(implicit - ctx: Ctx.Dest - ): PathRef = { - val isWin = scala.util.Properties.isWin - val isBatch = - isWin && !(org.jline.utils.OSUtils.IS_CYGWIN || org.jline.utils.OSUtils.IS_MSYSTEM) - val outputPath = ctx.dest / (if (isBatch) "run.bat" else "run") - val classPathStrs = classPath.map(_.toString) - - os.write(outputPath, launcherUniversalScript(mainClass, classPathStrs, classPathStrs, jvmArgs)) - - if (!isWin) { - val perms = Files.getPosixFilePermissions(outputPath.toNIO) - perms.add(PosixFilePermission.GROUP_EXECUTE) - perms.add(PosixFilePermission.OWNER_EXECUTE) - perms.add(PosixFilePermission.OTHERS_EXECUTE) - Files.setPosixFilePermissions(outputPath.toNIO, perms) - } - PathRef(outputPath) - } - - @deprecated("Use mill.api.JarManifest instead", "Mill after 0.11.0-M4") - type JarManifest = mill.api.JarManifest - @deprecated("Use mill.api.JarManifest instead", "Mill after 0.11.0-M4") - val JarManifest = mill.api.JarManifest - -} diff --git a/main/src/mill/modules/Util.scala b/main/src/mill/modules/Util.scala deleted file mode 100644 index 34dcfefa7d2..00000000000 --- a/main/src/mill/modules/Util.scala +++ /dev/null @@ -1,93 +0,0 @@ -package mill.modules - -import coursier.Repository -import mill.{Agg, BuildInfo} -import mill.api.{Ctx, IO, Loose, PathRef, Result} - -object Util { - - private val LongMillProps = new java.util.Properties() - - { - val millOptionsPath = sys.props("MILL_OPTIONS_PATH") - if (millOptionsPath != null) - LongMillProps.load(new java.io.FileInputStream(millOptionsPath)) - } - - def cleanupScaladoc(v: String) = { - v.linesIterator.map( - _.dropWhile(_.isWhitespace) - .stripPrefix("/**") - .stripPrefix("*/") - .stripPrefix("*") - .stripSuffix("**/") - .stripSuffix("*/") - .dropWhile(_.isWhitespace) - ).toArray - .dropWhile(_.isEmpty) - .reverse - .dropWhile(_.isEmpty) - .reverse - } - - def download(url: String, dest: os.RelPath = os.rel / "download")(implicit - ctx: Ctx.Dest - ): PathRef = { - val out = ctx.dest / dest - - val website = new java.net.URI(url).toURL - val rbc = java.nio.channels.Channels.newChannel(website.openStream) - try { - val fos = new java.io.FileOutputStream(out.toIO) - try { - fos.getChannel.transferFrom(rbc, 0, java.lang.Long.MAX_VALUE) - PathRef(out) - } finally { - fos.close() - } - } finally { - rbc.close() - } - } - - def downloadUnpackZip(url: String, dest: os.RelPath = os.rel / "unpacked")(implicit - ctx: Ctx.Dest - ) = { - - val tmpName = if (dest == os.rel / "tmp.zip") "tmp2.zip" else "tmp.zip" - val downloaded = download(url, os.rel / tmpName) - IO.unpackZip(downloaded.path, dest) - } - - /** - * Deprecated helper method, intended to allow runtime resolution and in-development-tree testings of mill plugins possible. - * This design has issues and will probably replaced. - */ - def millProjectModule( - artifact: String, - repositories: Seq[Repository], - resolveFilter: os.Path => Boolean = _ => true, - // this should correspond to the mill runtime Scala version - artifactSuffix: String = "_2.13" - ): Result[Agg[PathRef]] = { - - mill.modules.Jvm.resolveDependencies( - repositories = repositories, - deps = Seq( - coursier.Dependency( - coursier.Module( - coursier.Organization("com.lihaoyi"), - coursier.ModuleName(artifact + artifactSuffix) - ), - BuildInfo.millVersion - ) - ), - force = Nil, - resolveFilter = resolveFilter - ) - } - - def millProperty(key: String): Option[String] = - Option(sys.props(key)) // System property has priority - .orElse(Option(LongMillProps.getProperty(key))) -} diff --git a/main/src/mill/modules/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala similarity index 98% rename from main/src/mill/modules/CoursierSupport.scala rename to main/util/src/mill/util/CoursierSupport.scala index f2b9c5b3960..6a4328810dc 100644 --- a/main/src/mill/modules/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -1,12 +1,12 @@ -package mill.modules +package mill.util + import coursier.cache.ArtifactError import coursier.parse.RepositoryParser import coursier.util.{Gather, Task} import coursier.{Dependency, Repository, Resolution} -import mill.Agg import mill.api.{Ctx, PathRef, Result} - +import mill.api.Loose.Agg import java.io.File import java.nio.file.NoSuchFileException import scala.annotation.tailrec @@ -138,7 +138,7 @@ trait CoursierSupport { |-------------------------------------------- | |For additional information on library dependencies, see the docs at - |${mill.BuildInfo.millDocUrl}/mill/Library_Dependencies.html""".stripMargin + |${mill.api.BuildInfo.millDocUrl}/mill/Library_Dependencies.html""".stripMargin val errLines = errs.map { case ((module, vsn), errMsgs) => s" ${module.trim}:$vsn \n\t" + errMsgs.mkString("\n\t") @@ -194,7 +194,7 @@ trait CoursierSupport { if (errors.isEmpty) { Result.Success( - mill.Agg.from( + Agg.from( successes.map(os.Path(_)).filter(_.ext == "jar").map(PathRef(_, quick = true)) ).filter(x => resolveFilter(x.path)) ++ localTestDeps.flatten ) diff --git a/main/util/src/mill/util/Jvm.scala b/main/util/src/mill/util/Jvm.scala index dee80cfde08..a80baada025 100644 --- a/main/util/src/mill/util/Jvm.scala +++ b/main/util/src/mill/util/Jvm.scala @@ -1,15 +1,240 @@ package mill.util + +import mill.api.BuildInfo import mill.api.Loose.Agg -object Jvm { +import mill.api._ +import mill.main.client.InputPumper +import os.SubProcess +import upickle.default.{ReadWriter => RW} - def inprocess[T]( +import java.io._ +import java.lang.reflect.Modifier +import java.net.URI +import java.nio.file.attribute.PosixFilePermission +import java.nio.file.{FileSystems, Files, StandardOpenOption} +import java.util.Collections +import java.util.jar.{Attributes, JarFile, Manifest} +import scala.jdk.CollectionConverters._ +import scala.util.Properties.isWin +import scala.util.Using + +object Jvm extends CoursierSupport { + + /** + * Runs a JVM subprocess with the given configuration and returns a + * [[os.CommandResult]] with it's aggregated output and error streams + */ + def callSubprocess( + mainClass: String, classPath: Agg[os.Path], - classLoaderOverrideSbtTesting: Boolean, - isolated: Boolean, - closeContextClassLoaderWhenDone: Boolean, - body: ClassLoader => T - )(implicit ctx: mill.api.Ctx.Home): T = { + jvmArgs: Seq[String] = Seq.empty, + envArgs: Map[String, String] = Map.empty, + mainArgs: Seq[String] = Seq.empty, + workingDir: os.Path = null, + streamOut: Boolean = true + )(implicit ctx: Ctx) = { + + val commandArgs = + Vector(javaExe) ++ + jvmArgs ++ + Vector("-cp", classPath.iterator.mkString(java.io.File.pathSeparator), mainClass) ++ + mainArgs + + val workingDir1 = Option(workingDir).getOrElse(ctx.dest) + os.makeDir.all(workingDir1) + + os.proc(commandArgs).call(cwd = workingDir1, env = envArgs) + } + + /** + * Resolves a tool to a path under the currently used JDK (if known). + */ + def jdkTool(toolName: String): String = { + sys.props + .get("java.home") + .map(h => + if (isWin) new File(h, s"bin\\${toolName}.exe") + else new File(h, s"bin/${toolName}") + ) + .filter(f => f.exists()) + .fold(toolName)(_.getAbsolutePath()) + + } + + def javaExe: String = jdkTool("java") + + /** + * Runs a JVM subprocess with the given configuration and streams + * it's stdout and stderr to the console. + * @param mainClass The main class to run + * @param classPath The classpath + * @param JvmArgs Arguments given to the forked JVM + * @param envArgs Environment variables used when starting the forked JVM + * @param workingDir The working directory to be used by the forked JVM + * @param background `true` if the forked JVM should be spawned in background + * @param useCpPassingJar When `false`, the `-cp` parameter is used to pass the classpath + * to the forked JVM. + * When `true`, a temporary empty JAR is created + * which contains a `Class-Path` manifest entry containing the actual classpath. + * This might help with long classpaths on OS'es (like Windows) + * which only supports limited command-line length + */ + def runSubprocess( + mainClass: String, + classPath: Agg[os.Path], + jvmArgs: Seq[String] = Seq.empty, + envArgs: Map[String, String] = Map.empty, + mainArgs: Seq[String] = Seq.empty, + workingDir: os.Path = null, + background: Boolean = false, + useCpPassingJar: Boolean = false + )(implicit ctx: Ctx): Unit = { + + val cp = + if (useCpPassingJar && !classPath.iterator.isEmpty) { + val passingJar = os.temp(prefix = "run-", suffix = ".jar", deleteOnExit = false) + ctx.log.debug( + s"Creating classpath passing jar '${passingJar}' with Class-Path: ${classPath.iterator.map( + _.toNIO.toUri().toURL().toExternalForm() + ).mkString(" ")}" + ) + createClasspathPassingJar(passingJar, classPath) + Agg(passingJar) + } else { + classPath + } + + val args = + Vector(javaExe) ++ + jvmArgs ++ + Vector("-cp", cp.iterator.mkString(java.io.File.pathSeparator), mainClass) ++ + mainArgs + + ctx.log.debug(s"Run subprocess with args: ${args.map(a => s"'${a}'").mkString(" ")}") + + if (background) spawnSubprocess(args, envArgs, workingDir, background = true) + else runSubprocess(args, envArgs, workingDir) + } + + /** + * Runs a generic subprocess and waits for it to terminate. + */ + def runSubprocess(commandArgs: Seq[String], envArgs: Map[String, String], workingDir: os.Path) = { + val process = spawnSubprocess(commandArgs, envArgs, workingDir, background = false) + val shutdownHook = new Thread("subprocess-shutdown") { + override def run(): Unit = { + System.err.println("Host JVM shutdown. Forcefully destroying subprocess ...") + process.destroy() + } + } + Runtime.getRuntime().addShutdownHook(shutdownHook) + try { + process.waitFor() + } catch { + case e: InterruptedException => + System.err.println("Interrupted. Forcefully destroying subprocess ...") + process.destroy() + // rethrow + throw e + } finally { + Runtime.getRuntime().removeShutdownHook(shutdownHook) + } + if (process.exitCode() == 0) () + else throw new Exception("Interactive Subprocess Failed (exit code " + process.exitCode() + ")") + } + + /** + * Spawns a generic subprocess, streaming the stdout and stderr to the + * console. If the System.out/System.err have been substituted, makes sure + * that the subprocess's stdout and stderr streams go to the subtituted + * streams + */ + def spawnSubprocess( + commandArgs: Seq[String], + envArgs: Map[String, String], + workingDir: os.Path, + background: Boolean = false + ): SubProcess = { + // If System.in is fake, then we pump output manually rather than relying + // on `os.Inherit`. That is because `os.Inherit` does not follow changes + // to System.in/System.out/System.err, so the subprocess's streams get sent + // to the parent process's origin outputs even if we want to direct them + // elsewhere + + if (!SystemStreams.isOriginal()) { + val process = os.proc(commandArgs).spawn( + cwd = workingDir, + env = envArgs, + stdin = if (!background) os.Pipe else "", + stdout = if (!background) os.Pipe else workingDir / "stdout.log", + stderr = if (!background) os.Pipe else workingDir / "stderr.log" + ) + + val sources = Seq( + (process.stdout, System.out, "spawnSubprocess.stdout", false, () => true), + (process.stderr, System.err, "spawnSubprocess.stderr", false, () => true), + (System.in, process.stdin, "spawnSubprocess.stdin", true, () => process.isAlive()) + ) + + for ((std, dest, name, checkAvailable, runningCheck) <- sources) { + val t = new Thread( + new InputPumper(std, dest, checkAvailable, () => runningCheck()), + name + ) + t.setDaemon(true) + t.start() + } + + process + } else { + os.proc(commandArgs).spawn( + cwd = workingDir, + env = envArgs, + stdin = if (!background) os.Inherit else "", + stdout = if (!background) os.Inherit else workingDir / "stdout.log", + stderr = if (!background) os.Inherit else workingDir / "stderr.log" + ) + } + } + + def runLocal( + mainClass: String, + classPath: Agg[os.Path], + mainArgs: Seq[String] = Seq.empty + )(implicit ctx: Ctx): Unit = { + inprocess( + classPath, + classLoaderOverrideSbtTesting = false, + isolated = true, + closeContextClassLoaderWhenDone = true, + cl => { + getMainMethod(mainClass, cl).invoke(null, mainArgs.toArray) + } + ) + } + + private def getMainMethod(mainClassName: String, cl: ClassLoader) = { + val mainClass = cl.loadClass(mainClassName) + val method = mainClass.getMethod("main", classOf[Array[String]]) + // jvm allows the actual main class to be non-public and to run a method in the non-public class, + // we need to make it accessible + method.setAccessible(true) + val modifiers = method.getModifiers + if (!Modifier.isPublic(modifiers)) + throw new NoSuchMethodException(mainClassName + ".main is not public") + if (!Modifier.isStatic(modifiers)) + throw new NoSuchMethodException(mainClassName + ".main is not static") + method + } + + def inprocess[T]( + classPath: Agg[os.Path], + classLoaderOverrideSbtTesting: Boolean, + isolated: Boolean, + closeContextClassLoaderWhenDone: Boolean, + body: ClassLoader => T + )(implicit ctx: mill.api.Ctx.Home): T = { val urls = classPath.map(_.toIO.toURI.toURL) val cl = if (classLoaderOverrideSbtTesting) { @@ -35,4 +260,129 @@ object Jvm { } } } + + def createManifest(mainClass: Option[String]): mill.api.JarManifest = { + mainClass.foldLeft(mill.api.JarManifest.MillDefault)((m, c) => + m.add((java.util.jar.Attributes.Name.MAIN_CLASS.toString, c)) + ) + } + + /** + * Create a jar file containing all files from the specified input Paths, + * called out.jar in the implicit ctx.dest folder. An optional main class may + * be provided for the jar. An optional filter function may also be provided to + * selectively include/exclude specific files. + * @param inputPaths - `Agg` of `os.Path`s containing files to be included in the jar + * @param fileFilter - optional file filter to select files to be included. + * Given a `os.Path` (from inputPaths) and a `os.RelPath` for the individual file, + * return true if the file is to be included in the jar. + * @param ctx - implicit `Ctx.Dest` used to determine the output directory for the jar. + * @return - a `PathRef` for the created jar. + */ + def createJar( + inputPaths: Agg[os.Path], + manifest: mill.api.JarManifest = mill.api.JarManifest.MillDefault, + fileFilter: (os.Path, os.RelPath) => Boolean = (_, _) => true + )(implicit ctx: Ctx.Dest): PathRef = { + val outputPath = ctx.dest / "out.jar" + createJar( + jar = outputPath, + inputPaths = inputPaths, + manifest = manifest, + fileFilter = fileFilter + ) + PathRef(outputPath) + } + + def createJar( + jar: os.Path, + inputPaths: Agg[os.Path], + manifest: mill.api.JarManifest, + fileFilter: (os.Path, os.RelPath) => Boolean + ): Unit = + JarOps.jar(jar, inputPaths, manifest, fileFilter, includeDirs = true, timestamp = None) + + def createClasspathPassingJar(jar: os.Path, classpath: Agg[os.Path]): Unit = { + createJar( + jar = jar, + inputPaths = Agg(), + manifest = mill.api.JarManifest.MillDefault.add( + "Class-Path" -> classpath.iterator.map(_.toNIO.toUri().toURL().toExternalForm()).mkString( + " " + ) + ), + fileFilter = (_, _) => true + ) + } + + + def universalScript( + shellCommands: String, + cmdCommands: String, + shebang: Boolean = false + ): String = { + Seq( + if (shebang) "#!/usr/bin/env sh" else "", + "@ 2>/dev/null # 2>nul & echo off & goto BOF\r", + ":", + shellCommands.replaceAll("\r\n|\n", "\n"), + "exit", + Seq( + "", + ":BOF", + "setlocal", + "@echo off", + cmdCommands.replaceAll("\r\n|\n", "\r\n"), + "endlocal", + "exit /B %errorlevel%", + "" + ).mkString("\r\n") + ).filterNot(_.isEmpty).mkString("\n") + } + + def launcherUniversalScript( + mainClass: String, + shellClassPath: Agg[String], + cmdClassPath: Agg[String], + jvmArgs: Seq[String], + shebang: Boolean = false + ): String = { + universalScript( + shellCommands = + s"""exec java ${jvmArgs.mkString(" ")} $$JAVA_OPTS -cp "${shellClassPath.iterator.mkString( + ":" + )}" '$mainClass' "$$@"""", + cmdCommands = + s"""java ${jvmArgs.mkString(" ")} %JAVA_OPTS% -cp "${cmdClassPath.iterator.mkString( + ";" + )}" $mainClass %*""", + shebang = shebang + ) + } + def createLauncher(mainClass: String, classPath: Agg[os.Path], jvmArgs: Seq[String])(implicit + ctx: Ctx.Dest + ): PathRef = { + val isWin = scala.util.Properties.isWin + val isBatch = + isWin && !(org.jline.utils.OSUtils.IS_CYGWIN || org.jline.utils.OSUtils.IS_MSYSTEM) + val outputPath = ctx.dest / (if (isBatch) "run.bat" else "run") + val classPathStrs = classPath.map(_.toString) + + os.write(outputPath, launcherUniversalScript(mainClass, classPathStrs, classPathStrs, jvmArgs)) + + if (!isWin) { + val perms = Files.getPosixFilePermissions(outputPath.toNIO) + perms.add(PosixFilePermission.GROUP_EXECUTE) + perms.add(PosixFilePermission.OWNER_EXECUTE) + perms.add(PosixFilePermission.OTHERS_EXECUTE) + Files.setPosixFilePermissions(outputPath.toNIO, perms) + } + PathRef(outputPath) + } + + @deprecated("Use mill.api.JarManifest instead", "Mill after 0.11.0-M4") + type JarManifest = mill.api.JarManifest + @deprecated("Use mill.api.JarManifest instead", "Mill after 0.11.0-M4") + val JarManifest = mill.api.JarManifest + } diff --git a/main/util/src/mill/util/Util.scala b/main/util/src/mill/util/Util.scala index 4900a1cba11..3e0f52f1631 100644 --- a/main/util/src/mill/util/Util.scala +++ b/main/util/src/mill/util/Util.scala @@ -1,6 +1,9 @@ package mill.util -import mill.api.SystemStreams + +import coursier.Repository +import mill.api.Loose.Agg +import mill.api.{BuildInfo, Ctx, IO, Loose, PathRef, Result} object Util { @@ -11,4 +14,90 @@ object Util { val windowsPlatform = System.getProperty("os.name").startsWith("Windows") val java9OrAbove = !System.getProperty("java.specification.version").startsWith("1.") + + + private val LongMillProps = new java.util.Properties() + + { + val millOptionsPath = sys.props("MILL_OPTIONS_PATH") + if (millOptionsPath != null) + LongMillProps.load(new java.io.FileInputStream(millOptionsPath)) + } + + def cleanupScaladoc(v: String) = { + v.linesIterator.map( + _.dropWhile(_.isWhitespace) + .stripPrefix("/**") + .stripPrefix("*/") + .stripPrefix("*") + .stripSuffix("**/") + .stripSuffix("*/") + .dropWhile(_.isWhitespace) + ).toArray + .dropWhile(_.isEmpty) + .reverse + .dropWhile(_.isEmpty) + .reverse + } + + def download(url: String, dest: os.RelPath = os.rel / "download")(implicit + ctx: Ctx.Dest + ): PathRef = { + val out = ctx.dest / dest + + val website = new java.net.URI(url).toURL + val rbc = java.nio.channels.Channels.newChannel(website.openStream) + try { + val fos = new java.io.FileOutputStream(out.toIO) + try { + fos.getChannel.transferFrom(rbc, 0, java.lang.Long.MAX_VALUE) + PathRef(out) + } finally { + fos.close() + } + } finally { + rbc.close() + } + } + + def downloadUnpackZip(url: String, dest: os.RelPath = os.rel / "unpacked")(implicit + ctx: Ctx.Dest + ) = { + + val tmpName = if (dest == os.rel / "tmp.zip") "tmp2.zip" else "tmp.zip" + val downloaded = download(url, os.rel / tmpName) + IO.unpackZip(downloaded.path, dest) + } + + /** + * Deprecated helper method, intended to allow runtime resolution and in-development-tree testings of mill plugins possible. + * This design has issues and will probably replaced. + */ + def millProjectModule( + artifact: String, + repositories: Seq[Repository], + resolveFilter: os.Path => Boolean = _ => true, + // this should correspond to the mill runtime Scala version + artifactSuffix: String = "_2.13" + ): Result[Agg[PathRef]] = { + + mill.util.Jvm.resolveDependencies( + repositories = repositories, + deps = Seq( + coursier.Dependency( + coursier.Module( + coursier.Organization("com.lihaoyi"), + coursier.ModuleName(artifact + artifactSuffix) + ), + BuildInfo.millVersion + ) + ), + force = Nil, + resolveFilter = resolveFilter + ) + } + + def millProperty(key: String): Option[String] = + Option(sys.props(key)) // System property has priority + .orElse(Option(LongMillProps.getProperty(key))) } diff --git a/main/test/src/mill/modules/JvmTests.scala b/main/util/test/src/mill/util/JvmTests.scala similarity index 97% rename from main/test/src/mill/modules/JvmTests.scala rename to main/util/test/src/mill/util/JvmTests.scala index 8711e1da2fc..fc31a0c2572 100644 --- a/main/test/src/mill/modules/JvmTests.scala +++ b/main/util/test/src/mill/util/JvmTests.scala @@ -1,10 +1,10 @@ -package mill.modules - -import java.util.jar.{Attributes, JarFile} +package mill.util import mill.Agg import utest.{TestSuite, Tests, test} +import java.util.jar.{Attributes, JarFile} + object JvmTests extends TestSuite { val tests = Tests { diff --git a/runner/src/mill/runner/MillBuildRootModule.scala b/runner/src/mill/runner/MillBuildRootModule.scala index 523ae9ac17e..f7e77ac923a 100644 --- a/runner/src/mill/runner/MillBuildRootModule.scala +++ b/runner/src/mill/runner/MillBuildRootModule.scala @@ -4,8 +4,8 @@ import coursier.Repository import mill._ import mill.api.{Loose, PathRef, Result, internal} import mill.define.{Caller, Discover, Target, Task} -import mill.modules.CoursierSupport -import mill.modules.Util.millProjectModule +import mill.util.CoursierSupport +import mill.util.Util.millProjectModule import mill.scalalib.{BoundDep, DepSyntax, Lib, ScalaModule} import mill.scalalib.api.Versions import os.{Path, rel} diff --git a/scalajslib/src/mill/scalajslib/ScalaJSModule.scala b/scalajslib/src/mill/scalajslib/ScalaJSModule.scala index 2c7cc94e478..7183dfce181 100644 --- a/scalajslib/src/mill/scalajslib/ScalaJSModule.scala +++ b/scalajslib/src/mill/scalajslib/ScalaJSModule.scala @@ -35,7 +35,7 @@ trait ScalaJSModule extends scalalib.ScalaModule { outer => def scalaJSWorkerVersion = T { ZincWorkerUtil.scalaJSWorkerVersion(scalaJSVersion()) } def scalaJSWorkerClasspath = T { - mill.modules.Util.millProjectModule( + mill.util.Util.millProjectModule( artifact = s"mill-scalajslib-worker-${scalaJSWorkerVersion()}", repositories = repositoriesTask(), resolveFilter = _.toString.contains("mill-scalajslib-worker") diff --git a/main/src/mill/modules/Assembly.scala b/scalalib/src/mill/scalalib/Assembly.scala similarity index 62% rename from main/src/mill/modules/Assembly.scala rename to scalalib/src/mill/scalalib/Assembly.scala index 34881a907db..40cfa916b53 100644 --- a/main/src/mill/modules/Assembly.scala +++ b/scalalib/src/mill/scalalib/Assembly.scala @@ -1,14 +1,20 @@ -package mill.modules +package mill.scalalib import com.eed3si9n.jarjarabrams.{ShadePattern, Shader} -import java.io.{ByteArrayInputStream, Closeable, InputStream} -import java.util.jar.JarFile -import java.util.regex.Pattern - import mill.Agg +import mill.api.{Ctx, IO, PathRef} import os.Generator + +import java.io.{ByteArrayInputStream, InputStream, SequenceInputStream} +import java.net.URI +import java.nio.file.attribute.PosixFilePermission +import java.nio.file.{FileSystems, Files, StandardOpenOption} +import java.util.Collections +import java.util.jar.JarFile +import java.util.regex.Pattern import scala.jdk.CollectionConverters._ import scala.tools.nsc.io.Streamable +import scala.util.Using object Assembly { @@ -164,19 +170,103 @@ object Assembly { type UnopenedInputStream = () => InputStream - private[modules] sealed trait GroupedEntry { + private[scalalib] sealed trait GroupedEntry { def append(entry: UnopenedInputStream): GroupedEntry } - private[modules] object AppendEntry { + private[scalalib] object AppendEntry { val empty: AppendEntry = AppendEntry(Nil, defaultSeparator) } - private[modules] case class AppendEntry(inputStreams: Seq[UnopenedInputStream], separator: String) + private[scalalib] case class AppendEntry(inputStreams: Seq[UnopenedInputStream], separator: String) extends GroupedEntry { def append(inputStream: UnopenedInputStream): GroupedEntry = copy(inputStreams = inputStreams :+ inputStream) } - private[modules] case class WriteOnceEntry(inputStream: UnopenedInputStream) + private[scalalib] case class WriteOnceEntry(inputStream: UnopenedInputStream) extends GroupedEntry { def append(entry: UnopenedInputStream): GroupedEntry = this } + + def createAssembly( + inputPaths: Agg[os.Path], + manifest: mill.api.JarManifest = mill.api.JarManifest.MillDefault, + prependShellScript: String = "", + base: Option[os.Path] = None, + assemblyRules: Seq[Assembly.Rule] = Assembly.defaultRules + )(implicit ctx: Ctx.Dest with Ctx.Log): PathRef = { + val tmp = ctx.dest / "out-tmp.jar" + + val baseUri = "jar:" + tmp.toIO.getCanonicalFile.toURI.toASCIIString + val hm = base.fold(Map("create" -> "true")) { b => + os.copy(b, tmp) + Map.empty + } + Using.resource(FileSystems.newFileSystem(URI.create(baseUri), hm.asJava)) { zipFs => + val manifestPath = zipFs.getPath(JarFile.MANIFEST_NAME) + Files.createDirectories(manifestPath.getParent) + val manifestOut = Files.newOutputStream( + manifestPath, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.CREATE + ) + manifest.build.write(manifestOut) + manifestOut.close() + + val (mappings, resourceCleaner) = Assembly.loadShadedClasspath(inputPaths, assemblyRules) + try { + Assembly.groupAssemblyEntries(mappings, assemblyRules).foreach { + case (mapping, entry) => + val path = zipFs.getPath(mapping).toAbsolutePath + entry match { + case entry: AppendEntry => + val separated = entry.inputStreams + .flatMap(inputStream => + Seq(new ByteArrayInputStream(entry.separator.getBytes), inputStream()) + ) + val cleaned = if (Files.exists(path)) separated else separated.drop(1) + val concatenated = + new SequenceInputStream(Collections.enumeration(cleaned.asJava)) + writeEntry(path, concatenated, append = true) + case entry: WriteOnceEntry => writeEntry(path, entry.inputStream(), append = false) + } + } + } finally { + resourceCleaner() + } + } + + val output = ctx.dest / "out.jar" + // Prepend shell script and make it executable + if (prependShellScript.isEmpty) os.move(tmp, output) + else { + val lineSep = if (!prependShellScript.endsWith("\n")) "\n\r\n" else "" + os.write(output, prependShellScript + lineSep) + os.write.append(output, os.read.inputStream(tmp)) + + if (!scala.util.Properties.isWin) { + os.perms.set( + output, + os.perms(output) + + PosixFilePermission.GROUP_EXECUTE + + PosixFilePermission.OWNER_EXECUTE + + PosixFilePermission.OTHERS_EXECUTE + ) + } + } + + PathRef(output) + } + + private def writeEntry(p: java.nio.file.Path, + inputStream: InputStream, + append: Boolean): Unit = { + if (p.getParent != null) Files.createDirectories(p.getParent) + val options = + if (append) Seq(StandardOpenOption.APPEND, StandardOpenOption.CREATE) + else Seq(StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE) + + val outputStream = java.nio.file.Files.newOutputStream(p, options: _*) + IO.stream(inputStream, outputStream) + outputStream.close() + inputStream.close() + } } diff --git a/scalalib/src/mill/scalalib/GenIdeaImpl.scala b/scalalib/src/mill/scalalib/GenIdeaImpl.scala index cec51d44746..2db8f208b8d 100755 --- a/scalalib/src/mill/scalalib/GenIdeaImpl.scala +++ b/scalalib/src/mill/scalalib/GenIdeaImpl.scala @@ -13,7 +13,7 @@ import mill.api.Ctx.{Home, Log} import mill.api.{PathRef, Result, Strict} import mill.define._ import mill.eval.Evaluator -import mill.modules.Util +import mill.util.Util import mill.scalalib.GenIdeaModule.{IdeaConfigFile, JavaFacet} import mill.util.Classpath import mill.{BuildInfo, T, scalalib} diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 873c59ea529..128c47c649d 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -11,7 +11,8 @@ import coursier.util.ModuleMatcher import mainargs.Flag import mill.api.Loose.Agg import mill.api.{JarManifest, PathRef, Result, internal} -import mill.modules.{Assembly, Jvm} +import mill.util.Jvm +import mill.scalalib.Assembly import mill.scalalib.api.CompilationResult import mill.scalalib.bsp.{BspBuildTarget, BspModule} import mill.scalalib.publish.Artifact @@ -251,7 +252,7 @@ trait JavaModule finalMainClassOpt().toOption match { case None => "" case Some(cls) => - mill.modules.Jvm.launcherUniversalScript( + mill.util.Jvm.launcherUniversalScript( mainClass = cls, shellClassPath = Agg("$0"), cmdClassPath = Agg("%~dpnx0"), @@ -438,7 +439,7 @@ trait JavaModule * upstream dependencies do not change */ def upstreamAssembly: T[PathRef] = T { - Jvm.createAssembly( + Assembly.createAssembly( upstreamAssemblyClasspath().map(_.path), manifest(), assemblyRules = assemblyRules @@ -450,7 +451,7 @@ trait JavaModule * classfiles from this module and all it's upstream modules and dependencies */ def assembly: T[PathRef] = T { - Jvm.createAssembly( + Assembly.createAssembly( Agg.from(localClasspath().map(_.path)), manifest(), prependShellScript(), diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala index 5fc823e345c..b1f9c690920 100644 --- a/scalalib/src/mill/scalalib/Lib.scala +++ b/scalalib/src/mill/scalalib/Lib.scala @@ -33,7 +33,7 @@ object Lib { ] = None ): (Seq[Dependency], Resolution) = { val depSeq = deps.iterator.toSeq - mill.modules.Jvm.resolveDependenciesMetadata( + mill.util.Jvm.resolveDependenciesMetadata( repositories = repositories, deps = depSeq.map(_.dep), force = depSeq.filter(_.force).map(_.dep), @@ -63,7 +63,7 @@ object Lib { ] = None ): Result[Agg[PathRef]] = { val depSeq = deps.iterator.toSeq - mill.modules.Jvm.resolveDependencies( + mill.util.Jvm.resolveDependencies( repositories = repositories, deps = depSeq.map(_.dep), force = depSeq.filter(_.force).map(_.dep), diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index 3cd5b546161..6eea39d1549 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -4,7 +4,7 @@ package scalalib import mill.define.{Command, ExternalModule, Target, Task} import mill.api.{JarManifest, PathRef, Result} import mill.main.Tasks -import mill.modules.Jvm +import mill.util.Jvm import mill.scalalib.PublishModule.checkSonatypeCreds import mill.scalalib.publish.{Artifact, SonatypePublisher, VersionScheme} diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala index 40c7da2d7aa..15a193df530 100644 --- a/scalalib/src/mill/scalalib/ScalaModule.scala +++ b/scalalib/src/mill/scalalib/ScalaModule.scala @@ -3,8 +3,8 @@ package scalalib import scala.annotation.nowarn import mill.api.{DummyInputStream, JarManifest, PathRef, Result, internal} -import mill.modules.Jvm -import mill.modules.Jvm.createJar +import mill.util.Jvm +import mill.util.Jvm.createJar import mill.api.Loose.Agg import mill.scalalib.api.{CompilationResult, ZincWorkerUtil, Versions} diff --git a/scalalib/src/mill/scalalib/TestModule.scala b/scalalib/src/mill/scalalib/TestModule.scala index 0d6b9a1c81c..95f012c5fc7 100644 --- a/scalalib/src/mill/scalalib/TestModule.scala +++ b/scalalib/src/mill/scalalib/TestModule.scala @@ -3,7 +3,7 @@ package mill.scalalib import mill.{Agg, T} import mill.define.{Command, Task, TaskModule} import mill.api.{Ctx, Result} -import mill.modules.Jvm +import mill.util.Jvm import mill.scalalib.bsp.{BspBuildTarget, BspModule} import mill.testrunner.TestRunner diff --git a/scalalib/src/mill/scalalib/ZincWorkerModule.scala b/scalalib/src/mill/scalalib/ZincWorkerModule.scala index 562c4dca881..f0526e7db5c 100644 --- a/scalalib/src/mill/scalalib/ZincWorkerModule.scala +++ b/scalalib/src/mill/scalalib/ZincWorkerModule.scala @@ -10,7 +10,7 @@ import mill.define.{ExternalModule, Discover} import mill.scalalib.Lib.resolveDependencies import mill.scalalib.api.ZincWorkerUtil.{isBinaryBridgeAvailable, isDotty, isDottyOrScala3} import mill.scalalib.api.{ZincWorkerApi, ZincWorkerUtil, Versions} -import mill.modules.Util.millProjectModule +import mill.util.Util.millProjectModule /** * A default implementation of [[ZincWorkerModule]] diff --git a/scalalib/src/mill/scalalib/giter8/Giter8Module.scala b/scalalib/src/mill/scalalib/giter8/Giter8Module.scala index 3ecf8213340..b8f38aae0a8 100644 --- a/scalalib/src/mill/scalalib/giter8/Giter8Module.scala +++ b/scalalib/src/mill/scalalib/giter8/Giter8Module.scala @@ -2,7 +2,7 @@ package mill.scalalib.giter8 import mill.T import mill.define.{Command, Discover, ExternalModule} -import mill.modules.Jvm +import mill.util.Jvm import mill.scalalib.api.ZincWorkerUtil import mill.scalalib._ import mill.BuildInfo diff --git a/scalalib/src/mill/scalalib/publish/SonatypePublisher.scala b/scalalib/src/mill/scalalib/publish/SonatypePublisher.scala index c646f348357..ce1c31d97db 100644 --- a/scalalib/src/mill/scalalib/publish/SonatypePublisher.scala +++ b/scalalib/src/mill/scalalib/publish/SonatypePublisher.scala @@ -4,7 +4,7 @@ import java.math.BigInteger import java.security.MessageDigest import mill.api.Logger -import mill.modules.Jvm +import mill.util.Jvm import os.Shellable class SonatypePublisher( diff --git a/scalalib/test/src/mill/scalalib/HelloWorldTests.scala b/scalalib/test/src/mill/scalalib/HelloWorldTests.scala index 95b5f07c7ca..d56fa974f60 100644 --- a/scalalib/test/src/mill/scalalib/HelloWorldTests.scala +++ b/scalalib/test/src/mill/scalalib/HelloWorldTests.scala @@ -9,7 +9,7 @@ import mill._ import mill.api.Result import mill.define.NamedTask import mill.eval.{Evaluator, EvaluatorPaths} -import mill.modules.Assembly +import mill.scalalib.Assembly import mill.scalalib.api.ZincWorkerUtil import mill.scalalib.publish.{VersionControl, _} import mill.util.{TestEvaluator, TestUtil} diff --git a/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala b/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala index a3a60fbbba8..44804887c68 100644 --- a/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala +++ b/scalanativelib/src/mill/scalanativelib/ScalaNativeModule.scala @@ -5,8 +5,8 @@ import mainargs.Flag import mill.api.Loose.Agg import mill.api.{Result, internal} import mill.define.{Command, Target, Task} -import mill.modules.Jvm -import mill.modules.Util.millProjectModule +import mill.util.Jvm +import mill.util.Util.millProjectModule import mill.scalalib.api.ZincWorkerUtil import mill.scalalib.bsp.{ScalaBuildTarget, ScalaPlatform} import mill.scalalib.{ From 89723468818fc7e0ffe727533559d32c186fcac2 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 10 May 2023 08:48:15 +0800 Subject: [PATCH 2/7] update bootstrap --- ci/mill-bootstrap.patch | 69 +++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/ci/mill-bootstrap.patch b/ci/mill-bootstrap.patch index a3bf3b7587b..8bc15d9e842 100644 --- a/ci/mill-bootstrap.patch +++ b/ci/mill-bootstrap.patch @@ -1,5 +1,5 @@ diff --git a/build.sc b/build.sc -index 5d071823cd..be020eb125 100644 +index 0311fdf8b5..530e09941b 100644 --- a/build.sc +++ b/build.sc @@ -19,17 +19,18 @@ import com.github.lolgab.mill.mima.{ @@ -11,9 +11,10 @@ index 5d071823cd..be020eb125 100644 import mill.main.MainModule import mill.scalalib._ import mill.scalalib.publish._ - import mill.modules.Jvm +-import mill.modules.Jvm -import mill.define.SelectMode - ++import mill.util.Jvm +import mill.resolve.SelectMode +import $ivy.`com.lihaoyi::mill-contrib-buildinfo:` +import mill.contrib.buildinfo.BuildInfo @@ -25,7 +26,7 @@ index 5d071823cd..be020eb125 100644 import mill.api.{Logger, Result} import os.{CommandResult, SubprocessException} -@@ -164,12 +165,8 @@ object Deps { +@@ -165,12 +166,8 @@ object Deps { val requests = ivy"com.lihaoyi::requests:0.8.0" } @@ -40,7 +41,7 @@ index 5d071823cd..be020eb125 100644 def millBinPlatform: T[String] = T { val tag = millLastTag() if (tag.contains("-M")) tag -@@ -218,8 +215,8 @@ val buildBridgeScalaVersions = +@@ -219,8 +216,8 @@ val buildBridgeScalaVersions = if (!buildAllCompilerBridges) Seq() else bridgeScalaVersions @@ -51,7 +52,7 @@ index 5d071823cd..be020eb125 100644 def scalaVersion = crossScalaVersion def publishVersion = bridgeVersion def artifactName = T { "mill-scala-compiler-bridge" } -@@ -238,239 +235,25 @@ class BridgeModule(val crossScalaVersion: String) extends PublishModule with Cro +@@ -239,239 +236,25 @@ class BridgeModule(val crossScalaVersion: String) extends PublishModule with Cro def generatedSources = T { import mill.scalalib.api.ZincWorkerUtil.{grepJar, scalaBinaryVersion} val resolvedJars = resolveDeps( @@ -296,7 +297,7 @@ index 5d071823cd..be020eb125 100644 def commonPomSettings(artifactName: String) = { PomSettings( description = artifactName, -@@ -524,27 +307,8 @@ trait MillCoursierModule extends CoursierModule { +@@ -525,27 +308,8 @@ trait MillCoursierModule extends CoursierModule { ) } @@ -325,7 +326,7 @@ index 5d071823cd..be020eb125 100644 } /** A Module compiled with applied Mill-specific compiler plugins: mill-moduledefs. */ -@@ -855,7 +619,9 @@ object scalajslib extends MillModule with BuildInfo { +@@ -858,7 +622,9 @@ object scalajslib extends MillModule with BuildInfo { override def ivyDeps = Agg(Deps.sbtTestInterface) } object worker extends Cross[WorkerModule]("1") @@ -336,7 +337,7 @@ index 5d071823cd..be020eb125 100644 def testDepPaths = T { Seq(compile().classes) } override def moduleDeps = Seq(scalajslib.`worker-api`, main.client, main.api) override def ivyDeps = Agg( -@@ -919,8 +685,10 @@ object contrib extends MillModule { +@@ -922,8 +688,10 @@ object contrib extends MillModule { object api extends MillPublishModule @@ -349,7 +350,7 @@ index 5d071823cd..be020eb125 100644 override def sources = T.sources { // We want to avoid duplicating code as long as the Play APIs allow. // But if newer Play versions introduce incompatibilities, -@@ -1083,8 +851,10 @@ object scalanativelib extends MillModule { +@@ -1086,8 +854,10 @@ object scalanativelib extends MillModule { override def ivyDeps = Agg(Deps.sbtTestInterface) } object worker extends Cross[WorkerModule]("0.4") @@ -362,7 +363,7 @@ index 5d071823cd..be020eb125 100644 def testDepPaths = T { Seq(compile().classes) } override def moduleDeps = Seq(scalanativelib.`worker-api`) override def ivyDeps = scalaNativeWorkerVersion match { -@@ -1203,7 +973,10 @@ trait IntegrationTestModule extends MillScalaModule { +@@ -1206,7 +976,10 @@ trait IntegrationTestModule extends MillScalaModule { } } @@ -374,7 +375,7 @@ index 5d071823cd..be020eb125 100644 object local extends ModeModule object fork extends ModeModule object server extends ModeModule -@@ -1217,15 +990,15 @@ object example extends MillScalaModule { +@@ -1220,15 +993,15 @@ object example extends MillScalaModule { def moduleDeps = Seq(integration) @@ -398,7 +399,7 @@ index 5d071823cd..be020eb125 100644 def sources = T.sources() def testRepoRoot: T[PathRef] = T.source(millSourcePath) def compile = example.compile() -@@ -1275,7 +1048,7 @@ object example extends MillScalaModule { +@@ -1278,7 +1051,7 @@ object example extends MillScalaModule { val title = if (seenCode) "" else { @@ -407,7 +408,7 @@ index 5d071823cd..be020eb125 100644 val exampleDashed = examplePath.segments.mkString("-") val download = s"{mill-download-url}/$label-$exampleDashed.zip[download]" val browse = s"{mill-example-url}/$examplePath[browse]" -@@ -1306,9 +1079,9 @@ object example extends MillScalaModule { +@@ -1309,9 +1082,9 @@ object example extends MillScalaModule { } object integration extends MillScalaModule { @@ -420,8 +421,30 @@ index 5d071823cd..be020eb125 100644 def moduleDeps = Seq(scalalib, scalajslib, scalanativelib, runner.test) -@@ -1636,8 +1409,8 @@ object dev extends MillModule { - mill.modules.Jvm.createJar(Agg(), mill.modules.Jvm.JarManifest(manifestEntries)) +@@ -1390,7 +1163,7 @@ def launcherScript( + val millMainClass = "mill.main.client.MillClientMain" + val millClientMainClass = "mill.main.client.MillClientMain" + +- mill.modules.Jvm.universalScript( ++ Jvm.universalScript( + shellCommands = { + val jvmArgsStr = shellJvmArgs.mkString(" ") + def java(mainClass: String, passMillJvmOpts: Boolean) = { +@@ -1565,7 +1338,7 @@ object dev extends MillModule { + } + + override def assemblyRules = super.assemblyRules ++ Seq( +- mill.modules.Assembly.Rule.ExcludePattern("mill/local-test-overrides/.*") ++ mill.scalalib.Assembly.Rule.ExcludePattern("mill/local-test-overrides/.*") + ) + + lazy val allPublishModules = build.millInternal.modules.collect { +@@ -1636,16 +1409,16 @@ object dev extends MillModule { + "Created-By" -> "Scala mill", + "Class-Path" -> classpath + ) +- mill.modules.Jvm.createJar(Agg(), mill.modules.Jvm.JarManifest(manifestEntries)) ++ Jvm.createJar(Agg(), JarManifest(manifestEntries)) } - def run(args: String*) = T.command { @@ -431,7 +454,13 @@ index 5d071823cd..be020eb125 100644 case Nil => mill.api.Result.Failure("Need to pass in cwd as first argument to dev.run") case wd0 +: rest => val wd = os.Path(wd0, T.workspace) -@@ -1664,67 +1437,11 @@ object docs extends Module { + os.makeDir.all(wd) +- try mill.modules.Jvm.runSubprocess( ++ try Jvm.runSubprocess( + Seq(launcher().path.toString) ++ rest, + forkEnv(), + workingDir = wd +@@ -1667,67 +1440,11 @@ object docs extends Module { def moduleDeps = build.millInternal.modules.collect { case m: MillApiModule => m } def unidocSourceUrl = T { @@ -500,7 +529,7 @@ index 5d071823cd..be020eb125 100644 private val npmExe = if (scala.util.Properties.isWin) "npm.cmd" else "npm" private val antoraExe = if (scala.util.Properties.isWin) "antora.cmd" else "antora" def npmBase: T[os.Path] = T.persistent { T.dest } -@@ -1777,7 +1494,7 @@ object docs extends Module { +@@ -1780,7 +1497,7 @@ object docs extends Module { val contribReadmes = T.traverse(contrib.contribModules)(m => T.task { @@ -509,7 +538,7 @@ index 5d071823cd..be020eb125 100644 } )() -@@ -2009,7 +1726,7 @@ def exampleZips: Target[Seq[PathRef]] = T { +@@ -2012,7 +1729,7 @@ def exampleZips: Target[Seq[PathRef]] = T { examplePath = exampleMod.millSourcePath } yield { val example = examplePath.subRelativeTo(T.workspace) @@ -518,7 +547,7 @@ index 5d071823cd..be020eb125 100644 os.copy(examplePath, T.dest / exampleStr, createFolders = true) os.copy(bootstrapLauncher().path, T.dest / exampleStr / "mill") val zip = T.dest / s"$exampleStr.zip" -@@ -2019,51 +1736,10 @@ def exampleZips: Target[Seq[PathRef]] = T { +@@ -2022,51 +1739,10 @@ def exampleZips: Target[Seq[PathRef]] = T { } def uploadToGithub(authKey: String) = T.command { @@ -571,7 +600,7 @@ index 5d071823cd..be020eb125 100644 ev.withFailFast(false), Seq( "__.compile", -@@ -2076,7 +1752,8 @@ def validate(ev: Evaluator): Command[Unit] = T.command { +@@ -2079,7 +1755,8 @@ def validate(ev: Evaluator): Command[Unit] = T.command { "docs.localPages" ), selectMode = SelectMode.Separated From 43f86cf5e0565e546ab3d31f16e278e93e5f3843 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 10 May 2023 09:08:19 +0800 Subject: [PATCH 3/7] . --- ci/mill-bootstrap.patch | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ci/mill-bootstrap.patch b/ci/mill-bootstrap.patch index 8bc15d9e842..d906ff650dd 100644 --- a/ci/mill-bootstrap.patch +++ b/ci/mill-bootstrap.patch @@ -1,5 +1,5 @@ diff --git a/build.sc b/build.sc -index 0311fdf8b5..530e09941b 100644 +index 0311fdf8b5..eefefd4ded 100644 --- a/build.sc +++ b/build.sc @@ -19,17 +19,18 @@ import com.github.lolgab.mill.mima.{ @@ -439,12 +439,21 @@ index 0311fdf8b5..530e09941b 100644 ) lazy val allPublishModules = build.millInternal.modules.collect { +@@ -1585,7 +1358,7 @@ object dev extends MillModule { + val shellArgs = Seq("-DMILL_CLASSPATH=$0") ++ commonArgs + val cmdArgs = Seq(""""-DMILL_CLASSPATH=%~dpnx0"""") ++ commonArgs + os.move( +- Jvm.createAssembly( ++ mill.scalalib.Assembly.createAssembly( + devRunClasspath, + prependShellScript = launcherScript( + shellArgs, @@ -1636,16 +1409,16 @@ object dev extends MillModule { "Created-By" -> "Scala mill", "Class-Path" -> classpath ) - mill.modules.Jvm.createJar(Agg(), mill.modules.Jvm.JarManifest(manifestEntries)) -+ Jvm.createJar(Agg(), JarManifest(manifestEntries)) ++ Jvm.createJar(Agg(), Jvm.JarManifest(manifestEntries)) } - def run(args: String*) = T.command { From 563d58f6e1f5b3f3260cf5c1994cc5779381fd25 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 10 May 2023 09:12:34 +0800 Subject: [PATCH 4/7] . --- main/util/src/mill/util/CoursierSupport.scala | 1 - main/util/src/mill/util/Jvm.scala | 14 +++++------- main/util/src/mill/util/Util.scala | 18 +++++++-------- scalalib/src/mill/scalalib/Assembly.scala | 22 +++++++++---------- 4 files changed, 25 insertions(+), 30 deletions(-) diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index 6a4328810dc..1630facf6b5 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -1,6 +1,5 @@ package mill.util - import coursier.cache.ArtifactError import coursier.parse.RepositoryParser import coursier.util.{Gather, Task} diff --git a/main/util/src/mill/util/Jvm.scala b/main/util/src/mill/util/Jvm.scala index a80baada025..2e7dc5be46b 100644 --- a/main/util/src/mill/util/Jvm.scala +++ b/main/util/src/mill/util/Jvm.scala @@ -1,6 +1,5 @@ package mill.util - import mill.api.BuildInfo import mill.api.Loose.Agg import mill.api._ @@ -229,12 +228,12 @@ object Jvm extends CoursierSupport { } def inprocess[T]( - classPath: Agg[os.Path], - classLoaderOverrideSbtTesting: Boolean, - isolated: Boolean, - closeContextClassLoaderWhenDone: Boolean, - body: ClassLoader => T - )(implicit ctx: mill.api.Ctx.Home): T = { + classPath: Agg[os.Path], + classLoaderOverrideSbtTesting: Boolean, + isolated: Boolean, + closeContextClassLoaderWhenDone: Boolean, + body: ClassLoader => T + )(implicit ctx: mill.api.Ctx.Home): T = { val urls = classPath.map(_.toIO.toURI.toURL) val cl = if (classLoaderOverrideSbtTesting) { @@ -315,7 +314,6 @@ object Jvm extends CoursierSupport { ) } - def universalScript( shellCommands: String, cmdCommands: String, diff --git a/main/util/src/mill/util/Util.scala b/main/util/src/mill/util/Util.scala index 3e0f52f1631..17c564ed3f5 100644 --- a/main/util/src/mill/util/Util.scala +++ b/main/util/src/mill/util/Util.scala @@ -1,6 +1,5 @@ package mill.util - import coursier.Repository import mill.api.Loose.Agg import mill.api.{BuildInfo, Ctx, IO, Loose, PathRef, Result} @@ -15,7 +14,6 @@ object Util { val java9OrAbove = !System.getProperty("java.specification.version").startsWith("1.") - private val LongMillProps = new java.util.Properties() { @@ -41,7 +39,7 @@ object Util { } def download(url: String, dest: os.RelPath = os.rel / "download")(implicit - ctx: Ctx.Dest + ctx: Ctx.Dest ): PathRef = { val out = ctx.dest / dest @@ -61,7 +59,7 @@ object Util { } def downloadUnpackZip(url: String, dest: os.RelPath = os.rel / "unpacked")(implicit - ctx: Ctx.Dest + ctx: Ctx.Dest ) = { val tmpName = if (dest == os.rel / "tmp.zip") "tmp2.zip" else "tmp.zip" @@ -74,12 +72,12 @@ object Util { * This design has issues and will probably replaced. */ def millProjectModule( - artifact: String, - repositories: Seq[Repository], - resolveFilter: os.Path => Boolean = _ => true, - // this should correspond to the mill runtime Scala version - artifactSuffix: String = "_2.13" - ): Result[Agg[PathRef]] = { + artifact: String, + repositories: Seq[Repository], + resolveFilter: os.Path => Boolean = _ => true, + // this should correspond to the mill runtime Scala version + artifactSuffix: String = "_2.13" + ): Result[Agg[PathRef]] = { mill.util.Jvm.resolveDependencies( repositories = repositories, diff --git a/scalalib/src/mill/scalalib/Assembly.scala b/scalalib/src/mill/scalalib/Assembly.scala index 40cfa916b53..78baebdfa99 100644 --- a/scalalib/src/mill/scalalib/Assembly.scala +++ b/scalalib/src/mill/scalalib/Assembly.scala @@ -176,8 +176,10 @@ object Assembly { private[scalalib] object AppendEntry { val empty: AppendEntry = AppendEntry(Nil, defaultSeparator) } - private[scalalib] case class AppendEntry(inputStreams: Seq[UnopenedInputStream], separator: String) - extends GroupedEntry { + private[scalalib] case class AppendEntry( + inputStreams: Seq[UnopenedInputStream], + separator: String + ) extends GroupedEntry { def append(inputStream: UnopenedInputStream): GroupedEntry = copy(inputStreams = inputStreams :+ inputStream) } @@ -187,12 +189,12 @@ object Assembly { } def createAssembly( - inputPaths: Agg[os.Path], - manifest: mill.api.JarManifest = mill.api.JarManifest.MillDefault, - prependShellScript: String = "", - base: Option[os.Path] = None, - assemblyRules: Seq[Assembly.Rule] = Assembly.defaultRules - )(implicit ctx: Ctx.Dest with Ctx.Log): PathRef = { + inputPaths: Agg[os.Path], + manifest: mill.api.JarManifest = mill.api.JarManifest.MillDefault, + prependShellScript: String = "", + base: Option[os.Path] = None, + assemblyRules: Seq[Assembly.Rule] = Assembly.defaultRules + )(implicit ctx: Ctx.Dest with Ctx.Log): PathRef = { val tmp = ctx.dest / "out-tmp.jar" val baseUri = "jar:" + tmp.toIO.getCanonicalFile.toURI.toASCIIString @@ -256,9 +258,7 @@ object Assembly { PathRef(output) } - private def writeEntry(p: java.nio.file.Path, - inputStream: InputStream, - append: Boolean): Unit = { + private def writeEntry(p: java.nio.file.Path, inputStream: InputStream, append: Boolean): Unit = { if (p.getParent != null) Files.createDirectories(p.getParent) val options = if (append) Seq(StandardOpenOption.APPEND, StandardOpenOption.CREATE) From 0838a9b6e01f567c25fb9ca7899879a6ea8542bc Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 10 May 2023 20:48:10 -0700 Subject: [PATCH 5/7] . --- scalalib/src/mill/main/package.scala | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 scalalib/src/mill/main/package.scala diff --git a/scalalib/src/mill/main/package.scala b/scalalib/src/mill/main/package.scala new file mode 100644 index 00000000000..6fc2e8c683d --- /dev/null +++ b/scalalib/src/mill/main/package.scala @@ -0,0 +1,15 @@ +package mill + +package object main { + @deprecated("use mill.util.Jvm") + val Jvm = mill.util.Jvm + + @deprecated("use mill.util.Util") + val Util = mill.util.Util + + @deprecated("use mill.util.CoursierSupport") + val CoursierSupport = mill.util.CoursierSupport + + @deprecated("use mill.scalalib.Assembly") + val Assembly = mill.scalalib.Assembly +} From 6ae44a96984b2a42223aace13829b9e296bd3eac Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 10 May 2023 20:58:47 -0700 Subject: [PATCH 6/7] . --- scalalib/src/mill/{main => modules}/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename scalalib/src/mill/{main => modules}/package.scala (92%) diff --git a/scalalib/src/mill/main/package.scala b/scalalib/src/mill/modules/package.scala similarity index 92% rename from scalalib/src/mill/main/package.scala rename to scalalib/src/mill/modules/package.scala index 6fc2e8c683d..525edd7aa83 100644 --- a/scalalib/src/mill/main/package.scala +++ b/scalalib/src/mill/modules/package.scala @@ -1,6 +1,6 @@ package mill -package object main { +package object modules { @deprecated("use mill.util.Jvm") val Jvm = mill.util.Jvm From 14e9cbc5ec917b4d10c27f3a1bb8c722898da46d Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Thu, 11 May 2023 04:03:23 -0700 Subject: [PATCH 7/7] Update package.scala --- scalalib/src/mill/modules/package.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scalalib/src/mill/modules/package.scala b/scalalib/src/mill/modules/package.scala index 525edd7aa83..61b7c6c1d37 100644 --- a/scalalib/src/mill/modules/package.scala +++ b/scalalib/src/mill/modules/package.scala @@ -1,15 +1,15 @@ package mill package object modules { - @deprecated("use mill.util.Jvm") + @deprecated("use mill.util.Jvm", "Mill 0.11.0-M9") val Jvm = mill.util.Jvm - @deprecated("use mill.util.Util") + @deprecated("use mill.util.Util", "Mill 0.11.0-M9") val Util = mill.util.Util - @deprecated("use mill.util.CoursierSupport") + @deprecated("use mill.util.CoursierSupport", "Mill 0.11.0-M9") val CoursierSupport = mill.util.CoursierSupport - @deprecated("use mill.scalalib.Assembly") + @deprecated("use mill.scalalib.Assembly", "Mill 0.11.0-M9") val Assembly = mill.scalalib.Assembly }