From 1fe235c7b559c6a80f3ad5eda4a431417258f9ab Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Wed, 18 May 2022 16:00:22 +0200 Subject: [PATCH] add MainGenericCompiler add explicit tests of scalac flags address review comments --- .../src/dotty/tools/MainGenericCompiler.scala | 188 ++++++++++++++++++ .../src/dotty/tools/MainGenericRunner.scala | 31 +-- dist/bin/scala | 4 +- dist/bin/scalac | 84 ++++---- project/scripts/bootstrappedOnlyCmdTests | 14 ++ 5 files changed, 267 insertions(+), 54 deletions(-) create mode 100644 compiler/src/dotty/tools/MainGenericCompiler.scala diff --git a/compiler/src/dotty/tools/MainGenericCompiler.scala b/compiler/src/dotty/tools/MainGenericCompiler.scala new file mode 100644 index 000000000000..aa924a237f73 --- /dev/null +++ b/compiler/src/dotty/tools/MainGenericCompiler.scala @@ -0,0 +1,188 @@ +package dotty.tools + +import scala.language.unsafeNulls + +import scala.annotation.tailrec +import scala.io.Source +import scala.util.Try +import java.io.File +import java.lang.Thread +import scala.annotation.internal.sharable +import dotty.tools.dotc.util.ClasspathFromClassloader +import dotty.tools.runner.ObjectRunner +import dotty.tools.dotc.config.Properties.envOrNone +import dotty.tools.io.Jar +import dotty.tools.runner.ScalaClassLoader +import java.nio.file.Paths +import dotty.tools.dotc.config.CommandLineParser +import dotty.tools.scripting.StringDriver + +enum CompileMode: + case Guess + case Compile + case Decompile + case PrintTasty + case Script + case Repl + case Run + +case class CompileSettings( + verbose: Boolean = false, + classPath: List[String] = List.empty, + compileMode: CompileMode = CompileMode.Guess, + exitCode: Int = 0, + javaArgs: List[String] = List.empty, + javaProps: List[(String, String)] = List.empty, + scalaArgs: List[String] = List.empty, + residualArgs: List[String] = List.empty, + scriptArgs: List[String] = List.empty, + targetScript: String = "", + compiler: Boolean = false, + quiet: Boolean = false, + colors: Boolean = false, +) { + def withCompileMode(em: CompileMode): CompileSettings = this.compileMode match + case CompileMode.Guess => + this.copy(compileMode = em) + case _ => + println(s"compile_mode==[$compileMode], attempted overwrite by [$em]") + this.copy(exitCode = 1) + end withCompileMode + + def withScalaArgs(args: String*): CompileSettings = + this.copy(scalaArgs = scalaArgs.appendedAll(args.toList.filter(_.nonEmpty))) + + def withJavaArgs(args: String*): CompileSettings = + this.copy(javaArgs = javaArgs.appendedAll(args.toList.filter(_.nonEmpty))) + + def withJavaProps(args: (String, String)*): CompileSettings = + this.copy(javaProps = javaProps.appendedAll(args.toList)) + + def withResidualArgs(args: String*): CompileSettings = + this.copy(residualArgs = residualArgs.appendedAll(args.toList.filter(_.nonEmpty))) + + def withScriptArgs(args: String*): CompileSettings = + this.copy(scriptArgs = scriptArgs.appendedAll(args.toList.filter(_.nonEmpty))) + + def withTargetScript(file: String): CompileSettings = + Try(Source.fromFile(file)).toOption match + case Some(_) => this.copy(targetScript = file) + case None => + println(s"not found $file") + this.copy(exitCode = 2) + end withTargetScript + + def withCompiler: CompileSettings = + this.copy(compiler = true) + + def withQuiet: CompileSettings = + this.copy(quiet = true) + + def withColors: CompileSettings = + this.copy(colors = true) + + def withNoColors: CompileSettings = + this.copy(colors = false) +} + +object MainGenericCompiler { + + val classpathSeparator = File.pathSeparator + + @sharable val javaOption = raw"""-J(.*)""".r + @sharable val javaPropOption = raw"""-D(.+?)=(.?)""".r + @tailrec + def process(args: List[String], settings: CompileSettings): CompileSettings = args match + case Nil => + settings + case "--" :: tail => + process(Nil, settings.withResidualArgs(tail.toList*)) + case ("-v" | "-verbose" | "--verbose") :: tail => + process(tail, settings.withScalaArgs("-verbose")) + case ("-q" | "-quiet") :: tail => + process(tail, settings.withQuiet) + case "-repl" :: tail => + process(tail, settings.withCompileMode(CompileMode.Repl)) + case "-script" :: targetScript :: tail => + process(Nil, settings + .withCompileMode(CompileMode.Script) + .withJavaProps("script.path" -> targetScript) + .withTargetScript(targetScript) + .withScriptArgs(tail*)) + case "-compile" :: tail => + process(tail, settings.withCompileMode(CompileMode.Compile)) + case "-decompile" :: tail => + process(tail, settings.withCompileMode(CompileMode.Decompile)) + case "-print-tasty" :: tail => + process(tail, settings.withCompileMode(CompileMode.PrintTasty)) + case "-run" :: tail => + process(tail, settings.withCompileMode(CompileMode.Run)) + case "-colors" :: tail => + process(tail, settings.withColors) + case "-no-colors" :: tail => + process(tail, settings.withNoColors) + case "-with-compiler" :: tail => + process(tail, settings.withCompiler) + case ("-cp" | "-classpath" | "--class-path") :: cp :: tail => + val (tailargs, newEntries) = MainGenericRunner.processClasspath(cp, tail) + process(tailargs, settings.copy(classPath = settings.classPath ++ newEntries.filter(_.nonEmpty))) + case "-Oshort" :: tail => + // Nothing is to be done here. Request that the user adds the relevant flags manually. + // i.e this has no effect when MainGenericRunner is invoked programatically. + val addTC="-XX:+TieredCompilation" + val tStopAtLvl="-XX:TieredStopAtLevel=1" + println(s"ignoring deprecated -Oshort flag, please add `-J$addTC` and `-J$tStopAtLvl` flags manually") + process(tail, settings) + case javaOption(stripped) :: tail => + process(tail, settings.withJavaArgs(stripped)) + case javaPropOption(opt, value) :: tail => + process(tail, settings.withJavaProps(opt -> value)) + case arg :: tail => + process(tail, settings.withResidualArgs(arg)) + end process + + def main(args: Array[String]): Unit = + val settings = process(args.toList, CompileSettings()) + if settings.exitCode != 0 then System.exit(settings.exitCode) + + def classpathSetting = + if settings.classPath.isEmpty then List() + else List("-classpath", settings.classPath.mkString(classpathSeparator)) + + def reconstructedArgs() = + classpathSetting ++ settings.scalaArgs ++ settings.residualArgs + + def addJavaProps(): Unit = + settings.javaProps.foreach { (k, v) => sys.props(k) = v } + + def run(settings: CompileSettings): Unit = settings.compileMode match + case CompileMode.Compile => + addJavaProps() + val properArgs = reconstructedArgs() + dotty.tools.dotc.Main.main(properArgs.toArray) + case CompileMode.Decompile => + addJavaProps() + val properArgs = reconstructedArgs() + dotty.tools.dotc.decompiler.Main.main(properArgs.toArray) + case CompileMode.PrintTasty => + addJavaProps() + val properArgs = reconstructedArgs() + dotty.tools.dotc.core.tasty.TastyPrinter.main(properArgs.toArray) + case CompileMode.Script => // Naive copy from scalac bash script + addJavaProps() + val properArgs = + reconstructedArgs() + ++ List("-script", settings.targetScript) + ++ settings.scriptArgs + scripting.Main.main(properArgs.toArray) + case CompileMode.Repl | CompileMode.Run => + addJavaProps() + val properArgs = reconstructedArgs() + repl.Main.main(properArgs.toArray) + case CompileMode.Guess => + run(settings.withCompileMode(CompileMode.Compile)) + end run + + run(settings) + end main +} diff --git a/compiler/src/dotty/tools/MainGenericRunner.scala b/compiler/src/dotty/tools/MainGenericRunner.scala index dca7178c52f5..1dcab2d92a96 100644 --- a/compiler/src/dotty/tools/MainGenericRunner.scala +++ b/compiler/src/dotty/tools/MainGenericRunner.scala @@ -100,6 +100,20 @@ object MainGenericRunner { val classpathSeparator = File.pathSeparator + def processClasspath(cp: String, tail: List[String]): (List[String], List[String]) = + val cpEntries = cp.split(classpathSeparator).toList + val singleEntryClasspath: Boolean = cpEntries.take(2).size == 1 + val globdir: String = if singleEntryClasspath then cp.replaceAll("[\\\\/][^\\\\/]*$", "") else "" // slash/backslash agnostic + def validGlobbedJar(s: String): Boolean = s.startsWith(globdir) && ((s.toLowerCase.endsWith(".jar") || s.toLowerCase.endsWith(".zip"))) + if singleEntryClasspath && validGlobbedJar(cpEntries.head) then + // reassemble globbed wildcard classpath + // globdir is wildcard directory for globbed jar files, reconstruct the intended classpath + val cpJars = tail.takeWhile( f => validGlobbedJar(f) ) + val remainingArgs = tail.drop(cpJars.size) + (remainingArgs, cpEntries ++ cpJars) + else + (tail, cpEntries) + @sharable val javaOption = raw"""-J(.*)""".r @sharable val scalaOption = raw"""@.*""".r @sharable val colorOption = raw"""-color:.*""".r @@ -110,21 +124,8 @@ object MainGenericRunner { case "-run" :: fqName :: tail => process(tail, settings.withExecuteMode(ExecuteMode.Run).withTargetToRun(fqName)) case ("-cp" | "-classpath" | "--class-path") :: cp :: tail => - val cpEntries = cp.split(classpathSeparator).toList - val singleEntryClasspath: Boolean = cpEntries.take(2).size == 1 - val globdir: String = if singleEntryClasspath then cp.replaceAll("[\\\\/][^\\\\/]*$", "") else "" // slash/backslash agnostic - def validGlobbedJar(s: String): Boolean = s.startsWith(globdir) && ((s.toLowerCase.endsWith(".jar") || s.toLowerCase.endsWith(".zip"))) - val (tailargs, newEntries) = if singleEntryClasspath && validGlobbedJar(cpEntries.head) then - // reassemble globbed wildcard classpath - // globdir is wildcard directory for globbed jar files, reconstruct the intended classpath - val cpJars = tail.takeWhile( f => validGlobbedJar(f) ) - val remainingArgs = tail.drop(cpJars.size) - (remainingArgs, cpEntries ++ cpJars) - else - (tail, cpEntries) - + val (tailargs, newEntries) = processClasspath(cp, tail) process(tailargs, settings.copy(classPath = settings.classPath ++ newEntries.filter(_.nonEmpty))) - case ("-version" | "--version") :: _ => settings.copy( executeMode = ExecuteMode.Repl, @@ -170,7 +171,7 @@ object MainGenericRunner { val newSettings = if arg.startsWith("-") then settings else settings.withPossibleEntryPaths(arg).withModeShouldBePossibleRun process(tail, newSettings.withResidualArgs(arg)) end process - + def main(args: Array[String]): Unit = val scalaOpts = envOrNone("SCALA_OPTS").toArray.flatMap(_.split(" ")).filter(_.nonEmpty) val allArgs = scalaOpts ++ args diff --git a/dist/bin/scala b/dist/bin/scala index b3116b2706b3..d99fb24308d0 100755 --- a/dist/bin/scala +++ b/dist/bin/scala @@ -33,9 +33,9 @@ while [[ $# -gt 0 ]]; do -D*) # pass to scala as well: otherwise we lose it sometimes when we # need it, e.g. communicating with a server compiler. + # respect user-supplied -Dscala.usejavacp addJava "$1" addScala "$1" - # respect user-supplied -Dscala.usejavacp shift ;; -J*) @@ -47,7 +47,7 @@ while [[ $# -gt 0 ]]; do ;; -classpath*) if [ "$1" != "${1##* }" ]; then - # hashbang-combined args "-classpath 'lib/*'" + # -classpath and its value have been supplied in a single string e.g. "-classpath 'lib/*'" A=$1 ; shift # consume $1 before adding its substrings back set -- $A "$@" # split $1 on whitespace and put it back else diff --git a/dist/bin/scalac b/dist/bin/scalac index 6ebf0fe70ea8..11da4fdedc34 100644 --- a/dist/bin/scalac +++ b/dist/bin/scalac @@ -30,43 +30,54 @@ source "$PROG_HOME/bin/common" [ -z "$PROG_NAME" ] && PROG_NAME=$CompilerMain -withCompiler=true - while [[ $# -gt 0 ]]; do -case "$1" in - --) shift; for arg; do addResidual "$arg"; done; set -- ;; - -v|-verbose) verbose=true && addScala "-verbose" && shift ;; - -q|-quiet) quiet=true && shift ;; - + case "$1" in + --) + # pass all remaining arguments to scala, e.g. to avoid interpreting them here as -D or -J + while [[ $# -gt 0 ]]; do addScala "$1" && shift ; done + ;; + -script) + # pass all remaining arguments to scala, e.g. to avoid interpreting them here as -D or -J + while [[ $# -gt 0 ]]; do addScala "$1" && shift ; done + ;; # Optimize for short-running applications, see https://github.com/lampepfl/dotty/issues/222 - -Oshort) addJava "-XX:+TieredCompilation -XX:TieredStopAtLevel=1" && shift ;; - -repl) PROG_NAME="$ReplMain" && shift ;; - -script) PROG_NAME="$ScriptingMain" && target_script="$2" && shift && shift - while [[ $# -gt 0 ]]; do addScript "$1" && shift ; done ;; - -compile) PROG_NAME="$CompilerMain" && shift ;; - -decompile) PROG_NAME="$DecompilerMain" && shift ;; - -print-tasty) PROG_NAME="$TastyPrinterMain" && shift ;; - -run) PROG_NAME="$ReplMain" && shift ;; - -colors) colors=true && shift ;; - -no-colors) unset colors && shift ;; - -with-compiler) jvm_cp_args="$PSEP$DOTTY_COMP$PSEP$TASTY_CORE" && shift ;; - - # break out -D and -J options and add them to java_args so - # they reach the JVM in time to do some good. The -D options - # will be available as system properties. - -D*) addJava "$1" && shift ;; - -J*) addJava "${1:2}" && shift ;; - *) addResidual "$1" && shift ;; + -Oshort) + addScala "-Oshort" && \ + addJava "-XX:+TieredCompilation" && addJava "-XX:TieredStopAtLevel=1" && shift ;; + -D*) + # pass to scala as well: otherwise we lose it sometimes when we + # need it, e.g. communicating with a server compiler. + # respect user-supplied -Dscala.usejavacp + addJava "$1" + addScala "$1" + shift + ;; + -J*) + # as with -D, pass to scala even though it will almost + # never be used. + addJava "${1:2}" + addScala "$1" + shift + ;; + -classpath*) + if [ "$1" != "${1##* }" ]; then + # -classpath and its value have been supplied in a single string e.g. "-classpath 'lib/*'" + A=$1 ; shift # consume $1 before adding its substrings back + set -- $A "$@" # split $1 on whitespace and put it back + else + addScala "$1" + shift + fi + ;; + *) + addScala "$1" + shift + ;; esac done compilerJavaClasspathArgs -if [ "$PROG_NAME" == "$ScriptingMain" ]; then - setScriptName="-Dscript.path=$target_script" - scripting_string="-script $target_script ${script_args[@]}" -fi - [ -n "$script_trace" ] && set -x [ -z "${ConEmuPID-}" -o -n "${cygwin-}" ] && export MSYSTEM= PWD= # workaround for #12405 @@ -75,11 +86,10 @@ eval "\"$JAVACMD\"" \ ${JAVA_OPTS:-$default_java_opts} \ "${java_args[@]}" \ "-classpath \"$jvm_cp_args\"" \ - -Dscala.usejavacp=true \ - "$setScriptName" \ - "$PROG_NAME" \ - "${scala_args[@]}" \ - "${residual_args[@]}" \ - "${scripting_string-}" -scala_exit_status=$? + "-Dscala.usejavacp=true" \ + "-Dscala.home=$PROG_HOME" \ + "dotty.tools.MainGenericCompiler" \ + "${scala_args[@]}" +scala_exit_status=$? +onExit diff --git a/project/scripts/bootstrappedOnlyCmdTests b/project/scripts/bootstrappedOnlyCmdTests index 472426b23b20..8a0f5cf78f2f 100755 --- a/project/scripts/bootstrappedOnlyCmdTests +++ b/project/scripts/bootstrappedOnlyCmdTests @@ -47,12 +47,26 @@ echo "testing sbt scalac -decompile from file" ./bin/scalac -decompile -color:never "$OUT/$TASTY" > "$tmp" grep -qe "def main(args: scala.Array\[scala.Predef.String\]): scala.Unit =" "$tmp" +# check that `sbt scalac -print-tasty` runs +echo "testing sbt scalac -print-tasty from file" +./bin/scalac -print-tasty -color:never "$OUT/$TASTY" > "$tmp" +grep -qe "118: STRINGconst 32 \[hello world\]" "$tmp" + echo "testing loading tasty from .tasty file in jar" clear_out "$OUT" ./bin/scalac -d "$OUT/out.jar" "$SOURCE" ./bin/scalac -decompile -color:never "$OUT/out.jar" > "$tmp" grep -qe "def main(args: scala.Array\[scala.Predef.String\]): scala.Unit =" "$tmp" +echo "testing printing tasty from .tasty file in jar" +./bin/scalac -print-tasty -color:never "$OUT/out.jar" > "$tmp" +grep -qe "118: STRINGconst 32 \[hello world\]" "$tmp" + +echo "testing -script from scalac" +clear_out "$OUT" +./bin/scalac -script "$SOURCE" > "$tmp" +test "$EXPECTED_OUTPUT" = "$(cat "$tmp")" + echo "testing sbt scalac with suspension" clear_out "$OUT" "$SBT" "scala3-compiler-bootstrapped/scalac -d $OUT tests/pos-macros/macros-in-same-project-1/Bar.scala tests/pos-macros/macros-in-same-project-1/Foo.scala" > "$tmp"