From 7028f92716dbc2dce207e849f9c035f0417cc147 Mon Sep 17 00:00:00 2001 From: James Santucci Date: Wed, 2 Feb 2022 13:27:33 -0700 Subject: [PATCH 1/4] separate modules for compiler and plugin this change makes it so that I can separately test the classpath manipulation games to reproduce and the attempt to initialize from Scala. it turns out the problem is reproducible in pure scala! which is great because it probably means I don't have to think about dynamic classloading ever again. --- build.sbt | 43 +++++++++--- .../io/github/jisantuc/sbtse/Compiler.scala | 14 ++++ .../io/github/jisantuc/sbtse/Extracter.scala | 66 +++++++++++++++++++ .../jisantuc/sbtse/SourceExtractSpec.scala | 12 ++++ project/plugins.sbt | 1 + .../jisantuc/sbtse/SourceExtractPlugin.scala | 58 ++++++++++++++++ .../sbt-source-extract/simple/build.sbt | 0 .../simple/project/build.properties | 0 .../simple/project/plugins.sbt | 0 .../simple/src/main/scala/Main.scala | 0 .../sbt-test/sbt-source-extract/simple/test | 2 + .../jisantuc/sbtse/SourceextractPlugin.scala | 30 --------- src/sbt-test/sbt-source-extract/simple/test | 2 - .../jisantuc/sbtse/SourceextractSpec.scala | 5 -- 14 files changed, 187 insertions(+), 46 deletions(-) create mode 100644 compiler/src/main/scala/io/github/jisantuc/sbtse/Compiler.scala create mode 100644 compiler/src/main/scala/io/github/jisantuc/sbtse/Extracter.scala create mode 100644 compiler/src/test/scala/io/github/jisantuc/sbtse/SourceExtractSpec.scala create mode 100644 sbtse/src/main/scala/io/github/jisantuc/sbtse/SourceExtractPlugin.scala rename {src => sbtse/src}/sbt-test/sbt-source-extract/simple/build.sbt (100%) rename {src => sbtse/src}/sbt-test/sbt-source-extract/simple/project/build.properties (100%) rename {src => sbtse/src}/sbt-test/sbt-source-extract/simple/project/plugins.sbt (100%) rename {src => sbtse/src}/sbt-test/sbt-source-extract/simple/src/main/scala/Main.scala (100%) create mode 100644 sbtse/src/sbt-test/sbt-source-extract/simple/test delete mode 100644 src/main/scala/io/github/jisantuc/sbtse/SourceextractPlugin.scala delete mode 100644 src/sbt-test/sbt-source-extract/simple/test delete mode 100644 src/test/scala/io/github/jisantuc/sbtse/SourceextractSpec.scala diff --git a/build.sbt b/build.sbt index e8cbdb4..5d41814 100644 --- a/build.sbt +++ b/build.sbt @@ -8,9 +8,15 @@ addCommandAlias( ";scalafmtCheckAll; scalafmtSbtCheck; +test; +publishLocal; scripted" ) -// ScalaTest -libraryDependencies += "org.scalactic" %% "scalactic" % "3.2.9" % Test -libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.9" % Test +lazy val root = project.aggregate(compiler, `sbt-source-extract`) + +lazy val dependencies = Seq( + "org.scalactic" %% "scalactic" % "3.2.9" % Test, + "org.scalatest" %% "scalatest" % "3.2.9" % Test, + "org.typelevel" %% "cats-core" % "2.4.0" +) + +lazy val compilerClasspath = TaskKey[Classpath]("compiler-classpath") inThisBuild( List( @@ -23,7 +29,7 @@ inThisBuild( Developer( "jisantuc", "James Santucci", - "james.santucci@47deg.com", + "james.santucci@47d1eg.com", url("https://github.com/jisantuc") ) ) @@ -32,13 +38,32 @@ inThisBuild( (console / initialCommands) := """import io.github.jisantuc.sbtse._""" -enablePlugins(ScriptedPlugin, SbtPlugin) -// set up 'scripted; sbt plugin for testing sbt plugins -scriptedLaunchOpts ++= - Seq("-Xmx1024M", "-Dplugin.version=" + version.value) - ThisBuild / githubWorkflowBuild := Seq( WorkflowStep.Sbt(List("ci-test")) ) ThisBuild / githubWorkflowPublishTargetBranches := Seq.empty + +lazy val `sbt-source-extract` = (project in file(".")) + .settings( + compilerClasspath := { (compiler / Compile / fullClasspath) }.value, + buildInfoObject := "Meta", + buildInfoPackage := "io.github.jisantuc.sbtse", + buildInfoKeys := Seq( + version, + BuildInfoKey.map(compilerClasspath) { case (_, classFiles) ⇒ + ("compilerClasspath", classFiles.map(_.data)) + } + ), + libraryDependencies ++= dependencies, + // set up 'scripted; sbt plugin for testing sbt plugins + scriptedLaunchOpts ++= + Seq("-Xmx1024M", "-Dplugin.version=" + version.value) + ) + .dependsOn(compiler) + +lazy val compiler = (project in file("compiler")) + .settings( + libraryDependencies ++= dependencies + ) + .enablePlugins(BuildInfoPlugin, SbtPlugin) diff --git a/compiler/src/main/scala/io/github/jisantuc/sbtse/Compiler.scala b/compiler/src/main/scala/io/github/jisantuc/sbtse/Compiler.scala new file mode 100644 index 0000000..29537cd --- /dev/null +++ b/compiler/src/main/scala/io/github/jisantuc/sbtse/Compiler.scala @@ -0,0 +1,14 @@ +package io.github.jisantuc.sbtse + +class CompilerJava { + def compile(): Array[String] = { + locally(Compiler().test) + println("ok!") + Array.empty + } +} + +final case class Compiler() { + lazy val extracter = new Extracter() + val test = extracter.testFn() +} diff --git a/compiler/src/main/scala/io/github/jisantuc/sbtse/Extracter.scala b/compiler/src/main/scala/io/github/jisantuc/sbtse/Extracter.scala new file mode 100644 index 0000000..ceb579e --- /dev/null +++ b/compiler/src/main/scala/io/github/jisantuc/sbtse/Extracter.scala @@ -0,0 +1,66 @@ +package io.github.jisantuc.sbtse + +import java.io.File + +import scala.annotation.tailrec +import scala.collection.compat._ +import scala.reflect.internal.util.BatchSourceFile +import scala.tools.nsc._ +import scala.tools.nsc.doc.{Settings => _, _} +import scala.tools.nsc.doc.base.comment.Comment + +class Extracter { + private def classPathOfClass(className: String): List[String] = { + val resource = className.split('.').mkString("/", "/", ".class") + val path = getClass.getResource(resource).getPath + if (path.indexOf("file:") >= 0) { + val indexOfFile = path.indexOf("file:") + 5 + val indexOfSeparator = path.lastIndexOf('!') + List(path.substring(indexOfFile, indexOfSeparator)) + } else { + require(path.endsWith(resource)) + List(path.substring(0, path.length - resource.length + 1)) + } + } + + private lazy val compilerPath = + try classPathOfClass("scala.tools.nsc.Interpreter") + catch { + case e: Throwable => + throw new RuntimeException( + "Unable to load Scala interpreter from classpath (scala-compiler jar is missing?)", + e + ) + } + + private lazy val libPath = + try classPathOfClass("scala.AnyVal") + catch { + case e: Throwable => + throw new RuntimeException( + "Unable to load scala base object from classpath (scala-library jar is missing?)", + e + ) + } + + lazy val paths: List[String] = compilerPath ::: libPath + + // Same settings as + // https://github.com/scala-exercises/sbt-exercise/blob/v0.6.7/compiler/src/main/scala/org/scalaexercises/exercises/compiler/SourceTextExtraction.scal + // and + // https://github.com/scala-exercises/sbt-exercise/blob/v0.6.7/compiler/src/main/scala/org/scalaexercises/exercises/compiler/CompilerSettings.scala + // but with a vanilla Global as the embeddedDefaults type param + val defaultSettings = new Settings { + embeddedDefaults[Global.type] + Yrangepos.value = true + usejavacp.value = true + + bootclasspath.value = paths.mkString(File.pathSeparator) + classpath.value = paths.mkString(File.pathSeparator) + } + val global = new Global(defaultSettings) + + def testFn(): global.Run = { + new global.Run() + } +} diff --git a/compiler/src/test/scala/io/github/jisantuc/sbtse/SourceExtractSpec.scala b/compiler/src/test/scala/io/github/jisantuc/sbtse/SourceExtractSpec.scala new file mode 100644 index 0000000..9053c3c --- /dev/null +++ b/compiler/src/test/scala/io/github/jisantuc/sbtse/SourceExtractSpec.scala @@ -0,0 +1,12 @@ +package io.github.jisantuc.sbtse + +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers + +class SourceExtractTest extends AnyFunSpec with Matchers { + describe("run without crashing when initialized without classpath hackery") { + it("should do that") { + new CompilerJava().compile() + } + } +} diff --git a/project/plugins.sbt b/project/plugins.sbt index c73413a..83274ec 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,4 @@ libraryDependencies += "org.scala-sbt" %% "scripted-plugin" % sbtVersion.value addSbtPlugin("com.codecommit" % "sbt-github-actions" % "0.13.0") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6") +addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.10.0") diff --git a/sbtse/src/main/scala/io/github/jisantuc/sbtse/SourceExtractPlugin.scala b/sbtse/src/main/scala/io/github/jisantuc/sbtse/SourceExtractPlugin.scala new file mode 100644 index 0000000..dec84eb --- /dev/null +++ b/sbtse/src/main/scala/io/github/jisantuc/sbtse/SourceExtractPlugin.scala @@ -0,0 +1,58 @@ +package io.github.jisantuc.sbtse + +import sbt._ +import sbt.Keys._ +import sbt.plugins.JvmPlugin + +import cats.data.Ior +import cats.syntax.either._ +import scala.util.Random +import sbt.internal.inc.classpath.ClasspathUtil +import java.nio.file.Paths + +object SourceExtractPlugin extends AutoPlugin { + + override def trigger = allRequirements + override def requires = JvmPlugin + + object autoImport { + val testInit = TaskKey[Unit]("test-init", "run initialization to see if it crashes") + } + + import autoImport._ + + override lazy val projectSettings = Seq( + testInit := testTask.value + ) + + override lazy val buildSettings = Seq() + + override lazy val globalSettings = Seq() + + private def catching[A](f: => A)(msg: => String) = + Either.catchNonFatal(f).leftMap(e => Ior.both(msg, e)) + + + def testTask = Def.task { + val libraryClasspath = Attributed.data((Compile / fullClasspath).value) + val classpath = (Meta.compilerClasspath ++ libraryClasspath).distinct + val loader = ClasspathUtil.toLoader( + classpath.map(file => Paths.get(file.getAbsolutePath())), + null, + ClasspathUtil.createClasspathResources( + appPaths = Meta.compilerClasspath.map(file => Paths.get(file.getAbsolutePath())), + bootPaths = scalaInstance.value.allJars.map(file => Paths.get(file.getAbsolutePath())) + ) + ) + val result = for { + compilerClass <- catching(loader.loadClass(COMPILER_CLASS))( + "Unable to find exercise compiler class" + ) + compiler <- catching(compilerClass.newInstance.asInstanceOf[COMPILER])( + "Unable to create instance of exercise compiler" + ) + libraries <- libraryNames.toList.traverse(loadLibraryModule) + result <- libraries.traverse(invokeCompiler(compiler, _)) + } yield result + } +} diff --git a/src/sbt-test/sbt-source-extract/simple/build.sbt b/sbtse/src/sbt-test/sbt-source-extract/simple/build.sbt similarity index 100% rename from src/sbt-test/sbt-source-extract/simple/build.sbt rename to sbtse/src/sbt-test/sbt-source-extract/simple/build.sbt diff --git a/src/sbt-test/sbt-source-extract/simple/project/build.properties b/sbtse/src/sbt-test/sbt-source-extract/simple/project/build.properties similarity index 100% rename from src/sbt-test/sbt-source-extract/simple/project/build.properties rename to sbtse/src/sbt-test/sbt-source-extract/simple/project/build.properties diff --git a/src/sbt-test/sbt-source-extract/simple/project/plugins.sbt b/sbtse/src/sbt-test/sbt-source-extract/simple/project/plugins.sbt similarity index 100% rename from src/sbt-test/sbt-source-extract/simple/project/plugins.sbt rename to sbtse/src/sbt-test/sbt-source-extract/simple/project/plugins.sbt diff --git a/src/sbt-test/sbt-source-extract/simple/src/main/scala/Main.scala b/sbtse/src/sbt-test/sbt-source-extract/simple/src/main/scala/Main.scala similarity index 100% rename from src/sbt-test/sbt-source-extract/simple/src/main/scala/Main.scala rename to sbtse/src/sbt-test/sbt-source-extract/simple/src/main/scala/Main.scala diff --git a/sbtse/src/sbt-test/sbt-source-extract/simple/test b/sbtse/src/sbt-test/sbt-source-extract/simple/test new file mode 100644 index 0000000..6b311d6 --- /dev/null +++ b/sbtse/src/sbt-test/sbt-source-extract/simple/test @@ -0,0 +1,2 @@ +# check whether the test task runs without crashing +> test-init \ No newline at end of file diff --git a/src/main/scala/io/github/jisantuc/sbtse/SourceextractPlugin.scala b/src/main/scala/io/github/jisantuc/sbtse/SourceextractPlugin.scala deleted file mode 100644 index 2c3062a..0000000 --- a/src/main/scala/io/github/jisantuc/sbtse/SourceextractPlugin.scala +++ /dev/null @@ -1,30 +0,0 @@ -package io.github.jisantuc.sbtse - -import sbt._ -import sbt.Keys._ -import sbt.plugins.JvmPlugin - -import scala.util.Random - -object SourceExtractPlugin extends AutoPlugin { - - override def trigger = allRequirements - override def requires = JvmPlugin - - object autoImport { - val randomGenerator = - SettingKey[Random]("random-generator", "random number generator") - val randomNumber = TaskKey[Int]("random-number", "generate a random number") - } - - import autoImport._ - - override lazy val projectSettings = Seq( - randomGenerator := new Random(), - randomNumber := randomGenerator.value.nextInt(100) - ) - - override lazy val buildSettings = Seq() - - override lazy val globalSettings = Seq() -} diff --git a/src/sbt-test/sbt-source-extract/simple/test b/src/sbt-test/sbt-source-extract/simple/test deleted file mode 100644 index 69f0628..0000000 --- a/src/sbt-test/sbt-source-extract/simple/test +++ /dev/null @@ -1,2 +0,0 @@ -# check if we can compile -> show randomNumber \ No newline at end of file diff --git a/src/test/scala/io/github/jisantuc/sbtse/SourceextractSpec.scala b/src/test/scala/io/github/jisantuc/sbtse/SourceextractSpec.scala deleted file mode 100644 index 20f0f13..0000000 --- a/src/test/scala/io/github/jisantuc/sbtse/SourceextractSpec.scala +++ /dev/null @@ -1,5 +0,0 @@ -package io.github.jisantuc.sbtse - -class SourceExtractTest { - // write tests with your preferred framework -} From 20f736316e889ba7619a2b7125a1ae0078981d51 Mon Sep 17 00:00:00 2001 From: James Santucci Date: Wed, 2 Feb 2022 17:15:42 -0700 Subject: [PATCH 2/4] add reproduction --- build.sbt | 3 +- .../jisantuc/sbtse/SourceExtractPlugin.scala | 40 ++++++++++--------- .../sbt-source-extract/simple/build.sbt | 9 ++++- .../sbt-test/sbt-source-extract/simple/test | 2 +- 4 files changed, 33 insertions(+), 21 deletions(-) diff --git a/build.sbt b/build.sbt index 5d41814..1e9c6b0 100644 --- a/build.sbt +++ b/build.sbt @@ -44,7 +44,7 @@ ThisBuild / githubWorkflowBuild := Seq( ThisBuild / githubWorkflowPublishTargetBranches := Seq.empty -lazy val `sbt-source-extract` = (project in file(".")) +lazy val `sbt-source-extract` = (project in file("sbtse")) .settings( compilerClasspath := { (compiler / Compile / fullClasspath) }.value, buildInfoObject := "Meta", @@ -61,6 +61,7 @@ lazy val `sbt-source-extract` = (project in file(".")) Seq("-Xmx1024M", "-Dplugin.version=" + version.value) ) .dependsOn(compiler) + .enablePlugins(BuildInfoPlugin, SbtPlugin, ScriptedPlugin) lazy val compiler = (project in file("compiler")) .settings( diff --git a/sbtse/src/main/scala/io/github/jisantuc/sbtse/SourceExtractPlugin.scala b/sbtse/src/main/scala/io/github/jisantuc/sbtse/SourceExtractPlugin.scala index dec84eb..98dc016 100644 --- a/sbtse/src/main/scala/io/github/jisantuc/sbtse/SourceExtractPlugin.scala +++ b/sbtse/src/main/scala/io/github/jisantuc/sbtse/SourceExtractPlugin.scala @@ -6,6 +6,7 @@ import sbt.plugins.JvmPlugin import cats.data.Ior import cats.syntax.either._ +import cats.syntax.monadError._ import scala.util.Random import sbt.internal.inc.classpath.ClasspathUtil import java.nio.file.Paths @@ -16,13 +17,14 @@ object SourceExtractPlugin extends AutoPlugin { override def requires = JvmPlugin object autoImport { - val testInit = TaskKey[Unit]("test-init", "run initialization to see if it crashes") + val testInit = + TaskKey[Unit]("test-init", "run initialization to see if it crashes") } import autoImport._ override lazy val projectSettings = Seq( - testInit := testTask.value + testInit := locally(testTask.value) ) override lazy val buildSettings = Seq() @@ -32,27 +34,29 @@ object SourceExtractPlugin extends AutoPlugin { private def catching[A](f: => A)(msg: => String) = Either.catchNonFatal(f).leftMap(e => Ior.both(msg, e)) + private val COMPILER_CLASS = "io.github.jisantuc.sbtse.CompilerJava" + + private type COMPILER = { + def compile(): Array[String] + } def testTask = Def.task { val libraryClasspath = Attributed.data((Compile / fullClasspath).value) - val classpath = (Meta.compilerClasspath ++ libraryClasspath).distinct + val classpath = (Meta.compilerClasspath ++ libraryClasspath).distinct val loader = ClasspathUtil.toLoader( - classpath.map(file => Paths.get(file.getAbsolutePath())), - null, - ClasspathUtil.createClasspathResources( - appPaths = Meta.compilerClasspath.map(file => Paths.get(file.getAbsolutePath())), - bootPaths = scalaInstance.value.allJars.map(file => Paths.get(file.getAbsolutePath())) + classpath.map(file => Paths.get(file.getAbsolutePath())), + null, + ClasspathUtil.createClasspathResources( + appPaths = + Meta.compilerClasspath.map(file => Paths.get(file.getAbsolutePath())), + bootPaths = scalaInstance.value.allJars.map(file => + Paths.get(file.getAbsolutePath()) ) + ) ) - val result = for { - compilerClass <- catching(loader.loadClass(COMPILER_CLASS))( - "Unable to find exercise compiler class" - ) - compiler <- catching(compilerClass.newInstance.asInstanceOf[COMPILER])( - "Unable to create instance of exercise compiler" - ) - libraries <- libraryNames.toList.traverse(loadLibraryModule) - result <- libraries.traverse(invokeCompiler(compiler, _)) - } yield result + + val compilerClass = loader.loadClass(COMPILER_CLASS) + val compiler = compilerClass.newInstance.asInstanceOf[COMPILER] + compiler.compile() } } diff --git a/sbtse/src/sbt-test/sbt-source-extract/simple/build.sbt b/sbtse/src/sbt-test/sbt-source-extract/simple/build.sbt index a62047e..eeac86b 100644 --- a/sbtse/src/sbt-test/sbt-source-extract/simple/build.sbt +++ b/sbtse/src/sbt-test/sbt-source-extract/simple/build.sbt @@ -1,4 +1,11 @@ version := "0.1" scalaVersion := "2.12.15" -enablePlugins(SourceExtractPlugin) +lazy val simple = (project in file(".")) + .settings( + resolvers ++= Seq( + Resolver.sonatypeRepo("snapshots"), + Resolver.defaultLocal + ) + ) + .enablePlugins(SourceExtractPlugin) diff --git a/sbtse/src/sbt-test/sbt-source-extract/simple/test b/sbtse/src/sbt-test/sbt-source-extract/simple/test index 6b311d6..8f138a9 100644 --- a/sbtse/src/sbt-test/sbt-source-extract/simple/test +++ b/sbtse/src/sbt-test/sbt-source-extract/simple/test @@ -1,2 +1,2 @@ # check whether the test task runs without crashing -> test-init \ No newline at end of file +> simple / testInit \ No newline at end of file From 8ce17a05eb639a0012fbfc530dc6164643d9460b Mon Sep 17 00:00:00 2001 From: James Santucci Date: Thu, 3 Feb 2022 09:08:13 -0700 Subject: [PATCH 3/4] =?UTF-8?q?=5Fdon't=5F=20`usejavacp`=20after=20all=20?= =?UTF-8?q?=F0=9F=A4=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/scala/io/github/jisantuc/sbtse/Extracter.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/src/main/scala/io/github/jisantuc/sbtse/Extracter.scala b/compiler/src/main/scala/io/github/jisantuc/sbtse/Extracter.scala index ceb579e..34a9341 100644 --- a/compiler/src/main/scala/io/github/jisantuc/sbtse/Extracter.scala +++ b/compiler/src/main/scala/io/github/jisantuc/sbtse/Extracter.scala @@ -53,7 +53,7 @@ class Extracter { val defaultSettings = new Settings { embeddedDefaults[Global.type] Yrangepos.value = true - usejavacp.value = true + usejavacp.value = false bootclasspath.value = paths.mkString(File.pathSeparator) classpath.value = paths.mkString(File.pathSeparator) From 3ef0678c3a37dd08a4936d0ad9eea7e13f5a0f07 Mon Sep 17 00:00:00 2001 From: James Santucci Date: Thu, 3 Feb 2022 09:11:30 -0700 Subject: [PATCH 4/4] update settings comment --- .../main/scala/io/github/jisantuc/sbtse/Extracter.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compiler/src/main/scala/io/github/jisantuc/sbtse/Extracter.scala b/compiler/src/main/scala/io/github/jisantuc/sbtse/Extracter.scala index 34a9341..b403520 100644 --- a/compiler/src/main/scala/io/github/jisantuc/sbtse/Extracter.scala +++ b/compiler/src/main/scala/io/github/jisantuc/sbtse/Extracter.scala @@ -47,9 +47,10 @@ class Extracter { // Same settings as // https://github.com/scala-exercises/sbt-exercise/blob/v0.6.7/compiler/src/main/scala/org/scalaexercises/exercises/compiler/SourceTextExtraction.scal - // and - // https://github.com/scala-exercises/sbt-exercise/blob/v0.6.7/compiler/src/main/scala/org/scalaexercises/exercises/compiler/CompilerSettings.scala - // but with a vanilla Global as the embeddedDefaults type param + // except usejavacp.value is set to false. + // that setting controls whether to "Utilize the java.class.path in classpath resolution." + // clearly that was interacting with _something_ poorly (buildinfo plugin? I don't know!) + // but it'll probably be a bit before I understand what. val defaultSettings = new Settings { embeddedDefaults[Global.type] Yrangepos.value = true