diff --git a/bootstrap/build.sbt b/bootstrap/build.sbt index d079293f..0c3a6735 100644 --- a/bootstrap/build.sbt +++ b/bootstrap/build.sbt @@ -4,4 +4,6 @@ assemblyJarName in assembly := "ChatOverflow.jar" libraryDependencies += "org.scala-lang.modules" %% "scala-xml" % "1.1.1" libraryDependencies += "org.jline" % "jline-terminal-jansi" % "3.11.0" // used for terminal width -fork := true \ No newline at end of file +fork := true + +packageBin / includePom := false \ No newline at end of file diff --git a/build.sbt b/build.sbt index 99bf3415..bb8bb3ea 100644 --- a/build.sbt +++ b/build.sbt @@ -89,6 +89,9 @@ libraryDependencies += "com.fazecast" % "jSerialComm" % "[2.0.0,3.0.0)" // Socket.io libraryDependencies += "io.socket" % "socket.io-client" % "1.0.0" +// Coursier +libraryDependencies += "io.get-coursier" %% "coursier" % "2.0.0-RC3-2" + // --------------------------------------------------------------------------------------------------------------------- // PLUGIN FRAMEWORK DEFINITIONS // --------------------------------------------------------------------------------------------------------------------- @@ -135,6 +138,7 @@ Compile / packageBin := { } Compile / unmanagedJars := (crossTarget.value ** "chatoverflow-gui*.jar").classpath +packageBin / includePom := false // --------------------------------------------------------------------------------------------------------------------- // UTIL diff --git a/build/src/main/scala/org/codeoverflow/chatoverflow/build/PomInclusionPlugin.scala b/build/src/main/scala/org/codeoverflow/chatoverflow/build/PomInclusionPlugin.scala new file mode 100644 index 00000000..b75ee0b1 --- /dev/null +++ b/build/src/main/scala/org/codeoverflow/chatoverflow/build/PomInclusionPlugin.scala @@ -0,0 +1,47 @@ +package org.codeoverflow.chatoverflow.build + +import sbt._ +import sbt.Keys._ +import sbt.plugins.JvmPlugin + +/** + * A sbt plugin to automatically include the dependencies of a sbt project in the jar as a pom file called "dependencies.pom". + */ +object PomInclusionPlugin extends AutoPlugin { + + // Everything in autoImport will be visible to sbt project files + // They can set this value to false if they don't want to include their dependencies as a pom file + object autoImport { + val includePom = settingKey[Boolean]("Whether to include a pom file inside the jar with all dependencies.") + } + import autoImport._ + + // We require to have the Compile configuration and the packageBin task to override + override def requires = JvmPlugin + override def trigger = allRequirements + + // Adds our custom task before the packageBin task + override val projectSettings: Seq[Def.Setting[_]] = + inConfig(Compile)(Seq( + Compile / packageBin := { + addPomToOutput.value + (Compile / packageBin).value + } + )) + + // Sets default values + override def buildSettings: Seq[Def.Setting[_]] = inConfig(Compile)( + includePom in packageBin := true + ) + + // Just copies the pom resulted by makePom into the directory for compiled classes + // That way the file will be included in the jar + private lazy val addPomToOutput = Def.taskDyn { + if ((includePom in packageBin).value) Def.task { + val pomFile = (Compile / makePom).value + + IO.copyFile(pomFile, new File((Compile / classDirectory).value, "dependencies.pom")) + } else + Def.task {} // if disabled, do nothing + } +} \ No newline at end of file diff --git a/src/main/scala/org/codeoverflow/chatoverflow/framework/PluginFramework.scala b/src/main/scala/org/codeoverflow/chatoverflow/framework/PluginFramework.scala index 9f18032d..ed3aece0 100644 --- a/src/main/scala/org/codeoverflow/chatoverflow/framework/PluginFramework.scala +++ b/src/main/scala/org/codeoverflow/chatoverflow/framework/PluginFramework.scala @@ -7,6 +7,10 @@ import org.codeoverflow.chatoverflow.framework.helper.PluginLoader import org.codeoverflow.chatoverflow.framework.manager.PluginManagerStub import scala.collection.mutable.ListBuffer +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration._ +import scala.concurrent.{Await, Future} +import scala.util.Success /** * The plugin framework holds all plugin types important from the jar files in the plugin folder. @@ -62,6 +66,8 @@ class PluginFramework(pluginDirectoryPath: String) extends WithLogger { logger warn s"PluginType directory '$pluginDirectory' does not exist!" } else { + val futures = ListBuffer[Future[_]]() + // Get (new) jar file urls val jarFiles = getNewJarFiles(pluginDirectory) logger info s"Found ${jarFiles.length} new plugins." @@ -81,22 +87,28 @@ class PluginFramework(pluginDirectoryPath: String) extends WithLogger { } else { // Try to test the initiation of the plugin - try { - plugin.createPluginInstance(new PluginManagerStub) - logger info s"Successfully tested instantiation of plugin '${plugin.getName}'" - pluginTypes += plugin - } catch { - // Note that we catch not only exceptions, but also errors like NoSuchMethodError. Deep stuff - case _: Error => logger warn s"Error while test init of plugin '${plugin.getName}'." - case _: Exception => logger warn s"Exception while test init of plugin '${plugin.getName}'." + futures += plugin.getDependencyFuture andThen { + case Success(_) => + try { + plugin.createPluginInstance(new PluginManagerStub) + logger info s"Successfully tested instantiation of plugin '${plugin.getName}'" + pluginTypes += plugin + } catch { + // Note that we catch not only exceptions, but also errors like NoSuchMethodError. Deep stuff + case _: Error => logger warn s"Error while test init of plugin '${plugin.getName}'." + case _: Exception => logger warn s"Exception while test init of plugin '${plugin.getName}'." + } } } } } - } - logger info s"Loaded ${pluginTypes.length} plugin types in total: " + - s"${pluginTypes.map(pt => s"${pt.getName} (${pt.getAuthor})").mkString(", ")}" + // If plugins aren't done within this timeout they can still fetch everything in the background, they just won't be included in this summary + futures.foreach(f => Await.ready(f, 5.seconds)) + + logger info s"Loaded ${pluginTypes.length} plugin types in total: " + + s"${pluginTypes.map(pt => s"${pt.getName} (${pt.getAuthor})").mkString(", ")}" + } } /** diff --git a/src/main/scala/org/codeoverflow/chatoverflow/framework/PluginType.scala b/src/main/scala/org/codeoverflow/chatoverflow/framework/PluginType.scala index e34f56de..da4dcfdc 100644 --- a/src/main/scala/org/codeoverflow/chatoverflow/framework/PluginType.scala +++ b/src/main/scala/org/codeoverflow/chatoverflow/framework/PluginType.scala @@ -1,25 +1,31 @@ package org.codeoverflow.chatoverflow.framework +import java.io.File + import org.codeoverflow.chatoverflow.WithLogger import org.codeoverflow.chatoverflow.api.APIVersion import org.codeoverflow.chatoverflow.api.plugin.{Plugin, PluginManager} import org.codeoverflow.chatoverflow.framework.PluginCompatibilityState.PluginCompatibilityState +import scala.concurrent.Future + /** - * A plugin type is a container for all information about a plugin, everything in the 'plugin.xml' and the actual class. - * The plugins functionality and meta information can be accessed through this interface. - * - * @param name the name of the plugin, used for identifying - * @param author the author of the plugin, used for identifying - * @param version the version of the plugin - * @param majorAPIVersion the major api version, with which the plugin was developed - * @param minorAPIVersion the minor api version, with which the plugin was developed - * @param pluginClass the class of the plugin, used to create instances of this plugin. - * Needs to have a constructor with the signature of one PluginManager, - * otherwise instances can't be created from it. - */ + * A plugin type is a container for all information about a plugin, everything in the 'plugin.xml' and the actual class. + * The plugins functionality and meta information can be accessed through this interface. + * + * @param name the name of the plugin, used for identifying + * @param author the author of the plugin, used for identifying + * @param version the version of the plugin + * @param majorAPIVersion the major api version, with which the plugin was developed + * @param minorAPIVersion the minor api version, with which the plugin was developed + * @param pluginClass the class of the plugin, used to create instances of this plugin. + * Needs to have a constructor with the signature of one PluginManager, + * otherwise instances can't be created from it. + * @param pluginDependencies A future that completes when all dependencies, that the plugin has, are available and + * that returns a seq of the required dependencies files in the local coursier cache. + */ class PluginType(name: String, author: String, version: String, majorAPIVersion: Int, minorAPIVersion: Int, - metadata: PluginMetadata, pluginClass: Class[_ <: Plugin]) extends WithLogger { + metadata: PluginMetadata, pluginClass: Class[_ <: Plugin], pluginDependencies: Future[Seq[File]]) extends WithLogger { private var pluginVersionState = PluginCompatibilityState.Untested @@ -126,4 +132,12 @@ class PluginType(name: String, author: String, version: String, majorAPIVersion: * @return the PluginMetadata instance of this plugin */ def getMetadata: PluginMetadata = metadata + + /** + * Returns the future that will result in a sequence of jar files that represents the dependencies + * of this plugin including all sub-dependencies. + * + * @return the dependency future + */ + def getDependencyFuture: Future[Seq[File]] = pluginDependencies } diff --git a/src/main/scala/org/codeoverflow/chatoverflow/framework/helper/CoursierUtils.scala b/src/main/scala/org/codeoverflow/chatoverflow/framework/helper/CoursierUtils.scala new file mode 100644 index 00000000..6097fd59 --- /dev/null +++ b/src/main/scala/org/codeoverflow/chatoverflow/framework/helper/CoursierUtils.scala @@ -0,0 +1,58 @@ +package org.codeoverflow.chatoverflow.framework.helper + +import java.io.{File, InputStream} + +import coursier.Fetch +import coursier.cache.{CacheLogger, FileCache} +import coursier.core.Dependency +import coursier.maven.PomParser +import org.codeoverflow.chatoverflow.WithLogger + +import scala.io.Source + +/** + * A utility object containing some common code for use with Coursier. + */ +object CoursierUtils extends WithLogger { + + private object CoursierLogger extends CacheLogger { + override def downloadedArtifact(url: String, success: Boolean): Unit = { + logger debug (if (success) + s"Successfully downloaded $url" + else + s"Failed to download $url") + } + } + + private val cache = FileCache().noCredentials.withLogger(CoursierLogger) + + /** + * Extracts all dependencies out of the provided pom. Throws an exception if the pom is invalid. + * + * @param is the InputStream from which the pom is read + * @return a seq of all found dependencies + */ + def parsePom(is: InputStream): Seq[Dependency] = { + val pomFile = Source.fromInputStream(is) + val parser = coursier.core.compatibility.xmlParseSax(pomFile.mkString, new PomParser) + + parser.project match { + case Right(deps) => deps.dependencies.map(_._2) + case Left(errorMsg) => throw new IllegalArgumentException(s"Pom couldn't be parsed: $errorMsg") + } + } + + /** + * Resolves and fetches all passed dependencies and gives back a seq of all local files of these dependencies. + * + * @param dependencies all dependencies that you want to be fetched + * @return all local files for the passed dependencies + */ + def fetchDependencies(dependencies: Seq[Dependency]): Seq[File] = { + // IntelliJ may warn you that a implicit is missing. This is one of the many bugs in IntelliJ, the code compiles fine. + Fetch() + .withCache(cache) + .addDependencies(dependencies: _*) + .run() + } +} diff --git a/src/main/scala/org/codeoverflow/chatoverflow/framework/helper/PluginClassLoader.scala b/src/main/scala/org/codeoverflow/chatoverflow/framework/helper/PluginClassLoader.scala index 01823173..4b324d8f 100644 --- a/src/main/scala/org/codeoverflow/chatoverflow/framework/helper/PluginClassLoader.scala +++ b/src/main/scala/org/codeoverflow/chatoverflow/framework/helper/PluginClassLoader.scala @@ -2,9 +2,52 @@ package org.codeoverflow.chatoverflow.framework.helper import java.net.{URL, URLClassLoader} +import org.codeoverflow.chatoverflow.WithLogger + +/** + * This plugin class loader is used for plugin security policy checks, + * to expose the addURL method to the package inorder to add all required dependencies after dependency resolution + * and most importantly to isolate the plugin from the normal classpath and only access the classpath if it needs to load the ChatOverflow api. + * Also if this PluginClassLoader had access to the classpath the same classes of the classpath would have + * higher priority over the classes in this classloader which could be a problem if a plugin uses a newer version + * of a dependency that the framework. + * + * @param urls Takes an array of urls an creates a simple URLClassLoader with it + */ +class PluginClassLoader(urls: Array[URL]) extends URLClassLoader(urls, PluginClassLoader.platformClassloader) { + // Note the platform classloader in the constructor of the URLClassLoader as the parent. + // That way the classloader skips the app classloader with the classpath when it is asks it's parents for classes. + + protected[helper] override def addURL(url: URL): Unit = super.addURL(url) // just exposes this method to be package-private instead of class internal protected + + override def loadClass(name: String, resolve: Boolean): Class[_] = { + if (name.startsWith("org.codeoverflow.chatoverflow.api")) { + PluginClassLoader.appClassloader.loadClass(name) // Api needs to be loaded from the classpath + } else { + super.loadClass(name, resolve) // non api class. load it as normal + } + } +} + /** - * This plugin class loader does only exist for plugin security policy checks. - * - * @param urls Takes an array of urls an creates a simple URLClassLoader with it - */ -class PluginClassLoader(urls: Array[URL]) extends URLClassLoader(urls) \ No newline at end of file + * This companion object holds references to the app classloader (normal classloader, includes java and classpath) + * and to the extension/platform classloader depending on the java version that excludes the classpath, + * but still includes everything from java. + */ +private object PluginClassLoader extends WithLogger { + val appClassloader: ClassLoader = this.getClass.getClassLoader + val platformClassloader: ClassLoader = { + var current = appClassloader + while (current != null && !current.getClass.getName.contains("ExtClassLoader") && // ExtClassLoader is java < 9 + !current.getClass.getName.contains("PlatformClassLoader")) { // PlatformClassLoader is java >= 9 + current = current.getParent + } + + if (current != null) { + current + } else { + logger error "Platform classloader couldn't be found. Falling back to normal app classloader." + appClassloader + } + } +} \ No newline at end of file diff --git a/src/main/scala/org/codeoverflow/chatoverflow/framework/helper/PluginLoader.scala b/src/main/scala/org/codeoverflow/chatoverflow/framework/helper/PluginLoader.scala index 83fd5cb7..c31474e1 100644 --- a/src/main/scala/org/codeoverflow/chatoverflow/framework/helper/PluginLoader.scala +++ b/src/main/scala/org/codeoverflow/chatoverflow/framework/helper/PluginLoader.scala @@ -10,6 +10,9 @@ import org.reflections.scanners.SubTypesScanner import org.reflections.util.ConfigurationBuilder import scala.collection.JavaConverters._ +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future +import scala.util.{Failure, Success} import scala.xml.{Node, SAXParseException, XML} /** @@ -80,7 +83,8 @@ class PluginLoader(private val jar: File) extends WithLogger { majorString.toInt, minorString.toInt, PluginMetadata.fromXML(p), - cls + cls, + resolveDependencies(getString(p, "name")) )) } catch { // thrown by getString @@ -144,4 +148,29 @@ class PluginLoader(private val jar: File) extends WithLogger { None } } + + /** + * Creates a future which gets all dependencies from the included dependencies.pom, if existing, fetches them + * and adds their jar files to the classloader. + * + * @param pluginName the name of the plugin, only used for logging + * @return a future of all required jars for this plugin + */ + private def resolveDependencies(pluginName: String): Future[Seq[File]] = { + val pomIs = classloader.getResourceAsStream("dependencies.pom") + if (pomIs == null) { + return Future(Seq()) + } + + Future(CoursierUtils.parsePom(pomIs)) + .map(dependencies => dependencies.filter(_.module.name.value != "chatoverflow-api_2.12")) + .map(dependencies => CoursierUtils.fetchDependencies(dependencies)) + .andThen { + case Success(jarFiles) => + jarFiles.foreach(jar => classloader.addURL(jar.toURI.toURL)) + logger info s"Dependencies for the plugin $pluginName successfully resolved and fetched if missing." + case Failure(exception) => + logger warn s"Couldn't resolve and fetch dependencies for the plugin in $pluginName: $exception" + } + } } diff --git a/src/main/scala/org/codeoverflow/chatoverflow/instance/PluginInstance.scala b/src/main/scala/org/codeoverflow/chatoverflow/instance/PluginInstance.scala index cc03dbe4..17fc6a09 100644 --- a/src/main/scala/org/codeoverflow/chatoverflow/instance/PluginInstance.scala +++ b/src/main/scala/org/codeoverflow/chatoverflow/instance/PluginInstance.scala @@ -99,6 +99,11 @@ class PluginInstance(val instanceName: String, pluginType: PluginType) extends W } else { + if (!areDependenciesAvailable) { + logger error "Dependencies have either failed to resolve and fetch or aren't done yet." + return false + } + // This is set to false if any connector (aka input/output) is not ready. var allConnectorsReady = true @@ -224,6 +229,16 @@ class PluginInstance(val instanceName: String, pluginType: PluginType) extends W } } + /** + * Returns whether all dependencies are resolved and fetched which is required for the plugin to start. + * + * @return true if all dependencies are available, false if it has failed or aren't downloaded yet. + */ + def areDependenciesAvailable: Boolean = { + val opt = pluginType.getDependencyFuture.value + opt.isDefined && opt.get.isSuccess + } + /** * Returns if the plugin is currently executed (the thread is running) * diff --git a/src/main/scala/org/codeoverflow/chatoverflow/ui/web/rest/plugin/PluginInstanceController.scala b/src/main/scala/org/codeoverflow/chatoverflow/ui/web/rest/plugin/PluginInstanceController.scala index 5f706896..4d125139 100644 --- a/src/main/scala/org/codeoverflow/chatoverflow/ui/web/rest/plugin/PluginInstanceController.scala +++ b/src/main/scala/org/codeoverflow/chatoverflow/ui/web/rest/plugin/PluginInstanceController.scala @@ -46,6 +46,9 @@ class PluginInstanceController(implicit val swagger: Swagger) extends JsonServle } else if (!pluginInstance.get.getRequirements.getAccess.isComplete) { ResultMessage(success = false, "Not all required requirements have been set.") + } else if (!pluginInstance.get.areDependenciesAvailable) { + ResultMessage(success = false, "Dependencies have either failed to resolve and fetch or aren't done yet. Check logs for further information.") + } else if (!pluginInstance.get.start()) { ResultMessage(success = false, "Unable to start plugin.")