From 7b3f309cbeaafd616c7c60812d03e0a895e0de6e Mon Sep 17 00:00:00 2001 From: Daniel Huber Date: Sun, 19 May 2019 16:06:37 +0200 Subject: [PATCH 1/4] Update dependencyList.txt with new dependencies --- dependencyList.txt | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/dependencyList.txt b/dependencyList.txt index b8749372..9cead14d 100644 --- a/dependencyList.txt +++ b/dependencyList.txt @@ -1,11 +1,19 @@ [info] chatoverflow:chatoverflow_2.12:0.2 [info] chatoverflow-api:chatoverflow-api_2.12:1.0 -[info] com.fasterxml.jackson.core:jackson-annotations:2.8.0 -[info] com.fasterxml.jackson.core:jackson-core:2.8.4 -[info] com.fasterxml.jackson.core:jackson-databind:2.8.4 +[info] club.minnced:opus-java:1.0.4 +[info] club.minnced:opus-java-api:1.0.4 +[info] club.minnced:opus-java-natives:1.0.4 +[info] com.fasterxml.jackson.core:jackson-annotations:2.9.0 +[info] com.fasterxml.jackson.core:jackson-core:2.9.8 +[info] com.fasterxml.jackson.core:jackson-databind:2.9.8 +[info] com.fazecast:jSerialComm:2.5.1 [info] com.github.scopt:scopt_2.12:3.5.0 +[info] com.google.code.findbugs:jsr305:3.0.2 [info] com.google.guava:guava:20.0 [info] com.googlecode.juniversalchardet:juniversalchardet:1.0.3 +[info] com.neovisionaries:nv-websocket-client:2.5 +[info] com.squareup.okhttp3:okhttp:3.13.0 +[info] com.squareup.okio:okio:1.17.2 [info] com.thoughtworks.paranamer:paranamer:2.8 [info] com.typesafe:config:1.3.3 [info] com.typesafe.akka:akka-actor_2.12:2.5.18 @@ -16,6 +24,10 @@ [info] log4j:log4j:1.2.17 [info] net.bytebuddy:byte-buddy:1.6.11 [info] net.bytebuddy:byte-buddy-agent:1.6.11 +[info] net.dv8tion:JDA:4.ALPHA.0_82 +[info] net.java.dev.jna:jna:4.4.0 +[info] net.sf.trove4j:trove4j:3.0.3 +[info] org.apache.commons:commons-collections4:4.1 [info] org.apache.commons:commons-lang3:3.6 [info] org.apache.httpcomponents:httpclient:4.5.3 [info] org.apache.httpcomponents:httpcore:4.4.6 @@ -30,6 +42,7 @@ [info] org.eclipse.jetty:jetty-xml:9.4.6.v20170531 [info] org.hamcrest:hamcrest-core:1.3 [info] org.javassist:javassist:3.21.0-GA +[info] org.jetbrains:annotations:16.0.1 [info] org.json4s:json4s-ast_2.12:3.5.2 [info] org.json4s:json4s-core_2.12:3.5.2 [info] org.json4s:json4s-jackson_2.12:3.5.2 From e79be5e4d82a6fbd6bf4c5087e5a060e2c966db0 Mon Sep 17 00:00:00 2001 From: Daniel Huber Date: Thu, 23 May 2019 17:48:35 +0200 Subject: [PATCH 2/4] Dynamically resolve dependencies urls for Bootstrap Launcher The Bootstrap Launcher was designed to download anything from Maven Central. But since commit 97e67cb has added JDA for Discord the Bootstrap Launcher was not able to download it because it along with some of its dependencies are only hosted on JCenter. This commit adds the ability to resolve dependencies from other repos, but currently only JCenter and Maven Central are used. It still tries to get anything from Maven Central and only uses JCenter, or any other repo that may be used in the future, as a fallback if Maven Central doesn't have a dependency. --- project/Dependency.scala | 41 ++---------- project/DependencyResolver.scala | 106 +++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 36 deletions(-) create mode 100644 project/DependencyResolver.scala diff --git a/project/Dependency.scala b/project/Dependency.scala index 50e923bc..d299d330 100644 --- a/project/Dependency.scala +++ b/project/Dependency.scala @@ -1,5 +1,3 @@ -import java.net.{HttpURLConnection, URL} - import sbt.internal.util.ManagedLogger import scala.xml.Node @@ -46,7 +44,6 @@ class Dependency(dependencyString: String, logger: ManagedLogger) { */ private def create(): Unit = { val DependencyRegex = "([^:]+):([^:_]+)(_[^:]+)?:([^:]+)".r - val mavenCentralFormat = "http://central.maven.org/maven2/%s/%s/%s/%s.jar" dependencyString match { case DependencyRegex(depAuthor, depName, scalaVersion, depVersion) => @@ -54,45 +51,17 @@ class Dependency(dependencyString: String, logger: ManagedLogger) { this.version = depVersion val combinedName = if (scalaVersion != null) depName + scalaVersion else depName + val depTuple = (depAuthor, combinedName, depVersion) - // Create URL for maven central - url = mavenCentralFormat.format(depAuthor.replaceAll("\\.", "/"), s"$combinedName", - depVersion, s"$combinedName-$depVersion") + val status = DependencyResolver.resolve(depTuple, logger) - available = testURL(0, 3) + // Provide the url of the default repo if the dependency couldn't be resolved + url = status.getOrElse(DependencyResolver.getDefaultUrl(depTuple)) + available = status.isDefined case _ => logger warn s"Invalid dependency format: '$dependencyString'." } } - /** - * Tests, if the dependency url is available. Uses recursion to handle connection faults. - */ - private def testURL(recursionCount: Int, recursionLimit: Int): Boolean = { - - var status = -1 - - try { - - // Test if the url exists - val connection = new URL(url).openConnection.asInstanceOf[HttpURLConnection] - connection.setRequestMethod("HEAD") - connection.setConnectTimeout(200) - connection.setReadTimeout(200) - status = connection.getResponseCode - connection.disconnect() - - } catch { - case e: Exception => logger warn s"Error while testing dependency (attempt $recursionCount of $recursionLimit)" + - s" availability of ${this}: ${e.getMessage}" - } - - if (status != 200 && recursionCount + 1 <= recursionLimit) { - testURL(recursionCount + 1, recursionLimit) - } else { - status == 200 - } - } - } diff --git a/project/DependencyResolver.scala b/project/DependencyResolver.scala new file mode 100644 index 00000000..6df0f652 --- /dev/null +++ b/project/DependencyResolver.scala @@ -0,0 +1,106 @@ +import java.net.{HttpURLConnection, URL} + +import sbt.internal.util.ManagedLogger + +/** + * Holds logic to resolve dependencies with their author, name and version. + * To be used by the BootstrapUtility to get urls for dependencies from dependencyList.txt + */ +object DependencyResolver { + // Contains all repos with descending priority. + // All urls MUST have the trailing slash. + private val repos = Set( + "http://central.maven.org/maven2/", // used by almost anything + "http://jcenter.bintray.com/" // used by JDA and its dependencies like opus-java + ) + + /** + * Resolves the given dependency into a url, where it can be downloaded. + * Uses the private "repos" constant to determine possible urls. + * + * @param dependency the dependency to resolve. The tuple needs to consist of following + * 1. the author or organisation + * 2. the name with the scala version if needed + * 3. the version + * @param logger the sbt logger for exception warnings + * @return the resolved url if everything went well. + * In case of an error that persists after 3 tries + * the Option is empty and the error message has been logged. + */ + def resolve(dependency: (String, String, String), logger: ManagedLogger): Option[String] = { + for (repo <- repos) { + val url = buildUrl(dependency, repo) + val status = testURL(url, 1, 3) + + status match { + // Repo has produced 3 errors in a row! Give up, report error and move on to the next repo + case Left(e) => logger warn s"Error while testing dependency ${dependency.productIterator.mkString(":")}" + + s" availability at $url: ${e.getMessage}" + + // Repo returned with a 200 OK, the url is correct and we don't need to check the following repos + // If found is false e.g. by 404 Not Found we check the next repo for this dependency + case Right(found) => if (found) return Some(url) + } + } + + None // No repo has this dependency, can't resolve it + } + + /** + * Generates a default url for the provided dependency using the primary/first repo (Maven Central). + * It doesn't check whether this url actually works and is designed to be used as a fallback url. + * + * @param dependency the dependency for which to get the default url + * @return the default url of this dependency + */ + def getDefaultUrl(dependency: (String, String, String)): String = buildUrl(dependency, repos.head) + + /** + * Builds the url from the repo base url and the dependency tuple. + * + * @param dependency the dependency to convert to a url and append to the repo root + * @param repo the repo root with a trailing slash + * @return the fully build url + */ + private def buildUrl(dependency: (String, String, String), repo: String): String = { + val (author, name, version) = dependency + + s"$repo${author.replaceAll("\\.", "/")}/$name/$version/$name-$version.jar" + } + + /** + * Tests if the given url is available and returns 200 OK as http status code. + * Tries multiple times using recursion if a error occurs. Maximal retry count depends on recursionLimit. + * + * @param url the url to be checked + * @param recursionCount current retry count, should be 1 if called from anything else than itself + * @param recursionLimit maximal retries + * @return Either a exception in case a exception is thrown and the recursionLimit is reached + * or a boolean if no exception was thrown. + * True if the http status code was 200 and false otherwise. + */ + private def testURL(url: String, recursionCount: Int, recursionLimit: Int): Either[Exception, Boolean] = { + var status = -1 + + try { + + // Test if the url exists + val connection = new URL(url).openConnection.asInstanceOf[HttpURLConnection] + connection.setRequestMethod("HEAD") + connection.setConnectTimeout(400) // JCenter is very slow with 404s for some reason + connection.setReadTimeout(400) + status = connection.getResponseCode + connection.disconnect() + + } catch { + case e: Exception => + return if (recursionCount < recursionLimit) { + testURL(url, recursionCount + 1, recursionLimit) + } else { + Left(e) + } + } + + Right(status == 200) + } +} From 883736f57d983a64b8cd1247dc8ea10e739edcc0 Mon Sep 17 00:00:00 2001 From: Daniel Huber Date: Thu, 23 May 2019 18:24:17 +0200 Subject: [PATCH 3/4] Exclude opus-java from the Bootstrap dependency xml and close dependencyList.txt source --- project/BootstrapUtility.scala | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/project/BootstrapUtility.scala b/project/BootstrapUtility.scala index 2466aaa9..2c3be4f4 100644 --- a/project/BootstrapUtility.scala +++ b/project/BootstrapUtility.scala @@ -65,7 +65,9 @@ object BootstrapUtility { logger info "Found dependency file." // Load file, remove the info tag and create dependency objects - val input = scala.io.Source.fromFile(dependencyFile).getLines().toList + val dependencySource = scala.io.Source.fromFile(dependencyFile) + val input = dependencySource.getLines().toList + dependencySource.close() val lines = input.map(line => line.replaceFirst("\\[info\\] ", "")) logger info "Read dependencies successfully. Creating dependency list." @@ -74,10 +76,15 @@ object BootstrapUtility { logger info "Updating and modifying dependencies..." - // Modify dependencies: Remove ChatOverflow, add scala library - val depsWithoutChatOverflow = dependencies.filter(d => - d.nameWithoutScalaVersion != "chatoverflow" && d.nameWithoutScalaVersion != "chatoverflow-api") - val modifiedDependencies = depsWithoutChatOverflow ++ + // Modify dependencies: Remove ChatOverflow and opus-java, add scala library + // opus-java is a virtual package which instructs sbt or any other build tool to get opus-java-api and opus-java-native. + // Therefore it doesn't have a jar that needs to be downloaded and sbt includes the requested dependencies in the dependencyList. + // So we can just ignore it as it can't be resolved and only need to include the requested deps in our xml. + // Check https://github.com/discord-java/opus-java#opus-java-1 for more information on this. + val excludedDeps = List("chatoverflow", "chatoverflow-api", "opus-java") + val filteredDeps = dependencies.filter(d => !excludedDeps.contains(d.nameWithoutScalaVersion)) + + val modifiedDependencies = filteredDeps ++ List(new Dependency(s"org.scala-lang:scala-library:$scalaLibraryVersion", logger)) // Info output From 1388ee9cd432af23755e77b4fcf6446fd7bf7646 Mon Sep 17 00:00:00 2001 From: Daniel Huber Date: Fri, 24 May 2019 16:18:47 +0200 Subject: [PATCH 4/4] Get rid of dependencyList.txt and get dependencies directly from a sbt api --- .gitignore | 1 - Makefile | 1 - bootstrap/src/main/resources/dependencies.xml | 633 ------------------ build.sbt | 14 +- dependencyList.txt | 77 --- project/BootstrapUtility.scala | 69 +- 6 files changed, 41 insertions(+), 754 deletions(-) delete mode 100644 bootstrap/src/main/resources/dependencies.xml delete mode 100644 dependencyList.txt diff --git a/.gitignore b/.gitignore index 74253c4d..53ac882c 100644 --- a/.gitignore +++ b/.gitignore @@ -28,7 +28,6 @@ project/plugins/project/ /config/ # resources of the bootstrap project -/dependencyList.txt /bootstrap/src/main/resources/ /bin/ /lib/ diff --git a/Makefile b/Makefile index 5ca3c6ca..3fcec022 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,6 @@ simple_run: sbt package copy bootstrap_deploy: - sbt dependencyList | cut -d " " -f2 | grep ":" | sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[mGK]//g" | uniq > dependencyList.txt sbt compile sbt clean sbt package copy diff --git a/bootstrap/src/main/resources/dependencies.xml b/bootstrap/src/main/resources/dependencies.xml deleted file mode 100644 index 88ac482e..00000000 --- a/bootstrap/src/main/resources/dependencies.xml +++ /dev/null @@ -1,633 +0,0 @@ - - - - jackson-annotations - - - 2.8.0 - - - http://central.maven.org/maven2/com/fasterxml/jackson/core/jackson-annotations/2.8.0/jackson-annotations-2.8.0.jar - - - - jackson-core - - - 2.8.4 - - - http://central.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.8.4/jackson-core-2.8.4.jar - - - - jackson-databind - - - 2.8.4 - - - http://central.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/2.8.4/jackson-databind-2.8.4.jar - - - - scopt - - - 3.5.0 - - - http://central.maven.org/maven2/com/github/scopt/scopt_2.12/3.5.0/scopt_2.12-3.5.0.jar - - - - guava - - - 20.0 - - - http://central.maven.org/maven2/com/google/guava/guava/20.0/guava-20.0.jar - - - - juniversalchardet - - - 1.0.3 - - - http://central.maven.org/maven2/com/googlecode/juniversalchardet/juniversalchardet/1.0.3/juniversalchardet-1.0.3.jar - - - - paranamer - - - 2.8 - - - http://central.maven.org/maven2/com/thoughtworks/paranamer/paranamer/2.8/paranamer-2.8.jar - - - - config - - - 1.3.3 - - - http://central.maven.org/maven2/com/typesafe/config/1.3.3/config-1.3.3.jar - - - - akka-actor - - - 2.5.18 - - - http://central.maven.org/maven2/com/typesafe/akka/akka-actor_2.12/2.5.18/akka-actor_2.12-2.5.18.jar - - - - commons-codec - - - 1.10 - - - http://central.maven.org/maven2/commons-codec/commons-codec/1.10/commons-codec-1.10.jar - - - - commons-logging - - - 1.2 - - - http://central.maven.org/maven2/commons-logging/commons-logging/1.2/commons-logging-1.2.jar - - - - mime-util - - - 2.1.3 - - - http://central.maven.org/maven2/eu/medsea/mimeutil/mime-util/2.1.3/mime-util-2.1.3.jar - - - - javax.servlet-api - - - 3.1.0 - - - http://central.maven.org/maven2/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar - - - - log4j - - - 1.2.17 - - - http://central.maven.org/maven2/log4j/log4j/1.2.17/log4j-1.2.17.jar - - - - byte-buddy - - - 1.6.11 - - - http://central.maven.org/maven2/net/bytebuddy/byte-buddy/1.6.11/byte-buddy-1.6.11.jar - - - - byte-buddy-agent - - - 1.6.11 - - - http://central.maven.org/maven2/net/bytebuddy/byte-buddy-agent/1.6.11/byte-buddy-agent-1.6.11.jar - - - - commons-lang3 - - - 3.6 - - - http://central.maven.org/maven2/org/apache/commons/commons-lang3/3.6/commons-lang3-3.6.jar - - - - httpclient - - - 4.5.3 - - - http://central.maven.org/maven2/org/apache/httpcomponents/httpclient/4.5.3/httpclient-4.5.3.jar - - - - httpcore - - - 4.4.6 - - - http://central.maven.org/maven2/org/apache/httpcomponents/httpcore/4.4.6/httpcore-4.4.6.jar - - - - httpmime - - - 4.5.3 - - - http://central.maven.org/maven2/org/apache/httpcomponents/httpmime/4.5.3/httpmime-4.5.3.jar - - - - jetty-http - - - 9.4.6.v20170531 - - - http://central.maven.org/maven2/org/eclipse/jetty/jetty-http/9.4.6.v20170531/jetty-http-9.4.6.v20170531.jar - - - - jetty-io - - - 9.4.6.v20170531 - - - http://central.maven.org/maven2/org/eclipse/jetty/jetty-io/9.4.6.v20170531/jetty-io-9.4.6.v20170531.jar - - - - jetty-security - - - 9.4.6.v20170531 - - - http://central.maven.org/maven2/org/eclipse/jetty/jetty-security/9.4.6.v20170531/jetty-security-9.4.6.v20170531.jar - - - - jetty-server - - - 9.4.6.v20170531 - - - http://central.maven.org/maven2/org/eclipse/jetty/jetty-server/9.4.6.v20170531/jetty-server-9.4.6.v20170531.jar - - - - jetty-servlet - - - 9.4.6.v20170531 - - - http://central.maven.org/maven2/org/eclipse/jetty/jetty-servlet/9.4.6.v20170531/jetty-servlet-9.4.6.v20170531.jar - - - - jetty-util - - - 9.4.6.v20170531 - - - http://central.maven.org/maven2/org/eclipse/jetty/jetty-util/9.4.6.v20170531/jetty-util-9.4.6.v20170531.jar - - - - jetty-webapp - - - 9.4.6.v20170531 - - - http://central.maven.org/maven2/org/eclipse/jetty/jetty-webapp/9.4.6.v20170531/jetty-webapp-9.4.6.v20170531.jar - - - - jetty-xml - - - 9.4.6.v20170531 - - - http://central.maven.org/maven2/org/eclipse/jetty/jetty-xml/9.4.6.v20170531/jetty-xml-9.4.6.v20170531.jar - - - - hamcrest-core - - - 1.3 - - - http://central.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar - - - - javassist - - - 3.21.0-GA - - - http://central.maven.org/maven2/org/javassist/javassist/3.21.0-GA/javassist-3.21.0-GA.jar - - - - json4s-ast - - - 3.5.2 - - - http://central.maven.org/maven2/org/json4s/json4s-ast_2.12/3.5.2/json4s-ast_2.12-3.5.2.jar - - - - json4s-core - - - 3.5.2 - - - http://central.maven.org/maven2/org/json4s/json4s-core_2.12/3.5.2/json4s-core_2.12-3.5.2.jar - - - - json4s-jackson - - - 3.5.2 - - - http://central.maven.org/maven2/org/json4s/json4s-jackson_2.12/3.5.2/json4s-jackson_2.12-3.5.2.jar - - - - json4s-scalap - - - 3.5.2 - - - http://central.maven.org/maven2/org/json4s/json4s-scalap_2.12/3.5.2/json4s-scalap_2.12-3.5.2.jar - - - - mockito-core - - - 2.7.22 - - - http://central.maven.org/maven2/org/mockito/mockito-core/2.7.22/mockito-core-2.7.22.jar - - - - objenesis - - - 2.5 - - - http://central.maven.org/maven2/org/objenesis/objenesis/2.5/objenesis-2.5.jar - - - - pircbotx - - - 2.1 - - - http://central.maven.org/maven2/org/pircbotx/pircbotx/2.1/pircbotx-2.1.jar - - - - reflections - - - 0.9.11 - - - http://central.maven.org/maven2/org/reflections/reflections/0.9.11/reflections-0.9.11.jar - - - - scala-compiler - - - 2.12.5 - - - http://central.maven.org/maven2/org/scala-lang/scala-compiler/2.12.5/scala-compiler-2.12.5.jar - - - - scala-reflect - - - 2.12.5 - - - http://central.maven.org/maven2/org/scala-lang/scala-reflect/2.12.5/scala-reflect-2.12.5.jar - - - - scala-java8-compat - - - 0.8.0 - - - http://central.maven.org/maven2/org/scala-lang/modules/scala-java8-compat_2.12/0.8.0/scala-java8-compat_2.12-0.8.0.jar - - - - scala-parser-combinators - - - 1.0.6 - - - http://central.maven.org/maven2/org/scala-lang/modules/scala-parser-combinators_2.12/1.0.6/scala-parser-combinators_2.12-1.0.6.jar - - - - scala-xml - - - 1.0.6 - - - http://central.maven.org/maven2/org/scala-lang/modules/scala-xml_2.12/1.0.6/scala-xml_2.12-1.0.6.jar - - - - test-interface - - - 1.0 - - - http://central.maven.org/maven2/org/scala-sbt/test-interface/1.0/test-interface-1.0.jar - - - - scalatra-common - - - 2.6.5 - - - http://central.maven.org/maven2/org/scalatra/scalatra-common_2.12/2.6.5/scalatra-common_2.12-2.6.5.jar - - - - scalatra-json - - - 2.6.3 - - - http://central.maven.org/maven2/org/scalatra/scalatra-json_2.12/2.6.3/scalatra-json_2.12-2.6.3.jar - - - - scalatra-scalate - - - 2.6.5 - - - http://central.maven.org/maven2/org/scalatra/scalatra-scalate_2.12/2.6.5/scalatra-scalate_2.12-2.6.5.jar - - - - scalatra-specs2 - - - 2.6.5 - - - http://central.maven.org/maven2/org/scalatra/scalatra-specs2_2.12/2.6.5/scalatra-specs2_2.12-2.6.5.jar - - - - scalatra-test - - - 2.6.5 - - - http://central.maven.org/maven2/org/scalatra/scalatra-test_2.12/2.6.5/scalatra-test_2.12-2.6.5.jar - - - - scalatra - - - 2.6.5 - - - http://central.maven.org/maven2/org/scalatra/scalatra_2.12/2.6.5/scalatra_2.12-2.6.5.jar - - - - scalate-core - - - 1.8.0 - - - http://central.maven.org/maven2/org/scalatra/scalate/scalate-core_2.12/1.8.0/scalate-core_2.12-1.8.0.jar - - - - scalate-util - - - 1.8.0 - - - http://central.maven.org/maven2/org/scalatra/scalate/scalate-util_2.12/1.8.0/scalate-util_2.12-1.8.0.jar - - - - slf4j-api - - - 1.7.25 - - - http://central.maven.org/maven2/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar - - - - slf4j-log4j12 - - - 1.7.22 - - - http://central.maven.org/maven2/org/slf4j/slf4j-log4j12/1.7.22/slf4j-log4j12-1.7.22.jar - - - - classycle - - - 1.4.3 - - - http://central.maven.org/maven2/org/specs2/classycle/1.4.3/classycle-1.4.3.jar - - - - specs2-analysis - - - 4.0.1 - - - http://central.maven.org/maven2/org/specs2/specs2-analysis_2.12/4.0.1/specs2-analysis_2.12-4.0.1.jar - - - - specs2-common - - - 4.0.1 - - - http://central.maven.org/maven2/org/specs2/specs2-common_2.12/4.0.1/specs2-common_2.12-4.0.1.jar - - - - specs2-core - - - 4.0.1 - - - http://central.maven.org/maven2/org/specs2/specs2-core_2.12/4.0.1/specs2-core_2.12-4.0.1.jar - - - - specs2-fp - - - 4.0.1 - - - http://central.maven.org/maven2/org/specs2/specs2-fp_2.12/4.0.1/specs2-fp_2.12-4.0.1.jar - - - - specs2-matcher-extra - - - 4.0.1 - - - http://central.maven.org/maven2/org/specs2/specs2-matcher-extra_2.12/4.0.1/specs2-matcher-extra_2.12-4.0.1.jar - - - - specs2-matcher - - - 4.0.1 - - - http://central.maven.org/maven2/org/specs2/specs2-matcher_2.12/4.0.1/specs2-matcher_2.12-4.0.1.jar - - - - specs2-mock - - - 4.0.1 - - - http://central.maven.org/maven2/org/specs2/specs2-mock_2.12/4.0.1/specs2-mock_2.12-4.0.1.jar - - - - scala-library - - - 2.12.5 - - - http://central.maven.org/maven2/org/scala-lang/scala-library/2.12.5/scala-library-2.12.5.jar - - - \ No newline at end of file diff --git a/build.sbt b/build.sbt index bdf24932..d9f3a987 100644 --- a/build.sbt +++ b/build.sbt @@ -70,6 +70,18 @@ lazy val pluginFolderNames = settingKey[List[String]]("The folder names of all p lazy val pluginTargetFolderNames = settingKey[List[String]]("The folder names of compiled and packaged plugins. Remember to gitignore these!") lazy val apiProjectPath = settingKey[String]("The path to the api sub project. Remember to gitignore it!") +// Util task for bs, gets a dependency list kinda like "sbt dependencyList", but only includes deps required for runtime +lazy val getDependencyList = Def.task[List[ModuleID]] { + // only get deps required for runtime and not for anything else like testing + val updateReport = update.value.configuration(ConfigRef("runtime")) + + if (updateReport.isEmpty) { + List() + } else { + updateReport.get.modules.map(m => m.module).toList + } +} + // Plugin framework tasks lazy val create = TaskKey[Unit]("create", "Creates a new plugin. Interactive command using the console.") lazy val fetch = TaskKey[Unit]("fetch", "Searches for plugins in plugin directories, builds the plugin build file.") @@ -86,5 +98,5 @@ create := BuildUtility(streams.value.log).createPluginTask(pluginFolderNames.val fetch := BuildUtility(streams.value.log).fetchPluginsTask(pluginFolderNames.value, pluginBuildFileName.value, pluginTargetFolderNames.value, apiProjectPath.value) copy := BuildUtility(streams.value.log).copyPluginsTask(pluginFolderNames.value, pluginTargetFolderNames.value, scalaMajorVersion) -bs := BootstrapUtility.bootstrapGenTask(streams.value.log, s"$scalaMajorVersion$scalaMinorVersion") +bs := BootstrapUtility.bootstrapGenTask(streams.value.log, s"$scalaMajorVersion$scalaMinorVersion", getDependencyList.value) deploy := BootstrapUtility.prepareDeploymentTask(streams.value.log, scalaMajorVersion) \ No newline at end of file diff --git a/dependencyList.txt b/dependencyList.txt deleted file mode 100644 index 9cead14d..00000000 --- a/dependencyList.txt +++ /dev/null @@ -1,77 +0,0 @@ -[info] chatoverflow:chatoverflow_2.12:0.2 -[info] chatoverflow-api:chatoverflow-api_2.12:1.0 -[info] club.minnced:opus-java:1.0.4 -[info] club.minnced:opus-java-api:1.0.4 -[info] club.minnced:opus-java-natives:1.0.4 -[info] com.fasterxml.jackson.core:jackson-annotations:2.9.0 -[info] com.fasterxml.jackson.core:jackson-core:2.9.8 -[info] com.fasterxml.jackson.core:jackson-databind:2.9.8 -[info] com.fazecast:jSerialComm:2.5.1 -[info] com.github.scopt:scopt_2.12:3.5.0 -[info] com.google.code.findbugs:jsr305:3.0.2 -[info] com.google.guava:guava:20.0 -[info] com.googlecode.juniversalchardet:juniversalchardet:1.0.3 -[info] com.neovisionaries:nv-websocket-client:2.5 -[info] com.squareup.okhttp3:okhttp:3.13.0 -[info] com.squareup.okio:okio:1.17.2 -[info] com.thoughtworks.paranamer:paranamer:2.8 -[info] com.typesafe:config:1.3.3 -[info] com.typesafe.akka:akka-actor_2.12:2.5.18 -[info] commons-codec:commons-codec:1.10 -[info] commons-logging:commons-logging:1.2 -[info] eu.medsea.mimeutil:mime-util:2.1.3 -[info] javax.servlet:javax.servlet-api:3.1.0 -[info] log4j:log4j:1.2.17 -[info] net.bytebuddy:byte-buddy:1.6.11 -[info] net.bytebuddy:byte-buddy-agent:1.6.11 -[info] net.dv8tion:JDA:4.ALPHA.0_82 -[info] net.java.dev.jna:jna:4.4.0 -[info] net.sf.trove4j:trove4j:3.0.3 -[info] org.apache.commons:commons-collections4:4.1 -[info] org.apache.commons:commons-lang3:3.6 -[info] org.apache.httpcomponents:httpclient:4.5.3 -[info] org.apache.httpcomponents:httpcore:4.4.6 -[info] org.apache.httpcomponents:httpmime:4.5.3 -[info] org.eclipse.jetty:jetty-http:9.4.6.v20170531 -[info] org.eclipse.jetty:jetty-io:9.4.6.v20170531 -[info] org.eclipse.jetty:jetty-security:9.4.6.v20170531 -[info] org.eclipse.jetty:jetty-server:9.4.6.v20170531 -[info] org.eclipse.jetty:jetty-servlet:9.4.6.v20170531 -[info] org.eclipse.jetty:jetty-util:9.4.6.v20170531 -[info] org.eclipse.jetty:jetty-webapp:9.4.6.v20170531 -[info] org.eclipse.jetty:jetty-xml:9.4.6.v20170531 -[info] org.hamcrest:hamcrest-core:1.3 -[info] org.javassist:javassist:3.21.0-GA -[info] org.jetbrains:annotations:16.0.1 -[info] org.json4s:json4s-ast_2.12:3.5.2 -[info] org.json4s:json4s-core_2.12:3.5.2 -[info] org.json4s:json4s-jackson_2.12:3.5.2 -[info] org.json4s:json4s-scalap_2.12:3.5.2 -[info] org.mockito:mockito-core:2.7.22 -[info] org.objenesis:objenesis:2.5 -[info] org.pircbotx:pircbotx:2.1 -[info] org.reflections:reflections:0.9.11 -[info] org.scala-lang:scala-compiler:2.12.5 -[info] org.scala-lang:scala-reflect:2.12.5 -[info] org.scala-lang.modules:scala-java8-compat_2.12:0.8.0 -[info] org.scala-lang.modules:scala-parser-combinators_2.12:1.0.6 -[info] org.scala-lang.modules:scala-xml_2.12:1.0.6 -[info] org.scala-sbt:test-interface:1.0 -[info] org.scalatra:scalatra-common_2.12:2.6.5 -[info] org.scalatra:scalatra-json_2.12:2.6.3 -[info] org.scalatra:scalatra-scalate_2.12:2.6.5 -[info] org.scalatra:scalatra-specs2_2.12:2.6.5 -[info] org.scalatra:scalatra-test_2.12:2.6.5 -[info] org.scalatra:scalatra_2.12:2.6.5 -[info] org.scalatra.scalate:scalate-core_2.12:1.8.0 -[info] org.scalatra.scalate:scalate-util_2.12:1.8.0 -[info] org.slf4j:slf4j-api:1.7.25 -[info] org.slf4j:slf4j-log4j12:1.7.22 -[info] org.specs2:classycle:1.4.3 -[info] org.specs2:specs2-analysis_2.12:4.0.1 -[info] org.specs2:specs2-common_2.12:4.0.1 -[info] org.specs2:specs2-core_2.12:4.0.1 -[info] org.specs2:specs2-fp_2.12:4.0.1 -[info] org.specs2:specs2-matcher-extra_2.12:4.0.1 -[info] org.specs2:specs2-matcher_2.12:4.0.1 -[info] org.specs2:specs2-mock_2.12:4.0.1 \ No newline at end of file diff --git a/project/BootstrapUtility.scala b/project/BootstrapUtility.scala index 2c3be4f4..67044951 100644 --- a/project/BootstrapUtility.scala +++ b/project/BootstrapUtility.scala @@ -3,6 +3,7 @@ import java.nio.file.{Files, Paths} import BuildUtility.withTaskInfo import sbt.internal.util.ManagedLogger +import sbt.librarymanagement.ModuleID import scala.xml.XML @@ -11,7 +12,6 @@ import scala.xml.XML * Should be used once for every new public version of chat overflow. */ object BootstrapUtility { - val dependencyListFileName = "dependencyList.txt" val dependencyProjectBasePath = "bootstrap/src/main" val dependencyXMLFileName = s"$dependencyProjectBasePath/resources/dependencies.xml" @@ -21,11 +21,11 @@ object BootstrapUtility { * @param logger the sbt logger * @param scalaLibraryVersion the current scala library version */ - def bootstrapGenTask(logger: ManagedLogger, scalaLibraryVersion: String): Unit = { + def bootstrapGenTask(logger: ManagedLogger, scalaLibraryVersion: String, modules: List[ModuleID]): Unit = { withTaskInfo("BOOTSTRAP GENERATION", logger) { // Dependency management - val dependencyList = retrieveDependencies(logger, scalaLibraryVersion) + val dependencyList = retrieveDependencies(logger, scalaLibraryVersion, modules) saveDependencyXML(dependencyList, logger) } } @@ -48,54 +48,41 @@ object BootstrapUtility { } /** - * Uses a file called 'dependencyList.txt' with the output of the - * sbt command 'dependencyList' to retrieve all dependencies. + * Converts passed modules to a list of Dependency, resolves them to urls, filters out codeoverflow + * and adds the scala library to the list. */ - private def retrieveDependencies(logger: ManagedLogger, scalaLibraryVersion: String): List[Dependency] = { + private def retrieveDependencies(logger: ManagedLogger, scalaLibraryVersion: String, modules: List[ModuleID]): List[Dependency] = { logger info "Starting dependency retrieval." - val dependencyFile = new File(dependencyListFileName) + logger info "Creating dependency list and resolve dependencies." - if (!dependencyFile.exists()) { - logger error "No dependency file found. Please copy the output of the task 'dependencyList' into a file named 'dependencyList.txt' in the root folder." - List[Dependency]() - } else { - - logger info "Found dependency file." - - // Load file, remove the info tag and create dependency objects - val dependencySource = scala.io.Source.fromFile(dependencyFile) - val input = dependencySource.getLines().toList - dependencySource.close() - val lines = input.map(line => line.replaceFirst("\\[info\\] ", "")) - - logger info "Read dependencies successfully. Creating dependency list." - - val dependencies = for (line <- lines) yield new Dependency(line, logger) - - logger info "Updating and modifying dependencies..." + val dependencyList = for (dep <- modules) yield { + val dependencyString = s"${dep.organization}:${dep.name}:${dep.revision}" + new Dependency(dependencyString, logger) + } - // Modify dependencies: Remove ChatOverflow and opus-java, add scala library - // opus-java is a virtual package which instructs sbt or any other build tool to get opus-java-api and opus-java-native. - // Therefore it doesn't have a jar that needs to be downloaded and sbt includes the requested dependencies in the dependencyList. - // So we can just ignore it as it can't be resolved and only need to include the requested deps in our xml. - // Check https://github.com/discord-java/opus-java#opus-java-1 for more information on this. - val excludedDeps = List("chatoverflow", "chatoverflow-api", "opus-java") - val filteredDeps = dependencies.filter(d => !excludedDeps.contains(d.nameWithoutScalaVersion)) + logger info "Updating and modifying dependencies..." - val modifiedDependencies = filteredDeps ++ - List(new Dependency(s"org.scala-lang:scala-library:$scalaLibraryVersion", logger)) + // Modify dependencies: Remove ChatOverflow and opus-java, add scala library + // opus-java is a virtual package which instructs sbt or any other build tool to get opus-java-api and opus-java-native. + // Therefore it doesn't have a jar that needs to be downloaded and sbt includes the requested dependencies in the dependencyList. + // So we can just ignore it as it can't be resolved and only need to include the requested deps in our xml. + // Check https://github.com/discord-java/opus-java#opus-java-1 for more information on this. + val excludedDeps = List("chatoverflow", "chatoverflow-api", "opus-java") + val filteredDeps = dependencyList.filter(d => !excludedDeps.contains(d.nameWithoutScalaVersion)) - // Info output - logger info s"Found ${modifiedDependencies.length} dependencies." - if (modifiedDependencies.exists(d => !d.available)) { - logger warn "Found the following dependencies, that could not be retrieved online:" - logger warn modifiedDependencies.filter(d => !d.available).map(_.toString).mkString("\n") - } + val modifiedDependencies = filteredDeps ++ + List(new Dependency(s"org.scala-lang:scala-library:$scalaLibraryVersion", logger)) - modifiedDependencies + // Info output + logger info s"Found ${modifiedDependencies.length} dependencies." + if (modifiedDependencies.exists(d => !d.available)) { + logger warn "Found the following dependencies, that could not be retrieved online:" + logger warn modifiedDependencies.filter(d => !d.available).map(_.toString).mkString("\n") } + + modifiedDependencies } /**