From 1911f3ea77d01a2453882053810afc54f7167bdf Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Mon, 12 Aug 2024 18:02:47 +0200 Subject: [PATCH 1/3] Remove the need for the uber jar Signed-off-by: Fabio Di Fabio --- arithmetization/build.gradle | 24 +++---- gradle.properties | 2 +- gradle/dependency-management.gradle | 104 ++++++++++++++++------------ gradle/dist.gradle | 9 --- gradle/versions.gradle | 1 - reference-tests/build.gradle | 19 ++--- 6 files changed, 81 insertions(+), 78 deletions(-) delete mode 100644 gradle/versions.gradle diff --git a/arithmetization/build.gradle b/arithmetization/build.gradle index abbab084d..1f9f1870b 100644 --- a/arithmetization/build.gradle +++ b/arithmetization/build.gradle @@ -36,21 +36,21 @@ apply from: rootProject.file("gradle/trace-files.gradle") dependencies { // annotationProcessor generates the file META-INF/services/org.hyperledger.besu.plugin.BesuPlugin annotationProcessor 'com.google.auto.service:auto-service' - compileOnly "${besuArtifactGroup}:evm" - compileOnly 'com.google.auto.service:auto-service' - compileOnly 'com.google.auto.service:auto-service-annotations' + implementation 'com.google.auto.service:auto-service' + implementation 'com.google.auto.service:auto-service-annotations' + implementation "${besuArtifactGroup}:besu-datatypes" implementation "${besuArtifactGroup}:evm" implementation "${besuArtifactGroup}:plugin-api" - implementation "${besuArtifactGroup}:besu-datatypes" - implementation "${besuArtifactGroup}.internal:api:${besuVersion}" - implementation "${besuArtifactGroup}.internal:core:${besuVersion}" - implementation "${besuArtifactGroup}.internal:rlp:${besuVersion}" - implementation "${besuArtifactGroup}.internal:algorithms:${besuVersion}" + implementation "${besuArtifactGroup}.internal:algorithms" + implementation "${besuArtifactGroup}.internal:api" + implementation "${besuArtifactGroup}.internal:core" + implementation "${besuArtifactGroup}.internal:rlp" + implementation 'info.picocli:picocli' - compileOnly 'io.vertx:vertx-core' + implementation 'io.vertx:vertx-core' implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' @@ -60,12 +60,6 @@ dependencies { implementation 'io.tmio:tuweni-toml' implementation 'org.bouncycastle:bcprov-jdk18on' implementation 'org.hibernate.validator:hibernate-validator' - - testImplementation "${besuArtifactGroup}:evm" - testImplementation "${besuArtifactGroup}:besu-datatypes" - testImplementation "${besuArtifactGroup}.internal:core:${besuVersion}" - testImplementation "${besuArtifactGroup}.internal:rlp:${besuVersion}" - testImplementation "${besuArtifactGroup}.internal:core:${besuVersion}" } configurations { diff --git a/gradle.properties b/gradle.properties index 660d7dbcc..179e2d83d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -releaseVersion=0.3.0-rc2 +releaseVersion=0.3.0-rc2-local besuVersion=24.7-develop-c0029e6 besuArtifactGroup=io.consensys.linea-besu distributionIdentifier=linea-arithmetization diff --git a/gradle/dependency-management.gradle b/gradle/dependency-management.gradle index 020a032a7..ec5ae02b9 100644 --- a/gradle/dependency-management.gradle +++ b/gradle/dependency-management.gradle @@ -31,6 +31,7 @@ repositories { content { includeGroupByRegex('com\\.splunk\\..*') } } mavenCentral() + mavenLocal() } configurations.all { @@ -48,80 +49,97 @@ dependencyManagement { } dependencies { - dependencySet(group: 'io.tmio', version: '2.4.2') { - entry 'tuweni-bytes' - entry 'tuweni-net' - entry 'tuweni-toml' - entry 'tuweni-units' + // Besu dependencies + dependencySet(group: "${besuArtifactGroup}", version: "${besuVersion}") { + entry "besu-datatypes" + entry "evm" + entry "plugin-api" } - dependencySet(group: 'org.bouncycastle', version: '1.75') { - entry 'bcpkix-jdk18on' - entry 'bcprov-jdk18on' - entry 'jcajce' + // Besu internal dependencies + dependencySet(group: "${besuArtifactGroup}.internal", version: "${besuVersion}") { + entry "algorithms" + entry "api" + entry "config" + entry "core" + entry "dsl" + entry "eth" + entry "metrics-core" + entry "referencetests" + entry "rlp" + entry "testutil" + entry "trie" + entry "util" } + dependency 'commons-io:commons-io:2.11.0' + dependency 'com.fasterxml.jackson.core:jackson-databind:2.15.2' dependency 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.15.2' dependency 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.15.2' - dependency 'commons-io:commons-io:2.11.0' - dependency 'org.apache.commons:commons-lang3:3.13.0' - dependency 'com.google.guava:guava:32.1.2-jre' - - // Besu dependencies + dependency 'com.google.auto.service:auto-service:1.0.1' - dependencySet(group: "${besuArtifactGroup}", version: "${besuVersion}") { - entry "besu-datatypes" - entry "evm" - entry "plugin-api" - } + dependency 'com.google.guava:guava:32.1.2-jre' - // Logging dependencies - dependency 'org.slf4j:slf4j-api:2.0.7' - dependencySet(group: 'ch.qos.logback', version: '1.4.9') { - entry 'logback-core' - entry 'logback-classic' - } + dependency 'com.slack.api:slack-api-client:1.32.1' - // auto service - dependency 'com.google.auto.service:auto-service:1.0.1' + dependency 'com.splunk.logging:splunk-library-javalogging:1.11.5' dependencySet(group: 'info.picocli', version: '4.7.1') { entry 'picocli' entry 'picocli-codegen' } - dependencySet(group: 'org.web3j', version: '4.9.7') { - entry 'abi' - entry 'besu' - entry 'core' - entry 'crypto' + dependencySet(group: 'io.tmio', version: '2.4.2') { + entry 'tuweni-bytes' + entry 'tuweni-net' + entry 'tuweni-toml' + entry 'tuweni-units' } + dependency 'io.vertx:vertx-core:4.3.8' + dependency 'junit:junit:4.13.2' + + dependency 'org.apache.commons:commons-lang3:3.13.0' + + dependency 'org.assertj:assertj-core:3.25.1' + + dependency 'org.awaitility:awaitility:4.2.0' + + dependencySet(group: 'org.bouncycastle', version: '1.75') { + entry 'bcpkix-jdk18on' + entry 'bcprov-jdk18on' + entry 'jcajce' + } + + dependency 'org.hibernate.validator:hibernate-validator:8.0.1.Final' + dependency 'org.junit.vintage:junit-vintage-engine:5.10.2' dependencySet(group: 'org.junit.jupiter', version: '5.10.2') { entry 'junit-jupiter-api' entry 'junit-jupiter-engine' entry 'junit-jupiter-params' } + dependencySet(group: 'org.mockito', version: '5.12.0') { entry 'mockito-core' entry 'mockito-junit-jupiter' } - dependency 'org.assertj:assertj-core:3.25.1' - dependency 'org.awaitility:awaitility:4.2.0' - - dependency 'com.splunk.logging:splunk-library-javalogging:1.11.5' - - dependency 'io.vertx:vertx-core:4.3.8' - - dependency 'com.slack.api:slack-api-client:1.32.1' - - dependency 'org.hibernate.validator:hibernate-validator:8.0.1.Final' + // Logging dependencies + dependency 'org.slf4j:slf4j-api:2.0.7' + dependencySet(group: 'ch.qos.logback', version: '1.4.9') { + entry 'logback-core' + entry 'logback-classic' + } - dependency 'net.java.dev.jna:jna:5.12.1' + dependencySet(group: 'org.web3j', version: '4.9.7') { + entry 'abi' + entry 'besu' + entry 'core' + entry 'crypto' + } } } diff --git a/gradle/dist.gradle b/gradle/dist.gradle index f743c0595..3c636a7d5 100644 --- a/gradle/dist.gradle +++ b/gradle/dist.gradle @@ -1,5 +1,3 @@ -import de.undercouch.gradle.tasks.download.Download - /* * Copyright Consensys Software Inc. * @@ -40,13 +38,6 @@ jar { 'Implementation-Version': calculateVersion() ) } - - from { - configurations.runtimeClasspath.filter( {! (it.name =~ /log4j.*\.jar/ ) && ! (it.name =~ /vertx.*\.jar/ )} ) - .collect {it.isDirectory() ? it : zipTree(it) } - } - exclude 'META-INF/*.RSA', 'META-INF/*.SF', 'META-INF/*.DSA' - duplicatesStrategy(DuplicatesStrategy.INCLUDE) } // Takes the version, and if -SNAPSHOT is part of it replaces SNAPSHOT diff --git a/gradle/versions.gradle b/gradle/versions.gradle deleted file mode 100644 index 4e8e56174..000000000 --- a/gradle/versions.gradle +++ /dev/null @@ -1 +0,0 @@ -dependency 'net.java.dev.jna:jna:5.14.0' diff --git a/reference-tests/build.gradle b/reference-tests/build.gradle index 4f5366394..c2b9e8ccb 100644 --- a/reference-tests/build.gradle +++ b/reference-tests/build.gradle @@ -85,16 +85,17 @@ dependencies { testImplementation project(":arithmetization") testImplementation "${besuArtifactGroup}:besu-datatypes" testImplementation "${besuArtifactGroup}:evm" - testImplementation "${besuArtifactGroup}.internal:config:${besuVersion}" - testImplementation "${besuArtifactGroup}.internal:core:${besuVersion}" - testImplementation "${besuArtifactGroup}.internal:algorithms:${besuVersion}" - testImplementation "${besuArtifactGroup}.internal:metrics-core:${besuVersion}" - testImplementation "${besuArtifactGroup}.internal:referencetests:${besuVersion}" - testImplementation "${besuArtifactGroup}.internal:rlp:${besuVersion}" - testImplementation "${besuArtifactGroup}.internal:testutil:${besuVersion}" - testImplementation "${besuArtifactGroup}.internal:trie:$besuVersion" - testImplementation "${besuArtifactGroup}.internal:util:${besuVersion}" testImplementation "${besuArtifactGroup}:plugin-api" + testImplementation "${besuArtifactGroup}.internal:algorithms" + testImplementation "${besuArtifactGroup}.internal:config" + testImplementation "${besuArtifactGroup}.internal:core" + testImplementation "${besuArtifactGroup}.internal:metrics-core" + testImplementation "${besuArtifactGroup}.internal:referencetests" + testImplementation "${besuArtifactGroup}.internal:rlp" + testImplementation "${besuArtifactGroup}.internal:testutil" + testImplementation "${besuArtifactGroup}.internal:trie" + testImplementation "${besuArtifactGroup}.internal:util" + testImplementation 'io.tmio:tuweni-bytes' testImplementation 'io.tmio:tuweni-units' From e706fd00f147d9fd47b454a6734596c934188640 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 29 Nov 2024 16:03:33 +0100 Subject: [PATCH 2/3] Introduce pluginImplementation and pluginClasspath Signed-off-by: Fabio Di Fabio --- arithmetization/build.gradle | 28 ++++++-------- gradle/dependency-management.gradle | 12 ++++-- gradle/dist.gradle | 57 +++++++++++++++++++++++++++++ testing/build.gradle | 15 +++----- 4 files changed, 83 insertions(+), 29 deletions(-) diff --git a/arithmetization/build.gradle b/arithmetization/build.gradle index f06070fb2..72fba7f63 100644 --- a/arithmetization/build.gradle +++ b/arithmetization/build.gradle @@ -34,10 +34,15 @@ apply from: rootProject.file("gradle/lint.gradle") apply from: rootProject.file("gradle/trace-files.gradle") dependencies { + /** + * Use pluginImplementation for dependencies that are specific to this plugin + * and are not already provided by Besu. + * These dependencies are the only ones that are included in the final jar. + */ + pluginImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' + // annotationProcessor generates the file META-INF/services/org.hyperledger.besu.plugin.BesuPlugin annotationProcessor 'com.google.auto.service:auto-service' - implementation 'com.google.auto.service:auto-service' - implementation 'com.google.auto.service:auto-service-annotations' implementation "${besuArtifactGroup}:besu-datatypes" implementation "${besuArtifactGroup}:evm" @@ -48,28 +53,19 @@ dependencies { implementation "${besuArtifactGroup}.internal:core" implementation "${besuArtifactGroup}.internal:rlp" - implementation 'info.picocli:picocli' - - implementation 'io.vertx:vertx-core' - implementation 'io.vertx:vertx-web' + implementation 'com.google.auto.service:auto-service' - implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' + implementation 'info.picocli:picocli' implementation 'io.tmio:tuweni-bytes' implementation 'io.tmio:tuweni-units' implementation 'io.tmio:tuweni-toml' - implementation 'org.bouncycastle:bcprov-jdk18on' - implementation 'org.hibernate.validator:hibernate-validator' + + implementation 'io.vertx:vertx-web' testImplementation project(path: ':testing') - testImplementation 'org.junit.platform:junit-platform-launcher' -} -configurations { - installedJars { - transitive = false - } + testImplementation 'org.junit.platform:junit-platform-launcher' } apply from: rootProject.file("gradle/dist.gradle") diff --git a/gradle/dependency-management.gradle b/gradle/dependency-management.gradle index 5c059377d..7079c5972 100644 --- a/gradle/dependency-management.gradle +++ b/gradle/dependency-management.gradle @@ -30,12 +30,16 @@ repositories { mavenLocal() } -configurations.all { - resolutionStrategy { - cacheChangingModulesFor 0, 'seconds' +configurations { + pluginImplementation + resolvable('pluginClasspath') { + extendsFrom(pluginImplementation) } } +sourceSets.main.compileClasspath += configurations.pluginImplementation +sourceSets.main.runtimeClasspath += configurations.pluginImplementation + apply plugin: 'io.spring.dependency-management' dependencyManagement { @@ -54,6 +58,8 @@ dependencyManagement { entry 'logback-classic' } + dependency 'com.google.code.gson:gson:2.11.0' + dependency 'com.slack.api:slack-api-client:1.32.1' } } diff --git a/gradle/dist.gradle b/gradle/dist.gradle index 3c636a7d5..e3368b1e8 100644 --- a/gradle/dist.gradle +++ b/gradle/dist.gradle @@ -12,6 +12,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ +import de.undercouch.gradle.tasks.download.Download tasks.register('sourcesJar', Jar) { dependsOn classes @@ -27,7 +28,43 @@ tasks.register('javadocJar', Jar) { version = project.hasProperty('releaseVersion') ? project.getProperty('releaseVersion') : 'snapshot' +def lineaBesuDistTar = new File(new File(buildDir, "tmp"), rootProject.besuFilename) +tasks.register('downloadLineaBesu', Download) { + src rootProject.besuUrl + dest lineaBesuDistTar + onlyIfModified true +} + +tasks.register('copyLocalLineaBesu', Copy) { + onlyIf { + downloadLineaBesu.state.failure + } + def localLineaBesuDir = + project.hasProperty('useLocalLineaBesuDir') + ? file("${findProperty('useLocalLineaBesuDir')}".replaceFirst('^~', System.getProperty('user.home'))) + : new File(projectDir, "../../linea-besu") + + def localLineaBesuFile = new File("${localLineaBesuDir.absoluteFile}/build/distributions/${rootProject.besuFilename}") + doFirst { + if (!file(localLineaBesuFile).exists()) { + throw new GradleException("Could not download Linea Besu distribution from: " + rootProject.besuUrl + + ", and could not find it locally at ${localLineaBesuFile} either") + } + } + from localLineaBesuFile + into lineaBesuDistTar.parentFile +} + +task unTarLineaBesu(type: Copy) { + dependsOn downloadLineaBesu + dependsOn copyLocalLineaBesu + + from tarTree(lineaBesuDistTar) + into lineaBesuDistTar.parentFile +} + jar { + dependsOn unTarLineaBesu archiveBaseName = distributionIdentifier manifest { @@ -38,6 +75,26 @@ jar { 'Implementation-Version': calculateVersion() ) } + + // Get all the dependencies that are provided by Besu, removing the version and suffix + def lineaBesuLibDir = new File(lineaBesuDistTar.parentFile, rootProject.besuIdentifier + '/lib') + def lineaBesuLibs = [] + fileTree(dir: lineaBesuLibDir, include: '*.jar').visit { + FileVisitDetails details -> + def libPrefix = details.file.name =~ /(.*)(\-.*?)\.jar/ + lineaBesuLibs << libPrefix[0][1] + } + + from { + configurations.pluginClasspath.filter( + { + // include the dependency in the jar only if it is not already provided by Besu + def libName = it.name =~ /(.*)(\-.*?)\.jar/ + def libPrefix = libName[0][1] + !lineaBesuLibs.contains(libPrefix) + }).collect {it.isDirectory() ? it : zipTree(it) } + } + duplicatesStrategy('exclude') } // Takes the version, and if -SNAPSHOT is part of it replaces SNAPSHOT diff --git a/testing/build.gradle b/testing/build.gradle index 2723b1047..def0471af 100644 --- a/testing/build.gradle +++ b/testing/build.gradle @@ -16,8 +16,10 @@ apply from: rootProject.file("gradle/lint.gradle") dependencies { implementation project(path: ':arithmetization') + implementation "${besuArtifactGroup}:besu-datatypes" implementation "${besuArtifactGroup}:evm" + implementation "${besuArtifactGroup}:plugin-api" implementation "${besuArtifactGroup}.internal:clique" implementation "${besuArtifactGroup}.internal:common" implementation "${besuArtifactGroup}.internal:config" @@ -28,19 +30,12 @@ dependencies { implementation "${besuArtifactGroup}.internal:services" implementation "${besuArtifactGroup}.internal:testutil" implementation "${besuArtifactGroup}.internal:algorithms" - implementation "${besuArtifactGroup}:plugin-api" - implementation "com.google.guava:guava" + implementation 'com.google.code.gson:gson' + implementation 'com.google.guava:guava' - implementation 'org.junit.platform:junit-platform-launcher' - implementation 'org.junit.jupiter:junit-jupiter-api' - runtimeOnly 'org.junit.jupiter:junit-jupiter-engine' - implementation 'org.junit.jupiter:junit-jupiter-params' - runtimeOnly 'org.junit.vintage:junit-vintage-engine' - - implementation 'org.mockito:mockito-core' - implementation 'org.mockito:mockito-junit-jupiter' implementation 'org.assertj:assertj-core' + implementation 'org.junit.jupiter:junit-jupiter-api' } node { From 43a8649d38dae743c0c57cbbc17aa4017c009c66 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Mon, 2 Dec 2024 13:00:38 +0100 Subject: [PATCH 3/3] Add distPlugin task Signed-off-by: Fabio Di Fabio --- arithmetization/build.gradle | 1 + gradle/dist.gradle | 53 ++++++++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/arithmetization/build.gradle b/arithmetization/build.gradle index 72fba7f63..6de4224a6 100644 --- a/arithmetization/build.gradle +++ b/arithmetization/build.gradle @@ -15,6 +15,7 @@ plugins { id 'java' + id 'java-library-distribution' id 'common-plugins' id 'com.github.hierynomus.license' id "de.undercouch.download" diff --git a/gradle/dist.gradle b/gradle/dist.gradle index e3368b1e8..9e14e85e5 100644 --- a/gradle/dist.gradle +++ b/gradle/dist.gradle @@ -63,6 +63,22 @@ task unTarLineaBesu(type: Copy) { into lineaBesuDistTar.parentFile } +// Get all the dependencies that are provided by Besu, removing the version and suffix +def lineaBesuLibDir = new File(lineaBesuDistTar.parentFile, rootProject.besuIdentifier + '/lib') +def lineaBesuLibs = [] +fileTree(dir: lineaBesuLibDir, include: '*.jar').visit { + FileVisitDetails details -> + def libPrefix = details.file.name =~ dependencyNamePattern() + lineaBesuLibs << libPrefix[0][1] +} + +def excludeBesuProvidedDeps = { + // include the dependency in the jar only if it is not already provided by Besu + def libName = it.name =~ dependencyNamePattern() + def libPrefix = libName[0][1] + !lineaBesuLibs.contains(libPrefix) +} + jar { dependsOn unTarLineaBesu archiveBaseName = distributionIdentifier @@ -76,23 +92,10 @@ jar { ) } - // Get all the dependencies that are provided by Besu, removing the version and suffix - def lineaBesuLibDir = new File(lineaBesuDistTar.parentFile, rootProject.besuIdentifier + '/lib') - def lineaBesuLibs = [] - fileTree(dir: lineaBesuLibDir, include: '*.jar').visit { - FileVisitDetails details -> - def libPrefix = details.file.name =~ /(.*)(\-.*?)\.jar/ - lineaBesuLibs << libPrefix[0][1] - } from { - configurations.pluginClasspath.filter( - { - // include the dependency in the jar only if it is not already provided by Besu - def libName = it.name =~ /(.*)(\-.*?)\.jar/ - def libPrefix = libName[0][1] - !lineaBesuLibs.contains(libPrefix) - }).collect {it.isDirectory() ? it : zipTree(it) } + configurations.pluginClasspath.filter(excludeBesuProvidedDeps +).collect {it.isDirectory() ? it : zipTree(it) } } duplicatesStrategy('exclude') } @@ -108,7 +111,27 @@ def calculateVersion() { return version } +/** + * Create a distribution of the plugin, that only contains the plugin jar and the + * dependencies that are not provided by Besu itself, so that is can be simply + * extracted in the Besu plugins dir. + */ +tasks.register('distPlugin', Zip) { + dependsOn installDist + + from("${buildDir}/libs") + from { + configurations.pluginClasspath.filter( + excludeBesuProvidedDeps) + + } +} + static def getCheckedOutGitCommitHash() { def hashLength = 8 "git rev-parse HEAD".execute().text.take(hashLength) } + +static def dependencyNamePattern() { + /(.*)(\-.*?)\.jar/ +} \ No newline at end of file