Skip to content

Commit

Permalink
deps: update ort to v40
Browse files Browse the repository at this point in the history
ORT's reporter was migrated to ORT's new plugin API. This requires
several adaptions in the reporter worker and related tests.
  • Loading branch information
eclipse-apoapsis-bot authored and sschuberth committed Nov 21, 2024
1 parent ea0cfb7 commit 38d6f36
Show file tree
Hide file tree
Showing 10 changed files with 212 additions and 188 deletions.
2 changes: 1 addition & 1 deletion gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ log4j = "2.24.1"
logback = "1.5.12"
micrometer = "1.14.1"
mockk = "1.13.13"
ort = "39.0.0"
ort = "40.0.1"
postgres = "42.7.4"
qpidJmsClient = "2.6.1"
schemaKenerator = "1.6.0"
Expand Down
4 changes: 4 additions & 0 deletions workers/reporter/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ plugins {

// Apply third-party plugins.
alias(libs.plugins.jib)
alias(libs.plugins.ksp)
}

tasks.withType<JibTask> {
Expand All @@ -58,6 +59,9 @@ dependencies {
implementation(libs.ortReporter)
implementation(libs.typesafeConfig)

ksp(libs.ortPluginCompiler)
ksp(libs.ortReporter)

runtimeOnly(platform(projects.config))
runtimeOnly(platform(projects.secrets))
runtimeOnly(platform(projects.storage))
Expand Down
37 changes: 25 additions & 12 deletions workers/reporter/src/main/kotlin/reporter/OrtResultReporter.kt
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,34 @@ package org.eclipse.apoapsis.ortserver.workers.reporter

import java.io.File

import org.ossreviewtoolkit.model.config.PluginConfiguration
import org.ossreviewtoolkit.model.writeValue
import org.ossreviewtoolkit.plugins.api.OrtPlugin
import org.ossreviewtoolkit.plugins.api.OrtPluginOption
import org.ossreviewtoolkit.plugins.api.PluginDescriptor
import org.ossreviewtoolkit.reporter.Reporter
import org.ossreviewtoolkit.reporter.ReporterFactory
import org.ossreviewtoolkit.reporter.ReporterInput
import org.ossreviewtoolkit.utils.common.packZip

class OrtResultReporter : Reporter {
data class OrtResultReporterConfig(
/**
* Whether to compress the generated file. If set to `true` (which is the default), a Zip archive is generated which
* contains the ORT result file.
*/
@OrtPluginOption(defaultValue = "true")
val compressed: Boolean
)

@OrtPlugin(
id = "OrtResult",
displayName = "ORT Result Reporter",
description = "A reporter that creates an ORT result YAML file for the ORT run.",
factory = ReporterFactory::class
)
class OrtResultReporter(
override val descriptor: PluginDescriptor = OrtResultReporterFactory.descriptor,
private val config: OrtResultReporterConfig
) : Reporter {
companion object {
/**
* Name of the property that enables compression of the generated file. If set to *true* (which is the
Expand All @@ -42,20 +63,12 @@ class OrtResultReporter : Reporter {
private const val ARCHIVE_FILE_NAME = "ort-result.zip"
}

override val type = "OrtResult"

override fun generateReport(
input: ReporterInput,
outputDir: File,
config: PluginConfiguration
): List<Result<File>> {
val compressed = config.options.getOrDefault(COMPRESSED_PROPERTY, "true").toBooleanStrict()

override fun generateReport(input: ReporterInput, outputDir: File): List<Result<File>> {
val reportFile = runCatching {
val targetDir = outputDir.resolve("ort-result").apply { mkdir() }
val outputFile = targetDir.resolve(RESULT_FILE_NAME).apply { writeValue(input.ortResult) }

if (compressed) {
if (config.compressed) {
val archiveFile = outputDir.resolve(ARCHIVE_FILE_NAME)
targetDir.packZip(archiveFile)
archiveFile
Expand Down
16 changes: 9 additions & 7 deletions workers/reporter/src/main/kotlin/reporter/ReporterRunner.kt
Original file line number Diff line number Diff line change
Expand Up @@ -50,20 +50,20 @@ import org.ossreviewtoolkit.model.OrtResult
import org.ossreviewtoolkit.model.config.CopyrightGarbage
import org.ossreviewtoolkit.model.config.LicenseFilePatterns
import org.ossreviewtoolkit.model.config.PackageConfiguration
import org.ossreviewtoolkit.model.config.PluginConfiguration as OrtPluginConfiguration
import org.ossreviewtoolkit.model.config.Resolutions
import org.ossreviewtoolkit.model.licenses.DefaultLicenseInfoProvider
import org.ossreviewtoolkit.model.licenses.LicenseClassifications
import org.ossreviewtoolkit.model.licenses.LicenseInfoResolver
import org.ossreviewtoolkit.model.utils.DefaultResolutionProvider
import org.ossreviewtoolkit.model.utils.FileArchiver
import org.ossreviewtoolkit.plugins.api.PluginConfig
import org.ossreviewtoolkit.plugins.packageconfigurationproviders.api.CompositePackageConfigurationProvider
import org.ossreviewtoolkit.plugins.packageconfigurationproviders.api.PackageConfigurationProviderFactory
import org.ossreviewtoolkit.plugins.packageconfigurationproviders.api.SimplePackageConfigurationProvider
import org.ossreviewtoolkit.reporter.DefaultLicenseTextProvider
import org.ossreviewtoolkit.reporter.HowToFixTextProvider
import org.ossreviewtoolkit.reporter.LicenseTextProvider
import org.ossreviewtoolkit.reporter.Reporter
import org.ossreviewtoolkit.reporter.ReporterFactory
import org.ossreviewtoolkit.reporter.ReporterInput
import org.ossreviewtoolkit.utils.common.safeMkdirs
import org.ossreviewtoolkit.utils.config.setPackageConfigurations
Expand Down Expand Up @@ -200,14 +200,16 @@ class ReporterRunner(
logger.info("Generating the '$format' report...")

val result = runCatching {
val reporter = requireNotNull(Reporter.ALL[format]) {
val reporterFactory = requireNotNull(ReporterFactory.ALL[format]) {
"No reporter found for the configured format '$format'."
}

val reporterOptions = transformedOptions[reporter.type]?.mapToOrt()
?: OrtPluginConfiguration.EMPTY
val reporterConfig = transformedOptions[reporterFactory.descriptor.id]?.let { options ->
PluginConfig(options.options, options.secrets)
} ?: PluginConfig()

val reportFileResults = reporter.generateReport(reporterInput, outputDir, reporterOptions)
val reporter = reporterFactory.create(reporterConfig)
val reportFileResults = reporter.generateReport(reporterInput, outputDir)

val reportFiles = reportFileResults.mapNotNull { result ->
result.getOrElse {
Expand All @@ -222,7 +224,7 @@ class ReporterRunner(
}

result.getOrNull()?.let { (reporter, reportFiles) ->
val nameMapper = ReportNameMapper.create(config, reporter.type)
val nameMapper = ReportNameMapper.create(config, reporter.descriptor.id)
format to nameMapper.mapReportNames(reportFiles)
.also { reportStorage.storeReportFiles(runId, it) }
}
Expand Down
22 changes: 13 additions & 9 deletions workers/reporter/src/main/kotlin/reporter/RunStatisticsReporter.kt
Original file line number Diff line number Diff line change
Expand Up @@ -21,24 +21,28 @@ package org.eclipse.apoapsis.ortserver.workers.reporter

import java.io.File

import org.ossreviewtoolkit.model.config.PluginConfiguration
import org.ossreviewtoolkit.model.writeValue
import org.ossreviewtoolkit.plugins.api.OrtPlugin
import org.ossreviewtoolkit.plugins.api.PluginDescriptor
import org.ossreviewtoolkit.reporter.Reporter
import org.ossreviewtoolkit.reporter.ReporterFactory
import org.ossreviewtoolkit.reporter.ReporterInput

/**
* A custom reporter which exports the statistics of the run. Unlike ORT's EvaluatedModelReporter, which also exports
* statistics data, this reporter does not include any other information to the produced reports thus making them more
* memory-efficient.
*/
class RunStatisticsReporter : Reporter {
override val type = "RunStatistics"

override fun generateReport(
input: ReporterInput,
outputDir: File,
config: PluginConfiguration
): List<Result<File>> {
@OrtPlugin(
id = "RunStatistics",
displayName = "Run Statistics Reporter",
description = "A reporter that creates a JSON file with the statistics of the ORT run.",
factory = ReporterFactory::class
)
class RunStatisticsReporter(
override val descriptor: PluginDescriptor = RunStatisticsReporterFactory.descriptor
) : Reporter {
override fun generateReport(input: ReporterInput, outputDir: File): List<Result<File>> {
val outputFile = runCatching {
outputDir.resolve("statistics.json").apply { writeValue(input.statistics) }
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,17 @@ import org.ossreviewtoolkit.model.PackageType
import org.ossreviewtoolkit.model.Provenance
import org.ossreviewtoolkit.model.RepositoryProvenance
import org.ossreviewtoolkit.model.config.DownloaderConfiguration
import org.ossreviewtoolkit.model.config.PluginConfiguration
import org.ossreviewtoolkit.model.licenses.LicenseCategorization
import org.ossreviewtoolkit.model.licenses.LicenseClassifications
import org.ossreviewtoolkit.model.licenses.LicenseInfoResolver
import org.ossreviewtoolkit.model.licenses.LicenseView
import org.ossreviewtoolkit.model.licenses.ResolvedLicenseInfo
import org.ossreviewtoolkit.model.utils.createLicenseInfoResolver
import org.ossreviewtoolkit.plugins.api.OrtPlugin
import org.ossreviewtoolkit.plugins.api.OrtPluginOption
import org.ossreviewtoolkit.plugins.api.PluginDescriptor
import org.ossreviewtoolkit.reporter.Reporter
import org.ossreviewtoolkit.reporter.ReporterFactory
import org.ossreviewtoolkit.reporter.ReporterInput
import org.ossreviewtoolkit.utils.common.encodeOrUnknown
import org.ossreviewtoolkit.utils.common.packZip
Expand All @@ -54,12 +57,38 @@ import org.slf4j.LoggerFactory
const val SOURCE_BUNDLE_FILE_NAME = "source-bundle-archive.zip"
const val SOURCE_BUNDLE_SUB_DIR = "source_code_bundle"

data class SourceCodeBundleReporterConfig(
/**
* The license categories to consider when including packages in the source code bundle.
*/
val includedLicenseCategories: List<String>?,

/**
* The type of package to include in the source code bundle. Allowed values are "PROJECT" and "PACKAGE".
*/
@OrtPluginOption(defaultValue = "PROJECT", aliases = ["packageType"])
val packageTypes: List<String>
)

/**
* A custom reporter that creates source code bundles.
*/
@OrtPlugin(
id = "SourceCodeBundle",
displayName = "Source Code Bundle Reporter",
description = "A reporter that creates a source code bundle for the given ORT result.",
factory = ReporterFactory::class
)
class SourceCodeBundleReporter(
private val downloader: Downloader = Downloader(DownloaderConfiguration())
override val descriptor: PluginDescriptor = SourceCodeBundleReporterFactory.descriptor,
private val config: SourceCodeBundleReporterConfig,
private val downloader: Downloader
) : Reporter {
constructor(
descriptor: PluginDescriptor = SourceCodeBundleReporterFactory.descriptor,
config: SourceCodeBundleReporterConfig
) : this(descriptor, config, Downloader(DownloaderConfiguration()))

companion object {
/**
* Name of the property that specifies the type of package for source code bundle
Expand All @@ -71,11 +100,6 @@ class SourceCodeBundleReporter(
*/
const val INCLUDED_LICENSE_CATEGORIES_PROPERTY = "includedLicenseCategories"

/**
* Name of the property that specifies the type of package for source code bundle
*/
const val REPORTER_NAME = "SourceCodeBundle"

/** A file filter that simply includes all files. */
private val includeAllFilter: (File) -> Boolean = { true }

Expand Down Expand Up @@ -107,22 +131,16 @@ class SourceCodeBundleReporter(
}
}

override val type = REPORTER_NAME

override fun generateReport(
input: ReporterInput,
outputDir: File,
config: PluginConfiguration
): List<Result<File>> {
override fun generateReport(input: ReporterInput, outputDir: File): List<Result<File>> {
log.info("Preparing a source code bundle for repository '${input.ortResult.repository.vcsProcessed.url}'")

val outputFile = runCatching {
downloadSourceCode(
input.ortResult,
outputDir,
input.licenseClassifications,
config.options[INCLUDED_LICENSE_CATEGORIES_PROPERTY]?.takeUnless { it.isEmpty() }?.split(",").orEmpty(),
config.options.getOrDefault(PACKAGE_TYPE_PROPERTY, "PROJECT")
config.includedLicenseCategories.orEmpty(),
config.packageTypes
)
}

Expand All @@ -134,16 +152,16 @@ class SourceCodeBundleReporter(
outputDir: File,
licenseClassifications: LicenseClassifications,
includedLicenseCategories: List<String>,
packageType: String
packageTypes: List<String>
): File {
val allPackages = buildList {
if (packageType.contains(PackageType.PROJECT.name, true)) {
if (packageTypes.any { it.equals(PackageType.PROJECT.name, ignoreCase = true) }) {
val projects = consolidateProjectPackagesByVcs(ortResult.getProjects(true)).keys
log.info("Found ${projects.size} project(s) in the ORT result.")
addAll(projects)
}

if (packageType.contains(PackageType.PACKAGE.name, true)) {
if (packageTypes.any { it.equals(PackageType.PACKAGE.name, ignoreCase = true) }) {
val packages = ortResult.getPackages(true).map { it.metadata }
log.info("Found ${packages.size} packages(s) in the ORT result.")
addAll(packages)
Expand Down
19 changes: 7 additions & 12 deletions workers/reporter/src/test/kotlin/reporter/OrtResultReporterTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,15 @@ import io.kotest.matchers.types.beInstanceOf
import org.eclipse.apoapsis.ortserver.workers.common.OrtTestData

import org.ossreviewtoolkit.model.OrtResult
import org.ossreviewtoolkit.model.config.PluginConfiguration
import org.ossreviewtoolkit.model.readValue
import org.ossreviewtoolkit.reporter.Reporter
import org.ossreviewtoolkit.reporter.ReporterFactory
import org.ossreviewtoolkit.reporter.ReporterInput
import org.ossreviewtoolkit.utils.common.unpack

class OrtResultReporterTest : WordSpec({
"The OrtResultReporter" should {
"create a correct ORT result file" {
val reporter = OrtResultReporter()
val reporter = OrtResultReporter(config = OrtResultReporterConfig(compressed = false))

// Set the options of the advisor configuration to null because the configured empty map will be serialized
// as null. Changing the value in OrtTestData is not possible, because the options in the server model are
Expand All @@ -60,11 +59,7 @@ class OrtResultReporterTest : WordSpec({

val input = ReporterInput(ortResult = ortResult)

val reportFileResults = reporter.generateReport(
input,
tempdir(),
PluginConfiguration(options = mapOf(OrtResultReporter.COMPRESSED_PROPERTY to "false"))
)
val reportFileResults = reporter.generateReport(input, tempdir())

reportFileResults.shouldBeSingleton {
it shouldBeSuccess { reportFile ->
Expand All @@ -76,7 +71,7 @@ class OrtResultReporterTest : WordSpec({
}

"create a compressed ORT result file" {
val reporter = OrtResultReporter()
val reporter = OrtResultReporter(config = OrtResultReporterConfig(compressed = true))

val advisorRun = OrtTestData.advisorRun.copy(
config = OrtTestData.advisorConfiguration.copy(config = null)
Expand Down Expand Up @@ -107,10 +102,10 @@ class OrtResultReporterTest : WordSpec({
}

"be found by the service loader" {
val reporter = OrtResultReporter()
val pluginId = OrtResultReporterFactory.descriptor.id

Reporter.ALL should containAnyKeys(reporter.type)
Reporter.ALL[reporter.type] should beInstanceOf<OrtResultReporter>()
ReporterFactory.ALL should containAnyKeys(pluginId)
ReporterFactory.ALL[pluginId] should beInstanceOf<OrtResultReporterFactory>()
}
}
})
Loading

0 comments on commit 38d6f36

Please sign in to comment.