feat(package-managers): Start to implement a BitBake analyzer

Add the beginnings of an analyzer for BitBake [1], see [2] for context. It
basically works by making the build inherit from the "create-spdx" class
[3] to create SPDX 2.2 files and post-processing them via ORT's SPDX
document file analyzer.

This initial implementations has several known limitations still. First
of all, as the "create-spdx" class cannot be used without building, and
builds are not cached, so the analyzer is very slow. Secondly, SPDX
external documents refs cannot be resolved yet. This requires some
post-processing of the SPDX document files before passing them on,
notably by adjusting the `SPDX_NAMESPACE_PREFIX` variable [4].

[1]: https://docs.yoctoproject.org/bitbake/
[2]: #722
[3]: https://docs.yoctoproject.org/dev/dev-manual/sbom.html
[4]: https://docs.yoctoproject.org/ref-manual/variables.html#term-SPDX_NAMESPACE_PREFIX

Signed-off-by: Sebastian Schuberth <sebastian@doubleopen.org>

analyzer/src/funTest/kotlin/PackageManagerFunTest.kt

@@ -39,6 +39,7 @@ import org.ossreviewtoolkit.model.config.PathExcludeReason
 class PackageManagerFunTest : WordSpec({
     val definitionFiles = listOf(
         "bazel/MODULE.bazel",
+        "bitbake/recipe.bb",
         "bower/bower.json",
         "bundler/Gemfile",
         "cargo/Cargo.toml",

plugins/package-managers/bitbake/README.md

@@ -0,0 +1,13 @@
+# About
+
+This is a package manager plugin for the [OSS Review Toolkit][ORT] to analyze [Yocto] projects managed by [BitBake].
+It supersedes the combination of the [meta-doubleopen] and [do-convert] projects by relying on upstream [SBOM] generation in [SPDX] format, and converting the generated files to an ORT analyzer result file via ORT's [SPDX document file analyzer].
+
+[ORT]: https://github.com/oss-review-toolkit/ort
+[BitBake]: https://docs.yoctoproject.org/bitbake.html
+[Yocto]: https://www.yoctoproject.org/
+[meta-doubleopen]: https://github.com/doubleopen-project/meta-doubleopen
+[do-convert]: https://github.com/doubleopen-project/do-convert
+[SBOM]: https://docs.yoctoproject.org/dev/dev-manual/sbom.html
+[SPDX]: https://spdx.dev/
+[SPDX document file analyzer]: https://oss-review-toolkit.org/ort/docs/tools/analyzer

plugins/package-managers/bitbake/build.gradle.kts

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+plugins {
+    // Apply precompiled plugins.
+    id("ort-library-conventions")
+}
+
+dependencies {
+    api(project(":analyzer"))
+    api(project(":model"))
+
+    implementation(project(":utils:common-utils"))
+    implementation(project(":plugins:package-managers:spdx-package-manager"))
+
+    funTestImplementation(project(":downloader"))
+    funTestImplementation(project(":plugins:version-control-systems:git-version-control-system"))
+    funTestImplementation(testFixtures(project(":analyzer")))
+}

plugins/package-managers/bitbake/src/funTest/kotlin/BitBakeToolFunTest.kt

@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.plugins.packagemanagers.bitbake
+
+import io.kotest.core.spec.style.WordSpec
+import io.kotest.engine.spec.tempdir
+import io.kotest.matchers.collections.beEmpty
+import io.kotest.matchers.collections.shouldHaveSize
+import io.kotest.matchers.result.shouldBeSuccess
+import io.kotest.matchers.should
+import io.kotest.matchers.shouldBe
+import io.kotest.matchers.string.shouldMatch
+
+import org.ossreviewtoolkit.analyzer.Analyzer
+import org.ossreviewtoolkit.analyzer.create
+import org.ossreviewtoolkit.model.Identifier
+import org.ossreviewtoolkit.model.VcsInfo
+import org.ossreviewtoolkit.model.VcsType
+import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
+import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.Git
+import org.ossreviewtoolkit.utils.test.ExpensiveTag
+import org.ossreviewtoolkit.utils.test.shouldNotBeNull
+
+class BitBakeToolFunTest : WordSpec({
+    "BitBake" should {
+        "get the version correctly" {
+            val bitBake = create("BitBake") as BitBake
+
+            val version = bitBake.getBitBakeVersion(tempdir())
+
+            version shouldMatch "\\d+\\.\\d+\\.\\d+"
+        }
+    }
+
+    "Analyzing recipes from Poky" should {
+        val projectDir = tempdir()
+        val pokyVcsInfo = VcsInfo(VcsType.GIT, "https://git.yoctoproject.org/poky", "kirkstone-4.0.17")
+
+        Git().run {
+            val workingTree = initWorkingTree(projectDir, pokyVcsInfo)
+            updateWorkingTree(workingTree, pokyVcsInfo.revision)
+        } shouldBeSuccess pokyVcsInfo.revision
+
+        "create an SPDX file for the 'quilt-native' package" {
+            val recipeFileName = "quilt-native_0.67.bb"
+            val result = Analyzer(AnalyzerConfiguration()).run {
+                val fileInfo = findManagedFiles(projectDir)
+                val singleFileInfo = fileInfo.copy(
+                    managedFiles = fileInfo.managedFiles.map { (packageManager, definitionsFiles) ->
+                        packageManager to definitionsFiles.filter { it.name == recipeFileName }
+                    }.toMap()
+                )
+                analyze(singleFileInfo)
+            }
+
+            result.analyzer?.result shouldNotBeNull {
+                projects shouldHaveSize 1
+
+                with(projects.single()) {
+                    id shouldBe Identifier("BitBake:OpenEmbedded ():quilt-native:0.67")
+                    declaredLicenses shouldBe setOf("GPL-2.0-only")
+                    homepageUrl shouldBe "http://savannah.nongnu.org/projects/quilt/"
+                    scopes should beEmpty()
+                }
+            }
+        }
+
+        "create a SPDX files for the 'xmlto' package".config(tags = setOf(ExpensiveTag)) {
+            val recipeFileName = "xmlto_0.0.28.bb"
+            val result = Analyzer(AnalyzerConfiguration()).run {
+                val fileInfo = findManagedFiles(projectDir)
+                val singleFileInfo = fileInfo.copy(
+                    managedFiles = fileInfo.managedFiles.map { (packageManager, definitionsFiles) ->
+                        packageManager to definitionsFiles.filter { it.name == recipeFileName }
+                    }.toMap()
+                )
+                analyze(singleFileInfo)
+            }
+
+            result.analyzer?.result shouldNotBeNull {
+                projects shouldHaveSize 90
+            }
+        }
+    }
+})

plugins/package-managers/bitbake/src/main/kotlin/BitBake.kt

@@ -0,0 +1,166 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.plugins.packagemanagers.bitbake
+
+import java.io.File
+
+import kotlin.time.measureTime
+
+import org.apache.logging.log4j.kotlin.logger
+
+import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory
+import org.ossreviewtoolkit.analyzer.PackageManager
+import org.ossreviewtoolkit.analyzer.PackageManagerResult
+import org.ossreviewtoolkit.model.ProjectAnalyzerResult
+import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
+import org.ossreviewtoolkit.model.config.RepositoryConfiguration
+import org.ossreviewtoolkit.plugins.packagemanagers.spdx.SpdxDocumentFile
+import org.ossreviewtoolkit.utils.common.ProcessCapture
+import org.ossreviewtoolkit.utils.common.getCommonParentFile
+import org.ossreviewtoolkit.utils.common.safeDeleteRecursively
+import org.ossreviewtoolkit.utils.common.withoutPrefix
+import org.ossreviewtoolkit.utils.ort.createOrtTempDir
+import org.ossreviewtoolkit.utils.ort.createOrtTempFile
+
+/**
+ * A package manager that uses OpenEmbedded's "bitbake" tool to create SPDX SBOMs [1][2] e.g. for Yocto distributions,
+ * and post-processes these into ORT analyzer results.
+ *
+ * [1]: https://docs.yoctoproject.org/dev/dev-manual/sbom.html
+ * [2]: https://dev.to/angrymane/create-spdx-with-yocto-2od9
+ */
+class BitBake(
+    name: String,
+    analysisRoot: File,
+    analyzerConfig: AnalyzerConfiguration,
+    repoConfig: RepositoryConfiguration
+) : PackageManager(name, analysisRoot, analyzerConfig, repoConfig) {
+    class Factory : AbstractPackageManagerFactory<BitBake>("BitBake") {
+        override val globsForDefinitionFiles = listOf("*.bb")
+
+        override fun create(
+            analysisRoot: File,
+            analyzerConfig: AnalyzerConfiguration,
+            repoConfig: RepositoryConfiguration
+        ) = BitBake(type, analysisRoot, analyzerConfig, repoConfig)
+    }
+
+    private val scriptFile by lazy { extractResourceToTempFile(BITBAKE_SCRIPT_NAME).apply { setExecutable(true) } }
+    private val spdxConfFile by lazy { extractResourceToTempFile(SPDX_CONF_NAME) }
+
+    private val spdxManager by lazy { SpdxDocumentFile(name, analysisRoot, analyzerConfig, repoConfig) }
+
+    override fun resolveDependencies(definitionFiles: List<File>, labels: Map<String, String>): PackageManagerResult {
+        val commonDefinitionDir = getCommonParentFile(definitionFiles)
+        val workingDir = requireNotNull(commonDefinitionDir.searchUpwardsForFile(INIT_SCRIPT_NAME)) {
+            "No '$INIT_SCRIPT_NAME' script file found for directory '$commonDefinitionDir'."
+        }
+
+        logger.info { "Determined the working directory to be '$workingDir'." }
+
+        val localVersion = getBitBakeVersion(workingDir)
+        val globalVersion = createOrtTempDir().let { dir ->
+            getBitBakeVersion(dir).also { dir.safeDeleteRecursively(force = true) }
+        }
+
+        if (localVersion != globalVersion) {
+            logger.warn { "Local $managerName version $localVersion differs from global version $globalVersion." }
+        }
+
+        val deployDirs = mutableSetOf<File>()
+
+        definitionFiles.forEach { definitionFile ->
+            val target = definitionFile.nameWithoutExtension.substringBeforeLast('_')
+
+            val deployDir = getDeployDir(workingDir, target)
+            deployDirs += deployDir
+
+            val spdxFile = deployDir.findSpdxFiles().find { it.name == "recipe-$target.spdx.json" }
+            if (spdxFile != null) {
+                logger.info { "Not creating SPDX files for target '$target' as it already exists at '$spdxFile'." }
+            } else {
+                logger.info { "Creating SPDX files for target '$target'..." }
+
+                // This implicitly triggers the build and can take a very long time.
+                val duration = measureTime { createSpdx(workingDir, target) }
+
+                logger.info { "Creating SPDX files for target '$target' took $duration." }
+            }
+        }
+
+        if (!scriptFile.delete()) logger.warn { "Unable to delete the temporary '$scriptFile' file." }
+        if (!spdxConfFile.delete()) logger.warn { "Unable to delete the temporary '$spdxConfFile' file." }
+
+        val commonDeployDir = deployDirs.singleOrNull() ?: getCommonParentFile(deployDirs)
+        val spdxFiles = commonDeployDir.findSpdxFiles().toList()
+
+        logger.info { "Found ${spdxFiles.size} SPDX file(s) in '$commonDeployDir'." }
+
+        return spdxManager.resolveDependencies(spdxFiles, labels)
+    }
+
+    override fun resolveDependencies(definitionFile: File, labels: Map<String, String>): List<ProjectAnalyzerResult> =
+        throw NotImplementedError("This function is not supported for $managerName.")
+
+    private fun getDeployDir(workingDir: File, target: String): File {
+        val bitbakeEnv = runBitBake(workingDir, "-e", target)
+        return bitbakeEnv.stdout.lineSequence().mapNotNull { it.withoutPrefix("DEPLOY_DIR=") }.first()
+            .let { File(it.removeSurrounding("\"")) }
+    }
+
+    private fun createSpdx(workingDir: File, target: String) =
+        runBitBake(workingDir, "-r", spdxConfFile.absolutePath, "-c", "create_spdx", target)
+
+    private fun File.findSpdxFiles() = resolve("spdx").walk().filter { it.isFile && it.name.endsWith(".spdx.json") }
+
+    private fun runBitBake(workingDir: File, vararg args: String): ProcessCapture =
+        ProcessCapture(scriptFile.absolutePath, workingDir.absolutePath, *args, workingDir = workingDir)
+            .requireSuccess()
+
+    internal fun getBitBakeVersion(workingDir: File): String =
+        runBitBake(workingDir, "--version").stdout.lineSequence().first {
+            it.startsWith("BitBake Build Tool")
+        }.substringAfterLast(' ')
+
+    private fun extractResourceToTempFile(resourceName: String): File {
+        val prefix = resourceName.substringBefore('.')
+        val suffix = resourceName.substringAfter(prefix)
+        val scriptFile = createOrtTempFile(prefix, suffix)
+        val script = checkNotNull(javaClass.getResource("/$resourceName")).readText()
+
+        return scriptFile.apply { writeText(script) }
+    }
+}
+
+private const val INIT_SCRIPT_NAME = "oe-init-build-env"
+private const val BITBAKE_SCRIPT_NAME = "bitbake.sh"
+private const val SPDX_CONF_NAME = "spdx.conf"
+
+private fun File.searchUpwardsForFile(searchFileName: String): File? {
+    if (!isDirectory) return null
+
+    var currentDir: File? = absoluteFile
+
+    while (currentDir != null && !currentDir.resolve(searchFileName).isFile) {
+        currentDir = currentDir.parentFile
+    }
+
+    return currentDir
+}

plugins/package-managers/bitbake/src/main/resources/META-INF/services/org.ossreviewtoolkit.analyzer.PackageManagerFactory

@@ -0,0 +1 @@
+org.ossreviewtoolkit.plugins.packagemanagers.bitbake.BitBake$Factory

plugins/package-managers/bitbake/src/main/resources/bitbake.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# License-Filename: LICENSE
+
+BUILD_DIR=$1; shift
+
+if [ -z "$BBPATH" ] && [ -f oe-init-build-env ]; then
+    # Initialize the build environment.
+    . oe-init-build-env "$BUILD_DIR"
+fi
+
+bitbake "$@"