Add an initial BitBake analyzer implementation

Dockerfile

@@ -421,6 +421,17 @@
 COPY --from=bazelbuild /opt/bazel /opt/bazel
 COPY --from=bazelbuild /opt/go/bin/buildozer /opt/go/bin/buildozer
 
+#------------------------------------------------------------------------
+# BITBAKE
+FROM base as bitbakebuild
+
+ARG BITBAKE_VERSION
+
+RUN git clone --branch $BITBAKE_VERSION --depth 1 https://git.openembedded.org/bitbake /opt/bitbake
+
+FROM scratch as bitbake
+COPY --from=bitbakebuild /opt/bitbake /opt/bitbake
+
 #------------------------------------------------------------------------
 # ORT
 FROM base as ortbuild
@@ -558,6 +569,21 @@
 COPY --from=bazel $BAZEL_HOME $BAZEL_HOME
 COPY --from=bazel --chown=$USER:$USER /opt/go/bin/buildozer /opt/go/bin/buildozer
 
+# BitBake
+ENV BITBAKE_HOME=/opt/bitbake
+ENV PATH=$PATH:$BITBAKE_HOME/bin
+
+COPY --from=bitbake $BITBAKE_HOME $BITBAKE_HOME
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    sudo apt-get update && \
+    DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends \
+        bzip2 chrpath cpio diffstat gawk lz4 patch zstd \
+    && sudo rm -rf /var/lib/apt/lists/*
+
+RUN syft $BITBAKE_HOME -o spdx-json --output json=/usr/share/doc/ort/ort-bitbake.spdx.json
+
 #------------------------------------------------------------------------
 # Runtime container with minimal selection of supported package managers pre-installed.
 FROM minimal-tools as minimal

analyzer/src/funTest/kotlin/PackageManagerFunTest.kt

@@ -39,6 +39,7 @@ import org.ossreviewtoolkit.model.config.PathExcludeReason
 class PackageManagerFunTest : WordSpec({
     val definitionFiles = listOf(
         "bazel/MODULE.bazel",
+        "bitbake/recipe.bb",
         "bower/bower.json",
         "bundler/Gemfile",
         "cargo/Cargo.toml",

docker/versions.dockerfile

@@ -1,6 +1,7 @@
 ARG ANDROID_CMD_VERSION=11076708
 ARG ASKALONO_VERSION=0.4.6
 ARG BAZELISK_VERSION=1.20.0
+ARG BITBAKE_VERSION=2.8.0
 ARG BOWER_VERSION=1.8.14
 ARG BOYTERLC_VERSION=1.3.1
 ARG COCOAPODS_VERSION=1.15.2

plugins/package-managers/bitbake/README.md

@@ -0,0 +1,13 @@
+# About
+
+This is a package manager plugin for the [OSS Review Toolkit][ORT] to analyze [Yocto] projects managed by [BitBake].
+It supersedes the combination of the [meta-doubleopen] and [do-convert] projects by relying on upstream [SBOM] generation in [SPDX] format, and converting the generated files to an ORT analyzer result file via ORT's [SPDX document file analyzer].
+
+[ORT]: https://github.com/oss-review-toolkit/ort
+[BitBake]: https://docs.yoctoproject.org/bitbake.html
+[Yocto]: https://www.yoctoproject.org/
+[meta-doubleopen]: https://github.com/doubleopen-project/meta-doubleopen
+[do-convert]: https://github.com/doubleopen-project/do-convert
+[SBOM]: https://docs.yoctoproject.org/dev/dev-manual/sbom.html
+[SPDX]: https://spdx.dev/
+[SPDX document file analyzer]: https://oss-review-toolkit.org/ort/docs/tools/analyzer

plugins/package-managers/bitbake/build.gradle.kts

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+plugins {
+    // Apply precompiled plugins.
+    id("ort-library-conventions")
+}
+
+dependencies {
+    api(projects.analyzer)
+    api(projects.model)
+
+    implementation(projects.utils.commonUtils)
+    implementation(projects.plugins.packageManagers.spdxPackageManager)
+
+    funTestImplementation(projects.downloader)
+    funTestImplementation(projects.plugins.versionControlSystems.gitVersionControlSystem)
+    funTestImplementation(testFixtures(projects.analyzer))
+}

plugins/package-managers/bitbake/src/funTest/kotlin/BitBakeFunTest.kt

@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.plugins.packagemanagers.bitbake
+
+import io.kotest.core.spec.style.WordSpec
+import io.kotest.engine.spec.tempdir
+import io.kotest.matchers.collections.beEmpty
+import io.kotest.matchers.collections.shouldHaveSize
+import io.kotest.matchers.result.shouldBeSuccess
+import io.kotest.matchers.should
+import io.kotest.matchers.shouldBe
+import io.kotest.matchers.string.shouldMatch
+
+import org.ossreviewtoolkit.analyzer.Analyzer
+import org.ossreviewtoolkit.analyzer.create
+import org.ossreviewtoolkit.model.Identifier
+import org.ossreviewtoolkit.model.VcsInfo
+import org.ossreviewtoolkit.model.VcsType
+import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
+import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.Git
+import org.ossreviewtoolkit.utils.test.ExpensiveTag
+import org.ossreviewtoolkit.utils.test.shouldNotBeNull
+
+class BitBakeFunTest : WordSpec({
+    "BitBake" should {
+        "get the version correctly" {
+            val bitBake = create("BitBake") as BitBake
+
+            val version = bitBake.getBitBakeVersion(tempdir())
+
+            version shouldMatch "\\d+\\.\\d+\\.\\d+"
+        }
+    }
+
+    "Analyzing recipes from Poky" should {
+        val projectDir = tempdir()
+        val pokyVcsInfo = VcsInfo(VcsType.GIT, "https://git.yoctoproject.org/poky", "kirkstone-4.0.17")
+
+        Git().run {
+            val workingTree = initWorkingTree(projectDir, pokyVcsInfo)
+            updateWorkingTree(workingTree, pokyVcsInfo.revision)
+        } shouldBeSuccess pokyVcsInfo.revision
+
+        "create an SPDX file for the 'quilt-native' package" {
+            val recipeFileName = "quilt-native_0.67.bb"
+            val result = Analyzer(AnalyzerConfiguration()).run {
+                val fileInfo = findManagedFiles(projectDir)
+                val singleFileInfo = fileInfo.copy(
+                    managedFiles = fileInfo.managedFiles.map { (packageManager, definitionsFiles) ->
+                        packageManager to definitionsFiles.filter { it.name == recipeFileName }
+                    }.toMap()
+                )
+                analyze(singleFileInfo)
+            }
+
+            result.analyzer?.result shouldNotBeNull {
+                projects shouldHaveSize 1
+
+                with(projects.single()) {
+                    id shouldBe Identifier("BitBake:OpenEmbedded ():quilt-native:0.67")
+                    declaredLicenses shouldBe setOf("GPL-2.0-only")
+                    homepageUrl shouldBe "http://savannah.nongnu.org/projects/quilt/"
+                    scopes should beEmpty()
+                }
+            }
+        }
+
+        "create a SPDX files for the 'xmlto' package".config(tags = setOf(ExpensiveTag)) {
+            val recipeFileName = "xmlto_0.0.28.bb"
+            val result = Analyzer(AnalyzerConfiguration()).run {
+                val fileInfo = findManagedFiles(projectDir)
+                val singleFileInfo = fileInfo.copy(
+                    managedFiles = fileInfo.managedFiles.map { (packageManager, definitionsFiles) ->
+                        packageManager to definitionsFiles.filter { it.name == recipeFileName }
+                    }.toMap()
+                )
+                analyze(singleFileInfo)
+            }
+
+            result.analyzer?.result shouldNotBeNull {
+                projects shouldHaveSize 90
+            }
+        }
+    }
+})

plugins/package-managers/bitbake/src/main/kotlin/BitBake.kt

@@ -0,0 +1,166 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.plugins.packagemanagers.bitbake
+
+import java.io.File
+
+import kotlin.time.measureTime
+
+import org.apache.logging.log4j.kotlin.logger
+
+import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory
+import org.ossreviewtoolkit.analyzer.PackageManager
+import org.ossreviewtoolkit.analyzer.PackageManagerResult
+import org.ossreviewtoolkit.model.ProjectAnalyzerResult
+import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
+import org.ossreviewtoolkit.model.config.RepositoryConfiguration
+import org.ossreviewtoolkit.plugins.packagemanagers.spdx.SpdxDocumentFile
+import org.ossreviewtoolkit.utils.common.ProcessCapture
+import org.ossreviewtoolkit.utils.common.getCommonParentFile
+import org.ossreviewtoolkit.utils.common.safeDeleteRecursively
+import org.ossreviewtoolkit.utils.common.withoutPrefix
+import org.ossreviewtoolkit.utils.ort.createOrtTempDir
+import org.ossreviewtoolkit.utils.ort.createOrtTempFile
+
+/**
+ * A package manager that uses OpenEmbedded's "bitbake" tool to create SPDX SBOMs [1][2] e.g. for Yocto distributions,
+ * and post-processes these into ORT analyzer results.
+ *
+ * [1]: https://docs.yoctoproject.org/dev/dev-manual/sbom.html
+ * [2]: https://dev.to/angrymane/create-spdx-with-yocto-2od9
+ */
+class BitBake(
+    name: String,
+    analysisRoot: File,
+    analyzerConfig: AnalyzerConfiguration,
+    repoConfig: RepositoryConfiguration
+) : PackageManager(name, analysisRoot, analyzerConfig, repoConfig) {
+    class Factory : AbstractPackageManagerFactory<BitBake>("BitBake") {
+        override val globsForDefinitionFiles = listOf("*.bb")
+
+        override fun create(
+            analysisRoot: File,
+            analyzerConfig: AnalyzerConfiguration,
+            repoConfig: RepositoryConfiguration
+        ) = BitBake(type, analysisRoot, analyzerConfig, repoConfig)
+    }
+
+    private val scriptFile by lazy { extractResourceToTempFile(BITBAKE_SCRIPT_NAME).apply { setExecutable(true) } }
+    private val spdxConfFile by lazy { extractResourceToTempFile(SPDX_CONF_NAME) }
+
+    private val spdxManager by lazy { SpdxDocumentFile(name, analysisRoot, analyzerConfig, repoConfig) }
+
+    override fun resolveDependencies(definitionFiles: List<File>, labels: Map<String, String>): PackageManagerResult {
+        val commonDefinitionDir = getCommonParentFile(definitionFiles)
+        val workingDir = requireNotNull(commonDefinitionDir.searchUpwardsForFile(INIT_SCRIPT_NAME)) {
+            "No '$INIT_SCRIPT_NAME' script file found for directory '$commonDefinitionDir'."
+        }
+
+        logger.info { "Determined the working directory to be '$workingDir'." }
+
+        val localVersion = getBitBakeVersion(workingDir)
+        val globalVersion = createOrtTempDir().let { dir ->
+            getBitBakeVersion(dir).also { dir.safeDeleteRecursively(force = true) }
+        }
+
+        if (localVersion != globalVersion) {
+            logger.warn { "Local $managerName version $localVersion differs from global version $globalVersion." }
+        }
+
+        val deployDirs = mutableSetOf<File>()
+
+        definitionFiles.forEach { definitionFile ->
+            val target = definitionFile.nameWithoutExtension.substringBeforeLast('_')
+
+            val deployDir = getDeployDir(workingDir, target)
+            deployDirs += deployDir
+
+            val spdxFile = deployDir.findSpdxFiles().find { it.name == "recipe-$target.spdx.json" }
+            if (spdxFile != null) {
+                logger.info { "Not creating SPDX files for target '$target' as it already exists at '$spdxFile'." }
+            } else {
+                logger.info { "Creating SPDX files for target '$target'..." }
+
+                // This implicitly triggers the build and can take a very long time.
+                val duration = measureTime { createSpdx(workingDir, target) }
+
+                logger.info { "Creating SPDX files for target '$target' took $duration." }
+            }
+        }
+
+        if (!scriptFile.delete()) logger.warn { "Unable to delete the temporary '$scriptFile' file." }
+        if (!spdxConfFile.delete()) logger.warn { "Unable to delete the temporary '$spdxConfFile' file." }
+
+        val commonDeployDir = deployDirs.singleOrNull() ?: getCommonParentFile(deployDirs)
+        val spdxFiles = commonDeployDir.findSpdxFiles().toList()
+
+        logger.info { "Found ${spdxFiles.size} SPDX file(s) in '$commonDeployDir'." }
+
+        return spdxManager.resolveDependencies(spdxFiles, labels)
+    }
+
+    override fun resolveDependencies(definitionFile: File, labels: Map<String, String>): List<ProjectAnalyzerResult> =
+        throw NotImplementedError("This function is not supported for $managerName.")
+
+    private fun getDeployDir(workingDir: File, target: String): File {
+        val bitbakeEnv = runBitBake(workingDir, "-e", target)
+        return bitbakeEnv.stdout.lineSequence().mapNotNull { it.withoutPrefix("DEPLOY_DIR=") }.first()
+            .let { File(it.removeSurrounding("\"")) }
+    }
+
+    private fun createSpdx(workingDir: File, target: String) =
+        runBitBake(workingDir, "-r", spdxConfFile.absolutePath, "-c", "create_spdx", target)
+
+    private fun File.findSpdxFiles() = resolve("spdx").walk().filter { it.isFile && it.name.endsWith(".spdx.json") }
+
+    private fun runBitBake(workingDir: File, vararg args: String): ProcessCapture =
+        ProcessCapture(scriptFile.absolutePath, workingDir.absolutePath, *args, workingDir = workingDir)
+            .requireSuccess()
+
+    internal fun getBitBakeVersion(workingDir: File): String =
+        runBitBake(workingDir, "--version").stdout.lineSequence().first {
+            it.startsWith("BitBake Build Tool")
+        }.substringAfterLast(' ')
+
+    private fun extractResourceToTempFile(resourceName: String): File {
+        val prefix = resourceName.substringBefore('.')
+        val suffix = resourceName.substringAfter(prefix)
+        val scriptFile = createOrtTempFile(prefix, suffix)
+        val script = checkNotNull(javaClass.getResource("/$resourceName")).readText()
+
+        return scriptFile.apply { writeText(script) }
+    }
+}
+
+private const val INIT_SCRIPT_NAME = "oe-init-build-env"
+private const val BITBAKE_SCRIPT_NAME = "bitbake.sh"
+private const val SPDX_CONF_NAME = "spdx.conf"
+
+private fun File.searchUpwardsForFile(searchFileName: String): File? {
+    if (!isDirectory) return null
+
+    var currentDir: File? = absoluteFile
+
+    while (currentDir != null && !currentDir.resolve(searchFileName).isFile) {
+        currentDir = currentDir.parentFile
+    }
+
+    return currentDir
+}

plugins/package-managers/bitbake/src/main/resources/META-INF/services/org.ossreviewtoolkit.analyzer.PackageManagerFactory

@@ -0,0 +1 @@
+org.ossreviewtoolkit.plugins.packagemanagers.bitbake.BitBake$Factory