From 03560a58b357023745b1717b29330437e837b476 Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Thu, 7 Nov 2024 14:00:17 +0100 Subject: [PATCH] refactor(node)!: Make `Npm` separate from `Yarn` Analog to 0eb1eea, remove the inheritance between the two managers and re-write large parts of `Npm` to extract all needed information solely based on the output of the `npm` CLI command, instead of relying on the file hierarchy under the `node_modules` directory. This reduces complexity and makes the implementation(s) easy to understand, maintain and change in isolation. Note: The handling of the `installIssues` in `Yarn` has been used only for `Npm`, which is why that code is moved from `Yarn` to `Npm`. Signed-off-by: Frank Viernau --- .../node/src/main/kotlin/npm/ModuleInfo.kt | 43 ++++++ .../node/src/main/kotlin/npm/Npm.kt | 126 +++++++++++++++++- .../main/kotlin/npm/NpmDependencyHandler.kt | 75 +++++++++++ .../node/src/main/kotlin/yarn/Yarn.kt | 29 +--- 4 files changed, 241 insertions(+), 32 deletions(-) create mode 100644 plugins/package-managers/node/src/main/kotlin/npm/ModuleInfo.kt create mode 100644 plugins/package-managers/node/src/main/kotlin/npm/NpmDependencyHandler.kt diff --git a/plugins/package-managers/node/src/main/kotlin/npm/ModuleInfo.kt b/plugins/package-managers/node/src/main/kotlin/npm/ModuleInfo.kt new file mode 100644 index 000000000000..fd76f0918a46 --- /dev/null +++ b/plugins/package-managers/node/src/main/kotlin/npm/ModuleInfo.kt @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagemanagers.node.npm + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json + +private val JSON = Json { ignoreUnknownKeys = true } + +internal fun parseNpmList(json: String): ModuleInfo = JSON.decodeFromString(json) + +@Serializable +internal data class ModuleInfo( + val name: String = "", + val version: String = "", + val path: String? = null, + @SerialName("_id") + val id: String? = null, + @SerialName("_dependencies") + val dependencyConstraints: Map = emptyMap(), + val dependencies: Map = emptyMap(), + val optional: Boolean = false, + val dev: Boolean = false, + val resolved: String? = null +) diff --git a/plugins/package-managers/node/src/main/kotlin/npm/Npm.kt b/plugins/package-managers/node/src/main/kotlin/npm/Npm.kt index cceb0f2a7550..666557b2b36a 100644 --- a/plugins/package-managers/node/src/main/kotlin/npm/Npm.kt +++ b/plugins/package-managers/node/src/main/kotlin/npm/Npm.kt @@ -22,22 +22,32 @@ package org.ossreviewtoolkit.plugins.packagemanagers.node.npm import java.io.File +import java.util.LinkedList import org.apache.logging.log4j.kotlin.logger import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory +import org.ossreviewtoolkit.analyzer.PackageManager +import org.ossreviewtoolkit.analyzer.PackageManagerResult +import org.ossreviewtoolkit.model.DependencyGraph import org.ossreviewtoolkit.model.Issue +import org.ossreviewtoolkit.model.Project +import org.ossreviewtoolkit.model.ProjectAnalyzerResult import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.config.AnalyzerConfiguration import org.ossreviewtoolkit.model.config.PackageManagerConfiguration import org.ossreviewtoolkit.model.config.RepositoryConfiguration +import org.ossreviewtoolkit.model.utils.DependencyGraphBuilder import org.ossreviewtoolkit.plugins.packagemanagers.node.NodePackageManager import org.ossreviewtoolkit.plugins.packagemanagers.node.NpmDetection import org.ossreviewtoolkit.plugins.packagemanagers.node.PackageJson import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackageJson -import org.ossreviewtoolkit.plugins.packagemanagers.node.yarn.Yarn +import org.ossreviewtoolkit.plugins.packagemanagers.node.parseProject +import org.ossreviewtoolkit.utils.common.CommandLineTool import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.common.ProcessCapture +import org.ossreviewtoolkit.utils.common.collectMessages +import org.ossreviewtoolkit.utils.common.stashDirectories import org.ossreviewtoolkit.utils.common.withoutPrefix import org.semver4j.RangesList @@ -57,7 +67,7 @@ class Npm( analysisRoot: File, analyzerConfig: AnalyzerConfiguration, repoConfig: RepositoryConfiguration -) : Yarn(name, analysisRoot, analyzerConfig, repoConfig) { +) : PackageManager(name, analysisRoot, analyzerConfig, repoConfig), CommandLineTool { companion object { /** Name of the configuration option to toggle legacy peer dependency support. */ const val OPTION_LEGACY_PEER_DEPS = "legacyPeerDeps" @@ -74,10 +84,54 @@ class Npm( } private val legacyPeerDeps = options[OPTION_LEGACY_PEER_DEPS].toBoolean() - private val npmViewCache = mutableMapOf() + private val handler = NpmDependencyHandler(this) + private val graphBuilder by lazy { DependencyGraphBuilder(handler) } + + override fun resolveDependencies(definitionFile: File, labels: Map): List = + stashDirectories(definitionFile.resolveSibling("node_modules")).use { + resolveDependencies(definitionFile) + } + + private fun resolveDependencies(definitionFile: File): List { + val workingDir = definitionFile.parentFile + val installIssues = installDependencies(workingDir) + + if (installIssues.any { it.severity == Severity.ERROR }) { + val project = runCatching { + parseProject(definitionFile, analysisRoot, managerName) + }.getOrElse { + logger.error { "Failed to parse project information: ${it.collectMessages()}" } + Project.EMPTY + } - override fun hasLockfile(projectDir: File) = NodePackageManager.NPM.hasLockfile(projectDir) + return listOf(ProjectAnalyzerResult(project, emptySet(), installIssues)) + } + + val project = parseProject(definitionFile, analysisRoot, managerName) + val projectModuleInfo = listModules(workingDir).undoDeduplication() + + val scopeNames = Scope.entries + .filterNot { excludes.isScopeExcluded(it.descriptor) } + .mapTo(mutableSetOf()) { scope -> + val scopeName = scope.descriptor + val qualifiedScopeName = DependencyGraph.qualifyScope(project, scopeName) + + projectModuleInfo.getScopeDependencies(scope).forEach { dependency -> + graphBuilder.addDependency(qualifiedScopeName, dependency) + } + + scopeName + } + + return ProjectAnalyzerResult( + project = project.copy(scopeNames = scopeNames), + packages = emptySet(), + issues = installIssues + ).let { listOf(it) } + } + + private fun hasLockfile(projectDir: File) = NodePackageManager.NPM.hasLockfile(projectDir) override fun command(workingDir: File?) = if (Os.isWindows) "npm.cmd" else "npm" @@ -92,7 +146,16 @@ class Npm( checkVersion() } - override fun getRemotePackageDetails(workingDir: File, packageName: String): PackageJson? { + override fun createPackageManagerResult(projectResults: Map>) = + PackageManagerResult(projectResults, graphBuilder.build(), graphBuilder.packages()) + + private fun listModules(workingDir: File): ModuleInfo { + val json = run(workingDir, "list", "--depth", "Infinity", "--json", "--long").stdout + + return parseNpmList(json) + } + + internal fun getRemotePackageDetails(workingDir: File, packageName: String): PackageJson? { npmViewCache[packageName]?.let { return it } return runCatching { @@ -106,7 +169,9 @@ class Npm( }.getOrNull() } - override fun runInstall(workingDir: File): ProcessCapture { + private fun installDependencies(workingDir: File): List { + requireLockfile(workingDir) { hasLockfile(workingDir) } + val options = listOfNotNull( "--ignore-scripts", "--no-audit", @@ -114,8 +179,55 @@ class Npm( ) val subcommand = if (hasLockfile(workingDir)) "ci" else "install" - return ProcessCapture(workingDir, command(workingDir), subcommand, *options.toTypedArray()) + + val process = ProcessCapture(workingDir, command(workingDir), subcommand, *options.toTypedArray()) + + return process.extractNpmIssues() + } +} + +private enum class Scope(val descriptor: String) { + DEPENDENCIES("dependencies"), + DEV_DEPENDENCIES("devDependencies") +} + +private fun ModuleInfo.getScopeDependencies(scope: Scope) = + when (scope) { + Scope.DEPENDENCIES -> dependencies.values.filter { !it.dev } + Scope.DEV_DEPENDENCIES -> dependencies.values.filter { it.dev && !it.optional } + } + +private fun ModuleInfo.undoDeduplication(): ModuleInfo { + val replacements = getNonDeduplicatedModuleInfosForId() + + fun ModuleInfo.undoDeduplicationRec(ancestorsIds: Set = emptySet()): ModuleInfo { + val dependencyAncestorIds = ancestorsIds + setOfNotNull(id) + val dependencies = (replacements[id] ?: this) + .dependencies + .filter { it.value.id !in dependencyAncestorIds } // break cycles. + .mapValues { it.value.undoDeduplicationRec(dependencyAncestorIds) } + + return copy(dependencies = dependencies) + } + + return undoDeduplicationRec() +} + +private fun ModuleInfo.getNonDeduplicatedModuleInfosForId(): Map { + val queue = LinkedList().apply { add(this@getNonDeduplicatedModuleInfosForId) } + val result = mutableMapOf() + + while (queue.isNotEmpty()) { + val info = queue.removeFirst() + + if (info.id != null && info.dependencyConstraints.keys.subtract(info.dependencies.keys).isEmpty()) { + result[info.id] = info + } + + queue += info.dependencies.values } + + return result } internal fun List.groupLines(vararg markers: String): List { diff --git a/plugins/package-managers/node/src/main/kotlin/npm/NpmDependencyHandler.kt b/plugins/package-managers/node/src/main/kotlin/npm/NpmDependencyHandler.kt new file mode 100644 index 000000000000..bc28c34b2803 --- /dev/null +++ b/plugins/package-managers/node/src/main/kotlin/npm/NpmDependencyHandler.kt @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagemanagers.node.npm + +import java.io.File + +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.model.Issue +import org.ossreviewtoolkit.model.Package +import org.ossreviewtoolkit.model.PackageLinkage +import org.ossreviewtoolkit.model.utils.DependencyHandler +import org.ossreviewtoolkit.plugins.packagemanagers.node.PackageJson +import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackage +import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackageJson +import org.ossreviewtoolkit.utils.common.realFile + +internal class NpmDependencyHandler(private val npm: Npm) : DependencyHandler { + private val packageJsonCache = mutableMapOf() + + override fun identifierFor(dependency: ModuleInfo): Identifier { + val type = npm.managerName.takeIf { dependency.isProject } ?: "NPM" + val namespace = dependency.name.substringBeforeLast("/", "") + val name = dependency.name.substringAfterLast("/") + val version = if (dependency.isProject) { + readPackageJson(dependency.packageJsonFile).version.orEmpty() + } else { + dependency.version.takeUnless { it.startsWith("link:") || it.startsWith("file:") }.orEmpty() + } + + return Identifier(type, namespace, name, version) + } + + override fun dependenciesFor(dependency: ModuleInfo): List = + dependency.dependencies.values.filter { it.isInstalled } + + override fun linkageFor(dependency: ModuleInfo): PackageLinkage = + PackageLinkage.DYNAMIC.takeUnless { dependency.isProject } ?: PackageLinkage.PROJECT_DYNAMIC + + override fun createPackage(dependency: ModuleInfo, issues: MutableCollection): Package? = + dependency.takeUnless { it.isProject || !it.isInstalled }?.let { + parsePackage( + workingDir = it.workingDir, + packageJsonFile = it.packageJsonFile, + getRemotePackageDetails = npm::getRemotePackageDetails + ) + } + + private fun readPackageJson(packageJsonFile: File): PackageJson = + packageJsonCache.getOrPut(packageJsonFile.realFile()) { parsePackageJson(packageJsonFile) } +} + +private val ModuleInfo.workingDir: File get() = File(path) + +private val ModuleInfo.isInstalled: Boolean get() = path != null + +private val ModuleInfo.isProject: Boolean get() = resolved == null + +private val ModuleInfo.packageJsonFile: File get() = File(path, "package.json") diff --git a/plugins/package-managers/node/src/main/kotlin/yarn/Yarn.kt b/plugins/package-managers/node/src/main/kotlin/yarn/Yarn.kt index 141b6e37685a..824f5661959e 100644 --- a/plugins/package-managers/node/src/main/kotlin/yarn/Yarn.kt +++ b/plugins/package-managers/node/src/main/kotlin/yarn/Yarn.kt @@ -44,7 +44,6 @@ import org.ossreviewtoolkit.model.Identifier import org.ossreviewtoolkit.model.Issue import org.ossreviewtoolkit.model.Project import org.ossreviewtoolkit.model.ProjectAnalyzerResult -import org.ossreviewtoolkit.model.Severity import org.ossreviewtoolkit.model.config.AnalyzerConfiguration import org.ossreviewtoolkit.model.config.RepositoryConfiguration import org.ossreviewtoolkit.model.createAndLogIssue @@ -53,14 +52,12 @@ import org.ossreviewtoolkit.model.utils.DependencyGraphBuilder import org.ossreviewtoolkit.plugins.packagemanagers.node.NodePackageManager import org.ossreviewtoolkit.plugins.packagemanagers.node.NpmDetection import org.ossreviewtoolkit.plugins.packagemanagers.node.PackageJson -import org.ossreviewtoolkit.plugins.packagemanagers.node.npm.extractNpmIssues import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackageJson import org.ossreviewtoolkit.plugins.packagemanagers.node.parseProject import org.ossreviewtoolkit.plugins.packagemanagers.node.splitNpmNamespaceAndName import org.ossreviewtoolkit.utils.common.CommandLineTool import org.ossreviewtoolkit.utils.common.DiskCache import org.ossreviewtoolkit.utils.common.Os -import org.ossreviewtoolkit.utils.common.ProcessCapture import org.ossreviewtoolkit.utils.common.alsoIfNull import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.common.fieldNamesOrEmpty @@ -178,25 +175,12 @@ open class Yarn( // Actually installing the dependencies is the easiest way to get the metadata of all transitive // dependencies (i.e. their respective "package.json" files). As NPM uses a global cache, the same // dependency is only ever downloaded once. - val installIssues = installDependencies(workingDir) - - if (installIssues.any { it.severity == Severity.ERROR }) { - val project = runCatching { - parseProject(definitionFile, analysisRoot, managerName) - }.getOrElse { - logger.error { "Failed to parse project information: ${it.collectMessages()}" } - Project.EMPTY - } - - return listOf(ProjectAnalyzerResult(project, emptySet(), installIssues)) - } + installDependencies(workingDir) val projectDirs = findWorkspaceSubmodules(workingDir).toSet() + definitionFile.parentFile return projectDirs.map { projectDir -> - val issues = mutableListOf().apply { - if (projectDir == workingDir) addAll(installIssues) - } + val issues = mutableListOf() val project = runCatching { parseProject(projectDir.resolve("package.json"), analysisRoot, managerName) @@ -367,17 +351,12 @@ open class Yarn( /** * Install dependencies using the given package manager command. */ - private fun installDependencies(workingDir: File): List { + private fun installDependencies(workingDir: File) { requireLockfile(workingDir) { hasLockfile(workingDir) } // Install all NPM dependencies to enable NPM to list dependencies. - val process = runInstall(workingDir) - - return process.extractNpmIssues() - } - - protected open fun runInstall(workingDir: File): ProcessCapture = run(workingDir, "install", "--ignore-scripts", "--ignore-engines", "--immutable") + } internal open fun getRemotePackageDetails(workingDir: File, packageName: String): PackageJson? { yarnInfoCache.read(packageName)?.let { return parsePackageJson(it) }