Skip to content

Commit

Permalink
refactor(node)!: Make Npm separate from Yarn
Browse files Browse the repository at this point in the history
Analog to 0eb1eea, remove the inheritance between the two managers and
re-write large parts of `Npm` to extract all needed information solely
based on the output of the `npm` CLI command, instead of relying on the
file hierarchy under the `node_modules` directory.

This reduces complexity and makes the implementation(s) easy to
understand, maintain and change in isolation.

Note: The handling of the `installIssues` in `Yarn` has been used only
for `Npm`, which is why that code is moved from `Yarn` to `Npm`.

Signed-off-by: Frank Viernau <frank_viernau@epam.com>
  • Loading branch information
fviernau committed Nov 8, 2024
1 parent 0d06faa commit 03560a5
Show file tree
Hide file tree
Showing 4 changed files with 241 additions and 32 deletions.
43 changes: 43 additions & 0 deletions plugins/package-managers/node/src/main/kotlin/npm/ModuleInfo.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.plugins.packagemanagers.node.npm

import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json

private val JSON = Json { ignoreUnknownKeys = true }

internal fun parseNpmList(json: String): ModuleInfo = JSON.decodeFromString(json)

@Serializable
internal data class ModuleInfo(
val name: String = "",
val version: String = "",
val path: String? = null,
@SerialName("_id")
val id: String? = null,
@SerialName("_dependencies")
val dependencyConstraints: Map<String, String> = emptyMap(),
val dependencies: Map<String, ModuleInfo> = emptyMap(),
val optional: Boolean = false,
val dev: Boolean = false,
val resolved: String? = null
)
126 changes: 119 additions & 7 deletions plugins/package-managers/node/src/main/kotlin/npm/Npm.kt
Original file line number Diff line number Diff line change
Expand Up @@ -22,22 +22,32 @@
package org.ossreviewtoolkit.plugins.packagemanagers.node.npm

import java.io.File
import java.util.LinkedList

import org.apache.logging.log4j.kotlin.logger

import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory
import org.ossreviewtoolkit.analyzer.PackageManager
import org.ossreviewtoolkit.analyzer.PackageManagerResult
import org.ossreviewtoolkit.model.DependencyGraph
import org.ossreviewtoolkit.model.Issue
import org.ossreviewtoolkit.model.Project
import org.ossreviewtoolkit.model.ProjectAnalyzerResult
import org.ossreviewtoolkit.model.Severity
import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
import org.ossreviewtoolkit.model.config.PackageManagerConfiguration
import org.ossreviewtoolkit.model.config.RepositoryConfiguration
import org.ossreviewtoolkit.model.utils.DependencyGraphBuilder
import org.ossreviewtoolkit.plugins.packagemanagers.node.NodePackageManager
import org.ossreviewtoolkit.plugins.packagemanagers.node.NpmDetection
import org.ossreviewtoolkit.plugins.packagemanagers.node.PackageJson
import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackageJson
import org.ossreviewtoolkit.plugins.packagemanagers.node.yarn.Yarn
import org.ossreviewtoolkit.plugins.packagemanagers.node.parseProject
import org.ossreviewtoolkit.utils.common.CommandLineTool
import org.ossreviewtoolkit.utils.common.Os
import org.ossreviewtoolkit.utils.common.ProcessCapture
import org.ossreviewtoolkit.utils.common.collectMessages
import org.ossreviewtoolkit.utils.common.stashDirectories
import org.ossreviewtoolkit.utils.common.withoutPrefix

import org.semver4j.RangesList
Expand All @@ -57,7 +67,7 @@ class Npm(
analysisRoot: File,
analyzerConfig: AnalyzerConfiguration,
repoConfig: RepositoryConfiguration
) : Yarn(name, analysisRoot, analyzerConfig, repoConfig) {
) : PackageManager(name, analysisRoot, analyzerConfig, repoConfig), CommandLineTool {
companion object {
/** Name of the configuration option to toggle legacy peer dependency support. */
const val OPTION_LEGACY_PEER_DEPS = "legacyPeerDeps"
Expand All @@ -74,10 +84,54 @@ class Npm(
}

private val legacyPeerDeps = options[OPTION_LEGACY_PEER_DEPS].toBoolean()

private val npmViewCache = mutableMapOf<String, PackageJson>()
private val handler = NpmDependencyHandler(this)
private val graphBuilder by lazy { DependencyGraphBuilder(handler) }

override fun resolveDependencies(definitionFile: File, labels: Map<String, String>): List<ProjectAnalyzerResult> =
stashDirectories(definitionFile.resolveSibling("node_modules")).use {
resolveDependencies(definitionFile)
}

private fun resolveDependencies(definitionFile: File): List<ProjectAnalyzerResult> {
val workingDir = definitionFile.parentFile
val installIssues = installDependencies(workingDir)

if (installIssues.any { it.severity == Severity.ERROR }) {
val project = runCatching {
parseProject(definitionFile, analysisRoot, managerName)
}.getOrElse {
logger.error { "Failed to parse project information: ${it.collectMessages()}" }
Project.EMPTY
}

override fun hasLockfile(projectDir: File) = NodePackageManager.NPM.hasLockfile(projectDir)
return listOf(ProjectAnalyzerResult(project, emptySet(), installIssues))
}

val project = parseProject(definitionFile, analysisRoot, managerName)
val projectModuleInfo = listModules(workingDir).undoDeduplication()

val scopeNames = Scope.entries
.filterNot { excludes.isScopeExcluded(it.descriptor) }
.mapTo(mutableSetOf()) { scope ->
val scopeName = scope.descriptor
val qualifiedScopeName = DependencyGraph.qualifyScope(project, scopeName)

projectModuleInfo.getScopeDependencies(scope).forEach { dependency ->
graphBuilder.addDependency(qualifiedScopeName, dependency)
}

scopeName
}

return ProjectAnalyzerResult(
project = project.copy(scopeNames = scopeNames),
packages = emptySet(),
issues = installIssues
).let { listOf(it) }
}

private fun hasLockfile(projectDir: File) = NodePackageManager.NPM.hasLockfile(projectDir)

override fun command(workingDir: File?) = if (Os.isWindows) "npm.cmd" else "npm"

Expand All @@ -92,7 +146,16 @@ class Npm(
checkVersion()
}

override fun getRemotePackageDetails(workingDir: File, packageName: String): PackageJson? {
override fun createPackageManagerResult(projectResults: Map<File, List<ProjectAnalyzerResult>>) =
PackageManagerResult(projectResults, graphBuilder.build(), graphBuilder.packages())

private fun listModules(workingDir: File): ModuleInfo {
val json = run(workingDir, "list", "--depth", "Infinity", "--json", "--long").stdout

return parseNpmList(json)
}

internal fun getRemotePackageDetails(workingDir: File, packageName: String): PackageJson? {
npmViewCache[packageName]?.let { return it }

return runCatching {
Expand All @@ -106,16 +169,65 @@ class Npm(
}.getOrNull()
}

override fun runInstall(workingDir: File): ProcessCapture {
private fun installDependencies(workingDir: File): List<Issue> {
requireLockfile(workingDir) { hasLockfile(workingDir) }

val options = listOfNotNull(
"--ignore-scripts",
"--no-audit",
"--legacy-peer-deps".takeIf { legacyPeerDeps }
)

val subcommand = if (hasLockfile(workingDir)) "ci" else "install"
return ProcessCapture(workingDir, command(workingDir), subcommand, *options.toTypedArray())

val process = ProcessCapture(workingDir, command(workingDir), subcommand, *options.toTypedArray())

return process.extractNpmIssues()
}
}

private enum class Scope(val descriptor: String) {
DEPENDENCIES("dependencies"),
DEV_DEPENDENCIES("devDependencies")
}

private fun ModuleInfo.getScopeDependencies(scope: Scope) =
when (scope) {
Scope.DEPENDENCIES -> dependencies.values.filter { !it.dev }
Scope.DEV_DEPENDENCIES -> dependencies.values.filter { it.dev && !it.optional }
}

private fun ModuleInfo.undoDeduplication(): ModuleInfo {
val replacements = getNonDeduplicatedModuleInfosForId()

fun ModuleInfo.undoDeduplicationRec(ancestorsIds: Set<String> = emptySet()): ModuleInfo {
val dependencyAncestorIds = ancestorsIds + setOfNotNull(id)
val dependencies = (replacements[id] ?: this)
.dependencies
.filter { it.value.id !in dependencyAncestorIds } // break cycles.
.mapValues { it.value.undoDeduplicationRec(dependencyAncestorIds) }

return copy(dependencies = dependencies)
}

return undoDeduplicationRec()
}

private fun ModuleInfo.getNonDeduplicatedModuleInfosForId(): Map<String, ModuleInfo> {
val queue = LinkedList<ModuleInfo>().apply { add(this@getNonDeduplicatedModuleInfosForId) }
val result = mutableMapOf<String, ModuleInfo>()

while (queue.isNotEmpty()) {
val info = queue.removeFirst()

if (info.id != null && info.dependencyConstraints.keys.subtract(info.dependencies.keys).isEmpty()) {
result[info.id] = info
}

queue += info.dependencies.values
}

return result
}

internal fun List<String>.groupLines(vararg markers: String): List<String> {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
/*
* Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.plugins.packagemanagers.node.npm

import java.io.File

import org.ossreviewtoolkit.model.Identifier
import org.ossreviewtoolkit.model.Issue
import org.ossreviewtoolkit.model.Package
import org.ossreviewtoolkit.model.PackageLinkage
import org.ossreviewtoolkit.model.utils.DependencyHandler
import org.ossreviewtoolkit.plugins.packagemanagers.node.PackageJson
import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackage
import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackageJson
import org.ossreviewtoolkit.utils.common.realFile

internal class NpmDependencyHandler(private val npm: Npm) : DependencyHandler<ModuleInfo> {
private val packageJsonCache = mutableMapOf<File, PackageJson>()

override fun identifierFor(dependency: ModuleInfo): Identifier {
val type = npm.managerName.takeIf { dependency.isProject } ?: "NPM"
val namespace = dependency.name.substringBeforeLast("/", "")
val name = dependency.name.substringAfterLast("/")
val version = if (dependency.isProject) {
readPackageJson(dependency.packageJsonFile).version.orEmpty()
} else {
dependency.version.takeUnless { it.startsWith("link:") || it.startsWith("file:") }.orEmpty()
}

return Identifier(type, namespace, name, version)
}

override fun dependenciesFor(dependency: ModuleInfo): List<ModuleInfo> =
dependency.dependencies.values.filter { it.isInstalled }

override fun linkageFor(dependency: ModuleInfo): PackageLinkage =
PackageLinkage.DYNAMIC.takeUnless { dependency.isProject } ?: PackageLinkage.PROJECT_DYNAMIC

override fun createPackage(dependency: ModuleInfo, issues: MutableCollection<Issue>): Package? =
dependency.takeUnless { it.isProject || !it.isInstalled }?.let {
parsePackage(
workingDir = it.workingDir,
packageJsonFile = it.packageJsonFile,
getRemotePackageDetails = npm::getRemotePackageDetails
)
}

private fun readPackageJson(packageJsonFile: File): PackageJson =
packageJsonCache.getOrPut(packageJsonFile.realFile()) { parsePackageJson(packageJsonFile) }
}

private val ModuleInfo.workingDir: File get() = File(path)

private val ModuleInfo.isInstalled: Boolean get() = path != null

private val ModuleInfo.isProject: Boolean get() = resolved == null

private val ModuleInfo.packageJsonFile: File get() = File(path, "package.json")
29 changes: 4 additions & 25 deletions plugins/package-managers/node/src/main/kotlin/yarn/Yarn.kt
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ import org.ossreviewtoolkit.model.Identifier
import org.ossreviewtoolkit.model.Issue
import org.ossreviewtoolkit.model.Project
import org.ossreviewtoolkit.model.ProjectAnalyzerResult
import org.ossreviewtoolkit.model.Severity
import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
import org.ossreviewtoolkit.model.config.RepositoryConfiguration
import org.ossreviewtoolkit.model.createAndLogIssue
Expand All @@ -53,14 +52,12 @@ import org.ossreviewtoolkit.model.utils.DependencyGraphBuilder
import org.ossreviewtoolkit.plugins.packagemanagers.node.NodePackageManager
import org.ossreviewtoolkit.plugins.packagemanagers.node.NpmDetection
import org.ossreviewtoolkit.plugins.packagemanagers.node.PackageJson
import org.ossreviewtoolkit.plugins.packagemanagers.node.npm.extractNpmIssues
import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackageJson
import org.ossreviewtoolkit.plugins.packagemanagers.node.parseProject
import org.ossreviewtoolkit.plugins.packagemanagers.node.splitNpmNamespaceAndName
import org.ossreviewtoolkit.utils.common.CommandLineTool
import org.ossreviewtoolkit.utils.common.DiskCache
import org.ossreviewtoolkit.utils.common.Os
import org.ossreviewtoolkit.utils.common.ProcessCapture
import org.ossreviewtoolkit.utils.common.alsoIfNull
import org.ossreviewtoolkit.utils.common.collectMessages
import org.ossreviewtoolkit.utils.common.fieldNamesOrEmpty
Expand Down Expand Up @@ -178,25 +175,12 @@ open class Yarn(
// Actually installing the dependencies is the easiest way to get the metadata of all transitive
// dependencies (i.e. their respective "package.json" files). As NPM uses a global cache, the same
// dependency is only ever downloaded once.
val installIssues = installDependencies(workingDir)

if (installIssues.any { it.severity == Severity.ERROR }) {
val project = runCatching {
parseProject(definitionFile, analysisRoot, managerName)
}.getOrElse {
logger.error { "Failed to parse project information: ${it.collectMessages()}" }
Project.EMPTY
}

return listOf(ProjectAnalyzerResult(project, emptySet(), installIssues))
}
installDependencies(workingDir)

val projectDirs = findWorkspaceSubmodules(workingDir).toSet() + definitionFile.parentFile

return projectDirs.map { projectDir ->
val issues = mutableListOf<Issue>().apply {
if (projectDir == workingDir) addAll(installIssues)
}
val issues = mutableListOf<Issue>()

val project = runCatching {
parseProject(projectDir.resolve("package.json"), analysisRoot, managerName)
Expand Down Expand Up @@ -367,17 +351,12 @@ open class Yarn(
/**
* Install dependencies using the given package manager command.
*/
private fun installDependencies(workingDir: File): List<Issue> {
private fun installDependencies(workingDir: File) {
requireLockfile(workingDir) { hasLockfile(workingDir) }

// Install all NPM dependencies to enable NPM to list dependencies.
val process = runInstall(workingDir)

return process.extractNpmIssues()
}

protected open fun runInstall(workingDir: File): ProcessCapture =
run(workingDir, "install", "--ignore-scripts", "--ignore-engines", "--immutable")
}

internal open fun getRemotePackageDetails(workingDir: File, packageName: String): PackageJson? {
yarnInfoCache.read(packageName)?.let { return parsePackageJson(it) }
Expand Down

0 comments on commit 03560a5

Please sign in to comment.