refactor(node): Move two model mapping functions to NpmSupport

The functions `parseProject()` and `parsePackage()` are used by mutliple package managers. Move them to `NpmSupport` which contains common code to account for that. Signed-off-by: Frank Viernau <frank_viernau@epam.com>
oss-review-toolkit · Nov 6, 2024 · f566a2d · f566a2d
1 parent 9d63529
commit f566a2d
Show file tree

Hide file tree

Showing 6 changed files with 154 additions and 156 deletions.
diff --git a/plugins/package-managers/node/src/main/kotlin/Npm.kt b/plugins/package-managers/node/src/main/kotlin/Npm.kt
@@ -26,32 +26,13 @@ import java.io.File
 import org.apache.logging.log4j.kotlin.logger
 
 import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory
-import org.ossreviewtoolkit.analyzer.PackageManager.Companion.getFallbackProjectName
-import org.ossreviewtoolkit.analyzer.PackageManager.Companion.processPackageVcs
-import org.ossreviewtoolkit.analyzer.PackageManager.Companion.processProjectVcs
-import org.ossreviewtoolkit.downloader.VcsHost
-import org.ossreviewtoolkit.downloader.VersionControlSystem
-import org.ossreviewtoolkit.model.Hash
-import org.ossreviewtoolkit.model.Identifier
-import org.ossreviewtoolkit.model.Package
-import org.ossreviewtoolkit.model.Project
-import org.ossreviewtoolkit.model.RemoteArtifact
-import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
 import org.ossreviewtoolkit.model.config.PackageManagerConfiguration
 import org.ossreviewtoolkit.model.config.RepositoryConfiguration
-import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NON_EXISTING_SEMVER
 import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NodePackageManager
 import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NpmDetection
-import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.expandNpmShortcutUrl
-import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.fixNpmDownloadUrl
-import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.mapNpmLicenses
-import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.parseNpmAuthor
-import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.parseNpmVcsInfo
-import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.splitNpmNamespaceAndName
 import org.ossreviewtoolkit.utils.common.Os
 import org.ossreviewtoolkit.utils.common.ProcessCapture
-import org.ossreviewtoolkit.utils.common.realFile
 import org.ossreviewtoolkit.utils.common.withoutPrefix
 
 import org.semver4j.RangesList
@@ -197,137 +178,3 @@ internal fun List<String>.groupLines(vararg markers: String): List<String> {
         nonFooterLines.map { it.trim() }
     }
 }
-
-/**
- * Construct a [Package] by parsing its _package.json_ file and - if applicable - querying additional
- * content via the `npm view` command. The result is a [Pair] with the raw identifier and the new package.
- */
-internal fun parsePackage(
-    workingDir: File,
-    packageJsonFile: File,
-    getRemotePackageDetails: (workingDir: File, packageName: String) -> PackageJson?
-): Package {
-    val packageJson = parsePackageJson(packageJsonFile)
-
-    // The "name" and "version" fields are only required if the package is going to be published, otherwise they are
-    // optional, see
-    // - https://docs.npmjs.com/cli/v10/configuring-npm/package-json#name
-    // - https://docs.npmjs.com/cli/v10/configuring-npm/package-json#version
-    // So, projects analyzed by ORT might not have these fields set.
-    val rawName = packageJson.name.orEmpty() // TODO: Fall back to a generated name if the name is unset.
-    val (namespace, name) = splitNpmNamespaceAndName(rawName)
-    val version = packageJson.version ?: NON_EXISTING_SEMVER
-
-    val declaredLicenses = packageJson.licenses.mapNpmLicenses()
-    val authors = parseNpmAuthor(packageJson.authors.firstOrNull()) // TODO: parse all authors.
-
-    var description = packageJson.description.orEmpty()
-    var homepageUrl = packageJson.homepage.orEmpty()
-
-    // Note that all fields prefixed with "_" are considered private to NPM and should not be relied on.
-    var downloadUrl = expandNpmShortcutUrl(packageJson.resolved.orEmpty()).ifEmpty {
-        // If the normalized form of the specified dependency contains a URL as the version, expand and use it.
-        val fromVersion = packageJson.from.orEmpty().substringAfterLast('@')
-        expandNpmShortcutUrl(fromVersion).takeIf { it != fromVersion }.orEmpty()
-    }
-
-    var hash = Hash.create(packageJson.integrity.orEmpty())
-
-    var vcsFromPackage = parseNpmVcsInfo(packageJson)
-
-    val id = Identifier("NPM", namespace, name, version)
-
-    val hasIncompleteData = description.isEmpty() || homepageUrl.isEmpty() || downloadUrl.isEmpty()
-        || hash == Hash.NONE || vcsFromPackage == VcsInfo.EMPTY
-
-    if (hasIncompleteData) {
-        getRemotePackageDetails(workingDir, "$rawName@$version")?.let { details ->
-            if (description.isEmpty()) description = details.description.orEmpty()
-            if (homepageUrl.isEmpty()) homepageUrl = details.homepage.orEmpty()
-
-            details.dist?.let { dist ->
-                if (downloadUrl.isEmpty() || hash == Hash.NONE) {
-                    downloadUrl = dist.tarball.orEmpty()
-                    hash = Hash.create(dist.shasum.orEmpty())
-                }
-            }
-
-            // Do not replace but merge, because it happens that `package.json` has VCS info while
-            // `npm view` doesn't, for example for dependencies hosted on GitLab package registry.
-            vcsFromPackage = vcsFromPackage.merge(parseNpmVcsInfo(details))
-        }
-    }
-
-    downloadUrl = downloadUrl.fixNpmDownloadUrl()
-
-    val vcsFromDownloadUrl = VcsHost.parseUrl(downloadUrl)
-    if (vcsFromDownloadUrl.url != downloadUrl) {
-        vcsFromPackage = vcsFromPackage.merge(vcsFromDownloadUrl)
-    }
-
-    val module = Package(
-        id = id,
-        authors = authors,
-        declaredLicenses = declaredLicenses,
-        description = description,
-        homepageUrl = homepageUrl,
-        binaryArtifact = RemoteArtifact.EMPTY,
-        sourceArtifact = RemoteArtifact(
-            url = VcsHost.toArchiveDownloadUrl(vcsFromDownloadUrl) ?: downloadUrl,
-            hash = hash
-        ),
-        vcs = vcsFromPackage,
-        vcsProcessed = processPackageVcs(vcsFromPackage, homepageUrl)
-    )
-
-    require(module.id.name.isNotEmpty()) {
-        "Generated package info for '${id.toCoordinates()}' has no name."
-    }
-
-    require(module.id.version.isNotEmpty()) {
-        "Generated package info for '${id.toCoordinates()}' has no version."
-    }
-
-    return module
-}
-
-internal fun parseProject(packageJsonFile: File, analysisRoot: File, managerName: String): Project {
-    Npm.logger.debug { "Parsing project info from '$packageJsonFile'." }
-
-    val packageJson = parsePackageJson(packageJsonFile)
-
-    val rawName = packageJson.name.orEmpty()
-    val (namespace, name) = splitNpmNamespaceAndName(rawName)
-
-    val projectName = name.ifBlank {
-        getFallbackProjectName(analysisRoot, packageJsonFile).also {
-            Npm.logger.warn { "'$packageJsonFile' does not define a name, falling back to '$it'." }
-        }
-    }
-
-    val version = packageJson.version.orEmpty()
-    if (version.isBlank()) {
-        Npm.logger.warn { "'$packageJsonFile' does not define a version." }
-    }
-
-    val declaredLicenses = packageJson.licenses.mapNpmLicenses()
-    val authors = parseNpmAuthor(packageJson.authors.firstOrNull()) // TODO: parse all authors.
-    val homepageUrl = packageJson.homepage.orEmpty()
-    val projectDir = packageJsonFile.parentFile.realFile()
-    val vcsFromPackage = parseNpmVcsInfo(packageJson)
-
-    return Project(
-        id = Identifier(
-            type = managerName,
-            namespace = namespace,
-            name = projectName,
-            version = version
-        ),
-        definitionFilePath = VersionControlSystem.getPathInfo(packageJsonFile.realFile()).path,
-        authors = authors,
-        declaredLicenses = declaredLicenses,
-        vcs = vcsFromPackage,
-        vcsProcessed = processProjectVcs(projectDir, vcsFromPackage, homepageUrl),
-        homepageUrl = homepageUrl
-    )
-}
diff --git a/plugins/package-managers/node/src/main/kotlin/Yarn.kt b/plugins/package-managers/node/src/main/kotlin/Yarn.kt
@@ -54,6 +54,7 @@ import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NodePackageManage
 import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NpmDetection
 import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NpmModuleInfo
 import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.YarnDependencyHandler
+import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.parseProject
 import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.splitNpmNamespaceAndName
 import org.ossreviewtoolkit.utils.common.CommandLineTool
 import org.ossreviewtoolkit.utils.common.DiskCache

diff --git a/plugins/package-managers/node/src/main/kotlin/pnpm/Pnpm.kt b/plugins/package-managers/node/src/main/kotlin/pnpm/Pnpm.kt
@@ -33,9 +33,9 @@ import org.ossreviewtoolkit.model.config.RepositoryConfiguration
 import org.ossreviewtoolkit.model.utils.DependencyGraphBuilder
 import org.ossreviewtoolkit.plugins.packagemanagers.node.PackageJson
 import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackageJson
-import org.ossreviewtoolkit.plugins.packagemanagers.node.parseProject
 import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NodePackageManager
 import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.NpmDetection
+import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.parseProject
 import org.ossreviewtoolkit.utils.common.CommandLineTool
 import org.ossreviewtoolkit.utils.common.Os
 import org.ossreviewtoolkit.utils.common.stashDirectories

diff --git a/plugins/package-managers/node/src/main/kotlin/pnpm/PnpmDependencyHandler.kt b/plugins/package-managers/node/src/main/kotlin/pnpm/PnpmDependencyHandler.kt
@@ -27,9 +27,9 @@ import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.PackageLinkage
 import org.ossreviewtoolkit.model.utils.DependencyHandler
 import org.ossreviewtoolkit.plugins.packagemanagers.node.PackageJson
-import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackage
 import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackageJson
 import org.ossreviewtoolkit.plugins.packagemanagers.node.pnpm.ModuleInfo.Dependency
+import org.ossreviewtoolkit.plugins.packagemanagers.node.utils.parsePackage
 import org.ossreviewtoolkit.utils.common.realFile
 
 internal class PnpmDependencyHandler(private val pnpm: Pnpm) : DependencyHandler<Dependency> {

diff --git a/plugins/package-managers/node/src/main/kotlin/utils/NpmSupport.kt b/plugins/package-managers/node/src/main/kotlin/utils/NpmSupport.kt
@@ -19,10 +19,27 @@
 
 package org.ossreviewtoolkit.plugins.packagemanagers.node.utils
 
+import java.io.File
+
+import org.apache.logging.log4j.kotlin.logger
+
+import org.ossreviewtoolkit.analyzer.PackageManager.Companion.getFallbackProjectName
+import org.ossreviewtoolkit.analyzer.PackageManager.Companion.processPackageVcs
+import org.ossreviewtoolkit.analyzer.PackageManager.Companion.processProjectVcs
 import org.ossreviewtoolkit.analyzer.parseAuthorString
+import org.ossreviewtoolkit.downloader.VcsHost
+import org.ossreviewtoolkit.downloader.VersionControlSystem
+import org.ossreviewtoolkit.model.Hash
+import org.ossreviewtoolkit.model.Identifier
+import org.ossreviewtoolkit.model.Package
+import org.ossreviewtoolkit.model.Project
+import org.ossreviewtoolkit.model.RemoteArtifact
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
+import org.ossreviewtoolkit.plugins.packagemanagers.node.Npm
 import org.ossreviewtoolkit.plugins.packagemanagers.node.PackageJson
+import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackageJson
+import org.ossreviewtoolkit.utils.common.realFile
 import org.ossreviewtoolkit.utils.common.toUri
 import org.ossreviewtoolkit.utils.spdx.SpdxConstants
 
@@ -137,6 +154,140 @@ internal fun parseNpmVcsInfo(packageJson: PackageJson): VcsInfo {
     )
 }
 
+/**
+ * Construct a [Package] by parsing its _package.json_ file and - if applicable - querying additional
+ * content via the `npm view` command. The result is a [Pair] with the raw identifier and the new package.
+ */
+internal fun parsePackage(
+    workingDir: File,
+    packageJsonFile: File,
+    getRemotePackageDetails: (workingDir: File, packageName: String) -> PackageJson?
+): Package {
+    val packageJson = parsePackageJson(packageJsonFile)
+
+    // The "name" and "version" fields are only required if the package is going to be published, otherwise they are
+    // optional, see
+    // - https://docs.npmjs.com/cli/v10/configuring-npm/package-json#name
+    // - https://docs.npmjs.com/cli/v10/configuring-npm/package-json#version
+    // So, projects analyzed by ORT might not have these fields set.
+    val rawName = packageJson.name.orEmpty() // TODO: Fall back to a generated name if the name is unset.
+    val (namespace, name) = splitNpmNamespaceAndName(rawName)
+    val version = packageJson.version ?: NON_EXISTING_SEMVER
+
+    val declaredLicenses = packageJson.licenses.mapNpmLicenses()
+    val authors = parseNpmAuthor(packageJson.authors.firstOrNull()) // TODO: parse all authors.
+
+    var description = packageJson.description.orEmpty()
+    var homepageUrl = packageJson.homepage.orEmpty()
+
+    // Note that all fields prefixed with "_" are considered private to NPM and should not be relied on.
+    var downloadUrl = expandNpmShortcutUrl(packageJson.resolved.orEmpty()).ifEmpty {
+        // If the normalized form of the specified dependency contains a URL as the version, expand and use it.
+        val fromVersion = packageJson.from.orEmpty().substringAfterLast('@')
+        expandNpmShortcutUrl(fromVersion).takeIf { it != fromVersion }.orEmpty()
+    }
+
+    var hash = Hash.create(packageJson.integrity.orEmpty())
+
+    var vcsFromPackage = parseNpmVcsInfo(packageJson)
+
+    val id = Identifier("NPM", namespace, name, version)
+
+    val hasIncompleteData = description.isEmpty() || homepageUrl.isEmpty() || downloadUrl.isEmpty()
+        || hash == Hash.NONE || vcsFromPackage == VcsInfo.EMPTY
+
+    if (hasIncompleteData) {
+        getRemotePackageDetails(workingDir, "$rawName@$version")?.let { details ->
+            if (description.isEmpty()) description = details.description.orEmpty()
+            if (homepageUrl.isEmpty()) homepageUrl = details.homepage.orEmpty()
+
+            details.dist?.let { dist ->
+                if (downloadUrl.isEmpty() || hash == Hash.NONE) {
+                    downloadUrl = dist.tarball.orEmpty()
+                    hash = Hash.create(dist.shasum.orEmpty())
+                }
+            }
+
+            // Do not replace but merge, because it happens that `package.json` has VCS info while
+            // `npm view` doesn't, for example for dependencies hosted on GitLab package registry.
+            vcsFromPackage = vcsFromPackage.merge(parseNpmVcsInfo(details))
+        }
+    }
+
+    downloadUrl = downloadUrl.fixNpmDownloadUrl()
+
+    val vcsFromDownloadUrl = VcsHost.parseUrl(downloadUrl)
+    if (vcsFromDownloadUrl.url != downloadUrl) {
+        vcsFromPackage = vcsFromPackage.merge(vcsFromDownloadUrl)
+    }
+
+    val module = Package(
+        id = id,
+        authors = authors,
+        declaredLicenses = declaredLicenses,
+        description = description,
+        homepageUrl = homepageUrl,
+        binaryArtifact = RemoteArtifact.EMPTY,
+        sourceArtifact = RemoteArtifact(
+            url = VcsHost.toArchiveDownloadUrl(vcsFromDownloadUrl) ?: downloadUrl,
+            hash = hash
+        ),
+        vcs = vcsFromPackage,
+        vcsProcessed = processPackageVcs(vcsFromPackage, homepageUrl)
+    )
+
+    require(module.id.name.isNotEmpty()) {
+        "Generated package info for '${id.toCoordinates()}' has no name."
+    }
+
+    require(module.id.version.isNotEmpty()) {
+        "Generated package info for '${id.toCoordinates()}' has no version."
+    }
+
+    return module
+}
+
+internal fun parseProject(packageJsonFile: File, analysisRoot: File, managerName: String): Project {
+    Npm.logger.debug { "Parsing project info from '$packageJsonFile'." }
+
+    val packageJson = parsePackageJson(packageJsonFile)
+
+    val rawName = packageJson.name.orEmpty()
+    val (namespace, name) = splitNpmNamespaceAndName(rawName)
+
+    val projectName = name.ifBlank {
+        getFallbackProjectName(analysisRoot, packageJsonFile).also {
+            Npm.logger.warn { "'$packageJsonFile' does not define a name, falling back to '$it'." }
+        }
+    }
+
+    val version = packageJson.version.orEmpty()
+    if (version.isBlank()) {
+        Npm.logger.warn { "'$packageJsonFile' does not define a version." }
+    }
+
+    val declaredLicenses = packageJson.licenses.mapNpmLicenses()
+    val authors = parseNpmAuthor(packageJson.authors.firstOrNull()) // TODO: parse all authors.
+    val homepageUrl = packageJson.homepage.orEmpty()
+    val projectDir = packageJsonFile.parentFile.realFile()
+    val vcsFromPackage = parseNpmVcsInfo(packageJson)
+
+    return Project(
+        id = Identifier(
+            type = managerName,
+            namespace = namespace,
+            name = projectName,
+            version = version
+        ),
+        definitionFilePath = VersionControlSystem.getPathInfo(packageJsonFile.realFile()).path,
+        authors = authors,
+        declaredLicenses = declaredLicenses,
+        vcs = vcsFromPackage,
+        vcsProcessed = processProjectVcs(projectDir, vcsFromPackage, homepageUrl),
+        homepageUrl = homepageUrl
+    )
+}
+
 /**
  * Split the given [rawName] of a module to a pair with namespace and name.
  */

diff --git a/plugins/package-managers/node/src/main/kotlin/utils/YarnDependencyHandler.kt b/plugins/package-managers/node/src/main/kotlin/utils/YarnDependencyHandler.kt
@@ -28,7 +28,6 @@ import org.ossreviewtoolkit.model.PackageLinkage
 import org.ossreviewtoolkit.model.Project
 import org.ossreviewtoolkit.model.utils.DependencyHandler
 import org.ossreviewtoolkit.plugins.packagemanagers.node.Yarn
-import org.ossreviewtoolkit.plugins.packagemanagers.node.parsePackage
 
 /**
  * A data class storing information about a specific NPM module and its dependencies.