🐛 Use direct endpoint instead of search to find repository URL from n…

…pm database (ossf#4118) * Update endpoint used when getting repo from npm to solve ossf#3166 Signed-off-by: aklevans <alexklevans@gmail.com> * Update test files to account for endpoint change when getting repo from npm Signed-off-by: aklevans <alexklevans@gmail.com> * Fix linter issues Signed-off-by: aklevans <alexklevans@gmail.com> * Added unit tests for ossf#3166 and ossf#2441 Signed-off-by: aklevans <alexklevans@gmail.com> * fix linter issues and reduce mock json output in package_manager_test to only include necessary data Signed-off-by: aklevans <alexklevans@gmail.com> * fix linter issues in package_managers.go Signed-off-by: aklevans <alexklevans@gmail.com> * convert windows line breaks to linux Signed-off-by: aklevans <alexklevans@gmail.com> * reduce test case size, still has windows line breaks Signed-off-by: aklevans <alexklevans@gmail.com> * Fix unit tests Signed-off-by: aklevans <alexklevans@gmail.com> * attempt linter fix Signed-off-by: aklevans <alexklevans@gmail.com> * Fix linter issues stemming from windows line breaks Signed-off-by: aklevans <alexklevans@gmail.com> * Remove magic number and rename variable to be more accurate Signed-off-by: aklevans <alexklevans@gmail.com> --------- Signed-off-by: aklevans <alexklevans@gmail.com> Signed-off-by: aklevans <105876795+aklevans@users.noreply.github.com> Signed-off-by: balteraivshay <avishay.balter@gmail.com>
balteravishay · Jun 12, 2024 · 4af6463 · 4af6463
1 parent 4c7e4f4
commit 4af6463
Show file tree

Hide file tree

Showing 2 changed files with 51 additions and 56 deletions.
diff --git a/cmd/package_managers.go b/cmd/package_managers.go
@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net/http"
 	"regexp"
 	"strings"
 
@@ -106,14 +107,10 @@ func fetchGitRepositoryFromPackageManagers(npm, pypi, rubygems, nuget string,
 	return packageMangerResponse{}, nil
 }
 
-type npmSearchResults struct {
-	Objects []struct {
-		Package struct {
-			Links struct {
-				Repository string `json:"repository"`
-			} `json:"links"`
-		} `json:"package"`
-	} `json:"objects"`
+type npmResult struct {
+	Repository struct {
+		URL string `json:"url"`
+	} `json:"repository"`
 }
 
 type pypiSearchResults struct {
@@ -129,23 +126,24 @@ type rubyGemsSearchResults struct {
 
 // Gets the GitHub repository URL for the npm package.
 func fetchGitRepositoryFromNPM(packageName string, packageManager pmc.Client) (string, error) {
-	npmSearchURL := "https://registry.npmjs.org/-/v1/search?text=%s&size=1"
-	resp, err := packageManager.Get(npmSearchURL, packageName)
+	npmGetURL := "https://registry.npmjs.org/%s/latest"
+
+	resp, err := packageManager.Get(npmGetURL, packageName)
 	if err != nil {
 		return "", sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("failed to get npm package json: %v", err))
 	}
 
 	defer resp.Body.Close()
-	v := &npmSearchResults{}
+	v := &npmResult{}
 	err = json.NewDecoder(resp.Body).Decode(v)
 	if err != nil {
 		return "", sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("failed to parse npm package json: %v", err))
 	}
-	if len(v.Objects) == 0 {
+	if resp.StatusCode == http.StatusNotFound || v.Repository.URL == "" {
 		return "", sce.WithMessage(sce.ErrScorecardInternal,
 			fmt.Sprintf("could not find source repo for npm package: %s", packageName))
 	}
-	return v.Objects[0].Package.Links.Repository, nil
+	return strings.TrimPrefix(strings.TrimSuffix(v.Repository.URL, ".git"), "git+"), nil
 }
 
 func findGitRepositoryInPYPIResponse(packageName string, response io.Reader) (string, error) {

diff --git a/cmd/package_managers_test.go b/cmd/package_managers_test.go
@@ -48,53 +48,48 @@ func Test_fetchGitRepositoryFromNPM(t *testing.T) {
 				packageName: "npm-package",
 				result: `
 {
-  "objects": [
-    {
-      "package": {
-        "name": "@pulumi/pulumi",
-        "scope": "pulumi",
-        "version": "3.26.0",
-        "description": "Pulumi's Node.js SDK",
-        "date": "2022-03-09T14:05:40.682Z",
-        "links": {
-          "homepage": "https://github.com/pulumi/pulumi#readme",
-          "repository": "https://github.com/pulumi/pulumi",
-          "bugs": "https://github.com/pulumi/pulumi/issues"
-        },
-        "publisher": {
-          "username": "pulumi-bot",
-          "email": "bot@pulumi.com"
-        },
-        "maintainers": [
-          {
-            "username": "joeduffy",
-            "email": "joe@pulumi.com"
-          },
-          {
-            "username": "pulumi-bot",
-            "email": "bot@pulumi.com"
-          }
-        ]
-      },
-      "score": {
-        "final": 0.4056031974977145,
-        "detail": {
-          "quality": 0.7308571951451065,
-          "popularity": 0.19908392082147397,
-          "maintenance": 0.3333333333333333
-        }
-      },
-      "searchScore": 0.00090895034
-    }
-  ],
-  "total": 380,
-  "time": "Wed Mar 09 2022 18:11:10 GMT+0000 (Coordinated Universal Time)"
+  "name": "@pulumi/pulumi",
+  "version": "3.116.1",
+  "description": "Pulumi's Node.js SDK",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/pulumi/pulumi.git",
+    "directory": "sdk/nodejs"
+  }
+ 
 }
 				`,
 			},
 			want:    "https://github.com/pulumi/pulumi",
 			wantErr: false,
 		},
+		{
+			name: "fetchGitRepositoryFromNPM",
+
+			args: args{
+				packageName: "left-pad",
+				result: `
+{
+  "name": "left-pad",
+  "version": "1.3.0",
+  "description": "String left pad",
+  "main": "index.js",
+  "types": "index.d.ts",
+  "scripts": {
+    "test": "node test",
+    "bench": "node perf/perf.js"
+  },
+  "repository": {
+    "url": "git+ssh://git@github.com/stevemao/left-pad.git",
+    "type": "git"
+  }
+}
+        `,
+			},
+			want:    "ssh://git@github.com/stevemao/left-pad",
+			wantErr: false,
+		},
 		{
 			name: "fetchGitRepositoryFromNPM_error",
 
@@ -109,8 +104,10 @@ func Test_fetchGitRepositoryFromNPM(t *testing.T) {
 			name: "fetchGitRepositoryFromNPM_error",
 
 			args: args{
-				packageName: "npm-package",
-				result:      "foo",
+				packageName: "https://github.com/airbnb/lottie-web",
+				result: `
+        {"code":"ResourceNotFound","message":"/https:/github.com/airbnb/lottie-web does not exist"}
+        `,
 			},
 			want:    "",
 			wantErr: true,