Add attribute to track private packages #3102

Reference: #3102 Reference: #1514 Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
aboutcode-org · May 30, 2024 · 1869154 · 1869154
1 parent 5d34442
commit 1869154
Show file tree

Hide file tree

Showing 1,140 changed files with 12,061 additions and 3,601 deletions.
diff --git a/src/packagedcode/bower.py b/src/packagedcode/bower.py
@@ -29,8 +29,10 @@ def parse(cls, location, package_only=False):
         with io.open(location, encoding='utf-8') as loc:
             package_data = json.load(loc)
 
-        # note: having no name is not a problem for private packages. See #1514
         name = package_data.get('name')
+        is_private = False
+        if not name:
+            is_private = True
 
         description = package_data.get('description')
         version = package_data.get('version')
@@ -99,5 +101,6 @@ def parse(cls, location, package_only=False):
             homepage_url=homepage_url,
             vcs_url=vcs_url,
             dependencies=dependencies,
+            is_private=is_private,
         )
         yield models.PackageData.from_data(package_data, package_only)
diff --git a/src/packagedcode/models.py b/src/packagedcode/models.py
@@ -682,6 +682,14 @@ class PackageData(IdentifiablePackageData):
              'package type or datafile format.'
     )
 
+    is_private = Boolean(
+        default=False,
+        label='is resolved flag',
+        help='True if the associated package for this package manifest '
+             'is never meant to be published to the corresponding package '
+             'repository, and is a private package.'
+    )
+
     extra_data = Mapping(
         label='extra data',
         help='A mapping of arbitrary extra package data.',

diff --git a/src/packagedcode/npm.py b/src/packagedcode/npm.py
@@ -212,7 +212,17 @@ def update_dependencies_by_purl(
 
                 if scope in metadata_deps :
                     dep_package = dependecies_by_purl.get(dep_purl)
-                    dep_package.is_optional = metadata.get("optional")
+                    if dep_package:
+                        dep_package.is_optional = metadata.get("optional")
+                    else:
+                        dep_package = models.DependentPackage(
+                            purl=dep_purl,
+                            scope=scope,
+                            is_runtime=is_runtime,
+                            is_optional=metadata.get("optional"),
+                            is_resolved=is_resolved,
+                        )
+                        dependecies_by_purl[dep_purl] = dep_package
                     continue
 
                 # pnpm has peer dependencies also sometimes in version?
@@ -266,7 +276,11 @@ def _parse(cls, json_data, package_only=False):
 
         namespace, name = split_scoped_package_name(name)
 
-        urls = get_urls(namespace, name, version)
+        is_private = json_data.get('private') or False
+        if is_private:
+            urls = {}
+        else:
+            urls = get_urls(namespace, name, version)
         package_data = dict(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,
@@ -276,6 +290,7 @@ def _parse(cls, json_data, package_only=False):
             version=version or None,
             description=json_data.get('description', '').strip() or None,
             homepage_url=homepage_url,
+            is_private=is_private,
             **urls,
         )
         package = models.PackageData.from_data(package_data, package_only)

diff --git a/src/packagedcode/phpcomposer.py b/src/packagedcode/phpcomposer.py
@@ -111,6 +111,7 @@ def build_package_data(package_data, package_only=False):
         repository_homepage_url=get_repository_homepage_url(ns, name),
         api_data_url=get_api_data_url(ns, name),
         primary_language=PhpComposerJsonHandler.default_primary_language,
+        is_private=is_private,
     )
     package = models.PackageData.from_data(package_mapping, package_only)
 

diff --git a/tests/formattedcode/data/common/manifests-expected.json b/tests/formattedcode/data/common/manifests-expected.json
@@ -73,6 +73,7 @@
       "source_packages": [
         "pkg:maven/javax.persistence/persistence-api@1.0?classifier=sources"
       ],
+      "is_private": false,
       "extra_data": {},
       "repository_homepage_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/",
       "repository_download_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/persistence-api-1.0.jar",
@@ -174,6 +175,7 @@
       "extracted_license_statement": "- Apache-2.0\n- type: Apache 2.0\n  url: https://github.com/spenceralger/grunt-esvm/blob/master/LICENSE.md\n",
       "notice_text": null,
       "source_packages": [],
+      "is_private": false,
       "extra_data": {},
       "repository_homepage_url": "https://www.npmjs.com/package/grunt-esvm",
       "repository_download_url": "https://registry.npmjs.org/grunt-esvm/-/grunt-esvm-3.2.8.tgz",
@@ -251,6 +253,7 @@
       "extracted_license_statement": "- MIT\n",
       "notice_text": null,
       "source_packages": [],
+      "is_private": false,
       "extra_data": {},
       "repository_homepage_url": "https://www.npmjs.com/package/angular-compare-validator",
       "repository_download_url": "https://registry.npmjs.org/angular-compare-validator/-/angular-compare-validator-0.1.1.tgz",
@@ -892,6 +895,7 @@
             "pkg:maven/javax.persistence/persistence-api@1.0?classifier=sources"
           ],
           "file_references": [],
+          "is_private": false,
           "extra_data": {},
           "dependencies": [],
           "repository_homepage_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/",
@@ -1096,6 +1100,7 @@
           "notice_text": null,
           "source_packages": [],
           "file_references": [],
+          "is_private": false,
           "extra_data": {},
           "dependencies": [
             {
@@ -1390,6 +1395,7 @@
           "notice_text": null,
           "source_packages": [],
           "file_references": [],
+          "is_private": false,
           "extra_data": {},
           "dependencies": [
             {
@@ -1639,6 +1645,7 @@
           "notice_text": null,
           "source_packages": [],
           "file_references": [],
+          "is_private": false,
           "extra_data": {},
           "dependencies": [
             {

diff --git a/tests/formattedcode/data/common/manifests-expected.jsonlines b/tests/formattedcode/data/common/manifests-expected.jsonlines
@@ -12,19 +12,19 @@
           "--package": true
         },
         "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-        "output_format_version": "3.0.0",
+        "output_format_version": "3.1.0",
         "message": null,
         "errors": [],
         "warnings": [],
         "extra_data": {
           "system_environment": {
             "operating_system": "linux",
             "cpu_architecture": "64",
-            "platform": "Linux-5.15.0-89-generic-x86_64-with-glibc2.29",
-            "platform_version": "#99~20.04.1-Ubuntu SMP Thu Nov 2 15:16:47 UTC 2023",
-            "python_version": "3.8.10 (default, Nov 22 2023, 10:22:35) \n[GCC 9.4.0]"
+            "platform": "Linux-5.15.0-106-generic-x86_64-with-glibc2.35",
+            "platform_version": "#116-Ubuntu SMP Wed Apr 17 09:17:56 UTC 2024",
+            "python_version": "3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0]"
           },
-          "spdx_license_list_version": "3.22",
+          "spdx_license_list_version": "3.23",
           "files_count": 4
         }
       }
@@ -105,6 +105,7 @@
         "source_packages": [
           "pkg:maven/javax.persistence/persistence-api@1.0?classifier=sources"
         ],
+        "is_private": false,
         "extra_data": {},
         "repository_homepage_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/",
         "repository_download_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/persistence-api-1.0.jar",
@@ -206,6 +207,7 @@
         "extracted_license_statement": "- Apache-2.0\n- type: Apache 2.0\n  url: https://github.com/spenceralger/grunt-esvm/blob/master/LICENSE.md\n",
         "notice_text": null,
         "source_packages": [],
+        "is_private": false,
         "extra_data": {},
         "repository_homepage_url": "https://www.npmjs.com/package/grunt-esvm",
         "repository_download_url": "https://registry.npmjs.org/grunt-esvm/-/grunt-esvm-3.2.8.tgz",
@@ -283,6 +285,7 @@
         "extracted_license_statement": "- MIT\n",
         "notice_text": null,
         "source_packages": [],
+        "is_private": false,
         "extra_data": {},
         "repository_homepage_url": "https://www.npmjs.com/package/angular-compare-validator",
         "repository_download_url": "https://registry.npmjs.org/angular-compare-validator/-/angular-compare-validator-0.1.1.tgz",
@@ -938,6 +941,7 @@
               "pkg:maven/javax.persistence/persistence-api@1.0?classifier=sources"
             ],
             "file_references": [],
+            "is_private": false,
             "extra_data": {},
             "dependencies": [],
             "repository_homepage_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/",
@@ -1150,6 +1154,7 @@
             "notice_text": null,
             "source_packages": [],
             "file_references": [],
+            "is_private": false,
             "extra_data": {},
             "dependencies": [
               {
@@ -1452,6 +1457,7 @@
             "notice_text": null,
             "source_packages": [],
             "file_references": [],
+            "is_private": false,
             "extra_data": {},
             "dependencies": [
               {
@@ -1709,6 +1715,7 @@
             "notice_text": null,
             "source_packages": [],
             "file_references": [],
+            "is_private": false,
             "extra_data": {},
             "dependencies": [
               {

diff --git a/tests/formattedcode/data/common/manifests-expected.yaml b/tests/formattedcode/data/common/manifests-expected.yaml
@@ -21,18 +21,18 @@ headers:
       for any legal advice.
       ScanCode is a free software code scanning tool from nexB Inc. and others.
       Visit https://github.com/nexB/scancode-toolkit/ for support and download.
-    output_format_version: 3.0.0
+    output_format_version: 3.1.0
     message:
     errors: []
     warnings: []
     extra_data:
       system_environment:
         operating_system: linux
         cpu_architecture: 64
-        platform: Linux-5.15.0-89-generic-x86_64-with-glibc2.29
-        platform_version: '#99~20.04.1-Ubuntu SMP Thu Nov 2 15:16:47 UTC 2023'
-        python_version: "3.8.10 (default, Nov 22 2023, 10:22:35) \n[GCC 9.4.0]"
-      spdx_license_list_version: '3.22'
+        platform: Linux-5.15.0-106-generic-x86_64-with-glibc2.35
+        platform_version: '#116-Ubuntu SMP Wed Apr 17 09:17:56 UTC 2024'
+        python_version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0]
+      spdx_license_list_version: '3.23'
       files_count: 4
 summary:
   declared_license_expression: apache-2.0 AND cddl-1.0 AND mit
@@ -130,6 +130,7 @@ packages:
     notice_text:
     source_packages:
       - pkg:maven/javax.persistence/persistence-api@1.0?classifier=sources
+    is_private: no
     extra_data: {}
     repository_homepage_url: https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/
     repository_download_url: https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/persistence-api-1.0.jar
@@ -215,6 +216,7 @@ packages:
         url: https://github.com/spenceralger/grunt-esvm/blob/master/LICENSE.md
     notice_text:
     source_packages: []
+    is_private: no
     extra_data: {}
     repository_homepage_url: https://www.npmjs.com/package/grunt-esvm
     repository_download_url: https://registry.npmjs.org/grunt-esvm/-/grunt-esvm-3.2.8.tgz
@@ -280,6 +282,7 @@ packages:
       - MIT
     notice_text:
     source_packages: []
+    is_private: no
     extra_data: {}
     repository_homepage_url: https://www.npmjs.com/package/angular-compare-validator
     repository_download_url: https://registry.npmjs.org/angular-compare-validator/-/angular-compare-validator-0.1.1.tgz
@@ -1791,6 +1794,7 @@ files:
         source_packages:
           - pkg:maven/javax.persistence/persistence-api@1.0?classifier=sources
         file_references: []
+        is_private: no
         extra_data: {}
         dependencies: []
         repository_homepage_url: https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/
@@ -1995,6 +1999,7 @@ files:
         notice_text:
         source_packages: []
         file_references: []
+        is_private: no
         extra_data: {}
         dependencies:
           - purl: pkg:npm/bluebird
@@ -2269,6 +2274,7 @@ files:
         notice_text:
         source_packages: []
         file_references: []
+        is_private: no
         extra_data: {}
         dependencies:
           - purl: pkg:npm/%40angular/core
@@ -2512,6 +2518,7 @@ files:
         notice_text:
         source_packages: []
         file_references: []
+        is_private: no
         extra_data: {}
         dependencies:
           - purl: pkg:pypi/numpy