Prefer using PKG-INFO from .egg-info in assemble #3083

* Add test for checking that the .egg-info PKG-INFO is the only Package source reported * Update test expectations Signed-off-by: Jono Yang <jyang@nexb.com>
aboutcode-org · Sep 6, 2022 · d6b68d2 · d6b68d2
1 parent 2e971c4
commit d6b68d2
Show file tree

Hide file tree

Showing 19 changed files with 10,158 additions and 183 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,6 +11,7 @@
 /src/*.egg-info
 *.egg-info
 !tests/packagedcode/data/pypi/source-package/pip-22.0.4/src/pip.egg-info
+!tests/packagedcode/data/pypi/unpacked_sdist/prefer-egg-info-pkg-info/celery/celery.egg-info
 /dist
 /build
 /bin

diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py
@@ -29,6 +29,7 @@
 import pkginfo2
 from commoncode import fileutils
 from commoncode.fileutils import as_posixpath
+from commoncode.resource import Resource
 from packaging.specifiers import SpecifierSet
 from packageurl import PackageURL
 from packaging import markers
@@ -133,6 +134,16 @@ def create_package_from_package_data(package_data, datafile_path):
     return package
 
 
+def is_egg_info_directory(resource):
+    """
+    Return True if `resource` is a Python .egg-info directory
+    """
+    return (
+        isinstance(resource, Resource)
+        and resource.path.endswith('.egg-info')
+    )
+
+
 class BaseExtractedPythonLayout(BasePypiHandler):
     """
     Base class for development repos, sdist tarballs and other related extracted
@@ -152,7 +163,33 @@ def assemble(cls, package_data, resource, codebase, package_adder):
 
         package_resource = None
         if resource.name == 'PKG-INFO':
+            # Initially use current Resource as `package_resource`.
+            # We'll want update `package_resource` with the Resource of a
+            # PKG-INFO file that's in an .egg-info Directory.
             package_resource = resource
+            # We want to use the PKG-INFO file from an .egg-info directory, as
+            # the package info collected from a *.egg_info/PKG-INFO file has
+            # dependency information that a PKG-INFO from the root of a Python
+            # project lacks.
+            parent_resource = resource.parent(codebase)
+            if not is_egg_info_directory(parent_resource):
+                # If we are not in an .egg-info directory, we assume we are at
+                # the root of a Python codebase and we want to find the
+                # .egg_info dir
+                egg_info_dir = None
+                for sibling in resource.siblings(codebase):
+                    if sibling.path.endswith('.egg-info'):
+                        egg_info_dir = sibling
+                        break
+
+                # If we find the .egg_info dir, then we look for the PKG-INFO
+                # file in it and use that as our package_resource
+                if egg_info_dir:
+                    for child in egg_info_dir.children(codebase):
+                        if not child.name == 'PKG-INFO':
+                            continue
+                        package_resource = child
+                        break
         elif resource.name in datafile_name_patterns:
             if resource.has_parent():
                 siblings = resource.siblings(codebase)
@@ -221,7 +258,14 @@ def assemble(cls, package_data, resource, codebase, package_adder):
                 package.license_expression = compute_normalized_license(package.declared_license)
             package_uid = package.package_uid
 
-            root = package_resource.parent(codebase)
+            package_resource_parent = package_resource.parent(codebase)
+            if is_egg_info_directory(package_resource_parent):
+                root = package_resource_parent.parent(codebase)
+            else:
+                # We're assuming that our package resource is already at the
+                # root
+                root = package_resource_parent
+
             if root:
                 for py_res in cls.walk_pypi(resource=root, codebase=codebase):
                     if py_res.is_dir:

diff --git a/tests/packagedcode/data/about/aboutfiles.expected.json b/tests/packagedcode/data/about/aboutfiles.expected.json
@@ -1,30 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json": "<file>",
-        "--package": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 3
-      }
-    }
-  ],
   "dependencies": [],
   "packages": [
     {

diff --git a/tests/packagedcode/data/build/bazel/end2end-expected.json b/tests/packagedcode/data/build/bazel/end2end-expected.json
@@ -1,30 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json-pp": "<file>",
-        "--package": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 6
-      }
-    }
-  ],
   "dependencies": [],
   "packages": [
     {

diff --git a/tests/packagedcode/data/build/buck/end2end-expected.json b/tests/packagedcode/data/build/buck/end2end-expected.json
@@ -1,30 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json-pp": "<file>",
-        "--package": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 7
-      }
-    }
-  ],
   "dependencies": [],
   "packages": [
     {

diff --git a/tests/packagedcode/data/cocoapods/assemble/solo/Podfile-expected.json b/tests/packagedcode/data/cocoapods/assemble/solo/Podfile-expected.json
@@ -1,30 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json": "<file>",
-        "--package": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 1
-      }
-    }
-  ],
   "dependencies": [],
   "packages": [],
   "files": [

diff --git a/tests/packagedcode/data/cocoapods/assemble/solo/RxDataSources.podspec-expected.json b/tests/packagedcode/data/cocoapods/assemble/solo/RxDataSources.podspec-expected.json
@@ -1,30 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json": "<file>",
-        "--package": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 1
-      }
-    }
-  ],
   "dependencies": [],
   "packages": [
     {

diff --git a/tests/packagedcode/data/npm/get_package_resources.scan.expected.json b/tests/packagedcode/data/npm/get_package_resources.scan.expected.json
@@ -1,30 +1,4 @@
 {
-  "headers": [
-    {
-      "tool_name": "scancode-toolkit",
-      "options": {
-        "input": "<path>",
-        "--json": "<file>",
-        "--package": true
-      },
-      "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
-      "output_format_version": "2.0.0",
-      "message": null,
-      "errors": [],
-      "warnings": [],
-      "extra_data": {
-        "system_environment": {
-          "operating_system": "linux",
-          "cpu_architecture": "64",
-          "platform": "Linux-5.4.0-109-generic-x86_64-with-Ubuntu-18.04-bionic",
-          "platform_version": "#123~18.04.1-Ubuntu SMP Fri Apr 8 09:48:52 UTC 2022",
-          "python_version": "3.6.9 (default, Mar 15 2022, 13:55:28) \n[GCC 8.4.0]"
-        },
-        "spdx_license_list_version": "3.16",
-        "files_count": 3
-      }
-    }
-  ],
   "dependencies": [],
   "packages": [
     {