Prefer using PKG-INFO from .egg-info in assemble #3083

    * Update test expectations

Signed-off-by: Jono Yang <jyang@nexb.com>

src/packagedcode/pypi.py

@@ -152,7 +152,36 @@ def assemble(cls, package_data, resource, codebase, package_adder):
 
         package_resource = None
         if resource.name == 'PKG-INFO':
+            # Initially use current Resource as `package_resource`.
+            # We'll want update `package_resource` with the Resource of a
+            # PKG-INFO file that's in an .egg-info Directory.
             package_resource = resource
+            # We want to use the PKG-INFO file from an .egg-info directory, as
+            # the package info collected from a *.egg_info/PKG-INFO file has
+            # dependency information that a PKG-INFO from the root of a Python
+            # project lacks.
+            parent_resource = resource.parent(codebase)
+            if (
+                parent_resource
+                and not parent_resource.path.endswith('.egg-info')
+            ):
+                # If we are not in an .egg-info directory, we assume we are at
+                # the root of a Python codebase and we want to find the
+                # .egg_info dir
+                egg_info_dir = None
+                for sibling in resource.siblings(codebase):
+                    if sibling.path.endswith('.egg-info'):
+                        egg_info_dir = sibling
+                        break
+
+                # If we find the .egg_info dir, then we look for the PKG-INFO
+                # file in it and use that as our package_resource
+                if egg_info_dir:
+                    for child in egg_info_dir.children(codebase):
+                        if not child.name == 'PKG-INFO':
+                            continue
+                        package_resource = child
+                        break
         elif resource.name in datafile_name_patterns:
             if resource.has_parent():
                 siblings = resource.siblings(codebase)
@@ -221,7 +250,17 @@ def assemble(cls, package_data, resource, codebase, package_adder):
                 package.license_expression = compute_normalized_license(package.declared_license)
             package_uid = package.package_uid
 
-            root = package_resource.parent(codebase)
+            package_resource_parent = package_resource.parent(codebase)
+            if (
+                package_resource_parent
+                and package_resource_parent.path.endswith('.egg-info')
+            ):
+                root = package_resource_parent.parent(codebase)
+            else:
+                # We're assuming that our package resource is already at the
+                # root
+                root = package_resource_parent
+
             if root:
                 for py_res in cls.walk_pypi(resource=root, codebase=codebase):
                     if py_res.is_dir:

tests/packagedcode/data/pypi/site-packages/site-packages-expected.json

@@ -1,5 +1,61 @@
 {
   "dependencies": [
+    {
+      "purl": "pkg:pypi/six",
+      "extracted_requirement": "six>=1.10.0",
+      "scope": "install",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": false,
+      "resolved_package": {},
+      "extra_data": {},
+      "dependency_uid": "pkg:pypi/six?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "for_package_uid": "pkg:pypi/pyjpstring@0.0.3?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "datafile_path": "setup.py",
+      "datasource_id": "pypi_setup_py"
+    },
+    {
+      "purl": "pkg:pypi/zenhan",
+      "extracted_requirement": "zenhan>=0.5.2",
+      "scope": "install",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": false,
+      "resolved_package": {},
+      "extra_data": {},
+      "dependency_uid": "pkg:pypi/zenhan?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "for_package_uid": "pkg:pypi/pyjpstring@0.0.3?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "datafile_path": "setup.py",
+      "datasource_id": "pypi_setup_py"
+    },
+    {
+      "purl": "pkg:pypi/django",
+      "extracted_requirement": "django",
+      "scope": "test",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": false,
+      "resolved_package": {},
+      "extra_data": {},
+      "dependency_uid": "pkg:pypi/django?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "for_package_uid": "pkg:pypi/pyjpstring@0.0.3?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "datafile_path": "setup.py",
+      "datasource_id": "pypi_setup_py"
+    },
+    {
+      "purl": "pkg:pypi/pytest@2.9.1",
+      "extracted_requirement": "pytest==2.9.1",
+      "scope": "test",
+      "is_runtime": true,
+      "is_optional": false,
+      "is_resolved": true,
+      "resolved_package": {},
+      "extra_data": {},
+      "dependency_uid": "pkg:pypi/pytest@2.9.1?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "for_package_uid": "pkg:pypi/pyjpstring@0.0.3?uuid=fixed-uid-done-for-testing-5642512d1758",
+      "datafile_path": "setup.py",
+      "datasource_id": "pypi_setup_py"
+    },
     {
       "purl": "pkg:pypi/colorama",
       "extracted_requirement": "colorama; platform_system == \"Windows\"",
@@ -38,7 +94,7 @@
       "qualifiers": {},
       "subpath": null,
       "primary_language": "Python",
-      "description": "Python japanese string utilities.\nUNKNOWN",
+      "description": "Python japanese string utilities.",
       "release_date": null,
       "parties": [
         {
@@ -76,10 +132,10 @@
       "api_data_url": "https://pypi.org/pypi/PyJPString/0.0.3/json",
       "package_uid": "pkg:pypi/pyjpstring@0.0.3?uuid=fixed-uid-done-for-testing-5642512d1758",
       "datafile_paths": [
-        "PKG-INFO"
+        "setup.py"
       ],
       "datasource_ids": [
-        "pypi_sdist_pkginfo"
+        "pypi_setup_py"
       ],
       "purl": "pkg:pypi/pyjpstring@0.0.3"
     },

tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-expected.json

@@ -9,7 +9,7 @@
       "qualifiers": {},
       "subpath": null,
       "primary_language": "Python",
-      "description": "The PyPA recommended tool for installing Python packages.\npip - The Python Package Installer\n==================================\n\n.. image:: https://img.shields.io/pypi/v/pip.svg\n   :target: https://pypi.org/project/pip/\n\n.. image:: https://readthedocs.org/projects/pip/badge/?version=latest\n   :target: https://pip.pypa.io/en/latest\n\npip is the `package installer`_ for Python. You can use pip to install packages from the `Python Package Index`_ and other indexes.\n\nPlease take a look at our documentation for how to install and use pip:\n\n* `Installation`_\n* `Usage`_\n\nWe release updates regularly, with a new version every 3 months. Find more details in our documentation:\n\n* `Release notes`_\n* `Release process`_\n\nIn pip 20.3, we've `made a big improvement to the heart of pip`_; `learn more`_. We want your input, so `sign up for our user experience research studies`_ to help us do it right.\n\n**Note**: pip 21.0, in January 2021, removed Python 2 support, per pip's `Python 2 support policy`_. Please migrate to Python 3.\n\nIf you find bugs, need help, or want to talk to the developers, please use our mailing lists or chat rooms:\n\n* `Issue tracking`_\n* `Discourse channel`_\n* `User IRC`_\n\nIf you want to get involved head over to GitHub to get the source code, look at our development documentation and feel free to jump on the developer mailing lists and chat rooms:\n\n* `GitHub page`_\n* `Development documentation`_\n* `Development mailing list`_\n* `Development IRC`_\n\nCode of Conduct\n---------------\n\nEveryone interacting in the pip project's codebases, issue trackers, chat\nrooms, and mailing lists is expected to follow the `PSF Code of Conduct`_.\n\n.. _package installer: https://packaging.python.org/guides/tool-recommendations/\n.. _Python Package Index: https://pypi.org\n.. _Installation: https://pip.pypa.io/en/stable/installation/\n.. _Usage: https://pip.pypa.io/en/stable/\n.. _Release notes: https://pip.pypa.io/en/stable/news.html\n.. _Release process: https://pip.pypa.io/en/latest/development/release-process/\n.. _GitHub page: https://github.com/pypa/pip\n.. _Development documentation: https://pip.pypa.io/en/latest/development\n.. _made a big improvement to the heart of pip: https://pyfound.blogspot.com/2020/11/pip-20-3-new-resolver.html\n.. _learn more: https://pip.pypa.io/en/latest/user_guide/#changes-to-the-pip-dependency-resolver-in-20-3-2020\n.. _sign up for our user experience research studies: https://pyfound.blogspot.com/2020/03/new-pip-resolver-to-roll-out-this-year.html\n.. _Python 2 support policy: https://pip.pypa.io/en/latest/development/release-process/#python-2-support\n.. _Issue tracking: https://github.com/pypa/pip/issues\n.. _Discourse channel: https://discuss.python.org/c/packaging\n.. _Development mailing list: https://mail.python.org/mailman3/lists/distutils-sig.python.org/\n.. _User IRC: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa\n.. _Development IRC: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa-dev\n.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md",
+      "description": "The PyPA recommended tool for installing Python packages.",
       "release_date": null,
       "parties": [
         {
@@ -45,7 +45,7 @@
       "code_view_url": "https://github.com/pypa/pip",
       "vcs_url": null,
       "copyright": null,
-      "license_expression": "mit AND mit",
+      "license_expression": "mit",
       "declared_license": {
         "license": "MIT",
         "classifiers": [
@@ -56,17 +56,18 @@
       "source_packages": [],
       "extra_data": {
         "Documentation": "https://pip.pypa.io",
-        "Changelog": "https://pip.pypa.io/en/stable/news/"
+        "Changelog": "https://pip.pypa.io/en/stable/news/",
+        "python_requires": ">=3.7"
       },
       "repository_homepage_url": "https://pypi.org/project/pip",
       "repository_download_url": "https://pypi.org/packages/source/p/pip/pip-22.0.4.tar.gz",
       "api_data_url": "https://pypi.org/pypi/pip/22.0.4/json",
       "package_uid": "pkg:pypi/pip@22.0.4?uuid=fixed-uid-done-for-testing-5642512d1758",
       "datafile_paths": [
-        "PKG-INFO"
+        "setup.py"
       ],
       "datasource_ids": [
-        "pypi_sdist_pkginfo"
+        "pypi_setup_py"
       ],
       "purl": "pkg:pypi/pip@22.0.4"
     }