Merge pull request #2887 from nexB/fix-2886-packages

Do not raise exception on package data mismatch #2886

src/packagedcode/models.py

@@ -86,14 +86,14 @@
 SCANCODE_DEBUG_PACKAGE_API = os.environ.get('SCANCODE_DEBUG_PACKAGE_API', False)
 
 TRACE = False or SCANCODE_DEBUG_PACKAGE_API
-TRACE_MERGING = True or SCANCODE_DEBUG_PACKAGE_API
+TRACE_MERGING = False or SCANCODE_DEBUG_PACKAGE_API
 
 def logger_debug(*args):
     pass
 
 logger = logging.getLogger(__name__)
 
-if TRACE:
+if TRACE or TRACE_MERGING:
     import logging
 
     logging.basicConfig(stream=sys.stdout)
@@ -838,12 +838,26 @@ def populate_package_data(self, package_data_by_path, uuid):
         """
         Create a package instance object from one or multiple package data.
         """
+        paths_with_mismatch = []
+
         for path, package_data in package_data_by_path.items():
+            success = True
             if TRACE:
                 logger.debug('Merging package manifest data for: {}'.format(path))
                 logger.debug('package manifest data: {}'.format(repr(package_data)))
+
+            success = self.update(package_data.copy())
+            if not success:
+                paths_with_mismatch.append(path)
+
+                if TRACE:
+                    logger.debug('Failed to merge package manifest data for: {}'.format(path))
+                continue
+
             self.package_data_files.append(path)
-            self.update(package_data.copy())
+
+        for path in paths_with_mismatch:
+            package_data_by_path.pop(path)
 
         self.package_data_files = tuple(self.package_data_files)
 
@@ -931,6 +945,9 @@ def get_file_patterns(self, manifests):
 
     def update(self, package_data, replace=False):
         """
+        Returns False if there's a type/name/version mismatch between the package
+        and the package_data, otherwise returns True if update is successful.
+
         Update the Package object with data from the `package_data`
         object.
         When an `package_instance` field has no value one side and and the
@@ -963,13 +980,15 @@ def update(self, package_data, replace=False):
             # These fields has to be same across the package_data
             if existing_field in ('name', 'version', 'type', 'primary_language'):
                 if existing_value and new_value and existing_value != new_value:
-                    raise Exception(
-                        '\n'.join([
-                            'Mismatched {}:'.format(existing_field),
-                            '    existing_value: {}'.format(existing_value),
-                            '    new_value: {}'.format(new_value)
-                        ])
-                    )
+                    if TRACE_MERGING:
+                        logger.debug(
+                            '\n'.join([
+                                'Mismatched {}:'.format(existing_field),
+                                '    existing_value: {}'.format(existing_value),
+                                '    new_value: {}'.format(new_value)
+                            ])
+                        )
+                    return False
 
             if not new_value:
                 if TRACE_MERGING:
@@ -1059,6 +1078,9 @@ def update(self, package_data, replace=False):
             if TRACE_MERGING:
                 logger.debug('  Nothing done')
 
+        # Return True if package data update is successful
+        return True
+
 
 # Package types
 # NOTE: this is somewhat redundant with extractcode archive handlers

tests/packagedcode/data/instance/pypi-with-test-manifests/LICENSE

@@ -0,0 +1,19 @@
+Copyright Jason R. Coombs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE.

tests/packagedcode/data/instance/pypi-with-test-manifests/MANIFEST.in

@@ -0,0 +1,17 @@
+recursive-include setuptools *.py *.exe *.xml *.tmpl
+recursive-include tests *.py
+recursive-include setuptools/tests *.html
+recursive-include docs *.py *.txt *.rst *.conf *.css *.css_t Makefile indexsidebar.html
+recursive-include setuptools/_vendor *.py *.txt
+recursive-include pkg_resources *.py *.txt
+recursive-include pkg_resources/tests/data *
+recursive-include tools *
+recursive-include changelog.d *
+include *.py
+include *.rst
+include MANIFEST.in
+include LICENSE
+include launcher.c
+include msvc-build-launcher.cmd
+include pytest.ini
+include tox.ini

tests/packagedcode/data/instance/pypi-with-test-manifests/PKG-INFO

@@ -0,0 +1,93 @@
+Metadata-Version: 2.1
+Name: setuptools
+Version: 58.2.0
+Summary: Easily download, build, install, upgrade, and uninstall Python packages
+Home-page: https://github.com/pypa/setuptools
+Author: Python Packaging Authority
+Author-email: distutils-sig@python.org
+License: UNKNOWN
+Project-URL: Documentation, https://setuptools.readthedocs.io/
+Keywords: CPAN PyPI distutils eggs package management
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Archiving :: Packaging
+Classifier: Topic :: System :: Systems Administration
+Classifier: Topic :: Utilities
+Requires-Python: >=3.6
+Provides-Extra: testing
+Provides-Extra: docs
+Provides-Extra: ssl
+Provides-Extra: certs
+License-File: LICENSE
+
+.. image:: https://img.shields.io/pypi/v/setuptools.svg
+   :target: `PyPI link`_
+
+.. image:: https://img.shields.io/pypi/pyversions/setuptools.svg
+   :target: `PyPI link`_
+
+.. _PyPI link: https://pypi.org/project/setuptools
+
+.. image:: https://github.com/pypa/setuptools/workflows/tests/badge.svg
+   :target: https://github.com/pypa/setuptools/actions?query=workflow%3A%22tests%22
+   :alt: tests
+
+.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
+   :target: https://github.com/psf/black
+   :alt: Code style: Black
+
+.. image:: https://img.shields.io/readthedocs/setuptools/latest.svg
+    :target: https://setuptools.readthedocs.io
+
+.. image:: https://img.shields.io/badge/skeleton-2021-informational
+   :target: https://blog.jaraco.com/skeleton
+
+.. image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg?logo=codecov&logoColor=white
+   :target: https://codecov.io/gh/pypa/setuptools
+
+.. image:: https://tidelift.com/badges/github/pypa/setuptools?style=flat
+   :target: https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=readme
+
+See the `Installation Instructions
+<https://packaging.python.org/installing/>`_ in the Python Packaging
+User's Guide for instructions on installing, upgrading, and uninstalling
+Setuptools.
+
+Questions and comments should be directed to the `distutils-sig
+mailing list <http://mail.python.org/pipermail/distutils-sig/>`_.
+Bug reports and especially tested patches may be
+submitted directly to the `bug tracker
+<https://github.com/pypa/setuptools/issues>`_.
+
+
+Code of Conduct
+===============
+
+Everyone interacting in the setuptools project's codebases, issue trackers,
+chat rooms, and mailing lists is expected to follow the
+`PSF Code of Conduct <https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md>`_.
+
+
+For Enterprise
+==============
+
+Available as part of the Tidelift Subscription.
+
+Setuptools and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use.
+
+`Learn more <https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=referral&utm_campaign=github>`_.
+
+
+Security Contact
+================
+
+To report a security vulnerability, please use the
+`Tidelift security contact <https://tidelift.com/security>`_.
+Tidelift will coordinate the fix and disclosure.
+
+

tests/packagedcode/data/instance/pypi-with-test-manifests/README.rst

@@ -0,0 +1,64 @@
+.. image:: https://img.shields.io/pypi/v/setuptools.svg
+   :target: `PyPI link`_
+
+.. image:: https://img.shields.io/pypi/pyversions/setuptools.svg
+   :target: `PyPI link`_
+
+.. _PyPI link: https://pypi.org/project/setuptools
+
+.. image:: https://github.com/pypa/setuptools/workflows/tests/badge.svg
+   :target: https://github.com/pypa/setuptools/actions?query=workflow%3A%22tests%22
+   :alt: tests
+
+.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
+   :target: https://github.com/psf/black
+   :alt: Code style: Black
+
+.. image:: https://img.shields.io/readthedocs/setuptools/latest.svg
+    :target: https://setuptools.readthedocs.io
+
+.. image:: https://img.shields.io/badge/skeleton-2021-informational
+   :target: https://blog.jaraco.com/skeleton
+
+.. image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg?logo=codecov&logoColor=white
+   :target: https://codecov.io/gh/pypa/setuptools
+
+.. image:: https://tidelift.com/badges/github/pypa/setuptools?style=flat
+   :target: https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=readme
+
+See the `Installation Instructions
+<https://packaging.python.org/installing/>`_ in the Python Packaging
+User's Guide for instructions on installing, upgrading, and uninstalling
+Setuptools.
+
+Questions and comments should be directed to the `distutils-sig
+mailing list <http://mail.python.org/pipermail/distutils-sig/>`_.
+Bug reports and especially tested patches may be
+submitted directly to the `bug tracker
+<https://github.com/pypa/setuptools/issues>`_.
+
+
+Code of Conduct
+===============
+
+Everyone interacting in the setuptools project's codebases, issue trackers,
+chat rooms, and mailing lists is expected to follow the
+`PSF Code of Conduct <https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md>`_.
+
+
+For Enterprise
+==============
+
+Available as part of the Tidelift Subscription.
+
+Setuptools and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use.
+
+`Learn more <https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=referral&utm_campaign=github>`_.
+
+
+Security Contact
+================
+
+To report a security vulnerability, please use the
+`Tidelift security contact <https://tidelift.com/security>`_.
+Tidelift will coordinate the fix and disclosure.

tests/packagedcode/data/instance/pypi-with-test-manifests/pyproject.toml

@@ -0,0 +1,61 @@
+[build-system]
+requires = [
+    "wheel",
+]
+build-backend = "setuptools.build_meta"
+backend-path = ["."]
+
+[tool.black]
+skip-string-normalization = true
+
+[tool.setuptools_scm]
+
+[pytest.enabler.black]
+#addopts = "--black"
+
+[pytest.enabler.mypy]
+#addopts = "--mypy"
+
+[pytest.enabler.flake8]
+addopts = "--flake8"
+
+[pytest.enabler.cov]
+addopts = "--cov"
+
+[pytest.enabler.xdist]
+addopts = "-n auto"
+
+[tool.towncrier]
+    package = "setuptools"
+    package_dir = "setuptools"
+    filename = "CHANGES.rst"
+    directory = "changelog.d"
+    title_format = "v{version}"
+    issue_format = "#{issue}"
+    template = "towncrier_template.rst"
+    underlines = ["-", "^"]
+
+    [[tool.towncrier.type]]
+        directory = "deprecation"
+        name = "Deprecations"
+        showcontent = true
+
+    [[tool.towncrier.type]]
+        directory = "breaking"
+        name = "Breaking Changes"
+        showcontent = true
+
+    [[tool.towncrier.type]]
+        directory = "change"
+        name = "Changes"
+        showcontent = true
+
+    [[tool.towncrier.type]]
+        directory = "doc"
+        name = "Documentation changes"
+        showcontent = true
+
+    [[tool.towncrier.type]]
+        directory = "misc"
+        name = "Misc"
+        showcontent = true