diff --git a/src/packagedcode/models.py b/src/packagedcode/models.py index ed76b5159df..42bf0f2684a 100644 --- a/src/packagedcode/models.py +++ b/src/packagedcode/models.py @@ -86,14 +86,14 @@ SCANCODE_DEBUG_PACKAGE_API = os.environ.get('SCANCODE_DEBUG_PACKAGE_API', False) TRACE = False or SCANCODE_DEBUG_PACKAGE_API -TRACE_MERGING = True or SCANCODE_DEBUG_PACKAGE_API +TRACE_MERGING = False or SCANCODE_DEBUG_PACKAGE_API def logger_debug(*args): pass logger = logging.getLogger(__name__) -if TRACE: +if TRACE or TRACE_MERGING: import logging logging.basicConfig(stream=sys.stdout) @@ -838,12 +838,26 @@ def populate_package_data(self, package_data_by_path, uuid): """ Create a package instance object from one or multiple package data. """ + paths_with_mismatch = [] + for path, package_data in package_data_by_path.items(): + success = True if TRACE: logger.debug('Merging package manifest data for: {}'.format(path)) logger.debug('package manifest data: {}'.format(repr(package_data))) + + success = self.update(package_data.copy()) + if not success: + paths_with_mismatch.append(path) + + if TRACE: + logger.debug('Failed to merge package manifest data for: {}'.format(path)) + continue + self.package_data_files.append(path) - self.update(package_data.copy()) + + for path in paths_with_mismatch: + package_data_by_path.pop(path) self.package_data_files = tuple(self.package_data_files) @@ -931,6 +945,9 @@ def get_file_patterns(self, manifests): def update(self, package_data, replace=False): """ + Returns False if there's a type/name/version mismatch between the package + and the package_data, otherwise returns True if update is successful. + Update the Package object with data from the `package_data` object. When an `package_instance` field has no value one side and and the @@ -963,13 +980,15 @@ def update(self, package_data, replace=False): # These fields has to be same across the package_data if existing_field in ('name', 'version', 'type', 'primary_language'): if existing_value and new_value and existing_value != new_value: - raise Exception( - '\n'.join([ - 'Mismatched {}:'.format(existing_field), - ' existing_value: {}'.format(existing_value), - ' new_value: {}'.format(new_value) - ]) - ) + if TRACE_MERGING: + logger.debug( + '\n'.join([ + 'Mismatched {}:'.format(existing_field), + ' existing_value: {}'.format(existing_value), + ' new_value: {}'.format(new_value) + ]) + ) + return False if not new_value: if TRACE_MERGING: @@ -1059,6 +1078,9 @@ def update(self, package_data, replace=False): if TRACE_MERGING: logger.debug(' Nothing done') + # Return True if package data update is successful + return True + # Package types # NOTE: this is somewhat redundant with extractcode archive handlers diff --git a/tests/packagedcode/data/instance/pypi-with-test-manifests/LICENSE b/tests/packagedcode/data/instance/pypi-with-test-manifests/LICENSE new file mode 100644 index 00000000000..353924be0e5 --- /dev/null +++ b/tests/packagedcode/data/instance/pypi-with-test-manifests/LICENSE @@ -0,0 +1,19 @@ +Copyright Jason R. Coombs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/tests/packagedcode/data/instance/pypi-with-test-manifests/MANIFEST.in b/tests/packagedcode/data/instance/pypi-with-test-manifests/MANIFEST.in new file mode 100644 index 00000000000..3e8f09de37a --- /dev/null +++ b/tests/packagedcode/data/instance/pypi-with-test-manifests/MANIFEST.in @@ -0,0 +1,17 @@ +recursive-include setuptools *.py *.exe *.xml *.tmpl +recursive-include tests *.py +recursive-include setuptools/tests *.html +recursive-include docs *.py *.txt *.rst *.conf *.css *.css_t Makefile indexsidebar.html +recursive-include setuptools/_vendor *.py *.txt +recursive-include pkg_resources *.py *.txt +recursive-include pkg_resources/tests/data * +recursive-include tools * +recursive-include changelog.d * +include *.py +include *.rst +include MANIFEST.in +include LICENSE +include launcher.c +include msvc-build-launcher.cmd +include pytest.ini +include tox.ini diff --git a/tests/packagedcode/data/instance/pypi-with-test-manifests/PKG-INFO b/tests/packagedcode/data/instance/pypi-with-test-manifests/PKG-INFO new file mode 100644 index 00000000000..b5f284935c0 --- /dev/null +++ b/tests/packagedcode/data/instance/pypi-with-test-manifests/PKG-INFO @@ -0,0 +1,93 @@ +Metadata-Version: 2.1 +Name: setuptools +Version: 58.2.0 +Summary: Easily download, build, install, upgrade, and uninstall Python packages +Home-page: https://github.com/pypa/setuptools +Author: Python Packaging Authority +Author-email: distutils-sig@python.org +License: UNKNOWN +Project-URL: Documentation, https://setuptools.readthedocs.io/ +Keywords: CPAN PyPI distutils eggs package management +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: System :: Archiving :: Packaging +Classifier: Topic :: System :: Systems Administration +Classifier: Topic :: Utilities +Requires-Python: >=3.6 +Provides-Extra: testing +Provides-Extra: docs +Provides-Extra: ssl +Provides-Extra: certs +License-File: LICENSE + +.. image:: https://img.shields.io/pypi/v/setuptools.svg + :target: `PyPI link`_ + +.. image:: https://img.shields.io/pypi/pyversions/setuptools.svg + :target: `PyPI link`_ + +.. _PyPI link: https://pypi.org/project/setuptools + +.. image:: https://github.com/pypa/setuptools/workflows/tests/badge.svg + :target: https://github.com/pypa/setuptools/actions?query=workflow%3A%22tests%22 + :alt: tests + +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black + :alt: Code style: Black + +.. image:: https://img.shields.io/readthedocs/setuptools/latest.svg + :target: https://setuptools.readthedocs.io + +.. image:: https://img.shields.io/badge/skeleton-2021-informational + :target: https://blog.jaraco.com/skeleton + +.. image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg?logo=codecov&logoColor=white + :target: https://codecov.io/gh/pypa/setuptools + +.. image:: https://tidelift.com/badges/github/pypa/setuptools?style=flat + :target: https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=readme + +See the `Installation Instructions +`_ in the Python Packaging +User's Guide for instructions on installing, upgrading, and uninstalling +Setuptools. + +Questions and comments should be directed to the `distutils-sig +mailing list `_. +Bug reports and especially tested patches may be +submitted directly to the `bug tracker +`_. + + +Code of Conduct +=============== + +Everyone interacting in the setuptools project's codebases, issue trackers, +chat rooms, and mailing lists is expected to follow the +`PSF Code of Conduct `_. + + +For Enterprise +============== + +Available as part of the Tidelift Subscription. + +Setuptools and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use. + +`Learn more `_. + + +Security Contact +================ + +To report a security vulnerability, please use the +`Tidelift security contact `_. +Tidelift will coordinate the fix and disclosure. + + diff --git a/tests/packagedcode/data/instance/pypi-with-test-manifests/README.rst b/tests/packagedcode/data/instance/pypi-with-test-manifests/README.rst new file mode 100644 index 00000000000..691351763a8 --- /dev/null +++ b/tests/packagedcode/data/instance/pypi-with-test-manifests/README.rst @@ -0,0 +1,64 @@ +.. image:: https://img.shields.io/pypi/v/setuptools.svg + :target: `PyPI link`_ + +.. image:: https://img.shields.io/pypi/pyversions/setuptools.svg + :target: `PyPI link`_ + +.. _PyPI link: https://pypi.org/project/setuptools + +.. image:: https://github.com/pypa/setuptools/workflows/tests/badge.svg + :target: https://github.com/pypa/setuptools/actions?query=workflow%3A%22tests%22 + :alt: tests + +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black + :alt: Code style: Black + +.. image:: https://img.shields.io/readthedocs/setuptools/latest.svg + :target: https://setuptools.readthedocs.io + +.. image:: https://img.shields.io/badge/skeleton-2021-informational + :target: https://blog.jaraco.com/skeleton + +.. image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg?logo=codecov&logoColor=white + :target: https://codecov.io/gh/pypa/setuptools + +.. image:: https://tidelift.com/badges/github/pypa/setuptools?style=flat + :target: https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=readme + +See the `Installation Instructions +`_ in the Python Packaging +User's Guide for instructions on installing, upgrading, and uninstalling +Setuptools. + +Questions and comments should be directed to the `distutils-sig +mailing list `_. +Bug reports and especially tested patches may be +submitted directly to the `bug tracker +`_. + + +Code of Conduct +=============== + +Everyone interacting in the setuptools project's codebases, issue trackers, +chat rooms, and mailing lists is expected to follow the +`PSF Code of Conduct `_. + + +For Enterprise +============== + +Available as part of the Tidelift Subscription. + +Setuptools and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use. + +`Learn more `_. + + +Security Contact +================ + +To report a security vulnerability, please use the +`Tidelift security contact `_. +Tidelift will coordinate the fix and disclosure. diff --git a/tests/packagedcode/data/instance/pypi-with-test-manifests/pkg_resources/tests/data/my-test-package-zip/my-test-package.zip b/tests/packagedcode/data/instance/pypi-with-test-manifests/pkg_resources/tests/data/my-test-package-zip/my-test-package.zip new file mode 100644 index 00000000000..81f9a0170f7 Binary files /dev/null and b/tests/packagedcode/data/instance/pypi-with-test-manifests/pkg_resources/tests/data/my-test-package-zip/my-test-package.zip differ diff --git a/tests/packagedcode/data/instance/pypi-with-test-manifests/pyproject.toml b/tests/packagedcode/data/instance/pypi-with-test-manifests/pyproject.toml new file mode 100644 index 00000000000..70e3473d44d --- /dev/null +++ b/tests/packagedcode/data/instance/pypi-with-test-manifests/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = [ + "wheel", +] +build-backend = "setuptools.build_meta" +backend-path = ["."] + +[tool.black] +skip-string-normalization = true + +[tool.setuptools_scm] + +[pytest.enabler.black] +#addopts = "--black" + +[pytest.enabler.mypy] +#addopts = "--mypy" + +[pytest.enabler.flake8] +addopts = "--flake8" + +[pytest.enabler.cov] +addopts = "--cov" + +[pytest.enabler.xdist] +addopts = "-n auto" + +[tool.towncrier] + package = "setuptools" + package_dir = "setuptools" + filename = "CHANGES.rst" + directory = "changelog.d" + title_format = "v{version}" + issue_format = "#{issue}" + template = "towncrier_template.rst" + underlines = ["-", "^"] + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "change" + name = "Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "doc" + name = "Documentation changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "misc" + name = "Misc" + showcontent = true diff --git a/tests/packagedcode/data/instance/pypi-with-test-manifests/setup.cfg b/tests/packagedcode/data/instance/pypi-with-test-manifests/setup.cfg new file mode 100644 index 00000000000..6959fbb5116 --- /dev/null +++ b/tests/packagedcode/data/instance/pypi-with-test-manifests/setup.cfg @@ -0,0 +1,133 @@ +[metadata] +name = setuptools +version = 58.2.0 +author = Python Packaging Authority +author_email = distutils-sig@python.org +description = Easily download, build, install, upgrade, and uninstall Python packages +long_description = file:README.rst +url = https://github.com/pypa/setuptools +classifiers = + Development Status :: 5 - Production/Stable + Intended Audience :: Developers + License :: OSI Approved :: MIT License + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Topic :: Software Development :: Libraries :: Python Modules + Topic :: System :: Archiving :: Packaging + Topic :: System :: Systems Administration + Topic :: Utilities +keywords = CPAN PyPI distutils eggs package management +project_urls = + Documentation = https://setuptools.readthedocs.io/ + +[options] +packages = find_namespace: +python_requires = >=3.6 +install_requires = + +[options.packages.find] +exclude = + build* + dist* + docs* + tests* + *.tests + tools* + +[options.extras_require] +testing = + pytest >= 4.6 + pytest-checkdocs >= 2.4 + pytest-flake8 + pytest-black >= 0.3.7; \ + python_implementation != "PyPy" + pytest-cov + pytest-mypy; \ + python_implementation != "PyPy" + pytest-enabler >= 1.0.1 + + mock + flake8-2020 + virtualenv>=13.0.0 + pytest-virtualenv>=1.2.7 + wheel + paver + pip>=19.1 # For proper file:// URLs support. + jaraco.envs + pytest-xdist + sphinx + jaraco.path>=3.2.0 +docs = + sphinx + jaraco.packaging >= 8.2 + rst.linker >= 1.9 + jaraco.tidelift >= 1.4 + + pygments-github-lexers==0.0.5 + sphinx-inline-tabs + sphinxcontrib-towncrier + furo +ssl = +certs = + +[options.entry_points] +distutils.commands = + alias = setuptools.command.alias:alias + bdist_egg = setuptools.command.bdist_egg:bdist_egg + bdist_rpm = setuptools.command.bdist_rpm:bdist_rpm + build_clib = setuptools.command.build_clib:build_clib + build_ext = setuptools.command.build_ext:build_ext + build_py = setuptools.command.build_py:build_py + develop = setuptools.command.develop:develop + dist_info = setuptools.command.dist_info:dist_info + easy_install = setuptools.command.easy_install:easy_install + egg_info = setuptools.command.egg_info:egg_info + install = setuptools.command.install:install + install_egg_info = setuptools.command.install_egg_info:install_egg_info + install_lib = setuptools.command.install_lib:install_lib + install_scripts = setuptools.command.install_scripts:install_scripts + rotate = setuptools.command.rotate:rotate + saveopts = setuptools.command.saveopts:saveopts + sdist = setuptools.command.sdist:sdist + setopt = setuptools.command.setopt:setopt + test = setuptools.command.test:test + upload_docs = setuptools.command.upload_docs:upload_docs +setuptools.finalize_distribution_options = + parent_finalize = setuptools.dist:_Distribution.finalize_options + keywords = setuptools.dist:Distribution._finalize_setup_keywords +distutils.setup_keywords = + eager_resources = setuptools.dist:assert_string_list + namespace_packages = setuptools.dist:check_nsp + extras_require = setuptools.dist:check_extras + install_requires = setuptools.dist:check_requirements + tests_require = setuptools.dist:check_requirements + setup_requires = setuptools.dist:check_requirements + python_requires = setuptools.dist:check_specifier + entry_points = setuptools.dist:check_entry_points + test_suite = setuptools.dist:check_test_suite + zip_safe = setuptools.dist:assert_bool + package_data = setuptools.dist:check_package_data + exclude_package_data = setuptools.dist:check_package_data + include_package_data = setuptools.dist:assert_bool + packages = setuptools.dist:check_packages + dependency_links = setuptools.dist:assert_string_list + test_loader = setuptools.dist:check_importable + test_runner = setuptools.dist:check_importable + use_2to3 = setuptools.dist:invalid_unless_false +egg_info.writers = + PKG-INFO = setuptools.command.egg_info:write_pkg_info + requires.txt = setuptools.command.egg_info:write_requirements + entry_points.txt = setuptools.command.egg_info:write_entries + eager_resources.txt = setuptools.command.egg_info:overwrite_arg + namespace_packages.txt = setuptools.command.egg_info:overwrite_arg + top_level.txt = setuptools.command.egg_info:write_toplevel_names + depends.txt = setuptools.command.egg_info:warn_depends_obsolete + dependency_links.txt = setuptools.command.egg_info:overwrite_arg + +[egg_info] +tag_build = +tag_date = 0 + +[sdist] +formats = zip + diff --git a/tests/packagedcode/data/instance/pypi-with-test-manifests/setup.py b/tests/packagedcode/data/instance/pypi-with-test-manifests/setup.py new file mode 100755 index 00000000000..c6affe9779f --- /dev/null +++ b/tests/packagedcode/data/instance/pypi-with-test-manifests/setup.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python + +import os +import sys +import textwrap + +import setuptools +from setuptools.command.install import install + +here = os.path.dirname(__file__) + + +package_data = dict( + setuptools=['script (dev).tmpl', 'script.tmpl', 'site-patch.py'], +) + +force_windows_specific_files = ( + os.environ.get("SETUPTOOLS_INSTALL_WINDOWS_SPECIFIC_FILES", "1").lower() + not in ("", "0", "false", "no") +) + +include_windows_files = sys.platform == 'win32' or force_windows_specific_files + +if include_windows_files: + package_data.setdefault('setuptools', []).extend(['*.exe']) + package_data.setdefault('setuptools.command', []).extend(['*.xml']) + + +def pypi_link(pkg_filename): + """ + Given the filename, including md5 fragment, construct the + dependency link for PyPI. + """ + root = 'https://files.pythonhosted.org/packages/source' + name, sep, rest = pkg_filename.partition('-') + parts = root, name[0], name, pkg_filename + return '/'.join(parts) + + +class install_with_pth(install): + """ + Custom install command to install a .pth file for distutils patching. + + This hack is necessary because there's no standard way to install behavior + on startup (and it's debatable if there should be one). This hack (ab)uses + the `extra_path` behavior in Setuptools to install a `.pth` file with + implicit behavior on startup to give higher precedence to the local version + of `distutils` over the version from the standard library. + + Please do not replicate this behavior. + """ + + _pth_name = 'distutils-precedence' + _pth_contents = textwrap.dedent(""" + import os + var = 'SETUPTOOLS_USE_DISTUTILS' + enabled = os.environ.get(var, 'stdlib') == 'local' + enabled and __import__('_distutils_hack').add_shim() + """).lstrip().replace('\n', '; ') + + def initialize_options(self): + install.initialize_options(self) + self.extra_path = self._pth_name, self._pth_contents + + def finalize_options(self): + install.finalize_options(self) + self._restore_install_lib() + + def _restore_install_lib(self): + """ + Undo secondary effect of `extra_path` adding to `install_lib` + """ + suffix = os.path.relpath(self.install_lib, self.install_libbase) + + if suffix.strip() == self._pth_contents.strip(): + self.install_lib = self.install_libbase + + +setup_params = dict( + cmdclass={'install': install_with_pth}, + package_data=package_data, +) + +if __name__ == '__main__': + # allow setup.py to run from another directory + here and os.chdir(here) + dist = setuptools.setup(**setup_params) diff --git a/tests/packagedcode/data/instance/pypi-with-test-manifests/tox.ini b/tests/packagedcode/data/instance/pypi-with-test-manifests/tox.ini new file mode 100644 index 00000000000..8a48a1cdb3b --- /dev/null +++ b/tests/packagedcode/data/instance/pypi-with-test-manifests/tox.ini @@ -0,0 +1,55 @@ +[tox] +envlist = python +minversion = 3.2 +# https://github.com/jaraco/skeleton/issues/6 +tox_pip_extensions_ext_venv_update = true +toxworkdir={env:TOX_WORK_DIR:.tox} + +[testenv] +deps = + # workaround for sphinx-doc/sphinx#9562 + # TODO: remove after Sphinx>4.1.2 is available. + sphinx@git+https://github.com/sphinx-doc/sphinx; python_version>="3.10" +commands = + pytest {posargs} +usedevelop = True +extras = testing +passenv = + windir # required for test_pkg_resources + +[testenv:docs] +extras = + docs + testing +changedir = docs +commands = + python -m sphinx -W --keep-going . {toxinidir}/build/html + +[testenv:finalize] +skip_install = True +deps = + towncrier + bump2version +passenv = * +commands = + python tools/finalize.py + +[testenv:release] +skip_install = True +deps = + build + twine>=3 + jaraco.develop>=7.1 +passenv = + TWINE_PASSWORD + GITHUB_TOKEN +setenv = + TWINE_USERNAME = {env:TWINE_USERNAME:__token__} +commands = + python -m bootstrap + python -c "import shutil; shutil.rmtree('dist', ignore_errors=True)" + # unset tag_build and tag_date pypa/setuptools#2500 + python setup.py egg_info -Db "" saveopts + python -m build + python -m twine upload dist/* + python -m jaraco.develop.create-github-release diff --git a/tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests.json b/tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests.json new file mode 100644 index 00000000000..7beb728b313 --- /dev/null +++ b/tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests.json @@ -0,0 +1,524 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "options": { + "input": "", + "--json": "", + "--package": true, + "--processes": "-1", + "--strip-root": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "output_format_version": "3.0.0", + "message": null, + "errors": [], + "extra_data": { + "spdx_license_list_version": "3.16", + "files_count": 9 + } + } + ], + "dependencies": [ + { + "purl": "pkg:pypi/towncrier", + "extracted_requirement": "towncrier", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {}, + "dependency_uuid": "pkg:pypi/towncrier", + "for_package": "pkg:pypi/setuptools@58.2.0", + "lockfile": "tox.ini" + }, + { + "purl": "pkg:pypi/bump2version", + "extracted_requirement": "bump2version", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {}, + "dependency_uuid": "pkg:pypi/bump2version", + "for_package": "pkg:pypi/setuptools@58.2.0", + "lockfile": "tox.ini" + }, + { + "purl": "pkg:pypi/build", + "extracted_requirement": "build", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {}, + "dependency_uuid": "pkg:pypi/build", + "for_package": "pkg:pypi/setuptools@58.2.0", + "lockfile": "tox.ini" + }, + { + "purl": "pkg:pypi/twine", + "extracted_requirement": ">=3", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {}, + "dependency_uuid": "pkg:pypi/twine", + "for_package": "pkg:pypi/setuptools@58.2.0", + "lockfile": "tox.ini" + }, + { + "purl": "pkg:pypi/jaraco.develop", + "extracted_requirement": ">=7.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {}, + "dependency_uuid": "pkg:pypi/jaraco.develop", + "for_package": "pkg:pypi/setuptools@58.2.0", + "lockfile": "tox.ini" + } + ], + "packages": [ + { + "type": "pypi", + "namespace": null, + "name": "setuptools", + "version": "58.2.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "Easily download, build, install, upgrade, and uninstall Python packages\n.. image:: https://img.shields.io/pypi/v/setuptools.svg\n :target: `PyPI link`_\n\n.. image:: https://img.shields.io/pypi/pyversions/setuptools.svg\n :target: `PyPI link`_\n\n.. _PyPI link: https://pypi.org/project/setuptools\n\n.. image:: https://github.com/pypa/setuptools/workflows/tests/badge.svg\n :target: https://github.com/pypa/setuptools/actions?query=workflow%3A%22tests%22\n :alt: tests\n\n.. image:: https://img.shields.io/badge/code%20style-black-000000.svg\n :target: https://github.com/psf/black\n :alt: Code style: Black\n\n.. image:: https://img.shields.io/readthedocs/setuptools/latest.svg\n :target: https://setuptools.readthedocs.io\n\n.. image:: https://img.shields.io/badge/skeleton-2021-informational\n :target: https://blog.jaraco.com/skeleton\n\n.. image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg?logo=codecov&logoColor=white\n :target: https://codecov.io/gh/pypa/setuptools\n\n.. image:: https://tidelift.com/badges/github/pypa/setuptools?style=flat\n :target: https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=readme\n\nSee the `Installation Instructions\n`_ in the Python Packaging\nUser's Guide for instructions on installing, upgrading, and uninstalling\nSetuptools.\n\nQuestions and comments should be directed to the `distutils-sig\nmailing list `_.\nBug reports and especially tested patches may be\nsubmitted directly to the `bug tracker\n`_.\n\n\nCode of Conduct\n===============\n\nEveryone interacting in the setuptools project's codebases, issue trackers,\nchat rooms, and mailing lists is expected to follow the\n`PSF Code of Conduct `_.\n\n\nFor Enterprise\n==============\n\nAvailable as part of the Tidelift Subscription.\n\nSetuptools and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use.\n\n`Learn more `_.\n\n\nSecurity Contact\n================\n\nTo report a security vulnerability, please use the\n`Tidelift security contact `_.\nTidelift will coordinate the fix and disclosure.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Python Packaging Authority", + "email": "distutils-sig@python.org", + "url": null + } + ], + "keywords": [ + "CPAN PyPI distutils eggs package management", + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: System :: Archiving :: Packaging", + "Topic :: System :: Systems Administration", + "Topic :: Utilities" + ], + "homepage_url": "https://github.com/pypa/setuptools", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "mit", + "declared_license": { + "classifiers": [ + "License :: OSI Approved :: MIT License" + ] + }, + "notice_text": null, + "contains_source_code": null, + "source_packages": [], + "extra_data": {}, + "package_uuid": "pkg:pypi/setuptools@58.2.0", + "package_data_files": [ + "PKG-INFO", + "setup.py", + "tox.ini" + ], + "files": [ + "LICENSE", + "MANIFEST.in", + "PKG-INFO", + "pyproject.toml", + "README.rst", + "setup.cfg", + "setup.py", + "tox.ini", + "pkg_resources/tests/data/my-test-package-zip/my-test-package.zip" + ], + "purl": "pkg:pypi/setuptools@58.2.0", + "repository_homepage_url": "https://pypi.org/project/https://pypi.org", + "repository_download_url": "https://pypi.org/packages/source/s/setuptools/setuptools-58.2.0.tar.gz", + "api_data_url": "https://pypi.org/pypi/setuptools/58.2.0/json" + }, + { + "type": "pypi", + "namespace": null, + "name": "my-test-package", + "version": "1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "contains_source_code": null, + "source_packages": [], + "extra_data": {}, + "package_uuid": "pkg:pypi/my-test-package@1.0", + "package_data_files": [ + "pkg_resources/tests/data/my-test-package-zip/my-test-package.zip" + ], + "files": [ + "pkg_resources/tests/data/my-test-package-zip/my-test-package.zip" + ], + "purl": "pkg:pypi/my-test-package@1.0", + "repository_homepage_url": "https://pypi.org/project/https://pypi.org", + "repository_download_url": "https://pypi.org/packages/source/m/my-test-package/my-test-package-1.0.tar.gz", + "api_data_url": "https://pypi.org/pypi/my-test-package/1.0/json" + } + ], + "files": [ + { + "path": "LICENSE", + "type": "file", + "package_data": [], + "for_packages": [ + "pkg:pypi/setuptools@58.2.0" + ], + "scan_errors": [] + }, + { + "path": "MANIFEST.in", + "type": "file", + "package_data": [], + "for_packages": [ + "pkg:pypi/setuptools@58.2.0" + ], + "scan_errors": [] + }, + { + "path": "PKG-INFO", + "type": "file", + "package_data": [ + { + "type": "pypi", + "namespace": null, + "name": "setuptools", + "version": "58.2.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "Easily download, build, install, upgrade, and uninstall Python packages\n.. image:: https://img.shields.io/pypi/v/setuptools.svg\n :target: `PyPI link`_\n\n.. image:: https://img.shields.io/pypi/pyversions/setuptools.svg\n :target: `PyPI link`_\n\n.. _PyPI link: https://pypi.org/project/setuptools\n\n.. image:: https://github.com/pypa/setuptools/workflows/tests/badge.svg\n :target: https://github.com/pypa/setuptools/actions?query=workflow%3A%22tests%22\n :alt: tests\n\n.. image:: https://img.shields.io/badge/code%20style-black-000000.svg\n :target: https://github.com/psf/black\n :alt: Code style: Black\n\n.. image:: https://img.shields.io/readthedocs/setuptools/latest.svg\n :target: https://setuptools.readthedocs.io\n\n.. image:: https://img.shields.io/badge/skeleton-2021-informational\n :target: https://blog.jaraco.com/skeleton\n\n.. image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg?logo=codecov&logoColor=white\n :target: https://codecov.io/gh/pypa/setuptools\n\n.. image:: https://tidelift.com/badges/github/pypa/setuptools?style=flat\n :target: https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=readme\n\nSee the `Installation Instructions\n`_ in the Python Packaging\nUser's Guide for instructions on installing, upgrading, and uninstalling\nSetuptools.\n\nQuestions and comments should be directed to the `distutils-sig\nmailing list `_.\nBug reports and especially tested patches may be\nsubmitted directly to the `bug tracker\n`_.\n\n\nCode of Conduct\n===============\n\nEveryone interacting in the setuptools project's codebases, issue trackers,\nchat rooms, and mailing lists is expected to follow the\n`PSF Code of Conduct `_.\n\n\nFor Enterprise\n==============\n\nAvailable as part of the Tidelift Subscription.\n\nSetuptools and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use.\n\n`Learn more `_.\n\n\nSecurity Contact\n================\n\nTo report a security vulnerability, please use the\n`Tidelift security contact `_.\nTidelift will coordinate the fix and disclosure.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Python Packaging Authority", + "email": "distutils-sig@python.org", + "url": null + } + ], + "keywords": [ + "CPAN PyPI distutils eggs package management", + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: System :: Archiving :: Packaging", + "Topic :: System :: Systems Administration", + "Topic :: Utilities" + ], + "homepage_url": "https://github.com/pypa/setuptools", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "mit", + "declared_license": { + "classifiers": [ + "License :: OSI Approved :: MIT License" + ] + }, + "notice_text": null, + "contains_source_code": null, + "source_packages": [], + "extra_data": {}, + "dependencies": [], + "purl": "pkg:pypi/setuptools@58.2.0", + "repository_homepage_url": "https://pypi.org/project/https://pypi.org", + "repository_download_url": "https://pypi.org/packages/source/s/setuptools/setuptools-58.2.0.tar.gz", + "api_data_url": "https://pypi.org/pypi/setuptools/58.2.0/json" + } + ], + "for_packages": [ + "pkg:pypi/setuptools@58.2.0" + ], + "scan_errors": [] + }, + { + "path": "README.rst", + "type": "file", + "package_data": [], + "for_packages": [ + "pkg:pypi/setuptools@58.2.0" + ], + "scan_errors": [] + }, + { + "path": "pkg_resources", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "pkg_resources/tests", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "pkg_resources/tests/data", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "pkg_resources/tests/data/my-test-package-zip", + "type": "directory", + "package_data": [], + "for_packages": [], + "scan_errors": [] + }, + { + "path": "pkg_resources/tests/data/my-test-package-zip/my-test-package.zip", + "type": "file", + "package_data": [ + { + "type": "pypi", + "namespace": null, + "name": "my-test-package", + "version": "1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": {}, + "notice_text": null, + "contains_source_code": null, + "source_packages": [], + "extra_data": {}, + "dependencies": [], + "purl": "pkg:pypi/my-test-package@1.0", + "repository_homepage_url": "https://pypi.org/project/https://pypi.org", + "repository_download_url": "https://pypi.org/packages/source/m/my-test-package/my-test-package-1.0.tar.gz", + "api_data_url": "https://pypi.org/pypi/my-test-package/1.0/json" + } + ], + "for_packages": [ + "pkg:pypi/setuptools@58.2.0", + "pkg:pypi/my-test-package@1.0" + ], + "scan_errors": [] + }, + { + "path": "pyproject.toml", + "type": "file", + "package_data": [], + "for_packages": [ + "pkg:pypi/setuptools@58.2.0" + ], + "scan_errors": [] + }, + { + "path": "setup.cfg", + "type": "file", + "package_data": [], + "for_packages": [ + "pkg:pypi/setuptools@58.2.0" + ], + "scan_errors": [] + }, + { + "path": "setup.py", + "type": "file", + "package_data": [ + { + "type": "pypi", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": {}, + "notice_text": null, + "contains_source_code": null, + "source_packages": [], + "extra_data": {}, + "dependencies": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + } + ], + "for_packages": [ + "pkg:pypi/setuptools@58.2.0" + ], + "scan_errors": [] + }, + { + "path": "tox.ini", + "type": "file", + "package_data": [ + { + "type": "pypi", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": null, + "declared_license": null, + "notice_text": null, + "contains_source_code": null, + "source_packages": [], + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:pypi/towncrier", + "extracted_requirement": "towncrier", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/bump2version", + "extracted_requirement": "bump2version", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/build", + "extracted_requirement": "build", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/twine", + "extracted_requirement": ">=3", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {} + }, + { + "purl": "pkg:pypi/jaraco.develop", + "extracted_requirement": ">=7.1", + "scope": "install", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {} + } + ], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null + } + ], + "for_packages": [ + "pkg:pypi/setuptools@58.2.0" + ], + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/packagedcode/test_package_instance.py b/tests/packagedcode/test_package_instance.py index 464ea1c4b3c..f16312986c8 100644 --- a/tests/packagedcode/test_package_instance.py +++ b/tests/packagedcode/test_package_instance.py @@ -37,6 +37,13 @@ def test_package_instance_scan_python_with_uuid(self): run_scan_click(['--package', '--strip-root', '--processes', '-1', test_dir, '--json', result_file]) check_json_scan(expected_file, result_file, regen=False, ignore_instance_uuid=True) + def test_package_instance_scan_python_with_test_manifests(self): + test_dir = self.get_test_loc('instance/pypi-with-test-manifests') + result_file = self.get_temp_file('json') + expected_file = self.get_test_loc('instance/python-package-instance-expected-with-test-manifests.json') + run_scan_click(['--package', '--strip-root', '--processes', '-1', test_dir, '--json', result_file]) + check_json_scan(expected_file, result_file, regen=True, remove_instance_uuid=True) + def test_package_data_merge_generic(self, regen=False): input_file = self.get_test_loc('instance/python-manifests-click-scanned.json') expected_file = self.get_test_loc('instance/python-manifests-click-scanned-result.json')