From 0b1e07f91aada201088605a84ea394182ce0f10e Mon Sep 17 00:00:00 2001 From: Jan Kowalleck Date: Tue, 13 Dec 2022 21:13:17 +0100 Subject: [PATCH] feat: add support for poetry lock format v2.0 (#469) Signed-off-by: tewfik-ghariani Signed-off-by: Jan Kowalleck Co-authored-by: tewfik-ghariani --- cyclonedx_py/parser/poetry.py | 15 +++++- poetry.lock | 46 ++++++++++++------- pyproject.toml | 1 + ...ck-simple.txt => poetry-lock11-simple.txt} | 0 tests/fixtures/poetry-lock20-simple.txt | 18 ++++++++ tests/test_parser_poetry.py | 21 +++++---- 6 files changed, 74 insertions(+), 27 deletions(-) rename tests/fixtures/{poetry-lock-simple.txt => poetry-lock11-simple.txt} (100%) create mode 100644 tests/fixtures/poetry-lock20-simple.txt diff --git a/cyclonedx_py/parser/poetry.py b/cyclonedx_py/parser/poetry.py index b299d23c..a88d07fc 100644 --- a/cyclonedx_py/parser/poetry.py +++ b/cyclonedx_py/parser/poetry.py @@ -42,6 +42,13 @@ def __init__( debug_message('loading poetry_lock_contents') poetry_lock = load_toml(poetry_lock_contents) + poetry_lock_metadata = poetry_lock['metadata'] + try: + poetry_lock_version = tuple(int(p) for p in str(poetry_lock_metadata['lock-version']).split('.')) + except Exception: + poetry_lock_version = (0,) + debug_message('detected poetry_lock_version: {!r}', poetry_lock_version) + debug_message('processing poetry_lock') for package in poetry_lock['package']: debug_message('processing package: {!r}', package) @@ -51,7 +58,12 @@ def __init__( name=package['name'], bom_ref=bom_ref, version=package['version'], purl=purl ) - for file_metadata in poetry_lock['metadata']['files'][package['name']]: + debug_message('detecting package_files') + package_files = package['files'] \ + if poetry_lock_version >= (2,) \ + else poetry_lock_metadata['files'][package['name']] + debug_message('processing package_files: {!r}', package_files) + for file_metadata in package_files: debug_message('processing file_metadata: {!r}', file_metadata) try: component.external_references.add(ExternalReference( @@ -64,7 +76,6 @@ def __init__( # @todo traceback and details to the output? debug_message('Warning: suppressed {!r}', error) del error - self._components.append(component) diff --git a/poetry.lock b/poetry.lock index 6867f5f4..8b29ecf2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -7,10 +7,10 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "zope.interface"] +tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six"] [[package]] name = "autopep8" @@ -74,6 +74,14 @@ packageurl-python = ">=0.9" sortedcontainers = ">=2.4.0,<3.0.0" toml = ">=0.10.0,<0.11.0" +[[package]] +name = "ddt" +version = "1.6.0" +description = "Data-Driven/Decorated Tests" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "distlib" version = "0.3.4" @@ -163,8 +171,8 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] +docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pep517", "pyfakefs", "pytest (>=4.6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy"] [[package]] name = "importlib-resources" @@ -178,8 +186,8 @@ python-versions = ">=3.6" zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-black (>=0.3.7)", "pytest-mypy"] +docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"] +testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy"] [[package]] name = "isort" @@ -190,10 +198,10 @@ optional = false python-versions = ">=3.6.1,<4.0" [package.extras] -pipfile_deprecated_finder = ["pipreqs", "requirementslib"] -requirements_deprecated_finder = ["pipreqs", "pip-api"] colors = ["colorama (>=0.4.3,<0.5.0)"] +pipfile_deprecated_finder = ["pipreqs", "requirementslib"] plugins = ["setuptools"] +requirements_deprecated_finder = ["pip-api", "pipreqs"] [[package]] name = "mccabe" @@ -264,8 +272,8 @@ python-versions = ">=3.6.*" packaging = "<22.0.0" [package.extras] -docs = ["Sphinx (>=3.3.1)", "sphinx-rtd-theme (>=0.5.0)", "doc8 (>=0.8.1)"] -testing = ["pytest (>=6,!=7.0.0)", "pytest-xdist (>=2)", "aboutcode-toolkit (>=6.0.0)", "black"] +docs = ["Sphinx (>=3.3.1)", "doc8 (>=0.8.1)", "sphinx-rtd-theme (>=0.5.0)"] +testing = ["aboutcode-toolkit (>=6.0.0)", "black", "pytest (>=6,!=7.0.0)", "pytest-xdist (>=2)"] [[package]] name = "platformdirs" @@ -382,7 +390,7 @@ virtualenv = ">=16.0.0,<20.0.0 || >20.0.0,<20.0.1 || >20.0.1,<20.0.2 || >20.0.2, [package.extras] docs = ["pygments-github-lexers (>=0.0.5)", "sphinx (>=2.0.0)", "sphinxcontrib-autoprogram (>=0.1.5)", "towncrier (>=18.5.0)"] -testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pytest (>=4.0.0)", "pytest-cov (>=2.5.1)", "pytest-mock (>=1.10.0)", "pytest-randomly (>=1.0.0)", "psutil (>=5.6.1)", "pathlib2 (>=2.3.3)"] +testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pathlib2 (>=2.3.3)", "psutil (>=5.6.1)", "pytest (>=4.0.0)", "pytest-cov (>=2.5.1)", "pytest-mock (>=1.10.0)", "pytest-randomly (>=1.0.0)"] [[package]] name = "typed-ast" @@ -434,7 +442,7 @@ six = ">=1.9.0,<2" [package.extras] docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=21.3)"] -testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"] +testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "packaging (>=20.0)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)"] [[package]] name = "zipp" @@ -445,13 +453,13 @@ optional = false python-versions = ">=3.6" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] +docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"] +testing = ["func-timeout", "jaraco.itertools", "pytest (>=4.6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy"] [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "8a8e2bb08219bd7ca43a369b055bbd2a4e3cf6641aac201293783b14df200a00" +content-hash = "d89fa891e36ed31f8085e3be38147ff80c62cd79684977124ca5a1dbb01ad885" [metadata.files] attrs = [ @@ -569,6 +577,10 @@ cyclonedx-python-lib = [ {file = "cyclonedx-python-lib-3.1.0.tar.gz", hash = "sha256:39e9d36347d4dc736474ab4f3a7cd7bc91050c9315df698f83a6d8bbcb290744"}, {file = "cyclonedx_python_lib-3.1.0-py3-none-any.whl", hash = "sha256:3c79f32bb7d6ed34eac3308dbc8f2a77fbd1fd3779991173a147d866eaa7423e"}, ] +ddt = [ + {file = "ddt-1.6.0-py2.py3-none-any.whl", hash = "sha256:e3c93b961a108b4f4d5a6c7f2263513d928baf3bb5b32af8e1c804bfb041141d"}, + {file = "ddt-1.6.0.tar.gz", hash = "sha256:f71b348731b8c78c3100bffbd951a769fbd439088d1fdbb3841eee019af80acd"}, +] distlib = [ {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"}, {file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"}, diff --git a/pyproject.toml b/pyproject.toml index b41927dc..98834565 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ toml = "^0.10.0" [tool.poetry.dev-dependencies] autopep8 = "^1.6.0" isort = { version = "^5.10.0", python = ">= 3.6.1" } +ddt = "^1.6.0" tox = "^3.25.1" coverage = [ { python = ">=3.6,<3.7", version = "^6.2", extras = ["toml"] }, diff --git a/tests/fixtures/poetry-lock-simple.txt b/tests/fixtures/poetry-lock11-simple.txt similarity index 100% rename from tests/fixtures/poetry-lock-simple.txt rename to tests/fixtures/poetry-lock11-simple.txt diff --git a/tests/fixtures/poetry-lock20-simple.txt b/tests/fixtures/poetry-lock20-simple.txt new file mode 100644 index 00000000..0f365cb0 --- /dev/null +++ b/tests/fixtures/poetry-lock20-simple.txt @@ -0,0 +1,18 @@ +# This file is automatically @generated by Poetry and should not be changed by hand. + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "b97d7a3bc03286e93fd688187cbdcd469fe0e5108cdc7936c432995f983f478c" diff --git a/tests/test_parser_poetry.py b/tests/test_parser_poetry.py index abb3655a..7dd0e27e 100644 --- a/tests/test_parser_poetry.py +++ b/tests/test_parser_poetry.py @@ -20,15 +20,19 @@ import os from unittest import TestCase +from ddt import data, ddt + from cyclonedx_py.parser.poetry import PoetryFileParser +@ddt class TestPoetryParser(TestCase): - def test_simple(self) -> None: - tests_poetry_lock_file = os.path.join(os.path.dirname(__file__), 'fixtures/poetry-lock-simple.txt') - - parser = PoetryFileParser(poetry_lock_filename=tests_poetry_lock_file) + @data('poetry-lock11-simple.txt', + 'poetry-lock20-simple.txt') + def test_simple(self, lock_file_name: str) -> None: + poetry_lock_filename = os.path.join(os.path.dirname(__file__), 'fixtures', lock_file_name) + parser = PoetryFileParser(poetry_lock_filename=poetry_lock_filename) self.assertEqual(1, parser.component_count()) component = next(filter(lambda c: c.name == 'toml', parser.get_components()), None) self.assertIsNotNone(component) @@ -37,10 +41,11 @@ def test_simple(self) -> None: self.assertEqual('0.10.2', component.version) self.assertEqual(2, len(component.external_references), f'{component.external_references}') - def test_simple_purl_bom_ref(self) -> None: - tests_poetry_lock_file = os.path.join(os.path.dirname(__file__), 'fixtures/poetry-lock-simple.txt') - - parser = PoetryFileParser(poetry_lock_filename=tests_poetry_lock_file, use_purl_bom_ref=True) + @data('poetry-lock11-simple.txt', + 'poetry-lock20-simple.txt') + def test_simple_purl_bom_ref(self, lock_file_name: str) -> None: + poetry_lock_filename = os.path.join(os.path.dirname(__file__), 'fixtures', lock_file_name) + parser = PoetryFileParser(poetry_lock_filename=poetry_lock_filename, use_purl_bom_ref=True) self.assertEqual(1, parser.component_count()) component = next(filter(lambda c: c.name == 'toml', parser.get_components()), None) self.assertIsNotNone(component)