From 8801b5147a11929c415f19b232f8b7b2b28d1d93 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 29 Oct 2024 13:52:02 +0530 Subject: [PATCH 1/3] Bump scancode-toolkit version to v32.3.0 Signed-off-by: Ayan Sinha Mahapatra --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index b76a30bea..00b424042 100644 --- a/setup.cfg +++ b/setup.cfg @@ -72,7 +72,7 @@ install_requires = # Docker container-inspector==33.0.0 # ScanCode-toolkit - scancode-toolkit[packages]==32.2.1 + scancode-toolkit[packages]==32.3.0 extractcode[full]==31.0.0 commoncode==32.0.0 packageurl-python==0.15.6 From dde1fc5b7a9d0bc3db1d0a65b1585a530bb20065 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 21 Oct 2024 14:52:35 +0700 Subject: [PATCH 2/3] Rename dependency and license match attributes * Rename is_resolved to is_pinned for dependencies * Rename spdx_license_expression to license_expression_spdx for license matches Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/admin.py | 4 +- scanpipe/api/serializers.py | 4 +- scanpipe/filters.py | 8 +- ...ename_discovered_dependencies_attribute.py | 44 +++ scanpipe/models.py | 6 +- scanpipe/pipelines/find_vulnerabilities.py | 2 +- scanpipe/pipes/purldb.py | 4 +- scanpipe/pipes/resolve.py | 2 +- .../templates/scanpipe/dependency_list.html | 4 +- .../templates/scanpipe/project_charts.html | 6 +- scanpipe/tests/__init__.py | 6 +- .../data/asgiref/asgiref-3.3.0_fixtures.json | 36 +- ...asgiref-3.3.0_load_inventory_expected.json | 40 +- .../asgiref-3.3.0_scanpipe_output.json | 36 +- .../asgiref/asgiref-3.3.0_toolkit_scan.json | 40 +- .../asgiref-3.3.0_walk_test_fixtures.json | 36 +- .../tests/data/d2d/about_files/expected.json | 2 +- .../tests/data/d2d/flume-ng-node-d2d.json | 158 ++++---- .../resolved_dependencies_cocoapods.json | 122 +++--- ...ved_dependencies_npm_inspect_packages.json | 40 +- .../resolved_dependencies_nuget.json | 336 ++++++++--------- .../resolved_dependencies_pip.json | 58 +-- ..._dependencies_poetry_inspect_packages.json | 54 +-- ...d_dependencies_swift_inspect_packages.json | 40 +- .../docker/alpine_3_15_4_scan_codebase.json | 28 +- .../data/docker/centos_scan_codebase.json | 346 ++++++++++-------- .../data/docker/debian_scan_codebase.json | 14 +- .../gcr_io_distroless_base_scan_codebase.json | 26 +- .../openpdf-parent-1.3.11_scan_package.json | 50 +-- .../data/manifests/package.expected.json | 12 +- .../request_post_response.json | 2 +- .../rootfs/basic-rootfs_root_filesystems.json | 14 +- ...-0.6.0-py3-none-any.whl_scan_codebase.json | 50 +-- .../scancode/is-npm-1.0.0_scan_codebase.json | 10 +- .../scancode/is-npm-1.0.0_scan_package.json | 16 +- .../is-npm-1.0.0_scan_package_summary.json | 4 +- .../data/scancode/is-npm-1.0.0_summary.json | 4 +- .../multiple-is-npm-1.0.0_scan_package.json | 28 +- ...ple-is-npm-1.0.0_scan_package_summary.json | 4 +- .../scancode/package_assembly_codebase.json | 4 +- scanpipe/tests/pipes/test_input.py | 2 +- scanpipe/tests/pipes/test_resolve.py | 2 +- scanpipe/tests/test_api.py | 2 +- scanpipe/tests/test_filters.py | 6 +- scanpipe/tests/test_models.py | 4 +- scanpipe/views.py | 8 +- 46 files changed, 904 insertions(+), 820 deletions(-) create mode 100644 scanpipe/migrations/0068_rename_discovered_dependencies_attribute.py diff --git a/scanpipe/admin.py b/scanpipe/admin.py index 628ef564e..f8272c44d 100644 --- a/scanpipe/admin.py +++ b/scanpipe/admin.py @@ -152,7 +152,7 @@ class DiscoveredDependencyAdmin(ScanPipeBaseAdmin): "scope", "is_runtime", "is_optional", - "is_resolved", + "is_pinned", "is_direct", "project", ] @@ -171,7 +171,7 @@ class DiscoveredDependencyAdmin(ScanPipeBaseAdmin): "scope", "is_runtime", "is_optional", - "is_resolved", + "is_pinned", "is_direct", ] ordering = ["project", "dependency_uid"] diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py index 5da4f1186..ded09bef5 100644 --- a/scanpipe/api/serializers.py +++ b/scanpipe/api/serializers.py @@ -268,7 +268,7 @@ def get_discovered_dependencies_summary(self, project): "total": base_qs.count(), "is_runtime": base_qs.filter(is_runtime=True).count(), "is_optional": base_qs.filter(is_optional=True).count(), - "is_resolved": base_qs.filter(is_resolved=True).count(), + "is_pinned": base_qs.filter(is_pinned=True).count(), } def get_codebase_relations_summary(self, project): @@ -448,7 +448,7 @@ class Meta: "scope", "is_runtime", "is_optional", - "is_resolved", + "is_pinned", "is_direct", "dependency_uid", "for_package_uid", diff --git a/scanpipe/filters.py b/scanpipe/filters.py index a440bba5e..8c8be0eeb 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -739,7 +739,7 @@ class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): "scope", "is_runtime", "is_optional", - "is_resolved", + "is_pinned", "is_direct", "datasource_id", "is_vulnerable", @@ -760,7 +760,7 @@ class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): "scope", "is_runtime", "is_optional", - "is_resolved", + "is_pinned", "is_direct", "for_package", "resolved_to_package", @@ -775,7 +775,7 @@ class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): datasource_id = ModelFieldValuesFilter() is_runtime = StrictBooleanFilter() is_optional = StrictBooleanFilter() - is_resolved = StrictBooleanFilter() + is_pinned = StrictBooleanFilter() is_direct = StrictBooleanFilter() is_vulnerable = IsVulnerable(field_name="affected_by_vulnerabilities") @@ -794,7 +794,7 @@ class Meta: "scope", "is_runtime", "is_optional", - "is_resolved", + "is_pinned", "is_direct", "datasource_id", "is_vulnerable", diff --git a/scanpipe/migrations/0068_rename_discovered_dependencies_attribute.py b/scanpipe/migrations/0068_rename_discovered_dependencies_attribute.py new file mode 100644 index 000000000..f8a03e3b0 --- /dev/null +++ b/scanpipe/migrations/0068_rename_discovered_dependencies_attribute.py @@ -0,0 +1,44 @@ +# Generated by Django 5.0.7 on 2024-10-21 07:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scanpipe", "0067_discoveredpackage_notes"), + ] + + operations = [ + migrations.AlterModelOptions( + name="discovereddependency", + options={ + "ordering": [ + "-is_runtime", + "-is_pinned", + "is_optional", + "dependency_uid", + "for_package", + "datafile_resource", + "datasource_id", + ], + "verbose_name": "discovered dependency", + "verbose_name_plural": "discovered dependencies", + }, + ), + migrations.RemoveIndex( + model_name="discovereddependency", + name="scanpipe_di_is_reso_10570c_idx", + ), + migrations.RenameField( + model_name="discovereddependency", + old_name="is_resolved", + new_name="is_pinned", + ), + migrations.AddIndex( + model_name="discovereddependency", + index=models.Index( + fields=["is_pinned"], name="scanpipe_di_is_pinn_5667b2_idx" + ), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index 9dc86428f..fd84836f0 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -3700,7 +3700,7 @@ class DiscoveredDependency( default=False, help_text=_("True if this dependency is an optional dependency"), ) - is_resolved = models.BooleanField( + is_pinned = models.BooleanField( default=False, help_text=_( "True if this dependency version requirement has been pinned " @@ -3722,7 +3722,7 @@ class Meta: verbose_name_plural = "discovered dependencies" ordering = [ "-is_runtime", - "-is_resolved", + "-is_pinned", "is_optional", "dependency_uid", "for_package", @@ -3733,7 +3733,7 @@ class Meta: models.Index(fields=["scope"]), models.Index(fields=["is_runtime"]), models.Index(fields=["is_optional"]), - models.Index(fields=["is_resolved"]), + models.Index(fields=["is_pinned"]), models.Index(fields=["is_direct"]), ] constraints = [ diff --git a/scanpipe/pipelines/find_vulnerabilities.py b/scanpipe/pipelines/find_vulnerabilities.py index 7a6323b98..6a48812a2 100644 --- a/scanpipe/pipelines/find_vulnerabilities.py +++ b/scanpipe/pipelines/find_vulnerabilities.py @@ -62,7 +62,7 @@ def lookup_packages_vulnerabilities(self): def lookup_dependencies_vulnerabilities(self): """Check for vulnerabilities for each of the project's discovered dependency.""" - dependencies = self.project.discovereddependencies.filter(is_resolved=True) + dependencies = self.project.discovereddependencies.filter(is_pinned=True) vulnerablecode.fetch_vulnerabilities( packages=dependencies, ignore_set=self.project.ignored_vulnerabilities_set, diff --git a/scanpipe/pipes/purldb.py b/scanpipe/pipes/purldb.py index ba1d2181a..801e79325 100644 --- a/scanpipe/pipes/purldb.py +++ b/scanpipe/pipes/purldb.py @@ -289,7 +289,7 @@ def feed_purldb(packages, chunk_size, logger=logger.info): def get_unique_resolved_purls(project): """Return PURLs from project's resolved DiscoveredDependencies.""" - packages_resolved = project.discovereddependencies.filter(is_resolved=True) + packages_resolved = project.discovereddependencies.filter(is_pinned=True) distinct_results = packages_resolved.values("type", "namespace", "name", "version") @@ -300,7 +300,7 @@ def get_unique_resolved_purls(project): def get_unique_unresolved_purls(project): """Return PURLs from project's unresolved DiscoveredDependencies.""" packages_unresolved = project.discovereddependencies.filter( - is_resolved=False + is_pinned=False ).exclude(extracted_requirement="*") distinct_unresolved_results = packages_unresolved.values( diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index e841176de..55bf93197 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -151,7 +151,7 @@ def create_dependencies_from_packages_extra_data(project): resolved_to_package=resolved_to_package, datafile_resource=datafile_resource, is_runtime=True, - is_resolved=True, + is_pinned=True, is_direct=True, ) created_count += 1 diff --git a/scanpipe/templates/scanpipe/dependency_list.html b/scanpipe/templates/scanpipe/dependency_list.html index 92a3de0b6..b089df1a6 100644 --- a/scanpipe/templates/scanpipe/dependency_list.html +++ b/scanpipe/templates/scanpipe/dependency_list.html @@ -52,10 +52,10 @@ {{ dependency.is_optional }} - {{ dependency.is_resolved }} + {{ dependency.is_pinned }} - {{ dependency.is_direct }} + {{ dependency.is_direct }} {% if dependency.for_package %} diff --git a/scanpipe/templates/scanpipe/project_charts.html b/scanpipe/templates/scanpipe/project_charts.html index 9da3108ac..20db76198 100644 --- a/scanpipe/templates/scanpipe/project_charts.html +++ b/scanpipe/templates/scanpipe/project_charts.html @@ -35,7 +35,7 @@

-
+
{% endif %} @@ -82,7 +82,7 @@

{{ dependency_type|json_script:"dependency_type" }} {{ dependency_is_runtime|json_script:"dependency_is_runtime" }} {{ dependency_is_optional|json_script:"dependency_is_optional" }} - {{ dependency_is_resolved|json_script:"dependency_is_resolved" }} + {{ dependency_is_pinned|json_script:"dependency_is_pinned" }}