Remove distro specific pipes #438

    * Move get_installed_packages to rootfs.py
    * Use get_package_data instead of get_package_info
    * Rename all instances of packages to package_data when scanning for application packages
    * Update test docker images and test results
    * Add test for basic rootfs

Signed-off-by: Jono Yang <jyang@nexb.com>

scanpipe/pipes/docker.py

@@ -22,11 +22,9 @@
 
 import logging
 import posixpath
-from functools import partial
 from pathlib import Path
 
 from container_inspector.image import Image
-from packagedcode import plugin_package
 
 from scanpipe import pipes
 from scanpipe.pipes import rootfs
@@ -140,15 +138,6 @@ def create_codebase_resources(project, image):
         )
 
 
-def package_getter(root_dir, **kwargs):
-    """
-    Returns installed package objects.
-    """
-    packages = plugin_package.get_installed_packages(root_dir)
-    for package in packages:
-        yield package.purl, package
-
-
 def scan_image_for_system_packages(project, image, detect_licenses=True):
     """
     Given a `project` and an `image` - this scans the `image` layer by layer for
@@ -162,10 +151,10 @@ def scan_image_for_system_packages(project, image, detect_licenses=True):
         raise rootfs.DistroNotFound(f"Distro not found.")
 
     distro_id = image.distro.identifier
-    if distro_id not in rootfs.PACKAGE_GETTER_BY_DISTRO:
+    if distro_id not in rootfs.SUPPORTED_DISTROS:
         raise rootfs.DistroNotSupported(f'Distro "{distro_id}" is not supported.')
 
-    installed_packages = image.get_installed_packages(package_getter)
+    installed_packages = image.get_installed_packages(rootfs.package_getter)
 
     for i, (purl, package, layer) in enumerate(installed_packages):
         logger.info(f"Creating package #{i}: {purl}")

scanpipe/pipes/rootfs.py

@@ -23,36 +23,32 @@
 import fnmatch
 import logging
 import os
-from functools import partial
 
 from django.core.exceptions import ObjectDoesNotExist
 from django.db.models import Q
 
 import attr
 from commoncode.ignore import default_ignores
 from container_inspector.distro import Distro
+from packagedcode import plugin_package
 
 from scanpipe import pipes
-from scanpipe.pipes import alpine
-from scanpipe.pipes import debian
-from scanpipe.pipes import rpm
-from scanpipe.pipes import windows
 
 logger = logging.getLogger(__name__)
 
-PACKAGE_GETTER_BY_DISTRO = {
-    "alpine": alpine.package_getter,
-    "debian": partial(debian.package_getter, distro="debian"),
-    "ubuntu": partial(debian.package_getter, distro="ubuntu"),
-    "rhel": rpm.package_getter,
-    "centos": rpm.package_getter,
-    "fedora": rpm.package_getter,
-    "sles": rpm.package_getter,
-    "opensuse": rpm.package_getter,
-    "opensuse-tumbleweed": rpm.package_getter,
-    "photon": rpm.package_getter,
-    "windows": windows.package_getter,
-}
+SUPPORTED_DISTROS = [
+    "alpine",
+    "debian",
+    "ubuntu",
+    "rhel",
+    "centos",
+    "fedora",
+    "sles",
+    "opensuse",
+    "opensuse-tumbleweed",
+    "photon",
+    "windows",
+]
 
 
 class DistroNotFound(Exception):
@@ -198,6 +194,15 @@ def has_hash_diff(install_file, codebase_resource):
     return False
 
 
+def package_getter(root_dir, **kwargs):
+    """
+    Returns installed package objects.
+    """
+    packages = plugin_package.get_installed_packages(root_dir)
+    for package in packages:
+        yield package.purl, package
+
+
 def scan_rootfs_for_system_packages(project, rootfs, detect_licenses=True):
     """
     Given a `project` Project and a `rootfs` RootFs, scan the `rootfs` for
@@ -211,23 +216,23 @@ def scan_rootfs_for_system_packages(project, rootfs, detect_licenses=True):
         raise DistroNotFound(f"Distro not found.")
 
     distro_id = rootfs.distro.identifier
-    if distro_id not in PACKAGE_GETTER_BY_DISTRO:
+    if distro_id not in SUPPORTED_DISTROS:
         raise DistroNotSupported(f'Distro "{distro_id}" is not supported.')
 
-    package_getter = partial(
-        PACKAGE_GETTER_BY_DISTRO[distro_id],
-        distro=distro_id,
-        detect_licenses=detect_licenses,
-    )
+    logger.info(f"rootfs location: {rootfs.location}")
 
     installed_packages = rootfs.get_installed_packages(package_getter)
 
     for i, (purl, package) in enumerate(installed_packages):
         logger.info(f"Creating package #{i}: {purl}")
         created_package = pipes.update_or_create_package(project, package.to_dict())
 
+        installed_files = []
+        if hasattr(package, "resources"):
+            installed_files = package.resources
+
         # We have no files for this installed package, we cannot go further.
-        if not package.installed_files:
+        if not installed_files:
             logger.info(f"  No installed_files for: {purl}")
             continue
 
@@ -236,7 +241,7 @@ def scan_rootfs_for_system_packages(project, rootfs, detect_licenses=True):
 
         codebase_resources = project.codebaseresources.all()
 
-        for install_file in package.installed_files:
+        for install_file in installed_files:
             rootfs_path = pipes.normalize_path(install_file.path)
             logger.info(f"   installed file rootfs_path: {rootfs_path}")
 

scanpipe/pipes/scancode.py

@@ -200,14 +200,14 @@ def scan_file(location, with_threading=True):
     return _scan_resource(location, scanners, with_threading)
 
 
-def scan_for_package_info(location, with_threading=True):
+def scan_for_package_data(location, with_threading=True):
     """
     Runs a package scan on provided `location` using the scancode-toolkit direct API.
 
     Returns a dict of scan `results` and a list of `errors`.
     """
     scanners = [
-        Scanner("packages", scancode_api.get_package_info),
+        Scanner("package_data", scancode_api.get_package_data),
     ]
     return _scan_resource(location, scanners, with_threading)
 
@@ -319,7 +319,7 @@ def scan_for_application_packages(project):
     resource_qs = project.codebaseresources.no_status()
     _scan_and_save(
         resource_qs=resource_qs,
-        scan_func=scan_for_package_info,
+        scan_func=scan_for_package_data,
         save_func=save_scan_package_results,
     )
 

scanpipe/tests/data/is-npm-1.0.0_scan_package.json

@@ -103,7 +103,7 @@
     "license_clarity_score": {
       "score": 90,
       "declared_license": true,
-      "precise_license_detection": true,
+      "identification_precision": true,
       "has_license_text": false,
       "declared_copyrights": true,
       "conflicting_license_categories": false,

scanpipe/tests/data/is-npm-1.0.0_scan_package_summary.json

@@ -3,7 +3,7 @@
   "license_clarity_score": {
     "score": 90,
     "declared_license": true,
-    "precise_license_detection": true,
+    "identification_precision": true,
     "has_license_text": false,
     "declared_copyrights": true,
     "conflicting_license_categories": false,

scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package.json

@@ -172,7 +172,7 @@
     "license_clarity_score": {
       "score": 90,
       "declared_license": true,
-      "precise_license_detection": true,
+      "identification_precision": true,
       "has_license_text": false,
       "declared_copyrights": true,
       "conflicting_license_categories": false,

scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package_summary.json

@@ -3,7 +3,7 @@
   "license_clarity_score": {
     "score": 90,
     "declared_license": true,
-    "precise_license_detection": true,
+    "identification_precision": true,
     "has_license_text": false,
     "declared_copyrights": true,
     "conflicting_license_categories": false,