Get debian distro info from clues

We will now deduce and populate debian distro information, i.e. their namespace from clues present in version/maintainer info. Reference: aboutcode-org/scancode.io#899 Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
aboutcode-org · Jan 29, 2024 · 1cd5d02 · 1cd5d02
1 parent f70bbb7
commit 1cd5d02
Show file tree

Hide file tree

Showing 11 changed files with 823 additions and 12,818 deletions.
diff --git a/src/packagedcode/debian.py b/src/packagedcode/debian.py
@@ -305,6 +305,7 @@ def assemble(cls, package_data, resource, codebase, package_adder):
                 package.update(
                     package_data=package_data,
                     datafile_path=res.path,
+                    check_compatible=False,
                     replace=False,
                     include_version=False,
                     include_qualifiers=False,
@@ -628,6 +629,16 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
     if keyword:
         keywords.append(keyword)
 
+    # Get distro/namespace information from clues in package data
+    if not distro:
+        for clue, namespace in version_clues_for_namespace.items():
+            if clue in version:
+                distro = namespace
+
+        for clue, namespace in maintainer_clues_for_namespace.items():
+            if clue in maintainer:
+                distro = namespace
+
     source_packages = []
     source = debian_data.get('source')
     if source:
@@ -656,6 +667,18 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
     )
 
 
+version_clues_for_namespace = {
+    'deb': 'debian',
+    'ubuntu': 'ubuntu',
+}
+
+
+maintainer_clues_for_namespace = {
+    'packages.debian.org': 'debian',
+    'lists.ubuntu.com': 'ubuntu',
+}
+
+
 ignored_root_dirs = {
     '/.',
     '/bin',

diff --git a/src/packagedcode/models.py b/src/packagedcode/models.py
@@ -119,9 +119,10 @@
 """
 
 SCANCODE_DEBUG_PACKAGE = os.environ.get('SCANCODE_DEBUG_PACKAGE', False)
+SCANCODE_DEBUG_PACKAGE_ASSEMBLY = os.environ.get('SCANCODE_DEBUG_PACKAGE_ASSEMBLY', False)
 
 TRACE = SCANCODE_DEBUG_PACKAGE
-TRACE_UPDATE = SCANCODE_DEBUG_PACKAGE
+TRACE_UPDATE = SCANCODE_DEBUG_PACKAGE_ASSEMBLY
 
 
 def logger_debug(*args):
@@ -1537,6 +1538,7 @@ def update(
         self,
         package_data,
         datafile_path,
+        check_compatible=True,
         replace=False,
         include_version=True,
         include_qualifiers=False,
@@ -1568,7 +1570,7 @@ def update(
         if isinstance(package_data, dict):
             package_data = PackageData.from_dict(package_data)
 
-        if not is_compatible(
+        if check_compatible and not is_compatible(
             purl1=self,
             purl2=package_data,
             include_version=include_version,

diff --git a/tests/packagedcode/data/debian/basic-rootfs-expected.json b/tests/packagedcode/data/debian/basic-rootfs-expected.json
@@ -2,7 +2,7 @@
   "packages": [
     {
       "type": "deb",
-      "namespace": null,
+      "namespace": "ubuntu",
       "name": "libncurses5",
       "version": "6.1-1ubuntu1.18.04",
       "qualifiers": {
@@ -134,7 +134,7 @@
       "extracted_license_statement": null,
       "notice_text": null,
       "source_packages": [
-        "pkg:deb/ncurses"
+        "pkg:deb/ubuntu/ncurses"
       ],
       "extra_data": {
         "multi_arch": "same",
@@ -180,7 +180,7 @@
       "repository_homepage_url": null,
       "repository_download_url": null,
       "api_data_url": null,
-      "package_uid": "pkg:deb/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758",
+      "package_uid": "pkg:deb/ubuntu/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758",
       "datafile_paths": [
         "basic-rootfs.tar.gz/var/lib/dpkg/status",
         "basic-rootfs.tar.gz/usr/share/doc/libncurses5/copyright",
@@ -191,11 +191,11 @@
         "debian_copyright_in_package",
         "debian_installed_md5sums"
       ],
-      "purl": "pkg:deb/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64"
+      "purl": "pkg:deb/ubuntu/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64"
     },
     {
       "type": "deb",
-      "namespace": null,
+      "namespace": "ubuntu",
       "name": "libndp0",
       "version": "1.4-2ubuntu0.16.04.1",
       "qualifiers": {
@@ -291,7 +291,7 @@
       "extracted_license_statement": "- LGPL-2.1+\n- LGPL-2.1+\n- LGPL-2.1+\n",
       "notice_text": null,
       "source_packages": [
-        "pkg:deb/libndp"
+        "pkg:deb/ubuntu/libndp"
       ],
       "extra_data": {
         "multi_arch": "same",
@@ -319,7 +319,7 @@
       "repository_homepage_url": null,
       "repository_download_url": null,
       "api_data_url": null,
-      "package_uid": "pkg:deb/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758",
+      "package_uid": "pkg:deb/ubuntu/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758",
       "datafile_paths": [
         "basic-rootfs.tar.gz/var/lib/dpkg/status",
         "basic-rootfs.tar.gz/usr/share/doc/libndp0/copyright",
@@ -330,7 +330,7 @@
         "debian_copyright_in_package",
         "debian_installed_md5sums"
       ],
-      "purl": "pkg:deb/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64"
+      "purl": "pkg:deb/ubuntu/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64"
     }
   ],
   "dependencies": [],
@@ -507,7 +507,7 @@
         }
       ],
       "for_packages": [
-        "pkg:deb/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
+        "pkg:deb/ubuntu/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
       ],
       "scan_errors": []
     },
@@ -619,7 +619,7 @@
         }
       ],
       "for_packages": [
-        "pkg:deb/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
+        "pkg:deb/ubuntu/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
       ],
       "scan_errors": []
     },
@@ -738,7 +738,7 @@
         }
       ],
       "for_packages": [
-        "pkg:deb/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
+        "pkg:deb/ubuntu/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
       ],
       "scan_errors": []
     },
@@ -820,7 +820,7 @@
         }
       ],
       "for_packages": [
-        "pkg:deb/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
+        "pkg:deb/ubuntu/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64&uuid=fixed-uid-done-for-testing-5642512d1758"
       ],
       "scan_errors": []
     },
@@ -830,7 +830,7 @@
       "package_data": [
         {
           "type": "deb",
-          "namespace": null,
+          "namespace": "ubuntu",
           "name": "libncurses5",
           "version": "6.1-1ubuntu1.18.04",
           "qualifiers": {
@@ -873,7 +873,7 @@
           "extracted_license_statement": null,
           "notice_text": null,
           "source_packages": [
-            "pkg:deb/ncurses"
+            "pkg:deb/ubuntu/ncurses"
           ],
           "file_references": [],
           "extra_data": {
@@ -884,11 +884,11 @@
           "repository_download_url": null,
           "api_data_url": null,
           "datasource_id": "debian_installed_status_db",
-          "purl": "pkg:deb/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64"
+          "purl": "pkg:deb/ubuntu/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64"
         },
         {
           "type": "deb",
-          "namespace": null,
+          "namespace": "ubuntu",
           "name": "libndp0",
           "version": "1.4-2ubuntu0.16.04.1",
           "qualifiers": {
@@ -931,7 +931,7 @@
           "extracted_license_statement": null,
           "notice_text": null,
           "source_packages": [
-            "pkg:deb/libndp"
+            "pkg:deb/ubuntu/libndp"
           ],
           "file_references": [],
           "extra_data": {
@@ -942,7 +942,7 @@
           "repository_download_url": null,
           "api_data_url": null,
           "datasource_id": "debian_installed_status_db",
-          "purl": "pkg:deb/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64"
+          "purl": "pkg:deb/ubuntu/libndp0@1.4-2ubuntu0.16.04.1?architecture=amd64"
         }
       ],
       "for_packages": [],

diff --git a/tests/packagedcode/data/debian/basic/status.expected b/tests/packagedcode/data/debian/basic/status.expected
@@ -1,7 +1,7 @@
 [
   {
     "type": "deb",
-    "namespace": null,
+    "namespace": "ubuntu",
     "name": "libncurses5",
     "version": "6.1-1ubuntu1.18.04",
     "qualifiers": {
@@ -44,7 +44,7 @@
     "extracted_license_statement": null,
     "notice_text": null,
     "source_packages": [
-      "pkg:deb/ncurses"
+      "pkg:deb/ubuntu/ncurses"
     ],
     "file_references": [],
     "extra_data": {
@@ -55,11 +55,11 @@
     "repository_download_url": null,
     "api_data_url": null,
     "datasource_id": "debian_installed_status_db",
-    "purl": "pkg:deb/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64"
+    "purl": "pkg:deb/ubuntu/libncurses5@6.1-1ubuntu1.18.04?architecture=amd64"
   },
   {
     "type": "deb",
-    "namespace": null,
+    "namespace": "ubuntu",
     "name": "libcom-err2",
     "version": "1.44.1-1ubuntu1.1",
     "qualifiers": {
@@ -102,7 +102,7 @@
     "extracted_license_statement": null,
     "notice_text": null,
     "source_packages": [
-      "pkg:deb/e2fsprogs"
+      "pkg:deb/ubuntu/e2fsprogs"
     ],
     "file_references": [],
     "extra_data": {
@@ -113,11 +113,11 @@
     "repository_download_url": null,
     "api_data_url": null,
     "datasource_id": "debian_installed_status_db",
-    "purl": "pkg:deb/libcom-err2@1.44.1-1ubuntu1.1?architecture=amd64"
+    "purl": "pkg:deb/ubuntu/libcom-err2@1.44.1-1ubuntu1.1?architecture=amd64"
   },
   {
     "type": "deb",
-    "namespace": null,
+    "namespace": "ubuntu",
     "name": "libapt-pkg5.0",
     "version": "1.6.11",
     "qualifiers": {
@@ -160,7 +160,7 @@
     "extracted_license_statement": null,
     "notice_text": null,
     "source_packages": [
-      "pkg:deb/apt"
+      "pkg:deb/ubuntu/apt"
     ],
     "file_references": [],
     "extra_data": {
@@ -171,11 +171,11 @@
     "repository_download_url": null,
     "api_data_url": null,
     "datasource_id": "debian_installed_status_db",
-    "purl": "pkg:deb/libapt-pkg5.0@1.6.11?architecture=amd64"
+    "purl": "pkg:deb/ubuntu/libapt-pkg5.0@1.6.11?architecture=amd64"
   },
   {
     "type": "deb",
-    "namespace": null,
+    "namespace": "ubuntu",
     "name": "libaudit1",
     "version": "1:2.8.2-1ubuntu1",
     "qualifiers": {
@@ -218,7 +218,7 @@
     "extracted_license_statement": null,
     "notice_text": null,
     "source_packages": [
-      "pkg:deb/audit"
+      "pkg:deb/ubuntu/audit"
     ],
     "file_references": [],
     "extra_data": {
@@ -229,11 +229,11 @@
     "repository_download_url": null,
     "api_data_url": null,
     "datasource_id": "debian_installed_status_db",
-    "purl": "pkg:deb/libaudit1@1:2.8.2-1ubuntu1?architecture=amd64"
+    "purl": "pkg:deb/ubuntu/libaudit1@1:2.8.2-1ubuntu1?architecture=amd64"
   },
   {
     "type": "deb",
-    "namespace": null,
+    "namespace": "ubuntu",
     "name": "perl-base",
     "version": "5.26.1-6ubuntu0.3",
     "qualifiers": {
@@ -276,7 +276,7 @@
     "extracted_license_statement": null,
     "notice_text": null,
     "source_packages": [
-      "pkg:deb/perl"
+      "pkg:deb/ubuntu/perl"
     ],
     "file_references": [],
     "extra_data": {},
@@ -285,11 +285,11 @@
     "repository_download_url": null,
     "api_data_url": null,
     "datasource_id": "debian_installed_status_db",
-    "purl": "pkg:deb/perl-base@5.26.1-6ubuntu0.3?architecture=amd64"
+    "purl": "pkg:deb/ubuntu/perl-base@5.26.1-6ubuntu0.3?architecture=amd64"
   },
   {
     "type": "deb",
-    "namespace": null,
+    "namespace": "ubuntu",
     "name": "libudev1",
     "version": "237-3ubuntu10.22",
     "qualifiers": {
@@ -332,7 +332,7 @@
     "extracted_license_statement": null,
     "notice_text": null,
     "source_packages": [
-      "pkg:deb/systemd"
+      "pkg:deb/ubuntu/systemd"
     ],
     "file_references": [],
     "extra_data": {
@@ -343,6 +343,6 @@
     "repository_download_url": null,
     "api_data_url": null,
     "datasource_id": "debian_installed_status_db",
-    "purl": "pkg:deb/libudev1@237-3ubuntu10.22?architecture=amd64"
+    "purl": "pkg:deb/ubuntu/libudev1@237-3ubuntu10.22?architecture=amd64"
   }
 ]