Skip to content

Commit

Permalink
Improve debian package detection
Browse files Browse the repository at this point in the history
Detect and store more attributes from debian .dsc metadata
files. Also properly detect and create packages from control
and md5sums files.

Reference: aboutcode-org/scancode.io#1151
Reference: aboutcode-org/purldb#3

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
  • Loading branch information
AyanSinhaMahapatra committed Apr 4, 2024
1 parent cafcbcf commit 43dc430
Showing 11 changed files with 772 additions and 11 deletions.
19 changes: 17 additions & 2 deletions src/packagedcode/debian.py
Original file line number Diff line number Diff line change
@@ -127,7 +127,7 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder)
class DebianControlFileInExtractedDebHandler(models.DatafileHandler):
datasource_id = 'debian_control_extracted_deb'
default_package_type = 'deb'
path_patterns = ('*/control.tar.gz-extract/control',)
path_patterns = ('*/control.tar.gz-extract/control','*/control.tar.xz-extract/control')
description = 'Debian control file - extracted layout'
documentation_url = 'https://www.debian.org/doc/debian-policy/ch-controlfields.html'

@@ -590,7 +590,7 @@ def parse_debian_files_list(location, datasource_id, package_type):
else:
name = None
# For DebianMd5sumFilelistInPackageHandler we cannot infer name
if not name == "md5sums":
if not filename == "md5sums":
name = filename

file_references = []
@@ -663,6 +663,19 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
party = models.Party(role='maintainer', name=maintainer_name, email=maintainer_email)
parties.append(party)

uploaders = debian_data.get('uploaders')
if uploaders:
for uploader in uploaders.split(", "):
uploader_name, uploader_email = parse_debian_maintainers(uploader)
party = models.Party(role='uploader', name=uploader_name, email=uploader_email)
parties.append(party)

vcs_url = debian_data.get('vcs-git')
if vcs_url and ' ' in vcs_url:
vcs_url = vcs_url.split(' ')[0]

code_view_url = debian_data.get('vcs-browser')

keywords = []
keyword = debian_data.get('section')
if keyword:
@@ -712,6 +725,8 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
qualifiers=qualifiers,
description=description,
homepage_url=homepage_url,
vcs_url=vcs_url,
code_view_url=code_view_url,
size=size,
source_packages=source_packages,
keywords=keywords,
11 changes: 9 additions & 2 deletions tests/packagedcode/data/debian/control.expected.json
Original file line number Diff line number Diff line change
@@ -140,6 +140,13 @@
"name": "Bdale Garbee",
"email": "bdale@gag.com",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Carl Worth",
"email": "cworth@cworth.org",
"url": null
}
],
"keywords": [
@@ -153,8 +160,8 @@
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"vcs_url": null,
"code_view_url": "https://salsa.debian.org/debian/gzip",
"vcs_url": "https://salsa.debian.org/debian/gzip.git",
"copyright": null,
"holder": null,
"declared_license_expression": null,
Original file line number Diff line number Diff line change
@@ -19,6 +19,13 @@
"name": "Debian Adduser Developers",
"email": "adduser@packages.debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Marc Haber",
"email": "mh+debian-packages@zugschlus.de",
"url": null
}
],
"keywords": [],
@@ -30,8 +37,8 @@
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"vcs_url": null,
"code_view_url": "https://salsa.debian.org/debian/adduser",
"vcs_url": "https://salsa.debian.org/debian/adduser.git",
"copyright": null,
"holder": null,
"declared_license_expression": null,
@@ -84,6 +91,13 @@
"name": "Debian Adduser Developers",
"email": "adduser@packages.debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Marc Haber",
"email": "mh+debian-packages@zugschlus.de",
"url": null
}
],
"keywords": [],
@@ -95,8 +109,8 @@
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"vcs_url": null,
"code_view_url": "https://salsa.debian.org/debian/adduser",
"vcs_url": "https://salsa.debian.org/debian/adduser.git",
"copyright": null,
"holder": null,
"declared_license_expression": null,
49 changes: 49 additions & 0 deletions tests/packagedcode/data/debian/dsc_files/zsh_5.7.1-1+deb10u1.dsc
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA512

Format: 3.0 (quilt)
Source: zsh
Binary: zsh-common, zsh, zsh-doc, zsh-static, zsh-dev
Architecture: any all
Version: 5.7.1-1+deb10u1
Maintainer: Debian Zsh Maintainers <pkg-zsh-devel@lists.alioth.debian.org>
Uploaders: Michael Prokop <mika@debian.org>, Axel Beckert <abe@debian.org>, Frank Terbeck <ft@bewatermyfriend.org>, Richard Hartmann <richih@debian.org>
Homepage: https://www.zsh.org/
Standards-Version: 4.3.0
Vcs-Browser: https://salsa.debian.org/debian/zsh
Vcs-Git: https://salsa.debian.org/debian/zsh.git -b debian
Testsuite: autopkgtest
Testsuite-Triggers: adequate
Build-Depends: bsdmainutils, cm-super-minimal, debhelper-compat (= 12), dpkg-dev (>= 1.16.2~), ghostscript, groff, groff-base, libcap-dev [linux-any], libncursesw5-dev, libpcre3-dev, texinfo (>= 5~), texlive-fonts-recommended, texlive-latex-base, texlive-latex-recommended, yodl (>= 3.08.01) | yodl (<< 3.08.00)
Package-List:
zsh deb shells optional arch=any
zsh-common deb shells optional arch=all
zsh-dev deb libdevel optional arch=any
zsh-doc deb doc optional arch=all
zsh-static deb shells optional arch=any
Checksums-Sha1:
b2fd47fdb878aa681edc974864e37baae9b0d6b7 2776796 zsh_5.7.1.orig.tar.xz
14a8d38d3fae5b8eec0b124be5c943a60c4e8fec 87028 zsh_5.7.1-1+deb10u1.debian.tar.xz
Checksums-Sha256:
439aafb4341522c307a67a2680e95fadb1b35a5c7f332089b9cc5154496570ca 2776796 zsh_5.7.1.orig.tar.xz
ecbe22ed6a2b8dcaf10eff02b6b66583ce8d108a936624fc424c72188dea1ddd 87028 zsh_5.7.1-1+deb10u1.debian.tar.xz
Files:
acc1a32ef5b3120ead5c6f0d011ceb76 2776796 zsh_5.7.1.orig.tar.xz
d0f5fe26d9548331d9757e26b95c7aaf 87028 zsh_5.7.1-1+deb10u1.debian.tar.xz

-----BEGIN PGP SIGNATURE-----

iQIzBAEBCgAdFiEERoyJeTtCmBnp12Ema+Zjx1o1yXUFAmIMYK4ACgkQa+Zjx1o1
yXWTgw//SzaiMwbl8gSiQ/mB7+KPCeqL0o44q8oVAbj0WWhlh6yVUdjIcoHk6Zr3
Om4b3C5pU1l64jQ/JH5GP4i/7NnHAttobDL5zh9yvXAjUkUSfhBB6mJPeKT2QlHY
3SsvWiwtZOU8hPdwW48ATFiSSiYka+hKEM5nr5f/kI++yg2Rg7d1b5svaD3BMytI
Hl25comMwLgPeL4aIB7j8UL0sQT2zb/O5CK4aZr3CPUxy9OzBAwpQuukNNsJJ3Zg
g+a6Y6ADLAdhkav1JeZIF6LQ4Q9sk4Cqkp+CFenvRtDv2BDdosOXwOnBUBlm1A2j
yPuOcPdgSDqctMohSWVehDqGZK8dPjIOVq/sX3yKzwtNWdil9aErjGWxQVRSMyfg
fqEaCHYdpRpfrE8QWk+7TqovRqv3PaAmY6wahEAd5buniRScDF2yobT95qDOMnfj
fR1YDDH0cRkePHt+VHQUzM1mO4c/5Obhxfd8uExR+8KpO4SnsT8z7wa1J1O+jWGg
88zuhJ2HUCDGHdVlpgksagnNkmbT1Oohml6YfTAtDO4eXXJeTPmSpOYNzRCMsN0D
V1tY+awJuR0VSNjo+3X7DKyC/pKJiCzk+MBVogmFvzfBwJhYiLbVnVElNic/2J2J
Rz6VMPKIYqKUTyIgsQML0IkK8D9Lpp1LHqLNbUVt2aWcYlqgPBU=
=cPZC
-----END PGP SIGNATURE-----
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
{
"packages": [
{
"type": "deb",
"namespace": "debian",
"name": "zsh",
"version": "5.7.1-1+deb10u1",
"qualifiers": {
"arch": "any all"
},
"subpath": null,
"primary_language": null,
"description": null,
"release_date": null,
"parties": [
{
"type": null,
"role": "maintainer",
"name": "Debian Zsh Maintainers",
"email": "pkg-zsh-devel@lists.alioth.debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Michael Prokop",
"email": "mika@debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Axel Beckert",
"email": "abe@debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Frank Terbeck",
"email": "ft@bewatermyfriend.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Richard Hartmann",
"email": "richih@debian.org",
"url": null
}
],
"keywords": [],
"homepage_url": "https://www.zsh.org/",
"download_url": null,
"size": null,
"sha1": null,
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": "https://salsa.debian.org/debian/zsh",
"vcs_url": "https://salsa.debian.org/debian/zsh.git",
"copyright": null,
"holder": null,
"declared_license_expression": null,
"declared_license_expression_spdx": null,
"license_detections": [],
"other_license_expression": null,
"other_license_expression_spdx": null,
"other_license_detections": [],
"extracted_license_statement": null,
"notice_text": null,
"source_packages": [
"pkg:deb/debian/zsh"
],
"extra_data": {},
"repository_homepage_url": null,
"repository_download_url": null,
"api_data_url": null,
"package_uid": "pkg:deb/debian/zsh@5.7.1-1%2Bdeb10u1?arch=any%20all&uuid=fixed-uid-done-for-testing-5642512d1758",
"datafile_paths": [
"zsh_5.7.1-1+deb10u1.dsc"
],
"datasource_ids": [
"debian_source_control_dsc"
],
"purl": "pkg:deb/debian/zsh@5.7.1-1%2Bdeb10u1?arch=any%20all"
}
],
"dependencies": [],
"files": [
{
"path": "zsh_5.7.1-1+deb10u1.dsc",
"type": "file",
"package_data": [
{
"type": "deb",
"namespace": "debian",
"name": "zsh",
"version": "5.7.1-1+deb10u1",
"qualifiers": {
"arch": "any all"
},
"subpath": null,
"primary_language": null,
"description": null,
"release_date": null,
"parties": [
{
"type": null,
"role": "maintainer",
"name": "Debian Zsh Maintainers",
"email": "pkg-zsh-devel@lists.alioth.debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Michael Prokop",
"email": "mika@debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Axel Beckert",
"email": "abe@debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Frank Terbeck",
"email": "ft@bewatermyfriend.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Richard Hartmann",
"email": "richih@debian.org",
"url": null
}
],
"keywords": [],
"homepage_url": "https://www.zsh.org/",
"download_url": null,
"size": null,
"sha1": null,
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": "https://salsa.debian.org/debian/zsh",
"vcs_url": "https://salsa.debian.org/debian/zsh.git",
"copyright": null,
"holder": null,
"declared_license_expression": null,
"declared_license_expression_spdx": null,
"license_detections": [],
"other_license_expression": null,
"other_license_expression_spdx": null,
"other_license_detections": [],
"extracted_license_statement": null,
"notice_text": null,
"source_packages": [
"pkg:deb/debian/zsh"
],
"file_references": [],
"extra_data": {},
"dependencies": [],
"repository_homepage_url": null,
"repository_download_url": null,
"api_data_url": null,
"datasource_id": "debian_source_control_dsc",
"purl": "pkg:deb/debian/zsh@5.7.1-1%2Bdeb10u1?arch=any%20all"
}
],
"for_packages": [
"pkg:deb/debian/zsh@5.7.1-1%2Bdeb10u1?arch=any%20all&uuid=fixed-uid-done-for-testing-5642512d1758"
],
"scan_errors": []
}
]
}
Loading

0 comments on commit 43dc430

Please sign in to comment.