Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve debian package detection #3723

Merged
merged 1 commit into from
Apr 12, 2024
Merged

Improve debian package detection #3723

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions src/packagedcode/debian.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder)
class DebianControlFileInExtractedDebHandler(models.DatafileHandler):
datasource_id = 'debian_control_extracted_deb'
default_package_type = 'deb'
path_patterns = ('*/control.tar.gz-extract/control',)
path_patterns = ('*/control.tar.gz-extract/control','*/control.tar.xz-extract/control')
description = 'Debian control file - extracted layout'
documentation_url = 'https://www.debian.org/doc/debian-policy/ch-controlfields.html'

Expand Down Expand Up @@ -590,7 +590,7 @@ def parse_debian_files_list(location, datasource_id, package_type):
else:
name = None
# For DebianMd5sumFilelistInPackageHandler we cannot infer name
if not name == "md5sums":
if not filename == "md5sums":
name = filename

file_references = []
Expand Down Expand Up @@ -663,6 +663,19 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
party = models.Party(role='maintainer', name=maintainer_name, email=maintainer_email)
parties.append(party)

uploaders = debian_data.get('uploaders')
if uploaders:
for uploader in uploaders.split(", "):
uploader_name, uploader_email = parse_debian_maintainers(uploader)
party = models.Party(role='uploader', name=uploader_name, email=uploader_email)
parties.append(party)

vcs_url = debian_data.get('vcs-git')
if vcs_url and ' ' in vcs_url:
vcs_url = vcs_url.split(' ')[0]

code_view_url = debian_data.get('vcs-browser')

keywords = []
keyword = debian_data.get('section')
if keyword:
Expand Down Expand Up @@ -712,6 +725,8 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
qualifiers=qualifiers,
description=description,
homepage_url=homepage_url,
vcs_url=vcs_url,
code_view_url=code_view_url,
size=size,
source_packages=source_packages,
keywords=keywords,
Expand Down
11 changes: 9 additions & 2 deletions tests/packagedcode/data/debian/control.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,13 @@
"name": "Bdale Garbee",
"email": "bdale@gag.com",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Carl Worth",
"email": "cworth@cworth.org",
"url": null
}
],
"keywords": [
Expand All @@ -153,8 +160,8 @@
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"vcs_url": null,
"code_view_url": "https://salsa.debian.org/debian/gzip",
"vcs_url": "https://salsa.debian.org/debian/gzip.git",
"copyright": null,
"holder": null,
"declared_license_expression": null,
Expand Down
22 changes: 18 additions & 4 deletions tests/packagedcode/data/debian/dsc_files/adduser_3.118+deb11u1.dsc.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@
"name": "Debian Adduser Developers",
"email": "adduser@packages.debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Marc Haber",
"email": "mh+debian-packages@zugschlus.de",
"url": null
}
],
"keywords": [],
Expand All @@ -30,8 +37,8 @@
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"vcs_url": null,
"code_view_url": "https://salsa.debian.org/debian/adduser",
"vcs_url": "https://salsa.debian.org/debian/adduser.git",
"copyright": null,
"holder": null,
"declared_license_expression": null,
Expand Down Expand Up @@ -84,6 +91,13 @@
"name": "Debian Adduser Developers",
"email": "adduser@packages.debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Marc Haber",
"email": "mh+debian-packages@zugschlus.de",
"url": null
}
],
"keywords": [],
Expand All @@ -95,8 +109,8 @@
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": null,
"vcs_url": null,
"code_view_url": "https://salsa.debian.org/debian/adduser",
"vcs_url": "https://salsa.debian.org/debian/adduser.git",
"copyright": null,
"holder": null,
"declared_license_expression": null,
Expand Down
49 changes: 49 additions & 0 deletions tests/packagedcode/data/debian/dsc_files/zsh_5.7.1-1+deb10u1.dsc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA512

Format: 3.0 (quilt)
Source: zsh
Binary: zsh-common, zsh, zsh-doc, zsh-static, zsh-dev
Architecture: any all
Version: 5.7.1-1+deb10u1
Maintainer: Debian Zsh Maintainers <pkg-zsh-devel@lists.alioth.debian.org>
Uploaders: Michael Prokop <mika@debian.org>, Axel Beckert <abe@debian.org>, Frank Terbeck <ft@bewatermyfriend.org>, Richard Hartmann <richih@debian.org>
Homepage: https://www.zsh.org/
Standards-Version: 4.3.0
Vcs-Browser: https://salsa.debian.org/debian/zsh
Vcs-Git: https://salsa.debian.org/debian/zsh.git -b debian
Testsuite: autopkgtest
Testsuite-Triggers: adequate
Build-Depends: bsdmainutils, cm-super-minimal, debhelper-compat (= 12), dpkg-dev (>= 1.16.2~), ghostscript, groff, groff-base, libcap-dev [linux-any], libncursesw5-dev, libpcre3-dev, texinfo (>= 5~), texlive-fonts-recommended, texlive-latex-base, texlive-latex-recommended, yodl (>= 3.08.01) | yodl (<< 3.08.00)
Package-List:
zsh deb shells optional arch=any
zsh-common deb shells optional arch=all
zsh-dev deb libdevel optional arch=any
zsh-doc deb doc optional arch=all
zsh-static deb shells optional arch=any
Checksums-Sha1:
b2fd47fdb878aa681edc974864e37baae9b0d6b7 2776796 zsh_5.7.1.orig.tar.xz
14a8d38d3fae5b8eec0b124be5c943a60c4e8fec 87028 zsh_5.7.1-1+deb10u1.debian.tar.xz
Checksums-Sha256:
439aafb4341522c307a67a2680e95fadb1b35a5c7f332089b9cc5154496570ca 2776796 zsh_5.7.1.orig.tar.xz
ecbe22ed6a2b8dcaf10eff02b6b66583ce8d108a936624fc424c72188dea1ddd 87028 zsh_5.7.1-1+deb10u1.debian.tar.xz
Files:
acc1a32ef5b3120ead5c6f0d011ceb76 2776796 zsh_5.7.1.orig.tar.xz
d0f5fe26d9548331d9757e26b95c7aaf 87028 zsh_5.7.1-1+deb10u1.debian.tar.xz

-----BEGIN PGP SIGNATURE-----

iQIzBAEBCgAdFiEERoyJeTtCmBnp12Ema+Zjx1o1yXUFAmIMYK4ACgkQa+Zjx1o1
yXWTgw//SzaiMwbl8gSiQ/mB7+KPCeqL0o44q8oVAbj0WWhlh6yVUdjIcoHk6Zr3
Om4b3C5pU1l64jQ/JH5GP4i/7NnHAttobDL5zh9yvXAjUkUSfhBB6mJPeKT2QlHY
3SsvWiwtZOU8hPdwW48ATFiSSiYka+hKEM5nr5f/kI++yg2Rg7d1b5svaD3BMytI
Hl25comMwLgPeL4aIB7j8UL0sQT2zb/O5CK4aZr3CPUxy9OzBAwpQuukNNsJJ3Zg
g+a6Y6ADLAdhkav1JeZIF6LQ4Q9sk4Cqkp+CFenvRtDv2BDdosOXwOnBUBlm1A2j
yPuOcPdgSDqctMohSWVehDqGZK8dPjIOVq/sX3yKzwtNWdil9aErjGWxQVRSMyfg
fqEaCHYdpRpfrE8QWk+7TqovRqv3PaAmY6wahEAd5buniRScDF2yobT95qDOMnfj
fR1YDDH0cRkePHt+VHQUzM1mO4c/5Obhxfd8uExR+8KpO4SnsT8z7wa1J1O+jWGg
88zuhJ2HUCDGHdVlpgksagnNkmbT1Oohml6YfTAtDO4eXXJeTPmSpOYNzRCMsN0D
V1tY+awJuR0VSNjo+3X7DKyC/pKJiCzk+MBVogmFvzfBwJhYiLbVnVElNic/2J2J
Rz6VMPKIYqKUTyIgsQML0IkK8D9Lpp1LHqLNbUVt2aWcYlqgPBU=
=cPZC
-----END PGP SIGNATURE-----
185 changes: 185 additions & 0 deletions tests/packagedcode/data/debian/dsc_files/zsh_5.7.1-1+deb10u1.dsc.expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
{
"packages": [
{
"type": "deb",
"namespace": "debian",
"name": "zsh",
"version": "5.7.1-1+deb10u1",
"qualifiers": {
"arch": "any all"
},
"subpath": null,
"primary_language": null,
"description": null,
"release_date": null,
"parties": [
{
"type": null,
"role": "maintainer",
"name": "Debian Zsh Maintainers",
"email": "pkg-zsh-devel@lists.alioth.debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Michael Prokop",
"email": "mika@debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Axel Beckert",
"email": "abe@debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Frank Terbeck",
"email": "ft@bewatermyfriend.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Richard Hartmann",
"email": "richih@debian.org",
"url": null
}
],
"keywords": [],
"homepage_url": "https://www.zsh.org/",
"download_url": null,
"size": null,
"sha1": null,
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": "https://salsa.debian.org/debian/zsh",
"vcs_url": "https://salsa.debian.org/debian/zsh.git",
"copyright": null,
"holder": null,
"declared_license_expression": null,
"declared_license_expression_spdx": null,
"license_detections": [],
"other_license_expression": null,
"other_license_expression_spdx": null,
"other_license_detections": [],
"extracted_license_statement": null,
"notice_text": null,
"source_packages": [
"pkg:deb/debian/zsh"
],
"extra_data": {},
"repository_homepage_url": null,
"repository_download_url": null,
"api_data_url": null,
"package_uid": "pkg:deb/debian/zsh@5.7.1-1%2Bdeb10u1?arch=any%20all&uuid=fixed-uid-done-for-testing-5642512d1758",
"datafile_paths": [
"zsh_5.7.1-1+deb10u1.dsc"
],
"datasource_ids": [
"debian_source_control_dsc"
],
"purl": "pkg:deb/debian/zsh@5.7.1-1%2Bdeb10u1?arch=any%20all"
}
],
"dependencies": [],
"files": [
{
"path": "zsh_5.7.1-1+deb10u1.dsc",
"type": "file",
"package_data": [
{
"type": "deb",
"namespace": "debian",
"name": "zsh",
"version": "5.7.1-1+deb10u1",
"qualifiers": {
"arch": "any all"
},
"subpath": null,
"primary_language": null,
"description": null,
"release_date": null,
"parties": [
{
"type": null,
"role": "maintainer",
"name": "Debian Zsh Maintainers",
"email": "pkg-zsh-devel@lists.alioth.debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Michael Prokop",
"email": "mika@debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Axel Beckert",
"email": "abe@debian.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Frank Terbeck",
"email": "ft@bewatermyfriend.org",
"url": null
},
{
"type": null,
"role": "uploader",
"name": "Richard Hartmann",
"email": "richih@debian.org",
"url": null
}
],
"keywords": [],
"homepage_url": "https://www.zsh.org/",
"download_url": null,
"size": null,
"sha1": null,
"md5": null,
"sha256": null,
"sha512": null,
"bug_tracking_url": null,
"code_view_url": "https://salsa.debian.org/debian/zsh",
"vcs_url": "https://salsa.debian.org/debian/zsh.git",
"copyright": null,
"holder": null,
"declared_license_expression": null,
"declared_license_expression_spdx": null,
"license_detections": [],
"other_license_expression": null,
"other_license_expression_spdx": null,
"other_license_detections": [],
"extracted_license_statement": null,
"notice_text": null,
"source_packages": [
"pkg:deb/debian/zsh"
],
"file_references": [],
"extra_data": {},
"dependencies": [],
"repository_homepage_url": null,
"repository_download_url": null,
"api_data_url": null,
"datasource_id": "debian_source_control_dsc",
"purl": "pkg:deb/debian/zsh@5.7.1-1%2Bdeb10u1?arch=any%20all"
}
],
"for_packages": [
"pkg:deb/debian/zsh@5.7.1-1%2Bdeb10u1?arch=any%20all&uuid=fixed-uid-done-for-testing-5642512d1758"
],
"scan_errors": []
}
]
}
Loading
Loading