Merged PR 7818847: Adding the support for uploading deb source packages

This PR adds the initial support for deb source packages, only the support for uploading and listing the sources packages is added in this PR, A later PR will add the support for publishing source packages.
This PR is based on the [upstream PR](pulp/pulp_deb#295), Creating a patch based on the changes, and adding the patch to our docker container image.
Here's a few examples of how the ccli can be used to manipulate source packages:

```
- pmc package upload ../../hello_sources/hello_2.10-2ubuntu2.dsc --source-artifact ../../hello_sources/
- pmc package deb_src list
- pmc package show --details content-deb-source_packages-f63f0696-919d-41e2-b023-fd7c9c995b19
```

A later PR will update the documentation, once we have the other part of the changes merged.

Related work items: #13082573

cli/pmc/artifact_uploader.py

@@ -0,0 +1,108 @@
+import re
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional
+
+import httpx
+from click import BadParameter
+from pydantic import AnyHttpUrl, ValidationError, parse_obj_as
+
+from pmc.client import client, client_context, create_client
+from pmc.context import PMCContext
+from pmc.utils import PulpTaskFailure
+
+SHA256_REGEX = r"?P<sha256>[a-fA-F0-9]{64}"
+ARTIFACT_EXISTS_ERROR = rf"Artifact with sha256 checksum of '{SHA256_REGEX}' already exists."
+
+
+class ArtifactUploader:
+ def __init__(
+ self,
+ context: PMCContext,
+ path_or_url: str,
+ ):
+ self.context = context
+
+ try:
+ self.url = parse_obj_as(AnyHttpUrl, path_or_url)
+ except ValidationError:
+ self.url = None
+ self.path = Path(path_or_url)
+ if not self.path.is_file() and not self.path.is_dir():
+ raise BadParameter(
+ f"Invalid url or non-existent file/directory for artifact {self.path}."
+ )
+
+ def _build_data(self) -> Dict[str, Any]:
+ """Build data dict for uploading artifact(s)."""
+ data: Dict[str, Any] = {}
+
+ if self.url:
+ data["url"] = self.url
+
+ return data
+
+ def _find_existing_artifact(self, error_resp: Any) -> Any:
+ """Attempt to find the existing artifact if it exists"""
+ error = error_resp.get("detail")
+
+ if match := re.search(ARTIFACT_EXISTS_ERROR, error["non_field_errors"]):
+ resp = client.get("/artifacts/", params=match.groupdict())
+ results = resp.json()["results"]
+ if len(results) == 1:
+ return results[0]
+
+ raise KeyError()
+
+ def _upload_artifact(self, data: Dict[str, Any], path: Optional[Path] = None) -> Any:
+ if path:
+ files = {"file": open(path, "rb")}
+ else:
+ files = None
+
+ # upload the artifact
+ resp = client.post("/artifacts/", params=data, files=files)
+ resp_json = resp.json()
+
+ if not httpx.codes.is_success(resp.status_code):
+ try:
+ return self._find_existing_artifact(resp)
+ except KeyError:
+ # we're not dealing with a failure due to an existing artifact
+ raise PulpTaskFailure(
+ {
+ "error": {
+ "traceback": resp_json["command_traceback"],
+ "description": resp_json["message"],
+ }
+ }
+ )
+
+ return resp_json
+
+ def _upload_artifacts(
+ self, data: Dict[str, Any], paths: Iterable[Path] = []
+ ) -> List[Dict[str, Any]]:
+ def set_context(context: PMCContext) -> None:
+ client_context.set(create_client(context))
+
+ artifacts = []
+ with ThreadPoolExecutor(
+ max_workers=5, initializer=set_context, initargs=(self.context,)
+ ) as executor:
+ futures = [executor.submit(self._upload_artifact, data, path) for path in paths]
+ for future in as_completed(futures):
+ artifacts.append(future.result())
+
+ return artifacts
+
+ def upload(self) -> List[Dict[str, Any]]:
+ """Perform the upload."""
+ data = self._build_data()
+
+ if self.url:
+ return [self._upload_artifact(data)]
+ elif self.path.is_dir():
+ return self._upload_artifacts(data, self.path.glob("*"))
+ else:
+ return [self._upload_artifact(data, self.path)]

cli/pmc/commands/package.py

@@ -1,20 +1,23 @@
 import hashlib
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 import typer
 
+from pmc.artifact_uploader import ArtifactUploader
 from pmc.client import client, handle_response, output_json
 from pmc.package_uploader import PackageUploader
 from pmc.schemas import LIMIT_OPT, OFFSET_OPT, ORDERING_OPT, PackageType
 from pmc.utils import UserFriendlyTyper, build_params, id_or_name
 
 app = UserFriendlyTyper()
 deb = UserFriendlyTyper()
+deb_src = UserFriendlyTyper()
 rpm = UserFriendlyTyper()
 python = UserFriendlyTyper()
 file = UserFriendlyTyper()
 
 app.add_typer(deb, name="deb", help="Manage deb packages")
+app.add_typer(deb_src, name="debsrc", help="Manage deb source packages")
 app.add_typer(rpm, name="rpm", help="Manage rpm packages")
 app.add_restricted_typer(python, name="python", help="Manage python packages")
 app.add_restricted_typer(file, name="file", help="Manage files")
@@ -81,6 +84,43 @@ def deb_list(
  _list(PackageType.deb, ctx, params)
 
 
+@deb_src.command(name="list")
+def deb_src_list(
+ ctx: typer.Context,
+ repository: Optional[str] = id_or_name("repositories", repo_option),
+ release: Optional[str] = id_or_name(
+ "repositories/%(repository)s/releases",
+ typer.Option(None, help="Name or Id. Only list packages in this apt release."),
+ ),
+ name: Optional[str] = name_option,
+ version: Optional[str] = typer.Option(None),
+ arch: Optional[str] = typer.Option(None),
+ relative_path: Optional[str] = typer.Option(None),
+ limit: int = LIMIT_OPT,
+ offset: int = OFFSET_OPT,
+ ordering: str = ORDERING_OPT,
+) -> None:
+ """
+ List debian source packages matching the specified optional filters.
+
+ \b
+ - pmc package debsrc list
+ - pmc package debsrc list --version=2.10-2ubuntu2
+ """
+ params = build_params(
+ limit,
+ offset,
+ ordering=ordering,
+ repository=repository,
+ release=release,
+ source=name,
+ version=version,
+ architecture=arch,
+ relative_path=relative_path,
+ )
+ _list(PackageType.deb_src, ctx, params)
+
+
 @rpm.command(name="list")
 def rpm_list(
  ctx: typer.Context,
@@ -193,8 +233,16 @@ def upload(
  relative_path: Optional[str] = typer.Option(
  None, help="Manually specify the relative path of the package (files packages only)."
  ),
+ source_artifact: Optional[List[str]] = typer.Option(
+ None, help="URL to an artifact, path to an artifact, or path to a directory of artifacts."
+ ),
 ) -> None:
  """Upload a package."""
+ if source_artifact:
+ for artifact in source_artifact:
+ artifact_uploader = ArtifactUploader(ctx.obj, artifact)
+ artifact_uploader.upload()
+
  uploader = PackageUploader(ctx.obj, package, ignore_signature, file_type, relative_path)
  packages = uploader.upload()
  if ctx.obj.id_only and len(packages) == 1:

cli/pmc/package_uploader.py

@@ -21,6 +21,10 @@
  r"There is already a deb package with relative path '(?P<relative_path>\S+?)' "
  rf"and sha256 '({SHA256_REGEX})'"
  ),
+ PackageType.deb_src: (
+ r"There is already a DSC file with version '(?P<version>\S+?)' "
+ r"and source name '(?P<name>\S+?)'"
+ ),
  PackageType.rpm: (
  r"There is already a package with: arch=(?P<arch>\S+?), checksum_type=sha256, "
  rf"epoch=(?P<epoch>\S+?), name=(?P<name>\S+?), pkgId=({SHA256_REGEX}), "
@@ -47,10 +51,11 @@ def __init__(
  self.url = parse_obj_as(AnyHttpUrl, path_or_url)
  except ValidationError:
  self.url = None
- try:
- self.path = Path(path_or_url)
- except FileNotFoundError:
- raise BadParameter("Invalid path/url for package.")
+ self.path = Path(path_or_url)
+ if not self.path.is_file() and not self.path.is_dir():
+ raise BadParameter(
+ f"Invalid url or non-existent file/directory for package(s) {self.path}."
+ )
 
  if self.path.is_dir():
  if self.relative_path:

cli/pmc/schemas.py

@@ -97,6 +97,7 @@ class PackageType(StringEnum):
  """Type of packages."""
 
  deb = "deb"
+ deb_src = "deb_src"
  rpm = "rpm"
  python = "python"
  file = "file"

cli/tests/assets/hello_2.10-2ubuntu2.dsc

@@ -0,0 +1,43 @@
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA512
+
+Format: 3.0 (quilt)
+Source: hello
+Binary: hello
+Architecture: any
+Version: 2.10-2ubuntu2
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Homepage: http://www.gnu.org/software/hello/
+Standards-Version: 4.3.0
+Testsuite: autopkgtest
+Build-Depends: debhelper-compat (= 9)
+Package-List:
+ hello deb devel optional arch=any
+Checksums-Sha1:
+ f7bebf6f9c62a2295e889f66e05ce9bfaed9ace3 725946 hello_2.10.orig.tar.gz
+ 560b84044b18d69dda0ad03e8e3792d6c571b763 6560 hello_2.10-2ubuntu2.debian.tar.xz
+Checksums-Sha256:
+ 31e066137a962676e89f69d1b65382de95a7ef7d914b8cb956f41ea72e0f516b 725946 hello_2.10.orig.tar.gz
+ 2ae08bfe430c90986e74afbffdc7f98d1920527fd292963a567cb8c29d645b07 6560 hello_2.10-2ubuntu2.debian.tar.xz
+Files:
+ 6cd0ffea3884a4e79330338dcc2987d6 725946 hello_2.10.orig.tar.gz
+ 446c355fd690fbebb49bdd51eeec6f6f 6560 hello_2.10-2ubuntu2.debian.tar.xz
+Original-Maintainer: Santiago Vila <sanvila@debian.org>
+
+-----BEGIN PGP SIGNATURE-----
+
+iQJOBAEBCgA4FiEErEg/aN5yj0PyIC/KVo0w8yGyEz0FAlzmJ0caHHN0ZXZlLmxh
+bmdhc2VrQHVidW50dS5jb20ACgkQVo0w8yGyEz2rkxAAg+yJexJtgzKhXiAqYTHW
+9Us3VMVicCzxDJbeJN11J45Bx1d1L3m+pCHgDpyFnYGJDCuaABRE3HUURIOWbEVX
+Zd5Jns6Aj28wIXPj35ju9swb5K7rghYnHJ++QViwaRDcB+KrEJyRprevDTBA37b7
+KwYfY/hE7ZD8sMYeWHq5Ao814umOpNh7ruw2Jpc4GOcHEpcpt23cFLPKGumWVtWQ
+PNvYg94yId8J8fcF2NTrp681i0QXmtCIT53d7HLpyBlXcA8IlrpSRBTpcwXnIUAv
+MGTwYFQSEWYUw57H4XGzhJU1jGti5YgtIk78tce3r9m14oshVKXVJ0WsT1YZBAiY
+ngFg7yb1jREMf+uNgCZZXFbXLMndyGoZls6UywvyoEQOiCMDCb7eeaXN2qrcRWpu
+tF3p+alC2HAxWsgvv9A2pvhB86YMqDiOGs6+V4KuUleQpWhIlGQEIsasNqtVPKi0
+64r1DwEbPYGJ28KSQd4YQEBYy1tcaiqwymfsdkDKD/a2xXCpMAFmJp35guLuezcs
+RhiKEBkcW4i7nLLfW35N6zsMoL0kLC3GDmglQW4klFnMM+bqs909kXG3J41E9N9J
+d9k+0doufnui6P/SxYlAmMpf7tkaAZAk0nXaC6/Hem9fDSqa6dP+mcZjzUZ+D9Jg
+JMZ93pTsFDrozldams1rDKM=
+=xqJQ
+-----END PGP SIGNATURE-----

cli/tests/commands/test_package.py

@@ -64,6 +64,17 @@ def test_deb_list(deb_package: Any) -> None:
  _assert_package_list_not_empty("deb", {"file": "tests/assets/signed-by-us.deb"})
 
 
+def test_deb_src_list(deb_src_package: Any) -> None:
+ _assert_package_list_not_empty("debsrc")
+ _assert_package_list_not_empty("debsrc", {"name": deb_src_package["name"]})
+ _assert_package_list_empty("debsrc", {"name": "notarealpackagename"})
+ _assert_package_list_not_empty("debsrc", {"version": deb_src_package["version"]})
+ _assert_package_list_empty("debsrc", {"arch": "flux64"})
+ _assert_package_list_not_empty("debsrc", {"relative-path": deb_src_package["relative_path"]})
+ _assert_package_list_empty("debsrc", {"relative-path": "nonexistingrelativepath"})
+ assert len(deb_src_package["artifacts"]) == len(set(deb_src_package["artifacts"].values())) == 3
+
+
 def test_rpm_list(rpm_package: Any) -> None:
  _assert_package_list_not_empty("rpm")
  _assert_package_list_not_empty("rpm", {"name": rpm_package["name"]})
@@ -165,6 +176,13 @@ def test_invalid_deb_package_upload() -> None:
  assert "Unable to find global header" in result.stdout
 
 
+def test_deb_src_package_upload_no_artifacts() -> None:
+ become(Role.Package_Admin)
+ result = invoke_command(package_upload_command("hello_2.10-2ubuntu2.dsc"))
+ assert result.exit_code != 0
+ assert "A source file is listed in the DSC file but is not yet available" in result.stdout
+
+
 def test_ignore_signature(forced_unsigned_package: Any) -> None:
  """This empty test ensures forcing an unsigned package works by exercising the fixture."""
  pass
@@ -208,6 +226,13 @@ def test_duplicate_deb_package(deb_package: Any) -> None:
  assert json.loads(result.stdout)[0]["id"] == deb_package["id"]
 
 
+def test_duplicate_deb_src_package(deb_src_package: Any) -> None:
+ become(Role.Package_Admin)
+ result = invoke_command(package_upload_command("hello_2.10-2ubuntu2.dsc"))
+ assert result.exit_code == 0
+ assert json.loads(result.stdout)[0]["id"] == deb_src_package["id"]
+
+
 def test_duplicate_rpm_package(deb_package: Any) -> None:
  become(Role.Package_Admin)
  result = invoke_command(package_upload_command("signed-by-us.deb"))

cli/tests/conftest.py

@@ -208,25 +208,36 @@ def _my_cmd(action: str) -> List[str]:
 
 
 def package_upload_command(
- package_name: str, unsigned: Optional[bool] = False, file_type: Optional[str] = None
+ package_name: str,
+ unsigned: Optional[bool] = False,
+ file_type: Optional[str] = None,
+ source_artifacts: Optional[List[str]] = None,
 ) -> List[str]:
+ asset_path = Path.cwd() / "tests" / "assets"
  package = Path.cwd() / "tests" / "assets" / package_name
- cmd = ["package", "upload", str(package)]
+ cmd = ["package", "upload", str(asset_path / package)]
 
  if unsigned:
  cmd.append("--ignore-signature")
 
  if file_type:
  cmd += ["--type", file_type]
 
+ if source_artifacts:
+ for artifact in source_artifacts:
+ cmd += ["--source-artifact", str(asset_path / artifact)]
+
  return cmd
 
 
 @contextmanager
 def _package_manager(
- package_name: str, unsigned: Optional[bool] = False, file_type: Optional[str] = None
+ package_name: str,
+ unsigned: Optional[bool] = False,
+ file_type: Optional[str] = None,
+ source_artifacts: Optional[List[str]] = None,
 ) -> Generator[Any, None, None]:
- cmd = package_upload_command(package_name, unsigned, file_type)
+ cmd = package_upload_command(package_name, unsigned, file_type, source_artifacts)
  with _object_manager(cmd, Role.Package_Admin, False) as p:
  yield p
 
@@ -237,6 +248,16 @@ def deb_package(orphan_cleanup: None) -> Generator[Any, None, None]:
  yield p[0]
 
 
+@pytest.fixture()
+def deb_src_package(orphan_cleanup: None) -> Generator[Any, None, None]:
+ with _package_manager(
+ "hello_2.10-2ubuntu2.dsc",
+ True,
+ source_artifacts=["hello_2.10.orig.tar.gz", "hello_2.10-2ubuntu2.debian.tar.xz"],
+ ) as p:
+ yield p[0]
+
+
 @pytest.fixture()
 def zst_deb_package(orphan_cleanup: None) -> Generator[Any, None, None]:
  with _package_manager("signed-by-us-zst-compressed.deb") as p:

pulp/Dockerfile

@@ -122,6 +122,7 @@ RUN patch -d /usr/lib/python3.9/site-packages/ -p1 < /tmp/pulp_deb/0008-Merged-P
 RUN patch -d /usr/lib/python3.9/site-packages/ -p1 < /tmp/pulp_deb/0009-Prevent-sync-from-breaking-if-a-custom-field-is-blan.patch # https://github.com/pulp/pulp_deb/pull/695
 RUN patch -d /usr/lib/python3.9/site-packages/ -p1 < /tmp/pulp_deb/0010-Store-release-signing-overrides-on-repo-to-avoid-syn.patch # https://github.com/pulp/pulp_deb/pull/689
 RUN patch -d /usr/lib/python3.9/site-packages/ -p1 < /tmp/pulp_deb/0011-Add-repo_key_fields-constraint-to-PackageReleaseComp.patch # https://github.com/pulp/pulp_deb/pull/705
+RUN patch -d /usr/lib/python3.9/site-packages/ -p1 < /tmp/pulp_deb/0012-Initial-source-package-and-source-indices-support.patch # https://github.com/pulp/pulp_deb/pull/295
 
 # Apply our pulp_rpm patches until upstream accepts/releases our merge requests
 COPY container-assets/pulp_rpm /tmp/pulp_rpm