Python remote packages can be specified with requirements.txt files

fixes: #240

CHANGES/240.feature

@@ -0,0 +1 @@
+Python remote fields --includes/--excludes can now be specified with requirements.txt files

pulpcore/cli/python/remote.py

@@ -1,4 +1,6 @@
 import gettext
+import json
+from typing import Any, List, Optional
 
 import click
 
@@ -23,6 +25,27 @@
 _ = gettext.gettext
 
 
+def _package_list_callback(ctx: click.Context, param: click.Parameter, value: Optional[str]) -> Any:
+    """Parses the requirements file or JSON list for packages."""
+    if not value:
+        return value
+
+    if value.startswith("@"):
+        json_string = f'["{value[1:]}"]'
+    else:
+        json_string = value
+
+    try:
+        json_object = json.loads(json_string)
+    except json.decoder.JSONDecodeError:
+        raise click.ClickException(_("Failed to decode JSON: {}").format(json_string))
+    else:
+        package_list = list()
+        for packages in json_object:
+            package_list += parse_requirements_string(packages)
+        return package_list
+
+
 @click.group()
 @click.option(
     "-t",
@@ -45,8 +68,8 @@ def remote(ctx: click.Context, pulp_ctx: PulpContext, remote_type: str) -> None:
     click.option(
         "--policy", type=click.Choice(["immediate", "on_demand", "streamed"], case_sensitive=False)
     ),
-    click.option("--includes", callback=load_json_callback, help=_("Package allowlist")),
-    click.option("--excludes", callback=load_json_callback, help=_("Package blocklist")),
+    click.option("--includes", callback=_package_list_callback, help=_("Package allowlist")),
+    click.option("--excludes", callback=_package_list_callback, help=_("Package blocklist")),
     click.option("--prereleases", type=click.BOOL, default=True),
     pulp_option(
         "--keep-latest-packages", type=int, needs_plugins=[PluginRequirement("python", "3.2.0")]
@@ -71,4 +94,29 @@ def remote(ctx: click.Context, pulp_ctx: PulpContext, remote_type: str) -> None:
 )
 remote.add_command(label_command())
 
+
 # TODO Add support for 'from_bandersnatch' remote create endpoint
+def parse_requirements_string(requirements_string: str) -> List[str]:
+    """Parses the requirements string to find the packages listed."""
+    requirements_string = requirements_string.strip()
+    if not requirements_string or requirements_string.startswith("#"):
+        return []
+    requirements_string, *_ = requirements_string.split("#", maxsplit=1)
+    requirements_string = requirements_string.strip()
+    if requirements_string.endswith(".txt"):
+        *_, requirements_file = requirements_string.split()
+        return parse_requirements_file(requirements_file)
+    else:
+        return [requirements_string]
+
+
+def parse_requirements_file(requirements_file: str) -> List[str]:
+    """Parses the requirements.txt file."""
+    requirements = list()
+    try:
+        with click.open_file(requirements_file) as fp:
+            for line in fp.readlines():
+                requirements += parse_requirements_string(line)
+    except OSError:
+        raise click.ClickException(f"Failed to load content from {requirements_file}")
+    return requirements

tests/scripts/pulp_python/test_remote.sh

@@ -8,13 +8,32 @@ pulp debug has-plugin --name "python" --min-version "3.1.0.dev" || exit 3
 cleanup() {
   pulp python remote destroy --name "cli_test_python_remote" || true
   pulp python remote destroy --name "cli_test_complex_remote" || true
+  rm "python_test_requirements.txt" "python_test_requirements2.txt"
 }
 trap cleanup EXIT
 
 expect_succ pulp python remote list
 
 expect_succ pulp python remote create --name "cli_test_python_remote" --url "$PYTHON_REMOTE_URL"
+
+# Update includes using requirements file
+cat <<EOT >> python_test_requirements.txt
+# Hello there!
+Django>=4.0  # Comments don't affect it
+shelf-reader
+-r python_test_requirements2.txt  # Can recursively read requirements files
+EOT
+
+cat <<EOT >> python_test_requirements2.txt
+# Hi there!
+pulp_python
+EOT
+
+expect_succ pulp python remote update --name "cli_test_python_remote" --includes "@python_test_requirements.txt"
 expect_succ pulp python remote show --name "cli_test_python_remote"
+test "$(echo "$OUTPUT" | jq -r .includes[0])" = "Django>=4.0"
+test "$(echo "$OUTPUT" | jq -r .includes[1])" = "shelf-reader"
+test "$(echo "$OUTPUT" | jq -r .includes[2])" = "pulp_python"
 expect_succ pulp python remote list
 
 if [ "$(pulp debug has-plugin --name "python" --min-version "3.2.0.dev")" = "true" ]