diff --git a/.github/workflows/test-kubeflow.yaml b/.github/workflows/test-kubeflow.yaml index 13e0cea7c7..4d5a6f19c7 100644 --- a/.github/workflows/test-kubeflow.yaml +++ b/.github/workflows/test-kubeflow.yaml @@ -59,12 +59,7 @@ jobs: sudo snap install juju-helpers --classic - name: Enable kubeflow - run: | - set -eux - export KUBEFLOW_BUNDLE=edge - export KUBEFLOW_DEBUG=true - export KUBEFLOW_IGNORE_MIN_MEM=true - sg microk8s -c 'microk8s enable kubeflow' + run: sg microk8s -c 'microk8s enable kubeflow --debug --bundle=edge --ignore-min-mem --password=hunter2' - name: Test kubeflow run: | @@ -75,6 +70,8 @@ jobs: sudo pip3 install pytest sh kfp requests pyyaml git clone https://github.com/juju-solutions/bundle-kubeflow.git cd bundle-kubeflow + sudo microk8s status --wait-ready + sudo microk8s kubectl -n kube-system rollout status ds/calico-node trap 'sudo pkill -f svc/pipelines-api' SIGINT SIGTERM EXIT sudo microk8s kubectl -n kubeflow port-forward svc/pipelines-api 8888:8888 & (i=30; while ! curl localhost:8888 ; do ((--i)) || exit; sleep 1; done) @@ -170,6 +167,7 @@ jobs: export KUBEFLOW_BUNDLE=${{ matrix.bundle }} export KUBEFLOW_DEBUG=true export KUBEFLOW_IGNORE_MIN_MEM=true + export KUBEFLOW_AUTH_PASSWORD=hunter2 microk8s enable kubeflow EOF @@ -183,6 +181,8 @@ jobs: sudo pip3 install pytest sh kfp requests pyyaml git clone https://github.com/juju-solutions/bundle-kubeflow.git cd bundle-kubeflow + sudo microk8s status --wait-ready + sudo microk8s kubectl -n kube-system rollout status ds/calico-node trap 'sudo pkill -f svc/pipelines-api' SIGINT SIGTERM EXIT microk8s kubectl -n kubeflow port-forward svc/pipelines-api 8888:8888 & (i=30; while ! curl localhost:8888 ; do ((--i)) || exit; sleep 1; done) diff --git a/microk8s-resources/actions/disable.kubeflow.sh b/microk8s-resources/actions/disable.kubeflow.sh index b12d1e5494..fb2594b83f 100755 --- a/microk8s-resources/actions/disable.kubeflow.sh +++ b/microk8s-resources/actions/disable.kubeflow.sh @@ -1,14 +1,46 @@ -#!/usr/bin/env bash +#!/usr/bin/env python3 -set -eu +import os +import subprocess -source $SNAP/actions/common/utils.sh +import click -function disable_kubeflow() { - echo "Disabling Kubeflow..." - "$SNAP/microk8s-juju.wrapper" unregister -y uk8s || true - "$SNAP/microk8s-kubectl.wrapper" delete ns controller-uk8s kubeflow || true -} -disable_kubeflow +@click.command() +def kubeflow(): + click.echo("Disabling Kubeflow...") + env = os.environ.copy() + env["PATH"] += ":%s" % os.environ["SNAP"] + + click.echo("Unregistering model...") + try: + subprocess.run( + ['microk8s-juju.wrapper', 'unregister', '-y', 'uk8s'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + except subprocess.CalledProcessError: + pass + click.echo("Unregistering complete.") + + click.echo("Destroying namespace...") + try: + subprocess.check_call( + ['microk8s-kubectl.wrapper', 'delete', 'ns', 'controller-uk8s', 'kubeflow'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + except subprocess.CalledProcessError: + pass + click.echo("Destruction complete.") + + click.echo("Kubeflow is now disabled.") + + +if __name__ == "__main__": + kubeflow(prog_name='microk8s disable kubeflow') diff --git a/microk8s-resources/actions/enable.kubeflow.sh b/microk8s-resources/actions/enable.kubeflow.sh index b219754882..f496e627fb 100755 --- a/microk8s-resources/actions/enable.kubeflow.sh +++ b/microk8s-resources/actions/enable.kubeflow.sh @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import csv import json import os import random @@ -16,6 +15,7 @@ from pathlib import Path from urllib.error import URLError from urllib.parse import ParseResult, urlparse from urllib.request import urlopen +import click MIN_MEM_GB = 14 @@ -210,29 +210,42 @@ def get_hostname(): return "localhost" -def main(): - args = { - 'bundle': os.environ.get("KUBEFLOW_BUNDLE") or "cs:kubeflow-230", - 'channel': os.environ.get("KUBEFLOW_CHANNEL") or "stable", - 'debug': os.environ.get("KUBEFLOW_DEBUG") or "false", - 'hostname': os.environ.get("KUBEFLOW_HOSTNAME") or None, - 'ignore_min_mem': os.environ.get("KUBEFLOW_IGNORE_MIN_MEM") or "false", - 'no_proxy': os.environ.get("KUBEFLOW_NO_PROXY") or None, - 'password': os.environ.get("KUBEFLOW_AUTH_PASSWORD") or get_random_pass(), - } - for pair in list(csv.reader(sys.argv[1:]))[0]: - key, val = pair.split('=', maxsplit=1) - if key not in args: - print("Invalid argument `%s`." % key) - print("Valid arguments options are:\n - " + "\n - ".join(args.keys())) - sys.exit(1) - args[key] = val - - # Coerce the boolean args to actual bools - for arg in ['debug', 'ignore_min_mem']: - if not isinstance(args[arg], bool): - args[arg] = strtobool(args[arg]) - +@click.command() +@click.option( + '--bundle', + default='cs:kubeflow-230', + help='The Kubeflow bundle to deploy. Can be one of full, lite, edge, or a charm store URL.', +) +@click.option( + '--channel', + default='stable', + type=click.Choice(['stable', 'candidate', 'beta', 'edge']), + help='Which channel to deploy the bundle from. In most cases, this should be `stable`.', +) +@click.option( + '--debug/--no-debug', + default=False, + help='If true, shows more verbose output when enabling Kubeflow.', +) +@click.option( + '--hostname', + help='If set, this hostname is used instead of a hostname generated by MetalLB.', +) +@click.option( + '--ignore-min-mem/--no-ignore-min-mem', + default=False, + help='If set, overrides the minimum memory check.', +) +@click.option( + '--no-proxy', + help='Allows setting the juju-no-proxy configuration option.', +) +@click.password_option( + envvar='KUBEFLOW_AUTH_PASSWORD', + default=get_random_pass, + help='The Kubeflow dashboard password.', +) +def kubeflow(bundle, channel, debug, hostname, ignore_min_mem, no_proxy, password): if os.geteuid() == 0: print("This command can't be run as root.") print("Try `microk8s enable kubeflow` instead.") @@ -261,12 +274,17 @@ def main(): print("Couldn't determine total memory.") print("Kubeflow recommends at least %s GB of memory." % MIN_MEM_GB) - if total_mem < MIN_MEM_GB * 1024 * 1024 and not args['ignore_min_mem']: + if total_mem < MIN_MEM_GB * 1024 * 1024 and not ignore_min_mem: print("Kubeflow recommends at least %s GB of memory." % MIN_MEM_GB) - print( - "Run `KUBEFLOW_IGNORE_MIN_MEM=true microk8s.enable kubeflow`" - " if you'd like to proceed anyways." - ) + print("Use `--ignore-min-mem` if you'd like to proceed anyways.") + sys.exit(1) + + try: + juju("show-controller", "uk8s", die=False, stdout=False) + except subprocess.CalledProcessError: + pass + else: + print("Kubeflow has already been enabled.") sys.exit(1) # Allow specifying the bundle as one of the main types of kubeflow bundles @@ -274,23 +292,23 @@ def main(): # shoudn't have to specify a version for those bundles. However, allow the # user to specify a full charm store URL if they'd like, such as # `cs:kubeflow-lite-123`. - if args['bundle'] == 'full': + if bundle == 'full': bundle = 'cs:kubeflow-230' - elif args['bundle'] == 'lite': + elif bundle == 'lite': bundle = 'cs:kubeflow-lite-17' - elif args['bundle'] == 'edge': + elif bundle == 'edge': bundle = 'cs:kubeflow-edge-16' else: - bundle = args['bundle'] + bundle = bundle - run("microk8s-status.wrapper", "--wait-ready", debug=args['debug']) + run("microk8s-status.wrapper", "--wait-ready", debug=debug) run( 'microk8s-kubectl.wrapper', '-nkube-system', 'rollout', 'status', 'deployment.apps/calico-kube-controllers', - debug=args['debug'], + debug=debug, ) for service in [ @@ -301,16 +319,16 @@ def main(): "metallb:10.64.140.43-10.64.140.49", ]: print("Enabling %s..." % service) - run("microk8s-enable.wrapper", service, debug=args['debug']) + run("microk8s-enable.wrapper", service, debug=debug) - run("microk8s-status.wrapper", "--wait-ready", debug=args['debug']) + run("microk8s-status.wrapper", "--wait-ready", debug=debug) run( 'microk8s-kubectl.wrapper', '-nkube-system', 'rollout', 'status', 'ds/calico-node', - debug=args['debug'], + debug=debug, ) print("Waiting for DNS and storage plugins to finish setting up") @@ -322,31 +340,25 @@ def main(): "deployment/coredns", "deployment/hostpath-provisioner", "--timeout=10m", - debug=args['debug'], + debug=debug, ) - check_connectivity() + print("DNS and storage setup complete. Checking connectivity...") - try: - juju("show-controller", "uk8s", die=False, stdout=False) - except subprocess.CalledProcessError: - pass - else: - print("Kubeflow has already been enabled.") - sys.exit(1) + check_connectivity() print("Bootstrapping...") - if args['no_proxy'] is not None: - juju("bootstrap", "microk8s", "uk8s", "--config=juju-no-proxy=%s" % args['no_proxy']) + if no_proxy is not None: + juju("bootstrap", "microk8s", "uk8s", "--config=juju-no-proxy=%s" % no_proxy) juju("add-model", "kubeflow", "microk8s") - juju("model-config", "-m", "kubeflow", "juju-no-proxy=%s" % args['no_proxy']) + juju("model-config", "-m", "kubeflow", "juju-no-proxy=%s" % no_proxy) else: juju("bootstrap", "microk8s", "uk8s") juju("add-model", "kubeflow", "microk8s") print("Bootstrap complete.") print("Successfully bootstrapped, deploying...") - juju("deploy", bundle, "--channel", args['channel']) + juju("deploy", bundle, "--channel", channel) print("Kubeflow deployed.") print("Waiting for operator pods to become ready.") @@ -391,7 +403,7 @@ def main(): f.flush() run('microk8s-kubectl.wrapper', 'apply', '-f', f.name) - hostname = parse_hostname(args['hostname'] or get_hostname()) + hostname = parse_hostname(hostname or get_hostname()) if kubectl_exists('service/dex-auth'): juju("config", "dex-auth", "public-url=%s" % hostname.geturl()) @@ -407,7 +419,7 @@ def main(): "pod", "--timeout=30s", "--all", - debug=args['debug'], + debug=debug, times=100, ) @@ -428,7 +440,7 @@ def main(): microk8s juju config dex-auth static-password """ - % (hostname.geturl(), args['password']) + % (hostname.geturl(), password) ) ) else: @@ -447,4 +459,4 @@ def main(): if __name__ == "__main__": - main() + kubeflow(prog_name='microk8s enable kubeflow', auto_envvar_prefix='KUBEFLOW') diff --git a/microk8s-resources/wrappers/microk8s-disable.wrapper b/microk8s-resources/wrappers/microk8s-disable.wrapper index 99bc481eb3..eaaef96235 100755 --- a/microk8s-resources/wrappers/microk8s-disable.wrapper +++ b/microk8s-resources/wrappers/microk8s-disable.wrapper @@ -1,78 +1,14 @@ -#!/bin/bash +#!/usr/bin/env bash +set -eu export PATH="$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH" ARCH="$($SNAP/bin/uname -m)" -export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu" -export LD_LIBRARY_PATH=$SNAP_LIBRARY_PATH:$LD_LIBRARY_PATH +export IN_SNAP_LD_LIBRARY_PATH="$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu" +export PYTHONNOUSERSITE=false source $SNAP/actions/common/utils.sh -if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] -then - echo "This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s disable on the master." - exit 0 -fi - -if echo "$*" | grep -q -- '--help'; then - prog=$(basename -s.wrapper "$0" | tr - .) - echo "Usage: $prog ADDON..." - echo "Disable one or more ADDON included with microk8s" - echo "Example: $prog dns storage" - echo - echo "Available addons:" - echo - actions="$(find "${SNAP}/actions" -name '*.yaml' -or -name 'disable.*.sh')" - actions="$(echo "$actions" | sed -e 's/.*[/.]\([^.]*\)\..*/\1/' | sort | uniq)" - for action in $actions; do - echo " $action" - done - exit 0 -fi - -exit_if_stopped exit_if_no_permissions -result=1 -for addon in "$@"; do - # Making sure the cluster is up and running befor each addon - ${SNAP}/microk8s-status.wrapper --wait-ready --timeout 30 >/dev/null - action="$(addon_name $addon)" - arguments="$(addon_arguments $addon)" - # Check if the addon is already disabled, then skip. - if ${SNAP}/microk8s-status.wrapper --addon $action | grep "disabled" >/dev/null && - ! [ "$action" = "kubeflow" ] - then - echo "Addon $action is already disabled." - result=0 - continue - fi - - # If there is a script to execute for the action $1 run the script and ignore any yamls - if [ -f "${SNAP}/actions/disable.$action.sh" ]; then - "${SNAP}/actions/disable.$action.sh" "${arguments}" - disable_result="$?" - if [ "$disable_result" = "0" ] - then - result=0 - else - echo "Failed to disable $action" - exit 1 - fi - elif [ -f "${SNAP}/actions/$action.yaml" ]; then - echo "Disabling $action" - use_manifest $action delete - if [ "$use_manifest_result" = "0" ] - then - echo "$action disabled" - result=0 - else - echo "Failed to disable $action" - exit 1 - fi - else - echo "Nothing to do for $action" - exit 2 - fi -done -exit $result +LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/disable.py $@ diff --git a/microk8s-resources/wrappers/microk8s-enable.wrapper b/microk8s-resources/wrappers/microk8s-enable.wrapper index c42bfc1604..318cb2eee7 100755 --- a/microk8s-resources/wrappers/microk8s-enable.wrapper +++ b/microk8s-resources/wrappers/microk8s-enable.wrapper @@ -1,64 +1,14 @@ -#!/bin/bash +#!/usr/bin/env bash +set -eu export PATH="$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH" ARCH="$($SNAP/bin/uname -m)" -export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu" -export LD_LIBRARY_PATH=$SNAP_LIBRARY_PATH:$LD_LIBRARY_PATH +export IN_SNAP_LD_LIBRARY_PATH="$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu" +export PYTHONNOUSERSITE=false source $SNAP/actions/common/utils.sh -if [ -e ${SNAP_DATA}/var/lock/clustered.lock ] -then - echo "This MicroK8s deployment is acting as a node in a cluster. Please use the microk8s enable on the master." - exit 0 -fi - -if echo "$*" | grep -q -- '--help'; then - LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/enable.py $@ - exit -fi - -exit_if_stopped exit_if_no_permissions -result=1 -for addon in "$@"; do - # Making sure the cluster is up and running before each addon - action="$(addon_name $addon)" - arguments="$(addon_arguments $addon)" - # Check if the addon is already enabled, then skip. - if ${SNAP}/microk8s-status.wrapper --wait-ready --timeout 30 --addon $action | grep "enabled" >/dev/null; then - echo "Addon $action is already enabled." - result=0 - continue - fi - - # If there is a script to execute for the $action run the script and ignore any yamls - if [ -f "${SNAP}/actions/enable.$action.sh" ]; then - "${SNAP}/actions/enable.$action.sh" "${arguments}" - enable_result="$?" - if [ "$enable_result" = "0" ] - then - result=0 - else - echo "Failed to enable $action" - exit 1 - fi - elif [ -f "${SNAP}/actions/$action.yaml" ]; then - echo "Enabling $action" - use_manifest $action apply - if [ "$use_manifest_result" = "0" ] - then - echo "$action enabled" - result=0 - else - echo "Failed to enable $action" - exit 1 - fi - else - echo "Nothing to do for $action" - exit 2 - fi -done -exit $result +LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/enable.py $@ diff --git a/scripts/wrappers/common/utils.py b/scripts/wrappers/common/utils.py index 3a9e9339a4..33fb1dbf32 100644 --- a/scripts/wrappers/common/utils.py +++ b/scripts/wrappers/common/utils.py @@ -1,11 +1,14 @@ -import yaml +import getpass import json import os +import platform import subprocess import sys import time -import platform -import getpass +from pathlib import Path + +import click +import yaml kubeconfig = "--kubeconfig=" + os.path.expandvars("${SNAP_DATA}/credentials/client.config") @@ -17,6 +20,13 @@ def get_current_arch(): return arch_mapping[platform.machine()] +def snap_data() -> Path: + try: + return Path(os.environ['SNAP_DATA']) + except KeyError: + return Path('/var/snap/microk8s/current') + + def run(*args, die=True): # Add wrappers to $PATH env = os.environ.copy() @@ -102,13 +112,10 @@ def get_dqlite_info(): def is_cluster_locked(): - clusterLockFile = os.path.expandvars("${SNAP_DATA}/var/lock/clustered.lock") - if os.path.isfile(clusterLockFile): - print( - "This MicroK8s deployment is acting as a node in a cluster. " - "Please use the microk8s status on the master." - ) - exit(0) + if (snap_data() / 'var/lock/clustered.lock').exists(): + click.echo('This MicroK8s deployment is acting as a node in a cluster.') + click.echo('Please use the master node.') + sys.exit(1) def wait_for_ready(wait_ready, timeout): @@ -151,6 +158,12 @@ def exit_if_no_permission(): exit(1) +def ensure_started(): + if (snap_data() / 'var/lock/stopped.lock').exists(): + click.echo('microk8s is not running, try microk8s start', err=True) + sys.exit(1) + + def kubectl_get(cmd, namespace="--all-namespaces"): if namespace == "--all-namespaces": return run("kubectl", kubeconfig, "get", cmd, "--all-namespaces", die=False) @@ -211,3 +224,53 @@ def set_service_expected_to_start(service, start=True): else: fd = os.open(lock, os.O_CREAT, mode=0o700) os.close(fd) + + +def xable(action: str, addons: list, xabled_addons: list): + """Enables or disables the given addons. + + Collated into a single function since the logic is identical other than + the script names. + """ + actions = Path(__file__).absolute().parent / "../../../actions" + existing_addons = {sh.with_suffix('').name[7:] for sh in actions.glob('enable.*.sh')} + + # Backwards compatibility with enabling multiple addons at once, e.g. + # `microk8s.enable foo bar:"baz"` + if all(a.split(':')[0] in existing_addons for a in addons) and len(addons) > 1: + for addon in addons: + if addon in xabled_addons and addon != 'kubeflow': + click.echo("Addon %s is already %sd." % (addon, action)) + else: + addon, *args = addon.split(':') + subprocess.run([str(actions / ('%s.%s.sh' % (action, addon)))] + args) + + # The new way of xabling addons, that allows for unix-style argument passing, + # such as `microk8s.enable foo --bar`. + else: + addon, *args = addons[0].split(':') + + if addon in xabled_addons and addon != 'kubeflow': + click.echo("Addon %s is already %sd." % (addon, action)) + sys.exit(0) + + if addon not in existing_addons: + click.echo("Nothing to do for `%s`." % addon, err=True) + sys.exit(1) + + if args and addons[1:]: + click.echo( + "Can't pass string arguments and flag arguments simultaneously!\n" + "{0} an addon with only one argument style at a time:\n" + "\n" + " microk8s {1} foo:'bar'\n" + "or\n" + " microk8s {1} foo --bar\n".format(action.title(), action) + ) + sys.exit(1) + + script = [str(actions / ('%s.%s.sh' % (action, addon)))] + if args: + subprocess.run(script + args) + else: + subprocess.run(script + list(addons[1:])) diff --git a/scripts/wrappers/disable.py b/scripts/wrappers/disable.py new file mode 100755 index 0000000000..db37f3335c --- /dev/null +++ b/scripts/wrappers/disable.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +import click + +from common.utils import ensure_started, exit_if_no_permission, is_cluster_locked, xable +from status import get_status, get_available_addons, get_current_arch + + +@click.command(context_settings={'ignore_unknown_options': True}) +@click.argument('addons', nargs=-1, required=True) +def disable(addons): + """Disables one or more MicroK8s addons. + + For a list of available addons, run `microk8s status`. + + To see help for individual addons, run: + + microk8s disable ADDON -- --help + """ + + is_cluster_locked() + exit_if_no_permission() + ensure_started() + + _, disabled_addons = get_status(get_available_addons(get_current_arch()), True) + disabled_addons = {a['name'] for a in disabled_addons} + + xable('disable', addons, disabled_addons) + + +if __name__ == '__main__': + disable(prog_name='microk8s disable') diff --git a/scripts/wrappers/enable.py b/scripts/wrappers/enable.py index 0bd55e8a0b..b3cfbb5bc0 100755 --- a/scripts/wrappers/enable.py +++ b/scripts/wrappers/enable.py @@ -1,20 +1,32 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 -from common.utils import get_available_addons, get_current_arch +import click +from common.utils import ensure_started, exit_if_no_permission, is_cluster_locked, xable +from status import get_status, get_available_addons, get_current_arch -def print_console(addons): - print("Available Addons:") - for addon in addons: - print("{:>1} {:<20} # {}".format("", addon["name"], addon["description"])) +@click.command(context_settings={'ignore_unknown_options': True}) +@click.argument('addons', nargs=-1, required=True) +def enable(addons): + """Enables a MicroK8s addon. -def show_help(): - print("Usage: microk8s enable ADDON...") - print("Enable one or more ADDON included with microk8s") - print("Example: microk8s enable dns storage") + For a list of available addons, run `microk8s status`. + To see help for individual addons, run: -available_addons = get_available_addons(get_current_arch()) -show_help() -print_console(available_addons) + microk8s enable ADDON -- --help + """ + + is_cluster_locked() + exit_if_no_permission() + ensure_started() + + enabled_addons, _ = get_status(get_available_addons(get_current_arch()), True) + enabled_addons = {a['name'] for a in enabled_addons} + + xable('enable', addons, enabled_addons) + + +if __name__ == '__main__': + enable(prog_name='microk8s enable') diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 02a1644d76..6f5e6476f6 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -426,6 +426,7 @@ parts: - util-linux - zfsutils-linux - iproute2 + - python3-click source: . prime: - -README*