Switch microk8s enable wrapper to Python script (#1467)

* Switch microk8s enable/disable wrappers to Python script

For now, they're just clones of the old bash scripts, with the ability
to handle unix-style flag arguments. Using click however, will allow
expanding the enable scripts to be full click subcommands, instead
of just bash scripts.

.github/workflows/test-kubeflow.yaml

@@ -59,12 +59,7 @@ jobs:
         sudo snap install juju-helpers --classic
 
     - name: Enable kubeflow
-      run: |
-        set -eux
-        export KUBEFLOW_BUNDLE=edge
-        export KUBEFLOW_DEBUG=true
-        export KUBEFLOW_IGNORE_MIN_MEM=true
-        sg microk8s -c 'microk8s enable kubeflow'
+      run: sg microk8s -c 'microk8s enable kubeflow --debug --bundle=edge --ignore-min-mem --password=hunter2'
 
     - name: Test kubeflow
       run: |
@@ -75,6 +70,8 @@ jobs:
         sudo pip3 install pytest sh kfp requests pyyaml
         git clone https://github.com/juju-solutions/bundle-kubeflow.git
         cd bundle-kubeflow
+        sudo microk8s status --wait-ready
+        sudo microk8s kubectl -n kube-system rollout status ds/calico-node
         trap 'sudo pkill -f svc/pipelines-api' SIGINT SIGTERM EXIT
         sudo microk8s kubectl -n kubeflow port-forward svc/pipelines-api 8888:8888 &
         (i=30; while ! curl localhost:8888 ; do ((--i)) || exit; sleep 1; done)
@@ -170,6 +167,7 @@ jobs:
           export KUBEFLOW_BUNDLE=${{ matrix.bundle }}
           export KUBEFLOW_DEBUG=true
           export KUBEFLOW_IGNORE_MIN_MEM=true
+          export KUBEFLOW_AUTH_PASSWORD=hunter2
           microk8s enable kubeflow
         EOF
 
@@ -183,6 +181,8 @@ jobs:
           sudo pip3 install pytest sh kfp requests pyyaml
           git clone https://github.com/juju-solutions/bundle-kubeflow.git
           cd bundle-kubeflow
+          sudo microk8s status --wait-ready
+          sudo microk8s kubectl -n kube-system rollout status ds/calico-node
           trap 'sudo pkill -f svc/pipelines-api' SIGINT SIGTERM EXIT
           microk8s kubectl -n kubeflow port-forward svc/pipelines-api 8888:8888 &
           (i=30; while ! curl localhost:8888 ; do ((--i)) || exit; sleep 1; done)

microk8s-resources/actions/disable.kubeflow.sh

@@ -1,14 +1,46 @@
-#!/usr/bin/env bash
+#!/usr/bin/env python3
 
-set -eu
+import os
+import subprocess
 
-source $SNAP/actions/common/utils.sh
+import click
 
-function disable_kubeflow() {
-  echo "Disabling Kubeflow..."
-  "$SNAP/microk8s-juju.wrapper" unregister -y uk8s || true
-  "$SNAP/microk8s-kubectl.wrapper" delete ns controller-uk8s kubeflow || true
-}
 
-disable_kubeflow
+@click.command()
+def kubeflow():
+    click.echo("Disabling Kubeflow...")
 
+    env = os.environ.copy()
+    env["PATH"] += ":%s" % os.environ["SNAP"]
+
+    click.echo("Unregistering model...")
+    try:
+        subprocess.run(
+            ['microk8s-juju.wrapper', 'unregister', '-y', 'uk8s'],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=env,
+        )
+    except subprocess.CalledProcessError:
+        pass
+    click.echo("Unregistering complete.")
+
+    click.echo("Destroying namespace...")
+    try:
+        subprocess.check_call(
+            ['microk8s-kubectl.wrapper', 'delete', 'ns', 'controller-uk8s', 'kubeflow'],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=env,
+        )
+    except subprocess.CalledProcessError:
+        pass
+    click.echo("Destruction complete.")
+
+    click.echo("Kubeflow is now disabled.")
+
+
+if __name__ == "__main__":
+    kubeflow(prog_name='microk8s disable kubeflow')

microk8s-resources/actions/enable.kubeflow.sh

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import csv
 import json
 import os
 import random
@@ -16,6 +15,7 @@ from pathlib import Path
 from urllib.error import URLError
 from urllib.parse import ParseResult, urlparse
 from urllib.request import urlopen
+import click
 
 
 MIN_MEM_GB = 14
@@ -210,29 +210,42 @@ def get_hostname():
     return "localhost"
 
 
-def main():
-    args = {
-        'bundle': os.environ.get("KUBEFLOW_BUNDLE") or "cs:kubeflow-230",
-        'channel': os.environ.get("KUBEFLOW_CHANNEL") or "stable",
-        'debug': os.environ.get("KUBEFLOW_DEBUG") or "false",
-        'hostname': os.environ.get("KUBEFLOW_HOSTNAME") or None,
-        'ignore_min_mem': os.environ.get("KUBEFLOW_IGNORE_MIN_MEM") or "false",
-        'no_proxy': os.environ.get("KUBEFLOW_NO_PROXY") or None,
-        'password': os.environ.get("KUBEFLOW_AUTH_PASSWORD") or get_random_pass(),
-    }
-    for pair in list(csv.reader(sys.argv[1:]))[0]:
-        key, val = pair.split('=', maxsplit=1)
-        if key not in args:
-            print("Invalid argument `%s`." % key)
-            print("Valid arguments options are:\n - " + "\n - ".join(args.keys()))
-            sys.exit(1)
-        args[key] = val
-
-    # Coerce the boolean args to actual bools
-    for arg in ['debug', 'ignore_min_mem']:
-        if not isinstance(args[arg], bool):
-            args[arg] = strtobool(args[arg])
-
+@click.command()
+@click.option(
+    '--bundle',
+    default='cs:kubeflow-230',
+    help='The Kubeflow bundle to deploy. Can be one of full, lite, edge, or a charm store URL.',
+)
+@click.option(
+    '--channel',
+    default='stable',
+    type=click.Choice(['stable', 'candidate', 'beta', 'edge']),
+    help='Which channel to deploy the bundle from. In most cases, this should be `stable`.',
+)
+@click.option(
+    '--debug/--no-debug',
+    default=False,
+    help='If true, shows more verbose output when enabling Kubeflow.',
+)
+@click.option(
+    '--hostname',
+    help='If set, this hostname is used instead of a hostname generated by MetalLB.',
+)
+@click.option(
+    '--ignore-min-mem/--no-ignore-min-mem',
+    default=False,
+    help='If set, overrides the minimum memory check.',
+)
+@click.option(
+    '--no-proxy',
+    help='Allows setting the juju-no-proxy configuration option.',
+)
+@click.password_option(
+    envvar='KUBEFLOW_AUTH_PASSWORD',
+    default=get_random_pass,
+    help='The Kubeflow dashboard password.',
+)
+def kubeflow(bundle, channel, debug, hostname, ignore_min_mem, no_proxy, password):
     if os.geteuid() == 0:
         print("This command can't be run as root.")
         print("Try `microk8s enable kubeflow` instead.")
@@ -261,36 +274,41 @@ def main():
         print("Couldn't determine total memory.")
         print("Kubeflow recommends at least %s GB of memory." % MIN_MEM_GB)
 
-    if total_mem < MIN_MEM_GB * 1024 * 1024 and not args['ignore_min_mem']:
+    if total_mem < MIN_MEM_GB * 1024 * 1024 and not ignore_min_mem:
         print("Kubeflow recommends at least %s GB of memory." % MIN_MEM_GB)
-        print(
-            "Run `KUBEFLOW_IGNORE_MIN_MEM=true microk8s.enable kubeflow`"
-            " if you'd like to proceed anyways."
-        )
+        print("Use `--ignore-min-mem` if you'd like to proceed anyways.")
+        sys.exit(1)
+
+    try:
+        juju("show-controller", "uk8s", die=False, stdout=False)
+    except subprocess.CalledProcessError:
+        pass
+    else:
+        print("Kubeflow has already been enabled.")
         sys.exit(1)
 
     # Allow specifying the bundle as one of the main types of kubeflow bundles
     # that we create in the charm store, namely full, lite, or edge. The user
     # shoudn't have to specify a version for those bundles. However, allow the
     # user to specify a full charm store URL if they'd like, such as
     # `cs:kubeflow-lite-123`.
-    if args['bundle'] == 'full':
+    if bundle == 'full':
         bundle = 'cs:kubeflow-230'
-    elif args['bundle'] == 'lite':
+    elif bundle == 'lite':
         bundle = 'cs:kubeflow-lite-17'
-    elif args['bundle'] == 'edge':
+    elif bundle == 'edge':
         bundle = 'cs:kubeflow-edge-16'
     else:
-        bundle = args['bundle']
+        bundle = bundle
 
-    run("microk8s-status.wrapper", "--wait-ready", debug=args['debug'])
+    run("microk8s-status.wrapper", "--wait-ready", debug=debug)
     run(
         'microk8s-kubectl.wrapper',
         '-nkube-system',
         'rollout',
         'status',
         'deployment.apps/calico-kube-controllers',
-        debug=args['debug'],
+        debug=debug,
     )
 
     for service in [
@@ -301,16 +319,16 @@ def main():
         "metallb:10.64.140.43-10.64.140.49",
     ]:
         print("Enabling %s..." % service)
-        run("microk8s-enable.wrapper", service, debug=args['debug'])
+        run("microk8s-enable.wrapper", service, debug=debug)
 
-    run("microk8s-status.wrapper", "--wait-ready", debug=args['debug'])
+    run("microk8s-status.wrapper", "--wait-ready", debug=debug)
     run(
         'microk8s-kubectl.wrapper',
         '-nkube-system',
         'rollout',
         'status',
         'ds/calico-node',
-        debug=args['debug'],
+        debug=debug,
     )
 
     print("Waiting for DNS and storage plugins to finish setting up")
@@ -322,31 +340,25 @@ def main():
         "deployment/coredns",
         "deployment/hostpath-provisioner",
         "--timeout=10m",
-        debug=args['debug'],
+        debug=debug,
     )
 
-    check_connectivity()
+    print("DNS and storage setup complete. Checking connectivity...")
 
-    try:
-        juju("show-controller", "uk8s", die=False, stdout=False)
-    except subprocess.CalledProcessError:
-        pass
-    else:
-        print("Kubeflow has already been enabled.")
-        sys.exit(1)
+    check_connectivity()
 
     print("Bootstrapping...")
-    if args['no_proxy'] is not None:
-        juju("bootstrap", "microk8s", "uk8s", "--config=juju-no-proxy=%s" % args['no_proxy'])
+    if no_proxy is not None:
+        juju("bootstrap", "microk8s", "uk8s", "--config=juju-no-proxy=%s" % no_proxy)
         juju("add-model", "kubeflow", "microk8s")
-        juju("model-config", "-m", "kubeflow", "juju-no-proxy=%s" % args['no_proxy'])
+        juju("model-config", "-m", "kubeflow", "juju-no-proxy=%s" % no_proxy)
     else:
         juju("bootstrap", "microk8s", "uk8s")
         juju("add-model", "kubeflow", "microk8s")
     print("Bootstrap complete.")
 
     print("Successfully bootstrapped, deploying...")
-    juju("deploy", bundle, "--channel", args['channel'])
+    juju("deploy", bundle, "--channel", channel)
 
     print("Kubeflow deployed.")
     print("Waiting for operator pods to become ready.")
@@ -391,7 +403,7 @@ def main():
             f.flush()
             run('microk8s-kubectl.wrapper', 'apply', '-f', f.name)
 
-    hostname = parse_hostname(args['hostname'] or get_hostname())
+    hostname = parse_hostname(hostname or get_hostname())
 
     if kubectl_exists('service/dex-auth'):
         juju("config", "dex-auth", "public-url=%s" % hostname.geturl())
@@ -407,7 +419,7 @@ def main():
         "pod",
         "--timeout=30s",
         "--all",
-        debug=args['debug'],
+        debug=debug,
         times=100,
     )
 
@@ -428,7 +440,7 @@ def main():
             microk8s juju config dex-auth static-password
 
         """
-                % (hostname.geturl(), args['password'])
+                % (hostname.geturl(), password)
             )
         )
     else:
@@ -447,4 +459,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    kubeflow(prog_name='microk8s enable kubeflow', auto_envvar_prefix='KUBEFLOW')