From ec029da8a9f3ff8486fc257b74477df402d6e64e Mon Sep 17 00:00:00 2001 From: asmijafar20 Date: Thu, 22 Sep 2022 20:21:02 +0530 Subject: [PATCH 1/7] Add , , and commands. --- qhub/cli/main.py | 201 ++++++++++++++++++++++++++++++++++++++++++++++- qhub/keycloak.py | 18 +++-- 2 files changed, 208 insertions(+), 11 deletions(-) diff --git a/qhub/cli/main.py b/qhub/cli/main.py index 2f214d5a5..147ff8988 100644 --- a/qhub/cli/main.py +++ b/qhub/cli/main.py @@ -1,5 +1,7 @@ from pathlib import Path +from typing import Tuple + import rich import typer from click import Context @@ -24,7 +26,14 @@ verify, ) from qhub.utils import load_yaml +from qhub.cost import infracost_report +from qhub.upgrade import do_upgrade +from qhub.keycloak import do_keycloak +from zipfile import ZipFile +from kubernetes import client +from kubernetes import config as kube_config +from ruamel import yaml def enum_to_list(enum_cls): return [e.value for e in enum_cls] @@ -344,7 +353,7 @@ def init( @app.command() def validate( config: str = typer.Option( - None, + ..., "--config", "-c", help="qhub configuration yaml file path, please pass in as -c/--config flag", @@ -382,7 +391,7 @@ def render( help="output directory", ), config: str = typer.Option( - None, + ..., "-c", "--config", help="nebari configuration yaml file path", @@ -473,7 +482,7 @@ def deploy( @app.command() def destroy( - config: str = typer.Option(..., "-c", "--config", help="qhub configuration"), + config: str = typer.Option(..., "-c", "--config", help="qhub configuration file path"), output: str = typer.Option( "./" "-o", "--output", @@ -507,6 +516,192 @@ def destroy( destroy_configuration(config_yaml) +@app.command() +def cost( + path: str = typer.Option( + None, + "-p", + "--path", + help="Pass the path of your stages directory generated after rendering QHub configurations before deployment", + ), + dashboard: bool = typer.Option( + True, + "-d", + "--dashboard", + help="Enable the cost dashboard", + ), + file: str = typer.Option( + None, + "-f", + "--file", + help="Specify the path of the file to store the cost report", + ), + currency: str = typer.Option( + "USD", + "-c", + "--currency", + help="Specify the currency code to use in the cost report", + ), + compare: bool = typer.Option( + False, + "-cc", + "--compare", + help="Compare the cost report to a previously generated report", + ) +): + """ + Cost-Estimate + """ + infracost_report( + path=path, + dashboard=True, + file=file, + currency_code=currency, + compare=False, + ) + +@app.command() +def upgrade( + config: str = typer.Option( + ..., + "-c", + "--config", + help="qhub configuration file path", + ), + attempt_fixes: bool = typer.Option( + False, + "--attempt-fixes", + help="Attempt to fix the config for any incompatibilities between your old and new QHub versions.", + ) +): + """ + Upgrade + """ + config_filename = Path(config) + if not config_filename.is_file(): + raise ValueError( + f"passed in configuration filename={config_filename} must exist" + ) + + do_upgrade( + config_filename, + attempt_fixes=attempt_fixes + ) + +@app.command() +def keycloak( + config: str = typer.Option( + ..., + "-c", + "--config", + help="qhub configuration file path", + ), + add_user: Tuple[str,str] = typer.Option( + None, + "--add-user", + help="`--add-user [password]` or `listusers`", + ), + list_users: bool = typer.Option( + False, + "--listusers", + help="list current keycloak users", + ) +): + """ + Keycloak + """ + config_filename = Path(config) + if not config_filename.is_file(): + raise ValueError( + f"passed in configuration filename={config_filename} must exist" + ) + + do_keycloak( + config_filename, + add_user=add_user, + listusers=list_users, + ) + +@app.command() +def support( + config_filename: str = typer.Option( + ..., + "-c", + "--config", + help="qhub configuration file path", + ), + output: str = typer.Option( + "./qhub-support-logs.zip", + "-o", + "--output", + help="output filename", + ) +): + """ + Support + """ + + kube_config.load_kube_config() + + v1 = client.CoreV1Api() + + namespace = get_config_namespace(config=config_filename) + + pods = v1.list_namespaced_pod(namespace=namespace) + + for pod in pods.items: + Path(f"./log/{namespace}").mkdir(parents=True, exist_ok=True) + path = Path(f"./log/{namespace}/{pod.metadata.name}.txt") + with path.open(mode="wt") as file: + try: + file.write( + "%s\t%s\t%s\n" + % ( + pod.status.pod_ip, + namespace, + pod.metadata.name, + ) + ) + + # some pods are running multiple containers + containers = [ + _.name if len(pod.spec.containers) > 1 else None + for _ in pod.spec.containers + ] + + for container in containers: + if container is not None: + file.write(f"Container: {container}\n") + file.write( + v1.read_namespaced_pod_log( + name=pod.metadata.name, + namespace=namespace, + container=container, + ) + ) + + except client.exceptions.ApiException as e: + file.write("%s not available" % pod.metadata.name) + raise e + + with ZipFile(output, "w") as zip: + for file in list(Path(f"./log/{namespace}").glob("*.txt")): + print(file) + zip.write(file) + + + config_filename = Path(config_filename) + if not config_filename.is_file(): + raise ValueError( + f"passed in configuration filename={config_filename} must exist" + ) + + with config_filename.open() as f: + config = yaml.safe_load(f.read()) + + return config["namespace"] + + if __name__ == "__main__": app() diff --git a/qhub/keycloak.py b/qhub/keycloak.py index f836578da..2ae0ad71d 100644 --- a/qhub/keycloak.py +++ b/qhub/keycloak.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) -def do_keycloak(config_filename, *args): +def do_keycloak(config_filename, add_user, listusers): config = load_yaml(config_filename) verify(config) @@ -20,19 +20,21 @@ def do_keycloak(config_filename, *args): keycloak_admin = get_keycloak_admin_from_config(config) - if args[0] == "adduser": - if len(args) < 2: + if add_user is not None: + if len(add_user) < 2: raise ValueError( "keycloak command 'adduser' requires `username [password]`" ) - username = args[1] - password = args[2] if len(args) >= 3 else None + username, password = add_user + + if len(password) < 3: + password=None create_user(keycloak_admin, username, password, domain=config["domain"]) - elif args[0] == "listusers": - list_users(keycloak_admin) + elif listusers: + listusers(keycloak_admin) else: - raise ValueError(f"unknown keycloak command {args[0]}") + raise ValueError("unsupported keycloak argument, please use --help") def create_user( From 97baac9014d3107b33ed5da4a7e72a6dba24ec43 Mon Sep 17 00:00:00 2001 From: asmijafar20 Date: Thu, 22 Sep 2022 20:26:01 +0530 Subject: [PATCH 2/7] Add pre-commit --- qhub/cli/main.py | 64 ++++++++++++++++++++++++------------------------ qhub/keycloak.py | 4 +-- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/qhub/cli/main.py b/qhub/cli/main.py index 147ff8988..3bc8f5de8 100644 --- a/qhub/cli/main.py +++ b/qhub/cli/main.py @@ -1,11 +1,14 @@ from pathlib import Path - -from typing import Tuple +from typing import Tuple +from zipfile import ZipFile import rich import typer from click import Context +from kubernetes import client +from kubernetes import config as kube_config from rich import print +from ruamel import yaml from typer.core import TyperGroup from qhub.cli._init import ( @@ -14,8 +17,10 @@ check_project_name, handle_init, ) +from qhub.cost import infracost_report from qhub.deploy import deploy_configuration from qhub.destroy import destroy_configuration +from qhub.keycloak import do_keycloak from qhub.render import render_template from qhub.schema import ( AuthenticationEnum, @@ -25,15 +30,9 @@ TerraformStateEnum, verify, ) -from qhub.utils import load_yaml -from qhub.cost import infracost_report from qhub.upgrade import do_upgrade -from qhub.keycloak import do_keycloak -from zipfile import ZipFile +from qhub.utils import load_yaml -from kubernetes import client -from kubernetes import config as kube_config -from ruamel import yaml def enum_to_list(enum_cls): return [e.value for e in enum_cls] @@ -482,7 +481,9 @@ def deploy( @app.command() def destroy( - config: str = typer.Option(..., "-c", "--config", help="qhub configuration file path"), + config: str = typer.Option( + ..., "-c", "--config", help="qhub configuration file path" + ), output: str = typer.Option( "./" "-o", "--output", @@ -516,6 +517,7 @@ def destroy( destroy_configuration(config_yaml) + @app.command() def cost( path: str = typer.Option( @@ -547,7 +549,7 @@ def cost( "-cc", "--compare", help="Compare the cost report to a previously generated report", - ) + ), ): """ Cost-Estimate @@ -560,19 +562,20 @@ def cost( compare=False, ) + @app.command() def upgrade( config: str = typer.Option( ..., - "-c", - "--config", + "-c", + "--config", help="qhub configuration file path", ), attempt_fixes: bool = typer.Option( False, "--attempt-fixes", help="Attempt to fix the config for any incompatibilities between your old and new QHub versions.", - ) + ), ): """ Upgrade @@ -583,20 +586,18 @@ def upgrade( f"passed in configuration filename={config_filename} must exist" ) - do_upgrade( - config_filename, - attempt_fixes=attempt_fixes - ) + do_upgrade(config_filename, attempt_fixes=attempt_fixes) + @app.command() def keycloak( config: str = typer.Option( ..., - "-c", - "--config", + "-c", + "--config", help="qhub configuration file path", ), - add_user: Tuple[str,str] = typer.Option( + add_user: Tuple[str, str] = typer.Option( None, "--add-user", help="`--add-user [password]` or `listusers`", @@ -605,7 +606,7 @@ def keycloak( False, "--listusers", help="list current keycloak users", - ) + ), ): """ Keycloak @@ -617,30 +618,31 @@ def keycloak( ) do_keycloak( - config_filename, + config_filename, add_user=add_user, listusers=list_users, - ) + ) + @app.command() def support( config_filename: str = typer.Option( ..., - "-c", - "--config", + "-c", + "--config", help="qhub configuration file path", ), output: str = typer.Option( "./qhub-support-logs.zip", - "-o", - "--output", + "-o", + "--output", help="output filename", - ) + ), ): """ Support """ - + kube_config.load_kube_config() v1 = client.CoreV1Api() @@ -689,7 +691,6 @@ def support( print(file) zip.write(file) - config_filename = Path(config_filename) if not config_filename.is_file(): raise ValueError( @@ -702,6 +703,5 @@ def support( return config["namespace"] - if __name__ == "__main__": app() diff --git a/qhub/keycloak.py b/qhub/keycloak.py index 2ae0ad71d..aab379a43 100644 --- a/qhub/keycloak.py +++ b/qhub/keycloak.py @@ -20,7 +20,7 @@ def do_keycloak(config_filename, add_user, listusers): keycloak_admin = get_keycloak_admin_from_config(config) - if add_user is not None: + if add_user is not None: if len(add_user) < 2: raise ValueError( "keycloak command 'adduser' requires `username [password]`" @@ -29,7 +29,7 @@ def do_keycloak(config_filename, add_user, listusers): username, password = add_user if len(password) < 3: - password=None + password = None create_user(keycloak_admin, username, password, domain=config["domain"]) elif listusers: listusers(keycloak_admin) From 3d45ce05324ab4c310df3fcc72d0e11b25c021ed Mon Sep 17 00:00:00 2001 From: asmijafar20 Date: Thu, 22 Sep 2022 20:36:17 +0530 Subject: [PATCH 3/7] Add pre-commit --- qhub/cli/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qhub/cli/main.py b/qhub/cli/main.py index 3bc8f5de8..b1b940e59 100644 --- a/qhub/cli/main.py +++ b/qhub/cli/main.py @@ -691,6 +691,8 @@ def support( print(file) zip.write(file) + +def get_config_namespace(config): config_filename = Path(config_filename) if not config_filename.is_file(): raise ValueError( From e572eb36d6bd2c180e04b6177cc9c6bde50957c0 Mon Sep 17 00:00:00 2001 From: asmijafar20 Date: Thu, 22 Sep 2022 20:39:16 +0530 Subject: [PATCH 4/7] Add pre-commit --- qhub/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qhub/cli/main.py b/qhub/cli/main.py index b1b940e59..6a0baf043 100644 --- a/qhub/cli/main.py +++ b/qhub/cli/main.py @@ -693,7 +693,7 @@ def support( def get_config_namespace(config): - config_filename = Path(config_filename) + config_filename = Path(config) if not config_filename.is_file(): raise ValueError( f"passed in configuration filename={config_filename} must exist" From 4f39386254e749572b19221ac4f6fcf8a138c15d Mon Sep 17 00:00:00 2001 From: asmijafar20 Date: Thu, 22 Sep 2022 21:02:55 +0530 Subject: [PATCH 5/7] Add rich print --- qhub/keycloak.py | 7 +++++-- qhub/upgrade.py | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/qhub/keycloak.py b/qhub/keycloak.py index aab379a43..aa638d744 100644 --- a/qhub/keycloak.py +++ b/qhub/keycloak.py @@ -2,6 +2,7 @@ import os import keycloak +import rich from .schema import verify from .utils import load_yaml @@ -57,9 +58,11 @@ def create_user( {"type": "password", "value": password, "temporary": False} ] else: - print(f"Creating user={username} without password (none supplied)") + rich.print( + f"Creating user=[green]{username}[/green] without password (none supplied)" + ) keycloak_admin.create_user(payload) - print(f"Created user={username}") + rich.print(f"Created user=[green]{username}[/green]") def list_users(keycloak_admin: keycloak.KeycloakAdmin): diff --git a/qhub/upgrade.py b/qhub/upgrade.py index 4cb5a6ac1..c2c959a29 100644 --- a/qhub/upgrade.py +++ b/qhub/upgrade.py @@ -6,6 +6,7 @@ import string from abc import ABC +import rich from pydantic.error_wrappers import ValidationError from .schema import is_version_accepted, verify @@ -21,8 +22,8 @@ def do_upgrade(config_filename, attempt_fixes=False): try: verify(config) - print( - f"Your config file {config_filename} appears to be already up-to-date for qhub version {__version__}" + rich.print( + f"Your config file [purple]{config_filename}[/purple] appears to be already up-to-date for qhub version [green]{__version__}[/green]" ) return except (ValidationError, ValueError) as e: From 886e49b307cba2e6b5c9b55a31130b120c64fe8e Mon Sep 17 00:00:00 2001 From: asmijafar20 Date: Tue, 27 Sep 2022 00:04:56 +0530 Subject: [PATCH 6/7] Change do_keycloak function --- qhub/cli/keycloak.py | 61 ++++++++++++++++++++++++++++++++++++++++++++ qhub/cli/main.py | 5 +++- qhub/keycloak.py | 10 +++----- 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/qhub/cli/keycloak.py b/qhub/cli/keycloak.py index ebc638b46..1ac61d37a 100644 --- a/qhub/cli/keycloak.py +++ b/qhub/cli/keycloak.py @@ -3,6 +3,9 @@ from qhub.keycloak import do_keycloak +# from typing import Tuple + + logger = logging.getLogger(__name__) @@ -25,3 +28,61 @@ def handle_keycloak(args): ) do_keycloak(config_filename, *args.keycloak_action) + + +# import typer +# from dataclasses import dataclass + +# app = typer.Typer() + +# @dataclass +# class Common: +# config: str + +# @app.callback() +# def common(ctx: typer.Context, +# config: str = typer.Option( +# ..., +# envvar="APP_CONFIG" +# ) +# ): +# ctx.obj=Common( +# config +# ) + +# @app.command() +# def add_users(ctx: typer.Context, add_users: Tuple[str, str] = typer.Option( +# None, +# "--user", +# help="` [password]`" +# ) +# ): +# config_filename = Path(ctx.obj.config) +# if not config_filename.is_file(): +# raise ValueError( +# f"passed in configuration filename={config_filename} must exist" +# ) + +# do_keycloak( +# config_filename, +# add_user=add_users, +# listusers=False, +# ) + +# @app.command() +# def list_users(ctx: typer.Context): + +# config_filename = Path(ctx.obj.config) +# if not config_filename.is_file(): +# raise ValueError( +# f"passed in configuration filename={config_filename} must exist" +# ) + +# do_keycloak( +# config_filename, +# add_user=False, +# listusers=True, +# ) + +# if __name__=="__main__": +# app() diff --git a/qhub/cli/main.py b/qhub/cli/main.py index 6a0baf043..eaab9c2e8 100644 --- a/qhub/cli/main.py +++ b/qhub/cli/main.py @@ -53,6 +53,8 @@ def list_commands(self, ctx: Context): context_settings={"help_option_names": ["-h", "--help"]}, ) +# app.add_typer(keycloak.app,name="keycloak",help="keycloak") + @app.command() def guided_init( @@ -619,7 +621,8 @@ def keycloak( do_keycloak( config_filename, - add_user=add_user, + username=add_user[0], + password=add_user[1], listusers=list_users, ) diff --git a/qhub/keycloak.py b/qhub/keycloak.py index aa638d744..5ecd041dd 100644 --- a/qhub/keycloak.py +++ b/qhub/keycloak.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) -def do_keycloak(config_filename, add_user, listusers): +def do_keycloak(config_filename, username, password, listusers): config = load_yaml(config_filename) verify(config) @@ -21,19 +21,17 @@ def do_keycloak(config_filename, add_user, listusers): keycloak_admin = get_keycloak_admin_from_config(config) - if add_user is not None: - if len(add_user) < 2: + if username: + if len(username) < 2: raise ValueError( "keycloak command 'adduser' requires `username [password]`" ) - username, password = add_user - if len(password) < 3: password = None create_user(keycloak_admin, username, password, domain=config["domain"]) elif listusers: - listusers(keycloak_admin) + list_users(keycloak_admin) else: raise ValueError("unsupported keycloak argument, please use --help") From b025d65c93a7fa4bd2b505676fe24b4e4b5d0c47 Mon Sep 17 00:00:00 2001 From: iameskild Date: Mon, 26 Sep 2022 20:03:18 -0700 Subject: [PATCH 7/7] Create two keycloak subcommands --- qhub/cli/_keycloak.py | 47 +++++++++++++++++++++++++++++++++ qhub/cli/keycloak.py | 61 ------------------------------------------- qhub/cli/main.py | 42 ++--------------------------- qhub/keycloak.py | 14 +++++----- 4 files changed, 56 insertions(+), 108 deletions(-) create mode 100644 qhub/cli/_keycloak.py diff --git a/qhub/cli/_keycloak.py b/qhub/cli/_keycloak.py new file mode 100644 index 000000000..0ae76c212 --- /dev/null +++ b/qhub/cli/_keycloak.py @@ -0,0 +1,47 @@ +from pathlib import Path +from typing import Tuple + +import typer + +from qhub.keycloak import do_keycloak + +app_keycloak = typer.Typer() + + +@app_keycloak.command() +def add_user( + add_users: Tuple[str, str] = typer.Option( + ..., "--user", help="Provide both: " + ), + config_filename: str = typer.Option( + ..., + "-c", + "--config", + help="qhub configuration file path", + ), +): + """Add a user to Keycloak. User will be automatically added to the [italic]analyst[/italic] group.""" + if isinstance(config_filename, str): + config_filename = Path(config_filename) + + args = ["adduser", add_users[0], add_users[1]] + + do_keycloak(config_filename, *args) + + +@app_keycloak.command() +def list_users( + config_filename: str = typer.Option( + ..., + "-c", + "--config", + help="qhub configuration file path", + ) +): + """List the users in Keycloak.""" + if isinstance(config_filename, str): + config_filename = Path(config_filename) + + args = ["listusers"] + + do_keycloak(config_filename, *args) diff --git a/qhub/cli/keycloak.py b/qhub/cli/keycloak.py index 1ac61d37a..ebc638b46 100644 --- a/qhub/cli/keycloak.py +++ b/qhub/cli/keycloak.py @@ -3,9 +3,6 @@ from qhub.keycloak import do_keycloak -# from typing import Tuple - - logger = logging.getLogger(__name__) @@ -28,61 +25,3 @@ def handle_keycloak(args): ) do_keycloak(config_filename, *args.keycloak_action) - - -# import typer -# from dataclasses import dataclass - -# app = typer.Typer() - -# @dataclass -# class Common: -# config: str - -# @app.callback() -# def common(ctx: typer.Context, -# config: str = typer.Option( -# ..., -# envvar="APP_CONFIG" -# ) -# ): -# ctx.obj=Common( -# config -# ) - -# @app.command() -# def add_users(ctx: typer.Context, add_users: Tuple[str, str] = typer.Option( -# None, -# "--user", -# help="` [password]`" -# ) -# ): -# config_filename = Path(ctx.obj.config) -# if not config_filename.is_file(): -# raise ValueError( -# f"passed in configuration filename={config_filename} must exist" -# ) - -# do_keycloak( -# config_filename, -# add_user=add_users, -# listusers=False, -# ) - -# @app.command() -# def list_users(ctx: typer.Context): - -# config_filename = Path(ctx.obj.config) -# if not config_filename.is_file(): -# raise ValueError( -# f"passed in configuration filename={config_filename} must exist" -# ) - -# do_keycloak( -# config_filename, -# add_user=False, -# listusers=True, -# ) - -# if __name__=="__main__": -# app() diff --git a/qhub/cli/main.py b/qhub/cli/main.py index eaab9c2e8..5528b00a0 100644 --- a/qhub/cli/main.py +++ b/qhub/cli/main.py @@ -1,5 +1,4 @@ from pathlib import Path -from typing import Tuple from zipfile import ZipFile import rich @@ -17,10 +16,10 @@ check_project_name, handle_init, ) +from qhub.cli._keycloak import app_keycloak from qhub.cost import infracost_report from qhub.deploy import deploy_configuration from qhub.destroy import destroy_configuration -from qhub.keycloak import do_keycloak from qhub.render import render_template from qhub.schema import ( AuthenticationEnum, @@ -52,8 +51,7 @@ def list_commands(self, ctx: Context): rich_markup_mode="rich", context_settings={"help_option_names": ["-h", "--help"]}, ) - -# app.add_typer(keycloak.app,name="keycloak",help="keycloak") +app.add_typer(app_keycloak, name="keycloak", help="keycloak") @app.command() @@ -591,42 +589,6 @@ def upgrade( do_upgrade(config_filename, attempt_fixes=attempt_fixes) -@app.command() -def keycloak( - config: str = typer.Option( - ..., - "-c", - "--config", - help="qhub configuration file path", - ), - add_user: Tuple[str, str] = typer.Option( - None, - "--add-user", - help="`--add-user [password]` or `listusers`", - ), - list_users: bool = typer.Option( - False, - "--listusers", - help="list current keycloak users", - ), -): - """ - Keycloak - """ - config_filename = Path(config) - if not config_filename.is_file(): - raise ValueError( - f"passed in configuration filename={config_filename} must exist" - ) - - do_keycloak( - config_filename, - username=add_user[0], - password=add_user[1], - listusers=list_users, - ) - - @app.command() def support( config_filename: str = typer.Option( diff --git a/qhub/keycloak.py b/qhub/keycloak.py index 5ecd041dd..7934f566b 100644 --- a/qhub/keycloak.py +++ b/qhub/keycloak.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) -def do_keycloak(config_filename, username, password, listusers): +def do_keycloak(config_filename, *args): config = load_yaml(config_filename) verify(config) @@ -21,19 +21,19 @@ def do_keycloak(config_filename, username, password, listusers): keycloak_admin = get_keycloak_admin_from_config(config) - if username: - if len(username) < 2: + if args[0] == "adduser": + if len(args) < 2: raise ValueError( "keycloak command 'adduser' requires `username [password]`" ) - if len(password) < 3: - password = None + username = args[1] + password = args[2] if len(args) >= 3 else None create_user(keycloak_admin, username, password, domain=config["domain"]) - elif listusers: + elif args[0] == "listusers": list_users(keycloak_admin) else: - raise ValueError("unsupported keycloak argument, please use --help") + raise ValueError(f"unknown keycloak command {args[0]}") def create_user(