From a058e721eae49a8a775487dc6c283fc18d838b14 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Thu, 22 Aug 2024 15:03:53 +0530 Subject: [PATCH 01/21] Add YAML config file support Introduced YAML configuration file handling using `click-config-file` and `pyyaml`, ensuring a default config file is created if not present. Updated dependencies in `pyproject.toml` accordingly to support the new feature. --- pyproject.toml | 8 ++++--- src/hckr/cli/__init__.py | 45 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f3a7b3c..c4d54ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,11 +39,13 @@ dependencies = [ "pyarrow", # For saving data in Parquet format. "fastavro", # For handling Avro file format. "requests", -# "tomli", "packaging", - "kubernetes", + "kubernetes", # for k8s commands "yaspin", - "speedtest-cli" + "speedtest-cli", # for net speed command + # YAML config support + "click-config-file", + "pyyaml", ] [project.urls] diff --git a/src/hckr/cli/__init__.py b/src/hckr/cli/__init__.py index b233621..44c5c52 100644 --- a/src/hckr/cli/__init__.py +++ b/src/hckr/cli/__init__.py @@ -2,18 +2,22 @@ # # SPDX-License-Identifier: MIT import logging +import os +from pathlib import Path import click +import click_config_file +import yaml from click_repl import register_repl # type: ignore from hckr.cli.k8s.context import context from hckr.cli.k8s.namespace import namespace from hckr.cli.k8s.pod import pod -from .net import net from .crypto.fernet import fernet from .data import data from .info import info from .k8s import k8s +from .net import net from ..__about__ import __version__ from ..cli.cron import cron from ..cli.crypto import crypto @@ -42,11 +46,44 @@ def __init__(self): # Note: This object must have an empty constructor. # pass_info is a decorator for functions that pass 'Info' objects. pass_info = click.make_pass_decorator(Info, ensure=True) +# Define the default configuration file path +config_path = Path.home() / ".hckrcfg" + + +# Ensure the configuration file exists +def ensure_config_file(config_path: Path): + """Ensure the configuration file and its directory exist.""" + if not config_path.exists(): + config_path.parent.mkdir(parents=True, exist_ok=True) # Create the directory if it doesn't exist + config_path.touch(exist_ok=True) # Create the file if it doesn't exist + # Optionally, write some default configuration + default_config = { + 'hckr': { + 'version': f'hckr {__version__}' + }, + 'verbose': 3 + } + with config_path.open('w') as config_file: + yaml.dump(default_config, config_file) + + +ensure_config_file(config_path) + + +def yaml_loader(config_file, command): + """Load YAML configuration file.""" + print(config_file, command) + with open(config_file, 'r') as yaml_file: + return yaml.safe_load(yaml_file)[command] + @click.group( context_settings={"help_option_names": ["-h", "--help"]}, invoke_without_command=True, ) +@click_config_file.configuration_option(provider=yaml_loader, implicit=True, + # cmd_name=str(config_path), + config_file_name=str(config_path), default=config_path) @click.option( "--verbose", "-v", @@ -57,9 +94,9 @@ def __init__(self): # Note: This object must have an empty constructor. @click.pass_context @pass_info def cli( - _info: Info, - ctx: click.Context, - verbose: int, + _info: Info, + ctx: click.Context, + verbose: int, ): if verbose > 0: logging.basicConfig( From bd5c589c8849dbb2dfa0610cddb84d4e652e9eff Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Sat, 24 Aug 2024 00:18:10 +0530 Subject: [PATCH 02/21] Add configuration command group with set and get functionalities Introduced a new `configure` command group in the CLI for managing configurations. Implemented `set` and `get` commands to update and retrieve configuration values, respectively. Added necessary unit tests for these functionalities. --- src/hckr/cli/configure.py | 86 +++++++++++++++++++++++ src/hckr/utils/ConfigUtils.py | 125 ++++++++++++++++++++++++++++++++++ tests/cli/test_configure.py | 28 ++++++++ 3 files changed, 239 insertions(+) create mode 100644 src/hckr/cli/configure.py create mode 100644 src/hckr/utils/ConfigUtils.py create mode 100644 tests/cli/test_configure.py diff --git a/src/hckr/cli/configure.py b/src/hckr/cli/configure.py new file mode 100644 index 0000000..2678d3a --- /dev/null +++ b/src/hckr/cli/configure.py @@ -0,0 +1,86 @@ +# from ..utils.MessageUtils import * +import logging + +import click +import rich +from cron_descriptor import get_description # type: ignore +from rich.panel import Panel + +from ..utils import ConfigUtils, MessageUtils +from ..utils.ConfigUtils import load_config, config_path, ensure_config_file, DEFAULT_CONFIG, configMessage + + +@click.group( + help="Config commands", + context_settings={"help_option_names": ["-h", "--help"]}, +) +@click.pass_context +def configure(ctx): + """ + Defines a command group for configuration-related commands. + """ + ensure_config_file() + + +def common_config_options(func): + func = click.option("-c", "--config", help="Config instance, default: DEFAULT", default=DEFAULT_CONFIG)(func) + return func + + +@configure.command() +@common_config_options +@click.argument('key') +@click.argument('value') +def set(config, key, value): + """ + Sets a configuration value. + + Args: + config (str): The configuration instance name. Default is defined by DEFAULT_CONFIG. + key (str): The key of the config setting to change. + value (str): The value to set for the specified key. + + Example: + $ cli_tool configure set database_host 127.0.0.1 + """ + configMessage(config) + ConfigUtils.set_config_value(config, key, value) + rich.print( + Panel( + f"[{config}] {key} <- {value}", + expand=True, + title="Success", + ) + ) + +@configure.command() +@common_config_options +@click.argument('key') +def get(config, key): + """Get a configuration value.""" + configMessage(config) + try: + value = ConfigUtils.get_config_value(config, key) + rich.print( + Panel( + f"[{config}] {key} = {value}", + expand=True, + title="Success", + ) + ) + except ValueError as e: + rich.print( + Panel( + f"{e}", + expand=True, + title="Error", + ) + ) + + +@configure.command() +@common_config_options +def show(config): + """List configuration values.""" + configMessage(config) + ConfigUtils.list_config(config) diff --git a/src/hckr/utils/ConfigUtils.py b/src/hckr/utils/ConfigUtils.py new file mode 100644 index 0000000..ea67a46 --- /dev/null +++ b/src/hckr/utils/ConfigUtils.py @@ -0,0 +1,125 @@ +import configparser +import logging +from pathlib import Path + +import click +import rich +from rich.panel import Panel + +from . import MessageUtils +from ..__about__ import __version__ + +# Define the default configuration file path, this can't be changed, although user can have multile instances using --config +config_path = Path.home() / ".hckrcfg" +DEFAULT_CONFIG = "HCKR" + + +def load_config(): + """Load the INI configuration file.""" + config = configparser.ConfigParser() + config.read(config_path) + return config + + +def ensure_config_file(): + """ + Ensures the existence of a configuration file at the specified path. + + :param config_path: The path to the configuration file. + :return: None + + This function creates a configuration file at the specified path if it does not already exist. It also creates any necessary parent directories. The configuration file is empty initially, but a default configuration is written to it using configparser. The default configuration includes a 'DEFAULT' section with a 'version' option that contains the value of the __version__ global variable. + + Example Usage: + -------------- + ensure_config_file(Path('/path/to/config.ini')) + """ + if not config_path.exists(): + config_path.parent.mkdir(parents=True, exist_ok=True) + config_path.touch(exist_ok=True) + default_config = { + DEFAULT_CONFIG: { + 'version': f"{__version__}", + }, + } + config = configparser.ConfigParser() + config.read_dict(default_config) + with config_path.open('w') as config_file: + config.write(config_file) + MessageUtils.info(f"Creating default config file {config_path}") + else: + logging.debug(f"Config file {config_path} already exists ") + + +def set_config_value(section, key, value): + """ + Sets a configuration value in a configuration file. + + :param section: The name of the configuration section. + :param key: The key of the configuration value. + :param value: The value to set. + :return: None + + This function sets a configuration value in a configuration file. It first logs the action using the `logging.debug()` function. Then, it loads the configuration file using the `load_config()` function. If the configuration file does not have the specified section and the section is not the default section, it adds the section to the configuration file. Next, it sets the value for the key in the specified section. Finally, it writes the updated configuration file to disk. + + After setting the configuration value, the function displays an information message using the `MessageUtils.info()` function. + + Note: This function assumes that the `logging` and `MessageUtils` modules are imported and configured properly. + + Example Usage: + set_config_value("database", "username", "admin") + """ + logging.debug(f"Setting [{section}] {key} = {value}") + config = load_config() + if not config.has_section(section) and section != DEFAULT_CONFIG: + logging.debug(f"Adding section {section}") + config.add_section(section) + config.set(section, key, value) + with config_path.open('w') as config_file: + config.write(config_file) + + +def get_config_value(section, key) -> str: + """ + :param section: The section of the configuration file where the desired value is located. + :param key: The key of the value within the specified section. + :return: The value corresponding to the specified key within the specified section of the configuration file. + + """ + logging.debug(f"Getting [{section}] {key} ") + config = load_config() + if not config.has_section(section): + raise ValueError(f"Section '{section}' not found in the configuration.") + if not config.has_option(section, key): + raise ValueError(f"Key '{key}' not found in section '{section}'.") + return config.get(section, key) + + +def list_config(section): + """ + List Config + + This function takes a section parameter and lists the configuration values for that section from the loaded configuration file. If the section is found in the configuration file, it will print the section name and all key-value pairs associated with that section. If the section is not found, it will display an error message. + + :param section: The section name for which the configuration values should be listed. + :return: None + """ + config = load_config() + if config.has_section(section): + rich.print( + Panel( + "\n".join([f"{key} = {value}" for key, value in config.items(section)]) if config.items( + section) else "NOTHING FOUND", + expand=True, + title="HCKR", + ) + ) + else: + MessageUtils.warning(f"Config '{section}' not found.") + + +def configMessage(config): + if config == DEFAULT_CONFIG: + MessageUtils.info(f"Using default config: [magenta]{DEFAULT_CONFIG}") + else: + MessageUtils.info(f"Using config: [magenta]{config}") diff --git a/tests/cli/test_configure.py b/tests/cli/test_configure.py new file mode 100644 index 0000000..5f6483f --- /dev/null +++ b/tests/cli/test_configure.py @@ -0,0 +1,28 @@ +import string +from distutils.command.config import config +import random +from click.testing import CliRunner + +from hckr.cli.configure import set, get + + +def _get_random_string(length): + letters = string.ascii_lowercase + result_str = ''.join(random.choice(letters) for i in range(length)) + return result_str + + +# DEFAULT CONFIG GET AND SET +def test_configure_get_set_default(): + runner = CliRunner() + _key = f"key_{_get_random_string(5)}" + _value = f"value_{_get_random_string(5)}" + result = runner.invoke(set, [_key, _value]) + assert result.exit_code == 0 + print(result.output) + + assert ( + f"Set [DEFAULT] {_key} = {_value}" + in result.output + ) + result = runner.invoke(get, [_key, _value]) From 9c429b6ca12ae5176a1665f8cc31ca6fa5149359 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Sat, 24 Aug 2024 00:44:13 +0530 Subject: [PATCH 03/21] Refactor configuration management and add DB config command Remove redundant configurations and modularize configuration handling. Add a new CLI command to configure database credentials interactively, enhancing user flexibility. --- src/hckr/cli/__init__.py | 41 ++------------------ src/hckr/cli/configure.py | 38 +++++++++++++++---- src/hckr/utils/ConfigUtils.py | 71 +++++++++++++++++++++++++++++------ 3 files changed, 94 insertions(+), 56 deletions(-) diff --git a/src/hckr/cli/__init__.py b/src/hckr/cli/__init__.py index 44c5c52..c84d010 100644 --- a/src/hckr/cli/__init__.py +++ b/src/hckr/cli/__init__.py @@ -2,14 +2,11 @@ # # SPDX-License-Identifier: MIT import logging -import os -from pathlib import Path import click -import click_config_file -import yaml from click_repl import register_repl # type: ignore +from hckr.cli.configure import configure from hckr.cli.k8s.context import context from hckr.cli.k8s.namespace import namespace from hckr.cli.k8s.pod import pod @@ -46,44 +43,11 @@ def __init__(self): # Note: This object must have an empty constructor. # pass_info is a decorator for functions that pass 'Info' objects. pass_info = click.make_pass_decorator(Info, ensure=True) -# Define the default configuration file path -config_path = Path.home() / ".hckrcfg" - - -# Ensure the configuration file exists -def ensure_config_file(config_path: Path): - """Ensure the configuration file and its directory exist.""" - if not config_path.exists(): - config_path.parent.mkdir(parents=True, exist_ok=True) # Create the directory if it doesn't exist - config_path.touch(exist_ok=True) # Create the file if it doesn't exist - # Optionally, write some default configuration - default_config = { - 'hckr': { - 'version': f'hckr {__version__}' - }, - 'verbose': 3 - } - with config_path.open('w') as config_file: - yaml.dump(default_config, config_file) - - -ensure_config_file(config_path) - - -def yaml_loader(config_file, command): - """Load YAML configuration file.""" - print(config_file, command) - with open(config_file, 'r') as yaml_file: - return yaml.safe_load(yaml_file)[command] - @click.group( context_settings={"help_option_names": ["-h", "--help"]}, invoke_without_command=True, ) -@click_config_file.configuration_option(provider=yaml_loader, implicit=True, - # cmd_name=str(config_path), - config_file_name=str(config_path), default=config_path) @click.option( "--verbose", "-v", @@ -140,6 +104,9 @@ def cli( # NETWORK command cli.add_command(net) +# config +cli.add_command(configure) + # implementing this so that if the user just uses `hckr` we show them something if __name__ == "__main__": cli() diff --git a/src/hckr/cli/configure.py b/src/hckr/cli/configure.py index 2678d3a..4deb6e7 100644 --- a/src/hckr/cli/configure.py +++ b/src/hckr/cli/configure.py @@ -6,8 +6,8 @@ from cron_descriptor import get_description # type: ignore from rich.panel import Panel -from ..utils import ConfigUtils, MessageUtils -from ..utils.ConfigUtils import load_config, config_path, ensure_config_file, DEFAULT_CONFIG, configMessage +from ..utils.ConfigUtils import load_config, config_path, ensure_config_file, DEFAULT_CONFIG, configMessage, \ + list_config, set_config_value, get_config_value @click.group( @@ -44,7 +44,7 @@ def set(config, key, value): $ cli_tool configure set database_host 127.0.0.1 """ configMessage(config) - ConfigUtils.set_config_value(config, key, value) + set_config_value(config, key, value) rich.print( Panel( f"[{config}] {key} <- {value}", @@ -53,6 +53,7 @@ def set(config, key, value): ) ) + @configure.command() @common_config_options @click.argument('key') @@ -60,7 +61,7 @@ def get(config, key): """Get a configuration value.""" configMessage(config) try: - value = ConfigUtils.get_config_value(config, key) + value = get_config_value(config, key) rich.print( Panel( f"[{config}] {key} = {value}", @@ -80,7 +81,30 @@ def get(config, key): @configure.command() @common_config_options -def show(config): +@click.option( + "-a", + "--all", + default=False, + is_flag=True, + help="Whether to show all configs (default: False)" +) +def show(config, all): """List configuration values.""" - configMessage(config) - ConfigUtils.list_config(config) + list_config(config, all) + + +@configure.command('db') +@click.option('--host', prompt=True, help='Database host') +@click.option('--port', prompt=True, help='Database port') +@click.option('--user', prompt=True, help='Database user') +@click.option('--password', prompt=True, hide_input=True, confirmation_prompt=True, help='Database password') +@click.option('--dbname', prompt=True, help='Database name') +@click.pass_context +def configure_db(ctx, host, port, user, password, dbname): + """Configure database credentials.""" + set_config_value('database', 'host', host) + set_config_value('database', 'port', port) + set_config_value('database', 'user', user) + set_config_value('database', 'password', password) + set_config_value('database', 'dbname', dbname) + click.echo("Database configuration saved successfully.") diff --git a/src/hckr/utils/ConfigUtils.py b/src/hckr/utils/ConfigUtils.py index ea67a46..36b7037 100644 --- a/src/hckr/utils/ConfigUtils.py +++ b/src/hckr/utils/ConfigUtils.py @@ -1,6 +1,7 @@ import configparser import logging from pathlib import Path +from readline import set_completer import click import rich @@ -11,7 +12,7 @@ # Define the default configuration file path, this can't be changed, although user can have multile instances using --config config_path = Path.home() / ".hckrcfg" -DEFAULT_CONFIG = "HCKR" +DEFAULT_CONFIG = "DEFAULT" def load_config(): @@ -95,7 +96,35 @@ def get_config_value(section, key) -> str: return config.get(section, key) -def list_config(section): +def show_config(config, section): + if section == DEFAULT_CONFIG: + rich.print( + Panel( + "\n".join([f"{key} = {value}" for key, value in config.items('DEFAULT')]) if config.items( + 'DEFAULT') else "NOTHING FOUND", + expand=True, + title=f"\[DEFAULT]", + ) + ) + elif config.has_section(section): + rich.print( + Panel( + "\n".join([f"{key} = {value}" for key, value in config.items(section)]) if config.items( + section) else "NOTHING FOUND", + expand=True, + title=f"\[{section}]", + ) + ) + else: + rich.print( + Panel( + f"config {section} not found", + expand=True, + title="Error", + ) + ) + +def list_config(section, all=False): """ List Config @@ -105,17 +134,14 @@ def list_config(section): :return: None """ config = load_config() - if config.has_section(section): - rich.print( - Panel( - "\n".join([f"{key} = {value}" for key, value in config.items(section)]) if config.items( - section) else "NOTHING FOUND", - expand=True, - title="HCKR", - ) - ) + if all: + MessageUtils.info("Listing all config") + show_config(config, DEFAULT_CONFIG) + for section in config.sections(): + show_config(config, section) else: - MessageUtils.warning(f"Config '{section}' not found.") + configMessage(section) + show_config(config, section) def configMessage(config): @@ -123,3 +149,24 @@ def configMessage(config): MessageUtils.info(f"Using default config: [magenta]{DEFAULT_CONFIG}") else: MessageUtils.info(f"Using config: [magenta]{config}") + + +# Function to retrieve database credentials +def get_db_creds(section): + config = load_config() + try: + host = config.get(section, 'host') + port = config.get(section, 'port') + user = config.get(section, 'user') + password = config.get(section, 'password') + dbname = config.get(section, 'dbname') + return { + 'host': host, + 'port': port, + 'user': user, + 'password': password, + 'dbname': dbname + } + except (configparser.NoSectionError, configparser.NoOptionError) as e: + click.echo(f"Error: {e}", err=True) + return None From 3d13dd29c39553810bfe18c3e7897f4c3ac8ebf5 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Sat, 24 Aug 2024 10:43:17 +0530 Subject: [PATCH 04/21] Add database utilities and commands Introduced `DbUtils` for handling database URLs based on configuration and added database command functionalities. Updated existing configurations to integrate with the new database utility functions and commands. Enhanced test coverage and consistency in config handling. --- pyproject.toml | 9 +++-- src/hckr/cli/__init__.py | 6 ++-- src/hckr/cli/configure.py | 65 ++++++++++++++++++++++++----------- src/hckr/cli/db.py | 40 +++++++++++++++++++++ src/hckr/utils/ConfigUtils.py | 45 +++++++++++++++--------- src/hckr/utils/DbUtils.py | 54 +++++++++++++++++++++++++++++ tests/cli/test_configure.py | 7 ++-- 7 files changed, 177 insertions(+), 49 deletions(-) create mode 100644 src/hckr/cli/db.py create mode 100644 src/hckr/utils/DbUtils.py diff --git a/pyproject.toml b/pyproject.toml index c4d54ec..367e62d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,9 +43,12 @@ dependencies = [ "kubernetes", # for k8s commands "yaspin", "speedtest-cli", # for net speed command - # YAML config support - "click-config-file", - "pyyaml", + + # SQL Support + "sqlalchemy", # for SQL ORM + "psycopg2", # for Postgres + "pymysql", # for MySQL + "sqlite", # for SQLITE ] [project.urls] diff --git a/src/hckr/cli/__init__.py b/src/hckr/cli/__init__.py index c84d010..5d004f8 100644 --- a/src/hckr/cli/__init__.py +++ b/src/hckr/cli/__init__.py @@ -58,9 +58,9 @@ def __init__(self): # Note: This object must have an empty constructor. @click.pass_context @pass_info def cli( - _info: Info, - ctx: click.Context, - verbose: int, + _info: Info, + ctx: click.Context, + verbose: int, ): if verbose > 0: logging.basicConfig( diff --git a/src/hckr/cli/configure.py b/src/hckr/cli/configure.py index 4deb6e7..0205c08 100644 --- a/src/hckr/cli/configure.py +++ b/src/hckr/cli/configure.py @@ -6,8 +6,17 @@ from cron_descriptor import get_description # type: ignore from rich.panel import Panel -from ..utils.ConfigUtils import load_config, config_path, ensure_config_file, DEFAULT_CONFIG, configMessage, \ - list_config, set_config_value, get_config_value +from ..utils import MessageUtils +from ..utils.ConfigUtils import ( + load_config, + config_path, + ensure_config_file, + DEFAULT_CONFIG, + configMessage, + list_config, + set_config_value, + get_config_value, +) @click.group( @@ -23,14 +32,19 @@ def configure(ctx): def common_config_options(func): - func = click.option("-c", "--config", help="Config instance, default: DEFAULT", default=DEFAULT_CONFIG)(func) + func = click.option( + "-c", + "--config", + help="Config instance, default: DEFAULT", + default=DEFAULT_CONFIG, + )(func) return func @configure.command() @common_config_options -@click.argument('key') -@click.argument('value') +@click.argument("key") +@click.argument("value") def set(config, key, value): """ Sets a configuration value. @@ -56,7 +70,7 @@ def set(config, key, value): @configure.command() @common_config_options -@click.argument('key') +@click.argument("key") def get(config, key): """Get a configuration value.""" configMessage(config) @@ -86,25 +100,34 @@ def get(config, key): "--all", default=False, is_flag=True, - help="Whether to show all configs (default: False)" + help="Whether to show all configs (default: False)", ) def show(config, all): """List configuration values.""" list_config(config, all) -@configure.command('db') -@click.option('--host', prompt=True, help='Database host') -@click.option('--port', prompt=True, help='Database port') -@click.option('--user', prompt=True, help='Database user') -@click.option('--password', prompt=True, hide_input=True, confirmation_prompt=True, help='Database password') -@click.option('--dbname', prompt=True, help='Database name') -@click.pass_context -def configure_db(ctx, host, port, user, password, dbname): +@configure.command("db") +@click.option("--config_name", prompt=True, help="Name of the config instance") +@click.option("--host", prompt=True, help="Database host") +@click.option("--port", prompt=True, help="Database port") +@click.option("--user", prompt=True, help="Database user") +@click.option( + "--password", + prompt=True, + hide_input=True, + confirmation_prompt=True, + help="Database password", +) +@click.option("--dbname", prompt=True, help="Database name") +def configure_db(config_name, host, port, user, password, dbname): """Configure database credentials.""" - set_config_value('database', 'host', host) - set_config_value('database', 'port', port) - set_config_value('database', 'user', user) - set_config_value('database', 'password', password) - set_config_value('database', 'dbname', dbname) - click.echo("Database configuration saved successfully.") + set_config_value(config_name, "host", host) + set_config_value(config_name, "port", port) + set_config_value(config_name, "user", user) + set_config_value(config_name, "password", password) + set_config_value(config_name, "dbname", dbname) + MessageUtils.success( + f"Database configuration saved successfully in config instance {config_name}" + ) + list_config(config_name) diff --git a/src/hckr/cli/db.py b/src/hckr/cli/db.py new file mode 100644 index 0000000..5a9f12c --- /dev/null +++ b/src/hckr/cli/db.py @@ -0,0 +1,40 @@ +import click +import rich +import speedtest # type: ignore +from rich.panel import Panel +from sqlalchemy import create_engine, text +from sqlalchemy.exc import SQLAlchemyError +from yaspin import yaspin # type: ignore + +from hckr.cli.configure import common_config_options +from hckr.utils import NetUtils, MessageUtils +from hckr.utils.DbUtils import get_db_url +from hckr.utils.NetUtils import get_ip_addresses + + +@click.group( + help="Database commands", + context_settings={"help_option_names": ["-h", "--help"]}, +) +def db(): + pass + + +@db.command() +@common_config_options +@click.argument('query') +@click.pass_context +def query(ctx, config, query): + """Execute a SQL query.""" + db_url = get_db_url(section=config) + if db_url: + engine = create_engine(db_url) + try: + with engine.connect() as connection: + result = connection.execute(text(query)) + for row in result: + click.echo(row) + except SQLAlchemyError as e: + click.echo(f"Error executing query: {e}", err=True) + else: + click.echo("Database credentials are not properly configured.", err=True) diff --git a/src/hckr/utils/ConfigUtils.py b/src/hckr/utils/ConfigUtils.py index 36b7037..52d3114 100644 --- a/src/hckr/utils/ConfigUtils.py +++ b/src/hckr/utils/ConfigUtils.py @@ -40,12 +40,12 @@ def ensure_config_file(): config_path.touch(exist_ok=True) default_config = { DEFAULT_CONFIG: { - 'version': f"{__version__}", + "version": f"{__version__}", }, } config = configparser.ConfigParser() config.read_dict(default_config) - with config_path.open('w') as config_file: + with config_path.open("w") as config_file: config.write(config_file) MessageUtils.info(f"Creating default config file {config_path}") else: @@ -76,7 +76,7 @@ def set_config_value(section, key, value): logging.debug(f"Adding section {section}") config.add_section(section) config.set(section, key, value) - with config_path.open('w') as config_file: + with config_path.open("w") as config_file: config.write(config_file) @@ -100,8 +100,13 @@ def show_config(config, section): if section == DEFAULT_CONFIG: rich.print( Panel( - "\n".join([f"{key} = {value}" for key, value in config.items('DEFAULT')]) if config.items( - 'DEFAULT') else "NOTHING FOUND", + ( + "\n".join( + [f"{key} = {value}" for key, value in config.items("DEFAULT")] + ) + if config.items("DEFAULT") + else "NOTHING FOUND" + ), expand=True, title=f"\[DEFAULT]", ) @@ -109,8 +114,13 @@ def show_config(config, section): elif config.has_section(section): rich.print( Panel( - "\n".join([f"{key} = {value}" for key, value in config.items(section)]) if config.items( - section) else "NOTHING FOUND", + ( + "\n".join( + [f"{key} = {value}" for key, value in config.items(section)] + ) + if config.items(section) + else "NOTHING FOUND" + ), expand=True, title=f"\[{section}]", ) @@ -124,6 +134,7 @@ def show_config(config, section): ) ) + def list_config(section, all=False): """ List Config @@ -155,17 +166,17 @@ def configMessage(config): def get_db_creds(section): config = load_config() try: - host = config.get(section, 'host') - port = config.get(section, 'port') - user = config.get(section, 'user') - password = config.get(section, 'password') - dbname = config.get(section, 'dbname') + host = config.get(section, "host") + port = config.get(section, "port") + user = config.get(section, "user") + password = config.get(section, "password") + dbname = config.get(section, "dbname") return { - 'host': host, - 'port': port, - 'user': user, - 'password': password, - 'dbname': dbname + "host": host, + "port": port, + "user": user, + "password": password, + "dbname": dbname, } except (configparser.NoSectionError, configparser.NoOptionError) as e: click.echo(f"Error: {e}", err=True) diff --git a/src/hckr/utils/DbUtils.py b/src/hckr/utils/DbUtils.py new file mode 100644 index 0000000..8b61bf0 --- /dev/null +++ b/src/hckr/utils/DbUtils.py @@ -0,0 +1,54 @@ +from configparser import NoSectionError, NoOptionError + +from sqlalchemy import create_engine, text +from sqlalchemy.exc import SQLAlchemyError +import click + +from hckr.utils import ConfigUtils + + +def get_db_url(section): + """ + This function retrieves a database URL based on the given configuration section. + :param section: The name of the configuration section to retrieve the database URL from. + :return: The database URL. + """ + config = ConfigUtils.load_config() + try: + db_type = config.get(section, 'type') + + if db_type == 'sqlite': + dbname = config.get(section, 'dbname') + return f"sqlite:///{dbname}" + + elif db_type == 'postgresql': + user = config.get(section, 'user') + password = config.get(section, 'password') + host = config.get(section, 'host') + port = config.get(section, 'port') + dbname = config.get(section, 'dbname') + return f"postgresql://{user}:{password}@{host}:{port}/{dbname}" + + elif db_type == 'mysql': + user = config.get(section, 'user') + password = config.get(section, 'password') + host = config.get(section, 'host') + port = config.get(section, 'port') + dbname = config.get(section, 'dbname') + return f"mysql+pymysql://{user}:{password}@{host}:{port}/{dbname}" + + elif db_type == 'snowflake': + user = config.get(section, 'user') + password = config.get(section, 'password') + account = config.get(section, 'account') + warehouse = config.get(section, 'warehouse') + role = config.get(section, 'role') + dbname = config.get(section, 'dbname') + return ( + f"snowflake://{user}:{password}@{account}/{dbname}?" + f"warehouse={warehouse}&role={role}" + ) + + except (NoSectionError, NoOptionError) as e: + click.echo(f"Error: {e}", err=True) + return None diff --git a/tests/cli/test_configure.py b/tests/cli/test_configure.py index 5f6483f..f7b0916 100644 --- a/tests/cli/test_configure.py +++ b/tests/cli/test_configure.py @@ -8,7 +8,7 @@ def _get_random_string(length): letters = string.ascii_lowercase - result_str = ''.join(random.choice(letters) for i in range(length)) + result_str = "".join(random.choice(letters) for i in range(length)) return result_str @@ -21,8 +21,5 @@ def test_configure_get_set_default(): assert result.exit_code == 0 print(result.output) - assert ( - f"Set [DEFAULT] {_key} = {_value}" - in result.output - ) + assert f"Set [DEFAULT] {_key} = {_value}" in result.output result = runner.invoke(get, [_key, _value]) From f3a8fd88fd2a931b03b3b55e234bbf2ae3862a65 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Sat, 24 Aug 2024 10:53:12 +0530 Subject: [PATCH 05/21] Add config command group and enhance database configuration Introduce a new 'config' command group for managing configuration values. This includes implementing commands to set, get, and show configuration values. Additionally, enhanced the database configuration options to include database type selection and more detailed prompts. --- src/hckr/cli/__init__.py | 2 + src/hckr/cli/config.py | 107 +++++++++++++++++++++++++++++++++ src/hckr/cli/configure.py | 108 +++++++--------------------------- src/hckr/utils/ConfigUtils.py | 16 +++++ 4 files changed, 146 insertions(+), 87 deletions(-) create mode 100644 src/hckr/cli/config.py diff --git a/src/hckr/cli/__init__.py b/src/hckr/cli/__init__.py index 5d004f8..494109c 100644 --- a/src/hckr/cli/__init__.py +++ b/src/hckr/cli/__init__.py @@ -7,6 +7,7 @@ from click_repl import register_repl # type: ignore from hckr.cli.configure import configure +from hckr.cli.config import config from hckr.cli.k8s.context import context from hckr.cli.k8s.namespace import namespace from hckr.cli.k8s.pod import pod @@ -105,6 +106,7 @@ def cli( cli.add_command(net) # config +cli.add_command(config) cli.add_command(configure) # implementing this so that if the user just uses `hckr` we show them something diff --git a/src/hckr/cli/config.py b/src/hckr/cli/config.py new file mode 100644 index 0000000..2fef65e --- /dev/null +++ b/src/hckr/cli/config.py @@ -0,0 +1,107 @@ +# from ..utils.MessageUtils import * +import logging + +import click +import rich +from cron_descriptor import get_description # type: ignore +from rich.panel import Panel + +from ..utils import MessageUtils +from ..utils.ConfigUtils import ( + load_config, + config_path, + ensure_config_file, + DEFAULT_CONFIG, + configMessage, + list_config, + set_config_value, + get_config_value, db_type_mapping, +) + + +@click.group( + help="Config commands", + context_settings={"help_option_names": ["-h", "--help"]}, +) +@click.pass_context +def config(ctx): + """ + Defines a command group for configuration-related commands. + """ + ensure_config_file() + + +def common_config_options(func): + func = click.option( + "-c", + "--config", + help="Config instance, default: DEFAULT", + default=DEFAULT_CONFIG, + )(func) + return func + + +@config.command() +@common_config_options +@click.argument("key") +@click.argument("value") +def set(config, key, value): + """ + Sets a configuration value. + + Args: + config (str): The configuration instance name. Default is defined by DEFAULT_CONFIG. + key (str): The key of the config setting to change. + value (str): The value to set for the specified key. + + Example: + $ cli_tool configure set database_host 127.0.0.1 + """ + configMessage(config) + set_config_value(config, key, value) + rich.print( + Panel( + f"[{config}] {key} <- {value}", + expand=True, + title="Success", + ) + ) + + +@config.command() +@common_config_options +@click.argument("key") +def get(config, key): + """Get a configuration value.""" + configMessage(config) + try: + value = get_config_value(config, key) + rich.print( + Panel( + f"[{config}] {key} = {value}", + expand=True, + title="Success", + ) + ) + except ValueError as e: + rich.print( + Panel( + f"{e}", + expand=True, + title="Error", + ) + ) + + +@config.command() +@common_config_options +@click.option( + "-a", + "--all", + default=False, + is_flag=True, + help="Whether to show all configs (default: False)", +) +def show(config, all): + """List configuration values.""" + list_config(config, all) diff --git a/src/hckr/cli/configure.py b/src/hckr/cli/configure.py index 0205c08..fab3dda 100644 --- a/src/hckr/cli/configure.py +++ b/src/hckr/cli/configure.py @@ -6,6 +6,7 @@ from cron_descriptor import get_description # type: ignore from rich.panel import Panel +from .config import common_config_options from ..utils import MessageUtils from ..utils.ConfigUtils import ( load_config, @@ -15,12 +16,12 @@ configMessage, list_config, set_config_value, - get_config_value, + get_config_value, db_type_mapping, ) @click.group( - help="Config commands", + help="easy configurations for other commands (eg. db)", context_settings={"help_option_names": ["-h", "--help"]}, ) @click.pass_context @@ -31,103 +32,36 @@ def configure(ctx): ensure_config_file() -def common_config_options(func): - func = click.option( - "-c", - "--config", - help="Config instance, default: DEFAULT", - default=DEFAULT_CONFIG, - )(func) - return func - - -@configure.command() -@common_config_options -@click.argument("key") -@click.argument("value") -def set(config, key, value): - """ - Sets a configuration value. - - Args: - config (str): The configuration instance name. Default is defined by DEFAULT_CONFIG. - key (str): The key of the config setting to change. - value (str): The value to set for the specified key. - - Example: - $ cli_tool configure set database_host 127.0.0.1 - """ - configMessage(config) - set_config_value(config, key, value) - rich.print( - Panel( - f"[{config}] {key} <- {value}", - expand=True, - title="Success", - ) - ) - - -@configure.command() -@common_config_options -@click.argument("key") -def get(config, key): - """Get a configuration value.""" - configMessage(config) - try: - value = get_config_value(config, key) - rich.print( - Panel( - f"[{config}] {key} = {value}", - expand=True, - title="Success", - ) - ) - except ValueError as e: - rich.print( - Panel( - f"{e}", - expand=True, - title="Error", - ) - ) - - -@configure.command() -@common_config_options +@configure.command("db") +@click.option("--config_name", prompt="Enter a name for this database configuration", + help="Name of the config instance") @click.option( - "-a", - "--all", - default=False, - is_flag=True, - help="Whether to show all configs (default: False)", + "--db_type", + prompt="Select the type of database (1=PostgreSQL, 2=MySQL, 3=SQLite, 4=Snowflake)", + type=click.Choice(["1", "2", "3", "4"]), + help="Database type", ) -def show(config, all): - """List configuration values.""" - list_config(config, all) - - -@configure.command("db") -@click.option("--config_name", prompt=True, help="Name of the config instance") -@click.option("--host", prompt=True, help="Database host") -@click.option("--port", prompt=True, help="Database port") -@click.option("--user", prompt=True, help="Database user") +@click.option("--host", prompt="Enter the database host (e.g., localhost, 127.0.0.1)", help="Database host") +@click.option("--port", prompt="Enter the database port (e.g., 5432)", help="Database port") +@click.option("--user", prompt="Enter the database user (e.g., root)", help="Database user") @click.option( "--password", - prompt=True, + prompt="Enter the database password (input hidden)", hide_input=True, confirmation_prompt=True, help="Database password", ) -@click.option("--dbname", prompt=True, help="Database name") -def configure_db(config_name, host, port, user, password, dbname): +@click.option("--dbname", prompt="Enter the name of the database", help="Database name") +def configure_db(config_name, db_type, host, port, user, password, dbname): """Configure database credentials.""" + + set_config_value(config_name, "db_type", db_type_mapping[db_type]) set_config_value(config_name, "host", host) set_config_value(config_name, "port", port) set_config_value(config_name, "user", user) set_config_value(config_name, "password", password) set_config_value(config_name, "dbname", dbname) - MessageUtils.success( - f"Database configuration saved successfully in config instance {config_name}" - ) + + MessageUtils.success(f"Database configuration saved successfully in config instance '{config_name}'") list_config(config_name) + MessageUtils.success(f"Now you can use config {config_name}, using -c/--config in hckr db commands") diff --git a/src/hckr/utils/ConfigUtils.py b/src/hckr/utils/ConfigUtils.py index 52d3114..80c79a9 100644 --- a/src/hckr/utils/ConfigUtils.py +++ b/src/hckr/utils/ConfigUtils.py @@ -1,5 +1,6 @@ import configparser import logging +from enum import Enum from pathlib import Path from readline import set_completer @@ -15,6 +16,21 @@ DEFAULT_CONFIG = "DEFAULT" +class DBType(str, Enum): + PostgreSQL = ("PostgreSQL",) + MySQL = ("MySQL",) + SQLite = ("SQLite",) + Snowflake = ("Snowflake",) + + +db_type_mapping = { + "1": DBType.PostgreSQL, + "2": DBType.MySQL, + "3": DBType.SQLite, + "4": DBType.Snowflake, +} + + def load_config(): """Load the INI configuration file.""" config = configparser.ConfigParser() From 6147396029f41870c5c9ef0d20fc932e26e42324 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Sat, 24 Aug 2024 14:08:19 +0530 Subject: [PATCH 06/21] Update DB config handling and fix import errors Standardize usage of double quotes in DbUtils.py for consistency. Fix missing imports and improve SQLite handling in config. Adjust pyproject.toml to replace psycopg2 with psycopg2-binary and add snowflake-sqlalchemy. --- pyproject.toml | 5 +- src/hckr/cli/config.py | 3 +- src/hckr/cli/configure.py | 99 ++++++++++++++++++++++++++++++----- src/hckr/cli/db.py | 2 +- src/hckr/utils/ConfigUtils.py | 3 ++ src/hckr/utils/DbUtils.py | 44 ++++++++-------- src/hckr/utils/FileUtils.py | 6 ++- 7 files changed, 120 insertions(+), 42 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 367e62d..b89351e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,9 +46,10 @@ dependencies = [ # SQL Support "sqlalchemy", # for SQL ORM - "psycopg2", # for Postgres + "psycopg2-binary", # for Postgres "pymysql", # for MySQL - "sqlite", # for SQLITE +# "sqlite", # for SQLITE + "snowflake-sqlalchemy", # For Snowflake ] [project.urls] diff --git a/src/hckr/cli/config.py b/src/hckr/cli/config.py index 2fef65e..1af9abc 100644 --- a/src/hckr/cli/config.py +++ b/src/hckr/cli/config.py @@ -15,7 +15,8 @@ configMessage, list_config, set_config_value, - get_config_value, db_type_mapping, + get_config_value, + db_type_mapping, ) diff --git a/src/hckr/cli/configure.py b/src/hckr/cli/configure.py index fab3dda..38892d3 100644 --- a/src/hckr/cli/configure.py +++ b/src/hckr/cli/configure.py @@ -16,7 +16,8 @@ configMessage, list_config, set_config_value, - get_config_value, db_type_mapping, + get_config_value, + db_type_mapping, DBType, ) @@ -32,6 +33,56 @@ def configure(ctx): ensure_config_file() +# @configure.command("db") +# @click.option( +# "--config_name", +# prompt="Enter a name for this database configuration", +# help="Name of the config instance", +# ) +# @click.option( +# "--db_type", +# prompt="Select the type of database (1=PostgreSQL, 2=MySQL, 3=SQLite, 4=Snowflake)", +# type=click.Choice(["1", "2", "3", "4"]), +# help="Database type", +# ) +# @click.option( +# "--host", +# prompt="Enter the database host (e.g., localhost, 127.0.0.1)", +# help="Database host", +# ) +# @click.option( +# "--port", prompt="Enter the database port (e.g., 5432)", help="Database port" +# ) +# @click.option( +# "--user", prompt="Enter the database user (e.g., root)", help="Database user" +# ) +# @click.option( +# "--password", +# prompt="Enter the database password (input hidden)", +# hide_input=True, +# confirmation_prompt=True, +# help="Database password", +# ) +# @click.option("--dbname", prompt="Enter the name of the database", help="Database name") +# def configure_db(config_name, db_type, host, port, user, password, dbname): +# """Configure database credentials.""" +# +# set_config_value(config_name, "db_type", db_type_mapping[db_type]) +# set_config_value(config_name, "host", host) +# set_config_value(config_name, "port", port) +# set_config_value(config_name, "user", user) +# set_config_value(config_name, "password", password) +# set_config_value(config_name, "dbname", dbname) +# +# MessageUtils.success( +# f"Database configuration saved successfully in config instance '{config_name}'" +# ) +# list_config(config_name) +# MessageUtils.success( +# f"Now you can use config {config_name}, using -c/--config in hckr db commands" +# ) + + @configure.command("db") @click.option("--config_name", prompt="Enter a name for this database configuration", help="Name of the config instance") @@ -41,27 +92,47 @@ def configure(ctx): type=click.Choice(["1", "2", "3", "4"]), help="Database type", ) -@click.option("--host", prompt="Enter the database host (e.g., localhost, 127.0.0.1)", help="Database host") -@click.option("--port", prompt="Enter the database port (e.g., 5432)", help="Database port") -@click.option("--user", prompt="Enter the database user (e.g., root)", help="Database user") +@click.option("--host", prompt=False, help="Database host") +@click.option("--port", prompt=False, help="Database port") +@click.option("--user", prompt=False, help="Database user") @click.option( "--password", - prompt="Enter the database password (input hidden)", + prompt=False, hide_input=True, confirmation_prompt=True, help="Database password", ) @click.option("--dbname", prompt="Enter the name of the database", help="Database name") -def configure_db(config_name, db_type, host, port, user, password, dbname): - """Configure database credentials.""" +@click.pass_context +def configure_db(ctx, config_name, db_type, host, port, user, password, dbname): + """Configure database credentials based on the selected database type.""" + selected_db_type = db_type_mapping[db_type] + # set_config_value(config_name, "db_type", str(selected_db_type)) + set_config_value(config_name, "type", selected_db_type) + + if selected_db_type in [DBType.PostgreSQL, DBType.MySQL, DBType.Snowflake]: + if not host: + host = click.prompt("Enter the database host (e.g., localhost, 127.0.0.1)") + if not port: + port = click.prompt("Enter the database port (e.g., 5432 for PostgreSQL)") + if not user: + user = click.prompt("Enter the database user (e.g., root)") + if not password: + password = click.prompt("Enter the database password (input hidden)", hide_input=True, + confirmation_prompt=True) + + set_config_value(config_name, "host", host) + set_config_value(config_name, "port", port) + set_config_value(config_name, "user", user) + set_config_value(config_name, "password", password) - set_config_value(config_name, "db_type", db_type_mapping[db_type]) - set_config_value(config_name, "host", host) - set_config_value(config_name, "port", port) - set_config_value(config_name, "user", user) - set_config_value(config_name, "password", password) - set_config_value(config_name, "dbname", dbname) + if selected_db_type == "SQLite": + # SQLite only requires a database file path + if not dbname: + dbname = click.prompt("Enter the path to the SQLite database file") + set_config_value(config_name, "dbname", dbname) + else: + set_config_value(config_name, "dbname", dbname) MessageUtils.success(f"Database configuration saved successfully in config instance '{config_name}'") list_config(config_name) - MessageUtils.success(f"Now you can use config {config_name}, using -c/--config in hckr db commands") diff --git a/src/hckr/cli/db.py b/src/hckr/cli/db.py index 5a9f12c..6efa37e 100644 --- a/src/hckr/cli/db.py +++ b/src/hckr/cli/db.py @@ -22,7 +22,7 @@ def db(): @db.command() @common_config_options -@click.argument('query') +@click.argument("query") @click.pass_context def query(ctx, config, query): """Execute a SQL query.""" diff --git a/src/hckr/utils/ConfigUtils.py b/src/hckr/utils/ConfigUtils.py index 80c79a9..6602db2 100644 --- a/src/hckr/utils/ConfigUtils.py +++ b/src/hckr/utils/ConfigUtils.py @@ -22,6 +22,9 @@ class DBType(str, Enum): SQLite = ("SQLite",) Snowflake = ("Snowflake",) + def __str__(self): + return self.value + db_type_mapping = { "1": DBType.PostgreSQL, diff --git a/src/hckr/utils/DbUtils.py b/src/hckr/utils/DbUtils.py index 8b61bf0..f7d4812 100644 --- a/src/hckr/utils/DbUtils.py +++ b/src/hckr/utils/DbUtils.py @@ -15,35 +15,35 @@ def get_db_url(section): """ config = ConfigUtils.load_config() try: - db_type = config.get(section, 'type') + db_type = config.get(section, "type") - if db_type == 'sqlite': - dbname = config.get(section, 'dbname') + if db_type == "sqlite": + dbname = config.get(section, "dbname") return f"sqlite:///{dbname}" - elif db_type == 'postgresql': - user = config.get(section, 'user') - password = config.get(section, 'password') - host = config.get(section, 'host') - port = config.get(section, 'port') - dbname = config.get(section, 'dbname') + elif db_type == "postgresql": + user = config.get(section, "user") + password = config.get(section, "password") + host = config.get(section, "host") + port = config.get(section, "port") + dbname = config.get(section, "dbname") return f"postgresql://{user}:{password}@{host}:{port}/{dbname}" - elif db_type == 'mysql': - user = config.get(section, 'user') - password = config.get(section, 'password') - host = config.get(section, 'host') - port = config.get(section, 'port') - dbname = config.get(section, 'dbname') + elif db_type == "mysql": + user = config.get(section, "user") + password = config.get(section, "password") + host = config.get(section, "host") + port = config.get(section, "port") + dbname = config.get(section, "dbname") return f"mysql+pymysql://{user}:{password}@{host}:{port}/{dbname}" - elif db_type == 'snowflake': - user = config.get(section, 'user') - password = config.get(section, 'password') - account = config.get(section, 'account') - warehouse = config.get(section, 'warehouse') - role = config.get(section, 'role') - dbname = config.get(section, 'dbname') + elif db_type == "snowflake": + user = config.get(section, "user") + password = config.get(section, "password") + account = config.get(section, "account") + warehouse = config.get(section, "warehouse") + role = config.get(section, "role") + dbname = config.get(section, "dbname") return ( f"snowflake://{user}:{password}@{account}/{dbname}?" f"warehouse={warehouse}&role={role}" diff --git a/src/hckr/utils/FileUtils.py b/src/hckr/utils/FileUtils.py index 90590a3..5e59d84 100644 --- a/src/hckr/utils/FileUtils.py +++ b/src/hckr/utils/FileUtils.py @@ -2,6 +2,8 @@ import os from pathlib import Path +from pyarrow._dataset import FileFormat + from hckr.utils.MessageUtils import error, info, colored, warning from enum import Enum @@ -17,7 +19,7 @@ class FileFormat(str, Enum): INVALID = "invalid" @staticmethod - def fileExtToFormat(file_path, file_extension): + def fileExtToFormat(file_path, file_extension) -> FileFormat: file_type_extension_map = { ".txt": FileFormat.TXT, ".text": FileFormat.TXT, @@ -120,7 +122,7 @@ def validate_file_extension(file_path, expected_extensions): # validate if file extension is one of given -def get_file_format_from_extension(file_path): +def get_file_format_from_extension(file_path) -> FileFormat: if not file_path: return FileFormat.INVALID # Extract the extension from the file path From af468765d8cb9b43db4e406d82e56cd1e1bec35c Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Sat, 24 Aug 2024 14:39:25 +0530 Subject: [PATCH 07/21] Add initialization command for config and refactor tests Introduced a new `init` command to create a configuration file if it doesn't exist. Refactored the configuration tests to separate default and custom configurations and added missing key/value test cases. Adjusted config handling to ensure the presence of a config file before setting up databases. --- src/hckr/cli/config.py | 19 +++++----- src/hckr/cli/configure.py | 65 +++++------------------------------ src/hckr/utils/ConfigUtils.py | 11 ++++-- tests/cli/test_configure.py | 63 +++++++++++++++++++++++++++++---- 4 files changed, 85 insertions(+), 73 deletions(-) diff --git a/src/hckr/cli/config.py b/src/hckr/cli/config.py index 1af9abc..d1fb93b 100644 --- a/src/hckr/cli/config.py +++ b/src/hckr/cli/config.py @@ -1,22 +1,17 @@ # from ..utils.MessageUtils import * -import logging import click import rich from cron_descriptor import get_description # type: ignore from rich.panel import Panel -from ..utils import MessageUtils from ..utils.ConfigUtils import ( - load_config, - config_path, - ensure_config_file, + init_config, DEFAULT_CONFIG, configMessage, list_config, set_config_value, get_config_value, - db_type_mapping, ) @@ -29,8 +24,7 @@ def config(ctx): """ Defines a command group for configuration-related commands. """ - ensure_config_file() - + pass def common_config_options(func): func = click.option( @@ -106,3 +100,12 @@ def get(config, key): def show(config, all): """List configuration values.""" list_config(config, all) + +@config.command() +def init(): + """ + Creates configuration file ~/.hckrcfg if it does not exist + + :return: None + """ + init_config() diff --git a/src/hckr/cli/configure.py b/src/hckr/cli/configure.py index 38892d3..665f71c 100644 --- a/src/hckr/cli/configure.py +++ b/src/hckr/cli/configure.py @@ -11,13 +11,13 @@ from ..utils.ConfigUtils import ( load_config, config_path, - ensure_config_file, + init_config, DEFAULT_CONFIG, configMessage, list_config, set_config_value, get_config_value, - db_type_mapping, DBType, + db_type_mapping, DBType, check_config, ) @@ -30,57 +30,9 @@ def configure(ctx): """ Defines a command group for configuration-related commands. """ - ensure_config_file() - - -# @configure.command("db") -# @click.option( -# "--config_name", -# prompt="Enter a name for this database configuration", -# help="Name of the config instance", -# ) -# @click.option( -# "--db_type", -# prompt="Select the type of database (1=PostgreSQL, 2=MySQL, 3=SQLite, 4=Snowflake)", -# type=click.Choice(["1", "2", "3", "4"]), -# help="Database type", -# ) -# @click.option( -# "--host", -# prompt="Enter the database host (e.g., localhost, 127.0.0.1)", -# help="Database host", -# ) -# @click.option( -# "--port", prompt="Enter the database port (e.g., 5432)", help="Database port" -# ) -# @click.option( -# "--user", prompt="Enter the database user (e.g., root)", help="Database user" -# ) -# @click.option( -# "--password", -# prompt="Enter the database password (input hidden)", -# hide_input=True, -# confirmation_prompt=True, -# help="Database password", -# ) -# @click.option("--dbname", prompt="Enter the name of the database", help="Database name") -# def configure_db(config_name, db_type, host, port, user, password, dbname): -# """Configure database credentials.""" -# -# set_config_value(config_name, "db_type", db_type_mapping[db_type]) -# set_config_value(config_name, "host", host) -# set_config_value(config_name, "port", port) -# set_config_value(config_name, "user", user) -# set_config_value(config_name, "password", password) -# set_config_value(config_name, "dbname", dbname) -# -# MessageUtils.success( -# f"Database configuration saved successfully in config instance '{config_name}'" -# ) -# list_config(config_name) -# MessageUtils.success( -# f"Now you can use config {config_name}, using -c/--config in hckr db commands" -# ) + if not check_config(): + MessageUtils.warning("Config file doesn't exists, Please run \n hckr config init " + "\n to create one") @configure.command("db") @@ -97,7 +49,7 @@ def configure(ctx): @click.option("--user", prompt=False, help="Database user") @click.option( "--password", - prompt=False, + prompt=False, # we will get this value later if it is not provided hide_input=True, confirmation_prompt=True, help="Database password", @@ -107,7 +59,6 @@ def configure(ctx): def configure_db(ctx, config_name, db_type, host, port, user, password, dbname): """Configure database credentials based on the selected database type.""" selected_db_type = db_type_mapping[db_type] - # set_config_value(config_name, "db_type", str(selected_db_type)) set_config_value(config_name, "type", selected_db_type) if selected_db_type in [DBType.PostgreSQL, DBType.MySQL, DBType.Snowflake]: @@ -116,9 +67,9 @@ def configure_db(ctx, config_name, db_type, host, port, user, password, dbname): if not port: port = click.prompt("Enter the database port (e.g., 5432 for PostgreSQL)") if not user: - user = click.prompt("Enter the database user (e.g., root)") + user = click.prompt("Enter the database user (e.g., root, admin, user)") if not password: - password = click.prompt("Enter the database password (input hidden)", hide_input=True, + password = click.prompt("Enter the database password (input hidden)", hide_input=False, confirmation_prompt=True) set_config_value(config_name, "host", host) diff --git a/src/hckr/utils/ConfigUtils.py b/src/hckr/utils/ConfigUtils.py index 6602db2..7c4c1fa 100644 --- a/src/hckr/utils/ConfigUtils.py +++ b/src/hckr/utils/ConfigUtils.py @@ -41,7 +41,11 @@ def load_config(): return config -def ensure_config_file(): +def check_config() -> bool: + return config_path.exists() + + +def init_config(): """ Ensures the existence of a configuration file at the specified path. @@ -61,6 +65,9 @@ def ensure_config_file(): DEFAULT_CONFIG: { "version": f"{__version__}", }, + "CUSTOM": { + "key": f"value", + }, } config = configparser.ConfigParser() config.read_dict(default_config) @@ -108,7 +115,7 @@ def get_config_value(section, key) -> str: """ logging.debug(f"Getting [{section}] {key} ") config = load_config() - if not config.has_section(section): + if section != DEFAULT_CONFIG and not config.has_section(section): raise ValueError(f"Section '{section}' not found in the configuration.") if not config.has_option(section, key): raise ValueError(f"Key '{key}' not found in section '{section}'.") diff --git a/tests/cli/test_configure.py b/tests/cli/test_configure.py index f7b0916..07bef42 100644 --- a/tests/cli/test_configure.py +++ b/tests/cli/test_configure.py @@ -1,9 +1,9 @@ -import string -from distutils.command.config import config import random +import string + from click.testing import CliRunner -from hckr.cli.configure import set, get +from hckr.cli.config import set, get, show def _get_random_string(length): @@ -12,14 +12,65 @@ def _get_random_string(length): return result_str -# DEFAULT CONFIG GET AND SET +# CONFIG GET AND SET def test_configure_get_set_default(): runner = CliRunner() _key = f"key_{_get_random_string(5)}" _value = f"value_{_get_random_string(5)}" result = runner.invoke(set, [_key, _value]) assert result.exit_code == 0 + assert f"[DEFAULT] {_key} <- {_value}" in result.output + + # testing get + result = runner.invoke(get, [_key]) + assert result.exit_code == 0 + assert f"[DEFAULT] {_key} = {_value}" in result.output + + +def test_configure_get_set_custom_config(): + runner = CliRunner() + _key = f"key_{_get_random_string(5)}" + _value = f"value_{_get_random_string(5)}" + _CONFIG = "CUSTOM" + result = runner.invoke(set, ['--config', _CONFIG, _key, _value]) + assert result.exit_code == 0 + assert f"[{_CONFIG}] {_key} <- {_value}" in result.output + + # testing get + result = runner.invoke(get, ['--config', _CONFIG, _key]) + assert result.exit_code == 0 + assert f"[{_CONFIG}] {_key} = {_value}" in result.output + + +def test_configure_show(): + runner = CliRunner() + _CONFIG = "CUSTOM" + result = runner.invoke(show) + assert result.exit_code == 0 + assert f"[DEFAULT]" in result.output + + result = runner.invoke(show, ['-all']) + + + +# NAGATIVE USE CASES +def test_configure_get_set_missing_key(): + runner = CliRunner() + result = runner.invoke(set, []) + print(result.output) + assert result.exit_code != 0 + assert f"Error: Missing argument 'KEY'" in result.output + + runner = CliRunner() + result = runner.invoke(get, []) print(result.output) + assert result.exit_code != 0 + assert f"Error: Missing argument 'KEY'" in result.output - assert f"Set [DEFAULT] {_key} = {_value}" in result.output - result = runner.invoke(get, [_key, _value]) + +def test_configure_set_missing_value(): + runner = CliRunner() + result = runner.invoke(set, ['key']) + print(result.output) + assert result.exit_code != 0 + assert f"Error: Missing argument 'VALUE'" in result.output From ce692879c5bf8af233df3cf2d1b5923239ae7d8c Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Sat, 24 Aug 2024 18:35:12 +0530 Subject: [PATCH 08/21] Add initialization command for config and refactor tests Introduced a new `init` command to create a configuration file if it doesn't exist. Refactored the configuration tests to separate default and custom configurations and added missing key/value test cases. Adjusted config handling to ensure the presence of a config file before setting up databases. --- src/hckr/cli/config.py | 15 ++++++++++++--- src/hckr/cli/configure.py | 27 ++++++++++++++++++--------- src/hckr/utils/ConfigUtils.py | 22 +++++++++++++++++----- src/hckr/utils/MessageUtils.py | 29 +++++++++++++++++++++++++++++ tests/cli/test_configure.py | 4 +++- 5 files changed, 79 insertions(+), 18 deletions(-) diff --git a/src/hckr/cli/config.py b/src/hckr/cli/config.py index d1fb93b..babbb6e 100644 --- a/src/hckr/cli/config.py +++ b/src/hckr/cli/config.py @@ -26,6 +26,7 @@ def config(ctx): """ pass + def common_config_options(func): func = click.option( "-c", @@ -101,11 +102,19 @@ def show(config, all): """List configuration values.""" list_config(config, all) + @config.command() -def init(): +@click.option( + "-o", + "--overwrite", + default=False, + is_flag=True, + help="Whether to delete and recreate .hckrcfg file (default: False)", +) +def init(overwrite): """ - Creates configuration file ~/.hckrcfg if it does not exist + Initializes the configuration for the application. :return: None """ - init_config() + init_config(overwrite) diff --git a/src/hckr/cli/configure.py b/src/hckr/cli/configure.py index 665f71c..d940ed0 100644 --- a/src/hckr/cli/configure.py +++ b/src/hckr/cli/configure.py @@ -1,5 +1,6 @@ # from ..utils.MessageUtils import * import logging +from symbol import assert_stmt import click import rich @@ -17,7 +18,9 @@ list_config, set_config_value, get_config_value, - db_type_mapping, DBType, check_config, + db_type_mapping, + DBType, + check_config, ) @@ -30,14 +33,15 @@ def configure(ctx): """ Defines a command group for configuration-related commands. """ - if not check_config(): - MessageUtils.warning("Config file doesn't exists, Please run \n hckr config init " - "\n to create one") + pass @configure.command("db") -@click.option("--config_name", prompt="Enter a name for this database configuration", - help="Name of the config instance") +@click.option( + "--config_name", + prompt="Enter a name for this database configuration", + help="Name of the config instance", +) @click.option( "--db_type", prompt="Select the type of database (1=PostgreSQL, 2=MySQL, 3=SQLite, 4=Snowflake)", @@ -69,8 +73,11 @@ def configure_db(ctx, config_name, db_type, host, port, user, password, dbname): if not user: user = click.prompt("Enter the database user (e.g., root, admin, user)") if not password: - password = click.prompt("Enter the database password (input hidden)", hide_input=False, - confirmation_prompt=True) + password = click.prompt( + "Enter the database password (input hidden)", + hide_input=False, + confirmation_prompt=True, + ) set_config_value(config_name, "host", host) set_config_value(config_name, "port", port) @@ -85,5 +92,7 @@ def configure_db(ctx, config_name, db_type, host, port, user, password, dbname): else: set_config_value(config_name, "dbname", dbname) - MessageUtils.success(f"Database configuration saved successfully in config instance '{config_name}'") + MessageUtils.success( + f"Database configuration saved successfully in config instance '{config_name}'" + ) list_config(config_name) diff --git a/src/hckr/utils/ConfigUtils.py b/src/hckr/utils/ConfigUtils.py index 7c4c1fa..45d4505 100644 --- a/src/hckr/utils/ConfigUtils.py +++ b/src/hckr/utils/ConfigUtils.py @@ -7,11 +7,12 @@ import click import rich from rich.panel import Panel - from . import MessageUtils +from .MessageUtils import PWarn, PMsg, PError, PSuccess, PInfo from ..__about__ import __version__ -# Define the default configuration file path, this can't be changed, although user can have multile instances using --config +# Define the default configuration file path, this can't be changed, although user can have multile instances using +# --config config_path = Path.home() / ".hckrcfg" DEFAULT_CONFIG = "DEFAULT" @@ -37,6 +38,11 @@ def __str__(self): def load_config(): """Load the INI configuration file.""" config = configparser.ConfigParser() + if not check_config(): + PWarn( + f"Config file [magenta]{config_path}[/magenta] doesn't exists, Please run init command to create one \n " + f"[bold green]hckr config init") + exit(0) config.read(config_path) return config @@ -45,7 +51,7 @@ def check_config() -> bool: return config_path.exists() -def init_config(): +def init_config(overwrite): """ Ensures the existence of a configuration file at the specified path. @@ -73,9 +79,15 @@ def init_config(): config.read_dict(default_config) with config_path.open("w") as config_file: config.write(config_file) - MessageUtils.info(f"Creating default config file {config_path}") + PSuccess(f"Config file created at {config_path}") + elif overwrite: + PInfo(f"Config file already exists at {config_path}, [magenta]-o/--overwrite[/magenta] passed \n" + f"[yellow]Deleting existing file") + config_path.unlink() + init_config(overwrite=False) else: - logging.debug(f"Config file {config_path} already exists ") + PWarn(f"Config file already exists at [yellow]{config_path}[/yellow]," + f" please pass [magenta]-o/--overwrite[/magenta] to recreate") def set_config_value(section, key, value): diff --git a/src/hckr/utils/MessageUtils.py b/src/hckr/utils/MessageUtils.py index c67ed11..e348841 100644 --- a/src/hckr/utils/MessageUtils.py +++ b/src/hckr/utils/MessageUtils.py @@ -1,8 +1,11 @@ import logging +import rich from rich import print import random +from rich.panel import Panel + def colored(msg, color, bold=True): if bold: @@ -29,6 +32,32 @@ def warning(msg, color=None): print(f"{warn_emoji()} [bold yellow]{msg}[/bold yellow]") +def PMsg(msg, title): + rich.print( + Panel( + msg, + expand=False, + title=title, + ) + ) + + +def PWarn(msg, title="[yellow]Warning"): + PMsg(msg, title) + + +def PSuccess(msg, title="[green]Success"): + PMsg(msg, title) + + +def PInfo(msg, title="[blue]Info"): + PMsg(msg, title) + + +def PError(msg, title="[red]Error"): + PMsg(msg, title) + + def info(msg, color=None): if color: print(f"{info_emoji()} [bold {color}] {msg}[/bold {color}]") diff --git a/tests/cli/test_configure.py b/tests/cli/test_configure.py index 07bef42..1c5de62 100644 --- a/tests/cli/test_configure.py +++ b/tests/cli/test_configure.py @@ -49,7 +49,9 @@ def test_configure_show(): assert result.exit_code == 0 assert f"[DEFAULT]" in result.output - result = runner.invoke(show, ['-all']) + result = runner.invoke(show, ['--all']) + assert f"[DEFAULT]" in result.output + assert f"[CUSTOM]" in result.output From 068dc342d2839f1bbcc55a6cb84a25d3bb340316 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Sat, 24 Aug 2024 20:21:16 +0530 Subject: [PATCH 09/21] Refactor database configuration, enhance tests, add logging Refactored database configuration handling to support multiple DB types and improve error messaging. Enhanced tests to validate configuration logic and renamed test files/methods for better clarity. Added logging and updated CLI command imports for improved code organization. --- src/hckr/cli/__init__.py | 5 ++ src/hckr/cli/configure.py | 70 +++++++++------- src/hckr/cli/db.py | 12 +-- src/hckr/utils/ConfigUtils.py | 44 +++++----- src/hckr/utils/DbUtils.py | 80 +++++++++++-------- .../cli/{test_configure.py => test_config.py} | 33 ++++---- 6 files changed, 135 insertions(+), 109 deletions(-) rename tests/cli/{test_configure.py => test_config.py} (67%) diff --git a/src/hckr/cli/__init__.py b/src/hckr/cli/__init__.py index 494109c..f295c4a 100644 --- a/src/hckr/cli/__init__.py +++ b/src/hckr/cli/__init__.py @@ -6,6 +6,7 @@ import click from click_repl import register_repl # type: ignore +from hckr.cli.db import db from hckr.cli.configure import configure from hckr.cli.config import config from hckr.cli.k8s.context import context @@ -109,6 +110,10 @@ def cli( cli.add_command(config) cli.add_command(configure) +# database +cli.add_command(db) + + # implementing this so that if the user just uses `hckr` we show them something if __name__ == "__main__": cli() diff --git a/src/hckr/cli/configure.py b/src/hckr/cli/configure.py index d940ed0..14d974f 100644 --- a/src/hckr/cli/configure.py +++ b/src/hckr/cli/configure.py @@ -1,27 +1,13 @@ -# from ..utils.MessageUtils import * -import logging -from symbol import assert_stmt - import click -import rich -from cron_descriptor import get_description # type: ignore -from rich.panel import Panel -from .config import common_config_options from ..utils import MessageUtils from ..utils.ConfigUtils import ( - load_config, - config_path, - init_config, - DEFAULT_CONFIG, - configMessage, list_config, set_config_value, - get_config_value, db_type_mapping, - DBType, - check_config, + DBType, ConfigType, ) +from ..utils.MessageUtils import PSuccess @click.group( @@ -58,24 +44,30 @@ def configure(ctx): confirmation_prompt=True, help="Database password", ) -@click.option("--dbname", prompt="Enter the name of the database", help="Database name") -@click.pass_context -def configure_db(ctx, config_name, db_type, host, port, user, password, dbname): +@click.option("--dbname", prompt=False, help="Database name") +@click.option("--schema", prompt=False, help="Database schema") +@click.option("--account", prompt=False, help="Snowflake Account Id") +@click.option("--warehouse", prompt=False, help="Snowflake warehouse") +@click.option("--role", prompt=False, help="Snowflake role") +def configure_db( + config_name, db_type, host, port, user, password, dbname, schema, warehouse, role +): """Configure database credentials based on the selected database type.""" + set_config_value(config_name, "config_type", ConfigType.DATABASE) selected_db_type = db_type_mapping[db_type] - set_config_value(config_name, "type", selected_db_type) + set_config_value(config_name, "database_type", selected_db_type) - if selected_db_type in [DBType.PostgreSQL, DBType.MySQL, DBType.Snowflake]: + if selected_db_type in [DBType.PostgreSQL, DBType.MySQL]: if not host: host = click.prompt("Enter the database host (e.g., localhost, 127.0.0.1)") if not port: port = click.prompt("Enter the database port (e.g., 5432 for PostgreSQL)") if not user: - user = click.prompt("Enter the database user (e.g., root, admin, user)") + user = click.prompt("Enter the database user (e.g., root)") if not password: password = click.prompt( "Enter the database password (input hidden)", - hide_input=False, + hide_input=True, confirmation_prompt=True, ) @@ -83,16 +75,40 @@ def configure_db(ctx, config_name, db_type, host, port, user, password, dbname): set_config_value(config_name, "port", port) set_config_value(config_name, "user", user) set_config_value(config_name, "password", password) + set_config_value(config_name, "dbname", dbname) - if selected_db_type == "SQLite": - # SQLite only requires a database file path + elif selected_db_type == DBType.SQLite: if not dbname: dbname = click.prompt("Enter the path to the SQLite database file") set_config_value(config_name, "dbname", dbname) - else: + + elif selected_db_type == DBType.Snowflake: + if not user: + user = click.prompt("Enter your Snowflake username") + if not password: + password = click.prompt( + "Enter your Snowflake password", + hide_input=True, + confirmation_prompt=True, + ) + if not dbname: + dbname = click.prompt("Enter the Snowflake database name") + if not schema: + schema = click.prompt("Enter the Snowflake schema name") + if not warehouse: + warehouse = click.prompt("Enter the Snowflake warehouse name") + if not role: + role = click.prompt("Enter the Snowflake role") + + set_config_value(config_name, "user", user) + set_config_value(config_name, "password", password) + set_config_value(config_name, "account", account) + set_config_value(config_name, "warehouse", warehouse) set_config_value(config_name, "dbname", dbname) + set_config_value(config_name, "schema", schema) + set_config_value(config_name, "role", role) - MessageUtils.success( + PSuccess( f"Database configuration saved successfully in config instance '{config_name}'" ) list_config(config_name) diff --git a/src/hckr/cli/db.py b/src/hckr/cli/db.py index 6efa37e..492f7dc 100644 --- a/src/hckr/cli/db.py +++ b/src/hckr/cli/db.py @@ -1,15 +1,11 @@ import click -import rich -import speedtest # type: ignore -from rich.panel import Panel from sqlalchemy import create_engine, text from sqlalchemy.exc import SQLAlchemyError from yaspin import yaspin # type: ignore -from hckr.cli.configure import common_config_options -from hckr.utils import NetUtils, MessageUtils +from hckr.cli.config import common_config_options from hckr.utils.DbUtils import get_db_url -from hckr.utils.NetUtils import get_ip_addresses +from hckr.utils.MessageUtils import PError @click.group( @@ -35,6 +31,6 @@ def query(ctx, config, query): for row in result: click.echo(row) except SQLAlchemyError as e: - click.echo(f"Error executing query: {e}", err=True) + PError(f"Error executing query: {e}") else: - click.echo("Database credentials are not properly configured.", err=True) + PError("Database credentials are not properly configured.") diff --git a/src/hckr/utils/ConfigUtils.py b/src/hckr/utils/ConfigUtils.py index 45d4505..240b7cc 100644 --- a/src/hckr/utils/ConfigUtils.py +++ b/src/hckr/utils/ConfigUtils.py @@ -27,6 +27,13 @@ def __str__(self): return self.value +class ConfigType(str, Enum): + DATABASE = ("database",) + + def __str__(self): + return self.value + + db_type_mapping = { "1": DBType.PostgreSQL, "2": DBType.MySQL, @@ -41,8 +48,9 @@ def load_config(): if not check_config(): PWarn( f"Config file [magenta]{config_path}[/magenta] doesn't exists, Please run init command to create one \n " - f"[bold green]hckr config init") - exit(0) + "[bold green]hckr config init" + ) + exit(1) config.read(config_path) return config @@ -70,6 +78,7 @@ def init_config(overwrite): default_config = { DEFAULT_CONFIG: { "version": f"{__version__}", + "config_type": "default", }, "CUSTOM": { "key": f"value", @@ -81,13 +90,17 @@ def init_config(overwrite): config.write(config_file) PSuccess(f"Config file created at {config_path}") elif overwrite: - PInfo(f"Config file already exists at {config_path}, [magenta]-o/--overwrite[/magenta] passed \n" - f"[yellow]Deleting existing file") + PInfo( + f"Config file already exists at {config_path}, [magenta]-o/--overwrite[/magenta] passed \n" + "[yellow]Deleting existing file" + ) config_path.unlink() init_config(overwrite=False) else: - PWarn(f"Config file already exists at [yellow]{config_path}[/yellow]," - f" please pass [magenta]-o/--overwrite[/magenta] to recreate") + PWarn( + f"Config file already exists at [yellow]{config_path}[/yellow]," + " please pass [magenta]-o/--overwrite[/magenta] to recreate" + ) def set_config_value(section, key, value): @@ -200,22 +213,3 @@ def configMessage(config): MessageUtils.info(f"Using config: [magenta]{config}") -# Function to retrieve database credentials -def get_db_creds(section): - config = load_config() - try: - host = config.get(section, "host") - port = config.get(section, "port") - user = config.get(section, "user") - password = config.get(section, "password") - dbname = config.get(section, "dbname") - return { - "host": host, - "port": port, - "user": user, - "password": password, - "dbname": dbname, - } - except (configparser.NoSectionError, configparser.NoOptionError) as e: - click.echo(f"Error: {e}", err=True) - return None diff --git a/src/hckr/utils/DbUtils.py b/src/hckr/utils/DbUtils.py index f7d4812..9cf2084 100644 --- a/src/hckr/utils/DbUtils.py +++ b/src/hckr/utils/DbUtils.py @@ -1,3 +1,4 @@ +import logging from configparser import NoSectionError, NoOptionError from sqlalchemy import create_engine, text @@ -5,6 +6,8 @@ import click from hckr.utils import ConfigUtils +from hckr.utils.ConfigUtils import ConfigType, DBType, load_config +from hckr.utils.MessageUtils import PError def get_db_url(section): @@ -15,40 +18,53 @@ def get_db_url(section): """ config = ConfigUtils.load_config() try: - db_type = config.get(section, "type") + config_type = config.get(section, "config_type") + if config_type != ConfigType.DATABASE: + PError(f"The configuration {section} is not database type\n" + " Please use [magenta]hckr configure db[/magenta] to configure database.") + exit(1) + db_type = config.get(section, "database_type") - if db_type == "sqlite": + if db_type == DBType.SQLite: dbname = config.get(section, "dbname") return f"sqlite:///{dbname}" - elif db_type == "postgresql": - user = config.get(section, "user") - password = config.get(section, "password") - host = config.get(section, "host") - port = config.get(section, "port") - dbname = config.get(section, "dbname") - return f"postgresql://{user}:{password}@{host}:{port}/{dbname}" + elif db_type in [DBType.PostgreSQL, DBType.MySQL]: + return _get_jdbc_url(config, section, db_type) - elif db_type == "mysql": - user = config.get(section, "user") - password = config.get(section, "password") - host = config.get(section, "host") - port = config.get(section, "port") - dbname = config.get(section, "dbname") - return f"mysql+pymysql://{user}:{password}@{host}:{port}/{dbname}" - - elif db_type == "snowflake": - user = config.get(section, "user") - password = config.get(section, "password") - account = config.get(section, "account") - warehouse = config.get(section, "warehouse") - role = config.get(section, "role") - dbname = config.get(section, "dbname") - return ( - f"snowflake://{user}:{password}@{account}/{dbname}?" - f"warehouse={warehouse}&role={role}" - ) - - except (NoSectionError, NoOptionError) as e: - click.echo(f"Error: {e}", err=True) - return None + elif db_type == DBType.Snowflake: + return _get_snowflake_url(config, section) + + except NoOptionError as e: + PError(f"Config {section} is not configured correctly\n {e}") + exit(1) + + +def _get_jdbc_url(config, section, db_type): + user = config.get(section, "user") + password = config.get(section, "password") + host = config.get(section, "host") + port = config.get(section, "port") + dbname = config.get(section, "dbname") + if db_type == DBType.PostgreSQL: + return f"postgresql://{user}:{password}@{host}:{port}/{dbname}" + elif db_type == DBType.MySQL: + return f"mysql+pymysql://{user}:{password}@{host}:{port}/{dbname}" + else: + logging.debug(f"Invalid db_type: {db_type} in get_jdbc_url()") + PError("Error occured while creating JDBC Url") + exit(1) + + +def _get_snowflake_url(config, section): + user = config.get(section, "user") + password = config.get(section, "password") + account = config.get(section, "account") + warehouse = config.get(section, "warehouse") + role = config.get(section, "role") + dbname = config.get(section, "dbname") + schema = config.get(section, 'schema') + return ( + f"snowflake://{user}:{password}@{account}/{dbname}/{schema}" + f"?warehouse={warehouse}&role={role}" + ) diff --git a/tests/cli/test_configure.py b/tests/cli/test_config.py similarity index 67% rename from tests/cli/test_configure.py rename to tests/cli/test_config.py index 1c5de62..7c5b8c9 100644 --- a/tests/cli/test_configure.py +++ b/tests/cli/test_config.py @@ -13,7 +13,7 @@ def _get_random_string(length): # CONFIG GET AND SET -def test_configure_get_set_default(): +def test_config_get_set_default(): runner = CliRunner() _key = f"key_{_get_random_string(5)}" _value = f"value_{_get_random_string(5)}" @@ -27,52 +27,51 @@ def test_configure_get_set_default(): assert f"[DEFAULT] {_key} = {_value}" in result.output -def test_configure_get_set_custom_config(): +def test_config_get_set_custom_config(): runner = CliRunner() _key = f"key_{_get_random_string(5)}" _value = f"value_{_get_random_string(5)}" _CONFIG = "CUSTOM" - result = runner.invoke(set, ['--config', _CONFIG, _key, _value]) + result = runner.invoke(set, ["--config", _CONFIG, _key, _value]) assert result.exit_code == 0 assert f"[{_CONFIG}] {_key} <- {_value}" in result.output # testing get - result = runner.invoke(get, ['--config', _CONFIG, _key]) + result = runner.invoke(get, ["--config", _CONFIG, _key]) assert result.exit_code == 0 assert f"[{_CONFIG}] {_key} = {_value}" in result.output -def test_configure_show(): +def test_config_show(): runner = CliRunner() _CONFIG = "CUSTOM" result = runner.invoke(show) assert result.exit_code == 0 - assert f"[DEFAULT]" in result.output + assert "[DEFAULT]" in result.output - result = runner.invoke(show, ['--all']) - assert f"[DEFAULT]" in result.output - assert f"[CUSTOM]" in result.output + result = runner.invoke(show, ["--all"]) + assert "[DEFAULT]" in result.output + assert "[CUSTOM]" in result.output - -# NAGATIVE USE CASES -def test_configure_get_set_missing_key(): +# NEGATIVE USE CASES +def test_config_get_set_missing_key(): runner = CliRunner() result = runner.invoke(set, []) print(result.output) assert result.exit_code != 0 - assert f"Error: Missing argument 'KEY'" in result.output + assert "Error: Missing argument 'KEY'" in result.output runner = CliRunner() result = runner.invoke(get, []) print(result.output) assert result.exit_code != 0 - assert f"Error: Missing argument 'KEY'" in result.output + assert "Error: Missing argument 'KEY'" in result.output -def test_configure_set_missing_value(): +def test_config_set_missing_value(): runner = CliRunner() - result = runner.invoke(set, ['key']) + result = runner.invoke(set, ["key"]) print(result.output) assert result.exit_code != 0 - assert f"Error: Missing argument 'VALUE'" in result.output + assert "Error: Missing argument 'VALUE'" in result.output From bbf4db8b0e6d0004a370cf86a5cc63c11bbd85d3 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Sat, 24 Aug 2024 20:24:05 +0530 Subject: [PATCH 10/21] Refactor query command to return and display DataFrame Refactor the `query` command to execute SQL on Snowflake and return results as a DataFrame. Added imports for `config_path`, `print_df_as_table`, and `pandas`. Replaced row-by-row output with a formatted DataFrame table display. --- src/hckr/cli/db.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/hckr/cli/db.py b/src/hckr/cli/db.py index 492f7dc..a1a4a82 100644 --- a/src/hckr/cli/db.py +++ b/src/hckr/cli/db.py @@ -4,9 +4,11 @@ from yaspin import yaspin # type: ignore from hckr.cli.config import common_config_options +from hckr.utils.ConfigUtils import config_path +from hckr.utils.DataUtils import print_df_as_table from hckr.utils.DbUtils import get_db_url from hckr.utils.MessageUtils import PError - +import pandas as pd @click.group( help="Database commands", @@ -21,15 +23,16 @@ def db(): @click.argument("query") @click.pass_context def query(ctx, config, query): - """Execute a SQL query.""" + """Execute a SQL query on Snowflake and return a DataFrame.""" db_url = get_db_url(section=config) + if db_url: engine = create_engine(db_url) try: with engine.connect() as connection: - result = connection.execute(text(query)) - for row in result: - click.echo(row) + # Execute the query and convert the result to a DataFrame + df = pd.read_sql_query(text(query), connection) + print_df_as_table(df) except SQLAlchemyError as e: PError(f"Error executing query: {e}") else: From 45e23974e0313d54a50e7309905e14c8d3dcc83f Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Sun, 25 Aug 2024 18:06:30 +0530 Subject: [PATCH 11/21] Enhance table printing in `print_df_as_table` Added parameters to control the number of rows and columns displayed in `print_df_as_table` function. Updated function calls to use these new parameters, allowing for more flexible and customized table prints. --- src/hckr/cli/db.py | 2 +- src/hckr/utils/DataUtils.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/hckr/cli/db.py b/src/hckr/cli/db.py index a1a4a82..77a5c47 100644 --- a/src/hckr/cli/db.py +++ b/src/hckr/cli/db.py @@ -32,7 +32,7 @@ def query(ctx, config, query): with engine.connect() as connection: # Execute the query and convert the result to a DataFrame df = pd.read_sql_query(text(query), connection) - print_df_as_table(df) + print_df_as_table(df, title=query, count=10, col_count=10) except SQLAlchemyError as e: PError(f"Error executing query: {e}") else: diff --git a/src/hckr/utils/DataUtils.py b/src/hckr/utils/DataUtils.py index b013b01..ed6e07d 100644 --- a/src/hckr/utils/DataUtils.py +++ b/src/hckr/utils/DataUtils.py @@ -21,10 +21,9 @@ def safe_faker_method(faker_instance, method_name, *args): raise ValueError(f"No such Faker method: {method_name}") -def print_df_as_table(df, title="Data Sample", count=3): - MAX_COLS_TO_SHOW = 10 +def print_df_as_table(df, title="Data Sample", count=3, col_count=10): ROWS_TO_SHOW = min(count, df.shape[0]) - COLS_TO_SHOW = min(MAX_COLS_TO_SHOW, df.shape[1]) + COLS_TO_SHOW = min(col_count, df.shape[1]) table = Table( show_header=True, title=title, @@ -36,14 +35,14 @@ def print_df_as_table(df, title="Data Sample", count=3): for column in df.columns[:COLS_TO_SHOW]: table.add_column(column, no_wrap=False, overflow="fold") msg = f"Data has total {colored(df.shape[0], 'yellow')} rows and {colored(df.shape[1], 'yellow')} columns, showing first {colored(ROWS_TO_SHOW, 'yellow')} rows" - if len(df.columns) > MAX_COLS_TO_SHOW: + if len(df.columns) > col_count: warning(f"{msg} and {colored(COLS_TO_SHOW, 'yellow')} columns") else: info(msg) # Add rows to the table for index, row in df.head(count).iterrows(): # Convert each row to string format, necessary to handle different data types - table.add_row(*[str(item) for item in row.values[:MAX_COLS_TO_SHOW]]) + table.add_row(*[str(item) for item in row.values[:col_count]]) rich.print(table) From 406f0f751ac8def1efa69478be5dd7818b907d5f Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Sun, 25 Aug 2024 18:10:12 +0530 Subject: [PATCH 12/21] Add options for number of rows and columns to query command. Enhanced the `query` command to allow specifying the number of rows and columns displayed in the output with `--num-rows` and `--num-cols` options. The default values for these options are set to 10. --- src/hckr/cli/db.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/hckr/cli/db.py b/src/hckr/cli/db.py index 77a5c47..3081d3f 100644 --- a/src/hckr/cli/db.py +++ b/src/hckr/cli/db.py @@ -21,8 +21,22 @@ def db(): @db.command() @common_config_options @click.argument("query") +@click.option( + "-nr", + "--num-rows", + default=10, + help="Number of rows to show.", + required=False, +) +@click.option( + "-nc", + "--num-cols", + default=10, + help="Number of cols to show.", + required=False, +) @click.pass_context -def query(ctx, config, query): +def query(ctx, config, query, num_rows, num_cols): """Execute a SQL query on Snowflake and return a DataFrame.""" db_url = get_db_url(section=config) @@ -32,7 +46,7 @@ def query(ctx, config, query): with engine.connect() as connection: # Execute the query and convert the result to a DataFrame df = pd.read_sql_query(text(query), connection) - print_df_as_table(df, title=query, count=10, col_count=10) + print_df_as_table(df, title=query, count=num_rows, col_count=num_cols) except SQLAlchemyError as e: PError(f"Error executing query: {e}") else: From bd318539419168ec12005f917a57323989863256 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Tue, 27 Aug 2024 15:58:29 +0530 Subject: [PATCH 13/21] Refactor message logging and remove redundant tests Enable logging message uncommented in MessageUtils and revised set_config_value to support override flag. Cleaned up redundant configuration tests and added an informational placeholder test. Refactor message utilities and update CLI config commands Refactored `PMsg` to `_PMsg` for internal use and added optional descriptions. Updated CLI config commands to leverage new utilities for success and error messaging. Enhanced configuration setting to include informative messages when adding new sections. Update version to 0.4.0.dev0 This updates the version from 0.3.3.dev0 to 0.4.0.dev0 in the __about__.py file. This change likely indicates new features or significant improvements. Refactor code formatting for improved readability This commit updates the code formatting by changing single quotes to double quotes in SQL queries and aligns parameters across multiple functions and fixtures. Additionally, it ensures better transaction handling in database operations by using context management to automatically commit DDL and non-data-returning DML queries. These changes enhance code readability and maintainability without altering functionality. Add CLI tests for DB queries and refactor query handling Introduced new tests in `test_db.py` to cover various database query scenarios. Refactored the `query` function in `db.py` to better handle data and non-data-returning queries. Configuration utility functions for Add configuration utilities and database command refactoring Introduced new configuration utilities for managing database settings. Updated the `DbUtils` module to use centralized constants for configuration options, and refactored `configure_db` command for improved readability and maintenance. Added comprehensive tests for various database options. Refactor config utilities and enhance config file verification Renamed config utilities path for better organization and clarity. Enhanced `config_exists` function to check if the config file is not only present but also non-empty. Improved messages and functionality in various utility methods, and slightly adjusted message display in `MessageUtils.py`. --- .gitignore | 1 + src/hckr/__about__.py | 2 +- src/hckr/cli/config.py | 28 ++---- src/hckr/cli/configure.py | 100 +++++++++------------ src/hckr/cli/db.py | 44 +++++++-- src/hckr/utils/DbUtils.py | 68 ++++++++------ src/hckr/utils/FileUtils.py | 4 +- src/hckr/utils/MessageUtils.py | 26 +++--- src/hckr/utils/{ => config}/ConfigUtils.py | 69 ++++++-------- src/hckr/utils/config/ConfigureUtils.py | 64 +++++++++++++ src/hckr/utils/config/Constants.py | 49 ++++++++++ tests/cli/test_cli.py | 19 ++-- tests/cli/test_configure.py | 39 ++++++++ tests/cli/test_db.py | 58 ++++++++++++ tests/conftest.py | 99 ++++++++++++++++++++ 15 files changed, 488 insertions(+), 182 deletions(-) rename src/hckr/utils/{ => config}/ConfigUtils.py (87%) create mode 100644 src/hckr/utils/config/ConfigureUtils.py create mode 100644 src/hckr/utils/config/Constants.py create mode 100644 tests/cli/test_configure.py create mode 100644 tests/cli/test_db.py create mode 100644 tests/conftest.py diff --git a/.gitignore b/.gitignore index 6bfce93..21de01b 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,4 @@ output/ !output/.gitkeep _build/ out/ +*.sqlite diff --git a/src/hckr/__about__.py b/src/hckr/__about__.py index 3503050..40f33e2 100644 --- a/src/hckr/__about__.py +++ b/src/hckr/__about__.py @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2024-present Ashish Patel # # SPDX-License-Identifier: MIT -__version__ = "0.3.3.dev0" +__version__ = "0.4.0.dev0" diff --git a/src/hckr/cli/config.py b/src/hckr/cli/config.py index babbb6e..911064c 100644 --- a/src/hckr/cli/config.py +++ b/src/hckr/cli/config.py @@ -3,9 +3,9 @@ import click import rich from cron_descriptor import get_description # type: ignore -from rich.panel import Panel -from ..utils.ConfigUtils import ( +from ..utils.MessageUtils import PError, PSuccess +from ..utils.config.ConfigUtils import ( init_config, DEFAULT_CONFIG, configMessage, @@ -55,13 +55,7 @@ def set(config, key, value): """ configMessage(config) set_config_value(config, key, value) - rich.print( - Panel( - f"[{config}] {key} <- {value}", - expand=True, - title="Success", - ) - ) + PSuccess(f"[{config}] {key} <- {value}") @config.command() @@ -72,21 +66,9 @@ def get(config, key): configMessage(config) try: value = get_config_value(config, key) - rich.print( - Panel( - f"[{config}] {key} = {value}", - expand=True, - title="Success", - ) - ) + PSuccess(f"[{config}] {key} = {value}") except ValueError as e: - rich.print( - Panel( - f"{e}", - expand=True, - title="Error", - ) - ) + PError(f"{e}") @config.command() diff --git a/src/hckr/cli/configure.py b/src/hckr/cli/configure.py index 14d974f..7365cdd 100644 --- a/src/hckr/cli/configure.py +++ b/src/hckr/cli/configure.py @@ -1,13 +1,27 @@ import click -from ..utils import MessageUtils -from ..utils.ConfigUtils import ( +from ..utils.config.ConfigUtils import ( list_config, set_config_value, - db_type_mapping, - DBType, ConfigType, + DBType, ) from ..utils.MessageUtils import PSuccess +from ..utils.config.ConfigureUtils import configure_host, configure_creds +from ..utils.config.Constants import ( + CONFIG_TYPE, + DB_TYPE, + ConfigType, + db_type_mapping, + DB_HOST, + DB_PORT, + DB_ACCOUNT, + DB_WAREHOUSE, + DB_SCHEMA, + DB_ROLE, + DB_PASSWORD, + DB_NAME, + DB_USER, +) @click.group( @@ -24,12 +38,12 @@ def configure(ctx): @configure.command("db") @click.option( - "--config_name", + "--config-name", prompt="Enter a name for this database configuration", help="Name of the config instance", ) @click.option( - "--db_type", + "--database-type", prompt="Select the type of database (1=PostgreSQL, 2=MySQL, 3=SQLite, 4=Snowflake)", type=click.Choice(["1", "2", "3", "4"]), help="Database type", @@ -44,69 +58,39 @@ def configure(ctx): confirmation_prompt=True, help="Database password", ) -@click.option("--dbname", prompt=False, help="Database name") +@click.option("--database-name", prompt=False, help="Database name") @click.option("--schema", prompt=False, help="Database schema") @click.option("--account", prompt=False, help="Snowflake Account Id") @click.option("--warehouse", prompt=False, help="Snowflake warehouse") @click.option("--role", prompt=False, help="Snowflake role") def configure_db( - config_name, db_type, host, port, user, password, dbname, schema, warehouse, role + config_name, + database_type, + host, + port, + user, + password, + database_name, + schema, + account, + warehouse, + role, ): """Configure database credentials based on the selected database type.""" - set_config_value(config_name, "config_type", ConfigType.DATABASE) - selected_db_type = db_type_mapping[db_type] - set_config_value(config_name, "database_type", selected_db_type) - if selected_db_type in [DBType.PostgreSQL, DBType.MySQL]: - if not host: - host = click.prompt("Enter the database host (e.g., localhost, 127.0.0.1)") - if not port: - port = click.prompt("Enter the database port (e.g., 5432 for PostgreSQL)") - if not user: - user = click.prompt("Enter the database user (e.g., root)") - if not password: - password = click.prompt( - "Enter the database password (input hidden)", - hide_input=True, - confirmation_prompt=True, - ) + set_config_value(config_name, CONFIG_TYPE, ConfigType.DATABASE) + selected_db_type = db_type_mapping[database_type] + set_config_value(config_name, DB_TYPE, selected_db_type) - set_config_value(config_name, "host", host) - set_config_value(config_name, "port", port) - set_config_value(config_name, "user", user) - set_config_value(config_name, "password", password) - set_config_value(config_name, "dbname", dbname) + configure_creds(config_name, password, selected_db_type, user) - elif selected_db_type == DBType.SQLite: - if not dbname: - dbname = click.prompt("Enter the path to the SQLite database file") - set_config_value(config_name, "dbname", dbname) + if not database_name: + database_name = click.prompt("Enter the database name") + set_config_value(config_name, DB_NAME, database_name) - elif selected_db_type == DBType.Snowflake: - if not user: - user = click.prompt("Enter your Snowflake username") - if not password: - password = click.prompt( - "Enter your Snowflake password", - hide_input=True, - confirmation_prompt=True, - ) - if not dbname: - dbname = click.prompt("Enter the Snowflake database name") - if not schema: - schema = click.prompt("Enter the Snowflake schema name") - if not warehouse: - warehouse = click.prompt("Enter the Snowflake warehouse name") - if not role: - role = click.prompt("Enter the Snowflake role") - - set_config_value(config_name, "user", user) - set_config_value(config_name, "password", password) - set_config_value(config_name, "account", account) - set_config_value(config_name, "warehouse", warehouse) - set_config_value(config_name, "dbname", dbname) - set_config_value(config_name, "schema", schema) - set_config_value(config_name, "role", role) + configure_host( + account, config_name, host, port, role, schema, selected_db_type, warehouse + ) PSuccess( f"Database configuration saved successfully in config instance '{config_name}'" diff --git a/src/hckr/cli/db.py b/src/hckr/cli/db.py index 3081d3f..9b91002 100644 --- a/src/hckr/cli/db.py +++ b/src/hckr/cli/db.py @@ -1,14 +1,14 @@ import click +import pandas as pd from sqlalchemy import create_engine, text from sqlalchemy.exc import SQLAlchemyError from yaspin import yaspin # type: ignore from hckr.cli.config import common_config_options -from hckr.utils.ConfigUtils import config_path from hckr.utils.DataUtils import print_df_as_table from hckr.utils.DbUtils import get_db_url -from hckr.utils.MessageUtils import PError -import pandas as pd +from hckr.utils.MessageUtils import PError, PInfo, PSuccess + @click.group( help="Database commands", @@ -36,17 +36,43 @@ def db(): required=False, ) @click.pass_context -def query(ctx, config, query, num_rows, num_cols): - """Execute a SQL query on Snowflake and return a DataFrame.""" +def query(ctx, config, query, num_rows=None, num_cols=None): + """Execute a SQL query on Snowflake and return a DataFrame or handle non-data-returning queries.""" db_url = get_db_url(section=config) - + query = query.strip() if db_url: engine = create_engine(db_url) try: with engine.connect() as connection: - # Execute the query and convert the result to a DataFrame - df = pd.read_sql_query(text(query), connection) - print_df_as_table(df, title=query, count=num_rows, col_count=num_cols) + # Normalize and determine the type of query + normalized_query = query.lower() + is_data_returning_query = normalized_query.startswith( + ("select", "desc", "describe", "show", "explain") + ) + is_ddl_query = normalized_query.startswith( + ("create", "alter", "drop", "truncate") + ) + + if is_data_returning_query: + # Execute and fetch results for queries that return data + df = pd.read_sql_query(text(query), connection) + + # Optionally limit rows and columns if specified + if num_rows is not None: + df = df.head(num_rows) + if num_cols is not None: + df = df.iloc[:, :num_cols] + + print_df_as_table(df, title=query) + return df + else: + # Execute DDL or non-data-returning DML queries + with connection.begin(): # this will automatically commit at the end + result = connection.execute(text(query)) + if is_ddl_query: + PInfo(query, "Success") + else: + PInfo(query, f"[Success] Rows affected: {result.rowcount}") except SQLAlchemyError as e: PError(f"Error executing query: {e}") else: diff --git a/src/hckr/utils/DbUtils.py b/src/hckr/utils/DbUtils.py index 9cf2084..4313a4d 100644 --- a/src/hckr/utils/DbUtils.py +++ b/src/hckr/utils/DbUtils.py @@ -1,13 +1,23 @@ import logging -from configparser import NoSectionError, NoOptionError +from configparser import NoOptionError -from sqlalchemy import create_engine, text -from sqlalchemy.exc import SQLAlchemyError -import click - -from hckr.utils import ConfigUtils -from hckr.utils.ConfigUtils import ConfigType, DBType, load_config from hckr.utils.MessageUtils import PError +from hckr.utils.config import ConfigUtils +from hckr.utils.config.Constants import ( + ConfigType, + DBType, + DB_NAME, + DB_USER, + CONFIG_TYPE, + DB_TYPE, + DB_PASSWORD, + DB_HOST, + DB_PORT, + DB_ACCOUNT, + DB_WAREHOUSE, + DB_ROLE, + DB_SCHEMA, +) def get_db_url(section): @@ -18,16 +28,18 @@ def get_db_url(section): """ config = ConfigUtils.load_config() try: - config_type = config.get(section, "config_type") + config_type = config.get(section, CONFIG_TYPE) if config_type != ConfigType.DATABASE: - PError(f"The configuration {section} is not database type\n" - " Please use [magenta]hckr configure db[/magenta] to configure database.") + PError( + f"The configuration {section} is not database type\n" + " Please use [magenta]hckr configure db[/magenta] to configure database." + ) exit(1) - db_type = config.get(section, "database_type") + db_type = config.get(section, DB_TYPE) if db_type == DBType.SQLite: - dbname = config.get(section, "dbname") - return f"sqlite:///{dbname}" + database_name = config.get(section, DB_NAME) + return f"sqlite:///{database_name}" elif db_type in [DBType.PostgreSQL, DBType.MySQL]: return _get_jdbc_url(config, section, db_type) @@ -41,15 +53,15 @@ def get_db_url(section): def _get_jdbc_url(config, section, db_type): - user = config.get(section, "user") - password = config.get(section, "password") - host = config.get(section, "host") - port = config.get(section, "port") - dbname = config.get(section, "dbname") + user = config.get(section, DB_USER) + password = config.get(section, DB_PASSWORD) + host = config.get(section, DB_HOST) + port = config.get(section, DB_PORT) + database_name = config.get(section, DB_NAME) if db_type == DBType.PostgreSQL: - return f"postgresql://{user}:{password}@{host}:{port}/{dbname}" + return f"postgresql://{user}:{password}@{host}:{port}/{database_name}" elif db_type == DBType.MySQL: - return f"mysql+pymysql://{user}:{password}@{host}:{port}/{dbname}" + return f"mysql+pymysql://{user}:{password}@{host}:{port}/{database_name}" else: logging.debug(f"Invalid db_type: {db_type} in get_jdbc_url()") PError("Error occured while creating JDBC Url") @@ -57,14 +69,14 @@ def _get_jdbc_url(config, section, db_type): def _get_snowflake_url(config, section): - user = config.get(section, "user") - password = config.get(section, "password") - account = config.get(section, "account") - warehouse = config.get(section, "warehouse") - role = config.get(section, "role") - dbname = config.get(section, "dbname") - schema = config.get(section, 'schema') + user = config.get(section, DB_USER) + password = config.get(section, DB_PASSWORD) + account = config.get(section, DB_ACCOUNT) + warehouse = config.get(section, DB_WAREHOUSE) + role = config.get(section, DB_ROLE) + database_name = config.get(section, DB_NAME) + schema = config.get(section, DB_SCHEMA) return ( - f"snowflake://{user}:{password}@{account}/{dbname}/{schema}" + f"snowflake://{user}:{password}@{account}/{database_name}/{schema}" f"?warehouse={warehouse}&role={role}" ) diff --git a/src/hckr/utils/FileUtils.py b/src/hckr/utils/FileUtils.py index 5e59d84..061d551 100644 --- a/src/hckr/utils/FileUtils.py +++ b/src/hckr/utils/FileUtils.py @@ -2,8 +2,6 @@ import os from pathlib import Path -from pyarrow._dataset import FileFormat - from hckr.utils.MessageUtils import error, info, colored, warning from enum import Enum @@ -19,7 +17,7 @@ class FileFormat(str, Enum): INVALID = "invalid" @staticmethod - def fileExtToFormat(file_path, file_extension) -> FileFormat: + def fileExtToFormat(file_path, file_extension): file_type_extension_map = { ".txt": FileFormat.TXT, ".text": FileFormat.TXT, diff --git a/src/hckr/utils/MessageUtils.py b/src/hckr/utils/MessageUtils.py index e348841..18cf7d0 100644 --- a/src/hckr/utils/MessageUtils.py +++ b/src/hckr/utils/MessageUtils.py @@ -1,5 +1,6 @@ import logging +import click import rich from rich import print import random @@ -32,30 +33,35 @@ def warning(msg, color=None): print(f"{warn_emoji()} [bold yellow]{msg}[/bold yellow]") -def PMsg(msg, title): +def _PMsg(msg, title, desc=None): + if desc: + title = f"{title}: {desc}" + + click.echo("\n") rich.print( Panel( msg, - expand=False, + expand=True, title=title, ) ) -def PWarn(msg, title="[yellow]Warning"): - PMsg(msg, title) +def PWarn(msg, title="[yellow]Warning", desc=None): + _PMsg(msg, title, desc) -def PSuccess(msg, title="[green]Success"): - PMsg(msg, title) +def PSuccess(msg, desc=None, title="[green]Success[/green]"): + _PMsg(msg, title, desc) -def PInfo(msg, title="[blue]Info"): - PMsg(msg, title) +def PInfo(msg, desc=None, title="[blue]Info"): + _PMsg(msg, title, desc) -def PError(msg, title="[red]Error"): - PMsg(msg, title) +def PError(msg, desc=None, title="[red]Error"): + _PMsg(msg, title, desc) + exit(1) def info(msg, color=None): diff --git a/src/hckr/utils/ConfigUtils.py b/src/hckr/utils/config/ConfigUtils.py similarity index 87% rename from src/hckr/utils/ConfigUtils.py rename to src/hckr/utils/config/ConfigUtils.py index 240b7cc..9f040b0 100644 --- a/src/hckr/utils/ConfigUtils.py +++ b/src/hckr/utils/config/ConfigUtils.py @@ -1,53 +1,25 @@ import configparser import logging -from enum import Enum -from pathlib import Path -from readline import set_completer -import click import rich from rich.panel import Panel -from . import MessageUtils -from .MessageUtils import PWarn, PMsg, PError, PSuccess, PInfo -from ..__about__ import __version__ + +from .. import MessageUtils +from ..MessageUtils import PWarn, PSuccess, PInfo, PError +from ...__about__ import __version__ +from .Constants import config_path, DBType, DEFAULT_CONFIG # Define the default configuration file path, this can't be changed, although user can have multile instances using # --config -config_path = Path.home() / ".hckrcfg" -DEFAULT_CONFIG = "DEFAULT" - - -class DBType(str, Enum): - PostgreSQL = ("PostgreSQL",) - MySQL = ("MySQL",) - SQLite = ("SQLite",) - Snowflake = ("Snowflake",) - - def __str__(self): - return self.value - - -class ConfigType(str, Enum): - DATABASE = ("database",) - - def __str__(self): - return self.value - - -db_type_mapping = { - "1": DBType.PostgreSQL, - "2": DBType.MySQL, - "3": DBType.SQLite, - "4": DBType.Snowflake, -} def load_config(): """Load the INI configuration file.""" config = configparser.ConfigParser() - if not check_config(): + if not config_exists(): PWarn( - f"Config file [magenta]{config_path}[/magenta] doesn't exists, Please run init command to create one \n " + f"Config file [magenta]{config_path}[/magenta] doesn't exists or empty," + f" Please run init command to create one \n " "[bold green]hckr config init" ) exit(1) @@ -55,8 +27,22 @@ def load_config(): return config -def check_config() -> bool: - return config_path.exists() +def config_exists() -> bool: + """ + Check if config file exists and is not empty. + + :return: True if config file exists and is not empty, False otherwise. + :rtype: bool + """ + if not config_path.exists(): + return False + if config_path.stat().st_size == 0: + return False + with config_path.open("r") as file: + content = file.read().strip() + if not content: + return False + return True def init_config(overwrite): @@ -103,7 +89,7 @@ def init_config(overwrite): ) -def set_config_value(section, key, value): +def set_config_value(section, key, value, override=False): """ Sets a configuration value in a configuration file. @@ -124,8 +110,9 @@ def set_config_value(section, key, value): logging.debug(f"Setting [{section}] {key} = {value}") config = load_config() if not config.has_section(section) and section != DEFAULT_CONFIG: - logging.debug(f"Adding section {section}") + PInfo(f"Config \[{section}] doesn't exist, Adding") config.add_section(section) + config.set(section, key, value) with config_path.open("w") as config_file: config.write(config_file) @@ -211,5 +198,3 @@ def configMessage(config): MessageUtils.info(f"Using default config: [magenta]{DEFAULT_CONFIG}") else: MessageUtils.info(f"Using config: [magenta]{config}") - - diff --git a/src/hckr/utils/config/ConfigureUtils.py b/src/hckr/utils/config/ConfigureUtils.py new file mode 100644 index 0000000..e436fcc --- /dev/null +++ b/src/hckr/utils/config/ConfigureUtils.py @@ -0,0 +1,64 @@ +from configparser import NoOptionError + +import click + +from .ConfigUtils import set_config_value +from .Constants import ( + ConfigType, + DBType, + DB_NAME, + DB_USER, + CONFIG_TYPE, + DB_TYPE, + DB_PASSWORD, + DB_HOST, + DB_PORT, + DB_ACCOUNT, + DB_WAREHOUSE, + DB_ROLE, + DB_SCHEMA, +) +from ..DbUtils import _get_jdbc_url, _get_snowflake_url +from ..MessageUtils import PError + + +def configure_host( + account, config_name, host, port, role, schema, selected_db_type, warehouse +): + if selected_db_type in [DBType.PostgreSQL, DBType.MySQL]: + if not host: + host = click.prompt("Enter the database host (e.g., localhost, 127.0.0.1)") + if not port: + port = click.prompt("Enter the database port (e.g., 5432 for PostgreSQL)") + + set_config_value(config_name, DB_HOST, host) + set_config_value(config_name, DB_PORT, port) + + elif selected_db_type == DBType.Snowflake: + if not account: + account = click.prompt("Enter the Snowflake Account Id") + if not schema: + schema = click.prompt("Enter the Snowflake schema name") + if not warehouse: + warehouse = click.prompt("Enter the Snowflake warehouse name") + if not role: + role = click.prompt("Enter the Snowflake role") + set_config_value(config_name, DB_ACCOUNT, account) + set_config_value(config_name, DB_WAREHOUSE, warehouse) + set_config_value(config_name, DB_SCHEMA, schema) + set_config_value(config_name, DB_ROLE, role) + + +def configure_creds(config_name, password, selected_db_type, user): + if selected_db_type != DBType.SQLite: + if not user: + user = click.prompt("Enter the database user (e.g., root)") + if not password: + password = click.prompt( + "Enter the database password (input hidden)", + hide_input=True, + confirmation_prompt=True, + ) + # common values + set_config_value(config_name, DB_USER, user) + set_config_value(config_name, DB_PASSWORD, password) diff --git a/src/hckr/utils/config/Constants.py b/src/hckr/utils/config/Constants.py new file mode 100644 index 0000000..699348e --- /dev/null +++ b/src/hckr/utils/config/Constants.py @@ -0,0 +1,49 @@ +from enum import Enum +from pathlib import Path + +config_path = Path.home() / ".hckrcfg" +DEFAULT_CONFIG = "DEFAULT" + + +class DBType(str, Enum): + PostgreSQL = ("PostgreSQL",) + MySQL = ("MySQL",) + SQLite = ("SQLite",) + Snowflake = ("Snowflake",) + + def __str__(self): + return self.value + + +class ConfigType(str, Enum): + DATABASE = ("database",) + + def __str__(self): + return self.value + + +db_type_mapping = { + "1": DBType.PostgreSQL, + "2": DBType.MySQL, + "3": DBType.SQLite, + "4": DBType.Snowflake, +} + +# =============== CONFIG CONSTANTS =================== # +# COMMON CONFIG Constants +CONFIG_TYPE = "config_type" + +# DATABASE CONFIG Constants +DB_TYPE = "type" +DB_HOST = "host" +DB_PORT = "port" +DB_USER = "username" +DB_PASSWORD = "password" + +DB_NAME = "database" +DB_SCHEMA = "schema" + +# SNOWFLAKE SPECIFIC +DB_ACCOUNT = "account" +DB_ROLE = "role" +DB_WAREHOUSE = "warehouse" diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 9a6d222..232a74d 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -18,13 +18,16 @@ def test_hckr(): -h, --help Show this message and exit. Commands: - cron cron commands - crypto crypto commands - data data related commands - hash hash commands - info info commands - k8s Kubernetes commands - net network commands - repl Start an interactive shell""" + config Config commands + configure easy configurations for other commands (eg. + cron cron commands + crypto crypto commands + data data related commands + db Database commands + hash hash commands + info info commands + k8s Kubernetes commands + net network commands + repl Start an interactive shell.""" in result.output ) diff --git a/tests/cli/test_configure.py b/tests/cli/test_configure.py new file mode 100644 index 0000000..0d28bbc --- /dev/null +++ b/tests/cli/test_configure.py @@ -0,0 +1,39 @@ +from hckr.cli.configure import configure_db +from hckr.utils.MessageUtils import _PMsg, PInfo + + +def test_configure_postgres(cli_runner, postgres_options): + result = cli_runner.invoke(configure_db, postgres_options) + print(result.output) + assert result.exit_code == 0 + assert "Database configuration saved successfully" in result.output + assert "[testdb_postgres]" in result.output + + +def test_configure_mysql(cli_runner, mysql_options): + result = cli_runner.invoke(configure_db, mysql_options) + print(result.output) + assert result.exit_code == 0 + assert "Database configuration saved successfully" in result.output + assert "[testdb_mysql]" in result.output + + +def test_configure_snowflake(cli_runner, snowflake_options): + result = cli_runner.invoke(configure_db, snowflake_options) + print(result.output) + assert result.exit_code == 0 + assert "Database configuration saved successfully" in result.output + assert "[testdb_snowflake]" in result.output + + +def test_configure_sqlite(cli_runner, sqlite_options): + result = cli_runner.invoke(configure_db, sqlite_options) + print(result.output) + assert result.exit_code == 0 + assert "Database configuration saved successfully" in result.output + assert "[testdb_sqlite]" in result.output + + +def test_test(): + hi = "ashish" + PInfo(f"hello {hi}") diff --git a/tests/cli/test_db.py b/tests/cli/test_db.py new file mode 100644 index 0000000..1d845d8 --- /dev/null +++ b/tests/cli/test_db.py @@ -0,0 +1,58 @@ +from click.testing import CliRunner + +from hckr.cli.db import query +from hckr.cli.configure import configure_db + + +def _run_query_and_assert(cli_runner, sql_query, value_assert=None): + result = cli_runner.invoke(query, [sql_query, "-c", "testdb_sqlite"]) + print(result.output) + if value_assert: + assert value_assert in result.output + + +def test_db_query_sqlite(cli_runner, sqlite_options): + # # configuring sqlite for testing + result = cli_runner.invoke(configure_db, sqlite_options) + assert result.exit_code == 0 + assert "Database configuration saved successfully" in result.output + assert "[testdb_sqlite]" in result.output + + _run_query_and_assert(cli_runner, "drop table if exists users") + + _run_query_and_assert( + cli_runner, + """ + CREATE TABLE users ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + email TEXT NOT NULL UNIQUE + ); + """, + ) + + _run_query_and_assert( + cli_runner, + "INSERT INTO users (name, email) VALUES ('Alice', 'alice@example.com');", + ) + + _run_query_and_assert(cli_runner, "select * from users;") + + +# NEGATIVE USE CASES + + +def test_db_query_missing_or_invalid_config(): + runner = CliRunner() + result = runner.invoke(query, ["select 1"]) + print(result.output) + assert result.exit_code != 0 + assert "The configuration DEFAULT is not database type" in result.output + + +def test_db_query_missing_query(): + runner = CliRunner() + result = runner.invoke(query) + print(result.output) + assert result.exit_code != 0 + assert "Error: Missing argument 'QUERY'" in result.output diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b3abe46 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: 2024-present Ashish Patel +# +# SPDX-License-Identifier: MIT +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from hckr.utils.config.ConfigUtils import init_config + +current_directory = Path(__file__).parent +SQLITE_DB = current_directory / "cli" / "resources" / "db" / "test_db.sqlite" + + +@pytest.fixture(scope="package") +def local_sqlite_db(): + return SQLITE_DB + + +@pytest.fixture(scope="package") +def cli_runner(): + init_config(overwrite=True) # recreate the config file + return CliRunner() + + +@pytest.fixture(scope="package") +def postgres_options(): + return [ + "--config-name", + "testdb_postgres", + "--database-type", + "1", + "--user", + "user", + "--password", + "password", + "--host", + "host.postgres", + "--port", + "11143", + "--database-name", + "defaultdb", + ] + + +@pytest.fixture(scope="package") +def mysql_options(): + return [ + "--config-name", + "testdb_mysql", + "--database-type", + "2", + "--user", + "user", + "--password", + "password", + "--host", + "host", + "--port", + "123", + "--database-name", + "defaultdb", + ] + + +@pytest.fixture(scope="package") +def snowflake_options(): + return [ + "--config-name", + "testdb_snowflake", + "--database-type", + "4", + "--user", + "user", + "--password", + "password", + "--database-name", + "database", + "--schema", + "PUBLIC", + "--account", + "account-id", + "--warehouse", + "COMPUTE_WH", + "--role", + "ACCOUNTADMIN", + ] + + +@pytest.fixture(scope="package") +def sqlite_options(): + return [ + "--config-name", + "testdb_sqlite", + "--database-type", + "3", + "--database-name", + f"{SQLITE_DB}", + ] From 8d4ec343f0d1a7ab47deba68689dd7ab96a58dbd Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Thu, 29 Aug 2024 16:46:19 +0530 Subject: [PATCH 14/21] Update src/hckr/cli/config.py Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/hckr/cli/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hckr/cli/config.py b/src/hckr/cli/config.py index 911064c..634d8b9 100644 --- a/src/hckr/cli/config.py +++ b/src/hckr/cli/config.py @@ -22,7 +22,7 @@ @click.pass_context def config(ctx): """ - Defines a command group for configuration-related commands. + Defines a command group for managing application configurations. This group includes commands to set, get, show, and initialize configuration values. """ pass From f7650310bdc19acacb49f4d1c214331b042e0d37 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Thu, 29 Aug 2024 16:59:16 +0530 Subject: [PATCH 15/21] Remove error exits and improve config CLI documentation Removed `exit(1)` calls from DbUtils.py to allow for better exception handling and application flow. Enhanced documentation for config commands in config.py for clearer guidance on usage and examples. --- src/hckr/cli/config.py | 92 ++++++++++++++++++++++++---- src/hckr/utils/DbUtils.py | 3 - src/hckr/utils/config/ConfigUtils.py | 12 ---- 3 files changed, 79 insertions(+), 28 deletions(-) diff --git a/src/hckr/cli/config.py b/src/hckr/cli/config.py index 911064c..7338d80 100644 --- a/src/hckr/cli/config.py +++ b/src/hckr/cli/config.py @@ -1,8 +1,6 @@ # from ..utils.MessageUtils import * import click -import rich -from cron_descriptor import get_description # type: ignore from ..utils.MessageUtils import PError, PSuccess from ..utils.config.ConfigUtils import ( @@ -43,16 +41,25 @@ def common_config_options(func): @click.argument("value") def set(config, key, value): """ - Sets a configuration value. + This command adds a new entry to the config file with key and value - Args: - config (str): The configuration instance name. Default is defined by DEFAULT_CONFIG. - key (str): The key of the config setting to change. - value (str): The value to set for the specified key. + **Example Usage**: - Example: - $ cli_tool configure set database_host 127.0.0.1 + * Setting a value inside DEFAULT config + Note - DEFAULT config is parent of all other configurations and others will inherit its values if not overridden + + .. code-block:: shell + + $ hckr config set database_host 127.0.0.1 + + * Similarly, we can also set a value in specific configuration, configuration will be created if not exists + + .. code-block:: shell + $ hckr config set database_host 127.0.0.1 --config MY_DATABASE + + **Command Reference**: """ + configMessage(config) set_config_value(config, key, value) PSuccess(f"[{config}] {key} <- {value}") @@ -62,7 +69,27 @@ def set(config, key, value): @common_config_options @click.argument("key") def get(config, key): - """Get a configuration value.""" + """ + This command returns value for a key in a configuration + + **Example Usage**: + + * Getting a value for key in DEFAULT config + + Note - DEFAULT config is parent of all other configurations and others will inherit its values if not overridden + + .. code-block:: shell + + $ hckr config get database_host + + * Similarly, we can also get a value in specific configuration + + .. code-block:: shell + $ hckr config get database_host --config MY_DATABASE + + **Command Reference**: + """ + configMessage(config) try: value = get_config_value(config, key) @@ -81,7 +108,32 @@ def get(config, key): help="Whether to show all configs (default: False)", ) def show(config, all): - """List configuration values.""" + """ + This command show list of all keys available in given configuration, + we can also see values in all configurations by providing -a/--all flag + + **Example Usage**: + + * Getting values for keys in DEFAULT config + + Note - DEFAULT config is parent of all other configurations and others will inherit its values if not overridden + + .. code-block:: shell + + $ hckr config show + + * Similarly, we can also get all values in a specific configuration using -c/--config flag + + .. code-block:: shell + $ hckr config show -c MY_DATABASE + + * Additionally, we can also see all configurations using -a/--all flag + + .. code-block:: shell + $ hckr config show --all + + **Command Reference**: + """ list_config(config, all) @@ -95,8 +147,22 @@ def show(config, all): ) def init(overwrite): """ - Initializes the configuration for the application. + This command Initializes the configuration for the application, + we can also use this option to overwrite existing config and reinitialize. + + **Example Usage**: + + * Initialising config file .hckrcfg with default settings + + .. code-block:: shell + + $ hckr config init + + * Similarly, we can also delete existing file and recreate using -o/--overwrite flag + + .. code-block:: shell + $ hckr config init --overwrite - :return: None + **Command Reference**: """ init_config(overwrite) diff --git a/src/hckr/utils/DbUtils.py b/src/hckr/utils/DbUtils.py index 4313a4d..13e7249 100644 --- a/src/hckr/utils/DbUtils.py +++ b/src/hckr/utils/DbUtils.py @@ -34,7 +34,6 @@ def get_db_url(section): f"The configuration {section} is not database type\n" " Please use [magenta]hckr configure db[/magenta] to configure database." ) - exit(1) db_type = config.get(section, DB_TYPE) if db_type == DBType.SQLite: @@ -49,7 +48,6 @@ def get_db_url(section): except NoOptionError as e: PError(f"Config {section} is not configured correctly\n {e}") - exit(1) def _get_jdbc_url(config, section, db_type): @@ -65,7 +63,6 @@ def _get_jdbc_url(config, section, db_type): else: logging.debug(f"Invalid db_type: {db_type} in get_jdbc_url()") PError("Error occured while creating JDBC Url") - exit(1) def _get_snowflake_url(config, section): diff --git a/src/hckr/utils/config/ConfigUtils.py b/src/hckr/utils/config/ConfigUtils.py index 9f040b0..78226ff 100644 --- a/src/hckr/utils/config/ConfigUtils.py +++ b/src/hckr/utils/config/ConfigUtils.py @@ -46,18 +46,6 @@ def config_exists() -> bool: def init_config(overwrite): - """ - Ensures the existence of a configuration file at the specified path. - - :param config_path: The path to the configuration file. - :return: None - - This function creates a configuration file at the specified path if it does not already exist. It also creates any necessary parent directories. The configuration file is empty initially, but a default configuration is written to it using configparser. The default configuration includes a 'DEFAULT' section with a 'version' option that contains the value of the __version__ global variable. - - Example Usage: - -------------- - ensure_config_file(Path('/path/to/config.ini')) - """ if not config_path.exists(): config_path.parent.mkdir(parents=True, exist_ok=True) config_path.touch(exist_ok=True) From 55ac9242f47db426c3468126ef9744b4b4bd6ed6 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Thu, 29 Aug 2024 17:06:31 +0530 Subject: [PATCH 16/21] Refactor query handling and remove unused imports Refactored SQL query handling for better readability and maintainability by streamlining conditional checks and removing redundant checks. Additionally, cleaned up various utility files by removing unnecessary imports and docstring comments. --- src/hckr/cli/configure.py | 11 +--- src/hckr/cli/db.py | 69 ++++++++++++------------- src/hckr/utils/config/ConfigUtils.py | 41 ++------------- src/hckr/utils/config/ConfigureUtils.py | 8 --- 4 files changed, 38 insertions(+), 91 deletions(-) diff --git a/src/hckr/cli/configure.py b/src/hckr/cli/configure.py index 7365cdd..211c3cc 100644 --- a/src/hckr/cli/configure.py +++ b/src/hckr/cli/configure.py @@ -1,26 +1,17 @@ import click +from ..utils.MessageUtils import PSuccess from ..utils.config.ConfigUtils import ( list_config, set_config_value, - DBType, ) -from ..utils.MessageUtils import PSuccess from ..utils.config.ConfigureUtils import configure_host, configure_creds from ..utils.config.Constants import ( CONFIG_TYPE, DB_TYPE, ConfigType, db_type_mapping, - DB_HOST, - DB_PORT, - DB_ACCOUNT, - DB_WAREHOUSE, - DB_SCHEMA, - DB_ROLE, - DB_PASSWORD, DB_NAME, - DB_USER, ) diff --git a/src/hckr/cli/db.py b/src/hckr/cli/db.py index 9b91002..af6f5c2 100644 --- a/src/hckr/cli/db.py +++ b/src/hckr/cli/db.py @@ -2,12 +2,11 @@ import pandas as pd from sqlalchemy import create_engine, text from sqlalchemy.exc import SQLAlchemyError -from yaspin import yaspin # type: ignore from hckr.cli.config import common_config_options from hckr.utils.DataUtils import print_df_as_table from hckr.utils.DbUtils import get_db_url -from hckr.utils.MessageUtils import PError, PInfo, PSuccess +from hckr.utils.MessageUtils import PError, PInfo @click.group( @@ -39,41 +38,41 @@ def db(): def query(ctx, config, query, num_rows=None, num_cols=None): """Execute a SQL query on Snowflake and return a DataFrame or handle non-data-returning queries.""" db_url = get_db_url(section=config) + if not db_url: + PError("Database credentials are not properly configured.") + query = query.strip() - if db_url: - engine = create_engine(db_url) - try: - with engine.connect() as connection: - # Normalize and determine the type of query - normalized_query = query.lower() - is_data_returning_query = normalized_query.startswith( - ("select", "desc", "describe", "show", "explain") - ) - is_ddl_query = normalized_query.startswith( - ("create", "alter", "drop", "truncate") - ) + engine = create_engine(db_url) + try: + with engine.connect() as connection: + # Normalize and determine the type of query + normalized_query = query.lower() + is_data_returning_query = normalized_query.startswith( + ("select", "desc", "describe", "show", "explain") + ) + is_ddl_query = normalized_query.startswith( + ("create", "alter", "drop", "truncate") + ) - if is_data_returning_query: - # Execute and fetch results for queries that return data - df = pd.read_sql_query(text(query), connection) + if is_data_returning_query: + # Execute and fetch results for queries that return data + df = pd.read_sql_query(text(query), connection) - # Optionally limit rows and columns if specified - if num_rows is not None: - df = df.head(num_rows) - if num_cols is not None: - df = df.iloc[:, :num_cols] + # Optionally limit rows and columns if specified + if num_rows is not None: + df = df.head(num_rows) + if num_cols is not None: + df = df.iloc[:, :num_cols] - print_df_as_table(df, title=query) - return df + print_df_as_table(df, title=query) + return df + else: + # Execute DDL or non-data-returning DML queries + with connection.begin(): # this will automatically commit at the end + result = connection.execute(text(query)) + if is_ddl_query: + PInfo(query, "Success") else: - # Execute DDL or non-data-returning DML queries - with connection.begin(): # this will automatically commit at the end - result = connection.execute(text(query)) - if is_ddl_query: - PInfo(query, "Success") - else: - PInfo(query, f"[Success] Rows affected: {result.rowcount}") - except SQLAlchemyError as e: - PError(f"Error executing query: {e}") - else: - PError("Database credentials are not properly configured.") + PInfo(query, f"[Success] Rows affected: {result.rowcount}") + except SQLAlchemyError as e: + PError(f"Error executing query: {e}") diff --git a/src/hckr/utils/config/ConfigUtils.py b/src/hckr/utils/config/ConfigUtils.py index 78226ff..bfe7a81 100644 --- a/src/hckr/utils/config/ConfigUtils.py +++ b/src/hckr/utils/config/ConfigUtils.py @@ -4,25 +4,21 @@ import rich from rich.panel import Panel +from .Constants import config_path, DEFAULT_CONFIG from .. import MessageUtils from ..MessageUtils import PWarn, PSuccess, PInfo, PError from ...__about__ import __version__ -from .Constants import config_path, DBType, DEFAULT_CONFIG - -# Define the default configuration file path, this can't be changed, although user can have multile instances using -# --config def load_config(): """Load the INI configuration file.""" config = configparser.ConfigParser() if not config_exists(): - PWarn( + PError( f"Config file [magenta]{config_path}[/magenta] doesn't exists or empty," f" Please run init command to create one \n " "[bold green]hckr config init" ) - exit(1) config.read(config_path) return config @@ -30,9 +26,6 @@ def load_config(): def config_exists() -> bool: """ Check if config file exists and is not empty. - - :return: True if config file exists and is not empty, False otherwise. - :rtype: bool """ if not config_path.exists(): return False @@ -80,20 +73,6 @@ def init_config(overwrite): def set_config_value(section, key, value, override=False): """ Sets a configuration value in a configuration file. - - :param section: The name of the configuration section. - :param key: The key of the configuration value. - :param value: The value to set. - :return: None - - This function sets a configuration value in a configuration file. It first logs the action using the `logging.debug()` function. Then, it loads the configuration file using the `load_config()` function. If the configuration file does not have the specified section and the section is not the default section, it adds the section to the configuration file. Next, it sets the value for the key in the specified section. Finally, it writes the updated configuration file to disk. - - After setting the configuration value, the function displays an information message using the `MessageUtils.info()` function. - - Note: This function assumes that the `logging` and `MessageUtils` modules are imported and configured properly. - - Example Usage: - set_config_value("database", "username", "admin") """ logging.debug(f"Setting [{section}] {key} = {value}") config = load_config() @@ -107,12 +86,6 @@ def set_config_value(section, key, value, override=False): def get_config_value(section, key) -> str: - """ - :param section: The section of the configuration file where the desired value is located. - :param key: The key of the value within the specified section. - :return: The value corresponding to the specified key within the specified section of the configuration file. - - """ logging.debug(f"Getting [{section}] {key} ") config = load_config() if section != DEFAULT_CONFIG and not config.has_section(section): @@ -134,7 +107,7 @@ def show_config(config, section): else "NOTHING FOUND" ), expand=True, - title=f"\[DEFAULT]", + title="\[DEFAULT]", ) ) elif config.has_section(section): @@ -162,14 +135,6 @@ def show_config(config, section): def list_config(section, all=False): - """ - List Config - - This function takes a section parameter and lists the configuration values for that section from the loaded configuration file. If the section is found in the configuration file, it will print the section name and all key-value pairs associated with that section. If the section is not found, it will display an error message. - - :param section: The section name for which the configuration values should be listed. - :return: None - """ config = load_config() if all: MessageUtils.info("Listing all config") diff --git a/src/hckr/utils/config/ConfigureUtils.py b/src/hckr/utils/config/ConfigureUtils.py index e436fcc..8ba7ef7 100644 --- a/src/hckr/utils/config/ConfigureUtils.py +++ b/src/hckr/utils/config/ConfigureUtils.py @@ -1,15 +1,9 @@ -from configparser import NoOptionError - import click from .ConfigUtils import set_config_value from .Constants import ( - ConfigType, DBType, - DB_NAME, DB_USER, - CONFIG_TYPE, - DB_TYPE, DB_PASSWORD, DB_HOST, DB_PORT, @@ -18,8 +12,6 @@ DB_ROLE, DB_SCHEMA, ) -from ..DbUtils import _get_jdbc_url, _get_snowflake_url -from ..MessageUtils import PError def configure_host( From 0beaf9f9f27537d00e4a993d4e307170979362fe Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Thu, 29 Aug 2024 17:08:27 +0530 Subject: [PATCH 17/21] Remove redundant import statement from config.py The commented-out import statement was removed to clean up the code. This change helps maintain readability and ensures there are no unnecessary lines in the file. --- src/hckr/cli/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/hckr/cli/config.py b/src/hckr/cli/config.py index 5853258..4eee502 100644 --- a/src/hckr/cli/config.py +++ b/src/hckr/cli/config.py @@ -1,5 +1,3 @@ -# from ..utils.MessageUtils import * - import click from ..utils.MessageUtils import PError, PSuccess From 470aeadcd3882e5462e2d3038d23728bfcb2bd10 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Thu, 29 Aug 2024 17:47:17 +0530 Subject: [PATCH 18/21] Add comprehensive configuration documentation and examples Introduced detailed Sphinx documentation for config and configure commands. Enhanced CLI help strings with examples and notes to guide users through the configuration process, including database setup. --- docs/source/commands/config/config.rst | 11 ++++++ docs/source/commands/config/configure.rst | 11 ++++++ docs/source/commands/config/index.rst | 23 +++++++++++++ docs/source/commands/index.rst | 7 ++++ src/hckr/cli/config.py | 6 +++- src/hckr/cli/configure.py | 42 ++++++++++++++++++++++- 6 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 docs/source/commands/config/config.rst create mode 100644 docs/source/commands/config/configure.rst create mode 100644 docs/source/commands/config/index.rst diff --git a/docs/source/commands/config/config.rst b/docs/source/commands/config/config.rst new file mode 100644 index 0000000..1a5442a --- /dev/null +++ b/docs/source/commands/config/config.rst @@ -0,0 +1,11 @@ +.. hckr documentation master file, created by + sphinx-quickstart on Wed Jun 12 20:06:39 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +.. important:: + ``DEFAULT`` config is the parent of all other configurations and others will inherit its values if not overridden + +.. click:: hckr.cli.config:config + :prog: hckr config + :nested: full diff --git a/docs/source/commands/config/configure.rst b/docs/source/commands/config/configure.rst new file mode 100644 index 0000000..9e0e769 --- /dev/null +++ b/docs/source/commands/config/configure.rst @@ -0,0 +1,11 @@ +.. hckr documentation master file, created by + sphinx-quickstart on Wed Jun 12 20:06:39 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + + .. important:::: + ``DEFAULT`` config is the parent of all other configurations and others will inherit its values if not overridden + +.. click:: hckr.cli.configure:configure + :prog: hckr configure + :nested: full diff --git a/docs/source/commands/config/index.rst b/docs/source/commands/config/index.rst new file mode 100644 index 0000000..c4b0e9e --- /dev/null +++ b/docs/source/commands/config/index.rst @@ -0,0 +1,23 @@ +.. hckr documentation master file, created by + sphinx-quickstart on Wed Jun 12 20:06:39 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + + +Configuration +=============== +``hckr`` supports various of config commands which helps in configuring other commands, eg. ``hckr db`` command. + +.. important:: + ``DEFAULT`` config is the parent of all other configurations and others will inherit its values if not overridden + +.. tip:: + More commands will be added in future updates. Stay tuned! + + +commands +--------------- +.. toctree:: + config + configure + diff --git a/docs/source/commands/index.rst b/docs/source/commands/index.rst index 372a943..ef87f6f 100644 --- a/docs/source/commands/index.rst +++ b/docs/source/commands/index.rst @@ -5,6 +5,13 @@ hckr :prog: hckr :nested: short +.. toctree:: + :maxdepth: 2 + :caption: Configure + :hidden: + + config/index + .. toctree:: :maxdepth: 2 :caption: Cron diff --git a/src/hckr/cli/config.py b/src/hckr/cli/config.py index 4eee502..4e0a10d 100644 --- a/src/hckr/cli/config.py +++ b/src/hckr/cli/config.py @@ -44,7 +44,6 @@ def set(config, key, value): **Example Usage**: * Setting a value inside DEFAULT config - Note - DEFAULT config is parent of all other configurations and others will inherit its values if not overridden .. code-block:: shell @@ -53,6 +52,7 @@ def set(config, key, value): * Similarly, we can also set a value in specific configuration, configuration will be created if not exists .. code-block:: shell + $ hckr config set database_host 127.0.0.1 --config MY_DATABASE **Command Reference**: @@ -83,6 +83,7 @@ def get(config, key): * Similarly, we can also get a value in specific configuration .. code-block:: shell + $ hckr config get database_host --config MY_DATABASE **Command Reference**: @@ -123,11 +124,13 @@ def show(config, all): * Similarly, we can also get all values in a specific configuration using -c/--config flag .. code-block:: shell + $ hckr config show -c MY_DATABASE * Additionally, we can also see all configurations using -a/--all flag .. code-block:: shell + $ hckr config show --all **Command Reference**: @@ -159,6 +162,7 @@ def init(overwrite): * Similarly, we can also delete existing file and recreate using -o/--overwrite flag .. code-block:: shell + $ hckr config init --overwrite **Command Reference**: diff --git a/src/hckr/cli/configure.py b/src/hckr/cli/configure.py index 211c3cc..397d2aa 100644 --- a/src/hckr/cli/configure.py +++ b/src/hckr/cli/configure.py @@ -67,7 +67,47 @@ def configure_db( warehouse, role, ): - """Configure database credentials based on the selected database type.""" + """ + This command configures database credentials based on the selected database type. + + .. hint:: + We currently support ``Postgres``, ``MySql``, ``Sqlite`` and ``Snowflake``, + Please feel free to raise an issue or Pull request if additional databases are needed. + + .. table:: Database Options mapping + :widths: auto + + ===== ===== + 1 Postgres + 2 Mysql + 3 Sqlite + 4 Snowflake + ===== ===== + + **Example Usage**: + + * Setting up your database configuration using Prompt values + + + .. code-block:: shell + + $ hckr configure db + + .. note:: + Using this cli will ask for information like database ``host``, ``port``, ``username`` etc. as per the database type selected. + + * Similarly, we can also provide all values using flag + + .. code-block:: shell + + $ hckr configure db --config-name test-config --database-type 3 --database-name mydb.sqlite + + .. note:: + here, we have configured ``Sqlite`` database with a name 'mydb.sqlite' by providing both flags in command line, + for other database types, we will have to provide required flags accordingly. + + **Command Reference**: + """ set_config_value(config_name, CONFIG_TYPE, ConfigType.DATABASE) selected_db_type = db_type_mapping[database_type] From bf8de9626ad63648431c7e03032cc0e2efeea361 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Thu, 29 Aug 2024 18:10:42 +0530 Subject: [PATCH 19/21] Add database command documentation and examples Updated the documentation to include a new section for database commands, with specific instructions and examples on how to configure and use these commands. Enhanced the `query` function docstring to provide detailed usage examples, including options for limiting rows and columns. --- docs/source/commands/config/configure.rst | 5 +--- docs/source/commands/database/db.rst | 7 +++++ docs/source/commands/database/index.rst | 15 ++++++++++ docs/source/commands/index.rst | 7 +++++ src/hckr/cli/db.py | 36 ++++++++++++++++++++++- 5 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 docs/source/commands/database/db.rst create mode 100644 docs/source/commands/database/index.rst diff --git a/docs/source/commands/config/configure.rst b/docs/source/commands/config/configure.rst index 9e0e769..801cba1 100644 --- a/docs/source/commands/config/configure.rst +++ b/docs/source/commands/config/configure.rst @@ -1,7 +1,4 @@ -.. hckr documentation master file, created by - sphinx-quickstart on Wed Jun 12 20:06:39 2024. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +.. _configuration: .. important:::: ``DEFAULT`` config is the parent of all other configurations and others will inherit its values if not overridden diff --git a/docs/source/commands/database/db.rst b/docs/source/commands/database/db.rst new file mode 100644 index 0000000..64e64e9 --- /dev/null +++ b/docs/source/commands/database/db.rst @@ -0,0 +1,7 @@ +.. important:: + Before using these commands you need to add your database configuration in config (``.hckrcfg`` file), please refer :ref:`Configuring your databases ` + + +.. click:: hckr.cli.db:db + :prog: hckr db + :nested: full diff --git a/docs/source/commands/database/index.rst b/docs/source/commands/database/index.rst new file mode 100644 index 0000000..5db3cef --- /dev/null +++ b/docs/source/commands/database/index.rst @@ -0,0 +1,15 @@ +Database +=============== +``hckr`` supports various of database commands. + +.. important:: + Before using these commands you need to add your database configuration in config (``.hckrcfg`` file), please refer :ref:`Configuring your database ` + +.. tip:: + More commands will be added in future updates. Stay tuned! + +commands +--------------- +.. toctree:: + db + diff --git a/docs/source/commands/index.rst b/docs/source/commands/index.rst index ef87f6f..1673572 100644 --- a/docs/source/commands/index.rst +++ b/docs/source/commands/index.rst @@ -34,6 +34,13 @@ hckr data/index +.. toctree:: + :maxdepth: 2 + :caption: Database + :hidden: + + database/index + .. toctree:: :maxdepth: 2 :caption: Hash diff --git a/src/hckr/cli/db.py b/src/hckr/cli/db.py index af6f5c2..6ecb482 100644 --- a/src/hckr/cli/db.py +++ b/src/hckr/cli/db.py @@ -36,7 +36,41 @@ def db(): ) @click.pass_context def query(ctx, config, query, num_rows=None, num_cols=None): - """Execute a SQL query on Snowflake and return a DataFrame or handle non-data-returning queries.""" + """ + This command executes a SQL query on your configured database and show you result in a table format ( in + ``SELECT/SHOW/DESC`` queries ) + + **Example Usage**: + + * Running a simple query on your configured database + + .. code-block:: shell + + $ hckr db query "select 1 as key, 'one' as value" -c testdb_sqlite + + .. table:: Output + :widths: auto + + ===== ========= + key value + ===== ========= + 1 one + ===== ========= + + + .. note:: + here, we have configured ``Sqlite`` database with in a config 'testdb_sqlite' + using :ref:`Configuring your databases ` + + + * Additionally, we can also limit number of records and columns returned by the query using ``-nr/--num-rows`` and ``-nc/--num-cols`` options + + .. code-block:: shell + + $ hckr db query "select 1 as key, 'one' as value" -c testdb_sqlite -nr 1 -nc 1 + + **Command Reference**: + """ db_url = get_db_url(section=config) if not db_url: PError("Database credentials are not properly configured.") From 62eab53435dfc714a09c52ffdc70c995a37b1266 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Thu, 29 Aug 2024 18:40:08 +0530 Subject: [PATCH 20/21] Refactor CLI tests to use cli_runner fixture Updated all CLI test functions to accept the cli_runner fixture instead of creating new CliRunner instances. This ensures better test consistency and readability. All occurrences of 'runner' were replaced with 'cli_runner' to reflect this change. --- tests/cli/test_config.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/tests/cli/test_config.py b/tests/cli/test_config.py index 7c5b8c9..5df50dd 100644 --- a/tests/cli/test_config.py +++ b/tests/cli/test_config.py @@ -13,51 +13,47 @@ def _get_random_string(length): # CONFIG GET AND SET -def test_config_get_set_default(): - runner = CliRunner() +def test_config_get_set_default(cli_runner): _key = f"key_{_get_random_string(5)}" _value = f"value_{_get_random_string(5)}" - result = runner.invoke(set, [_key, _value]) + result = cli_runner.invoke(set, [_key, _value]) assert result.exit_code == 0 assert f"[DEFAULT] {_key} <- {_value}" in result.output # testing get - result = runner.invoke(get, [_key]) + result = cli_runner.invoke(get, [_key]) assert result.exit_code == 0 assert f"[DEFAULT] {_key} = {_value}" in result.output -def test_config_get_set_custom_config(): - runner = CliRunner() +def test_config_get_set_custom_config(cli_runner): _key = f"key_{_get_random_string(5)}" _value = f"value_{_get_random_string(5)}" _CONFIG = "CUSTOM" - result = runner.invoke(set, ["--config", _CONFIG, _key, _value]) + result = cli_runner.invoke(set, ["--config", _CONFIG, _key, _value]) assert result.exit_code == 0 assert f"[{_CONFIG}] {_key} <- {_value}" in result.output # testing get - result = runner.invoke(get, ["--config", _CONFIG, _key]) + result = cli_runner.invoke(get, ["--config", _CONFIG, _key]) assert result.exit_code == 0 assert f"[{_CONFIG}] {_key} = {_value}" in result.output -def test_config_show(): - runner = CliRunner() +def test_config_show(cli_runner): _CONFIG = "CUSTOM" - result = runner.invoke(show) + result = cli_runner.invoke(show) assert result.exit_code == 0 assert "[DEFAULT]" in result.output - result = runner.invoke(show, ["--all"]) + result = cli_runner.invoke(show, ["--all"]) assert "[DEFAULT]" in result.output assert "[CUSTOM]" in result.output # NEGATIVE USE CASES -def test_config_get_set_missing_key(): - runner = CliRunner() - result = runner.invoke(set, []) +def test_config_get_set_missing_key(cli_runner): + result = cli_runner.invoke(set, []) print(result.output) assert result.exit_code != 0 assert "Error: Missing argument 'KEY'" in result.output @@ -69,9 +65,8 @@ def test_config_get_set_missing_key(): assert "Error: Missing argument 'KEY'" in result.output -def test_config_set_missing_value(): - runner = CliRunner() - result = runner.invoke(set, ["key"]) +def test_config_set_missing_value(cli_runner): + result = cli_runner.invoke(set, ["key"]) print(result.output) assert result.exit_code != 0 assert "Error: Missing argument 'VALUE'" in result.output From 7513969d18fe4c4046c9219d78360b5b22a8742f Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Thu, 29 Aug 2024 18:48:46 +0530 Subject: [PATCH 21/21] Update sonar exclusions to ignore specific Python files Added `src/hckr/__about__.py` and all `__init__.py` files to exclusion list to prevent them from being considered main source files in SonarQube analysis. This helps in focusing the static analysis on relevant pieces of code. --- sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index c0785d2..b7f3525 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -5,7 +5,7 @@ sonar.sources=src sonar.tests=tests # exclude for being considered main source files -sonar.exclusions=_build/**,docs/**,**/resources/**,.github/**,tests/**,out/** +sonar.exclusions=_build/**,docs/**,**/resources/**,.github/**,tests/**,out/**,src/hckr/__about__.py,**/__init__.py # Include test files sonar.test.inclusions=tests/**