diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1feceb75..d9738240 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -54,7 +54,7 @@ jobs: UV_SYSTEM_PYTHON: 1 run: | cp broker_settings.yaml.example ${BROKER_DIRECTORY}/broker_settings.yaml - uv pip install "broker[dev,docker] @ ." + uv pip install "broker[dev,podman] @ ." ls -l "$BROKER_DIRECTORY" broker --version pytest -v tests/ --ignore tests/functional diff --git a/broker/broker.py b/broker/broker.py index af9598e3..54b53ef8 100644 --- a/broker/broker.py +++ b/broker/broker.py @@ -68,6 +68,7 @@ def mp_split(*args, **kwargs): results = [] max_workers_count = self.MAX_WORKERS or count with self.EXECUTOR(max_workers=max_workers_count) as workers: + logger.warning(f"{self.func=}, {instance=}, {args=}, {kwargs=}") completed_futures = as_completed( workers.submit(self.func, instance, *args, **kwargs) for _ in range(count) ) diff --git a/broker/commands.py b/broker/commands.py index 98b9a442..e613467a 100644 --- a/broker/commands.py +++ b/broker/commands.py @@ -4,17 +4,28 @@ import signal import sys -import click from logzero import logger from rich.console import Console +from rich.syntax import Syntax from rich.table import Table +import rich_click as click from broker import exceptions, helpers, settings from broker.broker import Broker +from broker.config_manager import ConfigManager from broker.logger import LOG_LEVEL from broker.providers import PROVIDER_ACTIONS, PROVIDER_HELP, PROVIDERS signal.signal(signal.SIGINT, helpers.handle_keyboardinterrupt) +CONSOLE = Console() # rich console for pretty printing + +click.rich_click.SHOW_ARGUMENTS = True +click.rich_click.COMMAND_GROUPS = { + "broker": [ + {"name": "Core Actions", "commands": ["checkout", "checkin", "inventory"]}, + {"name": "Extras", "commands": ["execute", "extend", "providers", "config"]}, + ] +} def loggedcli(group=None, *cli_args, **cli_kwargs): @@ -47,7 +58,7 @@ def parse_labels(provider_labels): } -class ExceptionHandler(click.Group): +class ExceptionHandler(click.RichGroup): """Wraps click group to catch and handle raised exceptions.""" def __call__(self, *args, **kwargs): @@ -167,13 +178,9 @@ def provider_cmd(ctx, *args, **kwargs): # the actual subcommand def cli(version): """Command-line interface for interacting with providers.""" if version: - from importlib.metadata import version - from packaging.version import Version import requests - broker_version = version("broker") - # Check against the latest version published to PyPi try: latest_version = Version( @@ -181,19 +188,28 @@ def cli(version): "version" ] ) - if latest_version > Version(broker_version): + if latest_version > Version(ConfigManager.version): click.secho( f"A newer version of broker is available: {latest_version}", fg="yellow", ) except requests.exceptions.RequestException as err: logger.warning(f"Unable to check for latest version: {err}") - click.echo(f"Version: {broker_version}") - broker_directory = settings.BROKER_DIRECTORY.absolute() - click.echo(f"Broker Directory: {broker_directory}") - click.echo(f"Settings File: {settings.settings_path.absolute()}") - click.echo(f"Inventory File: {broker_directory}/inventory.yaml") - click.echo(f"Log File: {broker_directory}/logs/broker.log") + + # Create a rich table + table = Table(title=f"Broker {ConfigManager.version}") + + table.add_column("", justify="left", style="cyan", no_wrap=True) + table.add_column("Location", justify="left", style="magenta") + + table.add_row("Broker Directory", str(settings.BROKER_DIRECTORY.absolute())) + table.add_row("Settings File", str(settings.settings_path.absolute())) + table.add_row("Inventory File", f"{settings.BROKER_DIRECTORY.absolute()}/inventory.yaml") + table.add_row("Log File", f"{settings.BROKER_DIRECTORY.absolute()}/logs/broker.log") + + # Print the table + console = Console() + console.print(table) @loggedcli(context_settings={"allow_extra_args": True, "ignore_unknown_options": True}) @@ -288,7 +304,6 @@ def inventory(details, curated, sync, filter): inventory = helpers.load_inventory(filter=filter) helpers.emit({"inventory": inventory}) if curated: - console = Console() table = Table(title="Host Inventory") table.add_column("Id", justify="left", style="cyan", no_wrap=True) @@ -302,7 +317,7 @@ def inventory(details, curated, sync, filter): str(host["id"]), host["host"], host["provider"], host["action"], host["os"] ) - console.print(table) + CONSOLE.print(table) return for num, host in enumerate(inventory): if (display_name := host.get("hostname")) is None: @@ -400,3 +415,111 @@ def execute(ctx, background, nick, output_format, artifacts, args_file, provider logger.info(result) elif output_format == "yaml": click.echo(helpers.yaml_format(result)) + + +@cli.group(cls=ExceptionHandler) +def config(): + """View and manage Broker's configuration. + + Note: One important concept of these commands is the concept of a "chunk". + + A chunk is a part of the configuration file that can be accessed or updated. + Chunks are specified by their keys in the configuration file. + Nested chunks are separated by periods. + + e.g. broker config view AnsibleTower.instances.my_instance + """ + + +@loggedcli(group=config) +@click.argument("chunk", type=str, required=False) +@click.option("--no-syntax", is_flag=True, help="Disable syntax highlighting") +# context_settings={"allow_extra_args": True, "ignore_unknown_options": True} +def view(chunk, no_syntax): + """View all or part of the broker configuration.""" + result = helpers.yaml_format(ConfigManager(settings.settings_path).get(chunk)) + if no_syntax: + CONSOLE.print(result) + else: + CONSOLE.print(Syntax(result, "yaml", background_color="default")) + + +@loggedcli(group=config) +@click.argument("chunk", type=str, required=False) +def edit(chunk): + """Directly edit the broker configuration file. + + You can define the scope of the edit by specifying a chunk. + Otherwise, the entire configuration file will be opened. + """ + ConfigManager(settings.settings_path).edit(chunk) + + +@loggedcli(group=config, name="set") +@click.argument("chunk", type=str, required=True) +@click.argument("new-value", type=str, required=True) +def _set(chunk, new_value): + """Set a value in the Broker configuration file. + + These updates take the form of ` ` pairs. + You can also pass a yaml or json file containing the new contents of a chunk. + """ + new_value = helpers.resolve_file_args({"nv": new_value})["nv"] + ConfigManager(settings.settings_path).update(chunk, new_value) + + +@loggedcli(group=config) +def restore(): + """Restore the broker configuration file to the last backup.""" + ConfigManager(settings.settings_path).restore() + + +@loggedcli(group=config) +@click.argument("chunk", type=str, required=False) +def init(chunk): + """Initialize the broker configuration file from your local clone or GitHub. + + You can also init specific chunks by passing the chunk name. + """ + ConfigManager(settings.settings_path).init_config_file(chunk) + + +@loggedcli(group=config) +def nicks(): + """Get a list of nicks.""" + result = ConfigManager(settings.settings_path).nicks() + CONSOLE.print("\n".join(result)) + + +@loggedcli(group=config) +@click.argument("nick", type=str, required=True) +@click.option("--no-syntax", is_flag=True, help="Disable syntax highlighting") +def nick(nick, no_syntax): + """Get information about a specific nick.""" + result = helpers.yaml_format(ConfigManager(settings.settings_path).nicks(nick)) + if no_syntax: + CONSOLE.print(result) + else: + CONSOLE.print(Syntax(result, "yaml", background_color="default")) + + +@loggedcli(group=config) +def migrate(): + """Migrate the broker configuration file to the latest version.""" + ConfigManager(settings.settings_path).migrate() + + +@loggedcli(group=config) +@click.argument("chunk", type=str, required=False, default="base") +def validate(chunk): + """Validate top-level chunks of the broker configuration file. + + You can validate against the `base` settings by default or specify a provider. + + To validate everything, pass `all` + """ + try: + ConfigManager(settings.settings_path).validate(chunk, PROVIDERS) + logger.info("Validation passed!") + except exceptions.BrokerError as err: + logger.warning(f"Validation failed: {err}") diff --git a/broker/config_manager.py b/broker/config_manager.py new file mode 100644 index 00000000..05eec591 --- /dev/null +++ b/broker/config_manager.py @@ -0,0 +1,264 @@ +"""Module providing the functionality powering the `broker config` command.""" + +import importlib +from importlib.metadata import version +import inspect +import json +import os +from pathlib import Path +import pkgutil +from tempfile import NamedTemporaryFile + +import click +from logzero import logger +from packaging.version import Version +import yaml + +from broker import exceptions, helpers + +C_SEP = "." # chunk separator +GH_CFG = "https://raw.githubusercontent.com/SatelliteQE/broker/master/broker_settings.yaml.example" + + +def file_name_to_ver(file_name): + """Convert a versoin encoded filename `v0_6_0` to a `Version` object.""" + return Version(file_name[1:].replace("_", ".")) + + +class ConfigManager: + """Class to interact with Broker's configuration file. + + One important concept of these commands is the concept of a "chunk". + + A chunk is a part of the configuration file that can be accessed or updated. + Chunks are specified by their keys in the configuration file. + Nested chunks are separated by periods. + + e.g. broker config view AnsibleTower.instances.my_instance + """ + + version = version("broker") + + def __init__(self, settings_path=None): + self._interactive_mode = None + self._settings_path = settings_path + if settings_path: + self._cfg = yaml.safe_load(self._settings_path.read_text()) + + @property + def interactive_mode(self): + """Determine if Broker is running in interactive mode.""" + if self._interactive_mode is None: + self._interactive_mode = False + # GitHub action context + if "GITHUB_WORKFLOW" not in os.environ: + # determine if we're being ran from a CLI + for frame in inspect.stack()[::-1]: + if "/bin/broker" in frame.filename: + self._interactive_mode = True + break + return self._interactive_mode + + def _interactive_edit(self, chunk): + """Write the chunk data to a temporary file and open it in an editor.""" + with NamedTemporaryFile(mode="w+", suffix=".yaml") as tmp: + tmp.write(helpers.yaml_format(chunk)) + tmp.seek(0) + click.edit(filename=tmp.name) + tmp.seek(0) + new_data = tmp.read() + # first try to load it as yaml + try: + return yaml.safe_load(new_data) + except yaml.YAMLError: # then try json + try: + return json.loads(new_data) + except json.JSONDecodeError: # finally, just return the raw data + return new_data + + def _import_config(self, source, is_url=False): + """Initialize the broker settings file from a source.""" + proceed = True + if self.interactive_mode: + try: + proceed = click.confirm(f"Get example file from {source}?") + except click.core.Abort: + # We're likely in a different non-interactive environment (container?) + self._interactive_mode = False + if not proceed: + return + # get example file from source + if is_url: + import requests + + click.echo(f"Downloading example file from: {source}") + return requests.get(source, timeout=60).text + else: + return source.read_text() + + def _get_migrations(self): + """Construct a dict of all applicable migrations.""" + from broker import config_migrations + + config_version = Version(self._cfg.get("_version", "0.0.0")) + migrations = [] + for _, name, _ in pkgutil.iter_modules(config_migrations.__path__): + module = importlib.import_module(f"broker.config_migrations.{name}") + if hasattr(module, "run_migrations") and config_version < file_name_to_ver(name): + migrations.append(module) + return migrations + + def backup(self): + """Backup the current configuration file.""" + logger.debug( + f"Backing up the configuration file to {self._settings_path.with_suffix('.bak')}" + ) + self._settings_path.with_suffix(".bak").write_text(self._settings_path.read_text()) + + def restore(self): + """Restore the configuration file from a backup if it exists.""" + logger.debug( + f"Restoring the configuration file from {self._settings_path.with_suffix('.bak')}" + ) + backup_path = self._settings_path.with_suffix(".bak") + if not backup_path.exists(): + raise exceptions.UserError("No backup file found.") + self._settings_path.write_text(backup_path.read_text()) + + def edit(self, chunk=None, content=None): + """Open the config file in an editor.""" + if not self.interactive_mode: + raise exceptions.UserError( + "Attempted to edit the config in non-interactive mode.\n" + "Did you mean to use the `set` method instead?" + ) + content = content or self.get(chunk=chunk) + new_val = self._interactive_edit(content) + self.update(chunk, new_val) + + def get(self, chunk=None, curr_chunk=None): + """Get a chunk of Broker's config or the whole config.""" + if not curr_chunk: + curr_chunk = self._cfg + if not chunk: + return curr_chunk + if C_SEP in chunk: + curr, chunk = chunk.split(C_SEP, 1) + # curr = int(curr) if curr.isdigit() else curr + return self.get(chunk, curr_chunk=curr_chunk[curr]) + else: + # chunk = int(chunk) if chunk.isdigit() else chunk + try: + return curr_chunk[chunk] + except KeyError: + raise exceptions.UserError(f"Chunk '{chunk}' not found in the config.") + + def update(self, chunk, new_val, curr_chunk=None): + """Update a chunk of Broker's config or the whole config.""" + # Recursive down to find the chunk to update, then propagate the new value back up + if not curr_chunk: # we're at the top level, so update the config directly + if chunk is None: # the whole config is being updated + self._cfg = new_val + elif C_SEP in chunk: # the update needs to happen at a lower level + curr, chunk = chunk.split(C_SEP, 1) + self._cfg[curr] = self.update(chunk, new_val, curr_chunk=self._cfg[curr]) + else: + self._cfg[chunk] = new_val + # update the config file + self.backup() + self._settings_path.write_text( + yaml.dump(self._cfg, default_flow_style=False, sort_keys=False) + ) + else: # we're not at the top level, so keep going down + if C_SEP in chunk: + curr, chunk = chunk.split(C_SEP, 1) + curr_chunk[curr] = self.update(chunk, new_val, curr_chunk=curr_chunk[curr]) + else: + curr_chunk[chunk] = new_val + return curr_chunk + + def nicks(self, nick=None): + """Get a list of nicks or single nick information.""" + nicks = self.get("nicks") + if nick: + return nicks[nick] + return list(nicks.keys()) + + def init_config_file(self, chunk=None): + """Check for the existence of the config file and create it if it doesn't exist.""" + if self.interactive_mode and self._settings_path.exists() and not chunk: + # if the file exists, ask the user if they want to overwrite it + if ( + click.prompt( + f"Settings file already exists at {self._settings_path.absolute()}. Overwrite?", + type=click.Choice(["y", "n"]), + default="n", + ) + != "y" + ): + return + # get the example file from the local repo or GitHub + example_path = Path(__file__).parent.parent.joinpath("broker_settings.yaml.example") + raw_data = None + if example_path.exists(): + raw_data = self._import_config(example_path) + if not raw_data: + raw_data = self._import_config(GH_CFG, is_url=True) + if not raw_data: + raise exceptions.ConfigurationError( + f"Broker settings file not found at {self._settings_path.absolute()}." + ) + chunk_data = self.get(chunk, yaml.safe_load(raw_data)) + edited_chunk = self._interactive_edit(chunk_data) + self.update(chunk, edited_chunk) + + def migrate(self): + """Migrate the config from a previous version of Broker.""" + # get all available migrations + if not (migrations := self._get_migrations()): + logger.info("No migrations are applicable to your config.") + return + # run all migrations in order + working_config = self._cfg + for migration in sorted(migrations): + working_config = migration.run_migrations(working_config) + self.backup() + self._settings_path.write_text( + yaml.dump(working_config, default_flow_style=False, sort_keys=False) + ) + logger.info("Config migration complete.") + + def validate(self, chunk, providers=None): + """Validate a top-level chunk of Broker's config.""" + if "." in chunk: # only validate the top-level chunk + chunk = chunk.split(".")[0] + if chunk.lower() == "base": # validate the base config + return # this happens before we're called + if chunk.lower() == "ssh": + from broker.settings import settings + + settings.validators.validate(only="SSH") + return + if providers is None: + raise exceptions.UserError( + "Attempted to validate provider settings without passing providers." + ) + if ":" in chunk: + prov_name, instance = chunk.split(":") + providers[prov_name](**{prov_name: instance}) + if chunk == "all": + for prov_name, prov_cls in providers.items(): + if prov_name == "TestProvider": + continue + logger.info(f"Validating {prov_name} provider settings.") + try: # we want to suppress all exceptions here to allow each provider to validate + prov_cls() + except Exception as err: # noqa: BLE001 + logger.warning(f"Provider {prov_name} failed validation: {err}") + return + if chunk not in providers: + raise exceptions.UserError( + "I don't know how to validate that.\n" + "If it's important, it is likely covered in the base validations." + ) + providers[chunk]() diff --git a/broker/config_migrations/v0_6_0.py b/broker/config_migrations/v0_6_0.py new file mode 100644 index 00000000..35485177 --- /dev/null +++ b/broker/config_migrations/v0_6_0.py @@ -0,0 +1,62 @@ +"""Config migrations for versions older than 0.6.0 to 0.6.0.""" +from logzero import logger + + +def migrate_instances(config_dict): + """Migrate instances from a list of dicts to a dict of dicts.""" + logger.debug("Migrating instances from a list to a dict.") + for key, val in config_dict.items(): + if not isinstance(val, dict): + continue + if "instances" in val and isinstance(val["instances"], list): + old_instances = val.pop("instances") + val["instances"] = {} + for inst in old_instances: + val["instances"].update(inst) + config_dict[key] = val + return config_dict + + +def remove_testprovider(config_dict): + """Remove the testprovider from the config.""" + logger.debug("Removing the testprovider from the config.") + config_dict.pop("TestProvider", None) + return config_dict + + +def remove_test_nick(config_dict): + """Remove the test nick from the config.""" + logger.debug("Removing the test nick from the config.") + nicks = config_dict.get("nicks", {}) + nicks.pop("test_nick", None) + config_dict["nicks"] = nicks + return config_dict + + +def move_ssh_settings(config_dict): + """Move SSH settings from the top leve into its own chunk.""" + logger.debug("Moving SSH settings into their own section.") + ssh_settings = { + "backend": config_dict.pop("ssh_backend", "ssh2-python312"), + "host_username": config_dict.pop("host_username", "root"), + "host_password": config_dict.pop("host_password", "toor"), + "host_ipv6": config_dict.pop("host_ipv6", False), + "host_ipv4_fallback": config_dict.pop("host_ipv4_fallback", True), + } + if ssh_port := config_dict.pop("host_ssh_port", None): + ssh_settings["ssh_port"] = ssh_port + if ssh_key := config_dict.pop("host_ssh_key_filename", None): + ssh_settings["host_ssh_key_filename"] = ssh_key + config_dict["ssh"] = ssh_settings + return config_dict + + +def run_migrations(config_dict): + """Run all migrations.""" + logger.info("Running config migrations for 0.6.0.") + config_dict = migrate_instances(config_dict) + config_dict = remove_testprovider(config_dict) + config_dict = remove_test_nick(config_dict) + config_dict = move_ssh_settings(config_dict) + config_dict["_version"] = "0.6.0" + return config_dict diff --git a/broker/hosts.py b/broker/hosts.py index 1149469f..bbc0095d 100644 --- a/broker/hosts.py +++ b/broker/hosts.py @@ -17,9 +17,10 @@ from logzero import logger from broker.exceptions import HostError, NotImplementedError -from broker.session import ContainerSession, Session from broker.settings import settings +SETTINGS_VALIDATED = False + class Host: """Class representing a host that can be accessed via SSH or Bind. @@ -45,6 +46,11 @@ def __init__(self, **kwargs): ipv6 (bool): Whether or not to use IPv6. Defaults to False. ipv4_fallback (bool): Whether or not to fallback to IPv4 if IPv6 fails. Defaults to True. """ + global SETTINGS_VALIDATED # noqa: PLW0603 + if not SETTINGS_VALIDATED: + logger.debug("Validating ssh settings") + settings.validators.validate(only="SSH") + SETTINGS_VALIDATED = True logger.debug(f"Constructing host using {kwargs=}") self.hostname = kwargs.get("hostname") or kwargs.get("ip") if not self.hostname: @@ -56,13 +62,13 @@ def __init__(self, **kwargs): else: raise HostError("Host must be constructed with a hostname or ip") self.name = kwargs.pop("name", None) - self.username = kwargs.pop("username", settings.HOST_USERNAME) - self.password = kwargs.pop("password", settings.HOST_PASSWORD) - self.timeout = kwargs.pop("connection_timeout", settings.HOST_CONNECTION_TIMEOUT) - self.port = kwargs.pop("port", settings.HOST_SSH_PORT) - self.key_filename = kwargs.pop("key_filename", settings.HOST_SSH_KEY_FILENAME) - self.ipv6 = kwargs.pop("ipv6", settings.HOST_IPV6) - self.ipv4_fallback = kwargs.pop("ipv4_fallback", settings.HOST_IPV4_FALLBACK) + self.username = kwargs.pop("username", settings.SSH.HOST_USERNAME) + self.password = kwargs.pop("password", settings.SSH.HOST_PASSWORD) + self.timeout = kwargs.pop("connection_timeout", settings.SSH.HOST_CONNECTION_TIMEOUT) + self.port = kwargs.pop("port", settings.SSH.HOST_SSH_PORT) + self.key_filename = kwargs.pop("key_filename", settings.SSH.HOST_SSH_KEY_FILENAME) + self.ipv6 = kwargs.pop("ipv6", settings.SSH.HOST_IPV6) + self.ipv4_fallback = kwargs.pop("ipv4_fallback", settings.SSH.HOST_IPV4_FALLBACK) self.__dict__.update(kwargs) # Make every other kwarg an attribute self._session = None @@ -79,10 +85,11 @@ def session(self): If the session object does not exist, it will be created by calling the `connect` method. If the host is a non-SSH-enabled container host, a `ContainerSession` object will be created instead. """ - # This attribute may be missing after pickling - if not isinstance(getattr(self, "_session", None), Session): + if self._session is None: # Check to see if we're a non-ssh-enabled Container Host if hasattr(self, "_cont_inst") and not self._cont_inst.ports.get(22): + from broker.session import ContainerSession + runtime = "podman" if "podman" in str(self._cont_inst.client) else "docker" self._session = ContainerSession(self, runtime=runtime) else: @@ -110,6 +117,8 @@ def connect( ipv6 (bool): Whether or not to use IPv6. Defaults to False. ipv4_fallback (bool): Whether or not to fallback to IPv4 if IPv6 fails. Defaults to True. """ + from broker.session import Session + username = username or self.username password = password or self.password timeout = timeout or self.timeout @@ -135,8 +144,7 @@ def connect( def close(self): """Close the SSH connection to the host.""" - # This attribute may be missing after pickling - if isinstance(getattr(self, "_session", None), Session): + if self._session is not None: self._session.disconnect() self._session = None diff --git a/broker/providers/__init__.py b/broker/providers/__init__.py index 5f3ad5de..2712175f 100644 --- a/broker/providers/__init__.py +++ b/broker/providers/__init__.py @@ -116,7 +116,7 @@ def __init__(self, **kwargs): def _validate_settings(self, instance_name=None): """Load and validate provider settings. - Each provider's settings can include an instances list with specific instance + Each provider's settings can include an instances dict with specific instance details. One instance should have a "default" key set to True, if instances are defined. General provider settings should live on the top level for that provider. @@ -128,19 +128,17 @@ def _validate_settings(self, instance_name=None): if self._fresh_settings.get(section_name).get("instances"): fresh_settings = self._fresh_settings.get(section_name).copy() instance_name = instance_name or getattr(self, "instance", None) - # iterate through the instances and find the one that matches the instance_name - # if no instance matches, use the default instance - for candidate in fresh_settings.instances: - logger.debug("Checking %s against %s", instance_name, candidate) - if instance_name in candidate: - instance = candidate - break - elif candidate.values()[0].get("default") or len(fresh_settings.instances) == 1: - instance = candidate - self.instance, *_ = instance # store the instance name on the provider - fresh_settings.update(inst_vals := instance.values()[0]) + # first check to see if we have a direct match + if not (instance_values := fresh_settings.instances.get(instance_name)): + # if no direct match is found, or no instance is provided, find the default + for name, values in fresh_settings.instances.items(): + if values.get("default") or len(fresh_settings.instances) == 1: + instance_name, instance_values = name, values + break + self.instance = instance_name # store the instance name on the provider + fresh_settings.update(instance_values) settings[section_name] = fresh_settings - if not inst_vals.get("override_envars"): + if not instance_values.get("override_envars"): # if a provider instance doesn't want to override envars, load them settings.execute_loaders(loaders=[dynaconf.loaders.env_loader]) new_validators = [v for v in self._validators if v not in settings.validators] diff --git a/broker/session.py b/broker/session.py index 4abddfd9..a227c388 100644 --- a/broker/session.py +++ b/broker/session.py @@ -12,6 +12,7 @@ from pathlib import Path import tempfile +from dynaconf import Validator from logzero import logger from broker import helpers @@ -19,7 +20,7 @@ from broker.settings import settings SSH_BACKENDS = ("ssh2-python", "ssh2-python312", "ansible-pylibssh", "hussh") -SSH_BACKEND = settings.SSH_BACKEND +SSH_BACKEND = settings.BACKEND logger.debug(f"{SSH_BACKEND=}") @@ -56,6 +57,19 @@ def __init__(self, **kwargs): """Initialize a Session object.""" +# validate the ssh settings +validators = [ + Validator("SSH.HOST_USERNAME", default="root"), + Validator("SSH.HOST_PASSWORD", default="toor"), + Validator("SSH.HOST_CONNECTION_TIMEOUT", default=60), + Validator("SSH.HOST_SSH_PORT", default=22), + Validator("SSH.HOST_SSH_KEY_FILENAME", default=None), + Validator("SSH.HOST_IPV6", default=False), + Validator("SSH.HOST_IPV4_FALLBACK", default=True), + Validator("SSH.SSH_BACKEND", default="ssh2-python312"), +] + + class ContainerSession: """An approximation of ssh-based functionality from the Session class.""" diff --git a/broker/settings.py b/broker/settings.py index 0bd758b9..785e05af 100644 --- a/broker/settings.py +++ b/broker/settings.py @@ -6,88 +6,28 @@ validate_settings: Function to validate the settings file. INTERACTIVE_MODE: Whether or not Broker is running in interactive mode. BROKER_DIRECTORY: The directory where Broker looks for its files. + TEST_MODE: Whether or not Broker is running in a pytest session. settings_path: The path to the settings file. inventory_path: The path to the inventory file. """ -import inspect import os from pathlib import Path +import sys import click from dynaconf import Dynaconf, Validator from dynaconf.validator import ValidationError +from broker.config_manager import ConfigManager from broker.exceptions import ConfigurationError - -def init_settings(settings_path, source, interactive=False, is_url=False): - """Initialize the broker settings file.""" - proceed = not False - if interactive: - try: - proceed = ( - click.prompt( - f"Get example file from {source}?\n", - type=click.Choice(["y", "n"]), - default="y", - ) - == "y" - ) - except click.core.Abort: - # We're likely in a different non-interactive environment (container?) - global INTERACTIVE_MODE - proceed, INTERACTIVE_MODE = True, False - if proceed: - # get example file from source - if is_url: - import requests - - click.echo(f"Downloading example file from: {source}") - raw_file = requests.get(source, timeout=60) - settings_path.write_text(raw_file.text) - else: - example_file = source.read_text() - settings_path.write_text(example_file) - if INTERACTIVE_MODE: - try: - click.edit(filename=str(settings_path.absolute())) - except click.exceptions.ClickException: - click.secho( - f"Please edit the file {settings_path.absolute()} and add your settings.", - fg="yellow", - ) - return True - - -def init_settings_from_github(settings_path, interactive=False): - """Initialize the broker settings file.""" - raw_url = ( - "https://raw.githubusercontent.com/SatelliteQE/broker/master/broker_settings.yaml.example" - ) - return init_settings(settings_path, raw_url, interactive, is_url=True) - - -def init_settings_from_local_repo(settings_path, interactive=False): - """Initialize the broker settings file.""" - example_path = Path(__file__).parent.parent.joinpath("broker_settings.yaml.example") - if not example_path.exists(): - return - return init_settings(settings_path, example_path, interactive) - - -INTERACTIVE_MODE = False -# GitHub action context -if "GITHUB_WORKFLOW" not in os.environ: - # determine if we're being ran from a CLI - for frame in inspect.stack()[::-1]: - if "/bin/broker" in frame.filename: - INTERACTIVE_MODE = True - break - - +INTERACTIVE_MODE = ConfigManager().interactive_mode BROKER_DIRECTORY = Path.home().joinpath(".broker") +TEST_MODE = "pytest" in sys.modules -if "BROKER_DIRECTORY" in os.environ: +if TEST_MODE: # when in test mode, don't use the real broker directory + BROKER_DIRECTORY = Path("tests/data/") +elif "BROKER_DIRECTORY" in os.environ: envar_location = Path(os.environ["BROKER_DIRECTORY"]) if envar_location.is_dir(): BROKER_DIRECTORY = envar_location @@ -97,23 +37,36 @@ def init_settings_from_local_repo(settings_path, interactive=False): settings_path = BROKER_DIRECTORY.joinpath("broker_settings.yaml") inventory_path = BROKER_DIRECTORY.joinpath("inventory.yaml") +cfg_manager = ConfigManager(settings_path) if not settings_path.exists(): click.secho(f"Broker settings file not found at {settings_path.absolute()}.", fg="red") - if not (success := init_settings_from_local_repo(settings_path, interactive=INTERACTIVE_MODE)): - success = init_settings_from_github(settings_path, interactive=INTERACTIVE_MODE) - if not success: - raise ConfigurationError(f"Broker settings file not found at {settings_path.absolute()}.") + cfg_manager.init_config_file() + +if cfg_manager._get_migrations() and not TEST_MODE: + if INTERACTIVE_MODE: + click.secho( + "Broker settings file has pending migrations.\n" + "Continuing without running the migrations may cause errors.", + fg="red", + ) + if click.confirm("Would you like to run the migrations now?"): + cfg_manager.migrate() + else: + click.secho("Continuing without running migrations.", fg="yellow") + else: + cfg_manager.migrate() validators = [ - Validator("HOST_USERNAME", default="root"), - Validator("HOST_PASSWORD", default="toor"), - Validator("HOST_CONNECTION_TIMEOUT", default=60), - Validator("HOST_SSH_PORT", default=22), - Validator("HOST_SSH_KEY_FILENAME", default=None), - Validator("HOST_IPV6", default=False), - Validator("HOST_IPV4_FALLBACK", default=True), - Validator("SSH_BACKEND", default="ssh2-python312"), + Validator("SSH", is_type_of=dict), + Validator("SSH.HOST_USERNAME", default="root"), + Validator("SSH.HOST_PASSWORD", default="toor"), + Validator("SSH.HOST_CONNECTION_TIMEOUT", default=60), + Validator("SSH.HOST_SSH_PORT", default=22), + Validator("SSH.HOST_SSH_KEY_FILENAME", default=None), + Validator("SSH.HOST_IPV6", default=False), + Validator("SSH.HOST_IPV4_FALLBACK", default=True), + Validator("SSH.BACKEND", default="ssh2-python312"), Validator("LOGGING", is_type_of=dict), Validator( "LOGGING.CONSOLE_LEVEL", @@ -141,7 +94,7 @@ def init_settings_from_local_repo(settings_path, interactive=False): settings._loaders = [loader for loader in settings._loaders if "vault" not in loader] try: - settings.validators.validate() + settings.validators.validate(only="LOGGING") except ValidationError as err: raise ConfigurationError( f"Configuration error in {settings_path.absolute()}: {err.args[0]}" diff --git a/broker_settings.yaml.example b/broker_settings.yaml.example index cdb57e30..23a39852 100644 --- a/broker_settings.yaml.example +++ b/broker_settings.yaml.example @@ -1,19 +1,22 @@ # Broker settings +_version: 0.6.0 # different log levels for file and stdout logging: console_level: info file_level: debug -# Host Settings +# Host SSH Settings # These can be left alone if you're not using Broker as a library -host_username: root -host_password: "" -host_ssh_port: 22 -host_ssh_key_filename: "" -# Default all host ssh connections to IPv6 -host_ipv6: False -# If IPv6 connection attempts fail, fallback to IPv4 -host_ipv4_fallback: True -ssh_backend: ssh2-python312 +ssh: + # this is the library Broker should use to perform ssh actions + backend: ssh2-python312 + host_username: root + host_password: "" + host_ssh_port: 22 + host_ssh_key_filename: "" + # Default all host ssh connections to IPv6 + host_ipv6: False + # If IPv6 connection attempts fail, fallback to IPv4 + host_ipv4_fallback: True # Provider settings AnsibleTower: base_url: "https:///" @@ -30,14 +33,14 @@ AnsibleTower: results_limit: 50 Container: instances: - - docker: + docker: host_username: "" host_password: "" host_port: None runtime: docker network: null default: True - - remote: + remote: host: "" host_username: "" host_password: "" @@ -49,7 +52,7 @@ Container: auto_map_ports: False Foreman: instances: - - foreman1: + foreman1: foreman_url: https://test.fore.man foreman_username: admin foreman_password: secret @@ -57,7 +60,7 @@ Foreman: location: LOC verify: ./ca.crt default: true - - foreman2: + foreman2: foreman_url: https://other-test.fore.man foreman_username: admin foreman_password: secret @@ -67,25 +70,9 @@ Foreman: Beaker: hub_url: max_job_wait: 24h -TestProvider: - instances: - - test1: - foo: "bar" - default: True - - test2: - foo: "baz" - override_envars: True - - bad: - nothing: False - config_value: "something" # You can set a nickname as a shortcut for arguments nicks: - rhel7: - workflow: "deploy-rhel" - deploy_rhel_version: "7.9" + rhel9: + workflow: deploy-rhel + deploy_rhel_version: 9.4 notes: "Requested by broker" - test_nick: - test_action: "fake" - arg1: "abc" - arg2: 123 - arg3: True diff --git a/pyproject.toml b/pyproject.toml index cfc0fece..3d398857 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "packaging", "pyyaml", "rich", + "rich_click", "setuptools", "ssh2-python312", ] diff --git a/tests/data/broker_settings.yaml b/tests/data/broker_settings.yaml new file mode 100644 index 00000000..e2241c4a --- /dev/null +++ b/tests/data/broker_settings.yaml @@ -0,0 +1,58 @@ +_version: 0.6.0 +logging: + console_level: info + file_level: debug +host_username: root +host_password: +host_ssh_port: 22 +host_ssh_key_filename: +host_ipv6: false +host_ipv4_fallback: true +ssh_backend: ssh2-python312 +AnsibleTower: + base_url: https:/// + username: + token: + release_workflow: remove-vm + extend_workflow: extend-vm + new_expire_time: '+172800' + workflow_timeout: 3600 + results_limit: 50 +Container: + host_username: + host_password: + host_port: None + network: null + default: true + runtime: podman + results_limit: 50 + auto_map_ports: false +Foreman: + foreman_url: https://test.fore.man + foreman_username: admin + foreman_password: secret + organization: ORG + location: LOC + verify: ./ca.crt + name_prefix: broker +Beaker: + hub_url: null + max_job_wait: 24h +TestProvider: + instances: + test1: + foo: "bar" + default: True + test2: + foo: "baz" + override_envars: True + bad: + nothing: False + config_value: "something" +# You can set a nickname as a shortcut for arguments +nicks: + test_nick: + test_action: "fake" + arg1: "abc" + arg2: 123 + arg3: True diff --git a/tests/functional/test_containers.py b/tests/functional/test_containers.py index 2e91dca7..0847e868 100644 --- a/tests/functional/test_containers.py +++ b/tests/functional/test_containers.py @@ -20,13 +20,10 @@ def skip_if_not_configured(): @pytest.fixture(scope="module") -def temp_inventory(): - """Temporarily move the local inventory, then move it back when done""" - backup_path = inventory_path.rename(f"{inventory_path.absolute()}.bak") +def checkin_containers(): + """Checkin all containers checkout out by the tests.""" yield CliRunner().invoke(cli, ["checkin", "--all", "--filter", "_broker_provider