0.6 Changes

.github/workflows/codeql-analysis.yml

@@ -50,11 +50,8 @@ jobs:
 
       - name: Unit Tests
         env:
-          BROKER_DIRECTORY: "${{ github.workspace }}/broker_dir"
           UV_SYSTEM_PYTHON: 1
         run: |
-          cp broker_settings.yaml.example ${BROKER_DIRECTORY}/broker_settings.yaml
-          uv pip install "broker[dev,docker] @ ."
-          ls -l "$BROKER_DIRECTORY"
+          uv pip install "broker[dev,podman] @ ."
           broker --version
           pytest -v tests/ --ignore tests/functional

broker/binds/hussh.py

@@ -91,9 +91,7 @@ def remote_copy(self, source, dest_host, dest_path=None, ensure_dir=True):
 
     def run(self, command, timeout=0):
         """Run a command on the host and return the results."""
-        # TODO support timeout parameter
-        result = self.session.execute(command)
-
+        result = self.session.execute(command, timeout=helpers.translate_timeout(timeout))
         # Create broker Result from hussh SSHResult
         return helpers.Result(
             status=result.status,

broker/broker.py

@@ -24,6 +24,7 @@
 from broker import exceptions, helpers
 from broker.hosts import Host
 from broker.providers import PROVIDER_ACTIONS, PROVIDERS, _provider_imports
+from broker.settings import settings
 
 # load all the provider class so they are registered
 for _import in _provider_imports:
@@ -86,7 +87,8 @@ def _act(self, provider, method, checkout=False):
         method_obj = getattr(provider_inst, method)
         logger.debug(f"On {provider_inst=} executing {method_obj=} with params {self._kwargs=}.")
         # Overkill for a single action, cleaner than splitting the logic
-        with ThreadPoolExecutor() as workers:
+        max_workers = min(count, int(settings.thread_limit)) if settings.thread_limit else None
+        with ThreadPoolExecutor(max_workers=max_workers) as workers:
             tasks = [workers.submit(method_obj, **self._kwargs) for _ in range(count)]
             result = []
             for task in as_completed(tasks):
@@ -202,8 +204,8 @@ def checkin(self, sequential=False, host=None, in_context=False):
         if not hosts:
             logger.debug("Checkin called with no hosts, taking no action")
             return
-
-        with ThreadPoolExecutor(max_workers=1 if sequential else None) as workers:
+        max_workers = min(len(hosts), int(settings.thread_limit)) if settings.thread_limit else None
+        with ThreadPoolExecutor(max_workers=1 if sequential else max_workers) as workers:
             completed_checkins = as_completed(
                 # reversing over a copy of the list to avoid skipping
                 workers.submit(self._checkin, _host)

broker/commands.py

@@ -4,17 +4,28 @@
 import signal
 import sys
 
-import click
 from logzero import logger
 from rich.console import Console
+from rich.syntax import Syntax
 from rich.table import Table
+import rich_click as click
 
 from broker import exceptions, helpers, settings
 from broker.broker import Broker
+from broker.config_manager import ConfigManager
 from broker.logger import LOG_LEVEL
 from broker.providers import PROVIDER_ACTIONS, PROVIDER_HELP, PROVIDERS
 
 signal.signal(signal.SIGINT, helpers.handle_keyboardinterrupt)
+CONSOLE = Console(no_color=settings.settings.less_colors)  # rich console for pretty printing
+
+click.rich_click.SHOW_ARGUMENTS = True
+click.rich_click.COMMAND_GROUPS = {
+    "broker": [
+        {"name": "Core Actions", "commands": ["checkout", "checkin", "inventory"]},
+        {"name": "Extras", "commands": ["execute", "extend", "providers", "config"]},
+    ]
+}
 
 
 def loggedcli(group=None, *cli_args, **cli_kwargs):
@@ -47,7 +58,7 @@ def parse_labels(provider_labels):
     }
 
 
-class ExceptionHandler(click.Group):
+class ExceptionHandler(click.RichGroup):
     """Wraps click group to catch and handle raised exceptions."""
 
     def __call__(self, *args, **kwargs):
@@ -167,33 +178,37 @@ def provider_cmd(ctx, *args, **kwargs):  # the actual subcommand
 def cli(version):
     """Command-line interface for interacting with providers."""
     if version:
-        from importlib.metadata import version
-
         from packaging.version import Version
         import requests
 
-        broker_version = version("broker")
-
         # Check against the latest version published to PyPi
         try:
             latest_version = Version(
                 requests.get("https://pypi.org/pypi/broker/json", timeout=60).json()["info"][
                     "version"
                 ]
             )
-            if latest_version > Version(broker_version):
+            if latest_version > Version(ConfigManager.version):
                 click.secho(
                     f"A newer version of broker is available: {latest_version}",
                     fg="yellow",
                 )
         except requests.exceptions.RequestException as err:
             logger.warning(f"Unable to check for latest version: {err}")
-        click.echo(f"Version: {broker_version}")
-        broker_directory = settings.BROKER_DIRECTORY.absolute()
-        click.echo(f"Broker Directory: {broker_directory}")
-        click.echo(f"Settings File: {settings.settings_path.absolute()}")
-        click.echo(f"Inventory File: {broker_directory}/inventory.yaml")
-        click.echo(f"Log File: {broker_directory}/logs/broker.log")
+
+        # Create a rich table
+        table = Table(title=f"Broker {ConfigManager.version}")
+
+        table.add_column("", justify="left", style="cyan", no_wrap=True)
+        table.add_column("Location", justify="left", style="magenta")
+
+        table.add_row("Broker Directory", str(settings.BROKER_DIRECTORY.absolute()))
+        table.add_row("Settings File", str(settings.settings_path.absolute()))
+        table.add_row("Inventory File", f"{settings.BROKER_DIRECTORY.absolute()}/inventory.yaml")
+        table.add_row("Log File", f"{settings.BROKER_DIRECTORY.absolute()}/logs/broker.log")
+
+        # Print the table
+        CONSOLE.print(table)
 
 
 @loggedcli(context_settings={"allow_extra_args": True, "ignore_unknown_options": True})
@@ -248,78 +263,80 @@ def providers():
 
 
 @loggedcli()
-@click.argument("vm", type=str, nargs=-1)
+@click.argument("hosts", type=str, nargs=-1)
 @click.option("-b", "--background", is_flag=True, help="Run checkin in the background")
-@click.option("--all", "all_", is_flag=True, help="Select all VMs")
+@click.option("--all", "all_", is_flag=True, help="Select all hosts")
 @click.option("--sequential", is_flag=True, help="Run checkins sequentially")
 @click.option("--filter", type=str, help="Checkin only what matches the specified filter")
-def checkin(vm, background, all_, sequential, filter):
-    """Checkin or "remove" a VM or series of VM broker instances.
+def checkin(hosts, background, all_, sequential, filter):
+    """Checkin or "remove" a host or series of hosts.
 
-    COMMAND: broker checkin <vm hostname>|<local id>|--all
+    COMMAND: broker checkin <hostname>|<local id>|--all
     """
     if background:
         helpers.fork_broker()
     inventory = helpers.load_inventory(filter=filter)
     to_remove = []
     for num, host in enumerate(inventory):
-        if str(num) in vm or host.get("hostname") in vm or host.get("name") in vm or all_:
+        if str(num) in hosts or host.get("hostname") in hosts or host.get("name") in hosts or all_:
             to_remove.append(Broker().reconstruct_host(host))
     Broker(hosts=to_remove).checkin(sequential=sequential)
 
 
 @loggedcli()
 @click.option("--details", is_flag=True, help="Display all host details")
-@click.option("--curated", is_flag=True, help="Display curated host details")
+@click.option("--list", "_list", is_flag=True, help="Display only hostnames and local ids")
 @click.option(
     "--sync",
     type=str,
     help="Class-style name of a supported broker provider. (AnsibleTower)",
 )
 @click.option("--filter", type=str, help="Display only what matches the specified filter")
-def inventory(details, curated, sync, filter):
-    """Get a list of all VMs you've checked out showing hostname and local id.
+def inventory(details, _list, sync, filter):
+    """Display a table of hosts in your local inventory.
 
-    hostname pulled from list of dictionaries.
+    Inventory fields are configurable in Broker's settings file.
+
+    Run a sync for your providers to pull down your host information.
+
+    e.g. `broker inventory --sync AnsibleTower`
+
+    Note: Applying a filter will result in incorrect id's being displayed.
     """
     if sync:
         Broker.sync_inventory(provider=sync)
-    logger.info("Pulling local inventory")
     inventory = helpers.load_inventory(filter=filter)
     helpers.emit({"inventory": inventory})
-    if curated:
-        console = Console()
-        table = Table(title="Host Inventory")
-
-        table.add_column("Id", justify="left", style="cyan", no_wrap=True)
-        table.add_column("Host", justify="left", style="magenta")
-        table.add_column("Provider", justify="left", style="green")
-        table.add_column("Action", justify="left", style="yellow")
-        table.add_column("OS", justify="left", style="blue")
-
-        for host in helpers.get_host_inventory_fields(inventory, PROVIDER_ACTIONS):
-            table.add_row(
-                str(host["id"]), host["host"], host["provider"], host["action"], host["os"]
-            )
-
-        console.print(table)
+    # details is handled differently than the normal and list views
+    if details:
+        detailed = helpers.yaml_format(dict(enumerate(inventory)))
+        CONSOLE.print(Syntax(detailed, "yaml", background_color="default"))
         return
-    for num, host in enumerate(inventory):
-        if (display_name := host.get("hostname")) is None:
-            display_name = host.get("name")
-        # if we're filtering, then don't show an index.
-        # Otherwise, a user might perform an action on the incorrect (unfiltered) index.
-        index = f"{num}: " if filter is None else ""
-        if details:
-            logger.info(f"{index}{display_name}:\n{helpers.yaml_format(host)}")
-        else:
-            logger.info(f"{index}{display_name}")
+
+    inventory_fields = (
+        {"Host": settings.settings.inventory_list_vars}
+        if _list
+        else settings.settings.inventory_fields
+    )
+    curated_host_info = [
+        helpers.inventory_fields_to_dict(
+            inventory_fields=inventory_fields,
+            host_dict=host,
+            provider_actions=PROVIDER_ACTIONS,
+        )
+        for host in inventory
+    ]
+    table = helpers.dictlist_to_table(curated_host_info, "Host Inventory", _id=True)
+    if _list:
+        table.title = None
+        table.box = None
+    CONSOLE.print(table)
 
 
 @loggedcli()
-@click.argument("vm", type=str, nargs=-1)
+@click.argument("hosts", type=str, nargs=-1)
 @click.option("-b", "--background", is_flag=True, help="Run extend in the background")
-@click.option("--all", "all_", is_flag=True, help="Select all VMs")
+@click.option("--all", "all_", is_flag=True, help="Select all hosts")
 @click.option("--sequential", is_flag=True, help="Run extends sequentially")
 @click.option("--filter", type=str, help="Extend only what matches the specified filter")
 @click.option(
@@ -331,18 +348,18 @@ def inventory(details, curated, sync, filter):
     " labels (e.g. '-l k1=v1,k2=v2,k3=v3=z4').",
 )
 @provider_options
-def extend(vm, background, all_, sequential, filter, **kwargs):
+def extend(hosts, background, all_, sequential, filter, **kwargs):
     """Extend a host's lease time.
 
-    COMMAND: broker extend <vm hostname>|<vm name>|<local id>|--all
+    COMMAND: broker extend <hostname>|<host name>|<local id>|--all
     """
     broker_args = helpers.clean_dict(kwargs)
     if background:
         helpers.fork_broker()
     inventory = helpers.load_inventory(filter=filter)
     to_extend = []
     for num, host in enumerate(inventory):
-        if str(num) in vm or host["hostname"] in vm or host.get("name") in vm or all_:
+        if str(num) in hosts or host["hostname"] in hosts or host.get("name") in hosts or all_:
             to_extend.append(Broker().reconstruct_host(host))
     Broker(hosts=to_extend, **broker_args).extend(sequential=sequential)
 
@@ -400,3 +417,115 @@ def execute(ctx, background, nick, output_format, artifacts, args_file, provider
         logger.info(result)
     elif output_format == "yaml":
         click.echo(helpers.yaml_format(result))
+
+
+@cli.group(cls=ExceptionHandler)
+def config():
+    """View and manage Broker's configuration.
+
+    Note: One important concept of these commands is the concept of a "chunk".
+
+    A chunk is a part of the configuration file that can be accessed or updated.
+    Chunks are specified by their keys in the configuration file.
+    Nested chunks are separated by periods.
+
+    e.g. broker config view AnsibleTower.instances.my_instance
+    """
+
+
+@loggedcli(group=config)
+@click.argument("chunk", type=str, required=False)
+@click.option("--no-syntax", is_flag=True, help="Disable syntax highlighting")
+def view(chunk, no_syntax):
+    """View all or part of the broker configuration."""
+    result = helpers.yaml_format(ConfigManager(settings.settings_path).get(chunk))
+    if no_syntax:
+        CONSOLE.print(result)
+    else:
+        CONSOLE.print(Syntax(result, "yaml", background_color="default"))
+
+
+@loggedcli(group=config)
+@click.argument("chunk", type=str, required=False)
+def edit(chunk):
+    """Directly edit the broker configuration file.
+
+    You can define the scope of the edit by specifying a chunk.
+    Otherwise, the entire configuration file will be opened.
+    """
+    ConfigManager(settings.settings_path).edit(chunk)
+
+
+@loggedcli(group=config, name="set")
+@click.argument("chunk", type=str, required=True)
+@click.argument("new-value", type=str, required=True)
+def _set(chunk, new_value):
+    """Set a value in the Broker configuration file.
+
+    These updates take the form of `<chunk> <value>` pairs.
+    You can also pass a yaml or json file containing the new contents of a chunk.
+    """
+    new_value = helpers.resolve_file_args({"nv": new_value})["nv"]
+    ConfigManager(settings.settings_path).update(chunk, new_value)
+
+
+@loggedcli(group=config)
+def restore():
+    """Restore the broker configuration file to the last backup."""
+    ConfigManager(settings.settings_path).restore()
+
+
+@loggedcli(group=config)
+@click.argument("chunk", type=str, required=False)
+@click.option("--from", "_from", type=str, help="A file path or URL to initialize the config from.")
+def init(chunk=None, _from=None):
+    """Initialize the broker configuration file from your local clone or GitHub.
+
+    You can also init specific chunks by passing the chunk name.
+    Additionally, if you want to initialize from a file or URL, you can pass the `--from` flag.
+    Keep in mind that the file and url contents need to be valid yaml.
+    """
+    ConfigManager(settings.settings_path).init_config_file(chunk=chunk, _from=_from)
+
+
+@loggedcli(group=config)
+def nicks():
+    """Get a list of nicks."""
+    result = ConfigManager(settings.settings_path).nicks()
+    CONSOLE.print("\n".join(result))
+
+
+@loggedcli(group=config)
+@click.argument("nick", type=str, required=True)
+@click.option("--no-syntax", is_flag=True, help="Disable syntax highlighting")
+def nick(nick, no_syntax):
+    """Get information about a specific nick."""
+    result = helpers.yaml_format(ConfigManager(settings.settings_path).nicks(nick))
+    if no_syntax:
+        CONSOLE.print(result)
+    else:
+        CONSOLE.print(Syntax(result, "yaml", background_color="default"))
+
+
+@loggedcli(group=config)
+@click.option("-f", "--force-version", type=str, help="Force the migration to a specific version")
+def migrate(force_version=None):
+    """Migrate the broker configuration file to the latest version."""
+    ConfigManager(settings.settings_path).migrate(force_version=force_version)
+
+
+@loggedcli(group=config)
+@click.argument("chunk", type=str, required=False, default="base")
+def validate(chunk):
+    """Validate top-level chunks of the broker configuration file.
+
+    You can validate against the `base` settings by default or specify a provider.
+    You can also validate against a specific provider instance with `ProviderClass:instance_name`.
+
+    To validate everything, pass `all`
+    """
+    try:
+        ConfigManager(settings.settings_path).validate(chunk, PROVIDERS)
+        logger.info("Validation passed!")
+    except exceptions.BrokerError as err:
+        logger.warning(f"Validation failed: {err}")