Component Discovery With Reclass

CHANGELOG.md

@@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ### Changed
 
 * Reworked documentation ([#147])
+* Use Reclass to discover components ([#160])
 
 ## [v0.2.2]
 
@@ -159,4 +160,5 @@ Initial implementation
 [#155]: https://github.com/projectsyn/commodore/pull/155
 [#157]: https://github.com/projectsyn/commodore/pull/157
 [#158]: https://github.com/projectsyn/commodore/pull/158
+[#160]: https://github.com/projectsyn/commodore/pull/160
 [#161]: https://github.com/projectsyn/commodore/pull/161

commodore/compile.py

@@ -64,15 +64,16 @@ def _regular_setup(config, cluster_id):
         raise click.ClickException(f"While fetching cluster specification: {e}") from e
     customer_id = cluster['tenant']
 
-    # Fetch components and config
-    _fetch_global_config(config, cluster)
-    _fetch_customer_config(config, customer_id)
-    fetch_components(config)
     target_name = update_target(config, cluster)
     if target_name != 'cluster':
         raise click.ClickException(
             f"Only target with name 'cluster' is supported, got {target_name}")
 
+    # Fetch components and config
+    _fetch_global_config(config, cluster)
+    _fetch_customer_config(config, customer_id)
+    fetch_components(config)
+    update_target(config, cluster)
     # Fetch catalog
     catalog_repo = fetch_customer_catalog(config, cluster['gitRepo'])
 
@@ -138,6 +139,7 @@ def compile(config, cluster_id):
     # Compile kapitan inventory to extract component versions. Component
     # versions are assumed to be defined in the inventory key
     # 'parameters.component_versions'
+    reset_reclass_cache()
     kapitan_inventory = inventory_reclass('inventory')['nodes'][target_name]
     versions = kapitan_inventory['parameters'].get('component_versions', None)
     if versions and not config.local:

commodore/dependency_mgmt.py

@@ -3,6 +3,9 @@
 
 import click
 
+from kapitan.cached import reset_cache as reset_reclass_cache
+from kapitan.resources import inventory_reclass
+
 from . import git
 from .config import Component
 from .helpers import relsymlink, yaml_load
@@ -31,19 +34,15 @@ def _discover_components(cfg, inventory_path):
     Discover components in `inventory_path/`. Parse all classes found in
     inventory_path and look for class includes starting with `components.`.
     """
+    reset_reclass_cache()
+    kapitan_inventory = inventory_reclass(inventory_path, ignore_class_notfound=True)['nodes']['cluster']
     components = set()
-    inventory = P(inventory_path)
-    for classfile in inventory.glob('**/*.yml'):
-        if cfg.trace:
-            click.echo(f" > Discovering components in {classfile}")
-        classyaml = yaml_load(classfile)
-        if classyaml is not None:
-            for kls in classyaml.get('classes', []):
-                if kls.startswith('components.'):
-                    component = kls.split('.')[1]
-                    if cfg.debug:
-                        click.echo(f"   > Found component {component}")
-                    components.add(component)
+    for kls in kapitan_inventory['classes']:
+        if kls.startswith('components.'):
+            component = kls.split('.')[1]
+            if cfg.debug:
+                click.echo(f"   > Found component {component}")
+            components.add(component)
     return sorted(components)
 
 
@@ -88,12 +87,12 @@ def fetch_components(cfg):
     """
 
     click.secho('Discovering components...', bold=True)
-    component_names = _discover_components(cfg, 'inventory')
-    components = _read_component_urls(cfg, component_names)
-    click.secho('Fetching components...', bold=True)
     os.makedirs('inventory/classes/components', exist_ok=True)
     os.makedirs('inventory/classes/defaults', exist_ok=True)
     os.makedirs('dependencies/lib', exist_ok=True)
+    component_names = _discover_components(cfg, 'inventory')
+    components = _read_component_urls(cfg, component_names)
+    click.secho('Fetching components...', bold=True)
     for c in components:
         if cfg.debug:
             click.echo(f" > Fetching component {c.name}...")

docs/modules/ROOT/pages/reference/architecture.adoc

@@ -24,9 +24,11 @@ Commodore fetches the following dependencies:
 
 === Component discovery
 
-By default, Commodore searches all the classes in the global and tenant
-configurations for includes which start with `components.`.
-All of these includes are considered to be components that need to be fetched.
+To discover all required components, Commodore searches the inventory for
+includes which start with `components.`. In order to use reclass for the
+discovery, it's run with a special option to
+https://github.com/salt-formulas/reclass/blob/develop/README-extensions.rst#ignore-class-not-found[ignore missing classes].
+This leads to many "[WARNING] Reclass class not found" logs which can be ignored.
 Unless instructed otherwise, Commodore will fetch components from subfolder
 `commodore-components` in the
 xref:reference/cli.adoc#_catalog_compile[global Git base].