Merge pull request kapicorp#1059 from projectsyn/feat/reclass-rs

Add `reclass-rs` inventory backend

Dockerfile

@@ -36,7 +36,7 @@ COPY ./poetry.lock ./poetry.lock
 COPY ./README.md ./README.md
 
 # Installs and caches dependencies
-RUN poetry install --no-root --extras=gojsonnet
+RUN poetry install --no-root --extras=gojsonnet --extras=reclass-rs
 
 COPY ./kapitan ./kapitan
 

Makefile

@@ -16,7 +16,7 @@ test_docker:
 .PHONY: test_coverage
 test_coverage:
 	@echo ----- Testing code coverage -----
-	coverage run --source=kapitan --omit="*reclass*" -m unittest discover
+	coverage run --source=kapitan -m unittest discover
 	coverage report --fail-under=65 -m
 
 .PHONY: test_formatting

docs/pages/commands/kapitan_compile.md

@@ -168,7 +168,7 @@ The `--embed-refs` flags tells **Kapitan** to embed these references on compile,
 
         options:
           -h, --help            show this help message and exit
-          --inventory-backend {reclass}
+          --inventory-backend {reclass,reclass-rs}
                                 Select the inventory backend to use (default=reclass)
           --search-paths JPATH [JPATH ...], -J JPATH [JPATH ...]
                                 set search paths, default is ["."]

docs/pages/inventory/introduction.md

@@ -17,6 +17,11 @@ By combining [**target**](#targets) and [**classes**](#classes), the **Inventory
 !!! info
     The **Kapitan** **Inventory** is based on an open source project called [reclass](https://github.com/kapicorp/reclass) and you can find the full documentation on our Github clone. However we discourage you to look directly at the reclass documentation before you learn more about **Kapitan**, because **Kapitan** uses a fork of reclass and greatly simplifies the reclass experience.
 
+!!! info
+    Kapitan allows users to switch the inventory backend to [reclass-rs](https://github.com/projectsyn/reclass-rs). You can switch the backend to reclass-rs by passing `--inventory-backend=reclass-rs` on the command line. Alternatively, you can define the backend in the [.kapitan config file](../commands/kapitan_dotfile.md).
+
+    See the [reclass-rs inventory backend](reclass-rs.md) documentation for more details.
+
 !!! note
     Kapitan enforces very little structure for the **Inventory**, so that you can adapt it to your specific needs: this might be overwhelming at the beginning: don’t worry, we will explain best practice and give guidelines soon.
 

docs/pages/inventory/reclass-rs.md

@@ -0,0 +1,71 @@
+# The reclass-rs inventory backend
+
+## Overview
+
+[Reclass-rs](https://github.com/projectsyn/reclass-rs) is a reimplementation of Kapitan's Reclass fork in Rust.
+Please note that the Rust implementation doesn't support all the features of Kapitan's Reclass fork yet.
+
+However, reclass-rs improves rendering time for the inventory significantly, especially if you're making heavy use of parameter references in class includes.
+If some of the Reclass features or options that you're using are missing in reclass-rs, don't hesitate to open an issue in the [reclass-rs project](https://github.com/projectsyn/reclass-rs/issues/new?assignees=&labels=enhancement&projects=&template=03_missing_reclass_feature.md).
+
+## Installation
+
+The `reclass-rs` Python package is an optional dependency of Kapitan.
+You can install it as follows:
+
+```shell
+pip install kapitan[reclass-rs]
+```
+
+## Usage
+
+To use the reclass-rs inventory backend, you need to pass `--inventory-backend=reclass-rs` on the command line.
+If you want to permanently switch to the reclass-rs inventory backend, you can select the inventory backend in the [.kapitan config file](../commands/kapitan_dotfile.md):
+
+```yaml
+global:
+  inventory-backend: reclass-rs
+```
+
+## Performance comparison
+
+For the performance comparison, a real Kapitan inventory which makes heavy use of parameter interpolation in class includes was rendered with both Reclass and reclass-rs.
+The example inventory that was used for the performance comparison contains 325 classes and 56 targets.
+The example inventory renders to a total of 25MB of YAML.
+
+### Reclass
+
+```
+$ time kapitan inventory -v --inventory-backend=reclass > inv.yml
+[ ... some output omitted ... ]
+kapitan.resources DEBUG    Using reclass as inventory backend
+kapitan.inventory.inv_reclass DEBUG    Inventory reclass: No config file found. Using reclass inventory config defaults
+kapitan.inventory.inv_reclass DEBUG    Inventory rendering with reclass took 0:01:06.037057
+
+real    1m23.840s
+user    1m23.520s
+sys     0m0.287s
+```
+
+Reclass takes 1 minute and 6 seconds to render the example inventory.
+The rest of the runtime (roughly 18 seconds) is spent in writing the resulting 25MB of YAML to the output file.
+
+### reclass-rs
+
+```
+$ time kapitan inventory -v --inventory-backend=reclass-rs > inv-rs.yml
+[ ... some output omitted ... ]
+kapitan.resources DEBUG    Using reclass-rs as inventory backend
+kapitan.inventory.inv_reclass DEBUG    Inventory reclass: No config file found. Using reclass inventory config defaults
+reclass-config.yml entry 'storage_type=yaml_fs' not implemented yet, ignoring...
+reclass-config.yml entry 'inventory_base_uri=./inventory' not implemented yet, ignoring...
+reclass-config.yml entry 'allow_none_override=true' not implemented yet, ignoring...
+kapitan.inventory.inv_reclass_rs DEBUG    Inventory rendering with reclass-rs took 0:00:01.717107
+
+real    0m19.921s
+user    0m35.586s
+sys     0m1.066s
+```
+
+reclass-rs takes 1.7 seconds to render the example inventory.
+The rest of the runtime (roughly 18 seconds) is spent in writing the resulting 25MB of YAML to the output file.

kapitan/inventory/__init__.py

@@ -1,10 +1,12 @@
 from typing import Type
 
 from .inv_reclass import ReclassInventory
+from .inv_reclass_rs import ReclassRsInventory
 from .inventory import Inventory
 
 # Dict mapping values for command line flag `--inventory-backend` to the
 # associated `Inventory` subclass.
 AVAILABLE_BACKENDS: dict[str, Type[Inventory]] = {
     "reclass": ReclassInventory,
+    "reclass-rs": ReclassRsInventory,
 }

kapitan/inventory/inv_reclass.py

@@ -1,6 +1,8 @@
 import logging
 import os
 
+from datetime import datetime
+
 import reclass
 import reclass.core
 import yaml
@@ -24,9 +26,9 @@ def render_targets(self, targets: list = None, ignore_class_notfound: bool = Fal
 
         Does not throw errors if a class is not found while ignore_class_notfound is specified
         """
-        reclass_config = get_reclass_config(self.inventory_path)
-        reclass_config.setdefault("ignore_class_notfound", ignore_class_notfound)
-        reclass_config["compose_node_name"] = self.compose_target_name
+        reclass_config = get_reclass_config(
+            self.inventory_path, ignore_class_notfound, self.compose_target_name
+        )
 
         try:
             storage = reclass.get_storage(
@@ -37,7 +39,10 @@ def render_targets(self, targets: list = None, ignore_class_notfound: bool = Fal
             )
             class_mappings = reclass_config.get("class_mappings")  # this defaults to None (disabled)
             _reclass = reclass.core.Core(storage, class_mappings, reclass.settings.Settings(reclass_config))
+            start = datetime.now()
             rendered_inventory = _reclass.inventory()
+            elapsed = datetime.now() - start
+            logger.debug(f"Inventory rendering with reclass took {elapsed}")
 
             # store parameters and classes
             for target_name, rendered_target in rendered_inventory["nodes"].items():
@@ -54,15 +59,21 @@ def render_targets(self, targets: list = None, ignore_class_notfound: bool = Fal
             raise InventoryError(e.message)
 
 
-def get_reclass_config(inventory_path: str) -> dict:
+def get_reclass_config(
+    inventory_path: str,
+    ignore_class_notfound: bool = False,
+    compose_node_name: bool = False,
+    normalise_nodes_classes: bool = True,
+) -> dict:
     # set default values initially
     reclass_config = {
         "storage_type": "yaml_fs",
         "inventory_base_uri": inventory_path,
         "nodes_uri": "targets",
         "classes_uri": "classes",
-        "compose_node_name": False,
+        "compose_node_name": compose_node_name,
         "allow_none_override": True,
+        "ignore_class_notfound": ignore_class_notfound,
     }
     try:
         from yaml import CSafeLoader as YamlLoader
@@ -87,7 +98,8 @@ def get_reclass_config(inventory_path: str) -> dict:
         logger.debug("Inventory reclass: No config file found. Using reclass inventory config defaults")
 
     # normalise relative nodes_uri and classes_uri paths
-    for uri in ("nodes_uri", "classes_uri"):
-        reclass_config[uri] = os.path.normpath(os.path.join(inventory_path, reclass_config[uri]))
+    if normalise_nodes_classes:
+        for uri in ("nodes_uri", "classes_uri"):
+            reclass_config[uri] = os.path.normpath(os.path.join(inventory_path, reclass_config[uri]))
 
     return reclass_config

kapitan/inventory/inv_reclass_rs.py

@@ -0,0 +1,44 @@
+import logging
+import os
+import reclass_rs
+
+from datetime import datetime
+
+from kapitan.errors import InventoryError
+
+from .inventory import Inventory, InventoryTarget
+from .inv_reclass import get_reclass_config
+
+logger = logging.getLogger(__name__)
+
+
+class ReclassRsInventory(Inventory):
+    def _make_reclass_rs(self, ignore_class_notfound: bool):
+        # Get Reclass config options with the same method that's used for `ReclassInventory`, but
+        # disable the logic to normalise the `nodes_uri` and `classes_uri` options, since reclass-rs
+        # expects those fields to be relative to the inventory path.
+        config_dict = get_reclass_config(
+            self.inventory_path, ignore_class_notfound, self.compose_target_name, False
+        )
+
+        # Turn on verbose config loading only if Kapitan loglevel is set to at least DEBUG.
+        config = reclass_rs.Config.from_dict(
+            self.inventory_path, config_dict, logger.isEnabledFor(logging.DEBUG)
+        )
+        return reclass_rs.Reclass.from_config(config)
+
+    def render_targets(self, targets: list = None, ignore_class_notfound: bool = False):
+        try:
+            r = self._make_reclass_rs(ignore_class_notfound)
+            start = datetime.now()
+            inv = r.inventory()
+            elapsed = datetime.now() - start
+            logger.debug(f"Inventory rendering with reclass-rs took {elapsed}")
+
+            for target_name, nodeinfo in inv.nodes.items():
+                self.targets[target_name].parameters = nodeinfo.parameters
+                self.targets[target_name].classes = nodeinfo.classes
+
+        except ValueError as e:
+            logger.error(f"Reclass-rs error: {e}")
+            raise InventoryError(f"{e}")

mkdocs.yml

@@ -91,6 +91,7 @@ nav:
         - Classes: pages/inventory/classes.md
         - Parameters Interpolation: pages/inventory/parameters_interpolation.md
         - Advanced: pages/inventory/advanced.md
+        - reclass-rs Backend: pages/inventory/reclass-rs.md
       - Input Types:
           - Introduction: pages/input_types/introduction.md
           - Kadet: pages/input_types/kadet.md