Generate inventory using Nix

This commit replaces the update_inventory.py script with a Nix module
based on Flake Parts [1] which defines all configuration options
including types and default values.

Only values that are exposed by the module can be set in the
configuration. Exposed values have been removed from the default/
main.yaml in order to have only a single source of truth. As a result,
all variables (set or not) are rendered into the inventory.

The cluster initialization is now backed by Nix Flake templates.
Template files for new clusters can be found at ./nix/templates/
cluster-repo.

The cluster repository layout is changed in such a way that the
./config directory is completely handled by the user and the ./inventory
directory is completely generated and may be ignored from the VCS. A
./state directory is added which is both input and output of the
inventory generation and which has to be added to VCS.

                   +---------+
                   | ./state |
                   +--+---^--+
                      |   |
               +------v---+---------+
+----------+   |                    |   +-------------+
| ./config +--->     Nix module     +---> ./inventory |
+----------+   |                    |   +-------------+
               +--------------------+

Resource request and limit handling has been changed such that
unflattened arrays are used instead of individual options for each
value.

[1] https://flake.parts

.envrc.lib.sh

@@ -69,9 +69,9 @@ use_flake_if_nix() {
       _poetry_common "${flake_dir}"
       watch_file "${flake_dir}/nix/poetry.nix"
       if [ "${MINIMAL_ACCESS_VENV:-false}" == "true" ]; then
-        use flake "${flake_dir}?shallow=1#minimal"
+        use flake "${flake_dir}#minimal"
       else
-        use flake "${flake_dir}?shallow=1#${YAOOK_K8S_DEVSHELL:-default}"
+        use flake "${flake_dir}#${YAOOK_K8S_DEVSHELL:-default}"
       fi
       export NIX_FLAKE_ACTIVE="${NIX_FLAKE_ACTIVE}:${flake_dir}"
     else

.gitlab-ci.yml

@@ -179,6 +179,7 @@ build-docs-check:
     - export PATH="$VIRTUAL_ENV/bin:$PATH"
   script:
     - towncrier build --version x.x.x --keep
+    - nix build .#docsRST -o docs/user/reference/options
     - sphinx-build -W docs _build/html
     - mv _build/html public
   artifacts:

.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
       - id: trailing-whitespace
         exclude: '.*(\.drawio|\.svg)$'
       - id: end-of-file-fixer
-        exclude: '.*(\.drawio|\.svg|\.nix)$'
+        exclude: '.*(\.drawio|\.svg|\.nix)$' # Nix files are fixed by nix-fmt
       - id: mixed-line-ending
       - id: check-executables-have-shebangs
       - id: check-merge-conflict
@@ -36,13 +36,13 @@ repos:
         stages: [pre-commit, pre-push, manual]
       - id: check-flake
         name: check flake
-        files: "^flake.nix$"
+        files: .*\.nix$'
         entry: ci/lint/check-flake.sh
         language: script
         stages: [pre-commit, pre-push, manual]
       - id: nix-fmt
         name: nix-fmt
-        files: "^flake.nix$"
+        files: '.*\.nix$'
         entry: ci/lint/format-flake.sh
         language: script
         stages: [pre-commit, pre-push, manual]

CHANGELOG.rst

@@ -211,7 +211,7 @@ Breaking changes
   rather than across separate lists for each type of value.
 
   Furthermore you now have control over the whole name of Terraform nodes,
-  see :ref:`the documentation <cluster-configuration.configuring-terraform>`
+  see :ref:`the documentation <configuration-options.yk8s.terraform>`
   for further details.
 
   .. code:: diff

actions/apply-all.sh

@@ -4,6 +4,10 @@ actions_dir="$(dirname "$0")"
 
 # shellcheck source=actions/lib.sh
 . "$actions_dir/lib.sh"
+
+# Ensure that the latest config is deployed to the inventory
+"$actions_dir/update-inventory.sh"
+
 load_conf_vars
 
 check_venv

actions/apply-custom.sh

@@ -4,6 +4,10 @@ actions_dir="$(dirname "$0")"
 
 # shellcheck source=actions/lib.sh
 . "$actions_dir/lib.sh"
+
+# Ensure that the latest config is deployed to the inventory
+"$actions_dir/update-inventory.sh"
+
 load_conf_vars
 
 check_venv
@@ -14,9 +18,6 @@ require_vault_token
 
 install_prerequisites
 
-# Ensure that the latest config is deployed to the inventory
-python3 "$actions_dir/update_inventory.py"
-
 # Bring the wireguard interface up if configured so
 "$actions_dir/wg-up.sh"
 

actions/apply-k8s-core.sh

@@ -25,19 +25,18 @@ execute_playbook() {
   local playbook="$1"
   notef "Executing playbook $playbook\n"
 
+  # Ensure that the latest config is deployed to the inventory
+  "$actions_dir/update-inventory.sh"
+
   load_conf_vars
   check_venv
   check_conf_sanity
   require_vault_token
   install_prerequisites
 
-  # Ensure that the latest config is deployed to the inventory
-  python3 "$actions_dir/update_inventory.py"
   # Bring the wireguard interface up if configured so
   "$actions_dir/wg-up.sh"
 
-  set_kubeconfig
-
   pushd "$ansible_k8s_core_dir"
   # Include k8s-core roles
   ANSIBLE_ROLES_PATH="$ansible_k8s_core_dir/roles" \

actions/apply-k8s-supplements.sh

@@ -25,14 +25,15 @@ execute_playbook() {
   local playbook="$1"
   notef "Executing playbook $playbook\n"
 
+  # Ensure that the latest config is deployed to the inventory
+  "$actions_dir/update-inventory.sh"
+
   load_conf_vars
   check_conf_sanity
   check_venv
   require_vault_token
   install_prerequisites
 
-  # Ensure that the latest config is deployed to the inventory
-  python3 "$actions_dir/update_inventory.py"
   # Bring the wireguard interface up if configured so
   "$actions_dir/wg-up.sh"
 

actions/apply-prepare-gw.sh

@@ -4,6 +4,10 @@ actions_dir="$(dirname "$0")"
 
 # shellcheck source=actions/lib.sh
 . "$actions_dir/lib.sh"
+
+# Ensure that the latest config is deployed to the inventory
+"$actions_dir/update-inventory.sh"
+
 load_conf_vars
 
 check_venv
@@ -14,9 +18,6 @@ require_vault_token
 
 install_prerequisites
 
-# Ensure that the latest config is deployed to the inventory
-python3 "$actions_dir/update_inventory.py"
-
 if [ "${tf_usage:-true}" == 'false' ]; then
   errorf "It seems like you're not running on top of OpenStack,"
   errorf "because terraform.enabled is false."

actions/apply-terraform.sh

@@ -4,13 +4,14 @@ actions_dir="$(realpath "$(dirname "$0")")"
 
 # shellcheck source=actions/lib.sh
 . "$actions_dir/lib.sh"
+
+# Ensure that the latest config is deployed to the inventory
+"$actions_dir/update-inventory.sh"
+
 load_conf_vars
 
 check_venv
 
-# Ensure that the latest config is deployed to the inventory
-python3 "$actions_dir/update_inventory.py"
-
 if [ "$("$actions_dir/helpers/semver2.sh" "$(terraform -v -json | jq -r '.terraform_version')" "$terraform_min_version")" -lt 0 ]; then
     errorf 'Please upgrade Terraform to at least v'"$terraform_min_version"
     exit 5
@@ -166,7 +167,7 @@ if [ $rc == $RC_DISRUPTION ]; then
         # shellcheck disable=SC2016
         errorf 'terraform would delete or recreate a resource, but not all of the following is set' >&2
         errorf '  - MANAGED_K8S_DISRUPT_THE_HARBOUR=true' >&2
-        errorf "  - ${terraform_disruption_setting}=false in ${config_file}" >&2
+        errorf "  - terraform.prevent_disruption = false in the config" >&2
         errorf 'aborting due to destructive change without approval.' >&2
         exit 3
     fi

actions/destroy.sh

@@ -1,8 +1,13 @@
 #!/usr/bin/env bash
 set -euo pipefail
 actions_dir="$(realpath "$(dirname "$0")")"
+
 # shellcheck source=actions/lib.sh
 . "$actions_dir/lib.sh"
+
+# Ensure that the latest config is deployed to the inventory
+"$actions_dir/update-inventory.sh"
+
 load_conf_vars
 
 check_venv
@@ -92,4 +97,4 @@ if [ "$(jq -r .backend.type "$terraform_state_dir/.terraform/terraform.tfstate")
 fi
 
 # Purge the remaining terraform directory. Its existence is a condition for additional disruption checks.
-rm -f "$terraform_state_dir/config.tfvars.json"
+rm -fr "$terraform_state_dir"