From 47c3949477670b0926193d03ac5a6c0776b323c5 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Mon, 26 Aug 2024 14:42:04 +0200
Subject: [PATCH 01/19] Change plugins_needs_config list format

Makes easier diff when adding or removing plugins.
---
 roles/kubernetes/control-plane/vars/main.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/roles/kubernetes/control-plane/vars/main.yaml b/roles/kubernetes/control-plane/vars/main.yaml
index f888d6b0ce8..9997472c8b9 100644
--- a/roles/kubernetes/control-plane/vars/main.yaml
+++ b/roles/kubernetes/control-plane/vars/main.yaml
@@ -1,3 +1,5 @@
 ---
 # list of admission plugins that needs to be configured
-kube_apiserver_admission_plugins_needs_configuration: [EventRateLimit, PodSecurity]
+kube_apiserver_admission_plugins_needs_configuration:
+- EventRateLimit
+- PodSecurity

From d3402736d426da3c4ac68e178cc73af89032c9da Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Mon, 26 Aug 2024 14:43:00 +0200
Subject: [PATCH 02/19] Remove special case for PodNodeSelector

This is already handled by the previous task.
---
 roles/kubernetes/control-plane/tasks/kubeadm-setup.yml | 9 ---------
 roles/kubernetes/control-plane/vars/main.yaml          | 1 +
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml
index dfbe604a4c8..63ed29aafaf 100644
--- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml
+++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml
@@ -122,15 +122,6 @@
     - item in kube_apiserver_admission_plugins_needs_configuration
   loop: "{{ kube_apiserver_enable_admission_plugins }}"
 
-- name: Kubeadm | Configure default cluster podnodeslector
-  template:
-    src: "podnodeselector.yaml.j2"
-    dest: "{{ kube_config_dir }}/admission-controls/podnodeselector.yaml"
-    mode: "0640"
-  when:
-    - kube_apiserver_admission_plugins_podnodeselector_default_node_selector is defined
-    - kube_apiserver_admission_plugins_podnodeselector_default_node_selector | length > 0
-
 - name: Kubeadm | Check apiserver.crt SANs
   vars:
     apiserver_ips: "{{ apiserver_sans | map('ansible.utils.ipaddr') | reject('equalto', False) | list }}"
diff --git a/roles/kubernetes/control-plane/vars/main.yaml b/roles/kubernetes/control-plane/vars/main.yaml
index 9997472c8b9..d387a15bc14 100644
--- a/roles/kubernetes/control-plane/vars/main.yaml
+++ b/roles/kubernetes/control-plane/vars/main.yaml
@@ -3,3 +3,4 @@
 kube_apiserver_admission_plugins_needs_configuration:
 - EventRateLimit
 - PodSecurity
+- PodNodeSelector

From 5b057c7328ee61b607d3985f93b00af337c344cd Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Mon, 26 Aug 2024 14:53:20 +0200
Subject: [PATCH 03/19] Update list of admission plugins with a config file

---
 roles/kubernetes/control-plane/vars/main.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/roles/kubernetes/control-plane/vars/main.yaml b/roles/kubernetes/control-plane/vars/main.yaml
index d387a15bc14..3775d253a7c 100644
--- a/roles/kubernetes/control-plane/vars/main.yaml
+++ b/roles/kubernetes/control-plane/vars/main.yaml
@@ -1,6 +1,8 @@
 ---
 # list of admission plugins that needs to be configured
+# https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/
 kube_apiserver_admission_plugins_needs_configuration:
 - EventRateLimit
+- ImagePolicyWebhook
 - PodSecurity
 - PodNodeSelector

From a8b66fd207cd4768df49d11be6be0633cf69d658 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 10 Sep 2024 13:43:02 +0200
Subject: [PATCH 04/19] Deduplicate kubeadm_patches tasks

kubernetes/control-plane and kubernetes/kubeadm roles both push kubeadm
patches in the same way.
Extract that code and make it a dependency of both.
This is safe because it's only configuration for kubeadm, which only
takes effect when kubeadm is run.
---
 roles/kubernetes/control-plane/meta/main.yml      |  1 +
 .../control-plane/tasks/kubeadm-setup.yml         | 15 ---------------
 roles/kubernetes/kubeadm/meta/main.yml            |  3 +++
 roles/kubernetes/kubeadm/tasks/main.yml           | 15 ---------------
 roles/kubernetes/kubeadm_common/tasks/main.yml    | 15 +++++++++++++++
 5 files changed, 19 insertions(+), 30 deletions(-)
 create mode 100644 roles/kubernetes/kubeadm/meta/main.yml
 create mode 100644 roles/kubernetes/kubeadm_common/tasks/main.yml

diff --git a/roles/kubernetes/control-plane/meta/main.yml b/roles/kubernetes/control-plane/meta/main.yml
index 7d793f92f39..7b2cfe3655d 100644
--- a/roles/kubernetes/control-plane/meta/main.yml
+++ b/roles/kubernetes/control-plane/meta/main.yml
@@ -1,5 +1,6 @@
 ---
 dependencies:
+  - role: kubernetes/kubeadm_common
   - role: kubernetes/tokens
     when: kube_token_auth
     tags:
diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml
index c13b6e833d3..d7783036bcd 100644
--- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml
+++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml
@@ -176,21 +176,6 @@
     - apiserver_sans_ip_check.changed or apiserver_sans_host_check.changed
     - not kube_external_ca_mode
 
-- name: Kubeadm | Create directory to store kubeadm patches
-  file:
-    path: "{{ kubeadm_patches.dest_dir }}"
-    state: directory
-    mode: "0640"
-  when: kubeadm_patches is defined and kubeadm_patches.enabled
-
-- name: Kubeadm | Copy kubeadm patches from inventory files
-  copy:
-    src: "{{ kubeadm_patches.source_dir }}/"
-    dest: "{{ kubeadm_patches.dest_dir }}"
-    owner: "root"
-    mode: "0644"
-  when: kubeadm_patches is defined and kubeadm_patches.enabled
-
 - name: Kubeadm | Initialize first control plane node
   command: >-
     timeout -k {{ kubeadm_init_timeout }} {{ kubeadm_init_timeout }}
diff --git a/roles/kubernetes/kubeadm/meta/main.yml b/roles/kubernetes/kubeadm/meta/main.yml
new file mode 100644
index 00000000000..4e727855957
--- /dev/null
+++ b/roles/kubernetes/kubeadm/meta/main.yml
@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: kubernetes/kubeadm_common
diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml
index 9e01f5fe5fa..cb29e2a6a32 100644
--- a/roles/kubernetes/kubeadm/tasks/main.yml
+++ b/roles/kubernetes/kubeadm/tasks/main.yml
@@ -83,21 +83,6 @@
     mode: "0640"
   when: ('kube_control_plane' not in group_names)
 
-- name: Kubeadm | Create directory to store kubeadm patches
-  file:
-    path: "{{ kubeadm_patches.dest_dir }}"
-    state: directory
-    mode: "0640"
-  when: kubeadm_patches is defined and kubeadm_patches.enabled
-
-- name: Kubeadm | Copy kubeadm patches from inventory files
-  copy:
-    src: "{{ kubeadm_patches.source_dir }}/"
-    dest: "{{ kubeadm_patches.dest_dir }}"
-    owner: "root"
-    mode: "0644"
-  when: kubeadm_patches is defined and kubeadm_patches.enabled
-
 - name: Join to cluster if needed
   environment:
     PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}:/sbin"
diff --git a/roles/kubernetes/kubeadm_common/tasks/main.yml b/roles/kubernetes/kubeadm_common/tasks/main.yml
new file mode 100644
index 00000000000..b1f316e225d
--- /dev/null
+++ b/roles/kubernetes/kubeadm_common/tasks/main.yml
@@ -0,0 +1,15 @@
+---
+- name: Kubeadm | Create directory to store kubeadm patches
+  file:
+    path: "{{ kubeadm_patches.dest_dir }}"
+    state: directory
+    mode: "0640"
+  when: kubeadm_patches is defined and kubeadm_patches.enabled
+
+- name: Kubeadm | Copy kubeadm patches from inventory files
+  copy:
+    src: "{{ kubeadm_patches.source_dir }}/"
+    dest: "{{ kubeadm_patches.dest_dir }}"
+    owner: "root"
+    mode: "0644"
+  when: kubeadm_patches is defined and kubeadm_patches.enabled

From 8e254ec1e82377f4401775f5111353a4e0b43abb Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 10 Sep 2024 12:00:26 +0200
Subject: [PATCH 05/19] kubeadm: allow to provide patch inline in inventories

Specifying one directory for kubeadm patches is not ideal:
1. It does not allow working with multiples inventories easily
2. No ansible templating of the patch
3. Ansible path searching can sometimes be confusing

Instead, provide the patch directly in a variable, and add some quality
of life to handle components targeting and patch ordering more
explicitly (`target` and `type` which are translated to the kubeadm
scheme which is based on the file name)
---
 .../control-plane/tasks/kubeadm-upgrade.yml        |  4 ++--
 .../templates/kubeadm-config.v1beta3.yaml.j2       |  4 ++--
 .../templates/kubeadm-controlplane.v1beta3.yaml.j2 |  4 ++--
 .../templates/kubeadm-client.conf.v1beta3.j2       |  4 ++--
 roles/kubernetes/kubeadm_common/defaults/main.yml  | 14 ++++++++++++++
 roles/kubernetes/kubeadm_common/tasks/main.yml     | 12 +++++++-----
 6 files changed, 29 insertions(+), 13 deletions(-)
 create mode 100644 roles/kubernetes/kubeadm_common/defaults/main.yml

diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml
index 343724c473c..9609c2f3d85 100644
--- a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml
+++ b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml
@@ -18,7 +18,7 @@
     --ignore-preflight-errors=all
     --allow-experimental-upgrades
     --etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | bool | lower }}
-    {% if kubeadm_patches is defined and kubeadm_patches.enabled %}--patches={{ kubeadm_patches.dest_dir }}{% endif %}
+    {% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %}
     --force
   register: kubeadm_upgrade
   # Retry is because upload config sometimes fails
@@ -39,7 +39,7 @@
     --ignore-preflight-errors=all
     --allow-experimental-upgrades
     --etcd-upgrade={{ (etcd_deployment_type == "kubeadm") | bool | lower }}
-    {% if kubeadm_patches is defined and kubeadm_patches.enabled %}--patches={{ kubeadm_patches.dest_dir }}{% endif %}
+    {% if kubeadm_patches | length > 0 %}--patches={{ kubeadm_patches_dir }}{% endif %}
     --force
   register: kubeadm_upgrade
   # Retry is because upload config sometimes fails
diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2
index ca48a3a9116..9dd5e43764e 100644
--- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2
+++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2
@@ -28,9 +28,9 @@ nodeRegistration:
   kubeletExtraArgs:
     cloud-provider: external
 {% endif %}
-{% if kubeadm_patches is defined and kubeadm_patches.enabled %}
+{% if kubeadm_patches | length > 0 %}
 patches:
-  directory: {{ kubeadm_patches.dest_dir }}
+  directory: {{ kubeadm_patches_dir }}
 {% endif %}
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
diff --git a/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2
index cd19b5c2e5f..bc9f3bdf929 100644
--- a/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2
+++ b/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2
@@ -31,7 +31,7 @@ nodeRegistration:
 {% else %}
   taints: []
 {% endif %}
-{% if kubeadm_patches is defined and kubeadm_patches.enabled %}
+{% if kubeadm_patches | length > 0 %}
 patches:
-  directory: {{ kubeadm_patches.dest_dir }}
+  directory: {{ kubeadm_patches_dir }}
 {% endif %}
diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2
index 3b3bc57de46..5016df9c3dc 100644
--- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2
+++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2
@@ -38,7 +38,7 @@ nodeRegistration:
   - effect: NoSchedule
     key: node-role.kubernetes.io/calico-rr
 {% endif %}
-{% if kubeadm_patches is defined and kubeadm_patches.enabled %}
+{% if kubeadm_patches | length > 0 %}
 patches:
-  directory: {{ kubeadm_patches.dest_dir }}
+  directory: {{ kubeadm_patches_dir }}
 {% endif %}
diff --git a/roles/kubernetes/kubeadm_common/defaults/main.yml b/roles/kubernetes/kubeadm_common/defaults/main.yml
new file mode 100644
index 00000000000..f7d70691a27
--- /dev/null
+++ b/roles/kubernetes/kubeadm_common/defaults/main.yml
@@ -0,0 +1,14 @@
+---
+kubeadm_patches_dir: "{{ kube_config_dir }}/patches"
+kubeadm_patches: []
+# kubeadm_patches:
+# - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration
+#   type: strategic(default)|json|merge
+#   patch:
+#    metadata:
+#      annotations:
+#        example.com/test: "true"
+#      labels:
+#        example.com/prod_level: "{{ prod_level }}"
+# - ...
+# Patches are applied in the order they are specified.
diff --git a/roles/kubernetes/kubeadm_common/tasks/main.yml b/roles/kubernetes/kubeadm_common/tasks/main.yml
index b1f316e225d..0f8d3b0a0af 100644
--- a/roles/kubernetes/kubeadm_common/tasks/main.yml
+++ b/roles/kubernetes/kubeadm_common/tasks/main.yml
@@ -1,15 +1,17 @@
 ---
 - name: Kubeadm | Create directory to store kubeadm patches
   file:
-    path: "{{ kubeadm_patches.dest_dir }}"
+    path: "{{ kubeadm_patches_dir }}"
     state: directory
     mode: "0640"
-  when: kubeadm_patches is defined and kubeadm_patches.enabled
+  when: kubeadm_patches | length > 0
 
 - name: Kubeadm | Copy kubeadm patches from inventory files
   copy:
-    src: "{{ kubeadm_patches.source_dir }}/"
-    dest: "{{ kubeadm_patches.dest_dir }}"
+    content: "{{ item.patch | to_yaml }}"
+    dest: "{{ kubeadm_patches_dir }}/{{ item.target }}{{ suffix }}+{{ item.type | d('strategic') }}.yaml"
     owner: "root"
     mode: "0644"
-  when: kubeadm_patches is defined and kubeadm_patches.enabled
+  loop: "{{ kubeadm_patches }}"
+  loop_control:
+    index_var: suffix

From c87097fc35b491cfddabb4e779bab1fe3e9d0396 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 10 Sep 2024 17:34:04 +0200
Subject: [PATCH 06/19] Document how to use kubeadm patches

---
 docs/ansible/vars.md                          |  7 ++++++
 .../group_vars/k8s_cluster/k8s-cluster.yml    | 24 +++++++++++++++----
 .../kube-controller-manager+merge.yaml        |  8 -------
 .../sample/patches/kube-scheduler+merge.yaml  |  8 -------
 .../kubeadm_common/defaults/main.yml          |  6 +++++
 5 files changed, 32 insertions(+), 21 deletions(-)
 delete mode 100644 inventory/sample/patches/kube-controller-manager+merge.yaml
 delete mode 100644 inventory/sample/patches/kube-scheduler+merge.yaml

diff --git a/docs/ansible/vars.md b/docs/ansible/vars.md
index b172f4ada27..f8d040e12ea 100644
--- a/docs/ansible/vars.md
+++ b/docs/ansible/vars.md
@@ -337,6 +337,13 @@ in the form of dicts of key-value pairs of configuration parameters that will be
 * *kube_kubeadm_controller_extra_args*
 * *kube_kubeadm_scheduler_extra_args*
 
+### Kubeadm patches
+
+When extra flags are not sufficient and there is a need to further customize kubernetes components,
+[kubeadm patches](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches)
+can be used.
+You should use the [`kubeadm_patches` variable](../../roles/kubernetes/kubeadm_common/defaults/main.yml) for that purpose.
+
 ## App variables
 
 * *helm_version* - Only supports v3.x. Existing v2 installs (with Tiller) will not be modified and need to be removed manually.
diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml
index 522ddc58903..24f896818a7 100644
--- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml
+++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml
@@ -366,11 +366,25 @@ auto_renew_certificates: false
 # First Monday of each month
 # auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube_control_plane'].index(inventory_hostname) }}0:00"
 
-# kubeadm patches path
-kubeadm_patches:
-  enabled: false
-  source_dir: "{{ inventory_dir }}/patches"
-  dest_dir: "{{ kube_config_dir }}/patches"
+kubeadm_patches_dir: "{{ kube_config_dir }}/patches"
+kubeadm_patches: []
+# See https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches
+# Correspondance with this link
+# patchtype = type
+# target = target
+# suffix -> managed automatically
+# extension -> always "yaml"
+# kubeadm_patches:
+# - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration
+#   type: strategic(default)|json|merge
+#   patch:
+#    metadata:
+#      annotations:
+#        example.com/test: "true"
+#      labels:
+#        example.com/prod_level: "{{ prod_level }}"
+# - ...
+# Patches are applied in the order they are specified.
 
 # Set to true to remove the role binding to anonymous users created by kubeadm
 remove_anonymous_access: false
diff --git a/inventory/sample/patches/kube-controller-manager+merge.yaml b/inventory/sample/patches/kube-controller-manager+merge.yaml
deleted file mode 100644
index 3f0fbbcd5e8..00000000000
--- a/inventory/sample/patches/kube-controller-manager+merge.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-controller-manager
-  annotations:
-    prometheus.io/scrape: 'true'
-    prometheus.io/port: '10257'
diff --git a/inventory/sample/patches/kube-scheduler+merge.yaml b/inventory/sample/patches/kube-scheduler+merge.yaml
deleted file mode 100644
index 00f457237cf..00000000000
--- a/inventory/sample/patches/kube-scheduler+merge.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-scheduler
-  annotations:
-    prometheus.io/scrape: 'true'
-    prometheus.io/port: '10259'
diff --git a/roles/kubernetes/kubeadm_common/defaults/main.yml b/roles/kubernetes/kubeadm_common/defaults/main.yml
index f7d70691a27..acbcdcf5fae 100644
--- a/roles/kubernetes/kubeadm_common/defaults/main.yml
+++ b/roles/kubernetes/kubeadm_common/defaults/main.yml
@@ -1,6 +1,12 @@
 ---
 kubeadm_patches_dir: "{{ kube_config_dir }}/patches"
 kubeadm_patches: []
+# See https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches
+# Correspondance with this link
+# patchtype = type
+# target = target
+# suffix -> managed automatically
+# extension -> always "yaml"
 # kubeadm_patches:
 # - target: kube-apiserver|kube-controller-manager|kube-scheduler|etcd|kubeletconfiguration
 #   type: strategic(default)|json|merge

From d54356e113785677246fe3d3c1b017ca9e24f1db Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Tue, 10 Sep 2024 17:38:56 +0200
Subject: [PATCH 07/19] Add testcase for kubeadm_patches

---
 .../packet_ubuntu24-calico-etcd-datastore.yml   | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/files/packet_ubuntu24-calico-etcd-datastore.yml b/tests/files/packet_ubuntu24-calico-etcd-datastore.yml
index 4f35d2f8763..7a27ce56610 100644
--- a/tests/files/packet_ubuntu24-calico-etcd-datastore.yml
+++ b/tests/files/packet_ubuntu24-calico-etcd-datastore.yml
@@ -27,3 +27,20 @@ containerd_registries_mirrors:
         skip_verify: true
 
 calico_datastore: "etcd"
+
+# Test kubeadm patches
+kubeadm_patches:
+  - target: kube-apiserver
+    patch:
+      metadata:
+        annotations:
+          example.com/test: "true"
+        labels:
+          example.com/prod_level: "prep"
+  - target: kube-controller-manager
+    patch:
+      metadata:
+        annotations:
+          example.com/test: "false"
+        labels:
+          example.com/prod_level: "prep"

From 1e769b726092b365a0dc57fb50e974a43f812981 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Thu, 12 Sep 2024 13:56:12 +0200
Subject: [PATCH 08/19] Rework vagrant inventory handling + support for
 multiples inventories

The current way to handle a custom inventory in vagrant is a bit
hackish, copy files around and can break Vagrantfile parsing in
cornercase scenarios (removing vagrant inventories, or the inventory
copied into vagrant inventory).

Instead, simply pass additional inventories to the ansible-playbook
command lines as raw arguments with `-i`.
This also makes supporting multiples inventories trivial, so we add a
new `$inventories` variable for that purpose.
---
 Vagrantfile    | 24 ++++++------------------
 tests/Makefile |  3 ++-
 2 files changed, 8 insertions(+), 19 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 8b43088b06c..a07f1a55f87 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -55,6 +55,8 @@ $subnet ||= "172.18.8"
 $subnet_ipv6 ||= "fd3c:b398:0698:0756"
 $os ||= "ubuntu2004"
 $network_plugin ||= "flannel"
+$inventory ||= "inventory/sample"
+$inventories ||= [$inventory]
 # Setting multi_networking to true will install Multus: https://github.com/k8snetworkplumbingwg/multus-cni
 $multi_networking ||= "False"
 $download_run_once ||= "True"
@@ -93,19 +95,6 @@ if ! SUPPORTED_OS.key?($os)
 end
 
 $box = SUPPORTED_OS[$os][:box]
-# if $inventory is not set, try to use example
-$inventory = "inventory/sample" if ! $inventory
-$inventory = File.absolute_path($inventory, File.dirname(__FILE__))
-
-# if $inventory has a hosts.ini file use it, otherwise copy over
-# vars etc to where vagrant expects dynamic inventory to be
-if ! File.exist?(File.join(File.dirname($inventory), "hosts.ini"))
-  $vagrant_ansible = File.join(File.absolute_path($vagrant_dir), "provisioners", "ansible")
-  FileUtils.mkdir_p($vagrant_ansible) if ! File.exist?($vagrant_ansible)
-  $vagrant_inventory = File.join($vagrant_ansible,"inventory")
-  FileUtils.rm_f($vagrant_inventory)
-  FileUtils.ln_s($inventory, $vagrant_inventory)
-end
 
 if Vagrant.has_plugin?("vagrant-proxyconf")
   $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost"
@@ -286,14 +275,13 @@ Vagrant.configure("2") do |config|
           ansible.playbook = $playbook
           ansible.compatibility_mode = "2.0"
           ansible.verbose = $ansible_verbosity
-          $ansible_inventory_path = File.join( $inventory, "hosts.ini")
-          if File.exist?($ansible_inventory_path)
-            ansible.inventory_path = $ansible_inventory_path
-          end
           ansible.become = true
           ansible.limit = "all,localhost"
           ansible.host_key_checking = false
-          ansible.raw_arguments = ["--forks=#{$num_instances}", "--flush-cache", "-e ansible_become_pass=vagrant"]
+          ansible.raw_arguments = ["--forks=#{$num_instances}",
+                                   "--flush-cache",
+                                   "-e ansible_become_pass=vagrant"] +
+                                   $inventories.map {|inv| ["-i", inv]}.flatten
           ansible.host_vars = host_vars
           ansible.extra_vars = $extra_vars
           if $ansible_tags != ""
diff --git a/tests/Makefile b/tests/Makefile
index b7d64f19d98..f052978b15e 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -35,7 +35,8 @@ cleanup-packet:
 
 create-vagrant:
 	vagrant up
-	cp  $(CI_PROJECT_DIR)/inventory/sample/vagrant_ansible_inventory $(INVENTORY)
+	cp  $(CI_PROJECT_DIR)/.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory $(INVENTORY)
+
 
 delete-vagrant:
 	vagrant destroy -f

From 4bf2d7a2c2bb71724696dea3adadb56c6bccd7e5 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Thu, 12 Sep 2024 16:44:30 +0200
Subject: [PATCH 09/19] Rework vagrant documentation

- Use proper syntax highlighting for config.rb examples
- Consistent shell style ($ as prompt)
- Use only one way to do things
- Remove OS specific details
---
 docs/developers/vagrant.md | 87 +++++++++++++-------------------------
 1 file changed, 30 insertions(+), 57 deletions(-)

diff --git a/docs/developers/vagrant.md b/docs/developers/vagrant.md
index 83b5b999e9c..3fac5851658 100644
--- a/docs/developers/vagrant.md
+++ b/docs/developers/vagrant.md
@@ -29,7 +29,26 @@ You can override the default settings in the `Vagrantfile` either by
 directly modifying the `Vagrantfile` or through an override file.
 In the same directory as the `Vagrantfile`, create a folder called
 `vagrant` and create `config.rb` file in it.
-An example of how to configure this file is given below.
+
+Example:
+
+```ruby
+# vagrant/config.rb
+$instance_name_prefix = "kub"
+$vm_cpus = 1
+$num_instances = 3
+$os = "centos8-bento"
+$subnet = "10.0.20"
+$network_plugin = "flannel"
+
+$extra_vars = {
+    dns_domain: my.custom.domain
+}
+# or
+$extra_vars = "path/to/extra/vars/file.yml"
+```
+
+For all available options look at the Vagrantfile (search for "CONFIG")
 
 ## Use alternative OS for Vagrant
 
@@ -57,73 +76,33 @@ see [download documentation](/docs/advanced/downloads.md).
 ## Example use of Vagrant
 
 The following is an example of setting up and running kubespray using `vagrant`.
-For repeated runs, you could save the script to a file in the root of the
-kubespray and run it by executing `source <name_of_the_file>`.
+Customize your settings as shown, above, then run the commands:
 
 ```ShellSession
 # use virtualenv to install all python requirements
 VENVDIR=venv
-virtualenv --python=/usr/bin/python3.7 $VENVDIR
-source $VENVDIR/bin/activate
-pip install -r requirements.txt
-
-# prepare an inventory to test with
-INV=inventory/my_lab
-rm -rf ${INV}.bak &> /dev/null
-mv ${INV} ${INV}.bak &> /dev/null
-cp -a inventory/sample ${INV}
-rm -f ${INV}/hosts.ini
-
-# customize the vagrant environment
-mkdir vagrant
-cat << EOF > vagrant/config.rb
-\$instance_name_prefix = "kub"
-\$vm_cpus = 1
-\$num_instances = 3
-\$os = "centos8-bento"
-\$subnet = "10.0.20"
-\$network_plugin = "flannel"
-\$inventory = "$INV"
-\$shared_folders = { 'temp/docker_rpms' => "/var/cache/yum/x86_64/7/docker-ce/packages" }
-\$extra_vars = {
-    dns_domain: my.custom.domain
-}
-# or
-\$extra_vars = "path/to/extra/vars/file.yml"
-EOF
+$ virtualenv --python=/usr/bin/python3.7 $VENVDIR
+$ source $VENVDIR/bin/activate
+$ pip install -r requirements.txt
 
-# make the rpm cache
-mkdir -p temp/docker_rpms
+$ vagrant up
 
-vagrant up
-
-# make a copy of the downloaded docker rpm, to speed up the next provisioning run
-scp kub-1:/var/cache/yum/x86_64/7/docker-ce/packages/* temp/docker_rpms/
-
-# copy kubectl access configuration in place
-mkdir $HOME/.kube/ &> /dev/null
-ln -s $PWD/$INV/artifacts/admin.conf $HOME/.kube/config
+# Access the cluster
+$ export INV=.vagrant/provisionners/ansible/inventory
+$ export KUBECONFIG=${INV}/artifacts/admin.conf
 # make the kubectl binary available
-sudo ln -s $PWD/$INV/artifacts/kubectl /usr/local/bin/kubectl
-#or
-export PATH=$PATH:$PWD/$INV/artifacts
+$ export PATH=$PATH:$PWD/$INV/artifacts
 ```
 
 If a vagrant run failed and you've made some changes to fix the issue causing
 the fail, here is how you would re-run ansible:
 
 ```ShellSession
-ansible-playbook -vvv -i .vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory cluster.yml
+vagrant provision
 ```
 
 If all went well, you check if it's all working as expected:
 
-```ShellSession
-kubectl get nodes
-```
-
-The output should look like this:
-
 ```ShellSession
 $ kubectl get nodes
 NAME    STATUS   ROLES                  AGE     VERSION
@@ -134,12 +113,6 @@ kub-3   Ready    <none>                 3m7s    v1.22.5
 
 Another nice test is the following:
 
-```ShellSession
-kubectl get pods --all-namespaces -o wide
-```
-
-Which should yield something like the following:
-
 ```ShellSession
 $ kubectl get pods --all-namespaces -o wide
 NAMESPACE            NAME                                      READY   STATUS    RESTARTS   AGE     IP            NODE    NOMINATED NODE   READINESS GATES

From 8cb081a3d02e8855b26e9b5bbefe28b3122fad06 Mon Sep 17 00:00:00 2001
From: ChengHao Yang <17496418+tico88612@users.noreply.github.com>
Date: Sun, 15 Sep 2024 00:35:13 +0800
Subject: [PATCH 10/19] Fix: download hash crictl link (#11534)

Signed-off-by: ChengHao Yang <17496418+tico88612@users.noreply.github.com>
---
 scripts/download_hash.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/download_hash.py b/scripts/download_hash.py
index d4dbed5bbae..ecd901162bb 100644
--- a/scripts/download_hash.py
+++ b/scripts/download_hash.py
@@ -35,7 +35,7 @@ def version_compare(version):
     "ciliumcli_binary": "https://github.com/cilium/cilium-cli/releases/download/{version}/cilium-{os}-{arch}.tar.gz.sha256sum",
     "cni_binary": "https://github.com/containernetworking/plugins/releases/download/{version}/cni-plugins-{os}-{arch}-{version}.tgz.sha256",
     "containerd_archive": "https://github.com/containerd/containerd/releases/download/v{version}/containerd-{version}-{os}-{arch}.tar.gz.sha256sum",
-    "crictl": "https://github.com/kubernetes-sigs/cri-tools/releases/download/{version}/critest-{version}-{os}-{arch}.tar.gz.sha256",
+    "crictl": "https://github.com/kubernetes-sigs/cri-tools/releases/download/{version}/crictl-{version}-{os}-{arch}.tar.gz.sha256",
     "crio_archive": "https://storage.googleapis.com/cri-o/artifacts/cri-o.{arch}.{version}.tar.gz.sha256sum",
     "etcd_binary": "https://github.com/etcd-io/etcd/releases/download/{version}/SHA256SUMS",
     "kubeadm": "https://dl.k8s.io/release/{version}/bin/linux/{arch}/kubeadm.sha256",

From e5d2452828f794ad0ae3c182b4477bf7d33a8402 Mon Sep 17 00:00:00 2001
From: Kay Yan <kay.yan@daocloud.io>
Date: Sun, 15 Sep 2024 20:59:13 +0800
Subject: [PATCH 11/19] fix elastx cleanup security groups (#11411)

---
 scripts/openstack-cleanup/main.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/scripts/openstack-cleanup/main.py b/scripts/openstack-cleanup/main.py
index 28820de02ed..d10e4725b0f 100755
--- a/scripts/openstack-cleanup/main.py
+++ b/scripts/openstack-cleanup/main.py
@@ -37,17 +37,6 @@ def main():
     map_if_old(conn.compute.delete_server,
                conn.compute.servers())
 
-    print('Security groups...')
-    try:
-        map_if_old(conn.network.delete_security_group,
-                conn.network.security_groups())
-    except openstack.exceptions.ConflictException as ex:
-        # Need to delete port when security groups is in used
-        map_if_old(conn.network.delete_port,
-                   conn.network.ports())
-        map_if_old(conn.network.delete_security_group,
-                conn.network.security_groups())
-
     print('Ports...')
     try:
         map_if_old(conn.network.delete_port,
@@ -73,6 +62,17 @@ def main():
         map_if_old(conn.network.delete_port,
                    conn.network.ports())
 
+    print('Security groups...')
+    try:
+        map_if_old(conn.network.delete_security_group,
+                conn.network.security_groups())
+    except openstack.exceptions.ConflictException as ex:
+        # Need to delete port when security groups is in used
+        map_if_old(conn.network.delete_port,
+                   conn.network.ports())
+        map_if_old(conn.network.delete_security_group,
+                conn.network.security_groups())
+
     print('Subnets...')
     map_if_old(conn.network.delete_subnet,
                conn.network.subnets())

From 59dd71358518a45492f374194dfd43b66b52e5cf Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Sep 2024 08:31:14 +0100
Subject: [PATCH 12/19] Bump molecule from 24.8.0 to 24.9.0 (#11542)

Bumps [molecule](https://github.com/ansible-community/molecule) from 24.8.0 to 24.9.0.
- [Release notes](https://github.com/ansible-community/molecule/releases)
- [Commits](https://github.com/ansible-community/molecule/compare/v24.8.0...v24.9.0)

---
updated-dependencies:
- dependency-name: molecule
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 tests/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/requirements.txt b/tests/requirements.txt
index f840a1c4b85..dce3ea67d5d 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -3,7 +3,7 @@ ansible-lint==24.7.0
 apache-libcloud==3.8.0
 ara[server]==1.7.2
 dopy==0.3.7
-molecule==24.8.0
+molecule==24.9.0
 molecule-plugins[vagrant]==23.5.3
 pytest-testinfra==10.1.1
 python-vagrant==1.0.0

From c3de25c782308d3253150515bb88a0c37f141a86 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Wed, 18 Sep 2024 02:34:45 +0200
Subject: [PATCH 13/19] Move the CRI endpoint setting to kubelet config
 (#11550)

The `--container-runtime-endpoint` kubelet argument is deprecated in
favor of the config file alternative.
---
 roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 | 1 +
 roles/kubernetes/node/templates/kubelet.env.v1beta1.j2         | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2
index 870383c041b..325cc2fa11e 100644
--- a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2
+++ b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2
@@ -26,6 +26,7 @@ staticPodPath: {{ kube_manifest_dir }}
 cgroupDriver: {{ kubelet_cgroup_driver | default('systemd') }}
 containerLogMaxFiles: {{ kubelet_logfiles_max_nr }}
 containerLogMaxSize: {{ kubelet_logfiles_max_size }}
+containerRuntimeEndpoint : {{ cri_socket }}
 maxPods: {{ kubelet_max_pods }}
 podPidsLimit: {{ kubelet_pod_pids_limit }}
 address: {{ kubelet_bind_address }}
diff --git a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2
index a5aa369df57..643194bf193 100644
--- a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2
+++ b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2
@@ -11,7 +11,6 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 --config={{ kube_config_dir }}/kubelet-config.yaml \
 --kubeconfig={{ kube_config_dir }}/kubelet.conf \
 {# end kubeadm specific settings #}
---container-runtime-endpoint={{ cri_socket }} \
 --runtime-cgroups={{ kubelet_runtime_cgroups }} \
 {% endset %}
 

From e550118314750470cc0b770f6d6361961bd19cf0 Mon Sep 17 00:00:00 2001
From: Qasim Mehmood <18313886+ThisIsQasim@users.noreply.github.com>
Date: Wed, 18 Sep 2024 15:54:44 +0500
Subject: [PATCH 14/19] Allow setting annotations on ingress-nginx service
 (#11544)

---
 inventory/sample/group_vars/k8s_cluster/addons.yml            | 2 ++
 .../ingress_controller/ingress_nginx/defaults/main.yml        | 1 +
 .../ingress_nginx/templates/svc-ingress-nginx.yml.j2          | 4 ++++
 3 files changed, 7 insertions(+)

diff --git a/inventory/sample/group_vars/k8s_cluster/addons.yml b/inventory/sample/group_vars/k8s_cluster/addons.yml
index f3afed3243d..b436ca08034 100644
--- a/inventory/sample/group_vars/k8s_cluster/addons.yml
+++ b/inventory/sample/group_vars/k8s_cluster/addons.yml
@@ -104,6 +104,8 @@ gateway_api_enabled: false
 ingress_nginx_enabled: false
 # ingress_nginx_host_network: false
 # ingress_nginx_service_type: LoadBalancer
+# ingress_nginx_service_annotations:
+#   example.io/loadbalancerIPs: 1.2.3.4
 # ingress_nginx_service_nodeport_http: 30080
 # ingress_nginx_service_nodeport_https: 30081
 ingress_publish_status_address: ""
diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml b/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml
index 82943024b44..9d7a4c402d3 100644
--- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml
+++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml
@@ -4,6 +4,7 @@ ingress_nginx_host_network: false
 ingress_nginx_service_type: LoadBalancer
 ingress_nginx_service_nodeport_http: ""
 ingress_nginx_service_nodeport_https: ""
+ingress_nginx_service_annotations: {}
 ingress_publish_status_address: ""
 ingress_nginx_nodeselector:
   kubernetes.io/os: "linux"
diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx.yml.j2
index 072b9892b4b..ba51ffe94e7 100644
--- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx.yml.j2
+++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-ingress-nginx.yml.j2
@@ -7,6 +7,10 @@ metadata:
   labels:
     app.kubernetes.io/name: ingress-nginx
     app.kubernetes.io/part-of: ingress-nginx
+{% if ingress_nginx_service_annotations %}
+  annotations:
+    {{ ingress_nginx_service_annotations | to_nice_yaml(indent=2, width=1337) | indent(width=4) }}
+{% endif %}
 spec:
   type: {{ ingress_nginx_service_type }}
   ports:

From b3b00775ea1b0c29f8bb40c1b5e60de98b7cd158 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Wed, 18 Sep 2024 14:04:44 +0200
Subject: [PATCH 15/19] Remove unused test infrastructure (#11529)

These two files haven't been touched since 2016 and don't appear to be
referenced anywhere else
---
 tests/run-tests.sh |    8 -
 tests/shebang-unit | 1146 --------------------------------------------
 2 files changed, 1154 deletions(-)
 delete mode 100755 tests/run-tests.sh
 delete mode 100755 tests/shebang-unit

diff --git a/tests/run-tests.sh b/tests/run-tests.sh
deleted file mode 100755
index c20438e1025..00000000000
--- a/tests/run-tests.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#! /bin/bash
-
-# curl -# -C - -o shebang-unit https://raw.github.com/arpinum-oss/shebang-unit/master/releases/shebang-unit
-# chmod +x shebang-unit
-
-now=$(date +"%Y%m%d%H%M%S")
-mkdir -p ${PWD}/tests-results
-./shebang-unit --reporters=simple,junit --output-file=${PWD}/tests-results/junit_report-${now}.xml tests
diff --git a/tests/shebang-unit b/tests/shebang-unit
deleted file mode 100755
index 6f9a2bc6cbf..00000000000
--- a/tests/shebang-unit
+++ /dev/null
@@ -1,1146 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2015, Arpinum
-#
-# shebang-unit is free software: you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free Software
-# Foundation, either version 3 of the License, or (at your option) any later
-# version.
-#
-# shebang-unit is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# shebang-unit. If not, see http://www.gnu.org/licenses/lgpl.html.
-
-# shebang-unit all in one source file
-
-
-configuration__load() {
-  # yes/no representation used with shebang-unit parameters to activate
-  # stuff like colors
-  SBU_YES="yes"
-  SBU_NO="no"
-
-  # Colors for outputs
-  SBU_GREEN_COLOR_CODE="\\033[1;32m"
-  SBU_RED_COLOR_CODE="\\033[1;31m"
-  SBU_YELLOW_COLOR_CODE="\\033[1;33m"
-  SBU_DEFAULT_COLOR_CODE="\\e[0m"
-
-  # Functions coding coventions
-  SBU_GLOBAL_SETUP_FUNCTION_NAME="global_setup"
-  SBU_GLOBAL_TEARDOWN_FUNCTION_NAME="global_teardown"
-  SBU_SETUP_FUNCTION_NAME="setup"
-  SBU_TEARDOWN_FUNCTION_NAME="teardown"
-  SBU_FUNCTION_DECLARATION_REGEX="^[ ]*\(function\)\{0,1\}[ ]*\([A-Za-z0-9_-]\{1,\}\)[ ]*\(([ ]*)\)\{0,1\}[ ]*{"
-  SBU_PRIVATE_FUNCTION_NAME_REGEX="^_.*"
-
-  # Default configuration that can be modified with shebang-unit parameters
-  # For more information see shebang-unit usages
-  SBU_TEST_FILE_PATTERN="*_test.sh"
-  SBU_TEST_FUNCTION_PATTERN="*"
-  SBU_USE_COLORS="${SBU_YES}"
-  SBU_RANDOM_RUN="${SBU_NO}"
-  SBU_REPORTERS="simple"
-  SBU_JUNIT_REPORTER_OUTPUT_FILE="./junit_report.xml"
-
-  # Internal constants
-  SBU_SUCCESS_STATUS_CODE=0
-  SBU_FAILURE_STATUS_CODE=1
-  SBU_VALUE_SEPARATOR=","
-  SBU_TEMP_DIR="/tmp/.shebang-unit"
-  SBU_LAST_ASSERTION_MSG_KEY="last_assertion_message"
-  SBU_NO_RUN="${SBU_NO}"
-  SBU_STANDARD_FD=42
-  SBU_ERROR_FD=43
-}
-
-
-assertion__equal() {
-  if [[ "$1" != "$2" ]]; then
-    _assertion__failed "Actual: <$2>, expected: <$1>."
-  fi
-}
-
-assertion__different() {
-  if [[ "$1" == "$2" ]]; then
-    _assertion__failed "Both values are: <$1>."
-  fi
-}
-
-assertion__string_contains() {
-  if ! system__string_contains "$1" "$2"; then
-    _assertion__failed "String: <$1> does not contain: <$2>."
-  fi
-}
-
-assertion__string_does_not_contain() {
-  if system__string_contains "$1" "$2"; then
-    _assertion__failed "String: <$1> contains: <$2>."
-  fi
-}
-
-assertion__string_empty() {
-  if [[ -n "$1" ]]; then
-    _assertion__failed "String: <$1> is not empty."
-  fi
-}
-
-assertion__string_not_empty() {
-  if [[ -z "$1" ]]; then
-    _assertion__failed "The string is empty."
-  fi
-}
-
-assertion__array_contains() {
-  local element=$1
-  shift 1
-  if ! array__contains "${element}" "$@"; then
-    local array_as_string="$(system__pretty_print_array "$@")"
-    _assertion__failed \
-      "Array: <${array_as_string}> does not contain: <${element}>."
-  fi
-}
-
-assertion__array_does_not_contain() {
-  local element=$1
-  shift 1
-  if array__contains "${element}" "$@"; then
-    local array_as_string="$(system__pretty_print_array "$@")"
-    _assertion__failed \
-      "Array: <${array_as_string}> contains: <${element}>."
-  fi
-}
-
-assertion__status_code_is_success() {
-  if (( $1 != ${SBU_SUCCESS_STATUS_CODE} )); then
-    _assertion__failed \
-      "Status code is failure instead of success." "$2"
-  fi
-}
-
-assertion__status_code_is_failure() {
-  if (( $1 == ${SBU_SUCCESS_STATUS_CODE} )); then
-    _assertion__failed \
-      "Status code is success instead of failure." "$2"
-  fi
-}
-
-assertion__successful() {
-  "$@"
-  if (( $? != ${SBU_SUCCESS_STATUS_CODE} )); then
-    _assertion__failed "Command is failing instead of successful."
-  fi
-}
-
-assertion__failing() {
-  "$@"
-  if (( $? == ${SBU_SUCCESS_STATUS_CODE} )); then
-    _assertion__failed "Command is successful instead of failing."
-  fi
-}
-
-_assertion__failed() {
-  local message_to_use="$(_assertion__get_assertion_message_to_use "$1" "$2")"
-  system__print_line "Assertion failed. ${message_to_use}"
-  exit ${SBU_FAILURE_STATUS_CODE}
-}
-
-_assertion__get_assertion_message_to_use() {
-  local message=$1
-  local custom_messsage=$2
-  if [[ -n "${custom_messsage}" ]]; then
-    system__print "${message} ${custom_messsage}"
-  else
-    system__print "${message}"
-  fi
-}
-
-
-mock__make_function_do_nothing() {
-  mock__make_function_call "$1" ":"
-}
-
-mock__make_function_prints() {
-  local function=$1
-  local text=$2
-  eval "${function}() { printf "${text}"; }"
-}
-
-mock__make_function_call() {
-  local function_to_mock=$1
-  local function_to_call=$2
-  shift 2
-  eval "${function_to_mock}() { ${function_to_call} \"\$@\"; }"
-}
-
-
-runner__run_all_test_files() {
-  SBU_BASE_TEST_DIRECTORY=$1
-  reporter__test_files_start_running
-  timer__store_current_time "global_time"
-  results__test_files_start_running
-  _runner__run_all_test_files_with_pattern_in_directory "$1"
-  reporter__test_files_end_running "$(timer__get_time_elapsed "global_time")"
-  runner__tests_are_successful
-}
-
-_runner__run_all_test_files_with_pattern_in_directory() {
-  local file
-  local files
-  array__from_lines files <<< "$(_runner__get_test_files_in_directory "$1")"
-  for file in "${files[@]}"; do
-    file_runner__run_test_file "${file}"
-  done
-}
-
-_runner__get_test_files_in_directory() {
-  local files
-  array__from_lines files <<< "$(find "$1" -name "${SBU_TEST_FILE_PATTERN}" | sort)"
-  if [[ "${SBU_RANDOM_RUN}" == "${SBU_YES}"  ]]; then
-    array__from_lines files <<< "$(system__randomize_array "${files[@]}")"
-  fi
-  array__print "${files[@]}"
-}
-
-runner__tests_are_successful() {
-  (( $(results__get_failing_tests_count) == 0 \
-      && $(results__get_skipped_tests_count) == 0 ))
-}
-
-
-file_runner__run_test_file() {
-  local file=$1
-  local public_functions=($(parser__get_public_functions_in_file "${file}"))
-  local test_functions=($(_file_runner__get_test_functions))
-  reporter__test_file_starts_running "${file}" "${#test_functions[@]}"
-  ( source "${file}"
-    _file_runner__run_global_setup_if_exists \
-      && _file_runner__call_all_tests
-    _file_runner__run_global_teardown_if_exists )
-  _file_runner__check_if_global_setup_has_exited
-  reporter__test_file_ends_running
-}
-
-_file_runner__run_all_tests_if_global_setup_is_successful() {
-    _file_runner__call_all_tests
-}
-
-_file_runner__call_all_tests() {
-  local i
-  for (( i=0; i < ${#test_functions[@]}; i++ )); do
-    test_runner__run_test "${test_functions[${i}]}" "${public_functions[@]}"
-  done
-}
-
-_file_runner__skip_all_tests() {
-  local i
-  for (( i=0; i < ${#test_functions[@]}; i++ )); do
-    test_runner__skip_test "${test_functions[${i}]}" "${public_functions[@]}"
-  done
-}
-
-_file_runner__get_test_functions() {
-  local result=()
-  local test_function
-  for test_function in "${public_functions[@]}"; do
-    if _file_runner__function_is_a_test "${test_function}"\
-       && [[ "${test_function}" == ${SBU_TEST_FUNCTION_PATTERN} ]]; then
-      result+=("${test_function}")
-    fi
-  done
-  _file_runner__get_randomized_test_functions_if_needed "${result[@]}"
-}
-
-_file_runner__get_randomized_test_functions_if_needed() {
-  if [[ "${SBU_RANDOM_RUN}" == "${SBU_YES}" ]]; then
-    system__randomize_array "$@"
-  else
-    array__print "$@"
-  fi
-}
-
-_file_runner__run_global_setup_if_exists() {
-  database__put "sbu_current_global_setup_has_failed" "${SBU_YES}"
-  _file_runner__call_function_if_exists "${SBU_GLOBAL_SETUP_FUNCTION_NAME}" \
-    && database__put "sbu_current_global_setup_has_failed" "${SBU_NO}"
-}
-
-_file_runner__run_global_teardown_if_exists() {
-  _file_runner__call_function_if_exists "${SBU_GLOBAL_TEARDOWN_FUNCTION_NAME}"
-}
-
-_file_runner__function_is_a_test() {
-  ! array__contains "$1" \
-                      "${SBU_GLOBAL_SETUP_FUNCTION_NAME}" \
-                      "${SBU_GLOBAL_TEARDOWN_FUNCTION_NAME}" \
-                      "${SBU_SETUP_FUNCTION_NAME}" \
-                      "${SBU_TEARDOWN_FUNCTION_NAME}"
-}
-
-_file_runner__call_function_if_exists() {
-  local function=$1
-  shift 1
-  if array__contains "${function}" "${public_functions[@]}"; then
-    "${function}"
-  fi
-}
-
-_file_runner__check_if_global_setup_has_exited() {
-  local has_exited="$(database__get "sbu_current_global_setup_has_failed")"
-  if [[ "${has_exited}" == "${SBU_YES}" ]]; then
-    _file_runner__handle_failure_in_global_setup
-  fi
-}
-
-_file_runner__handle_failure_in_global_setup() {
-    reporter__global_setup_has_failed
-    _file_runner__skip_all_tests
-}
-
-
-parser__get_public_functions_in_file() {
-  _parser__find_functions_in_file "$1" \
-    | _parser__filter_private_functions \
-    | awk '{ print $1 }'
-}
-
-_parser__find_functions_in_file() {
-  grep -o "${SBU_FUNCTION_DECLARATION_REGEX}" "$1" \
-    | _parser__get_function_name_from_declaration
-}
-
-_parser__filter_private_functions() {
-  grep -v "${SBU_PRIVATE_FUNCTION_NAME_REGEX}"
-}
-
-_parser__get_function_name_from_declaration() {
-  sed "s/${SBU_FUNCTION_DECLARATION_REGEX}/\2/"
-}
-
-
-timer__store_current_time() {
-  local id=$1
-  database__put "sbu_beginning_date_$1" "$(system__get_date_in_seconds)"
-}
-
-timer__get_time_elapsed() {
-  local id=$1
-  local beginning_date="$(database__get "sbu_beginning_date_$1")"
-  local ending_date="$(system__get_date_in_seconds)"
-
-  [[ -n "${beginning_date}" ]] \
-    && system__print "$(( ending_date - beginning_date ))" \
-    || system__print "0"
-}
-
-
-results__test_files_start_running() {
-  database__put "sbu_successful_tests_count" "0"
-  database__put "sbu_failing_tests_count" "0"
-  database__put "sbu_skipped_tests_count" "0"
-}
-
-results__get_successful_tests_count() {
-  _results__get_tests_count_of_type "successful"
-}
-
-results__increment_successful_tests() {
-  _results__increment_tests_of_type "successful"
-}
-
-results__get_failing_tests_count() {
-  _results__get_tests_count_of_type "failing"
-}
-
-results__increment_failing_tests() {
-  _results__increment_tests_of_type "failing"
-}
-
-results__get_skipped_tests_count() {
-  _results__get_tests_count_of_type "skipped"
-}
-
-results__increment_skipped_tests() {
-  _results__increment_tests_of_type "skipped"
-}
-
-results__get_total_tests_count() {
-  local successes="$(results__get_successful_tests_count)"
-  local failures="$(results__get_failing_tests_count)"
-  local skipped="$(results__get_skipped_tests_count)"
-  system__print "$(( successes + failures + skipped ))"
-}
-
-_results__get_tests_count_of_type() {
-  local type=$1
-  database__get "sbu_${type}_tests_count"
-}
-
-_results__increment_tests_of_type() {
-  local type=$1
-  local count="$(results__get_${type}_tests_count)"
-  database__put "sbu_${type}_tests_count" "$(( count + 1 ))"
-}
-
-
-test_runner__run_test() {
-  local test_function=$1
-  shift 1
-  reporter__test_starts_running "${test_function}"
-  timer__store_current_time "test_time"
-  (
-    _test_runner__call_setup_if_exists "$@" \
-      && _test_runner__call_test_fonction "${test_function}"
-    local setup_and_test_code=$?
-    _test_runner__call_teardown_if_exists "$@"
-    (( $? == ${SBU_SUCCESS_STATUS_CODE} \
-    &&  ${setup_and_test_code} == ${SBU_SUCCESS_STATUS_CODE} ))
-  )
-  _test_runner__parse_test_function_result $?
-  reporter__test_ends_running "$(timer__get_time_elapsed "test_time")"
-}
-
-_test_runner__call_test_fonction() {
-  ( "$1" >&${SBU_STANDARD_FD} 2>&${SBU_ERROR_FD} )
-}
-
-_test_runner__call_setup_if_exists() {
-  _test_runner__call_function_if_exits "${SBU_SETUP_FUNCTION_NAME}" "$@"
-}
-
-_test_runner__call_teardown_if_exists() {
-  _test_runner__call_function_if_exits "${SBU_TEARDOWN_FUNCTION_NAME}" "$@"
-}
-
-_test_runner__parse_test_function_result() {
-  if (( $1 == ${SBU_SUCCESS_STATUS_CODE} )); then
-    results__increment_successful_tests
-    reporter__test_has_succeeded
-  else
-    results__increment_failing_tests
-    reporter__test_has_failed
-  fi
-}
-
-_test_runner__call_function_if_exits() {
-  local function=$1
-  shift 1
-  if array__contains "${function}" "$@"; then
-    "${function}"
-  fi
-}
-
-test_runner__skip_test() {
-  local test_function=$1
-  reporter__test_starts_running "${test_function}"
-  results__increment_skipped_tests
-  reporter__test_is_skipped "${test_function}"
-  reporter__test_ends_running 0
-}
-
-
-reporter__test_files_start_running() {
-  _reporter__initialise_file_descriptors
-  reporter__for_each_reporter \
-    _reporter__call_function "test_files_start_running" "$@"
-}
-
-_reporter__initialise_file_descriptors() {
-  eval "exec ${SBU_STANDARD_FD}>&1"
-  eval "exec ${SBU_ERROR_FD}>&2"
-}
-
-reporter__global_setup_has_failed() {
-  reporter__for_each_reporter \
-    _reporter__call_function "global_setup_has_failed" "$@"
-}
-
-reporter__test_file_starts_running() {
-  reporter__for_each_reporter \
-    _reporter__call_function "test_file_starts_running" "$@"
-}
-
-reporter__test_starts_running() {
-  reporter__for_each_reporter \
-    _reporter__call_function "test_starts_running" "$@"
-}
-
-reporter__test_has_succeeded() {
-  reporter__for_each_reporter \
-    _reporter__call_function "test_has_succeeded" "$@"
-}
-
-reporter__test_has_failed() {
-  reporter__for_each_reporter \
-    _reporter__call_function "test_has_failed" "$@"
-}
-
-reporter__test_is_skipped() {
-  reporter__for_each_reporter \
-    _reporter__call_function "test_is_skipped" "$@"
-}
-
-reporter__test_ends_running() {
-  reporter__for_each_reporter \
-    _reporter__call_function "test_ends_running" "$@"
-}
-
-reporter__test_file_ends_running() {
-  reporter__for_each_reporter \
-    _reporter__call_function "test_file_ends_running" "$@"
-}
-
-reporter__test_files_end_running() {
-  reporter__for_each_reporter \
-    _reporter__call_function "test_files_end_running" "$@"
-  _reporter__release_file_descriptors
-}
-
-_reporter__release_file_descriptors() {
-  eval "exec 1>&${SBU_STANDARD_FD} ${SBU_STANDARD_FD}>&-"
-  eval "exec 2>&${SBU_ERROR_FD} ${SBU_ERROR_FD}>&-"
-}
-
-_reporter__call_function() {
-  local function=$1
-  shift 1
-  "${reporter}_reporter__${function}" "$@"
-}
-
-reporter__for_each_reporter() {
-  local reporter
-  for reporter in ${SBU_REPORTERS//${SBU_VALUE_SEPARATOR}/ }; do
-    "$@"
-  done
-}
-
-reporter__print_with_color() {
-  system__print_with_color "$@" >&${SBU_STANDARD_FD}
-}
-
-reporter__print_line() {
-  system__print_line "$@" >&${SBU_STANDARD_FD}
-}
-
-reporter__print_line_with_color() {
-  system__print_line_with_color "$@" >&${SBU_STANDARD_FD}
-}
-
-reporter__print_new_line() {
-  system__print_new_line >&${SBU_STANDARD_FD}
-}
-
-reporter__get_color_code_for_tests_result() {
-  local color_code=${SBU_GREEN_COLOR_CODE}
-  if ! runner__tests_are_successful; then
-    color_code=${SBU_RED_COLOR_CODE}
-  fi
-  system__print "${color_code}"
-}
-
-reporter__get_test_file_relative_name() {
-  system__print "${1#${SBU_BASE_TEST_DIRECTORY}\/}"
-}
-
-
-simple_reporter__test_files_start_running() {
-  :
-}
-
-simple_reporter__test_file_starts_running() {
-  local relative_name="$(reporter__get_test_file_relative_name "$1")"
-  reporter__print_line "[File] ${relative_name}"
-}
-
-simple_reporter__global_setup_has_failed() {
-  reporter__print_line_with_color \
-    "Global setup has failed" ${SBU_YELLOW_COLOR_CODE}
-}
-
-simple_reporter__test_starts_running() {
-  reporter__print_line "[Test] $1"
-}
-
-simple_reporter__test_has_succeeded() {
-  reporter__print_line_with_color "OK" ${SBU_GREEN_COLOR_CODE}
-}
-
-simple_reporter__test_has_failed() {
-  reporter__print_line_with_color "KO" ${SBU_RED_COLOR_CODE}
-}
-
-simple_reporter__test_is_skipped() {
-  reporter__print_line_with_color "Skipped" ${SBU_YELLOW_COLOR_CODE}
-}
-
-simple_reporter__test_ends_running() {
-  :
-}
-
-simple_reporter__test_file_ends_running() {
-  reporter__print_new_line
-}
-
-simple_reporter__test_files_end_running() {
-  local time="in $1s"
-  reporter__print_line "[Results]"
-  local color="$(reporter__get_color_code_for_tests_result)"
-  local total_count="$(_simple_reporter__get_total_count_message)"
-  local failures_count="$(_simple_reporter__get_failures_count_message)"
-  local skipped_count="$(results__get_skipped_tests_count) skipped"
-  local message="${total_count}, ${failures_count}, ${skipped_count} ${time}"
-  reporter__print_line_with_color "${message}" "${color}"
-}
-
-_simple_reporter__get_total_count_message() {
-  local count="$(results__get_total_tests_count)"
-  system__print "${count} test$(_simple_reporter__get_agreement ${count})"
-}
-
-_simple_reporter__get_failures_count_message() {
-  local count="$(results__get_failing_tests_count)"
-  system__print "${count} failure$(_simple_reporter__get_agreement ${count})"
-}
-
-_simple_reporter__get_agreement() {
-  (( $1 > 1 )) \
-    && system__print "s" \
-    || system__print ""
-}
-
-
-dots_reporter__test_files_start_running() {
-  exec 1>/dev/null
-  exec 2>/dev/null
-}
-
-dots_reporter__test_file_starts_running() {
-  :
-}
-
-dots_reporter__global_setup_has_failed() {
-  :
-}
-
-dots_reporter__test_starts_running() {
-  :
-}
-
-dots_reporter__test_has_succeeded() {
-  reporter__print_with_color "." ${SBU_GREEN_COLOR_CODE}
-}
-
-dots_reporter__test_has_failed() {
-  reporter__print_with_color "F" ${SBU_RED_COLOR_CODE}
-}
-
-dots_reporter__test_is_skipped() {
-  reporter__print_with_color "S" ${SBU_YELLOW_COLOR_CODE}
-}
-
-dots_reporter__test_ends_running() {
-  :
-}
-
-dots_reporter__test_file_ends_running() {
-  :
-}
-
-dots_reporter__test_files_end_running() {
-  local color="$(reporter__get_color_code_for_tests_result)"
-  local texte="$(runner__tests_are_successful \
-                  && system__print "OK" \
-                  || system__print "KO")"
-  reporter__print_line_with_color "${texte}" "${color}"
-}
-
-
-junit_reporter__test_files_start_running() {
-  _junit_reporter__initialise_report_with \
-    "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>"
-  _junit_reporter__write_line_to_report "<testsuites>"
-}
-
-junit_reporter__test_file_starts_running() {
-  local file_name=$1
-  local test_count=$2
-  local suite_name="$(_junit_reporter__get_suite_name "${file_name}")"
-  database__put "sbu_current_suite_name" "${suite_name}"
-  _junit_reporter__write_line_to_report \
-    "  <testsuite name=\"${suite_name}\" tests=\"${test_count}\">"
-  _junit_reporter__delete_all_outputs_lines "suite"
-  _junit_reporter__redirect_outputs_to_database "suite"
-}
-
-junit_reporter__global_setup_has_failed() {
-  :
-}
-
-junit_reporter__test_starts_running() {
-  local suite_name="$(database__get "sbu_current_suite_name")"
-  local test_name="$(xml__encode_text "$1")"
-  _junit_reporter__write_line_to_report \
-    "    <testcase name=\"${test_name}\" classname=\"${suite_name}\" \
-time=\"\${sbu_current_test_time}\">"
-  _junit_reporter__delete_all_outputs_lines "test"
-  _junit_reporter__redirect_outputs_to_database "test"
-}
-
-junit_reporter__test_has_succeeded() {
-  :
-}
-
-junit_reporter__test_has_failed() {
-  _junit_reporter__write_line_to_report "      <failure>"
-  _junit_reporter__write_line_to_report "      </failure>"
-}
-
-junit_reporter__test_is_skipped() {
-  _junit_reporter__write_line_to_report "      <skipped>"
-  _junit_reporter__write_line_to_report "      </skipped>"
-}
-
-junit_reporter__test_ends_running() {
-  _junit_reporter__redirect_outputs_to_database "suite"
-  _junit_reporter__write_time_in_current_test_case_tag_in_report "$1"
-  _junit_reporter__flush_all_outputs_to_report_if_any "test"
-  _junit_reporter__write_line_to_report "    </testcase>"
-}
-
-_junit_reporter__write_time_in_current_test_case_tag_in_report() {
-  local test_time=$1
-  local report_content=$(cat "${SBU_JUNIT_REPORTER_OUTPUT_FILE}")
-  local content_with_time="$(system__substitute_variable \
-    "${report_content}" "sbu_current_test_time" "${test_time}")"
-  system__print_line \
-    "${content_with_time}" > "${SBU_JUNIT_REPORTER_OUTPUT_FILE}"
-}
-
-junit_reporter__test_file_ends_running() {
-  _junit_reporter__flush_all_outputs_to_report_if_any "suite"
-  _junit_reporter__write_line_to_report "  </testsuite>"
-  database__put "sbu_current_suite_name" ""
-}
-
-junit_reporter__test_files_end_running() {
-  _junit_reporter__write_line_to_report "</testsuites>"
-}
-
-_junit_reporter__get_suite_name() {
-  local relative_name="$(reporter__get_test_file_relative_name "$1")"
-  local dots_replaced_by_underscores="${relative_name//./_}"
-  local slashes_replaced_by_dots="${dots_replaced_by_underscores//\//.}"
-  xml__encode_text "${slashes_replaced_by_dots}"
-}
-
-_junit_reporter__initialise_report_with() {
-  system__print_line "$1" > "${SBU_JUNIT_REPORTER_OUTPUT_FILE}"
-}
-
-_junit_reporter__write_line_to_report() {
-  system__print_line "$1" >> "${SBU_JUNIT_REPORTER_OUTPUT_FILE}"
-}
-
-_junit_reporter__redirect_outputs_to_database() {
-  local scope=$1
-  exec 1>>\
-    "$(database__get_descriptor "sbu_current_${scope}_standard_ouputs_lines")"
-  exec 2>>\
-    "$(database__get_descriptor "sbu_current_${scope}_error_ouputs_lines")"
-}
-
-_junit_reporter__delete_all_outputs_lines() {
-  database__put "sbu_current_$1_standard_ouputs_lines"
-  database__put "sbu_current_$1_error_ouputs_lines"
-}
-
-_junit_reporter__flush_all_outputs_to_report_if_any() {
-  _junit_reporter__flush_outputs_to_report_if_any "$1" "standard"
-  _junit_reporter__flush_outputs_to_report_if_any "$1" "error"
-}
-
-_junit_reporter__flush_outputs_to_report_if_any() {
-  local scope=$1
-  local outputs_type=$2
-  local key="sbu_current_${scope}_${outputs_type}_ouputs_lines"
-  local outputs="$(database__get "${key}")"
-  if [[ -n "${outputs}" ]]; then
-    _junit_reporter__write_outputs_to_report \
-      "${scope}" "${outputs_type}" "${outputs}"
-    database__put "${key}" ""
-  fi
-}
-
-_junit_reporter__write_outputs_to_report() {
-  local scope=$1
-  local outputs_type=$2
-  local outputs=$3
-  local tag="$(_junit_reporter__get_tag_for_outputs_type "${outputs_type}")"
-  local indentation="$(_junit_reporter__get_indentation_for_scope "${scope}")"
-  _junit_reporter__write_line_to_report "${indentation}<${tag}>"
-  _junit_reporter__write_line_to_report "$(xml__encode_text "${outputs}")"
-  _junit_reporter__write_line_to_report "${indentation}</${tag}>"
-}
-
-_junit_reporter__get_tag_for_outputs_type() {
-  [[ "$1" == "standard" ]] \
-    && system__print "system-out" \
-    || system__print "system-err"
-}
-
-_junit_reporter__get_indentation_for_scope() {
-  [[ "$1" == "suite" ]] \
-    && system__print "    " \
-    || system__print "      "
-}
-
-
-xml__encode_text() {
-  local encoded=${1//\&/\&amp\;}
-  encoded=${encoded//\</\&lt\;}
-  encoded=${encoded//\>/\&gt\;}
-  encoded=${encoded//\"/\&quot\;}
-  encoded=${encoded//\'/\&apos\;}
-  system__print "${encoded}"
-}
-
-
-database__initialise() {
-  _SBU_DB_TOKEN="$(system__random)"
-  _database__ensure_directory_exists
-}
-
-database__release() {
-  rm -rf "$(_database__get_dir)"
-}
-
-database__put() {
-  _database__ensure_directory_exists
-  system__print "$2" > "$(_database__get_dir)/$1"
-}
-
-database__post() {
-  _database__ensure_directory_exists
-  system__print "$2" >> "$(_database__get_dir)/$1"
-}
-
-database__post_line() {
-  _database__ensure_directory_exists
-  system__print_line "$2" >> "$(_database__get_dir)/$1"
-}
-
-database__put_variable() {
-  _database__ensure_directory_exists
-  database__put "$1" "${!1}"
-}
-
-database__get() {
-  [[ -e "$(_database__get_dir)/$1" ]] && cat "$(_database__get_dir)/$1"
-}
-
-database__get_descriptor() {
-  system__print "$(_database__get_dir)/$1"
-}
-
-_database__ensure_directory_exists() {
-  mkdir -p "$(_database__get_dir)"
-}
-
-_database__get_dir() {
-  system__print "${SBU_TEMP_DIR}/database/${_SBU_DB_TOKEN}"
-}
-
-
-system__get_string_or_default() {
-  [[ -n "$1" ]] \
-    && system__print "$1" \
-    || system__print "$2"
-}
-
-system__get_date_in_seconds() {
-  date +%s
-}
-
-system__print_line_with_color() {
-  system__print_with_color "$@"
-  system__print_new_line
-}
-
-system__print_with_color() {
-  if [[ "${SBU_USE_COLORS}" == "${SBU_YES}" ]]; then
-    printf "$2$1${SBU_DEFAULT_COLOR_CODE}"
-  else
-    system__print "$1"
-  fi
-}
-
-system__print_line() {
-  system__print "$1"
-  system__print_new_line
-}
-
-system__print() {
-  printf "%s" "$1"
-}
-
-system__print_new_line() {
-  printf "\n"
-}
-
-array__contains() {
-  local value=$1
-  shift 1
-  local i
-  for (( i=1; i <= $#; i++ )); do
-    if [[ "${!i}" == "${value}" ]]; then
-      return ${SBU_SUCCESS_STATUS_CODE}
-    fi
-  done
-  return ${SBU_FAILURE_STATUS_CODE}
-}
-
-array__from_lines() {
-  local IFS=$'\n'
-  eval "$1=(\$(</dev/stdin))"
-}
-
-array__print() {
-  local element
-  for element in "$@"; do
-    system__print_line "${element}"
-  done
-}
-
-system__pretty_print_array() {
-  local array_as_string=""
-  local i
-  for (( i=1; i <= $#; i++ )); do
-    array_as_string+="${!i}, "
-  done
-  array_as_string=${array_as_string/%, /}
-  printf "[%s]" "${array_as_string}"
-}
-
-system__string_contains() {
-  [[ "$1" == *"$2"* ]]
-}
-
-system__randomize_array() {
-  local copy=("$@")
-  while (( ${#copy[@]} > 0 )); do
-    local random_index=$(( $(system__random) % ${#copy[@]} ))
-    system__print_line "${copy[${random_index}]}"
-    unset copy[${random_index}]
-    copy=("${copy[@]}")
-  done
-}
-
-system__random() {
-  system__print "${RANDOM}"
-}
-
-system__substitute_variable() {
-    local string=$1
-    local key="\$\{$2\}"
-    local value=$3
-    printf "%s" "${string//${key}/${value}}"
-}
-
-
-main__main() {
-  configuration__load
-  _main__initialise
-  local parsed_arguments=0
-  _main__parse_arguments "$@"
-  shift ${parsed_arguments}
-  _main__assert_only_one_argument_left $#
-  _main__assert_reporters_are_known
-  SBU_BASE_TEST_DIRECTORY=$1
-
-  if [[ "${SBU_NO_RUN}" != "${SBU_YES}" ]]; then
-    runner__run_all_test_files "$1"
-    return $?
-  fi
-}
-
-_main__initialise() {
-  database__initialise
-  trap _main__release EXIT
-}
-
-_main__release() {
-  database__release
-}
-
-_main__parse_arguments() {
-  local argument
-  for argument in "$@"; do
-    case "${argument}" in
-      -a|--api-cheat-sheet)
-      _main__print_api_cheat_sheet_and_exit
-      ;;
-      -c=*|--colors=*)
-      SBU_USE_COLORS="${argument#*=}"
-      (( parsed_arguments++ ))
-      ;;
-      -d=*|--random-run=*)
-      SBU_RANDOM_RUN="${argument#*=}"
-      (( parsed_arguments++ ))
-      ;;
-      -h|--help)
-      _main__print_full_usage
-      exit ${SBU_SUCCESS_STATUS_CODE}
-      ;;
-      -f=*|--file-pattern=*)
-      SBU_TEST_FILE_PATTERN="${argument#*=}"
-      (( parsed_arguments++ ))
-      ;;
-      --no-run)
-      SBU_NO_RUN="${SBU_YES}"
-      (( parsed_arguments++ ))
-      ;;
-      -o=*|--output-file=*)
-      SBU_JUNIT_REPORTER_OUTPUT_FILE="${argument#*=}"
-      (( parsed_arguments++ ))
-      ;;
-      -t=*|--test-pattern=*)
-      SBU_TEST_FUNCTION_PATTERN="${argument#*=}"
-      (( parsed_arguments++ ))
-      ;;
-      -r=*|--reporters=*)
-      SBU_REPORTERS="${argument#*=}"
-      (( parsed_arguments++ ))
-      ;;
-      -*|--*)
-      _main__print_illegal_option "${argument}"
-      _main__print_usage_and_exit_with_code ${SBU_FAILURE_STATUS_CODE}
-      ;;
-    esac
-  done
-}
-
- _main__assert_reporters_are_known() {
-  reporter__for_each_reporter _main__fail_if_reporter_unknown
-}
-
-_main__fail_if_reporter_unknown() {
-  if ! array__contains "${reporter}" "simple" "dots" "junit"; then
-    system__print_line \
-      "$(_main__get_script_name): unknown reporter <${reporter}>"
-    exit ${SBU_FAILURE_STATUS_CODE}
-  fi
-}
-
-_main__print_illegal_option() {
-  local option="${1%=*}"
-  option="${option#-}"
-  option="${option#-}"
-  system__print_line "$(_main__get_script_name): illegal option -- ${option}"
-}
-
-_main__assert_only_one_argument_left() {
-  if (( $1 > 1 )); then
-    system__print_line "$(_main__get_script_name): only one path is allowed"
-    _main__print_usage_and_exit_with_code ${SBU_FAILURE_STATUS_CODE}
-  fi
-}
-
-_main__get_script_name() {
-  basename "${BASH_SOURCE[0]}"
-}
-
-_main__print_usage_and_exit_with_code() {
-  _main__print_usage
-  exit $1
-}
-
-_main__print_full_usage() {
-  _main__print_usage
-  local script="$(_main__get_script_name)"
-  system__print_new_line
-  system__print_line "\
-[options]
-  -a, --api-cheat-sheet
-    print api cheat sheet like assertions
-  -c, --colors=${SBU_YES} or ${SBU_NO}
-    tests output with colors or no
-  -d, --random-run=${SBU_YES} or ${SBU_NO}
-    tests files and functions randomly run or no
-  -f, --file-pattern=<pattern>
-    pattern to filter test files
-  -h
-    print usage
-  -o, --output-file=<file>
-    output file for JUnit reporter
-  -r, --reporters=<reporter1,reporter2>
-    comma-separated reporters (simple, dots or junit)
-  -t, --test-pattern=<pattern>
-    pattern to filter test function in files
-
-[examples]
-  ${script} .
-    run all tests in current directory
-  ${script} -p=*test.sh sources/test
-    run all tests files ending with test.sh in sources/test"
-}
-
-_main__print_usage() {
-  system__print_line "\
-usage: $(_main__get_script_name) [options] path
-       run all tests in path"
-}
-
-_main__print_api_cheat_sheet_and_exit() {
-  system__print_line "\
-[assertions]
-  assertion__equal (value, other)
-    -> assert that <value> is equal to <other>
-  assertion__different (value, other)
-    -> assert that <value> is different from <other>
-  assertion__string_contains (string, substring)
-    -> assert that <string> contains <substring>
-  assertion__string_does_not_contain (string, substring)
-    -> assert that <string> does not contain <substring>
-  assertion__string_empty (string)
-    -> assert that <string> is empty
-  assertion__string_not_empty (string)
-    -> assert that <string> is not empty
-  assertion__array_contains (element, array[0], array[1], ...)
-    -> assert that the <array> contains the <element>
-  assertion__array_does_not_contain (element, array elements...)
-    -> assert that the <array> does not contain the <element>
-  assertion__successful (command)
-    -> assert that the <command> is successful
-  assertion__failing (command)
-    -> assert that the <command> is failing
-  assertion__status_code_is_success (code)
-    -> assert that the status <code> is 0
-  assertion__status_code_is_failure (code)
-    -> assert that the status <code> is not 0
-
-[special functions]
-  ${SBU_GLOBAL_SETUP_FUNCTION_NAME}
-    -> Executed before all tests in a file
-  ${SBU_GLOBAL_TEARDOWN_FUNCTION_NAME}
-    -> Executed after all tests in a file
-  ${SBU_SETUP_FUNCTION_NAME}
-    -> Executed before each test in a file
-  ${SBU_TEARDOWN_FUNCTION_NAME}
-    -> Executed after each test in a file
-
-[mocks]
-  mock__make_function_do_nothing (function_to_mock)
-    -> make function do nothing
-  mock__make_function_prints (function_to_mock, message)
-    -> make function prints a message
-  mock__make_function_call (function_to_mock, function_to_call)
-    -> make function call another function"
-  exit ${SBU_SUCCESS_STATUS_CODE}
-}
-
-
-main__main "$@"

From 76c42b4d3f61e80333c2237d09c09b56d0fdad70 Mon Sep 17 00:00:00 2001
From: Max Gautier <mg@max.gautier.name>
Date: Wed, 18 Sep 2024 14:04:50 +0200
Subject: [PATCH 16/19] CI: cleanup '-scale' tests infra (#11535)

There is actually no test using this since ad6fecefa879515578f3f7810ce6ebac14ac1d3d,
so there is no reason to keep that infra in our tests scripts.
---
 docs/developers/test_cases.md                              | 7 +------
 .../cloud_playbooks/roles/packet-ci/templates/inventory.j2 | 6 ++----
 tests/cloud_playbooks/roles/packet-ci/vars/main.yml        | 2 --
 tests/scripts/testcases_run.sh                             | 6 +++---
 tests/templates/fake_hosts.yml.j2                          | 3 ---
 5 files changed, 6 insertions(+), 18 deletions(-)
 delete mode 100644 tests/templates/fake_hosts.yml.j2

diff --git a/docs/developers/test_cases.md b/docs/developers/test_cases.md
index 1937e7ebe29..96d72ea3ba0 100644
--- a/docs/developers/test_cases.md
+++ b/docs/developers/test_cases.md
@@ -1,6 +1,6 @@
 # Node Layouts
 
-There are six node layout types: `default`, `separate`, `ha`, `scale`, `all-in-one`, and `node-etcd-client`.
+There are five node layout types: `default`, `separate`, `ha`, `all-in-one`, and `node-etcd-client`.
 
 `default` is a non-HA two nodes setup with one separate `kube_node`
 and the `etcd` group merged with the `kube_control_plane`.
@@ -11,11 +11,6 @@ and the `etcd` group merged with the `kube_control_plane`.
 `ha` layout consists of two etcd nodes, two control planes and a single worker node,
 with role intersection.
 
-`scale` layout can be combined with above layouts (`ha-scale`, `separate-scale`). It includes 200 fake hosts
-in the Ansible inventory. This helps test TLS certificate generation at scale
-to prevent regressions and profile certain long-running tasks. These nodes are
-never actually deployed, but certificates are generated for them.
-
 `all-in-one` layout use a single node for with `kube_control_plane`, `etcd` and `kube_node` merged.
 
 `node-etcd-client` layout consists of a 4 nodes cluster, all of them in `kube_node`, first 3 in `etcd` and only one `kube_control_plane`.
diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2
index cb697762944..e7adafccce9 100644
--- a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2
+++ b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2
@@ -3,7 +3,7 @@
 instance-{{ loop.index }} ansible_host={{instance.stdout}}
 {% endfor %}
 
-{% if mode is defined and mode in ["separate", "separate-scale"] %}
+{% if mode == "separate" %}
 [kube_control_plane]
 instance-1
 
@@ -12,7 +12,7 @@ instance-2
 
 [etcd]
 instance-3
-{% elif mode is defined and mode in ["ha", "ha-scale"] %}
+{% elif mode == "ha" %}
 [kube_control_plane]
 instance-1
 instance-2
@@ -103,5 +103,3 @@ kube_control_plane
 calico_rr
 
 [calico_rr]
-
-[fake_hosts]
diff --git a/tests/cloud_playbooks/roles/packet-ci/vars/main.yml b/tests/cloud_playbooks/roles/packet-ci/vars/main.yml
index 81aa17d9fc4..ba448587d48 100644
--- a/tests/cloud_playbooks/roles/packet-ci/vars/main.yml
+++ b/tests/cloud_playbooks/roles/packet-ci/vars/main.yml
@@ -1,9 +1,7 @@
 ---
 _vm_count_dict:
   separate: 3
-  separate-scale: 3
   ha: 3
-  ha-scale: 3
   ha-recover: 3
   ha-recover-noquorum: 3
   all-in-one: 1
diff --git a/tests/scripts/testcases_run.sh b/tests/scripts/testcases_run.sh
index 44ef4f04cc7..7559c4fb2b2 100755
--- a/tests/scripts/testcases_run.sh
+++ b/tests/scripts/testcases_run.sh
@@ -54,7 +54,7 @@ run_playbook () {
 playbook=$1
 shift
 # We can set --limit here and still pass it as supplemental args because `--limit`  is a 'last one wins' option
-ansible-playbook --limit "all:!fake_hosts" \
+ansible-playbook \
      $ANSIBLE_LOG_LEVEL \
     -e @${CI_TEST_SETTING} \
     -e @${CI_TEST_REGISTRY_MIRROR} \
@@ -85,8 +85,8 @@ fi
 
 # Test control plane recovery
 if [ "${RECOVER_CONTROL_PLANE_TEST}" != "false" ]; then
-    run_playbook reset.yml --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes
-    run_playbook recover-control-plane.yml -e etcd_retries=10 --limit "etcd:kube_control_plane:!fake_hosts"
+    run_playbook reset.yml --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}" -e reset_confirmation=yes
+    run_playbook recover-control-plane.yml -e etcd_retries=10 --limit "etcd:kube_control_plane"
 fi
 
 # Test collection build and install by installing our collection, emptying our repository, adding
diff --git a/tests/templates/fake_hosts.yml.j2 b/tests/templates/fake_hosts.yml.j2
deleted file mode 100644
index c172b78b070..00000000000
--- a/tests/templates/fake_hosts.yml.j2
+++ /dev/null
@@ -1,3 +0,0 @@
-ansible_default_ipv4:
-  address: 255.255.255.255
-ansible_hostname: "{{ '{{' }}inventory_hostname }}"

From 4323e5d039631558a25dd58069168f28b4a855bb Mon Sep 17 00:00:00 2001
From: Leeon Fu <48649837+leeonfu@users.noreply.github.com>
Date: Fri, 20 Sep 2024 02:14:37 +0800
Subject: [PATCH 17/19] Reset operation, disable service enabled and clean
 container's logs. (#11501)

---
 roles/reset/tasks/main.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml
index acafb5a760e..97bdbf5974a 100644
--- a/roles/reset/tasks/main.yml
+++ b/roles/reset/tasks/main.yml
@@ -3,6 +3,7 @@
   service:
     name: "{{ item }}"
     state: stopped
+    enabled: false
   with_items:
     - kubelet.service
     - cri-dockerd.service
@@ -290,6 +291,7 @@
     - /var/log/openvswitch
     - /var/log/ovn
     - /var/log/kube-ovn
+    - /var/log/containers
     - /etc/cni
     - /etc/nerdctl
     - "{{ nginx_config_dir }}"

From 1fa4bb733d6ce277d28fa680d8dcced8e38ec919 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 20 Sep 2024 09:33:39 +0100
Subject: [PATCH 18/19] Bump ansible from 9.8.0 to 9.10.0 (#11540)

Bumps [ansible](https://github.com/ansible-community/ansible-build-data) from 9.8.0 to 9.10.0.
- [Changelog](https://github.com/ansible-community/ansible-build-data/blob/main/docs/release-process.md)
- [Commits](https://github.com/ansible-community/ansible-build-data/compare/9.8.0...9.10.0)

---
updated-dependencies:
- dependency-name: ansible
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 93d2da5c582..5f7b3a5dcbf 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-ansible==9.8.0
+ansible==9.10.0
 # Needed for jinja2 json_query templating
 jmespath==1.0.1
 # Needed for ansible.utils.validate module

From 89ff0710e953b8330fc4ee1c837988cf6303a926 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 20 Sep 2024 10:45:58 +0100
Subject: [PATCH 19/19] Bump ansible-lint from 24.7.0 to 24.9.0 (#11541)

Bumps [ansible-lint](https://github.com/ansible/ansible-lint) from 24.7.0 to 24.9.0.
- [Release notes](https://github.com/ansible/ansible-lint/releases)
- [Commits](https://github.com/ansible/ansible-lint/compare/v24.7.0...v24.9.0)

---
updated-dependencies:
- dependency-name: ansible-lint
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 tests/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/requirements.txt b/tests/requirements.txt
index dce3ea67d5d..7ab3de5fb3f 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,5 +1,5 @@
 -r ../requirements.txt
-ansible-lint==24.7.0
+ansible-lint==24.9.0
 apache-libcloud==3.8.0
 ara[server]==1.7.2
 dopy==0.3.7