From ff8ff8a7217697481ad60303f4f6025f391004ff Mon Sep 17 00:00:00 2001 From: Tiago Castro Date: Fri, 6 Sep 2024 21:39:18 +0100 Subject: [PATCH 1/2] refactor(terraform): split master and worker Split master and worker name prefix. Parse master and worker cpu resources for lxd. Updates lxd to >=2.0.0. Signed-off-by: Tiago Castro --- terraform/cluster/main.tf | 14 ++++-- terraform/cluster/mod/k8s/kubeadm_config.yaml | 2 + terraform/cluster/mod/k8s/main.tf | 3 +- terraform/cluster/mod/libvirt/main.tf | 24 ++++++---- terraform/cluster/mod/lxd/main.tf | 46 ++++++++++++------- terraform/cluster/variables.tf | 35 ++++++++++---- 6 files changed, 85 insertions(+), 39 deletions(-) diff --git a/terraform/cluster/main.tf b/terraform/cluster/main.tf index 509a5969c..e8f9e818e 100644 --- a/terraform/cluster/main.tf +++ b/terraform/cluster/main.tf @@ -1,7 +1,8 @@ module "k8s" { source = "./mod/k8s" - num_nodes = var.num_nodes + num_nodes = local.num_nodes + master_nodes = local.master_nodes ssh_user = local.ssh_user private_key_path = local.ssh_key_priv node_list = module.provider.node_list @@ -21,12 +22,17 @@ module "provider" { # lxd and libvirt ssh_user = local.ssh_user ssh_key = local.ssh_key_pub - num_nodes = var.num_nodes + master_nodes = local.master_nodes + worker_nodes = var.worker_nodes + num_nodes = local.num_nodes worker_memory = var.worker_memory worker_vcpu = var.worker_vcpu master_memory = var.master_memory master_vcpu = var.master_vcpu - hostname_formatter = var.hostname_formatter + master_fmt = format("%s-%%d", var.master_prefix) + worker_fmt = format("%s-%%d", var.worker_prefix) + lxc_cached_image = var.lxc_cached_image + lxc_image = var.lxc_image # libvirt image_path = var.image_path @@ -47,6 +53,8 @@ locals { ssh_key_priv = var.ssh_key_priv == "" ? pathexpand("~/.ssh/id_rsa") : var.ssh_key_priv ssh_user = var.ssh_user == "" ? data.local_file.current_username.content : var.ssh_user qcow2_image = var.qcow2_image == "" ? pathexpand("~/terraform_images/ubuntu-20.04-server-cloudimg-amd64.img") : pathexpand(var.qcow2_image) + master_nodes = 1 + num_nodes = var.worker_nodes + local.master_nodes } resource "null_resource" "generate_current_username" { diff --git a/terraform/cluster/mod/k8s/kubeadm_config.yaml b/terraform/cluster/mod/k8s/kubeadm_config.yaml index c7c704f2d..4702db83b 100644 --- a/terraform/cluster/mod/k8s/kubeadm_config.yaml +++ b/terraform/cluster/mod/k8s/kubeadm_config.yaml @@ -38,3 +38,5 @@ apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration cgroupDriver: "systemd" failSwapOn: false +#cpuManagerPolicy: "static" +#reservedSystemCPUs: "1" diff --git a/terraform/cluster/mod/k8s/main.tf b/terraform/cluster/mod/k8s/main.tf index eb335516d..87c9424c8 100644 --- a/terraform/cluster/mod/k8s/main.tf +++ b/terraform/cluster/mod/k8s/main.tf @@ -4,6 +4,7 @@ variable "k8s_cluster_token" { variable "overlay_cidr" {} variable "num_nodes" {} +variable "master_nodes" {} variable "ssh_user" {} variable "private_key_path" {} @@ -38,7 +39,7 @@ resource "null_resource" "k8s" { provisioner "remote-exec" { inline = [ - count.index == 0 ? local.master : local.node + count.index < var.master_nodes ? local.master : local.node ] } } diff --git a/terraform/cluster/mod/libvirt/main.tf b/terraform/cluster/mod/libvirt/main.tf index d3fafc72c..3743b04c5 100644 --- a/terraform/cluster/mod/libvirt/main.tf +++ b/terraform/cluster/mod/libvirt/main.tf @@ -2,11 +2,14 @@ variable "image_path" {} variable "num_nodes" {} +variable "master_nodes" {} +variable "worker_nodes" {} variable "worker_memory" {} variable "worker_vcpu" {} variable "master_memory" {} variable "master_vcpu" {} -variable "hostname_formatter" {} +variable "master_fmt" {} +variable "worker_fmt" {} variable "ssh_user" {} variable "ssh_key" {} variable "private_key_path" {} @@ -16,6 +19,9 @@ variable "qcow2_image" {} variable "network_mode" {} variable "bridge_name" {} +variable "lxc_image" {} +variable "lxc_cached_image" {} + provider "libvirt" { uri = "qemu:///system" } @@ -37,7 +43,7 @@ resource "libvirt_volume" "ubuntu-qcow2" { # we want to, based of the first image, create 3 separate images each with their own cloud-init settings resource "libvirt_volume" "ubuntu-qcow2-resized" { - name = format(var.hostname_formatter, count.index + 1) + name = count.index < var.master_nodes ? format(var.master_fmt, count.index + 1) : format(var.worker_fmt, count.index + 1 - var.master_nodes) count = var.num_nodes base_volume_id = libvirt_volume.ubuntu-qcow2.id pool = libvirt_pool.ubuntu-pool.name @@ -50,7 +56,7 @@ resource "libvirt_volume" "pool-disk" { count = var.num_nodes pool = libvirt_pool.ubuntu-pool.name format = "raw" - size = var.pooldisk_size + size = count.index < var.master_nodes ? 0 : var.pooldisk_size } locals { @@ -58,7 +64,7 @@ locals { # passes them to a template file to be filled in user_data = [ for node_index in range(var.num_nodes) : templatefile("${path.module}/cloud_init.tmpl", { - ssh_user = var.ssh_user, ssh_key = var.ssh_key, hostname = format(var.hostname_formatter, node_index + 1) + ssh_user = var.ssh_user, ssh_key = var.ssh_key, hostname = node_index < var.master_nodes ? format(var.master_fmt, node_index + 1) : format(var.worker_fmt, node_index + 1 - var.master_nodes) }) ] # likewise for networking @@ -123,9 +129,9 @@ resource "libvirt_cloudinit_disk" "commoninit" { # create the actual VMs for the cluster resource "libvirt_domain" "ubuntu-domain" { count = var.num_nodes - name = format(var.hostname_formatter, count.index + 1) - memory = count.index == 0 ? var.master_memory : var.worker_memory - vcpu = count.index == 0 ? var.master_vcpu : var.worker_vcpu + name = count.index < var.master_nodes ? format(var.master_fmt, count.index + 1) : format(var.worker_fmt, count.index + 1 - var.master_nodes) + memory = count.index < var.master_nodes ? var.master_memory : var.worker_memory + vcpu = count.index < var.master_nodes ? var.master_vcpu : var.worker_vcpu autostart = true cloudinit = libvirt_cloudinit_disk.commoninit[count.index].id @@ -152,7 +158,7 @@ resource "libvirt_domain" "ubuntu-domain" { network_interface { network_name = var.network_mode == "default" ? "default" : "k8snet" - hostname = format(var.hostname_formatter, count.index + 1) + hostname = count.index < var.master_nodes ? format(var.master_fmt, count.index + 1) : format(var.worker_fmt, count.index + 1 - var.master_nodes) wait_for_lease = true } @@ -185,7 +191,7 @@ output "ks-cluster-nodes" { ${libvirt_domain.ubuntu-domain.0.name} ansible_host=${libvirt_domain.ubuntu-domain.0.network_interface.0.addresses.0} ansible_user=${var.ssh_user} ansible_ssh_private_key_file=${var.private_key_path} ansible_ssh_common_args='-o StrictHostKeyChecking=no' [nodes]%{for ip in libvirt_domain.ubuntu-domain.*~} -%{if ip.name != "${format(var.hostname_formatter, 1)}"}${ip.name} ansible_host=${ip.network_interface.0.addresses.0} ansible_user=${var.ssh_user} ansible_ssh_private_key_file=${var.private_key_path} ansible_ssh_common_args='-o StrictHostKeyChecking=no'%{endif} +%{if ip.name != "${format(var.master_fmt, 1)}"}${ip.name} ansible_host=${ip.network_interface.0.addresses.0} ansible_user=${var.ssh_user} ansible_ssh_private_key_file=${var.private_key_path} ansible_ssh_common_args='-o StrictHostKeyChecking=no'%{endif} %{endfor~} EOT } diff --git a/terraform/cluster/mod/lxd/main.tf b/terraform/cluster/mod/lxd/main.tf index 78b049a0e..187287c7e 100644 --- a/terraform/cluster/mod/lxd/main.tf +++ b/terraform/cluster/mod/lxd/main.tf @@ -4,13 +4,16 @@ provider "lxd" { } variable "num_nodes" {} +variable "master_nodes" {} +variable "worker_nodes" {} variable "worker_memory" {} variable "worker_vcpu" {} variable "master_memory" {} variable "master_vcpu" {} variable "ssh_user" {} variable "ssh_key" {} -variable "hostname_formatter" {} +variable "master_fmt" {} +variable "worker_fmt" {} variable "private_key_path" {} variable "pooldisk_size" {} @@ -20,9 +23,13 @@ variable "image_path" {} variable "disk_size" {} variable "qcow2_image" {} +variable "lxc_image" {} +variable "lxc_cached_image" {} + resource "lxd_cached_image" "ubuntu" { - source_remote = "ubuntu" - source_image = "jammy/amd64" + source_remote = split(":", var.lxc_image)[0] + source_image = split(":", var.lxc_image)[1] + count = var.lxc_cached_image ? 1 : 0 } locals { @@ -30,11 +37,12 @@ locals { # passes them to a template file to be filled in user_data = [ for node_index in range(var.num_nodes) : templatefile("${path.module}/cloud_init.tmpl", { - ssh_user = var.ssh_user, ssh_key = var.ssh_key, hostname = format(var.hostname_formatter, node_index + 1) + ssh_user = var.ssh_user, ssh_key = var.ssh_key, hostname = node_index < var.master_nodes ? format(var.master_fmt, node_index + 1) : format(var.worker_fmt, node_index + 1 - var.master_nodes) }) ] # likewise for networking network_config = templatefile("${path.module}/network_config.cfg", {}) + use_ipv4 = true } resource "null_resource" "lxd_init" { @@ -44,21 +52,26 @@ resource "null_resource" "lxd_init" { } resource "null_resource" "lxd_stop_force" { + triggers = { + master_nodes = var.master_nodes + master_fmt = var.master_fmt + worker_fmt = var.worker_fmt + } provisioner "local-exec" { when = destroy # todo: should use hostname_formatter - command = format("lxc stop ksnode-%d --force", count.index + 1) + command = format("lxc stop %s --force", count.index < self.triggers.master_nodes ? format(self.triggers.master_fmt, count.index + 1) : format(self.triggers.worker_fmt, count.index + 1 - self.triggers.master_nodes)) } count = var.num_nodes depends_on = [ - lxd_container.c8s + lxd_instance.c8s ] } -resource "lxd_container" "c8s" { +resource "lxd_instance" "c8s" { count = var.num_nodes - name = format(var.hostname_formatter, count.index + 1) - image = lxd_cached_image.ubuntu.fingerprint + name = count.index < var.master_nodes ? format(var.master_fmt, count.index + 1) : format(var.worker_fmt, count.index + 1 - var.master_nodes) + image = var.lxc_cached_image ? lxd_cached_image.ubuntu[0].fingerprint : var.lxc_image ephemeral = false # be careful with raw.lxc it has to be key=value\nkey=value @@ -74,9 +87,9 @@ resource "lxd_container" "c8s" { } limits = { - memory = format("%dMiB", count.index == 0 ? var.master_memory : var.worker_memory) + memory = format("%dMiB", count.index < var.master_nodes ? var.master_memory : var.worker_memory) # For the moment this doesn't as io-engine then can't set its core affinity... - # cpu = count.index == 0 ? var.master_vcpu : var.worker_vcpu + # cpu = count.index < var.master_nodes ? var.master_vcpu : var.worker_vcpu } device { @@ -93,7 +106,7 @@ resource "lxd_container" "c8s" { connection { type = "ssh" user = var.ssh_user - host = self.ip_address + host = local.use_ipv4 ? self.ipv4_address : self.ipv6_address private_key = file(var.private_key_path) } } @@ -106,22 +119,23 @@ resource "lxd_container" "c8s" { output "ks-cluster-nodes" { value = < Date: Thu, 12 Sep 2024 14:34:16 +0100 Subject: [PATCH 2/2] build: allow fmt on openapi prior to build Adds some indirection between lib.rs and the actual lib.rs Signed-off-by: Tiago Castro --- .gitignore | 1 + nix/pkgs/control-plane/cargo-project.nix | 1 + openapi/build.rs | 1 + openapi/src/lib.rs | 1 + scripts/rust/generate-openapi-bindings.sh | 5 +++++ 5 files changed, 9 insertions(+) create mode 100644 openapi/src/lib.rs diff --git a/.gitignore b/.gitignore index 7e04cc0c4..84f75f546 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ __pycache__ /openapi/* !/openapi/Cargo.toml !/openapi/build.rs +!/openapi/src/lib.rs /tests/bdd/autogen/ /terraform/cluster/ansible-hosts /terraform/cluster/current_user.txt diff --git a/nix/pkgs/control-plane/cargo-project.nix b/nix/pkgs/control-plane/cargo-project.nix index 3185f2743..492b94d4b 100644 --- a/nix/pkgs/control-plane/cargo-project.nix +++ b/nix/pkgs/control-plane/cargo-project.nix @@ -54,6 +54,7 @@ let "k8s" "openapi/Cargo.toml" "openapi/build.rs" + "openapi/src/lib.rs" "rpc" "scripts/rust/generate-openapi-bindings.sh" "scripts/rust/branch_ancestor.sh" diff --git a/openapi/build.rs b/openapi/build.rs index 25dcab748..da60fe13a 100644 --- a/openapi/build.rs +++ b/openapi/build.rs @@ -19,5 +19,6 @@ fn main() { // directive against files created during the build of build.rs?? // https://doc.rust-lang.org/cargo/reference/build-scripts.html#rerun-if-changed // println!("cargo:rerun-if-changed=."); + // println!("cargo:rerun-if-changed=./src/lib_.rs"); // println!("cargo:rerun-if-changed=version.txt"); } diff --git a/openapi/src/lib.rs b/openapi/src/lib.rs new file mode 100644 index 000000000..08ca4c0e8 --- /dev/null +++ b/openapi/src/lib.rs @@ -0,0 +1 @@ +include!("lib_.rs"); diff --git a/scripts/rust/generate-openapi-bindings.sh b/scripts/rust/generate-openapi-bindings.sh index 504aadb4d..2935b9947 100755 --- a/scripts/rust/generate-openapi-bindings.sh +++ b/scripts/rust/generate-openapi-bindings.sh @@ -140,6 +140,11 @@ fi # Cleanup the existing autogenerated code ( cd "$ROOTDIR"; git clean -f -e "!version.txt" -X "$REAL_TARGET" || true ) +( cd "$tmpd"; mv src/lib.rs src/lib_.rs; ) +if [ ! -f "$TARGET/src/lib.rs" ]; then + git restore "$TARGET/src/lib.rs" +fi +mv "$TARGET/src/lib.rs" "$tmpd/src/lib.rs" mv "$tmpd"/* "$TARGET"/ rm -rf "$tmpd"