diff --git a/README.md b/README.md index 4efb77e..519466b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Description -Provision VMs (and their resources, included networks) on a bare metal Linux, using Terraform on top of opensource tools +Provision VMs (and their resources, included networks) on a bare metal Linux, using Terraform on top of opensource tools such as Libvirt, QEMU and KVM. ## Requirements @@ -14,56 +14,66 @@ such as Libvirt, QEMU and KVM. > All the following commands are executed from the root path of the repository -1. Declare environment vars with the SSH connection parameters. -These can be declared as input vars inside a `.tfvars` file too - -```bash -export TF_VAR_SSH_HOST="XXX.XXX.XXX.XXX" -export TF_VAR_SSH_USERNAME="yourUsername" -export TF_VAR_SSH_PASSWORD="yourPassword" -export TF_VAR_SSH_PRIVATE_KEY_PATH="~/.ssh/id_ed25519" -``` - -2. Execute some REQUIRED previous scripts to bootstrap the host - -> By the moment, only recent Ubuntu versions are supported. -> Feel free to extend the OS support pushing your code to this repository. - -```bash -# Copy current SSH key into the target host -echo ${TF_VAR_SSH_PASSWORD} | ssh-copy-id -f ${TF_VAR_SSH_USERNAME}@${TF_VAR_SSH_HOST} - -# Give execution permissions to the helper scripts -chmod -R +x ./scripts - -# Connect to the host machine by SSH and install the dependencies using passwordless authentication -# (user with sudo privileges required) -ssh -i ${TF_VAR_SSH_PRIVATE_KEY_PATH} ${TF_VAR_SSH_USERNAME}@${TF_VAR_SSH_HOST} \ - "sudo bash ./scripts/prepare-host-ubuntu.sh ${TF_VAR_SSH_USERNAME}" -``` - -3. Configure Terraform to store the state of your infrastructure - -> We have configured S3 backend by default thinking on bare metal solutions like S3-compatible APIs provided by -> solutions like TrueNAS, which cover the problem of storing the .tfstate using the same approach as major cloud providers - -```bash -# Set the right parameters for your S3 storage -nano backends/config.s3.tfbackend - -# Init the process configuring your backend -terraform init -backend-config=backends/config.s3.tfbackend -``` - -4. Declare resources related to VMs to create. - -> Change the file called `data.tf`. -> Several examples can be found inside (yes, for Kubernetes) - -5. Create your VMs. -```bash -terraform apply -``` +1. Declare environment vars with the SSH connection parameters. + These can be declared as input vars inside a `.tfvars` file too + + ```bash + export TF_VAR_SSH_HOST="XXX.XXX.XXX.XXX" + export TF_VAR_SSH_USERNAME="yourUsername" + export TF_VAR_SSH_PASSWORD="yourPassword" + export TF_VAR_SSH_PRIVATE_KEY_PATH="~/.ssh/id_ed25519" + ``` + +2. Install some REQUIRED dependencies in local machine + + > To build the ISOs we need to have installed `mkisofs`. + + ```bash + sudo apt install mkisofs + ``` + +3. Execute some REQUIRED previous scripts to bootstrap the host + + > By the moment, only recent Ubuntu versions are supported. + > Feel free to extend the OS support by pushing your code to this repository. + + ```bash + # Copy current SSH key into the target host + echo ${TF_VAR_SSH_PASSWORD} | ssh-copy-id -f ${TF_VAR_SSH_USERNAME}@${TF_VAR_SSH_HOST} + + # Give execution permissions to the helper scripts + chmod -R +x ./scripts + + # Connect to the host machine by SSH and install the dependencies using passwordless authentication + # (user with sudo privileges required) + scp ./scripts/prepare-host-ubuntu.sh ${TF_VAR_SSH_USERNAME}@${TF_VAR_SSH_HOST}:/tmp + ssh ${TF_VAR_SSH_USERNAME}@${TF_VAR_SSH_HOST} "sudo bash ./tmp/prepare-host-ubuntu.sh ${TF_VAR_SSH_USERNAME}" + ``` + +4. Configure Terraform to store the state of your infrastructure + + > We have configured S3 backend by default thinking on bare metal solutions like S3-compatible APIs provided by + > solutions like TrueNAS, which cover the problem of storing the .tfstate using the same approach as major cloud + > providers + + ```bash + # Set the right parameters for your S3 storage + nano backends/config.s3.tfbackend + + # Init the process configuring your backend + terraform init -backend-config=backends/config.s3.tfbackend + ``` + +5. Declare resources related to VMs to create. + + > Change the file called `data.tf`. + > Several examples can be found inside (yes, for Kubernetes) + +6. Create your VMs. + + ```bash + terraform apply + ``` ## Security considerations @@ -72,11 +82,11 @@ This means that each instance has a different password and a different authorize They are stored in the `tfstate` so execute a `terraform state list` and then show the resource you need by using `terraform state show ยทยทยท` -When the `terraform apply` is complete, all the SSH private key files are exported in order +When the `terraform apply` is complete, all the SSH private key files are exported in order to allow you to access or manage them. -There is a special directory located in `files/input/external-ssh-keys`. -This was created for the special case that several well-known SSH keys must be authorized +There is a special directory located in `files/input/external-ssh-keys`. +This was created for the special case that several well-known SSH keys must be authorized in all the instances at the same time. This can be risky and must be used under your own responsibility. If you need it, place some `.pub` key files inside, and they will be directly configured and authorized in all the instances. diff --git a/scripts/prepare-host-ubuntu.sh b/scripts/prepare-host-ubuntu.sh index 1e846d5..b6ed506 100644 --- a/scripts/prepare-host-ubuntu.sh +++ b/scripts/prepare-host-ubuntu.sh @@ -129,8 +129,7 @@ function disable_qemu_security_driver () { esac } -# Disable security_driver parameter for Qemu -# Ref: https://github.com/dmacvicar/terraform-provider-libvirt/issues/546 +# Restart libvirt function restart_libvirt () { EXIT_CODE=0 diff --git a/terraform/data.tf b/terraform/data.tf index eafac95..9cc4043 100644 --- a/terraform/data.tf +++ b/terraform/data.tf @@ -62,14 +62,14 @@ locals { # Define the LoadBalancer kube-loadbalancer-0 = { - vcpu = 2 - memory = 2048 # 2GB - disk = 10000000000 # 10GB + vcpu = 2 + memory = 2048 # 2GB + disk = 10000000000 # 10GB networks = [ { name = "virnat0" address = "10.10.10.10/24" - },{ + }, { name = "external0" address = "192.168.0.210/24" } @@ -78,9 +78,9 @@ locals { # Define the masters kube-master-0 = { - vcpu = 2 - memory = 2048 # 2GB - disk = 10000000000 # 10GB + vcpu = 2 + memory = 2048 # 2GB + disk = 10000000000 # 10GB networks = [ { name = "virnat0" @@ -90,9 +90,9 @@ locals { } kube-master-1 = { - vcpu = 2 - memory = 2048 # 2GB - disk = 10000000000 # 10GB + vcpu = 2 + memory = 2048 # 2GB + disk = 10000000000 # 10GB networks = [ { name = "virnat0" @@ -102,9 +102,9 @@ locals { } kube-master-2 = { - vcpu = 2 - memory = 2048 # 2GB - disk = 10000000000 # 10GB + vcpu = 2 + memory = 2048 # 2GB + disk = 10000000000 # 10GB networks = [ { name = "virnat0" @@ -115,9 +115,9 @@ locals { # Define the workers kube-worker-0 = { - vcpu = 2 - memory = 2048 # 2GB - disk = 20000000000 # 20GB + vcpu = 2 + memory = 2048 # 2GB + disk = 20000000000 # 20GB networks = [ { name = "virnat0" diff --git a/terraform/main.tf b/terraform/main.tf index f576a73..05c2e8b 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -4,11 +4,11 @@ module "workload" { # Variables definition ssh_connection = { - host = var.SSH_HOST - username = var.SSH_USERNAME + host = var.SSH_HOST + username = var.SSH_USERNAME private_key_path = var.SSH_PRIVATE_KEY_PATH } - networks = local.networks + networks = local.networks instances = local.instances } diff --git a/terraform/modules/workload/instances.tf b/terraform/modules/workload/instances.tf index dea6ce3..8766f31 100644 --- a/terraform/modules/workload/instances.tf +++ b/terraform/modules/workload/instances.tf @@ -4,33 +4,33 @@ locals { # This makes a trustable list by looking for each defined network in the networks definition instance_networks_expanded = { for vm_name, vm_data in var.instances : - vm_name => [ - for _, vm_network in vm_data.networks : - merge( - { network_attachment = vm_network }, - { network_info = merge( - var.networks[vm_network.name], - { name = vm_network.name } - ) - }, + vm_name => [ + for _, vm_network in vm_data.networks : + merge( + { network_attachment = vm_network }, + { network_info = merge( + var.networks[vm_network.name], + { name = vm_network.name } ) - if length(try(var.networks[vm_network.name], {})) > 0 - ] + }, + ) + if length(try(var.networks[vm_network.name], {})) > 0 + ] } # Group instance's networks by type for easier attachments later instance_networks_grouped = { - for vm_name, vm_networks in local.instance_networks_expanded: - vm_name => { - nat = distinct([ - for _, vm_network in vm_networks: - vm_network if vm_network.network_info.mode == "nat" - ]) - macvtap = distinct([ - for _, vm_network in vm_networks: - vm_network if vm_network.network_info.mode == "macvtap" - ]) - } + for vm_name, vm_networks in local.instance_networks_expanded : + vm_name => { + nat = distinct([ + for _, vm_network in vm_networks : + vm_network if vm_network.network_info.mode == "nat" + ]) + macvtap = distinct([ + for _, vm_network in vm_networks : + vm_network if vm_network.network_info.mode == "macvtap" + ]) + } } } # Create all instances @@ -54,11 +54,11 @@ resource "libvirt_domain" "instance" { iterator = network content { network_id = libvirt_network.nat[network.value["network_attachment"]["name"]].id - hostname = each.key + hostname = each.key # Guest VM's virtualized network interface will claim the requested IP to the virtual NAT on the Host # On the system level, the interface in Linux is configured in DHCP mode by using cloud-init # WARNING: Addresses not in CIDR notation here - addresses = [split("/", network.value["network_attachment"]["address"])[0]] + addresses = [split("/", network.value["network_attachment"]["address"])[0]] wait_for_lease = true } } @@ -69,7 +69,7 @@ resource "libvirt_domain" "instance" { iterator = network content { - macvtap = network.value["network_info"]["interface"] + macvtap = network.value["network_info"]["interface"] hostname = each.key # Guest virtualized network interface is connected directly to a physical device on the Host, # As a result, requested IP address can only be claimed by the OS: Linux is configured in static mode by cloud-init @@ -102,5 +102,5 @@ resource "libvirt_domain" "instance" { } qemu_agent = true - autostart = true + autostart = true } diff --git a/terraform/modules/workload/networks.tf b/terraform/modules/workload/networks.tf index 4a1c307..dd89e1e 100644 --- a/terraform/modules/workload/networks.tf +++ b/terraform/modules/workload/networks.tf @@ -10,8 +10,8 @@ locals { resource "libvirt_network" "nat" { for_each = local.networks_nat - name = each.key - mode = "nat" + name = each.key + mode = "nat" bridge = each.key domain = join(".", [each.key, "local"]) @@ -20,7 +20,7 @@ resource "libvirt_network" "nat" { dhcp { enabled = true } dns { - enabled = true + enabled = true local_only = false } } diff --git a/terraform/modules/workload/outputs.tf b/terraform/modules/workload/outputs.tf index 309f53e..88b02cb 100644 --- a/terraform/modules/workload/outputs.tf +++ b/terraform/modules/workload/outputs.tf @@ -1,16 +1,16 @@ locals { # Prepare relevant information about recently modified instances instances_information = { - for instance, v in var.instances: - instance => merge(v, { - hostname = instance - user = "ubuntu" - password = random_string.instance_password[instance].result - ssh-keys = concat( - [tls_private_key.instance_ssh_key[instance].public_key_openssh], - local.instances_external_ssh_keys - ) - }) + for instance, v in var.instances : + instance => merge(v, { + hostname = instance + user = "ubuntu" + password = random_string.instance_password[instance].result + ssh-keys = concat( + [tls_private_key.instance_ssh_key[instance].public_key_openssh], + local.instances_external_ssh_keys + ) + }) } # Encode output in YAML and replace all the strange symbols on the keys @@ -23,17 +23,17 @@ locals { # Outputs all relevant information to connect to the instances output "instances_information" { sensitive = true - value = local.instances_information + value = local.instances_information } # Outputs instances' networks complete information output "instance_networks_expanded" { sensitive = true - value = local.instance_networks_expanded + value = local.instance_networks_expanded } # Output instance networks grouped by type output "instance_networks_grouped" { sensitive = true - value = local.instance_networks_grouped + value = local.instance_networks_grouped } \ No newline at end of file diff --git a/terraform/modules/workload/storage.tf b/terraform/modules/workload/storage.tf index 1d713df..74cf58b 100644 --- a/terraform/modules/workload/storage.tf +++ b/terraform/modules/workload/storage.tf @@ -1,15 +1,15 @@ # Create a dir where all the volumes will be created resource "libvirt_pool" "volume_pool" { - name = "volume_pool" + name = "metal-cloud-volume-pool" type = "dir" - path = "/home/${var.ssh_connection.username}/volume_pool" + path = "/opt/libvirt/metal-cloud-volume-pool" } # Fetch the latest ubuntu release image from their mirrors # DISCLAIMER: Using Ubuntu/Debian because the author's obsession resource "libvirt_volume" "os_image" { - name = "ubuntu-hirsute.qcow2" - source = "https://cloud-images.ubuntu.com/releases/hirsute/release/ubuntu-21.04-server-cloudimg-amd64.img" + name = "ubuntu-22.04.qcow2" + source = "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img" pool = libvirt_pool.volume_pool.name } @@ -49,30 +49,30 @@ locals { # List of SSH keys allowed on instances instances_external_ssh_keys = [ for i, v in fileset("${path.root}/files/input/external-ssh-keys", "*.pub") : - trimspace(file("${path.root}/files/input/external-ssh-keys/${v}")) + trimspace(file("${path.root}/files/input/external-ssh-keys/${v}")) ] # Parsed user-data config file for Cloud Init user_data = { for instance, _ in var.instances : - instance => templatefile("${path.module}/templates/cloud-init/user_data.cfg", { - hostname = instance - user = "ubuntu" - password = random_string.instance_password[instance].result - ssh-keys = concat( - [tls_private_key.instance_ssh_key[instance].public_key_openssh], - local.instances_external_ssh_keys - ) - }) + instance => templatefile("${path.module}/templates/cloud-init/user_data.cfg", { + hostname = instance + user = "ubuntu" + password = random_string.instance_password[instance].result + ssh-keys = concat( + [tls_private_key.instance_ssh_key[instance].public_key_openssh], + local.instances_external_ssh_keys + ) + }) } # Parsed network config file for Cloud Init network_config = { - for vm_name, vm_data in local.instance_networks_expanded: - vm_name => templatefile( - "${path.module}/templates/cloud-init/network_config.cfg", - { networks = vm_data } - ) + for vm_name, vm_data in local.instance_networks_expanded : + vm_name => templatefile( + "${path.module}/templates/cloud-init/network_config.cfg", + { networks = vm_data } + ) } } # Volume for bootstrapping instances using Cloud Init @@ -89,11 +89,11 @@ resource "libvirt_cloudinit_disk" "cloud_init" { resource "libvirt_volume" "kube_disk" { for_each = var.instances - name = join("", [each.key, ".qcow2"]) + name = join("", [each.key, ".qcow2"]) base_volume_id = libvirt_volume.os_image.id - pool = libvirt_pool.volume_pool.name + pool = libvirt_pool.volume_pool.name # 10GB (as bytes) as default - size = try(each.value.disk, 10*1000*1000*1000) + size = try(each.value.disk, 10 * 1000 * 1000 * 1000) } diff --git a/terraform/modules/workload/terraform.tf b/terraform/modules/workload/terraform.tf index c2f68e1..f78dca3 100644 --- a/terraform/modules/workload/terraform.tf +++ b/terraform/modules/workload/terraform.tf @@ -7,7 +7,7 @@ terraform { } tls = { - source = "hashicorp/tls" + source = "hashicorp/tls" version = "3.1.0" } } diff --git a/terraform/modules/workload/variables.tf b/terraform/modules/workload/variables.tf index 84d364d..da6d844 100644 --- a/terraform/modules/workload/variables.tf +++ b/terraform/modules/workload/variables.tf @@ -18,10 +18,10 @@ variable "ssh_connection" { # which networks of type NAT or macvtap will be created and attachable variable "networks" { type = map(object({ - mode = string + mode = string dhcp_address_blocks = list(string) - gateway_address = string - interface = string + gateway_address = string + interface = string })) description = "Networks definition block" } @@ -30,11 +30,11 @@ variable "networks" { # (and their resources) that will be created variable "instances" { type = map(object({ - vcpu = number + vcpu = number memory = number - disk = number + disk = number networks = list(object({ - name = string + name = string address = string })) })) diff --git a/terraform/variables.tf b/terraform/variables.tf index 28abc70..71efe98 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,13 +1,13 @@ # IP address to connect to the host variable "SSH_HOST" { - type = string + type = string description = "The IP of the SSH host to connect to" } # Username to be authenticated in the host # Warning: sudo permissions required variable "SSH_USERNAME" { - type = string + type = string description = "The username to be authenticated in the SSH host" } @@ -15,5 +15,5 @@ variable "SSH_USERNAME" { # This key will be used for API calls variable "SSH_PRIVATE_KEY_PATH" { description = "The path to the private key that will be authorized in the SSH host" - type = string + type = string }