From 4f5410c8e7dff061df32c858f5e293c66503a945 Mon Sep 17 00:00:00 2001 From: jim spring Date: Mon, 7 Sep 2020 21:59:13 +0000 Subject: [PATCH 1/3] add support to open/close api server --- .../kube_api_server_access.sh | 113 ++++++++++++++++++ cluster/azure/aks-api-server-access/main.tf | 48 ++++++++ cluster/azure/aks-api-server-access/output.tf | 7 ++ .../azure/aks-api-server-access/variables.tf | 32 +++++ cluster/azure/aks-gitops/main.tf | 52 +++++--- cluster/azure/aks-gitops/variables.tf | 18 ++- cluster/azure/aks/main.tf | 3 + cluster/azure/aks/variables.tf | 6 + cluster/common/flux/main.tf | 1 + cluster/common/flux/variables.tf | 5 + cluster/environments/azure-simple/main.tf | 68 ++++++----- .../azure-simple/terraform.tfvars | 4 +- .../environments/azure-simple/variables.tf | 12 ++ 13 files changed, 312 insertions(+), 57 deletions(-) create mode 100755 cluster/azure/aks-api-server-access/kube_api_server_access.sh create mode 100644 cluster/azure/aks-api-server-access/main.tf create mode 100644 cluster/azure/aks-api-server-access/output.tf create mode 100644 cluster/azure/aks-api-server-access/variables.tf diff --git a/cluster/azure/aks-api-server-access/kube_api_server_access.sh b/cluster/azure/aks-api-server-access/kube_api_server_access.sh new file mode 100755 index 0000000..631e7ed --- /dev/null +++ b/cluster/azure/aks-api-server-access/kube_api_server_access.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +MODE="" +IP="" +RESOURCE_GROUP="" +CLUSTER_NAME="" +MODE_FLAGS=0 +USE_IP_LIST=0 + +usage() { + echo "Usage: $?" + echo " -a add IP address" + echo " -r remove IP address" + echo " -i " + echo " -g " + echo " -n " + echo " -s " + echo "" + echo "To add an IP address:" + echo " $? -a -i -g -n " + echo "" + echo "To remove an IP address:" + echo " $? -r -i -g -n " + echo "" + echo "Omitting the '-i' flag will use the current external IP address discovered using IP Chicken" + echo "" + echo "It is possible to replace '-i' with '-s' for adding IP addresses. What '-s' does is replaces" + echo "all the values with the specified list. This can also be used to set the list to null, thus" + echo "opening up the IP address range to all" + exit 1 +} + +# remove an ip address from a list +subtract_ip() +{ + IP_TO_REMOVE=( $1 ) + IP_LIST=( $2 ) + OLDIFS="$IFS" + IFS=$'\n' + UPDATED_LIST=( $(grep -Fxv "${IP_TO_REMOVE[*]}" <<< "${IP_LIST[*]}") ) + IFS="$OLDIFS" + echo "${UPDATED_LIST[*]}" +} + +while getopts "ari:n:g:s:" OPTION; do + case $OPTION in + a) + MODE="add" + MODE_FLAGS=$((MODE_FLAGS+1)) + ;; + r) + MODE="remove" + MODE_FLAGS=$((MODE_FLAGS+1)) + ;; + i) + IP=$OPTARG + ;; + s) + IP_LIST=$OPTARG + USE_IP_LIST=1 + ;; + n) + CLUSTER_NAME=$OPTARG + ;; + g) + RESOURCE_GROUP=$OPTARG + ;; + esac +done + +# make sure the basics are set +if [[ -z "$RESOURCE_GROUP" || -z "$CLUSTER_NAME" || ! $MODE_FLAGS -eq 1 ]]; then + usage +fi + +# ensure that both an IP address and IP list are not both passed in +if [ ! -z "$IP_LIST" ] && [ ! -z "$IP" ]; then + echo "One can only use an IP or an IP list, not both" + usage +fi + +if [ $USE_IP_LIST -eq 0 ]; then + # handle case where we are working with a single IP address + if [ -z "$IP" ]; then + IP=`curl -s https://ipchicken.com | egrep -o '([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}' | sort -u` + IP="$IP/32" + fi + + # current IP address LIST + CURRENT_IP_ADDRESS_LIST=`az aks show -g jms-tst1-rg -n jmsfxclus | jq -c -r '.apiServerAccessProfile.authorizedIpRanges' | sed 's/\]//' | sed 's/\[//' | sed 's/"//g' | sed 's/,/ /g'` + FILTERED_IP_ADDRESS_LIST=$(subtract_ip "$IP" "$CURRENT_IP_ADDRESS_LIST") + if [ "$MODE" == "add" ]; then + # handle adding the IP + UPDATED_IP_ADDRESS_LIST="$FILTERED_IP_ADDRESS_LIST $IP" + else + UPDATED_IP_ADDRESS_LIST="$FILTERED_IP_ADDRESS_LIST" + fi + UPDATED_IP_ADDRESS_LIST=`echo $UPDATED_IP_ADDRESS_LIST | sed 's/ /,/g'` +else + # use the specified IP address liit + UPDATED_IP_ADDRESS_LIST="$IP_LIST" +fi + +# update the list +az aks update --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME --api-server-authorized-ip-ranges "$UPDATED_IP_ADDRESS_LIST" > /dev/null +if [ ! $? -eq 0 ]; then + echo "error updating api server ip ranges" + exit 1 +fi +echo "API Server authorized IPs updated to - \"$UPDATED_IP_ADDRESS_LIST\"" + +exit 0 + diff --git a/cluster/azure/aks-api-server-access/main.tf b/cluster/azure/aks-api-server-access/main.tf new file mode 100644 index 0000000..17fcce7 --- /dev/null +++ b/cluster/azure/aks-api-server-access/main.tf @@ -0,0 +1,48 @@ +module "common-provider" { + source = "../../common/provider" +} + +locals { + # does access to the api server need opening up? + api_server_access_needed = length(var.kube_api_server_authorized_ip_ranges) > 0 ? true : false + + # what does the opening the server look like? a specified IP will be added, if empty string, the whole api server will be open + api_server_temporary_access_allow_all = var.kube_api_server_temp_authorized_ip == "" ? true : false + + # current api server list + api_server_access_list = join(",", var.kube_api_server_authorized_ip_ranges) + + # setup the command line + api_access_script = "${path.module}/kube_api_server_access.sh" + + # open api server access + open_api_server_access_args = local.api_server_temporary_access_allow_all ? "-g ${var.resource_group_name} -n ${var.cluster_name} -s ''" : "-g ${var.resource_group_name} -n ${var.cluster_name} -a -i ${var.kube_api_server_temp_authorized_ip}" + + # close api server access + close_api_server_access_args = "-g ${var.resource_group_name} -n ${var.cluster_name} -a -s '${local.api_server_access_list}'" +} + +resource "null_resource" "open_api_server" { + count = local.api_server_access_needed ? 1 : 0 + + provisioner "local-exec" { + command = "${local.api_access_script} ${local.open_api_server_access_args}" + } + + triggers = { + kubeconfig_complete = var.kubeconfig_complete + } +} + +resource "null_resource" "close_api_server" { + count = local.api_server_access_needed ? 1 : 0 + + provisioner "local-exec" { + command = "${local.api_access_script} ${local.close_api_server_access_args}" + } + + triggers = { + flux_done = var.flux_done + kubediff_done = var.kubediff_done + } +} \ No newline at end of file diff --git a/cluster/azure/aks-api-server-access/output.tf b/cluster/azure/aks-api-server-access/output.tf new file mode 100644 index 0000000..758dd60 --- /dev/null +++ b/cluster/azure/aks-api-server-access/output.tf @@ -0,0 +1,7 @@ +output "api_server_open" { + value = join("",null_resource.open_api_server.*.id) +} + +output "api_server_closed" { + value = join("",null_resource.close_api_server.*.id) +} \ No newline at end of file diff --git a/cluster/azure/aks-api-server-access/variables.tf b/cluster/azure/aks-api-server-access/variables.tf new file mode 100644 index 0000000..8795b2f --- /dev/null +++ b/cluster/azure/aks-api-server-access/variables.tf @@ -0,0 +1,32 @@ +variable "resource_group_name" { + type = string +} + +variable "cluster_name" { + type = string +} + +variable "kube_api_server_authorized_ip_ranges" { + type = list(string) + default = [] +} + +variable "kube_api_server_temp_authorized_ip" { + type = string + default = "" +} + +variable "kubeconfig_complete" { + description = "Allows permissions to wait until kube cluster complete." + type = string +} + +variable "flux_done" { + description = "Is flux done running" + type = string +} + +variable "kubediff_done" { + description = "Is kubediff done running" + type = string +} \ No newline at end of file diff --git a/cluster/azure/aks-gitops/main.tf b/cluster/azure/aks-gitops/main.tf index fe372a2..3e3b846 100644 --- a/cluster/azure/aks-gitops/main.tf +++ b/cluster/azure/aks-gitops/main.tf @@ -1,28 +1,29 @@ data "azurerm_resource_group" "aksgitops" { - name = var.resource_group_name + name = var.resource_group_name } module "aks" { source = "../../azure/aks" - resource_group_name = data.azurerm_resource_group.aksgitops.name - cluster_name = var.cluster_name - agent_vm_count = var.agent_vm_count - agent_vm_size = var.agent_vm_size - dns_prefix = var.dns_prefix - vnet_subnet_id = var.vnet_subnet_id - ssh_public_key = var.ssh_public_key - msi_enabled = var.msi_enabled - service_principal_id = var.service_principal_id - service_principal_secret = var.service_principal_secret - service_cidr = var.service_cidr - dns_ip = var.dns_ip - docker_cidr = var.docker_cidr - kubernetes_version = var.kubernetes_version - kubeconfig_filename = var.kubeconfig_filename - network_policy = var.network_policy - network_plugin = var.network_plugin - oms_agent_enabled = var.oms_agent_enabled + resource_group_name = data.azurerm_resource_group.aksgitops.name + cluster_name = var.cluster_name + agent_vm_count = var.agent_vm_count + agent_vm_size = var.agent_vm_size + dns_prefix = var.dns_prefix + vnet_subnet_id = var.vnet_subnet_id + ssh_public_key = var.ssh_public_key + msi_enabled = var.msi_enabled + service_principal_id = var.service_principal_id + service_principal_secret = var.service_principal_secret + service_cidr = var.service_cidr + dns_ip = var.dns_ip + docker_cidr = var.docker_cidr + kubernetes_version = var.kubernetes_version + kubeconfig_filename = var.kubeconfig_filename + network_policy = var.network_policy + network_plugin = var.network_plugin + oms_agent_enabled = var.oms_agent_enabled + kube_api_server_authorized_ip_ranges = var.kube_api_server_authorized_ip_ranges tags = var.tags } @@ -43,6 +44,7 @@ module "flux" { flux_clone_dir = "${var.cluster_name}-flux" acr_enabled = var.acr_enabled gc_enabled = var.gc_enabled + api_server_available = module.api_server_access.api_server_open } module "kubediff" { @@ -51,3 +53,15 @@ module "kubediff" { kubeconfig_complete = module.aks.kubeconfig_done gitops_ssh_url = var.gitops_ssh_url } + +module "api_server_access" { + source = "../../azure/aks-api-server-access" + + resource_group_name = var.resource_group_name + cluster_name = var.cluster_name + kube_api_server_authorized_ip_ranges = var.kube_api_server_authorized_ip_ranges + kube_api_server_temp_authorized_ip = var.kube_api_server_temp_authorized_ip + kubeconfig_complete = module.aks.kubeconfig_done + flux_done = module.flux.flux_done + kubediff_done = module.kubediff.kubediff_done +} diff --git a/cluster/azure/aks-gitops/variables.tf b/cluster/azure/aks-gitops/variables.tf index 2abdc87..e9aa737 100644 --- a/cluster/azure/aks-gitops/variables.tf +++ b/cluster/azure/aks-gitops/variables.tf @@ -20,7 +20,7 @@ variable "cluster_name" { } variable "msi_enabled" { - type = bool + type = bool default = false } @@ -81,12 +81,12 @@ variable "vnet_subnet_id" { } variable "service_principal_id" { - type = string + type = string default = "" } variable "service_principal_secret" { - type = string + type = string default = "" } @@ -133,3 +133,15 @@ variable "tags" { default = {} } + +variable "kube_api_server_authorized_ip_ranges" { + description = "IPs allowed to contact the API server." + type = list(string) + default = [] +} + +variable "kube_api_server_temp_authorized_ip" { + description = "Temp IP allowed to contact the API server." + type = string + default = "" +} \ No newline at end of file diff --git a/cluster/azure/aks/main.tf b/cluster/azure/aks/main.tf index 3847270..b4d6e86 100644 --- a/cluster/azure/aks/main.tf +++ b/cluster/azure/aks/main.tf @@ -115,6 +115,9 @@ resource "azurerm_kubernetes_cluster" "cluster" { } } + # Control access to the API server end point + api_server_authorized_ip_ranges = var.kube_api_server_authorized_ip_ranges + tags = var.tags } diff --git a/cluster/azure/aks/variables.tf b/cluster/azure/aks/variables.tf index e69ec22..356f0e3 100644 --- a/cluster/azure/aks/variables.tf +++ b/cluster/azure/aks/variables.tf @@ -121,3 +121,9 @@ variable "tags" { default = {} } + +variable "kube_api_server_authorized_ip_ranges" { + description = "IPs allowed to contact the API server." + type = list(string) + default = [] +} diff --git a/cluster/common/flux/main.tf b/cluster/common/flux/main.tf index 097f462..712a6eb 100644 --- a/cluster/common/flux/main.tf +++ b/cluster/common/flux/main.tf @@ -12,5 +12,6 @@ resource "null_resource" "deploy_flux" { triggers = { enable_flux = var.enable_flux flux_recreate = var.flux_recreate + api_server_available = var.api_server_available } } diff --git a/cluster/common/flux/variables.tf b/cluster/common/flux/variables.tf index 824b783..bdca22e 100644 --- a/cluster/common/flux/variables.tf +++ b/cluster/common/flux/variables.tf @@ -91,3 +91,8 @@ variable "flux_clone_dir" { description = "Name of the directory to clone flux repo and deploy in the cluster." type = string } + +variable "api_server_available" { + description = "Has the api server proces run" + type = string +} diff --git a/cluster/environments/azure-simple/main.tf b/cluster/environments/azure-simple/main.tf index 3abfa53..5adbe1c 100644 --- a/cluster/environments/azure-simple/main.tf +++ b/cluster/environments/azure-simple/main.tf @@ -14,9 +14,9 @@ data "azurerm_resource_group" "cluster_rg" { module "vnet" { source = "../../azure/vnet" - resource_group_name = data.azurerm_resource_group.cluster_rg.name - vnet_name = var.vnet_name - address_space = var.address_space + resource_group_name = data.azurerm_resource_group.cluster_rg.name + vnet_name = var.vnet_name + address_space = var.address_space tags = { environment = "azure-simple" @@ -26,38 +26,40 @@ module "vnet" { module "subnet" { source = "../../azure/subnet" - subnet_name = "${var.cluster_name}-aks-subnet" - vnet_name = module.vnet.vnet_name - resource_group_name = data.azurerm_resource_group.cluster_rg.name - address_prefixes = [ var.subnet_prefix ] + subnet_name = "${var.cluster_name}-aks-subnet" + vnet_name = module.vnet.vnet_name + resource_group_name = data.azurerm_resource_group.cluster_rg.name + address_prefixes = [var.subnet_prefix] } module "aks-gitops" { source = "../../azure/aks-gitops" - acr_enabled = var.acr_enabled - agent_vm_count = var.agent_vm_count - agent_vm_size = var.agent_vm_size - cluster_name = var.cluster_name - dns_prefix = var.dns_prefix - flux_recreate = var.flux_recreate - gc_enabled = var.gc_enabled - gitops_ssh_url = var.gitops_ssh_url - gitops_ssh_key_path = var.gitops_ssh_key_path - gitops_path = var.gitops_path - gitops_poll_interval = var.gitops_poll_interval - gitops_label = var.gitops_label - gitops_url_branch = var.gitops_url_branch - ssh_public_key = var.ssh_public_key - resource_group_name = data.azurerm_resource_group.cluster_rg.name - service_principal_id = var.service_principal_id - service_principal_secret = var.service_principal_secret - vnet_subnet_id = module.subnet.subnet_id - service_cidr = var.service_cidr - dns_ip = var.dns_ip - docker_cidr = var.docker_cidr - network_plugin = var.network_plugin - network_policy = var.network_policy - oms_agent_enabled = var.oms_agent_enabled - kubernetes_version = var.kubernetes_version -} + acr_enabled = var.acr_enabled + agent_vm_count = var.agent_vm_count + agent_vm_size = var.agent_vm_size + cluster_name = var.cluster_name + dns_prefix = var.dns_prefix + flux_recreate = var.flux_recreate + gc_enabled = var.gc_enabled + gitops_ssh_url = var.gitops_ssh_url + gitops_ssh_key_path = var.gitops_ssh_key_path + gitops_path = var.gitops_path + gitops_poll_interval = var.gitops_poll_interval + gitops_label = var.gitops_label + gitops_url_branch = var.gitops_url_branch + ssh_public_key = var.ssh_public_key + resource_group_name = data.azurerm_resource_group.cluster_rg.name + service_principal_id = var.service_principal_id + service_principal_secret = var.service_principal_secret + vnet_subnet_id = module.subnet.subnet_id + service_cidr = var.service_cidr + dns_ip = var.dns_ip + docker_cidr = var.docker_cidr + network_plugin = var.network_plugin + network_policy = var.network_policy + oms_agent_enabled = var.oms_agent_enabled + kubernetes_version = var.kubernetes_version + kube_api_server_authorized_ip_ranges = var.kube_api_server_authorized_ip_ranges + kube_api_server_temp_authorized_ip = var.kube_api_server_temp_authorized_ip +} \ No newline at end of file diff --git a/cluster/environments/azure-simple/terraform.tfvars b/cluster/environments/azure-simple/terraform.tfvars index 66d50ca..c8844f3 100644 --- a/cluster/environments/azure-simple/terraform.tfvars +++ b/cluster/environments/azure-simple/terraform.tfvars @@ -4,9 +4,9 @@ agent_vm_count = "3" dns_prefix = "" service_principal_id = "" service_principal_secret = "" -ssh_public_key = "ssh-rsa ..." # from node-ssh-key.pub +ssh_public_key = "ssh-rsa ..." # from node-ssh-key.pub gitops_ssh_url = "git@github.com:/.git" # ssh url to manifest repo -gitops_ssh_key_path = "/home//.ssh/gitops-ssh-key" # path to private gitops repo key +gitops_ssh_key_path = "/home//.ssh/gitops-ssh-key" # path to private gitops repo key vnet_name = "" #-------------------------------------------------------------- diff --git a/cluster/environments/azure-simple/variables.tf b/cluster/environments/azure-simple/variables.tf index cffcd3b..26763b1 100644 --- a/cluster/environments/azure-simple/variables.tf +++ b/cluster/environments/azure-simple/variables.tf @@ -125,3 +125,15 @@ variable "oms_agent_enabled" { type = string default = "false" } + +variable "kube_api_server_authorized_ip_ranges" { + description = "IPs allowed to contact the API server." + type = list(string) + default = [] +} + +variable "kube_api_server_temp_authorized_ip" { + description = "Temporary IP to use to contact API server" + type = string + default = "" +} From 26cfa9bc646ef12a9bdc9f0035b5e85ba93726b3 Mon Sep 17 00:00:00 2001 From: jim spring Date: Mon, 7 Sep 2020 22:16:29 +0000 Subject: [PATCH 2/3] missing flag in the case of opening to all ips --- cluster/azure/aks-api-server-access/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/azure/aks-api-server-access/main.tf b/cluster/azure/aks-api-server-access/main.tf index 17fcce7..3758caf 100644 --- a/cluster/azure/aks-api-server-access/main.tf +++ b/cluster/azure/aks-api-server-access/main.tf @@ -16,7 +16,7 @@ locals { api_access_script = "${path.module}/kube_api_server_access.sh" # open api server access - open_api_server_access_args = local.api_server_temporary_access_allow_all ? "-g ${var.resource_group_name} -n ${var.cluster_name} -s ''" : "-g ${var.resource_group_name} -n ${var.cluster_name} -a -i ${var.kube_api_server_temp_authorized_ip}" + open_api_server_access_args = local.api_server_temporary_access_allow_all ? "-g ${var.resource_group_name} -n ${var.cluster_name} -a -s ''" : "-g ${var.resource_group_name} -n ${var.cluster_name} -a -i ${var.kube_api_server_temp_authorized_ip}" # close api server access close_api_server_access_args = "-g ${var.resource_group_name} -n ${var.cluster_name} -a -s '${local.api_server_access_list}'" From 46414c1e0da32abff650a802b13ac63ed2c2ae63 Mon Sep 17 00:00:00 2001 From: jim spring Date: Mon, 7 Sep 2020 22:31:04 +0000 Subject: [PATCH 3/3] need to pass in flux recreate --- cluster/azure/aks-api-server-access/main.tf | 3 ++- cluster/azure/aks-api-server-access/variables.tf | 7 ++++++- cluster/azure/aks-gitops/main.tf | 1 + cluster/common/flux/main.tf | 1 + 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cluster/azure/aks-api-server-access/main.tf b/cluster/azure/aks-api-server-access/main.tf index 3758caf..89c46cd 100644 --- a/cluster/azure/aks-api-server-access/main.tf +++ b/cluster/azure/aks-api-server-access/main.tf @@ -31,6 +31,7 @@ resource "null_resource" "open_api_server" { triggers = { kubeconfig_complete = var.kubeconfig_complete + flux_recreate = var.flux_recreate } } @@ -45,4 +46,4 @@ resource "null_resource" "close_api_server" { flux_done = var.flux_done kubediff_done = var.kubediff_done } -} \ No newline at end of file +} diff --git a/cluster/azure/aks-api-server-access/variables.tf b/cluster/azure/aks-api-server-access/variables.tf index 8795b2f..b24201b 100644 --- a/cluster/azure/aks-api-server-access/variables.tf +++ b/cluster/azure/aks-api-server-access/variables.tf @@ -26,7 +26,12 @@ variable "flux_done" { type = string } +variable "flux_recreate" { + description = "Does flux need recreating" + type = string +} + variable "kubediff_done" { description = "Is kubediff done running" type = string -} \ No newline at end of file +} diff --git a/cluster/azure/aks-gitops/main.tf b/cluster/azure/aks-gitops/main.tf index 3e3b846..507232e 100644 --- a/cluster/azure/aks-gitops/main.tf +++ b/cluster/azure/aks-gitops/main.tf @@ -63,5 +63,6 @@ module "api_server_access" { kube_api_server_temp_authorized_ip = var.kube_api_server_temp_authorized_ip kubeconfig_complete = module.aks.kubeconfig_done flux_done = module.flux.flux_done + flux_recreate = var.flux_recreate kubediff_done = module.kubediff.kubediff_done } diff --git a/cluster/common/flux/main.tf b/cluster/common/flux/main.tf index 712a6eb..3a773cc 100644 --- a/cluster/common/flux/main.tf +++ b/cluster/common/flux/main.tf @@ -13,5 +13,6 @@ resource "null_resource" "deploy_flux" { enable_flux = var.enable_flux flux_recreate = var.flux_recreate api_server_available = var.api_server_available + flux_image_tag = var.flux_image_tag } }