add ability to open up API server for flux when AKS configured to use it

cluster/azure/aks-api-server-access/kube_api_server_access.sh

@@ -0,0 +1,113 @@
+#!/bin/bash
+
+MODE=""
+IP=""
+RESOURCE_GROUP=""
+CLUSTER_NAME=""
+MODE_FLAGS=0
+USE_IP_LIST=0
+
+usage() {
+    echo "Usage: $?"
+    echo "  -a add IP address"
+    echo "  -r remove IP address"
+    echo "  -i <IP address>"
+    echo "  -g <Resource Group>"
+    echo "  -n <Cluster Name>"
+    echo "  -s <IP list>"
+    echo ""
+    echo "To add an IP address:"
+    echo "  $? -a -i <IP address> -g <Resource Group> -n <Cluster Name>"
+    echo ""
+    echo "To remove an IP address:"
+    echo "  $? -r -i <IP address> -g <Resource Group> -n <Cluster Name>"
+    echo ""
+    echo "Omitting the '-i' flag will use the current external IP address discovered using IP Chicken"
+    echo ""
+    echo "It is possible to replace '-i' with '-s' for adding IP addresses.  What '-s' does is replaces"
+    echo "all the values with the specified list.  This can also be used to set the list to null, thus"
+    echo "opening up the IP address range to all"
+    exit 1
+}
+
+# remove an ip address from a list
+subtract_ip()
+{
+    IP_TO_REMOVE=( $1 )
+    IP_LIST=( $2 )
+    OLDIFS="$IFS"
+    IFS=$'\n'
+    UPDATED_LIST=( $(grep -Fxv "${IP_TO_REMOVE[*]}" <<< "${IP_LIST[*]}") )
+    IFS="$OLDIFS"
+    echo "${UPDATED_LIST[*]}"
+}
+
+while getopts "ari:n:g:s:" OPTION; do
+    case $OPTION in
+    a)
+        MODE="add"
+        MODE_FLAGS=$((MODE_FLAGS+1))
+        ;;
+    r)
+        MODE="remove"
+        MODE_FLAGS=$((MODE_FLAGS+1))
+        ;;
+    i)
+        IP=$OPTARG
+        ;;
+    s)
+        IP_LIST=$OPTARG
+	USE_IP_LIST=1
+        ;;
+    n)
+        CLUSTER_NAME=$OPTARG
+        ;;
+    g)
+        RESOURCE_GROUP=$OPTARG
+        ;;
+    esac
+done
+
+# make sure the basics are set
+if [[ -z "$RESOURCE_GROUP" || -z "$CLUSTER_NAME" || ! $MODE_FLAGS -eq 1 ]]; then
+    usage
+fi
+
+# ensure that both an IP address and IP list are not both passed in
+if [ ! -z "$IP_LIST" ] && [ ! -z "$IP" ]; then
+    echo "One can only use an IP or an IP list, not both"
+    usage
+fi
+
+if [ $USE_IP_LIST -eq 0 ]; then
+    # handle case where we are working with a single IP address
+    if [ -z "$IP" ]; then
+        IP=`curl -s https://ipchicken.com | egrep -o '([[:digit:]]{1,3}\.){3}[[:digit:]]{1,3}' | sort -u`
+	IP="$IP/32"
+    fi
+
+    # current IP address LIST
+    CURRENT_IP_ADDRESS_LIST=`az aks show -g jms-tst1-rg -n jmsfxclus | jq -c -r '.apiServerAccessProfile.authorizedIpRanges' | sed 's/\]//' | sed 's/\[//' | sed 's/"//g' | sed 's/,/ /g'`
+    FILTERED_IP_ADDRESS_LIST=$(subtract_ip "$IP" "$CURRENT_IP_ADDRESS_LIST")
+    if [ "$MODE" == "add" ]; then
+        # handle adding the IP
+        UPDATED_IP_ADDRESS_LIST="$FILTERED_IP_ADDRESS_LIST $IP"
+    else
+        UPDATED_IP_ADDRESS_LIST="$FILTERED_IP_ADDRESS_LIST"
+    fi
+    UPDATED_IP_ADDRESS_LIST=`echo $UPDATED_IP_ADDRESS_LIST | sed 's/ /,/g'`
+else
+    # use the specified IP address liit
+    UPDATED_IP_ADDRESS_LIST="$IP_LIST"
+fi
+
+# update the list
+az aks update --resource-group $RESOURCE_GROUP --name $CLUSTER_NAME --api-server-authorized-ip-ranges "$UPDATED_IP_ADDRESS_LIST" > /dev/null
+if [ ! $? -eq 0 ]; then
+    echo "error updating api server ip ranges"
+    exit 1
+fi
+echo "API Server authorized IPs updated to - \"$UPDATED_IP_ADDRESS_LIST\""
+
+exit 0
+

cluster/azure/aks-api-server-access/main.tf

@@ -0,0 +1,49 @@
+module "common-provider" {
+  source = "../../common/provider"
+}
+
+locals {
+  # does access to the api server need opening up?
+  api_server_access_needed = length(var.kube_api_server_authorized_ip_ranges) > 0 ? true : false
+
+  # what does the opening the server look like?  a specified IP will be added, if empty string, the whole api server will be open
+  api_server_temporary_access_allow_all = var.kube_api_server_temp_authorized_ip == "" ? true : false
+
+  # current api server list
+  api_server_access_list = join(",", var.kube_api_server_authorized_ip_ranges)
+
+  # setup the command line
+  api_access_script = "${path.module}/kube_api_server_access.sh"
+
+  # open api server access
+  open_api_server_access_args = local.api_server_temporary_access_allow_all ? "-g ${var.resource_group_name} -n ${var.cluster_name} -a -s ''" : "-g ${var.resource_group_name} -n ${var.cluster_name} -a -i ${var.kube_api_server_temp_authorized_ip}"
+
+  # close api server access
+  close_api_server_access_args = "-g ${var.resource_group_name} -n ${var.cluster_name} -a -s '${local.api_server_access_list}'"
+}
+
+resource "null_resource" "open_api_server" {
+  count = local.api_server_access_needed ? 1 : 0
+
+  provisioner "local-exec" {
+    command = "${local.api_access_script} ${local.open_api_server_access_args}"
+  }
+
+  triggers = {
+    kubeconfig_complete = var.kubeconfig_complete
+    flux_recreate = var.flux_recreate
+  }
+}
+
+resource "null_resource" "close_api_server" {
+  count = local.api_server_access_needed ? 1 : 0
+
+  provisioner "local-exec" {
+    command = "${local.api_access_script} ${local.close_api_server_access_args}"
+  }
+
+  triggers = {
+    flux_done = var.flux_done
+    kubediff_done = var.kubediff_done
+  }
+}

cluster/azure/aks-api-server-access/output.tf

@@ -0,0 +1,7 @@
+output "api_server_open" {
+  value = join("",null_resource.open_api_server.*.id)
+}
+
+output "api_server_closed" {
+  value = join("",null_resource.close_api_server.*.id)
+}

cluster/azure/aks-api-server-access/variables.tf

@@ -0,0 +1,37 @@
+variable "resource_group_name" {
+  type = string
+}
+
+variable "cluster_name" {
+  type = string
+}
+
+variable "kube_api_server_authorized_ip_ranges" {
+  type = list(string)
+  default = []
+}
+
+variable "kube_api_server_temp_authorized_ip" {
+  type = string
+  default = ""
+}
+
+variable "kubeconfig_complete" {
+  description = "Allows permissions to wait until kube cluster complete."
+  type        = string
+}
+
+variable "flux_done" {
+  description = "Is flux done running"
+  type        = string
+}
+
+variable "flux_recreate" {
+  description = "Does flux need recreating"
+  type        = string
+}
+
+variable "kubediff_done" {
+  description = "Is kubediff done running"
+  type        = string
+}

cluster/azure/aks-gitops/main.tf

@@ -1,28 +1,29 @@
 data "azurerm_resource_group" "aksgitops" {
-    name = var.resource_group_name
+  name = var.resource_group_name
 }
 
 module "aks" {
   source = "../../azure/aks"
 
-  resource_group_name      = data.azurerm_resource_group.aksgitops.name
-  cluster_name             = var.cluster_name
-  agent_vm_count           = var.agent_vm_count
-  agent_vm_size            = var.agent_vm_size
-  dns_prefix               = var.dns_prefix
-  vnet_subnet_id           = var.vnet_subnet_id
-  ssh_public_key           = var.ssh_public_key
-  msi_enabled              = var.msi_enabled
-  service_principal_id     = var.service_principal_id
-  service_principal_secret = var.service_principal_secret
-  service_cidr             = var.service_cidr
-  dns_ip                   = var.dns_ip
-  docker_cidr              = var.docker_cidr
-  kubernetes_version       = var.kubernetes_version
-  kubeconfig_filename      = var.kubeconfig_filename
-  network_policy           = var.network_policy
-  network_plugin           = var.network_plugin
-  oms_agent_enabled        = var.oms_agent_enabled
+  resource_group_name                  = data.azurerm_resource_group.aksgitops.name
+  cluster_name                         = var.cluster_name
+  agent_vm_count                       = var.agent_vm_count
+  agent_vm_size                        = var.agent_vm_size
+  dns_prefix                           = var.dns_prefix
+  vnet_subnet_id                       = var.vnet_subnet_id
+  ssh_public_key                       = var.ssh_public_key
+  msi_enabled                          = var.msi_enabled
+  service_principal_id                 = var.service_principal_id
+  service_principal_secret             = var.service_principal_secret
+  service_cidr                         = var.service_cidr
+  dns_ip                               = var.dns_ip
+  docker_cidr                          = var.docker_cidr
+  kubernetes_version                   = var.kubernetes_version
+  kubeconfig_filename                  = var.kubeconfig_filename
+  network_policy                       = var.network_policy
+  network_plugin                       = var.network_plugin
+  oms_agent_enabled                    = var.oms_agent_enabled
+  kube_api_server_authorized_ip_ranges = var.kube_api_server_authorized_ip_ranges
 
   tags = var.tags
 }
@@ -43,6 +44,7 @@ module "flux" {
   flux_clone_dir       = "${var.cluster_name}-flux"
   acr_enabled          = var.acr_enabled
   gc_enabled           = var.gc_enabled
+  api_server_available = module.api_server_access.api_server_open
 }
 
 module "kubediff" {
@@ -51,3 +53,16 @@ module "kubediff" {
   kubeconfig_complete = module.aks.kubeconfig_done
   gitops_ssh_url      = var.gitops_ssh_url
 }
+
+module "api_server_access" {
+  source = "../../azure/aks-api-server-access"
+
+  resource_group_name = var.resource_group_name
+  cluster_name = var.cluster_name
+  kube_api_server_authorized_ip_ranges = var.kube_api_server_authorized_ip_ranges
+  kube_api_server_temp_authorized_ip = var.kube_api_server_temp_authorized_ip
+  kubeconfig_complete = module.aks.kubeconfig_done
+  flux_done = module.flux.flux_done
+  flux_recreate = var.flux_recreate
+  kubediff_done = module.kubediff.kubediff_done
+}

cluster/azure/aks-gitops/variables.tf

@@ -20,7 +20,7 @@ variable "cluster_name" {
 }
 
 variable "msi_enabled" {
-  type = bool
+  type    = bool
   default = false
 }
 
@@ -81,12 +81,12 @@ variable "vnet_subnet_id" {
 }
 
 variable "service_principal_id" {
-  type = string
+  type    = string
   default = ""
 }
 
 variable "service_principal_secret" {
-  type = string
+  type    = string
   default = ""
 }
 
@@ -133,3 +133,15 @@ variable "tags" {
 
   default = {}
 }
+
+variable "kube_api_server_authorized_ip_ranges" {
+  description = "IPs allowed to contact the API server."
+  type        = list(string)
+  default     = []
+}
+
+variable "kube_api_server_temp_authorized_ip" {
+  description = "Temp IP allowed to contact the API server."
+  type        = string
+  default     = ""
+}

cluster/azure/aks/main.tf

@@ -115,6 +115,9 @@ resource "azurerm_kubernetes_cluster" "cluster" {
     }
   }
 
+  # Control access to the API server end point
+  api_server_authorized_ip_ranges = var.kube_api_server_authorized_ip_ranges
+
   tags = var.tags
 }
 

cluster/azure/aks/variables.tf

@@ -121,3 +121,9 @@ variable "tags" {
 
   default = {}
 }
+
+variable "kube_api_server_authorized_ip_ranges" {
+  description = "IPs allowed to contact the API server."
+  type        = list(string)
+  default     = []
+}

cluster/common/flux/main.tf

@@ -12,5 +12,7 @@ resource "null_resource" "deploy_flux" {
   triggers = {
     enable_flux   = var.enable_flux
     flux_recreate = var.flux_recreate
+    api_server_available = var.api_server_available
+    flux_image_tag = var.flux_image_tag
   }
 }

cluster/common/flux/variables.tf

@@ -91,3 +91,8 @@ variable "flux_clone_dir" {
   description = "Name of the directory to clone flux repo and deploy in the cluster."
   type        = string
 }
+
+variable "api_server_available" {
+  description = "Has the api server proces run"
+  type        = string
+}