apps sc & wc: improve dry-run and deploy scripts

elastisys · Jun 1, 2021 · 34d6e2f · 34d6e2f
1 parent 6e56b4c
commit 34d6e2f
Show file tree

Hide file tree

Showing 6 changed files with 197 additions and 105 deletions.
diff --git a/WIP-CHANGELOG.md b/WIP-CHANGELOG.md
@@ -0,0 +1,3 @@
+### Changed
+
+- The dry-run and appy command now checks against the state of the cluster while ran.
diff --git a/bin/apps.bash b/bin/apps.bash
@@ -23,23 +23,45 @@ apps_init() {
 apps_run_sc() {
     log_info "Applying applications in service cluster"
 
-    (
-        : "${scripts_path:?Missing scripts path}"
-        : "${secrets[kube_config_sc]:?Missing service cluster kubeconfig}"
-        with_kubeconfig "${secrets[kube_config_sc]}" \
-            CONFIG_PATH="${CK8S_CONFIG_PATH}" "${scripts_path}/deploy-sc.sh"
-    )
+    if [ ${#} -eq 1 ]; then
+
+        (
+            : "${scripts_path:?Missing scripts path}"
+            : "${secrets[kube_config_sc]:?Missing service cluster kubeconfig}"
+            with_kubeconfig "${secrets[kube_config_sc]}" \
+                CONFIG_PATH="${CK8S_CONFIG_PATH}" "${scripts_path}/deploy-sc.sh" "${1}"
+        )
+    else
+        (
+            : "${scripts_path:?Missing scripts path}"
+            : "${secrets[kube_config_sc]:?Missing service cluster kubeconfig}"
+            with_kubeconfig "${secrets[kube_config_sc]}" \
+                CONFIG_PATH="${CK8S_CONFIG_PATH}" "${scripts_path}/deploy-sc.sh"
+        )
+
+    fi
 }
 
 apps_run_wc() {
     log_info "Applying applications in workload cluster"
 
-    (
-        : "${scripts_path:?Missing scripts path}"
-        : "${secrets[kube_config_wc]:?Missing workload cluster kubeconfig}"
-        with_kubeconfig "${secrets[kube_config_wc]}" \
-            CONFIG_PATH="${CK8S_CONFIG_PATH}" "${scripts_path}/deploy-wc.sh"
-    )
+    if [ ${#} -eq 1 ]; then
+
+        (
+            : "${scripts_path:?Missing scripts path}"
+            : "${secrets[kube_config_wc]:?Missing workload cluster kubeconfig}"
+            with_kubeconfig "${secrets[kube_config_wc]}" \
+                CONFIG_PATH="${CK8S_CONFIG_PATH}" "${scripts_path}/deploy-wc.sh" "${1}"
+        )
+    else
+        (
+            : "${scripts_path:?Missing scripts path}"
+            : "${secrets[kube_config_wc]:?Missing workload cluster kubeconfig}"
+            with_kubeconfig "${secrets[kube_config_wc]}" \
+                CONFIG_PATH="${CK8S_CONFIG_PATH}" "${scripts_path}/deploy-wc.sh"
+        )
+
+    fi
 }
 
 template_validate_sc() {
@@ -75,33 +97,65 @@ apps_sc() {
     # While it would be nice to have some template validation before `helmfile apply`,
     # at least Helmfile does "just in time" template validation. Not as nice,
     # but feels good enough until we figure out something smarter.
-    #
-    #[ "$1" != "--skip-template-validate" ] && template_validate_sc
-    apps_run_sc
+    #    #[ "$1" != "--skip-template-validate" ] && template_validate_sc
+
+    if [ ${#} -eq 2 ]; then
+
+        #[ "$1" != "--skip-template-validate" ] && template_validate_sc
+        apps_run_sc "$2"
+    else
+        #[ "$1" != "--skip-template-validate" ] && template_validate_sc
+        apps_run_sc
 
+    fi
     log_info "Applications applied successfully!"
+
 }
 
 apps_wc() {
     apps_init
-    # See rationale above.
-    #[ "$1" != "--skip-template-validate" ] && template_validate_wc
-    apps_run_wc
+    #
+    # The first few Charts install CRDs, which will make template validation
+    # fail. CRDs are "changes" to the Kubernetes API, hence validation against
+    # the Kubernetes API cannot be done. OTOH, manually adding the CRDs during
+    # bootstrap is error-prone and adds maintenance burden.
+    #
+    # While it would be nice to have some template validation before `helmfile apply`,
+    # at least Helmfile does "just in time" template validation. Not as nice,
+    # but feels good enough until we figure out something smarter.
+    #    #[ "$1" != "--skip-template-validate" ] && template_validate_sc
+
+    if [ ${#} -eq 2 ]; then
 
+        #[ "$1" != "--skip-template-validate" ] && template_validate_wc
+        apps_run_wc "$2"
+    else
+        #[ "$1" != "--skip-template-validate" ] && template_validate_wc
+        apps_run_wc
+
+    fi
     log_info "Applications applied successfully!"
 }
 
 #
 # ENTRYPOINT
 #
 
-
 if [[ $1 == "wc" ]]; then
     config_load "$1"
-    apps_wc "$2"
+    if [ ${#} -eq 3 ]; then
+        apps_wc "$2" "$3"
+    else
+        apps_wc "$2"
+    fi
+
 elif [[ $1 == "sc" ]]; then
     config_load "$1"
-    apps_sc "$2"
+    if [ ${#} -eq 3 ]; then
+        apps_sc "$2" "$3"
+    else
+        apps_sc "$2"
+    fi
 else
     echo "ERROR:  [$1] is an invalid argument:"
     echo "usage:   ck8s apps <wc|sc>. "

diff --git a/bin/ck8s b/bin/ck8s
@@ -14,8 +14,10 @@ usage() {
     echo "  bootstrap <wc|sc>                               bootstrap the cluster" 1>&2
     echo "  apps <wc|sc> [--skip-template-validate]         deploy the applications" 1>&2
     echo "  apply <wc|sc> [--skip-template-validate]        bootstrap and apps" 1>&2
+    echo "  apply-sync <wc|sc> [--skip-template-validate]   bootstrap and apps with sync" 1>&2
     echo "  test <wc|sc>                                    test the applications" 1>&2
     echo "  dry-run <wc|sc>                                 runs helmfile diff" 1>&2
+    echo "  dry-run-sync <wc|sc>                            runs helmfile template | kubectl diff -f -" 1>&2
     echo "  team add-pgp <fp>                               add a new PGP key to secrets" 1>&2
     echo "  team remove-pgp <fp>                            remove a PGP key from secrets and rotate the data encryption key" 1>&2
     # TODO: We might want to make this command less visible once we have proper
@@ -34,77 +36,86 @@ usage() {
 }
 
 case "${1}" in
-    init)
-        "${here}/init.bash"
-        ;;
-    bootstrap)
-        [[ "${2}" =~ ^(wc|sc)$ ]] || usage
-        "${here}/bootstrap.bash" "${2}"
-        ;;
-    apps)
-        [[ "${2}" =~ ^(wc|sc)$ ]] || usage
-        "${here}/apps.bash" "${2}" "${3}"
-        ;;
-    apply)
-        [[ "${2}" =~ ^(wc|sc)$ ]] || usage
-        "${here}/bootstrap.bash" "${2}"
-        "${here}/apps.bash" "${2}" "${3}"
-        ;;
-    test)
-        [[ "${2}" =~ ^(wc|sc)$ ]] || usage
-        "${here}/test.bash" "${2}"
-        ;;
-    dry-run)
-        [[ "${2}" =~ ^(wc|sc)$ ]] || usage
-        "${here}/dry-run.bash" "${2}"
-        ;;
-    team)
-        case "${2}" in
-            add-pgp|remove-pgp)
-                [ -n "${3}" ] || usage
-                "${here}/team.bash" "${2}" "${3}"
-                ;;
-            *) usage ;;
-        esac
-        ;;
-    ops)
-        case "${2}" in
-            kubectl)
-                [[ "${3}" =~ ^(wc|sc)$ ]] || usage
-                shift 2
-                "${here}/ops.bash" kubectl "${@}"
-            ;;
-            helm)
-                [[ "${3}" =~ ^(wc|sc)$ ]] || usage
-                shift 2
-                "${here}/ops.bash" helm "${@}"
-            ;;
-            helmfile)
-                [[ "${3}" =~ ^(wc|sc)$ ]] || usage
-                shift 2
-                "${here}/ops.bash" helmfile "${@}"
-            ;;
-            *) usage ;;
-        esac
+init)
+    "${here}/init.bash"
+    ;;
+bootstrap)
+    [[ "${2}" =~ ^(wc|sc)$ ]] || usage
+    "${here}/bootstrap.bash" "${2}"
+    ;;
+apps)
+    [[ "${2}" =~ ^(wc|sc)$ ]] || usage
+    "${here}/apps.bash" "${2}" "${3}"
+    ;;
+apply)
+    [[ "${2}" =~ ^(wc|sc)$ ]] || usage
+    "${here}/bootstrap.bash" "${2}"
+    "${here}/apps.bash" "${2}" "${3}"
+    ;;
+apply-sync)
+    [[ "${2}" =~ ^(wc|sc)$ ]] || usage
+    "${here}/bootstrap.bash" "${2}"
+    "${here}/apps.bash" "${2}" "${3}" "sync"
     ;;
-    s3cmd)
-        shift
-        : "${secrets[s3cfg_file]:?Missing s3cfg file}"
-        sops_exec_file "${secrets[s3cfg_file]}" 's3cmd --config="{}" '"${*}"
+test)
+    [[ "${2}" =~ ^(wc|sc)$ ]] || usage
+    "${here}/test.bash" "${2}"
     ;;
-    kubeconfig)
-        [[ "${2}" =~ ^(user|admin)$ ]] || usage
-        shift
-        "${here}/kubeconfig.bash" "${@}"
+dry-run)
+    [[ "${2}" =~ ^(wc|sc)$ ]] || usage
+    "${here}/dry-run.bash" "${2}"
+    ;;
+dry-run-sync)
+    [[ "${2}" =~ ^(wc|sc)$ ]] || usage
+    "${here}/dry-run.bash" "${2}" "sync"
+    ;;
+team)
+    case "${2}" in
+    add-pgp | remove-pgp)
+        [ -n "${3}" ] || usage
+        "${here}/team.bash" "${2}" "${3}"
+        ;;
+    *) usage ;;
+    esac
+    ;;
+ops)
+    case "${2}" in
+    kubectl)
+        [[ "${3}" =~ ^(wc|sc)$ ]] || usage
+        shift 2
+        "${here}/ops.bash" kubectl "${@}"
         ;;
-    completion)
-        [ -f "${here}/../completion/${2}" ] || usage
-        cat "${here}/../completion/${2}"
+    helm)
+        [[ "${3}" =~ ^(wc|sc)$ ]] || usage
+        shift 2
+        "${here}/ops.bash" helm "${@}"
         ;;
-    validate)
-        [[ "${2}" =~ ^(wc|sc)$ ]] || usage
-        config_load "$2"
-        echo "Config validation successful"
+    helmfile)
+        [[ "${3}" =~ ^(wc|sc)$ ]] || usage
+        shift 2
+        "${here}/ops.bash" helmfile "${@}"
         ;;
     *) usage ;;
+    esac
+    ;;
+s3cmd)
+    shift
+    : "${secrets[s3cfg_file]:?Missing s3cfg file}"
+    sops_exec_file "${secrets[s3cfg_file]}" 's3cmd --config="{}" '"${*}"
+    ;;
+kubeconfig)
+    [[ "${2}" =~ ^(user|admin)$ ]] || usage
+    shift
+    "${here}/kubeconfig.bash" "${@}"
+    ;;
+completion)
+    [ -f "${here}/../completion/${2}" ] || usage
+    cat "${here}/../completion/${2}"
+    ;;
+validate)
+    [[ "${2}" =~ ^(wc|sc)$ ]] || usage
+    config_load "$2"
+    echo "Config validation successful"
+    ;;
+*) usage ;;
 esac
diff --git a/bin/dry-run.bash b/bin/dry-run.bash
@@ -21,14 +21,28 @@ if [[ $1 == "sc" ]]; then
   log_info "Running helmfile diff on the service cluster"
   : "${config[config_file_sc]:?Missing service cluster config file}"
 
-  "${here}/ops.bash" helmfile sc diff
+  if [ ${#} -eq 2 ]; then
 
-elif [[ $1 == "wc" ]]; then
+    "${here}/ops.bash" helmfile sc template | "${here}/ops.bash" kubectl sc diff -f -
+  else
+    "${here}/ops.bash" helmfile sc diff
+
+  fi
+elif
+  [[ $1 == "wc" ]]
+then
   log_info "Running helmfile diff on the workload cluster"
   : "${config[config_file_wc]:?Missing workload cluster config file}"
 
-  "${here}/ops.bash" helmfile wc diff
+  if [ ${#} -eq 2 ]; then
+    "${here}/ops.bash" helmfile wc template | "${here}/ops.bash" kubectl wc diff -f -
+  else
+    "${here}/ops.bash" helmfile wc diff
+
+  fi
+
 else
+
   log_error "ERROR: unsupported option for dry-run. Supported options are <wc|sc>"
   exit 1
 fi
diff --git a/scripts/deploy-sc.sh b/scripts/deploy-sc.sh
@@ -10,30 +10,34 @@ source "${SCRIPTS_PATH}/../bin/common.bash"
 : "${secrets[secrets_file]:?Missing secrets}"
 
 alertTo=$(yq r -e "${config[config_file_sc]}" 'alerts.alertTo')
-if [[ "$alertTo" != "slack" && "$alertTo" != "null" && "$alertTo" != "opsgenie" ]]
-then
+if [[ "$alertTo" != "slack" && "$alertTo" != "null" && "$alertTo" != "opsgenie" ]]; then
     log_error "ERROR: alerts.alertTo must be set to one of slack, opsgenie or null."
     exit 1
 fi
 
-
 INTERACTIVE=${1:-""}
 
 objectStoreProvider=$(yq r -e "${config[config_file_sc]}" objectStorage.type)
 if [[ ${objectStoreProvider} == "s3" ]]; then
-  echo "Creating fluentd secrets" >&2
-  s3_access_key=$(sops_exec_file "${secrets[secrets_file]}" 'yq r -e {} objectStorage.s3.accessKey')
-  s3_secret_key=$(sops_exec_file "${secrets[secrets_file]}" 'yq r -e {} objectStorage.s3.secretKey')
-  kubectl create secret generic s3-credentials -n fluentd \
-      --from-literal=s3_access_key="${s3_access_key}" \
-      --from-literal=s3_secret_key="${s3_secret_key}" \
-      --dry-run=client -o yaml | kubectl apply -f -
+    echo "Creating fluentd secrets" >&2
+    s3_access_key=$(sops_exec_file "${secrets[secrets_file]}" 'yq r -e {} objectStorage.s3.accessKey')
+    s3_secret_key=$(sops_exec_file "${secrets[secrets_file]}" 'yq r -e {} objectStorage.s3.secretKey')
+    kubectl create secret generic s3-credentials -n fluentd \
+        --from-literal=s3_access_key="${s3_access_key}" \
+        --from-literal=s3_secret_key="${s3_secret_key}" \
+        --dry-run=client -o yaml | kubectl apply -f -
 fi
 
 echo "Installing helm charts" >&2
 cd "${SCRIPTS_PATH}/../helmfile"
 declare -a helmfile_opt_flags
 [[ -n "$INTERACTIVE" ]] && helmfile_opt_flags+=("$INTERACTIVE")
-helmfile -f . -e service_cluster "${helmfile_opt_flags[@]}" apply --suppress-diff
+
+if [ ${#} -eq 1 ]; then
+    helmfile -f . -e service_cluster sync
+else
+    helmfile -f . -e service_cluster apply --suppress-diff
+
+fi
 
 echo "Deploy sc completed!" >&2