apps sc & wc: improve dry-run and deploy scripts

elastisys · Jun 7, 2021 · f21e149 · f21e149
1 parent 417f3c1
commit f21e149
Show file tree

Hide file tree

Showing 7 changed files with 132 additions and 49 deletions.
diff --git a/WIP-CHANGELOG.md b/WIP-CHANGELOG.md
@@ -7,6 +7,7 @@
 
 - The sc-logs-retention cronjob now runs without error even if no backups were found for automatic removal
 - Harbor Swift authentication configuration options has moved from `citycloud` to `harbor.persistence.swift`.
+- The dry-run and apply command now checks against the state of the cluster while ran.
 
 ### Fixed
 

diff --git a/bin/apps.bash b/bin/apps.bash
@@ -23,23 +23,19 @@ apps_init() {
 apps_run_sc() {
     log_info "Applying applications in service cluster"
 
-    (
-        : "${scripts_path:?Missing scripts path}"
-        : "${secrets[kube_config_sc]:?Missing service cluster kubeconfig}"
-        with_kubeconfig "${secrets[kube_config_sc]}" \
-            CONFIG_PATH="${CK8S_CONFIG_PATH}" "${scripts_path}/deploy-sc.sh"
-    )
+    : "${scripts_path:?Missing scripts path}"
+    : "${secrets[kube_config_sc]:?Missing service cluster kubeconfig}"
+    with_kubeconfig "${secrets[kube_config_sc]}" \
+        CONFIG_PATH="${CK8S_CONFIG_PATH}" "${scripts_path}/deploy-sc.sh" "${1:-""}"
 }
 
 apps_run_wc() {
     log_info "Applying applications in workload cluster"
 
-    (
-        : "${scripts_path:?Missing scripts path}"
-        : "${secrets[kube_config_wc]:?Missing workload cluster kubeconfig}"
-        with_kubeconfig "${secrets[kube_config_wc]}" \
-            CONFIG_PATH="${CK8S_CONFIG_PATH}" "${scripts_path}/deploy-wc.sh"
-    )
+    : "${scripts_path:?Missing scripts path}"
+    : "${secrets[kube_config_wc]:?Missing workload cluster kubeconfig}"
+    with_kubeconfig "${secrets[kube_config_wc]}" \
+        CONFIG_PATH="${CK8S_CONFIG_PATH}" "${scripts_path}/deploy-wc.sh" "${1:-""}"
 }
 
 template_validate_sc() {
@@ -75,34 +71,75 @@ apps_sc() {
     # While it would be nice to have some template validation before `helmfile apply`,
     # at least Helmfile does "just in time" template validation. Not as nice,
     # but feels good enough until we figure out something smarter.
-    #
+    #    #[ "$1" != "--skip-template-validate" ] && template_validate_sc
+
     #[ "$1" != "--skip-template-validate" ] && template_validate_sc
-    apps_run_sc
+
+    if [ "$#" -gt 0 ] && [ "${*: -1}" = "sync" ]; then
+        apps_run_sc "${*: -1}"
+    else
+        apps_run_sc
+    fi
 
     log_info "Applications applied successfully!"
+
 }
 
 apps_wc() {
     apps_init
-    # See rationale above.
+    #
+    # The first few Charts install CRDs, which will make template validation
+    # fail. CRDs are "changes" to the Kubernetes API, hence validation against
+    # the Kubernetes API cannot be done. OTOH, manually adding the CRDs during
+    # bootstrap is error-prone and adds maintenance burden.
+    #
+    # While it would be nice to have some template validation before `helmfile apply`,
+    # at least Helmfile does "just in time" template validation. Not as nice,
+    # but feels good enough until we figure out something smarter.
+    #    #[ "$1" != "--skip-template-validate" ] && template_validate_sc
+
     #[ "$1" != "--skip-template-validate" ] && template_validate_wc
-    apps_run_wc
+
+    if [ "$#" -gt 0 ] && [ "${*: -1}" = "sync" ]; then
+        apps_run_wc "${*: -1}"
+    else
+        apps_run_wc
+    fi
 
     log_info "Applications applied successfully!"
 }
 
 #
 # ENTRYPOINT
 #
-
-
 if [[ $1 == "wc" ]]; then
     config_load "$1"
-    apps_wc "$2"
+
+    if [ ${#} -eq 3 ] && [ "$2" = "--skip-template-validate" ] && [ "$3" = "sync" ]; then
+        apps_wc "$2" "$3"
+    else
+        if [ ${#} -eq 2 ]; then
+            apps_wc "$2"
+        else
+            apps_wc
+        fi
+    fi
+
 elif [[ $1 == "sc" ]]; then
+
     config_load "$1"
-    apps_sc "$2"
+
+    if [ ${#} -eq 3 ] && [ "$2" = "--skip-template-validate" ] && [ "$3" = "sync" ]; then
+        apps_sc "$2" "$3"
+    else
+        if [ ${#} -eq 2 ]; then
+            apps_sc "$2"
+        else
+            apps_sc
+        fi
+    fi
 else
+
     echo "ERROR:  [$1] is an invalid argument:"
     echo "usage:   ck8s apps <wc|sc>. "
     exit 1

diff --git a/bin/ck8s b/bin/ck8s
@@ -13,9 +13,9 @@ usage() {
     echo "  init                                            initialize the config path" 1>&2
     echo "  bootstrap <wc|sc>                               bootstrap the cluster" 1>&2
     echo "  apps <wc|sc> [--skip-template-validate]         deploy the applications" 1>&2
-    echo "  apply <wc|sc> [--skip-template-validate]        bootstrap and apps" 1>&2
+    echo "  apply <wc|sc> [--sync] [--skip-template-validate] bootstrap and apps" 1>&2
     echo "  test <wc|sc>                                    test the applications" 1>&2
-    echo "  dry-run <wc|sc>                                 runs helmfile diff" 1>&2
+    echo "  dry-run <wc|sc> [--kubectl]                     runs helmfile diff" 1>&2
     echo "  team add-pgp <fp>                               add a new PGP key to secrets" 1>&2
     echo "  team remove-pgp <fp>                            remove a PGP key from secrets and rotate the data encryption key" 1>&2
     # TODO: We might want to make this command less visible once we have proper
@@ -33,6 +33,19 @@ usage() {
     exit 1
 }
 
+
+SYNC=""
+SKIP=""
+KUBECTL=""
+
+for arg in "$@"; do
+  case "$arg" in
+    "--skip-template-validate") SKIP="--skip-template-validate" ;;
+     "--sync") SYNC="sync" ;;
+      "--kubectl") KUBECTL="kubectl" ;;
+esac
+done
+
 case "${1}" in
     init)
         "${here}/init.bash"
@@ -43,21 +56,32 @@ case "${1}" in
         ;;
     apps)
         [[ "${2}" =~ ^(wc|sc)$ ]] || usage
-        "${here}/apps.bash" "${2}" "${3}"
+        "${here}/apps.bash" "${2}" $SKIP
         ;;
     apply)
-        [[ "${2}" =~ ^(wc|sc)$ ]] || usage
-        "${here}/bootstrap.bash" "${2}"
-        "${here}/apps.bash" "${2}" "${3}"
-        ;;
+
+    [[ "${2}" =~ ^(wc|sc)$ ]] || usage
+
+    "${here}/bootstrap.bash" "${2}"
+    "${here}/apps.bash" "${2}" $SKIP $SYNC
+
+    ;;
     test)
         [[ "${2}" =~ ^(wc|sc)$ ]] || usage
         "${here}/test.bash" "${2}"
         ;;
     dry-run)
-        [[ "${2}" =~ ^(wc|sc)$ ]] || usage
-        "${here}/dry-run.bash" "${2}"
-        ;;
+    [[ "${2}" =~ ^(wc|sc)$ ]] || usage
+
+   # if [ ${#} -eq 2 ]; then
+        "${here}/dry-run.bash" "${2}" $KUBECTL
+   # else
+
+    #    if [[ "${#}" -eq 3 ]] && [[ "${3}" = "--kubectl" ]] || usage; then
+   #         "${here}/dry-run.bash" "${2}" "kubectl"
+    #    fi
+   #fi
+    ;;
     team)
         case "${2}" in
             add-pgp|remove-pgp)

diff --git a/bin/dry-run.bash b/bin/dry-run.bash
@@ -21,14 +21,26 @@ if [[ $1 == "sc" ]]; then
   log_info "Running helmfile diff on the service cluster"
   : "${config[config_file_sc]:?Missing service cluster config file}"
 
-  "${here}/ops.bash" helmfile sc diff
-
-elif [[ $1 == "wc" ]]; then
+  if [ ${#} -eq 2 ] && [ "$2" = "kubectl" ]; then
+    "${here}/ops.bash" helmfile sc template | "${here}/ops.bash" kubectl sc diff -f -
+  else
+    "${here}/ops.bash" helmfile sc diff
+
+  fi
+elif
+  [[ $1 == "wc" ]]; then
   log_info "Running helmfile diff on the workload cluster"
   : "${config[config_file_wc]:?Missing workload cluster config file}"
 
-  "${here}/ops.bash" helmfile wc diff
+  if [ ${#} -eq 2 ] && [ "$2" = "kubectl" ]; then
+    "${here}/ops.bash" helmfile wc template | "${here}/ops.bash" kubectl wc diff -f -
+  else
+    "${here}/ops.bash" helmfile wc diff
+
+  fi
+
 else
+
   log_error "ERROR: unsupported option for dry-run. Supported options are <wc|sc>"
   exit 1
 fi
diff --git a/scripts/clean-wc.sh b/scripts/clean-wc.sh
@@ -57,7 +57,7 @@ PROM_CRDS=$(
         kubectl wc api-resources \
         --api-group=monitoring.coreos.com \
         -o name
-    )
+)
 if [ -n "$PROM_CRDS" ]; then
     # shellcheck disable=SC2086
     # We definitely want word splitting here.

diff --git a/scripts/deploy-sc.sh b/scripts/deploy-sc.sh
@@ -10,30 +10,34 @@ source "${SCRIPTS_PATH}/../bin/common.bash"
 : "${secrets[secrets_file]:?Missing secrets}"
 
 alertTo=$(yq r -e "${config[config_file_sc]}" 'alerts.alertTo')
-if [[ "$alertTo" != "slack" && "$alertTo" != "null" && "$alertTo" != "opsgenie" ]]
-then
+if [[ "$alertTo" != "slack" && "$alertTo" != "null" && "$alertTo" != "opsgenie" ]]; then
     log_error "ERROR: alerts.alertTo must be set to one of slack, opsgenie or null."
     exit 1
 fi
 
-
 INTERACTIVE=${1:-""}
 
 objectStoreProvider=$(yq r -e "${config[config_file_sc]}" objectStorage.type)
 if [[ ${objectStoreProvider} == "s3" ]]; then
-  echo "Creating fluentd secrets" >&2
-  s3_access_key=$(sops_exec_file "${secrets[secrets_file]}" 'yq r -e {} objectStorage.s3.accessKey')
-  s3_secret_key=$(sops_exec_file "${secrets[secrets_file]}" 'yq r -e {} objectStorage.s3.secretKey')
-  kubectl create secret generic s3-credentials -n fluentd \
-      --from-literal=s3_access_key="${s3_access_key}" \
-      --from-literal=s3_secret_key="${s3_secret_key}" \
-      --dry-run=client -o yaml | kubectl apply -f -
+    echo "Creating fluentd secrets" >&2
+    s3_access_key=$(sops_exec_file "${secrets[secrets_file]}" 'yq r -e {} objectStorage.s3.accessKey')
+    s3_secret_key=$(sops_exec_file "${secrets[secrets_file]}" 'yq r -e {} objectStorage.s3.secretKey')
+    kubectl create secret generic s3-credentials -n fluentd \
+        --from-literal=s3_access_key="${s3_access_key}" \
+        --from-literal=s3_secret_key="${s3_secret_key}" \
+        --dry-run=client -o yaml | kubectl apply -f -
 fi
 
 echo "Installing helm charts" >&2
 cd "${SCRIPTS_PATH}/../helmfile"
 declare -a helmfile_opt_flags
 [[ -n "$INTERACTIVE" ]] && helmfile_opt_flags+=("$INTERACTIVE")
-helmfile -f . -e service_cluster "${helmfile_opt_flags[@]}" apply --suppress-diff
+
+if [ ${#} -eq 1 ] && [ "$1" = "sync" ]; then
+    helmfile -f . -e service_cluster sync
+else
+    helmfile -f . -e service_cluster apply --suppress-diff
+
+fi
 
 echo "Deploy sc completed!" >&2
diff --git a/scripts/deploy-wc.sh b/scripts/deploy-wc.sh
@@ -26,14 +26,19 @@ kubectl -n fluentd create secret generic elasticsearch \
 # We use `create` here instead of `apply` to avoid overwriting any changes the
 # user may have done.
 kubectl create -f "${SCRIPTS_PATH}/../manifests/examples/fluentd/fluentd-extra-config.yaml" \
-    2> /dev/null || echo "fluentd-extra-config configmap already in place. Ignoring."
+    2>/dev/null || echo "fluentd-extra-config configmap already in place. Ignoring."
 kubectl create -f "${SCRIPTS_PATH}/../manifests/examples/fluentd/fluentd-extra-plugins.yaml" \
-    2> /dev/null || echo "fluentd-extra-plugins configmap already in place. Ignoring." >&2
+    2>/dev/null || echo "fluentd-extra-plugins configmap already in place. Ignoring." >&2
 
 echo "Installing helm charts" >&2
 cd "${SCRIPTS_PATH}/../helmfile"
 declare -a helmfile_opt_flags
 [[ -n "$INTERACTIVE" ]] && helmfile_opt_flags+=("$INTERACTIVE")
-helmfile -f . -e workload_cluster "${helmfile_opt_flags[@]}" apply --suppress-diff
+
+if [ ${#} -eq 1 ] && [ "$1" = "sync" ]; then
+    helmfile -f . -e workload_cluster sync
+else
+    helmfile -f . -e workload_cluster apply --suppress-diff
+fi
 
 echo "Deploy wc completed!" >&2