From 14aad4d6f576d13329379c5966b459df87a29a49 Mon Sep 17 00:00:00 2001 From: Steven Shi Date: Tue, 24 Nov 2020 22:32:18 -0800 Subject: [PATCH 01/11] add dev environment - initial commit --- templates/main.go | 1 + templates/skaffold.yaml | 19 ++++++ templates/start-dev-env.sh | 121 +++++++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 templates/skaffold.yaml create mode 100755 templates/start-dev-env.sh diff --git a/templates/main.go b/templates/main.go index 8c2fae7..d0a3d14 100644 --- a/templates/main.go +++ b/templates/main.go @@ -41,6 +41,7 @@ func main() { r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.Path)) + log.Printf("Hello, %q", html.EscapeString(r.URL.Path)) }) serverAddress := fmt.Sprintf("0.0.0.0:%s", os.Getenv("SERVER_PORT")) diff --git a/templates/skaffold.yaml b/templates/skaffold.yaml new file mode 100644 index 0000000..188f8ee --- /dev/null +++ b/templates/skaffold.yaml @@ -0,0 +1,19 @@ +apiVersion: skaffold/v2beta9 +kind: Config +metadata: + name: backend-service +build: + tagPolicy: + sha256: {} + artifacts: + - image: <% index .Params `accountId` %>.dkr.ecr.<% index .Params `region` %>.amazonaws.com/<% .Name %> +deploy: + kustomize: + paths: + - kubernetes.tmp/base +profiles: +- name: staging + deploy: + kustomize: + paths: + - kubernetes.tmp/overlays/staging diff --git a/templates/start-dev-env.sh b/templates/start-dev-env.sh new file mode 100755 index 0000000..bf271a1 --- /dev/null +++ b/templates/start-dev-env.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +# +# This script is to create a dev namespace on Staging environment +# +PROJECT_NAME=<% .Name %> +ENVIRONMENT=stage + +# common functions +function usage() { + echo + echo "Usage:" + echo " $0 " + echo " - IAM user: can be found by running 'aws iam get-group --group-name ${PROJECT_NAME}-developer-${ENVIRONMENT} | jq -r .Users[].UserName'" + exit 1 +} + +function command_exist() { + command -v ${1} >& /dev/null +} + +function error_exit() { + echo "ERROR : $1" + exit 2 +} + +function can_i() { + command=$1 + kubectl auth can-i $1 >& /dev/null || error_exit "No permission to $1" +} + +# Start +USER_NAME=$1 +DEV_PROJECT_ID=$2 +[[ -z "$USER_NAME" ]] && usage +[[ -z "$DEV_PROJECT_ID" ]] && DEV_PROJECT_ID="001" + +# Check installation: skaffold & telepresence +if ! command_exist skaffold || ! command_exist telepresence; then + if ! command_exist skaffold; then + error_exit "command 'skaffold' not found: please visit https://skaffold.dev/docs/install/" + fi + if ! command_exist kubectl; then + error_exit "command 'telepresence' not found. You can download it at https://www.telepresence.io/reference/install." + fi +fi + +# Validate input +NAMESPACE=${PROJECT_NAME} +SECRET_NAME=${PROJECT_NAME} +DEV_SECRET_NAME=dev-${USER_NAME} +DEV_SECRET_JSON=$(kubectl get secret ${DEV_SECRET_NAME} -n ${NAMESPACE} -o json) +[[ -z "${DEV_SECRET_JSON}" ]] && error_exit "The secret ${DEV_SECRET_NAME} is not existing. Please check secrets with 'kubectl get secrets'" + +# Setup dev namepsace +DEV_USER_PROJECT=${USER_NAME}-${DEV_PROJECT_ID} +DEV_NAMESPACE=${DEV_USER_PROJECT} +kubectl get namespace ${DEV_NAMESPACE} >& /dev/null || \ + (can_i "create namespace" && kubectl create namespace ${DEV_NAMESPACE} && \ + can_i "create deployment -n ${DEV_NAMESPACE}" && \ + can_i "create ingress -n ${DEV_NAMESPACE}" && \ + can_i "create service -n ${DEV_NAMESPACE}" && \ + can_i "create configmaps -n ${DEV_NAMESPACE}") +echo "Namespace: ${DEV_NAMESPACE}" + +# Setup dev secret from pre-configed one +kubectl get secret ${SECRET_NAME} -n ${DEV_NAMESPACE} >& /dev/null || \ + echo ${DEV_SECRET_JSON} | jq 'del(.metadata["namespace","creationTimestamp","resourceVersion","selfLink","uid"])' | sed "s/${DEV_SECRET_NAME}/${SECRET_NAME}/g" | kubectl apply -n ${DEV_NAMESPACE} -f - +echo "Secret: ${SECRET_NAME}" + +# Setup dev service account from pre-configured one +SERVICE_ACCOUNT=backend-service +kubectl get sa ${SERVICE_ACCOUNT} -n ${DEV_NAMESPACE} >& /dev/null || \ + kubectl get sa ${SERVICE_ACCOUNT} -n ${NAMESPACE} -o json | jq 'del(.metadata["namespace","creationTimestamp","resourceVersion","selfLink","uid"])' | kubectl apply -n ${DEV_NAMESPACE} -f - +echo "Service Account: ${SERVICE_ACCOUNT}" + +# Prepare varaibles +ACCOUNT_ID=<% index .Params `accountId` %> +REGION=<% index .Params `region` %> +CLUSTER=${PROJECT_NAME}-stage-${REGION} +DEPLOYMENT=${PROJECT_NAME} +REPO=${PROJECT_NAME} +EXT_HOSTNAME=<% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %> +MY_EXT_HOSTNAME=${DEV_USER_PROJECT}-${EXT_HOSTNAME} +ECR_REPO=${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${REPO} +VERSION_TAG=lastest-${USER_NAME} +DATABASE_NAME=<% index .Params `databaseName` %> +DEV_DATABASE_NAME=$(echo "dev-${USER_NAME}" | tr -dc 'A-Za-z0-9') + +CONFIG_ENVIRONMENT="staging" + +# Setup dev k8s manifests +cp -pr kubernetes kubernetes.tmp +echo "start manifest changes" +## image: set image definition +sed -i ".bak" "s|image: fake-image|image: ${ECR_REPO}:${VERSION_TAG}|g" kubernetes.tmp/base/deployment.yml +## ingress: create new domain with external DNS +sed -i ".bak" "s|${EXT_HOSTNAME}|${MY_EXT_HOSTNAME}|g" kubernetes.tmp/overlays/${CONFIG_ENVIRONMENT}/ingress.yml +## database_name: replace with new one +sed -i ".bak" "s|DATABASE_NAME=${DATABASE_NAME}|DATABASE_NAME=${DEV_DATABASE_NAME}|g" kubernetes.tmp/base/kustomization.yml + +# Build & Deploy dev environment with Skaffold +echo +echo "Check backend-service at http://${MY_EXT_HOSTNAME}/" +echo +echo "Intercept the traffic to local by running:" +echo " telepresence --swap-deployment ${DEPLOYMENT} --namespace ${DEV_NAMESPACE} --expose 80 --run go run main.go" +echo " or running in docker" +echo " docker build . -t ${DEPLOYMENT}-dev" +echo " telepresence --swap-deployment ${DEPLOYMENT} --namespace ${DEV_NAMESPACE} --expose 80 --docker-run --rm -it -v $(pwd):/usr/src/app ${DEPLOYMENT}-dev" +echo + +skaffold config set default-repo ${ECR_REPO} +aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${ECR_REPO} >& /dev/null || error_exit "Failed to login to AWS ECR" +## run in backgound +skaffold run --profile ${CONFIG_ENVIRONMENT} --namespace ${DEV_NAMESPACE} +## run in frontend +#skaffold dev --profile ${CONFIG_ENVIRONMENT} --namespace ${DEV_NAMESPACE} + +# Clean up +rm -rf kubernetes.tmp From f48da02722bc5c76fdc7ed57dae1cd8fe65d6bfe Mon Sep 17 00:00:00 2001 From: Steven Shi Date: Wed, 25 Nov 2020 12:58:07 -0800 Subject: [PATCH 02/11] more enhancement... --- templates/start-dev-env.sh | 90 +++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 39 deletions(-) diff --git a/templates/start-dev-env.sh b/templates/start-dev-env.sh index bf271a1..88a82a6 100755 --- a/templates/start-dev-env.sh +++ b/templates/start-dev-env.sh @@ -5,6 +5,8 @@ # PROJECT_NAME=<% .Name %> ENVIRONMENT=stage +ACCOUNT_ID=<% index .Params `accountId` %> +REGION=<% index .Params `region` %> # common functions function usage() { @@ -12,6 +14,7 @@ function usage() { echo "Usage:" echo " $0 " echo " - IAM user: can be found by running 'aws iam get-group --group-name ${PROJECT_NAME}-developer-${ENVIRONMENT} | jq -r .Users[].UserName'" + echo " - project id: can be 001, 002, or whatever id without space" exit 1 } @@ -35,7 +38,21 @@ DEV_PROJECT_ID=$2 [[ -z "$USER_NAME" ]] && usage [[ -z "$DEV_PROJECT_ID" ]] && DEV_PROJECT_ID="001" -# Check installation: skaffold & telepresence +echo '[Dev Environment]' + +# Validate cluster +CLUSTER=$(kubectl config current-context) +[[ ${CLUSTER} == "${PROJECT_NAME}-${ENVIRONMENT}-${REGION}" ]] || error_exit "Your kubernetes context ${CLUSTER} is not proper to run this script" +echo " Cluster: ${CLUSTER}" + +# Validate secret +NAMESPACE=${PROJECT_NAME} +SECRET_NAME=${PROJECT_NAME} +DEV_SECRET_NAME=dev-${USER_NAME} +DEV_SECRET_JSON=$(kubectl get secret ${DEV_SECRET_NAME} -n ${NAMESPACE} -o json) +[[ -z "${DEV_SECRET_JSON}" ]] && error_exit "The secret ${DEV_SECRET_NAME} is not existing. Please check secrets with 'kubectl get secrets'" + +# Check installations if ! command_exist skaffold || ! command_exist telepresence; then if ! command_exist skaffold; then error_exit "command 'skaffold' not found: please visit https://skaffold.dev/docs/install/" @@ -45,13 +62,6 @@ if ! command_exist skaffold || ! command_exist telepresence; then fi fi -# Validate input -NAMESPACE=${PROJECT_NAME} -SECRET_NAME=${PROJECT_NAME} -DEV_SECRET_NAME=dev-${USER_NAME} -DEV_SECRET_JSON=$(kubectl get secret ${DEV_SECRET_NAME} -n ${NAMESPACE} -o json) -[[ -z "${DEV_SECRET_JSON}" ]] && error_exit "The secret ${DEV_SECRET_NAME} is not existing. Please check secrets with 'kubectl get secrets'" - # Setup dev namepsace DEV_USER_PROJECT=${USER_NAME}-${DEV_PROJECT_ID} DEV_NAMESPACE=${DEV_USER_PROJECT} @@ -60,24 +70,22 @@ kubectl get namespace ${DEV_NAMESPACE} >& /dev/null || \ can_i "create deployment -n ${DEV_NAMESPACE}" && \ can_i "create ingress -n ${DEV_NAMESPACE}" && \ can_i "create service -n ${DEV_NAMESPACE}" && \ - can_i "create configmaps -n ${DEV_NAMESPACE}") -echo "Namespace: ${DEV_NAMESPACE}" + can_i "create secret -n ${DEV_NAMESPACE}" && \ + can_i "create configmap -n ${DEV_NAMESPACE}") +echo " Namespace: ${DEV_NAMESPACE}" # Setup dev secret from pre-configed one kubectl get secret ${SECRET_NAME} -n ${DEV_NAMESPACE} >& /dev/null || \ echo ${DEV_SECRET_JSON} | jq 'del(.metadata["namespace","creationTimestamp","resourceVersion","selfLink","uid"])' | sed "s/${DEV_SECRET_NAME}/${SECRET_NAME}/g" | kubectl apply -n ${DEV_NAMESPACE} -f - -echo "Secret: ${SECRET_NAME}" +echo " Secret: ${SECRET_NAME}" # Setup dev service account from pre-configured one SERVICE_ACCOUNT=backend-service kubectl get sa ${SERVICE_ACCOUNT} -n ${DEV_NAMESPACE} >& /dev/null || \ kubectl get sa ${SERVICE_ACCOUNT} -n ${NAMESPACE} -o json | jq 'del(.metadata["namespace","creationTimestamp","resourceVersion","selfLink","uid"])' | kubectl apply -n ${DEV_NAMESPACE} -f - -echo "Service Account: ${SERVICE_ACCOUNT}" -# Prepare varaibles -ACCOUNT_ID=<% index .Params `accountId` %> -REGION=<% index .Params `region` %> -CLUSTER=${PROJECT_NAME}-stage-${REGION} +# Setup dev k8s manifests, configuration, docker login etc +CONFIG_ENVIRONMENT="staging" DEPLOYMENT=${PROJECT_NAME} REPO=${PROJECT_NAME} EXT_HOSTNAME=<% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %> @@ -86,36 +94,40 @@ ECR_REPO=${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${REPO} VERSION_TAG=lastest-${USER_NAME} DATABASE_NAME=<% index .Params `databaseName` %> DEV_DATABASE_NAME=$(echo "dev-${USER_NAME}" | tr -dc 'A-Za-z0-9') - -CONFIG_ENVIRONMENT="staging" - -# Setup dev k8s manifests -cp -pr kubernetes kubernetes.tmp -echo "start manifest changes" +rm -rf kubernetes.tmp && cp -pr kubernetes kubernetes.tmp ## image: set image definition sed -i ".bak" "s|image: fake-image|image: ${ECR_REPO}:${VERSION_TAG}|g" kubernetes.tmp/base/deployment.yml ## ingress: create new domain with external DNS sed -i ".bak" "s|${EXT_HOSTNAME}|${MY_EXT_HOSTNAME}|g" kubernetes.tmp/overlays/${CONFIG_ENVIRONMENT}/ingress.yml ## database_name: replace with new one sed -i ".bak" "s|DATABASE_NAME=${DATABASE_NAME}|DATABASE_NAME=${DEV_DATABASE_NAME}|g" kubernetes.tmp/base/kustomization.yml +skaffold config set default-repo ${ECR_REPO} >& /dev/null +aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${ECR_REPO} >& /dev/null || error_exit "Failed to login to AWS ECR" +echo " Domain: ${MY_EXT_HOSTNAME}" +echo " Database Name: ${DEV_DATABASE_NAME}" -# Build & Deploy dev environment with Skaffold echo -echo "Check backend-service at http://${MY_EXT_HOSTNAME}/" echo -echo "Intercept the traffic to local by running:" -echo " telepresence --swap-deployment ${DEPLOYMENT} --namespace ${DEV_NAMESPACE} --expose 80 --run go run main.go" -echo " or running in docker" -echo " docker build . -t ${DEPLOYMENT}-dev" -echo " telepresence --swap-deployment ${DEPLOYMENT} --namespace ${DEV_NAMESPACE} --expose 80 --docker-run --rm -it -v $(pwd):/usr/src/app ${DEPLOYMENT}-dev" +echo "Now, you are ready to launch your backend service on Staging under your dev environment/namepsace '${DEV_NAMESPACE}'. This will allow you to debug your cloud backend service on the fly with your local laptop and daily favourite tools like vscode etc." +echo +echo " Step 1: launch your backend service, run the followings in a terminal:" +echo " > skaffold run --profile ${CONFIG_ENVIRONMENT} --namespace ${DEV_NAMESPACE}" +echo " or run in frontend:" +echo " > skaffold dev --profile ${CONFIG_ENVIRONMENT} --namespace ${DEV_NAMESPACE}" +echo +echo " to confirm the result:" +echo " > kubectl -n ${DEV_NAMESPACE} get pods # see pods running" +echo " > curl http://${MY_EXT_HOSTNAME}/ # see Hello" +echo +echo " Step 2: intercept the request to local, run the followings in a terminal:" +echo " > docker build . -t ${DEPLOYMENT}-dev" +echo " > telepresence --swap-deployment ${DEPLOYMENT} --namespace ${DEV_NAMESPACE} --expose 80 --docker-run --rm -it -v $(pwd):/usr/src/app ${DEPLOYMENT}-dev" +echo " to confirm the result (in another terminal):" +echo " > curl http://${MY_EXT_HOSTNAME}/ # see Hello in both terminals" +echo +echo "Then, you can change the code in your editor now, and you will see the live traffic from the browser redirected to your terminal immediately. After confirmed the changes running on Staging, you may just submit it to git repo which will trigger CI/CD workflow as before." +echo +echo "For more detailed information, please refer to:" +echo " telepresence: https://www.telepresence.io/ " +echo " skaffold: https://skaffold.dev/" echo - -skaffold config set default-repo ${ECR_REPO} -aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${ECR_REPO} >& /dev/null || error_exit "Failed to login to AWS ECR" -## run in backgound -skaffold run --profile ${CONFIG_ENVIRONMENT} --namespace ${DEV_NAMESPACE} -## run in frontend -#skaffold dev --profile ${CONFIG_ENVIRONMENT} --namespace ${DEV_NAMESPACE} - -# Clean up -rm -rf kubernetes.tmp From d1be0fe1320a06543e154ba3f7e6e1de0f3607ee Mon Sep 17 00:00:00 2001 From: Steven Shi Date: Mon, 30 Nov 2020 16:10:53 -0800 Subject: [PATCH 03/11] enhancement with kustomize(replace skaffold) and TP shell, etc. --- templates/skaffold.yaml | 19 ------ templates/start-dev-env.sh | 119 ++++++++++++++++--------------------- 2 files changed, 51 insertions(+), 87 deletions(-) delete mode 100644 templates/skaffold.yaml diff --git a/templates/skaffold.yaml b/templates/skaffold.yaml deleted file mode 100644 index 188f8ee..0000000 --- a/templates/skaffold.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: skaffold/v2beta9 -kind: Config -metadata: - name: backend-service -build: - tagPolicy: - sha256: {} - artifacts: - - image: <% index .Params `accountId` %>.dkr.ecr.<% index .Params `region` %>.amazonaws.com/<% .Name %> -deploy: - kustomize: - paths: - - kubernetes.tmp/base -profiles: -- name: staging - deploy: - kustomize: - paths: - - kubernetes.tmp/overlays/staging diff --git a/templates/start-dev-env.sh b/templates/start-dev-env.sh index 88a82a6..75478c4 100755 --- a/templates/start-dev-env.sh +++ b/templates/start-dev-env.sh @@ -12,8 +12,7 @@ REGION=<% index .Params `region` %> function usage() { echo echo "Usage:" - echo " $0 " - echo " - IAM user: can be found by running 'aws iam get-group --group-name ${PROJECT_NAME}-developer-${ENVIRONMENT} | jq -r .Users[].UserName'" + echo " $0 " echo " - project id: can be 001, 002, or whatever id without space" exit 1 } @@ -28,106 +27,90 @@ function error_exit() { } function can_i() { - command=$1 - kubectl auth can-i $1 >& /dev/null || error_exit "No permission to $1" + commands=$1 + IFS=',' read -r -a array <<< "$commands" + err=0 + for command in "${array[@]}" + do + kubectl --context ${CLUSTER_CONTEXT} auth can-i $command >& /dev/null || (echo "No permission to '$command'" && let "err+=1") + done + + [[ $err -gt 0 ]] && error_exit "Found $err permission errors. Please check with your administrator." } # Start -USER_NAME=$1 -DEV_PROJECT_ID=$2 -[[ -z "$USER_NAME" ]] && usage -[[ -z "$DEV_PROJECT_ID" ]] && DEV_PROJECT_ID="001" +# Validate current iam user +MY_USERNAME=$(aws sts get-caller-identity --output json | jq -r .Arn | cut -d/ -f2) +DEV_USERS=$(aws iam get-group --group-name ${PROJECT_NAME}-developer-${ENVIRONMENT} | jq -r .Users[].UserName) +[[ "${DEV_USERS[@]}" =~ "${MY_USERNAME}" ]] || error_exit "You (${MY_USERNAME}) are not in ${DEV_USERS}" + +DEV_PROJECT_ID=${1:-"001"} echo '[Dev Environment]' # Validate cluster -CLUSTER=$(kubectl config current-context) -[[ ${CLUSTER} == "${PROJECT_NAME}-${ENVIRONMENT}-${REGION}" ]] || error_exit "Your kubernetes context ${CLUSTER} is not proper to run this script" -echo " Cluster: ${CLUSTER}" +CLUSTER_CONTEXT=${PROJECT_NAME}-${ENVIRONMENT}-${REGION} +echo " Cluster context: ${CLUSTER_CONTEXT}" # Validate secret NAMESPACE=${PROJECT_NAME} SECRET_NAME=${PROJECT_NAME} -DEV_SECRET_NAME=dev-${USER_NAME} -DEV_SECRET_JSON=$(kubectl get secret ${DEV_SECRET_NAME} -n ${NAMESPACE} -o json) -[[ -z "${DEV_SECRET_JSON}" ]] && error_exit "The secret ${DEV_SECRET_NAME} is not existing. Please check secrets with 'kubectl get secrets'" +DEV_SECRET_NAME=devenv${PROJECT_NAME} +DEV_SECRET_JSON=$(kubectl --context ${CLUSTER_CONTEXT} get secret ${DEV_SECRET_NAME} -n ${NAMESPACE} -o json) +[[ -z "${DEV_SECRET_JSON}" ]] && error_exit "The secret ${DEV_SECRET_NAME} is not existing in namespace '${NAMESPACE}'." # Check installations -if ! command_exist skaffold || ! command_exist telepresence; then - if ! command_exist skaffold; then - error_exit "command 'skaffold' not found: please visit https://skaffold.dev/docs/install/" +if ! command_exist kustomize || ! command_exist telepresence; then + if ! command_exist kustomize; then + error_exit "command 'kustomize' not found: please visit https://kubectl.docs.kubernetes.io/installation/kustomize/" fi if ! command_exist kubectl; then - error_exit "command 'telepresence' not found. You can download it at https://www.telepresence.io/reference/install." + error_exit "command 'telepresence' not found. You can download it at https://www.telepresence.io/reference/install" fi fi # Setup dev namepsace -DEV_USER_PROJECT=${USER_NAME}-${DEV_PROJECT_ID} -DEV_NAMESPACE=${DEV_USER_PROJECT} -kubectl get namespace ${DEV_NAMESPACE} >& /dev/null || \ - (can_i "create namespace" && kubectl create namespace ${DEV_NAMESPACE} && \ - can_i "create deployment -n ${DEV_NAMESPACE}" && \ - can_i "create ingress -n ${DEV_NAMESPACE}" && \ - can_i "create service -n ${DEV_NAMESPACE}" && \ - can_i "create secret -n ${DEV_NAMESPACE}" && \ - can_i "create configmap -n ${DEV_NAMESPACE}") +DEV_NAMESPACE=${MY_USERNAME}-${DEV_PROJECT_ID} +kubectl --context ${CLUSTER_CONTEXT} get namespace ${DEV_NAMESPACE} >& /dev/null || \ + (can_i "create namespace,create deployment,create ingress,create service,create secret,create configmap" && \ + kubectl --context ${CLUSTER_CONTEXT} create namespace ${DEV_NAMESPACE}) echo " Namespace: ${DEV_NAMESPACE}" # Setup dev secret from pre-configed one -kubectl get secret ${SECRET_NAME} -n ${DEV_NAMESPACE} >& /dev/null || \ - echo ${DEV_SECRET_JSON} | jq 'del(.metadata["namespace","creationTimestamp","resourceVersion","selfLink","uid"])' | sed "s/${DEV_SECRET_NAME}/${SECRET_NAME}/g" | kubectl apply -n ${DEV_NAMESPACE} -f - +kubectl --context ${CLUSTER_CONTEXT} get secret ${SECRET_NAME} -n ${DEV_NAMESPACE} >& /dev/null || \ + echo ${DEV_SECRET_JSON} | jq 'del(.metadata["namespace","creationTimestamp","resourceVersion","selfLink","uid"])' | sed "s/${DEV_SECRET_NAME}/${SECRET_NAME}/g" | kubectl --context ${CLUSTER_CONTEXT} apply -n ${DEV_NAMESPACE} -f - echo " Secret: ${SECRET_NAME}" # Setup dev service account from pre-configured one SERVICE_ACCOUNT=backend-service -kubectl get sa ${SERVICE_ACCOUNT} -n ${DEV_NAMESPACE} >& /dev/null || \ - kubectl get sa ${SERVICE_ACCOUNT} -n ${NAMESPACE} -o json | jq 'del(.metadata["namespace","creationTimestamp","resourceVersion","selfLink","uid"])' | kubectl apply -n ${DEV_NAMESPACE} -f - +kubectl --context ${CLUSTER_CONTEXT} get sa ${SERVICE_ACCOUNT} -n ${DEV_NAMESPACE} >& /dev/null || \ + kubectl --context ${CLUSTER_CONTEXT} get sa ${SERVICE_ACCOUNT} -n ${NAMESPACE} -o json | jq 'del(.metadata["namespace","creationTimestamp","resourceVersion","selfLink","uid"])' | kubectl --context ${CLUSTER_CONTEXT} apply -n ${DEV_NAMESPACE} -f - # Setup dev k8s manifests, configuration, docker login etc CONFIG_ENVIRONMENT="staging" -DEPLOYMENT=${PROJECT_NAME} -REPO=${PROJECT_NAME} EXT_HOSTNAME=<% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %> -MY_EXT_HOSTNAME=${DEV_USER_PROJECT}-${EXT_HOSTNAME} -ECR_REPO=${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${REPO} -VERSION_TAG=lastest-${USER_NAME} +MY_EXT_HOSTNAME=${DEV_NAMESPACE}-${EXT_HOSTNAME} +ECR_REPO=${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${PROJECT_NAME} +VERSION_TAG=latest DATABASE_NAME=<% index .Params `databaseName` %> -DEV_DATABASE_NAME=$(echo "dev-${USER_NAME}" | tr -dc 'A-Za-z0-9') -rm -rf kubernetes.tmp && cp -pr kubernetes kubernetes.tmp -## image: set image definition -sed -i ".bak" "s|image: fake-image|image: ${ECR_REPO}:${VERSION_TAG}|g" kubernetes.tmp/base/deployment.yml -## ingress: create new domain with external DNS -sed -i ".bak" "s|${EXT_HOSTNAME}|${MY_EXT_HOSTNAME}|g" kubernetes.tmp/overlays/${CONFIG_ENVIRONMENT}/ingress.yml -## database_name: replace with new one -sed -i ".bak" "s|DATABASE_NAME=${DATABASE_NAME}|DATABASE_NAME=${DEV_DATABASE_NAME}|g" kubernetes.tmp/base/kustomization.yml -skaffold config set default-repo ${ECR_REPO} >& /dev/null -aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${ECR_REPO} >& /dev/null || error_exit "Failed to login to AWS ECR" +DEV_DATABASE_NAME=$(echo "dev${MY_USERNAME}" | tr -dc 'A-Za-z0-9') echo " Domain: ${MY_EXT_HOSTNAME}" echo " Database Name: ${DEV_DATABASE_NAME}" +# Apply manifests +(cd kubenetes/overlay/${CONFIG_ENVIRONMENT} && \ + kustomize build . | \ + sed "s|image: fake-image|image: ${ECR_REPO}:${VERSION_TAG}|g" | \ + sed "s|${EXT_HOSTNAME}|${MY_EXT_HOSTNAME}|g" | \ + sed "s|DATABASE_NAME=${DATABASE_NAME}|DATABASE_NAME=${DEV_DATABASE_NAME}|g" | \ + kubectl --context ${} -n ${DEV_NAMESPACE} apply -f - ) || error_exit "Failed to apply kubernetes manifests" + +# Starting dev environment with telepresence shell echo +telepresence --swap-deployment ${PROJECT_NAME} --namespace ${DEV_NAMESPACE} --expose 80 --run-shell + +# Ending dev environment echo -echo "Now, you are ready to launch your backend service on Staging under your dev environment/namepsace '${DEV_NAMESPACE}'. This will allow you to debug your cloud backend service on the fly with your local laptop and daily favourite tools like vscode etc." -echo -echo " Step 1: launch your backend service, run the followings in a terminal:" -echo " > skaffold run --profile ${CONFIG_ENVIRONMENT} --namespace ${DEV_NAMESPACE}" -echo " or run in frontend:" -echo " > skaffold dev --profile ${CONFIG_ENVIRONMENT} --namespace ${DEV_NAMESPACE}" -echo -echo " to confirm the result:" -echo " > kubectl -n ${DEV_NAMESPACE} get pods # see pods running" -echo " > curl http://${MY_EXT_HOSTNAME}/ # see Hello" -echo -echo " Step 2: intercept the request to local, run the followings in a terminal:" -echo " > docker build . -t ${DEPLOYMENT}-dev" -echo " > telepresence --swap-deployment ${DEPLOYMENT} --namespace ${DEV_NAMESPACE} --expose 80 --docker-run --rm -it -v $(pwd):/usr/src/app ${DEPLOYMENT}-dev" -echo " to confirm the result (in another terminal):" -echo " > curl http://${MY_EXT_HOSTNAME}/ # see Hello in both terminals" -echo -echo "Then, you can change the code in your editor now, and you will see the live traffic from the browser redirected to your terminal immediately. After confirmed the changes running on Staging, you may just submit it to git repo which will trigger CI/CD workflow as before." -echo -echo "For more detailed information, please refer to:" -echo " telepresence: https://www.telepresence.io/ " -echo " skaffold: https://skaffold.dev/" +kubectl --context ${CLUSTER_CONTEXT} delete namespaces/${DEV_NAMESPACE} +echo "Your dev environment on Staging has been deleted completely" echo From b8fd2fc3cd09b696e5a24a989037f59365cf1171 Mon Sep 17 00:00:00 2001 From: Steven Shi Date: Tue, 1 Dec 2020 16:41:10 -0800 Subject: [PATCH 04/11] finalizing the script and added section in README --- templates/README.md | 16 ++++++++++++ .../kubernetes/overlays/staging/ingress.yml | 26 ++++++++++++++++++- templates/start-dev-env.sh | 25 ++++++++++++++---- 3 files changed, 61 insertions(+), 6 deletions(-) diff --git a/templates/README.md b/templates/README.md index bbd5d06..75720c2 100644 --- a/templates/README.md +++ b/templates/README.md @@ -12,6 +12,22 @@ kubectl -n <% .Name %> get pods ### Configuring You can update the resource limits in the [kubernetes/base/deployment.yml][base-deployment], and control fine-grain customizations based on environment and specific deployments such as Scaling out your production replicas from the [overlays configurations][env-prod] +### Dev Environment +You can do fast local development of a single service, even if that service depends on other services in your cluster. Make a change to your service, save, and you can immediately see the new service in action. And, you can use any tools like IDE installed locally to test/debug/edit your service. + +Supposing your backend service (`<% .Name %>`, a web service, output `Hello`) is running on Staging cluster, you can `curl htts://<% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %>` and see `Hello`. Now, you want to change the service and verify it locally. + +Usually, you will test the service in local that accesses your local database. However, today, you need access data on Staging database and you are not allowed to access that database directly from your local machine. + +- Regular development workflow: + a. change code --> b. lite test on local --> c. git commit & auto-deploy to Staging --> d. verify the changes on Staging --> e. repeat a~d until done + +- New development workflow: + a. run `start-dev-env.sh` --> b. change code --> c. test on Staging with cloud DB --> d. repeat b~c until done --> e. git commit & auto-deploy to Staging + +Note: this script is powered by Telepresence (http://telepresence.io) and Kustomize. You may customize the script. + + ## Circle CI Your repository comes with a end-to-end CI/CD pipeline, which includes the following steps: 1. Checkout diff --git a/templates/kubernetes/overlays/staging/ingress.yml b/templates/kubernetes/overlays/staging/ingress.yml index 156d827..59d01d9 100644 --- a/templates/kubernetes/overlays/staging/ingress.yml +++ b/templates/kubernetes/overlays/staging/ingress.yml @@ -13,7 +13,31 @@ metadata: external-dns.alpha.kubernetes.io/hostname: <% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %> # CORS nginx.ingress.kubernetes.io/enable-cors: "true" - nginx.ingress.kubernetes.io/cors-allow-origin: "https://<% index .Params `stagingFrontendSubdomain` %><% index .Params `stagingHostRoot` %>/" + nginx.ingress.kubernetes.io/configuration-snippet: | + if ($http_origin ~* "^https?://((?:<% index .Params `stagingFrontendSubdomain` %><% index .Params `stagingHostRoot` %>)|(?:localhost))") { + set $cors "true"; + } + if ($request_method = 'OPTIONS') { + set $cors "${cors}options"; + } + + if ($cors = "true") { + add_header 'Access-Control-Allow-Origin' "$http_origin" always; + add_header 'Access-Control-Allow-Methods' 'GET, PUT, POST, DELETE, PATCH, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization' always; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always; + } + + if ($cors = "trueoptions") { + add_header 'Access-Control-Allow-Origin' "$http_origin"; + add_header 'Access-Control-Allow-Methods' 'GET, PUT, POST, DELETE, PATCH, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } spec: rules: diff --git a/templates/start-dev-env.sh b/templates/start-dev-env.sh index 75478c4..bc2644d 100755 --- a/templates/start-dev-env.sh +++ b/templates/start-dev-env.sh @@ -36,13 +36,16 @@ function can_i() { done [[ $err -gt 0 ]] && error_exit "Found $err permission errors. Please check with your administrator." + + echo "Permission checks: passed" + return 0 } # Start # Validate current iam user MY_USERNAME=$(aws sts get-caller-identity --output json | jq -r .Arn | cut -d/ -f2) DEV_USERS=$(aws iam get-group --group-name ${PROJECT_NAME}-developer-${ENVIRONMENT} | jq -r .Users[].UserName) -[[ "${DEV_USERS[@]}" =~ "${MY_USERNAME}" ]] || error_exit "You (${MY_USERNAME}) are not in ${DEV_USERS}" +[[ "${DEV_USERS[@]}" =~ "${MY_USERNAME}" ]] || error_exit "You (${MY_USERNAME}) are not in the ${PROJECT_NAME}-developer-${ENVIRONMENT} IAM group." DEV_PROJECT_ID=${1:-"001"} @@ -98,16 +101,28 @@ echo " Domain: ${MY_EXT_HOSTNAME}" echo " Database Name: ${DEV_DATABASE_NAME}" # Apply manifests -(cd kubenetes/overlay/${CONFIG_ENVIRONMENT} && \ +(cd kubernetes/overlays/${CONFIG_ENVIRONMENT} && \ kustomize build . | \ sed "s|image: fake-image|image: ${ECR_REPO}:${VERSION_TAG}|g" | \ sed "s|${EXT_HOSTNAME}|${MY_EXT_HOSTNAME}|g" | \ - sed "s|DATABASE_NAME=${DATABASE_NAME}|DATABASE_NAME=${DEV_DATABASE_NAME}|g" | \ - kubectl --context ${} -n ${DEV_NAMESPACE} apply -f - ) || error_exit "Failed to apply kubernetes manifests" + sed "s|DATABASE_NAME: ${DATABASE_NAME}|DATABASE_NAME: ${DEV_DATABASE_NAME}|g" | \ + kubectl --context ${CLUSTER_CONTEXT} -n ${DEV_NAMESPACE} apply -f - ) || error_exit "Failed to apply kubernetes manifests" + +# Confirm deployment +if ! kubectl --context ${CLUSTER_CONTEXT} -n ${DEV_NAMESPACE} rollout status deployment/${PROJECT_NAME} -w --timeout=180s ; then + echo "${PROJECT_NAME} rollout check failed:" + echo "${PROJECT_NAME} deployment:" + kubectl --context ${CLUSTER_CONTEXT} -n ${DEV_NAMESPACE} describe deployment ${PROJECT_NAME} + echo "${PROJECT_NAME} replicaset:" + kubectl --context ${CLUSTER_CONTEXT} -n ${DEV_NAMESPACE} describe rs -l app=${PROJECT_NAME} + echo "${PROJECT_NAME} pods:" + kubectl --context ${CLUSTER_CONTEXT} -n ${DEV_NAMESPACE} describe pod -l app=${PROJECT_NAME} + error_exit "Failed deployment. Leaving namespace ${DEV_NAMESPACE} for debugging" +fi # Starting dev environment with telepresence shell echo -telepresence --swap-deployment ${PROJECT_NAME} --namespace ${DEV_NAMESPACE} --expose 80 --run-shell +telepresence --context ${CLUSTER_CONTEXT} --swap-deployment ${PROJECT_NAME} --namespace ${DEV_NAMESPACE} --expose 80 --run-shell # Ending dev environment echo From c2ec78a20a14aa805bcf8865a3c6a970c048308d Mon Sep 17 00:00:00 2001 From: Steven Shi Date: Wed, 2 Dec 2020 15:00:10 -0800 Subject: [PATCH 05/11] DNS handling, and some message --- templates/start-dev-env.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/templates/start-dev-env.sh b/templates/start-dev-env.sh index bc2644d..bcac57b 100755 --- a/templates/start-dev-env.sh +++ b/templates/start-dev-env.sh @@ -90,7 +90,7 @@ kubectl --context ${CLUSTER_CONTEXT} get sa ${SERVICE_ACCOUNT} -n ${DEV_NAMESPAC kubectl --context ${CLUSTER_CONTEXT} get sa ${SERVICE_ACCOUNT} -n ${NAMESPACE} -o json | jq 'del(.metadata["namespace","creationTimestamp","resourceVersion","selfLink","uid"])' | kubectl --context ${CLUSTER_CONTEXT} apply -n ${DEV_NAMESPACE} -f - # Setup dev k8s manifests, configuration, docker login etc -CONFIG_ENVIRONMENT="staging" +CONFIG_ENVIRONMENT="devenv" EXT_HOSTNAME=<% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %> MY_EXT_HOSTNAME=${DEV_NAMESPACE}-${EXT_HOSTNAME} ECR_REPO=${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${PROJECT_NAME} @@ -120,6 +120,18 @@ if ! kubectl --context ${CLUSTER_CONTEXT} -n ${DEV_NAMESPACE} rollout status dep error_exit "Failed deployment. Leaving namespace ${DEV_NAMESPACE} for debugging" fi +# Verify until the ingress DNS gets ready +while ! nslookup ${MY_EXT_HOSTNAME} 8.8.8.8 >& /dev/null; do echo "waiting for domain '${MY_EXT_HOSTNAME}' to get resolved..."; sleep 5; done + +# Starting telepresence shell +echo +echo "Now you are ready to access your service at:" +echo +echo " https://${MY_EXT_HOSTNAME}" +echo +echo -n "You will get into dev environment shell which will proxy all the requests and environment variables from cluster to the local shell. Be noticied that the above URL access will get 502 bad gateway error until you launch the service by 'make run' in the shell. Press any key to continue ..." && read +echo + # Starting dev environment with telepresence shell echo telepresence --context ${CLUSTER_CONTEXT} --swap-deployment ${PROJECT_NAME} --namespace ${DEV_NAMESPACE} --expose 80 --run-shell From 3496efda589b61f328b1eb1c6152ae6a877ad714 Mon Sep 17 00:00:00 2001 From: Steven Shi Date: Wed, 2 Dec 2020 15:01:22 -0800 Subject: [PATCH 06/11] Update templates/README.md Co-authored-by: Bill Monkman --- templates/README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/templates/README.md b/templates/README.md index 75720c2..a912ca7 100644 --- a/templates/README.md +++ b/templates/README.md @@ -13,19 +13,19 @@ kubectl -n <% .Name %> get pods You can update the resource limits in the [kubernetes/base/deployment.yml][base-deployment], and control fine-grain customizations based on environment and specific deployments such as Scaling out your production replicas from the [overlays configurations][env-prod] ### Dev Environment -You can do fast local development of a single service, even if that service depends on other services in your cluster. Make a change to your service, save, and you can immediately see the new service in action. And, you can use any tools like IDE installed locally to test/debug/edit your service. - -Supposing your backend service (`<% .Name %>`, a web service, output `Hello`) is running on Staging cluster, you can `curl htts://<% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %>` and see `Hello`. Now, you want to change the service and verify it locally. - -Usually, you will test the service in local that accesses your local database. However, today, you need access data on Staging database and you are not allowed to access that database directly from your local machine. - -- Regular development workflow: - a. change code --> b. lite test on local --> c. git commit & auto-deploy to Staging --> d. verify the changes on Staging --> e. repeat a~d until done - -- New development workflow: - a. run `start-dev-env.sh` --> b. change code --> c. test on Staging with cloud DB --> d. repeat b~c until done --> e. git commit & auto-deploy to Staging - -Note: this script is powered by Telepresence (http://telepresence.io) and Kustomize. You may customize the script. +This project is set up with a local/cloud hybrid dev environment. This means you can do fast local development of a single service, even if that service depends on other resources in your cluster. +Make a change to your service, run it, and you can immediately see the new service in action in a real environment. You can also use any tools like your local IDE, debugger, etc. to test/debug/edit/run your service. + +Usually when developing you would run the service locally with a local database and any other dependencies running either locally or in containers using `docker-compose`, `minikube`, etc. +Now your service will have access to any dependencies within a namespace running in the EKS cluster, with access to resources there. +[Telepresence](https://telepresence.io) is used to provide this functionality. + + Development workflow: + + 1. Run `start-dev-env.sh` - You will be dropped into a shell that is the same as your local machine, but works as if it were running inside a pod in your k8s cluster + 2. Change code and run the server - As you run your local server, using local code, it will have access to remote dependencies, and will be sent traffic by the load balancer + 3. Test on your cloud environment with real dependencies - `https://-<% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %>` + 4. git commit & auto-deploy to Staging through the build pipeline ## Circle CI From 56b9dccb1d94843d870da7e8919b07fe68f3e0ef Mon Sep 17 00:00:00 2001 From: Steven Shi Date: Wed, 2 Dec 2020 15:59:41 -0800 Subject: [PATCH 07/11] added overlays/devenv with refined script and tag last-deployed, and backedn DNS check --- templates/.circleci/config.yml | 2 +- .../kubernetes/overlays/devenv/deployment.yml | 17 ++++++ .../kubernetes/overlays/devenv/ingress.yml | 54 +++++++++++++++++++ .../overlays/devenv/kustomization.yml | 15 ++++++ .../kubernetes/overlays/staging/ingress.yml | 26 +-------- templates/start-dev-env.sh | 9 ++-- 6 files changed, 93 insertions(+), 30 deletions(-) create mode 100644 templates/kubernetes/overlays/devenv/deployment.yml create mode 100644 templates/kubernetes/overlays/devenv/ingress.yml create mode 100644 templates/kubernetes/overlays/devenv/kustomization.yml diff --git a/templates/.circleci/config.yml b/templates/.circleci/config.yml index 7078e30..8f20667 100644 --- a/templates/.circleci/config.yml +++ b/templates/.circleci/config.yml @@ -166,7 +166,7 @@ jobs: echo 'export AWS_ECR_ACCOUNT_URL=<< parameters.account-id >>.dkr.ecr.<< parameters.region >>.amazonaws.com' >> $BASH_ENV - aws-ecr/build-and-push-image: repo: << parameters.repo >> - tag: $VERSION_TAG,latest + tag: $VERSION_TAG,latest,last-deployed deploy: executor: aws-eks/python3 diff --git a/templates/kubernetes/overlays/devenv/deployment.yml b/templates/kubernetes/overlays/devenv/deployment.yml new file mode 100644 index 0000000..62c8d0c --- /dev/null +++ b/templates/kubernetes/overlays/devenv/deployment.yml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: <% .Name %> +spec: + template: + spec: + containers: + - name: <% .Name %> + image: <% index .Params `accountId` %>.dkr.ecr.<% index .Params `region` %>.amazonaws.com/<% .Name %>:last-deployed + resources: + requests: + memory: 64Mi + cpu: 0.1 + limits: + memory: 256Mi + cpu: 1.0 diff --git a/templates/kubernetes/overlays/devenv/ingress.yml b/templates/kubernetes/overlays/devenv/ingress.yml new file mode 100644 index 0000000..59d01d9 --- /dev/null +++ b/templates/kubernetes/overlays/devenv/ingress.yml @@ -0,0 +1,54 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: <% .Name %> + annotations: + # nginx ingress + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/rewrite-target: /$1 + # cert-manager + ingress.kubernetes.io/ssl-redirect: "true" + cert-manager.io/cluster-issuer: clusterissuer-letsencrypt-production + # external-dns + external-dns.alpha.kubernetes.io/hostname: <% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %> + # CORS + nginx.ingress.kubernetes.io/enable-cors: "true" + nginx.ingress.kubernetes.io/configuration-snippet: | + if ($http_origin ~* "^https?://((?:<% index .Params `stagingFrontendSubdomain` %><% index .Params `stagingHostRoot` %>)|(?:localhost))") { + set $cors "true"; + } + if ($request_method = 'OPTIONS') { + set $cors "${cors}options"; + } + + if ($cors = "true") { + add_header 'Access-Control-Allow-Origin' "$http_origin" always; + add_header 'Access-Control-Allow-Methods' 'GET, PUT, POST, DELETE, PATCH, OPTIONS' always; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization' always; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always; + } + + if ($cors = "trueoptions") { + add_header 'Access-Control-Allow-Origin' "$http_origin"; + add_header 'Access-Control-Allow-Methods' 'GET, PUT, POST, DELETE, PATCH, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + add_header 'Content-Length' 0; + return 204; + } + +spec: + rules: + - host: <% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %> + http: + paths: + - path: /(.*) + backend: + serviceName: <% .Name %> + servicePort: http + tls: + - hosts: + - <% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %> + secretName: <% .Name %>-tls-secret diff --git a/templates/kubernetes/overlays/devenv/kustomization.yml b/templates/kubernetes/overlays/devenv/kustomization.yml new file mode 100644 index 0000000..8f7dc33 --- /dev/null +++ b/templates/kubernetes/overlays/devenv/kustomization.yml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +patchesStrategicMerge: +- deployment.yml + +resources: +- ../../base +- ingress.yml + +configMapGenerator: +- name: <% .Name %>-config + behavior: merge + literals: + - ENVIRONMENT=staging diff --git a/templates/kubernetes/overlays/staging/ingress.yml b/templates/kubernetes/overlays/staging/ingress.yml index 59d01d9..156d827 100644 --- a/templates/kubernetes/overlays/staging/ingress.yml +++ b/templates/kubernetes/overlays/staging/ingress.yml @@ -13,31 +13,7 @@ metadata: external-dns.alpha.kubernetes.io/hostname: <% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %> # CORS nginx.ingress.kubernetes.io/enable-cors: "true" - nginx.ingress.kubernetes.io/configuration-snippet: | - if ($http_origin ~* "^https?://((?:<% index .Params `stagingFrontendSubdomain` %><% index .Params `stagingHostRoot` %>)|(?:localhost))") { - set $cors "true"; - } - if ($request_method = 'OPTIONS') { - set $cors "${cors}options"; - } - - if ($cors = "true") { - add_header 'Access-Control-Allow-Origin' "$http_origin" always; - add_header 'Access-Control-Allow-Methods' 'GET, PUT, POST, DELETE, PATCH, OPTIONS' always; - add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization' always; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always; - } - - if ($cors = "trueoptions") { - add_header 'Access-Control-Allow-Origin' "$http_origin"; - add_header 'Access-Control-Allow-Methods' 'GET, PUT, POST, DELETE, PATCH, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization'; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; - add_header 'Access-Control-Max-Age' 1728000; - add_header 'Content-Type' 'text/plain charset=UTF-8'; - add_header 'Content-Length' 0; - return 204; - } + nginx.ingress.kubernetes.io/cors-allow-origin: "https://<% index .Params `stagingFrontendSubdomain` %><% index .Params `stagingHostRoot` %>/" spec: rules: diff --git a/templates/start-dev-env.sh b/templates/start-dev-env.sh index bcac57b..6bee3fb 100755 --- a/templates/start-dev-env.sh +++ b/templates/start-dev-env.sh @@ -47,7 +47,7 @@ MY_USERNAME=$(aws sts get-caller-identity --output json | jq -r .Arn | cut -d/ - DEV_USERS=$(aws iam get-group --group-name ${PROJECT_NAME}-developer-${ENVIRONMENT} | jq -r .Users[].UserName) [[ "${DEV_USERS[@]}" =~ "${MY_USERNAME}" ]] || error_exit "You (${MY_USERNAME}) are not in the ${PROJECT_NAME}-developer-${ENVIRONMENT} IAM group." -DEV_PROJECT_ID=${1:-"001"} +DEV_PROJECT_ID=${1:-""} echo '[Dev Environment]' @@ -73,7 +73,7 @@ if ! command_exist kustomize || ! command_exist telepresence; then fi # Setup dev namepsace -DEV_NAMESPACE=${MY_USERNAME}-${DEV_PROJECT_ID} +DEV_NAMESPACE=${MY_USERNAME}${DEV_PROJECT_ID} kubectl --context ${CLUSTER_CONTEXT} get namespace ${DEV_NAMESPACE} >& /dev/null || \ (can_i "create namespace,create deployment,create ingress,create service,create secret,create configmap" && \ kubectl --context ${CLUSTER_CONTEXT} create namespace ${DEV_NAMESPACE}) @@ -103,7 +103,6 @@ echo " Database Name: ${DEV_DATABASE_NAME}" # Apply manifests (cd kubernetes/overlays/${CONFIG_ENVIRONMENT} && \ kustomize build . | \ - sed "s|image: fake-image|image: ${ECR_REPO}:${VERSION_TAG}|g" | \ sed "s|${EXT_HOSTNAME}|${MY_EXT_HOSTNAME}|g" | \ sed "s|DATABASE_NAME: ${DATABASE_NAME}|DATABASE_NAME: ${DEV_DATABASE_NAME}|g" | \ kubectl --context ${CLUSTER_CONTEXT} -n ${DEV_NAMESPACE} apply -f - ) || error_exit "Failed to apply kubernetes manifests" @@ -121,7 +120,9 @@ if ! kubectl --context ${CLUSTER_CONTEXT} -n ${DEV_NAMESPACE} rollout status dep fi # Verify until the ingress DNS gets ready -while ! nslookup ${MY_EXT_HOSTNAME} 8.8.8.8 >& /dev/null; do echo "waiting for domain '${MY_EXT_HOSTNAME}' to get resolved..."; sleep 5; done +bash -c "while ! nslookup ${MY_EXT_HOSTNAME} >& /dev/null; do sleep 30; done; echo && echo \"Notice: your domain ${MY_EXT_HOSTNAME} gets propagated.\";" & +sleep 2 +echo # Starting telepresence shell echo From 639597c5d927583d0941af4fd738554481ef408a Mon Sep 17 00:00:00 2001 From: Steven Shi Date: Wed, 2 Dec 2020 16:31:36 -0800 Subject: [PATCH 08/11] Update templates/start-dev-env.sh Co-authored-by: Bill Monkman --- templates/start-dev-env.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/templates/start-dev-env.sh b/templates/start-dev-env.sh index 6bee3fb..3344067 100755 --- a/templates/start-dev-env.sh +++ b/templates/start-dev-env.sh @@ -120,7 +120,12 @@ if ! kubectl --context ${CLUSTER_CONTEXT} -n ${DEV_NAMESPACE} rollout status dep fi # Verify until the ingress DNS gets ready -bash -c "while ! nslookup ${MY_EXT_HOSTNAME} >& /dev/null; do sleep 30; done; echo && echo \"Notice: your domain ${MY_EXT_HOSTNAME} gets propagated.\";" & +if nslookup ${MY_EXT_HOSTNAME}x >& /dev/null; then + echo " Notice: your domain is ready to use." +else + echo " Notice: the first time you use this environment it may take up to 5 minutes for DNS to propagate before the hostname is available." + bash -c "while ! nslookup ${MY_EXT_HOSTNAME} >& /dev/null; do sleep 30; done; echo && echo \" Notice: your domain ${MY_EXT_HOSTNAME} is ready to use.\";" & +fi sleep 2 echo From 8b84cda7bd53ae90045e34661f6329c9a339cf39 Mon Sep 17 00:00:00 2001 From: Steven Shi Date: Wed, 2 Dec 2020 16:42:04 -0800 Subject: [PATCH 09/11] enhancements on removing dns etc. --- .../kubernetes/overlays/{devenv => dev}/deployment.yml | 0 templates/kubernetes/overlays/{devenv => dev}/ingress.yml | 3 +-- .../kubernetes/overlays/{devenv => dev}/kustomization.yml | 0 templates/kubernetes/overlays/production/ingress.yml | 2 -- templates/kubernetes/overlays/staging/ingress.yml | 2 -- templates/start-dev-env.sh | 7 +++---- 6 files changed, 4 insertions(+), 10 deletions(-) rename templates/kubernetes/overlays/{devenv => dev}/deployment.yml (100%) rename templates/kubernetes/overlays/{devenv => dev}/ingress.yml (92%) rename templates/kubernetes/overlays/{devenv => dev}/kustomization.yml (100%) diff --git a/templates/kubernetes/overlays/devenv/deployment.yml b/templates/kubernetes/overlays/dev/deployment.yml similarity index 100% rename from templates/kubernetes/overlays/devenv/deployment.yml rename to templates/kubernetes/overlays/dev/deployment.yml diff --git a/templates/kubernetes/overlays/devenv/ingress.yml b/templates/kubernetes/overlays/dev/ingress.yml similarity index 92% rename from templates/kubernetes/overlays/devenv/ingress.yml rename to templates/kubernetes/overlays/dev/ingress.yml index 59d01d9..fb72571 100644 --- a/templates/kubernetes/overlays/devenv/ingress.yml +++ b/templates/kubernetes/overlays/dev/ingress.yml @@ -9,10 +9,9 @@ metadata: # cert-manager ingress.kubernetes.io/ssl-redirect: "true" cert-manager.io/cluster-issuer: clusterissuer-letsencrypt-production - # external-dns - external-dns.alpha.kubernetes.io/hostname: <% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %> # CORS nginx.ingress.kubernetes.io/enable-cors: "true" + ## to support both frontend origin and 'localhost', need 'configuration-snippet' implementation here, because 'cors-allow-origin' field doesn't support multiple originss yet. nginx.ingress.kubernetes.io/configuration-snippet: | if ($http_origin ~* "^https?://((?:<% index .Params `stagingFrontendSubdomain` %><% index .Params `stagingHostRoot` %>)|(?:localhost))") { set $cors "true"; diff --git a/templates/kubernetes/overlays/devenv/kustomization.yml b/templates/kubernetes/overlays/dev/kustomization.yml similarity index 100% rename from templates/kubernetes/overlays/devenv/kustomization.yml rename to templates/kubernetes/overlays/dev/kustomization.yml diff --git a/templates/kubernetes/overlays/production/ingress.yml b/templates/kubernetes/overlays/production/ingress.yml index 6f27420..a1b6004 100644 --- a/templates/kubernetes/overlays/production/ingress.yml +++ b/templates/kubernetes/overlays/production/ingress.yml @@ -9,8 +9,6 @@ metadata: # cert-manager ingress.kubernetes.io/ssl-redirect: "true" cert-manager.io/cluster-issuer: clusterissuer-letsencrypt-production - # external-dns - external-dns.alpha.kubernetes.io/hostname: <% index .Params `productionBackendSubdomain` %><% index .Params `productionHostRoot` %> # CORS nginx.ingress.kubernetes.io/enable-cors: "true" nginx.ingress.kubernetes.io/cors-allow-origin: "https://<% index .Params `productionFrontendSubdomain` %><% index .Params `productionHostRoot` %>/" diff --git a/templates/kubernetes/overlays/staging/ingress.yml b/templates/kubernetes/overlays/staging/ingress.yml index 156d827..59a4977 100644 --- a/templates/kubernetes/overlays/staging/ingress.yml +++ b/templates/kubernetes/overlays/staging/ingress.yml @@ -9,8 +9,6 @@ metadata: # cert-manager ingress.kubernetes.io/ssl-redirect: "true" cert-manager.io/cluster-issuer: clusterissuer-letsencrypt-production - # external-dns - external-dns.alpha.kubernetes.io/hostname: <% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %> # CORS nginx.ingress.kubernetes.io/enable-cors: "true" nginx.ingress.kubernetes.io/cors-allow-origin: "https://<% index .Params `stagingFrontendSubdomain` %><% index .Params `stagingHostRoot` %>/" diff --git a/templates/start-dev-env.sh b/templates/start-dev-env.sh index 3344067..f83c632 100755 --- a/templates/start-dev-env.sh +++ b/templates/start-dev-env.sh @@ -90,7 +90,7 @@ kubectl --context ${CLUSTER_CONTEXT} get sa ${SERVICE_ACCOUNT} -n ${DEV_NAMESPAC kubectl --context ${CLUSTER_CONTEXT} get sa ${SERVICE_ACCOUNT} -n ${NAMESPACE} -o json | jq 'del(.metadata["namespace","creationTimestamp","resourceVersion","selfLink","uid"])' | kubectl --context ${CLUSTER_CONTEXT} apply -n ${DEV_NAMESPACE} -f - # Setup dev k8s manifests, configuration, docker login etc -CONFIG_ENVIRONMENT="devenv" +CONFIG_ENVIRONMENT="dev" EXT_HOSTNAME=<% index .Params `stagingBackendSubdomain` %><% index .Params `stagingHostRoot` %> MY_EXT_HOSTNAME=${DEV_NAMESPACE}-${EXT_HOSTNAME} ECR_REPO=${ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${PROJECT_NAME} @@ -120,14 +120,13 @@ if ! kubectl --context ${CLUSTER_CONTEXT} -n ${DEV_NAMESPACE} rollout status dep fi # Verify until the ingress DNS gets ready -if nslookup ${MY_EXT_HOSTNAME}x >& /dev/null; then +echo +if nslookup ${MY_EXT_HOSTNAME} >& /dev/null; then echo " Notice: your domain is ready to use." else echo " Notice: the first time you use this environment it may take up to 5 minutes for DNS to propagate before the hostname is available." bash -c "while ! nslookup ${MY_EXT_HOSTNAME} >& /dev/null; do sleep 30; done; echo && echo \" Notice: your domain ${MY_EXT_HOSTNAME} is ready to use.\";" & fi -sleep 2 -echo # Starting telepresence shell echo From 482216f0688e48fd7317d0108a47e690f2d95808 Mon Sep 17 00:00:00 2001 From: Steven Shi Date: Thu, 3 Dec 2020 12:29:45 -0800 Subject: [PATCH 10/11] Update templates/start-dev-env.sh Co-authored-by: Bill Monkman --- templates/start-dev-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/start-dev-env.sh b/templates/start-dev-env.sh index f83c632..5d458d5 100755 --- a/templates/start-dev-env.sh +++ b/templates/start-dev-env.sh @@ -134,7 +134,7 @@ echo "Now you are ready to access your service at:" echo echo " https://${MY_EXT_HOSTNAME}" echo -echo -n "You will get into dev environment shell which will proxy all the requests and environment variables from cluster to the local shell. Be noticied that the above URL access will get 502 bad gateway error until you launch the service by 'make run' in the shell. Press any key to continue ..." && read +echo -n "Your telepresence dev environment is now loading which will proxy all the requests and environment variables from the cloud EKS cluster to the local shell.\nNote that the above URL access will get a \"502 Bad Gateway\" error until you launch the service in the shell, at which point it will start receiving traffic." echo # Starting dev environment with telepresence shell From 91ce58385cce49d7b9cb43d69fd95a78b996e5d8 Mon Sep 17 00:00:00 2001 From: Steven Shi Date: Thu, 3 Dec 2020 12:35:29 -0800 Subject: [PATCH 11/11] added aws ecr put-image after kubectl apply successfuly --- templates/.circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/.circleci/config.yml b/templates/.circleci/config.yml index 8f20667..9b30a82 100644 --- a/templates/.circleci/config.yml +++ b/templates/.circleci/config.yml @@ -166,7 +166,7 @@ jobs: echo 'export AWS_ECR_ACCOUNT_URL=<< parameters.account-id >>.dkr.ecr.<< parameters.region >>.amazonaws.com' >> $BASH_ENV - aws-ecr/build-and-push-image: repo: << parameters.repo >> - tag: $VERSION_TAG,latest,last-deployed + tag: $VERSION_TAG,latest deploy: executor: aws-eks/python3 @@ -248,6 +248,9 @@ jobs: kubectl -n $NAMESPACE describe pod -l app=$DEPLOYMENT exit 1 fi + MANIFEST=$(aws ecr batch-get-image --region << parameters.region >> --repository-name << parameters.repo >> --image-ids imageTag=latest --query 'images[].imageManifest' --output text) + aws ecr put-image --region << parameters.region >> --repository-name << parameters.repo >> --image-tag last-deployed --image-manifest "$MANIFEST" + workflows: version: 2 # The main workflow. Check out the code, build it, push it, deploy to staging, test, deploy to production