Merge pull request #294 from mozmar/dp_updated_kops_and_helm

cleanup to allow k8s 1.6.4, helm 2.4.2, kops 1.6.1

k8s/install/etc/deis_s3.tf.template

@@ -82,7 +82,7 @@ output "registry-bucket" {
   value = "${var.registry-bucket-name}"
 }
 
-output "region" {
+output "s3-region" {
   value = "KOPS_REGION"
 }
 

k8s/install/stage1_functions.sh

@@ -58,6 +58,8 @@ run_kops() {
         --node-count=${KOPS_NODE_COUNT} \
         --node-size=${KOPS_NODE_SIZE} \
         --master-size=${KOPS_MASTER_SIZE} \
+        --master-volume-size=${KOPS_MASTER_VOLUME_SIZE_GB} \
+        --node-volume-size=${KOPS_NODE_VOLUME_SIZE_GB} \
         --ssh-public-key=${KOPS_PUBLIC_KEY} \
         --kubernetes-version=${KOPS_K8S_VERSION}
 }
@@ -86,6 +88,11 @@ render_tf_templates() {
         | sed s/KOPS_NAME/${KOPS_NAME}/g \
         > ./out/terraform/deis_s3.tf
 
+    cat ${KOPS_INSTALLER}/etc/tf_backend.template \
+        | sed s/TF_STATE_BUCKET/${TF_STATE_BUCKET}/g \
+        | sed s/KOPS_REGION/${KOPS_REGION}/g \
+        > ./out/terraform/tf_backend.tf
+
     if [ -z "${KOPS_EXISTING_RDS}" ]
     then
         echo "Creating new RDS instance"
@@ -116,15 +123,7 @@ setup_tf_s3_state_store() {
     aws s3 mb s3://${TF_STATE_BUCKET} --region ${KOPS_REGION} || true
 
     echo "Configuring Terraform to use an encrypted remote S3 bucket for state storage"
-    # store TF state in S3
-    terraform remote config \
-        -backend=s3 \
-        -backend-config="bucket=${TF_STATE_BUCKET}" \
-        -backend-config="key=${KOPS_SHORT_NAME}/terraform.tfstate" \
-        -backend-config="encrypt=1" \
-        -backend-config="region=${KOPS_REGION}"
-    echo "Encryption for TF state:"
-    aws s3api head-object --bucket=$TF_STATE_BUCKET --key=${KOPS_SHORT_NAME}/terraform.tfstate | jq -r .ServerSideEncryption
+    terraform init
     cd ../../
 }
 

k8s/install/stage2_functions.sh

@@ -76,7 +76,7 @@ install_dd() {
     echo "Installing Datadog"
     kubectl create namespace datadog | true
     kubectl create -f ${STAGE2_ETC_PATH}/dd-agent.yaml
-    kubectl create -f etc/datadog_statsd_svc.yaml
+    kubectl create -f ${KOPS_INSTALLER}/etc/datadog_statsd_svc.yaml
 }
 
 install_newrelic() {
@@ -135,37 +135,52 @@ customize_workflow() {
         rm -rf ./workflow/charts/${comp}
     done
 
+    # remove deleted components from requirements.yaml
+    cp ./workflow/requirements.yaml ./workflow/requirements.backup
+    y2j < ./workflow/requirements.yaml | \
+        jq 'del(.dependencies[] | select(.name=="fluentd"))' | \
+        jq 'del(.dependencies[] | select(.name=="redis"))' | \
+        jq 'del(.dependencies[] | select(.name=="logger"))' | \
+        jq 'del(.dependencies[] | select(.name=="monitor"))' | \
+        jq 'del(.dependencies[] | select(.name=="nsqd"))' | j2y > ./workflow/patched_requirements.yaml
+    # y2j/j2y flip out if you overwrite the file you're reading from, so
+    # write to a temp file first
+    cp ./workflow/patched_requirements.yaml ./workflow/requirements.yaml
+
     # SSL is handled at the ELB, but the ELB still wants to point to the Deis router SSL
     # port internally. We change the ssl port to be unencrypted (http) internally.
     # TODO: use a template with condition and submit upstream
     sed -i "s/6443/8080/" workflow/charts/router/templates/router-service.yaml
     echo "Workflow customized"
 }
 
+tf_out_cmd() {
+    (cd out/terraform && terraform output $1) 
+}
+
 install_workflow_chart() {
     # make sure we're running from a directory with config.sh
     check_cwd
     echo "Installing Deis Workflow"
     helm repo add deis https://charts.deis.com/workflow
     helm inspect values deis/workflow | sed -n '1!p' > workflow_config.yaml
-    TF_OUTPUT_CMD="terraform output --state ./out/terraform/.terraform/terraform.tfstate"
 
     # s3 settings
-    region=$(${TF_OUTPUT_CMD} region)
-    registry_bucket=$(${TF_OUTPUT_CMD} registry-bucket)
-    builder_bucket=$(${TF_OUTPUT_CMD} builder-bucket)
-    s3_accesskey=$(${TF_OUTPUT_CMD} deis_s3_accesskey)
-    s3_secretkey=$(${TF_OUTPUT_CMD} deis_s3_secretkey)
+    region=$(tf_out_cmd s3-region)
+    registry_bucket=$(tf_out_cmd registry-bucket)
+    builder_bucket=$(tf_out_cmd builder-bucket)
+    s3_accesskey=$(tf_out_cmd deis_s3_accesskey)
+    s3_secretkey=$(tf_out_cmd deis_s3_secretkey)
 
     # rds settings
     if [ -z "${KOPS_EXISTING_RDS}" ]
     then
         echo "Using new RDS instance"
-        pgsql_address=$(${TF_OUTPUT_CMD} pgsql_address)
-        pgsql_db_name=$(${TF_OUTPUT_CMD} pgsql_db_name)
-        pgsql_password=$(${TF_OUTPUT_CMD} pgsql_password)
-        pgsql_port=$(${TF_OUTPUT_CMD} pgsql_port)
-        pgsql_username=$(${TF_OUTPUT_CMD} pgsql_username)
+        pgsql_address=$(tf_out_cmd pgsql_address)
+        pgsql_db_name=$(tf_out_cmd pgsql_db_name)
+        pgsql_password=$(tf_out_cmd pgsql_password)
+        pgsql_port=$(tf_out_cmd pgsql_port)
+        pgsql_username=$(tf_out_cmd pgsql_username)
     else
         echo "Using existing RDS settings"
         pgsql_address=${KOPS_PGSQL_ADDRESS}