finalize dreams deployment to staging environment

charts-external/dreams/README.md

@@ -2,6 +2,9 @@
 
 Contains the Dreams web-app, DB and associated scripts
 
+* Dreams Staging: https://dreams.staging.midbun.org/
+* Dreams Production: **TODO**
+
 ## Secrets
 
 ```
@@ -17,4 +20,12 @@ Contains the Dreams web-app, DB and associated scripts
                                        --from-literal=SECRET_KEY_BASE=*** \
                                        --from-literal=SENDGRID_PASSWORD=*** \
                                        --from-literal=SENDGRID_USERNAME=***
+
+  kubectl create secret generic dreams-otherdb --from-literal=HOST=*** \
+                                               --from-literal=USER=*** \
+                                               --from-literal=PASSWORD=*** \
+                                               --from-literal=PORT=*** \
+                                               --from-literal=DATABASE=***
+
+kubectl create secret generic dreamsdb --from-literal=DATABASE_URL=postgres://***:***@***:5432/***
 ```

charts-external/dreams/templates/dreams.yaml

@@ -1,4 +1,4 @@
-{{ if .Values.enabled }}
+{{ if .Values.enabled }}{{ if .Values.enableDeployment }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -41,7 +41,7 @@ spec:
         - name: CREW_SIZE
           value: "true"
         - name: DATABASE_URL
-          value: postgres://dreamsdb
+          valueFrom: {"secretKeyRef":{"name":"dreamsdb", "key":"DATABASE_URL"}}
         - name: DISABLE_EDITING_DREAM
           value: "false"
         - name: EMAIL_FROM
@@ -98,4 +98,4 @@ spec:
           value: https://spark.midburn.org/api/userlogin
         - name: TZ
           value: Asia/Jerusalem
-{{ end }}
+{{ end }}{{ end }}

charts-external/dreams/templates/dreamsdb.yaml

@@ -31,11 +31,66 @@ spec:
         resources:
           requests:
             cpu: 20m
+        {{ if .Values.importDB }}
+        command:
+        - bash
+        - "-c"
+        - |
+          docker-entrypoint.sh postgres &
+          echo waiting for db...
+          while ! su postgres -c "psql -c 'select 1;'"; do sleep 1; echo .; done
+          echo db is ready
+          echo creating user
+          ! su postgres -c "createuser ${IMPORT_DB_USER}" && echo WARNING failed to create user
+          ! su postgres -c 'psql -c "alter role ${IMPORT_DB_USER} with password '"'${IMPORT_DB_PASSWORD}'"';"' \
+            && echo WARNING failed to alter role
+          if ! su postgres -c "psql -d ${IMPORT_DB_DATABASE} -c 'select 1 from users limit 1;'"; then
+            echo creating DB
+            ! su postgres -c 'psql -c "create database ${IMPORT_DB_DATABASE};"' \
+              && echo failed to create db && exit 1
+            echo importing from other DB...
+            importdb_pgexec() { PGPASSWORD="${IMPORT_DB_PASSWORD}" "${1}" -h "${IMPORT_DB_HOST}" -p "${IMPORT_DB_PORT}" -U ${IMPORT_DB_USER} -d "${IMPORT_DB_DATABASE}" "${@:2}";}
+            ! importdb_pgexec psql -c "select 1 from users limit 1;" && echo failed to connect to import db && exit 1
+            echo connected successfully, getting dump of import db...
+            ! importdb_pgexec pg_dump > pgdump && echo failed to get dump from import db && exit 1
+            echo importing dump to local db...
+            ! su postgres -c "psql -d ${IMPORT_DB_DATABASE} -f /pgdump" && echo failed to import from dump && exit 1
+          else
+            echo db already exists, not importing
+          fi
+          echo great success!
+          while true; do sleep 86400; done
+        {{ end }}
         env:
+        {{ if .Values.importDB }}
+        # when importing from other DB, we use same user credentials as in imported DB
+        - name: IMPORT_DB_HOST
+          valueFrom: {"secretKeyRef":{"name":"dreams-otherdb", "key":"HOST"}}
+        - name: IMPORT_DB_USER
+          valueFrom: {"secretKeyRef":{"name":"dreams-otherdb", "key":"USER"}}
+        - name: IMPORT_DB_PASSWORD
+          valueFrom: {"secretKeyRef":{"name":"dreams-otherdb", "key":"PASSWORD"}}
+        - name: IMPORT_DB_PORT
+          valueFrom: {"secretKeyRef":{"name":"dreams-otherdb", "key":"PORT"}}
+        - name: IMPORT_DB_DATABASE
+          valueFrom: {"secretKeyRef":{"name":"dreams-otherdb", "key":"DATABASE"}}
+        {{ else }}
+        # this creates a new, empty DB for development / testing
         - name: POSTGRES_USER
-          value: dreams
+          value: masterdreamer
         - name: POSTGRES_DB
-          value: dreams
+          value: dreamdb
         - name: POSTGRES_PASSWORD
           value: "123456"
+        {{ end }}
+      {{ if .Values.persistentStorageName }}
+        volumeMounts:
+        - name: dreamsdb
+          mountPath: /var/lib/postgresql/data
+      volumes:
+      - name: dreamsdb
+        nfs:
+          path: "/data/{{ .Values.global.environmentName }}/{{ .Values.persistentStorageName }}"
+          server: {{ .Values.global.persistentStorageIP | quote }}
+      {{ end }}
 {{ end }}

environments/staging/README.md

@@ -9,8 +9,9 @@
   * Deployed on push to `develop` branch of https://github.com/Midburn/Volunteers
 * Volunteers DB UI - https://staging.midburn.org/volunteers/mongoexpress
 * Profiles - https://profiles.staging.midburn.org/
-  * DEployed on push to `master` branch of https://github.com/orihoch/midburn-profiles-drupal
+  * Deployed on push to `master` branch of https://github.com/orihoch/midburn-profiles-drupal
 * Profiles Adminer - https://staging.midburn.org/profiles/adminer
+* Dreams - https://dreams.staging.midbun.org/
 
 ## Secrets
 

environments/staging/dreamsdb_persistent_storage_migration.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+source switch_environment.sh staging
+
+PERSISTENT_STORAGE_NAME="dreamsdb"
+
+echo "deleting existing ${PERSISTENT_STORAGE_NAME} data from persistent storage (if there is any)"
+read -p "press <Enter> to continue"
+! gcloud compute ssh midburn-k8s-persistent-storage-vm --command="bash -c 'sudo rm -rf /data/${K8S_ENVIRONMENT_NAME}/${PERSISTENT_STORAGE_NAME}'" && echo failed && exit 1
+
+echo "creating storage"
+gcloud compute ssh midburn-k8s-persistent-storage-vm --command="bash -c '
+    mkdir -p /data/${K8S_ENVIRONMENT_NAME}
+    ! mkdir /data/${K8S_ENVIRONMENT_NAME}/${PERSISTENT_STORAGE_NAME} && echo failed to create ${PERSISTENT_STORAGE_NAME} dir && exit 1
+    echo setting permissions
+    sudo chown -R root:root /data/${K8S_ENVIRONMENT_NAME}/${PERSISTENT_STORAGE_NAME}
+    echo great success
+    exit 0
+'"

environments/staging/values.yaml

@@ -24,6 +24,8 @@ traefik:
       main = "volunteers.staging.midburn.org"
     [[acme.domains]]
       main = "profiles.staging.midburn.org"
+    [[acme.domains]]
+      main = "dreams.staging.midburn.org"
   acmeEmail: ori@uumpa.com
   dnsProvider: route53
   AWS_ACCESS_KEY_ID: AKIAIJUE2QVTBJAN7INQ
@@ -38,6 +40,7 @@ traefik:
   backupJob: true
   enableLoadBalancer: true
   profilesHostsRule: "Host: profiles.staging.midburn.org;"
+  dreamsHostsRule: "Host: dreams.staging.midburn.org;"
 #  SSLcheckMainDomain: staging.midburn.org
 #  SSLcheckSparkDomain: spark.staging.midburn.org
 #  SSLcheckProfilesDomain: profiles.staging.midburn.org
@@ -96,4 +99,8 @@ bi:
 
 dreams:
   enabled: true
+  enableSecrets: true
+  importDB: true
   APP_URL: https://dreams.staging.midburn.org/
+  # persistent storage was setup using environments/staging/dreamsdb_persistent_storage_migration.sh
+  persistentStorageName: dreamsdb

templates/traefik-etc.yaml

@@ -64,6 +64,12 @@ data:
           url = "http://profiles-drupal"
       {{ end }}
 
+      {{ if .Values.dreams.enabled }}
+      [backends.dreams]
+        [backends.dreams.servers.server1]
+          url = "http://dreams:3000"
+      {{ end }}
+
     [frontends]
       {{ if .Values.nginx.enabled }}
       [frontends.default]
@@ -95,4 +101,12 @@ data:
             rule = "{{ .Values.traefik.profilesHostsRule }}"
       {{ end }}
 
+      {{ if .Values.dreams.enabled }}
+      [frontends.dreams]
+        backend="dreams"
+        passHostHeader = true
+          [frontends.dreams.routes.route1]
+            rule = "{{ .Values.traefik.dreamsHostsRule }}"
+      {{ end }}
+
 {{ end }}

values.yaml

@@ -36,3 +36,6 @@ bi:
   metabasePostgresqlResources: >
     {"requests": {"cpu": "20m", "memory": "200Mi"}, "limits": {"cpu": "50m", "memory": "500Mi"}}
   JAVA_TIMEZONE: Asia/Jerusalem
+
+dreams:
+  enableDeployment: true