Support deployment to separate gcp project

.semaphore/prod.yml

@@ -0,0 +1,29 @@
+---
+version: v1.0
+name: ISCO promotion
+agent:
+  machine:
+    type: e1-standard-2
+    os_image: ubuntu1804
+global_job_config:
+  secrets:
+    - name: isco
+  prologue:
+    commands:
+      - checkout
+      - export CI_COMMIT="${SEMAPHORE_GIT_SHA:0:7}"
+      - export CI_BRANCH="${SEMAPHORE_GIT_BRANCH}"
+      - export CI_TAG="${SEMAPHORE_GIT_TAG_NAME}"
+      - export CI_PULL_REQUEST="${SEMAPHORE_GIT_REF_TYPE/pull-request/true}"
+      - export CI_COMMIT_RANGE="${SEMAPHORE_GIT_COMMIT_RANGE}"
+      - export CLOUDSDK_CORE_DISABLE_PROMPTS=1
+      - export PROD_DEPLOY=1
+blocks:
+  - name: 'Promotion'
+    task:
+      jobs:
+        - name: Promote to production
+          run:
+            when: "branch = 'main'"
+          commands:
+            - ./ci/deploy.sh

.semaphore/semaphore.yml

@@ -7,11 +7,12 @@ agent:
     os_image: ubuntu1804
 global_job_config:
   secrets:
-    - name: GCP
+    - name: isco
     - name: docker-hub-credentials
     - name: coveralls
   prologue:
     commands:
+      - checkout
       - echo "${DOCKER_PASSWORD}" | docker login --username
         "${DOCKER_USERNAME}" --password-stdin
       - export CI_COMMIT="${SEMAPHORE_GIT_SHA:0:7}"
@@ -24,12 +25,9 @@ global_job_config:
       - export COVERALLS_REPO_TOKEN="${COVERALLS_ISCO_TOKEN}"
 blocks:
   - name: 'Build, Test & Deploy'
-    skip:
-      when: "tag =~ '.*'"
     task:
       prologue:
         commands:
-          - checkout
           - cache restore "npm-$(checksum frontend/package.json)"
           - cache restore "node-modules-$(checksum frontend/package.json)"
           - cache restore "pip-$(checksum backend/requirements.txt)"
@@ -48,12 +46,7 @@ blocks:
           commands:
             - ./ci/build.sh
             - ./ci/deploy.sh
+
+promotions:
   - name: "Promote to production"
-    run:
-      when: "tag =~ '.*'"
-    task:
-      jobs:
-        - name: Promote to production
-          commands:
-            - checkout
-            - ./ci/deploy.sh
+    pipeline_file: prod.yml

README.md

@@ -64,7 +64,7 @@ export CI_COMMIT='local'
 ./ci/build.sh
 ```
 
-Above command will generate two docker images with prefix `eu.gcr.io/akvo-lumen/isco` for backend and frontend
+Above command will generate two docker images for the backend and frontend
 
 ```bash
 docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d

backend/.dockerignore

@@ -0,0 +1,2 @@
+.coverage
+.pytest_cache

ci/build.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Required env vars:
+#   CI_COMMIT - commit hash
 #shellcheck disable=SC2039
 
 set -exuo pipefail
@@ -8,33 +10,36 @@ set -exuo pipefail
 BACKEND_CHANGES=0
 FRONTEND_CHANGES=0
 
-COMMIT_CONTENT=$(git diff --name-only "${CI_COMMIT_RANGE}")
-
-if grep -q "backend" <<< "${COMMIT_CONTENT}"
-then
-    BACKEND_CHANGES=1
-fi
-
-if grep -q "frontend" <<< "${COMMIT_CONTENT}"
-then
-    FRONTEND_CHANGES=1
-fi
-
-if grep -q "ci" <<< "${COMMIT_CONTENT}"
-then
-    BACKEND_CHANGES=1
-    FRONTEND_CHANGES=1
-fi
-
-if [[ "${CI_BRANCH}" ==  "main" && "${CI_PULL_REQUEST}" !=  "true" ]];
-then
-    BACKEND_CHANGES=1
-    FRONTEND_CHANGES=1
+if [[ -n "${FORCE_BUILD:-}" ]] ; then
+  BACKEND_CHANGES=1
+  FRONTEND_CHANGES=1
+else
+  COMMIT_CONTENT=$(git diff --name-only "${CI_COMMIT_RANGE}")
+
+  if grep -q "backend" <<< "${COMMIT_CONTENT}"
+  then
+      BACKEND_CHANGES=1
+  fi
+
+  if grep -q "frontend" <<< "${COMMIT_CONTENT}"
+  then
+      FRONTEND_CHANGES=1
+  fi
+
+  if grep -q "ci" <<< "${COMMIT_CONTENT}"
+  then
+      BACKEND_CHANGES=1
+      FRONTEND_CHANGES=1
+  fi
+
+  if [[ "${CI_BRANCH}" ==  "main" && "${CI_PULL_REQUEST}" !=  "true" ]];
+  then
+      BACKEND_CHANGES=1
+      FRONTEND_CHANGES=1
+  fi
 fi
 
 
-image_prefix="eu.gcr.io/akvo-lumen/isco"
-
 # Normal Docker Compose
 dc () {
     docker-compose \
@@ -53,24 +58,26 @@ frontend_build () {
     echo "PUBLIC_URL=/" > frontend/.env
     sed 's/"warn"/"error"/g' < frontend/.eslintrc.json > frontend/.eslintrc.prod.json
 
+    dc down
     dc run \
        --rm \
        --no-deps \
        frontend \
        bash release.sh
 
     docker build \
-        --tag "${image_prefix}/frontend:latest" \
-        --tag "${image_prefix}/frontend:${CI_COMMIT}" frontend
+        --tag "isco/frontend:latest" \
+        --tag "isco/frontend:${CI_COMMIT}" frontend
 }
 
 backend_build () {
 
     docker build \
-        --tag "${image_prefix}/backend:latest" \
-        --tag "${image_prefix}/backend:${CI_COMMIT}" backend
+        --tag "isco/backend:latest" \
+        --tag "isco/backend:${CI_COMMIT}" backend
 
     # Test and Code Quality
+    dc down
     dc -f docker-compose.test.yml \
         -p backend-test \
         run --rm -T backend ./test.sh

ci/deploy.sh

@@ -1,45 +1,87 @@
 #!/usr/bin/env bash
+# The required env vars require a registry_prefix depending on the deploy environment:
+#   prod: PROD_
+#   test: TEST_
+# For example the required var is CLOUDSDK_CORE_PROJECT.
+# In the prod environment it should be PROD_CLOUDSDK_CORE_PROJECT
+#
+# Required env vars:
+#   CLOUDSDK_CORE_PROJECT - ID of the GCP project
+#   CLOUDSDK_CONTAINER_CLUSTER - ID of the GKE cluster
+#   CLOUDSDK_COMPUTE_ZONE - the zone of the gke cluster
+#   GCP_DOCKER_HOST - Where to push the docker images to
+#   GCP_SERVICE_ACCOUNT_FILE - path to file containing GCP service account credentials
+#   IMAGE_PREFIX - The host (and path if necessary) to push the docker images to
+#
+# Optional env vars:
+#   PROD_DEPLOY - Make this a deployment to production
 set -exuo pipefail
 
-[[ "${CI_BRANCH}" !=  "main" && ! "${CI_TAG:=}" =~ promote.* ]] && { echo "Branch different than main and not a tag. Skip deploy"; exit 0; }
-[[ "${CI_PULL_REQUEST}" ==  "true" ]] && { echo "Pull request. Skip deploy"; exit 0; }
+[[ "${CI_BRANCH}" !=  "main" ]] && { echo "Branch must be 'main' to deploy"; exit 0; }
+[[ "${CI_PULL_REQUEST:-}" ==  "true" ]] && { echo "Pull request. Skip deploy"; exit 0; }
+
+export CLOUDSDK_CONTAINER_USE_CLIENT_CERTIFICATE=False
+
+generate_vars(){
+  PREFIX="$1"
+  TO_GEN=(
+    "CLOUDSDK_CORE_PROJECT"
+    "CLOUDSDK_CONTAINER_CLUSTER"
+    "CLOUDSDK_COMPUTE_ZONE"
+    "GCP_DOCKER_HOST"
+    "GCP_SERVICE_ACCOUNT_FILE"
+    "IMAGE_PREFIX"
+  )
+  for to_gen in "${TO_GEN[@]}" ; do
+      varname="${PREFIX}_${to_gen}"
+      # ${!varname} = give me value or variable with the name stored in varname
+      # bash... I know
+      echo "exporting $to_gen"
+      export "$to_gen"="${!varname}"
+  done
+}
 
 auth () {
-    gcloud auth activate-service-account --key-file=/home/semaphore/.secrets/gcp.json
-    gcloud config set project akvo-lumen
-    gcloud config set container/cluster europe-west1-d
-    gcloud config set compute/zone europe-west1-d
-    gcloud config set container/use_client_certificate False
-    gcloud auth configure-docker "eu.gcr.io"
+    gcloud auth activate-service-account --key-file="${GCP_SERVICE_ACCOUNT_FILE}"
+    gcloud auth configure-docker "${GCP_DOCKER_HOST}"
 }
 
 push_image () {
-    prefix="eu.gcr.io/akvo-lumen/isco"
-    docker push "${prefix}/${1}:${CI_COMMIT}"
-}
+    suffix="${1}:${CI_COMMIT}"
 
-prepare_deployment () {
-    cluster="test"
+    local_name="isco/$suffix"
+    remote_name="${IMAGE_PREFIX}/$suffix"
 
-    if [[ "${CI_TAG:=}" =~ promote.* ]]; then
-        cluster="production"
-    fi
+    docker tag $local_name $remote_name
+    docker push "${remote_name}"
+}
 
-    gcloud container clusters get-credentials "${cluster}"
+prepare_deployment () {
+    gcloud container clusters get-credentials "${CLOUDSDK_CONTAINER_CLUSTER}"
 
     sed "s/\${CI_COMMIT}/${CI_COMMIT}/g;" \
-        ci/k8s/deployment.yml.template \
-        | sed "s/\${BUCKET_FOLDER}/${cluster}/g;" > ci/k8s/deployment.yml
+        ci/k8s/deployment.template.yml \
+        | sed "s/\${BUCKET_FOLDER}/${CLOUDSDK_CONTAINER_CLUSTER}/g;" \
+        | sed "s|\${IMAGE_PREFIX}|${IMAGE_PREFIX}|g;" \
+        > ci/k8s/deployment.yml
 }
 
 apply_deployment () {
     kubectl apply -f ci/k8s/deployment.yml
     kubectl apply -f ci/k8s/service.yml
 }
 
+set +x  # Disable printing the variable values; values might be secret
+if [[ -n "${PROD_DEPLOY:-}" ]] ; then
+  generate_vars "PROD"
+else
+  generate_vars "TEST"
+fi
+set -x  # Renable it
+
 auth
 
-if [[ -z "${CI_TAG:=}" ]]; then
+if [[ -z "${PROD_DEPLOY:-}" ]] ; then
     push_image backend
     push_image frontend
 fi