Adapt deployment process to support our external geth instances

README.md

@@ -142,6 +142,18 @@ Do this if you want to use the virtual machine for deployment.
 
     This will run the `configure.yml` playbook for you.
 
+### Creating `concent-deployment` machine
+
+This step creates the `concent-deployment` machine meant to be used for deployment to `mainnet`, `testnet`, `staging` clusters environment and configuration machines, disks etc.
+
+- Run the `create-compute-instance-for-deployment-server.yml` playbook.
+
+    ``` bash
+    cd concent-deployment/cloud/
+    ansible-playbook create-compute-instance-for-deployment-server.yml  \
+        --inventory ../../concent-deployment-values/ansible_inventory   \
+        --user      $user
+    ```
 ### Configuring `concent-builder` machine
 
 Do this if you want to use the remote server for building and deploying.
@@ -157,6 +169,19 @@ Do this if you want to use the remote server for building and deploying.
 
     Where the `$user` shell variable contains the name of your shell account on the remote machine.
 
+### Configuring `concent-deployment` machine
+
+This step configures the `concent-deployment` machine.
+
+- Run the `configure-concent-deployment-server.yml` playbook.
+
+    ``` bash
+    cd concent-deployment/concent-builder/
+    ansible-playbook configure-concent-deployment-server.yml           \
+        --inventory ../../concent-deployment-values/ansible_inventory  \
+        --user      $user
+    ```
+
 ### Setting up Ethereum client on a separate machine
 This step installs and configures Geth on a separate machine in Google Compute Engine.
 This is optional since Geth can be deployed automatically as a part of a Concent cluster but when you have multiple clusters, having one shared instance of the client allows you to use less resources.
@@ -189,6 +214,11 @@ ansible-playbook create-vm-instances-for-geth.yml                   \
 This step must be performed separately for every user of the build server who needs to be able to access other parts of the project infrastructure on Google Cloud with `kubectl` or `gcloud`.
 It can be performed by user himself or an admin who can impersonate him with `sudo`.
 
+The `$cluster` variable determines which server the playbook will be executed on.
+For `concent-dev` it connects to `concent-builder`.
+For other values (`concent-staging`, `concent-testnet` or `concent-mainnet`) - to `concent-deployment-server`.
+This behavior applies to all playbooks, except for `build-test-and-push-containers.yml`.
+
 The `$user_name` variable below indicates the user account to be authorized.
 To perform this step you need to have the .vault files with encrypted secrets in your local `concent-secrets/` directory.
 Only cloud secrets are required in this case.
@@ -197,7 +227,7 @@ Ansible will prompt you for password required to decrypt them.
 ```bash
 cd concent-deployment/cloud/
 ansible-playbook configure-user-authentication-for-clusters.yml    \
-    --extra-vars user_name=$user_name                              \
+    --extra-vars "cluster=$cluster user_name=$user_name"           \
     --ask-vault-pass                                               \
     --inventory  ../../concent-deployment-values/ansible_inventory \
     --user       $user
@@ -219,7 +249,12 @@ ansible-playbook install-repositories.yml                          \
     --inventory  ../../concent-deployment-values/ansible_inventory \
     --user       $user
 
-ansible-playbook build-test-and-push.yml                           \
+ansible-playbook build-cluster-configuration.yml                   \
+    --extra-vars cluster=$cluster                                  \
+    --inventory  ../../concent-deployment-values/ansible_inventory \
+    --user       $user
+```
+ansible-playbook build-test-and-push-containers.yml                \
     --extra-vars cluster=$cluster                                  \
     --inventory  ../../concent-deployment-values/ansible_inventory \
     --user       $user

cloud/cluster-deploy-secrets.yml

@@ -1,5 +1,5 @@
 - hosts:
-    - concent-builder
+    - "{{ 'concent-deployment-server' if cluster in ['concent-mainnet', 'concent-testnet', 'concent-staging'] else 'concent-builder' }}"
   vars_files:
     - ../concent-builder/consts.yml
     - ../containers/versions.yml

cloud/cluster-remove-secrets.yml

@@ -1,5 +1,5 @@
 - hosts:
-    - concent-builder
+    - "{{ 'concent-deployment-server' if cluster in ['concent-mainnet', 'concent-testnet', 'concent-staging'] else 'concent-builder' }}"
   vars_files:
     - ../concent-builder/consts.yml
     - ../containers/versions.yml

cloud/configure-concent-deployment-server.yml

@@ -0,0 +1,69 @@
+- hosts:
+    - concent-deployment-server
+  vars_files:
+    - ../containers/versions.yml
+    - ../concent-builder/consts.yml
+    - ../concent-builder/repositories.yml
+    - "{{ deployment_values }}/var.yml"
+  roles:
+    - { role: upload_cluster_secrets, cluster: "concent-dev" }
+    - { role: upload_cluster_secrets, cluster: "concent-staging" }
+    - { role: upload_cluster_secrets, cluster: "concent-testnet" }
+    - { role: upload_cluster_secrets, deploy_cloud_secrets: yes  }
+  tasks:
+    - become:      yes
+      become_user: root
+      block:
+        - name:   include common prerequire tasks from `prerequire-configure-tasks.yml` file
+          include_tasks: ../concent-builder/common_tasks/prerequire-configure-tasks.yml
+
+        - name:   include task from `install-basic-utilities.yml` file
+          include_tasks: ../concent-builder/common_tasks/install-basic-utilities.yml
+
+        - name:   Install system updates for Debian
+          apt:  update_cache=yes
+
+        - name:    Install basic packages
+          apt:
+            name:
+              - ncdu
+              - tree
+              - htop
+              - tmux
+
+        - name:    Install dependencies
+          apt:
+            name:
+              - git
+              - kubectl={{ kubectl_version }}
+              - gcc
+              - postgresql-{{ postgres_version }}
+              - python3-psycopg2
+              - libssl-dev
+
+        - name:   include common postrequire tasks from `postrequire-configure-tasks.yml` file
+          include_tasks: ../concent-builder/common_tasks/postrequire-configure-tasks.yml
+
+    - block:
+        - name: Create directory for concent repositories
+          file:
+            path:  "{{ deployment_dir }}"
+            state: directory
+
+        - name: Clone concent repositories
+          git:
+            repo:    "{{ item.url }}"
+            dest:    "{{ deployment_dir }}/{{ item.name }}"
+            clone:   yes
+            update:  yes
+            version: master
+            force:   yes
+          with_items:
+            - { url: "{{ repositories['concent-deployment'].url }}", name: concent-deployment }
+            - { url: "{{ repositories['concent-deployment-values'].url }}", name: concent-deployment-values }
+
+        - name: Create symbolic link to concent-secrets in user home directory
+          file:
+            src:        "{{ data_dir }}/concent-secrets/"
+            dest:       "{{ deployment_dir }}/concent-secrets"
+            state:      link

cloud/configure-user-authentication-for-clusters.yml

@@ -1,8 +1,9 @@
 - hosts:
-    - concent-builder
+    - "{{ 'concent-deployment-server' if cluster in ['concent-mainnet', 'concent-testnet', 'concent-staging'] else 'concent-builder' }}"
   vars_files:
     - ../concent-builder/consts.yml
     - "{{ deployment_values }}/var.yml"
+    - "{{ deployment_values }}/var-{{ cluster }}.yml"
   tasks:
     - become:      yes
       become_user: root

cloud/create-compute-instance-for-deployment-server.yml

@@ -0,0 +1,20 @@
+- hosts:
+    - localhost
+  vars_files:
+    - ../concent-builder/consts.yml
+    - "{{ deployment_values }}/var.yml"
+  vars:
+    server_name: concent-deployment-server
+    name_of_ip: "{{ server_name }}-ip"
+  roles:
+    - reserve_static_ip
+  tasks:
+    - block:
+        - name:    Create a GCP instance for {{ server_name }}
+          command: >
+              gcloud compute instances create {{ server_name }}
+              --description  "Vm instance that contain {{ server_name }}"
+              --machine-type f1-micro
+              --address      {{ name_of_ip }}
+              --project      {{ gke.project }}
+              --zone         {{ gke.zone }}

cloud/create-databases.yml

@@ -1,5 +1,5 @@
 - hosts:
-    - concent-builder
+    - concent-deployment-server
   vars_files:
     - ../concent-builder/consts.yml
     - "{{ deployment_values }}/var.yml"

cloud/drop-databases.yml

@@ -1,5 +1,5 @@
 - hosts:
-    - concent-builder
+    - concent-deployment-server
   vars_files:
     - ../concent-builder/consts.yml
     - "{{ deployment_values }}/var.yml"

cloud/roles/reserve_static_ip/tasks/main.yml

@@ -0,0 +1,15 @@
+- name:   Check if static IP address already exists
+  shell: >
+      gcloud compute addresses list
+      --project {{ gke.project }}
+      | grep {{ name_of_ip }}
+  register:      ip_address_result
+  ignore_errors: yes
+
+- name:    Reserve static IP address for the "{{ server_name }}" instance
+  command: >
+      gcloud compute addresses create {{ name_of_ip }}
+      --description "Static IP attached to {{ server_name }} instance"
+      --project     {{ gke.project }}
+      --region      europe-west3
+  when: ip_address_result.stdout == ""

cloud/roles/upload_cluster_secrets/tasks/cloud-secrets.yml

@@ -0,0 +1,30 @@
+- become:      yes
+  become_user: root
+  block:
+    - name:    Create user that will be used for building stuff
+      user:
+        name:   "{{ shared_user }}"
+        state:  present
+
+    - name:    Create data_dir
+      file:
+        path:  "{{ item }}"
+        state: directory
+        owner: "{{ shared_user }}"
+        group: "{{ shared_user }}"
+        mode:  0777
+      with_items:
+        - "{{ data_dir }}"
+        - "{{ data_dir }}/concent-secrets/"
+        - "{{ data_dir }}/concent-secrets/cloud/"
+
+    - name: Upload secrets
+      copy:
+        src:     "{{ local_secret_dir }}/{{ item }}"
+        dest:    "{{ data_dir }}/concent-secrets/{{ item }}"
+        decrypt: no
+        owner:   "{{ shared_user }}"
+        group:   "{{ shared_user }}"
+      with_items:
+        - cloud/{{ gke.service_account_name }}-private-key.json.vault
+        - cloud/cloud-secrets.yml.vault

cloud/roles/upload_cluster_secrets/tasks/cluster-secrets.yml

@@ -0,0 +1,33 @@
+- become:      yes
+  become_user: root
+  block:
+    - name:    Create user that will be used for building stuff
+      user:
+        name:   "{{ shared_user }}"
+        state:  present
+
+    - name:    Create a global directory for data shared by all users
+      file:
+        path:  "{{ item }}"
+        state: directory
+        owner: "{{ shared_user }}"
+        group: "{{ shared_user }}"
+        mode:  0777
+      with_items:
+        - "{{ data_dir }}"
+        - "{{ data_dir }}/concent-secrets/"
+        - "{{ data_dir }}/concent-secrets/{{ cluster }}"
+
+    - name: Upload secrets
+      copy:
+        src:     "{{ local_secret_dir }}/{{ cluster }}/{{ item }}"
+        dest:    "{{ data_dir }}/concent-secrets/{{ cluster }}/{{ item }}"
+        decrypt: no
+        owner:   "{{ shared_user }}"
+        group:   "{{ shared_user }}"
+      with_items:
+        - cluster-secrets.yml.vault
+        - nginx-proxy-ssl.crt.vault
+        - nginx-proxy-ssl.key.vault
+        - nginx-storage-ssl.crt.vault
+        - nginx-storage-ssl.key.vault

cloud/roles/upload_cluster_secrets/tasks/main.yml

@@ -0,0 +1,5 @@
+- include_tasks: cluster-secrets.yml
+  when: cluster is defined and cluster in ['concent-dev', 'concent-staging', 'concent-testnet']
+
+- include_tasks: cloud-secrets.yml
+  when: deploy_cloud_secrets is defined and deploy_cloud_secrets

concent-builder/build-cluster-configuration.yml

@@ -0,0 +1,27 @@
+- hosts:
+    - "{{ 'concent-deployment-server' if cluster in ['concent-mainnet', 'concent-testnet', 'concent-staging'] else 'concent-builder' }}"
+  vars_files:
+    - consts.yml
+    - repositories.yml
+    - ../containers/versions.yml
+    - "{{ deployment_values }}/var.yml"
+    - "{{ deployment_values }}/var-{{ cluster }}.yml"
+  roles:
+    - clean_up_and_set_up_environment
+  tasks:
+    - name: Install var.yml from the local machine
+      copy:
+        src:   "{{ deployment_values }}/{{ item }}"
+        dest:  "{{ build_dir }}/concent-deployment/kubernetes/"
+      with_items:
+        - var.yml
+        - var-{{ cluster }}.yml
+
+    - name: Build cluster configuration
+      make:
+        chdir: "{{ build_dir }}/concent-deployment/kubernetes"
+        params:
+          CONCENT_SECRET_DIR: "{{ deployment_dir }}/concent-secrets/{{ cluster }}"
+          IMAGE_PREFIX:       "{{ image_prefix }}"
+          CLUSTER:            "{{ cluster }}"
+

concent-builder/build-test-and-push.yml → concent-builder/build-test-and-push-containers.yml

@@ -7,42 +7,13 @@
     - "{{ deployment_values }}/var.yml"
     - "{{ deployment_values }}/var-{{ cluster }}.yml"
   tasks:
-    - name: Remove the build directory to get rid of files from previous builds
-      file:
-        path:    "{{ build_dir }}"
-        state:   absent
-
-    - name: Check out working copy of concent-deployment repositories
-      git:
-        repo:    "{{ data_dir }}/concent-deployment"
-        dest:    "{{ build_dir }}/concent-deployment"
-        version: "{{ concent_versions[concent_version].versions['concent-deployment'] }}"
-        clone:   yes
-        update:  yes
-
-    - name: Install var.yml from the local machine
-      copy:
-        src:   "{{ deployment_values }}/{{ item }}"
-        dest:  "{{ build_dir }}/concent-deployment/kubernetes/"
-      with_items:
-        - var.yml
-        - var-{{ cluster }}.yml
-
     - block:
         - name: Build containers
           make:
             chdir: "{{ build_dir }}/concent-deployment/containers"
             params:
               IMAGE_PREFIX:       "{{ image_prefix }}"
 
-        - name: Build cluster configuration
-          make:
-            chdir: "{{ build_dir }}/concent-deployment/kubernetes"
-            params:
-              CONCENT_SECRET_DIR: "{{ deployment_dir }}/concent-secrets/{{ cluster }}"
-              IMAGE_PREFIX:       "{{ image_prefix }}"
-              CLUSTER:            "{{ cluster }}"
-
         - name:    Create custom network that allows communication between django and postgres containers
           command: docker network create {{ concent_versions[concent_version].gke.cluster }}-{{ image_prefix }}unittest-network
 

concent-builder/common_tasks/postrequire-configure-tasks.yml

@@ -18,6 +18,7 @@
 
 - name:   Install dependencies for pyenv
   apt:
+    default_release: testing
     name:
       - zlib1g-dev
       - libbz2-dev