Update on demand for the git cache on build server

README.md

@@ -156,13 +156,7 @@ All the instructions below assume that you're using the remote server.
 
 ### Building containers and cluster configuration
 
-``` bash
-cd concent-deployment/concent-builder/
-ansible-playbook install-repositories.yml                          \
-    --extra-vars cluster=$cluster                                  \
-    --inventory  ../../concent-deployment-values/ansible_inventory \
-    --user       $user
-
+```
 ansible-playbook build-test-and-push.yml                           \
     --extra-vars cluster=$cluster                                  \
     --inventory  ../../concent-deployment-values/ansible_inventory \
@@ -171,7 +165,15 @@ ansible-playbook build-test-and-push.yml                           \
 
 ### Deploying secrets
 Before you can deploy containers, you need to make sure that certificates, keys and passwords used to configure those containers are available on the cluster.
-Deploy them with:
+Upload them from your local machine with:
+
+``` bash
+cd concent-deployment/cloud/
+ansible-playbook install-secrets.yml                               \
+    --extra-vars cluster=$cluster                                  \
+    --inventory  ../../concent-deployment-values/ansible_inventory \
+    --user       $user
+
 
 ``` bash
 cd concent-deployment/cloud/

cloud/cluster-deploy-secrets.yml

@@ -12,7 +12,7 @@
           make:
             chdir: "{{ build_dir }}/concent-deployment/secrets"
             params:
-              CONCENT_SECRET_DIR: "{{ data_dir }}/concent-secrets/{{ cluster }}"
+              CONCENT_SECRET_DIR: "{{ secret_dir }}/{{ cluster }}"
               CLUSTER:            "{{ cluster }}"
 
         - name:    Configure kubectl to operate on the right cluster

concent-builder/install-repositories.yml → cloud/install-secrets.yml

@@ -1,28 +1,16 @@
 - hosts:
     - concent-builder
   vars_files:
-    - consts.yml
-    - repositories.yml
+    - ../concent-builder/consts.yml
     - "{{ deployment_values }}/var.yml"
     - "{{ deployment_values }}/var-{{ cluster }}.yml"
   tasks:
     - become:      yes
       become_user: "{{ shared_user }}"
       block:
-        - name: Clone concent repositories
-          git:
-            repo:           "{{ item.value.url }}"
-            dest:           "{{ data_dir }}/{{ item.key }}/"
-            clone:          yes
-            update:         yes
-            bare:           yes
-            # FIXME: Do not blindly accept the hostkey
-            accept_hostkey: yes
-          with_dict:        "{{ repositories }}"
-
         - name: Create a directory for secrets
           file:
-            path:  "{{ data_dir }}/concent-secrets/{{ item }}/"
+            path:  "{{ secret_dir }}/{{ item }}/"
             state: directory
           with_items:
             - "{{ cluster }}"
@@ -37,7 +25,7 @@
         - name: Upload secrets
           copy:
             src:   "{{ local_secret_dir }}/{{ cluster }}/{{ item }}"
-            dest:  "{{ data_dir }}/concent-secrets/{{ cluster }}/{{ item }}"
+            dest:  "{{ secret_dir }}/{{ cluster }}/{{ item }}"
             owner: "{{ shared_user }}"
             group: "{{ shared_user }}"
           with_items:
@@ -54,7 +42,7 @@
         - name: Activate the service account that has permissions to access the cluster
           command: gcloud auth activate-service-account                                                           \
               "{{ gke.service_account_name }}@{{ gke.project }}.iam.gserviceaccount.com"                          \
-              --key-file "{{ data_dir }}/concent-secrets/cloud/{{ gke.service_account_name }}-private-key.json"
+              --key-file "{{ secret_dir }}/cloud/{{ gke.service_account_name }}-private-key.json"
 
         - name: Configure kubectl to operate on the right cluster
           command: gcloud container clusters get-credentials \

concent-builder/build-test-and-push.yml

@@ -2,7 +2,6 @@
     - concent-builder
   vars_files:
     - consts.yml
-    - repositories.yml
     - "{{ deployment_values }}/var.yml"
     - "{{ deployment_values }}/var-{{ cluster }}.yml"
   tasks:
@@ -14,9 +13,15 @@
             path:    "{{ build_dir }}"
             state:   absent
 
+        - name:    Clone or update git repositories
+          service:
+            name: update-git-mirror
+            state: started
+          become_user: root
+
         - name: Check out working copy of concent-deployment repositories
           git:
-            repo:    "{{ data_dir }}/concent-deployment"
+            repo:    "{{ git_mirror_dir }}/concent-deployment"
             dest:    "{{ build_dir }}/concent-deployment"
             version: "{{ versions['concent-deployment'] }}"
             clone:   yes

concent-builder/configure.yml

@@ -105,9 +105,37 @@
             state:  present
             groups: docker
 
-        - name:    Create data_dir
+        - name:    Create git_mirror_dir and secret_dir
           file:
-            path:  "{{ data_dir }}"
+            path:  "{{ item.name }}"
             state: directory
-            owner: "{{ shared_user }}"
-            group: "{{ shared_user }}"
+            owner: "{{ item.owner }}"
+            group: "{{ item.group }}"
+            mode:  "{{ item.mode }}"
+          with_items:
+            - { name: "{{git_mirror_dir}}", owner: root, group: root, mode: "0755" }
+            - { name: "{{secret_dir}}", owner: "{{ shared_user }}", group: "{{ shared_user }}", mode: "0755" }
+
+        - name:   Copy scripts that update git repositories
+          template:
+            src:   "{{ item.name }}"
+            dest:  "{{ item.dest }}"
+            owner: root
+            group: root
+            mode:  0555
+          with_items:
+            - { name: repositories.sh, dest: /usr/local/lib/ }
+            - { name: update-git-mirror.sh.j2, dest: /usr/local/bin/update-git-mirror.sh }
+
+        - name:   Copy service that run update git repositories script
+          template:
+            src:   update-git-mirror.service.j2
+            dest:  /etc/systemd/system/update-git-mirror.service
+            owner: root
+            group: root
+            mode:  0555
+
+        - name:    Clone or update git repositories
+          service:
+            name: update-git-mirror
+            state: started

concent-builder/consts.yml

@@ -1,5 +1,6 @@
 shared_user:       builder
-data_dir:          /var/concent-data
+git_mirror_dir:    /var/git-mirror
+secret_dir:        /var/concent-secrets
 build_dir:         /home/{{ shared_user }}/build/{{ cluster }}
 local_secret_dir:  ../../concent-secrets
 deployment_values: ../../concent-deployment-values

concent-builder/repositories.sh

@@ -0,0 +1,9 @@
+#! /bin/bash -e
+
+declare -A repositories
+
+repositories=(
+    ["concent"]="https://github.com/golemfactory/concent.git"
+    ["concent-deployment"]="https://github.com/golemfactory/concent-deployment.git"
+    ["golem"]="https://github.com/golemfactory/golem.git"
+)

concent-builder/update-git-mirror.service.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Update git repository
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+WorkingDirectory={{ git_mirror_dir }}/
+ExecStart=/usr/local/bin/update-git-mirror.sh

concent-builder/update-git-mirror.sh.j2

@@ -0,0 +1,27 @@
+#! /bin/bash -e
+
+source /usr/local/lib/repositories.sh
+export repositories
+
+list_of_directories=(
+    $(ls {{ git_mirror_dir }}/)
+)
+
+# Check list of directories in `/var/git-mirror/` path with repositories list in the `repositories.sh` file.
+for directory in "${list_of_directories[@]}"; do
+    if [[ ! " ${!repositories[@]} " =~ " ${directory} " ]]; then
+        rm -rf "/var/git-mirror/$directory/"
+    fi
+done
+
+# Check if repositories from the `repositories.sh` file exist in `/var/git-mirror` path
+# and clone or update them.
+for repository in "${!repositories[@]}"; do
+    repository_path="/var/git-mirror/$repository/"
+    if [ ! -d $repository_path ]; then
+        git clone --mirror "${repositories[$repository]}" "$repository_path"
+    else
+        cd "$repository_path"
+        git fetch origin --prune
+    fi
+done

containers/package-builder/clone-or-update.sh.j2

@@ -4,10 +4,10 @@ repository=$1
 directory=$2
 
 if [[ "$repository" == "concent" ]]; then
-    repository_path=/var/concent-data/concent/
+    repository_path=/var/git-mirror/concent/
     repository_version="{{ concent_version }}"
 elif [[ "$repository" == "golem" ]]; then
-    repository_path=/var/concent-data/golem/
+    repository_path=/var/git-mirror/golem/
     repository_version="{{ golem_version }}"
 else
     echo "The '$repository' repository does not exist"