Allow to store validator keystore file in the docker swarm secret

CONFIGURATION.md

@@ -53,6 +53,8 @@ ORACLE_SHUTDOWN_CONTRACT_METHOD | Method signature to be used in the side chain
 ORACLE_FOREIGN_RPC_BLOCK_POLLING_LIMIT | Max length for the block range used in `eth_getLogs` requests for polling contract events for the Foreign chain. Infinite, if not provided. | `integer`
 ORACLE_HOME_RPC_BLOCK_POLLING_LIMIT | Max length for the block range used in `eth_getLogs` requests for polling contract events for the Home chain. Infinite, if not provided. | `integer`
 ORACLE_JSONRPC_ERROR_CODES | Override default JSON rpc error codes that can trigger RPC fallback to the next URL from the list (or a retry in case of a single RPC URL). Default is `-32603,-32002,-32005`. Should be a comma-separated list of negative integers. | `string`
+ORACLE_VALIDATOR_KEYSTORE_PATH | Path to the keystore v3 json file with the encrypted validator key. | `string`  
+ORACLE_VALIDATOR_KEYSTORE_PASSWORD | Password from the provided keystore file, oracle won't startup properly, if the provided password is invalid | `string`
 
 
 ## Monitor configuration

deployment-e2e/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7-stretch
+FROM python:3.7
 RUN curl -fsSL https://get.docker.com | sh
-RUN pip3 install docker molecule==2.22rc1 molecule[docker] flake8
+RUN pip3 install docker molecule[docker,ansible] pytest pytest-testinfra flake8
 WORKDIR mono/deployment-e2e

deployment-e2e/molecule/monitor/molecule.yml

@@ -3,12 +3,6 @@ dependency:
   name: galaxy
 driver:
   name: docker
-lint:
-  name: yamllint
-  enabled: True
-  options:
-    config-data:
-      ignore: ../../hosts.yml
 platforms:
   - name: monitor-host
     groups:
@@ -22,11 +16,6 @@ platforms:
       - /var/run/docker.sock:/var/run/docker.sock
 provisioner:
   name: ansible
-  lint:
-    name: ansible-lint
-    enabled: True
-    options:
-      r: ["bug"]
   playbooks:
     prepare: ../prepare.yml
     converge: ./converge.yml
@@ -37,14 +26,11 @@ provisioner:
         syslog_server_port: "udp://127.0.0.1:514"
 verifier:
   name: testinfra
-  lint:
-    name: flake8
   additional_files_or_dirs:
     - ../../tests/*
 scenario:
   name: monitor
   test_sequence:
-    - lint
     - cleanup
     - destroy
     - dependency

deployment-e2e/molecule/multiple/molecule.yml

@@ -3,12 +3,6 @@ dependency:
   name: galaxy
 driver:
   name: docker
-lint:
-  name: yamllint
-  enabled: True
-  options:
-    config-data:
-      ignore: ../../hosts.yml
 platforms:
   - name: multiple-host
     groups:
@@ -23,11 +17,6 @@ platforms:
       - /var/run/docker.sock:/var/run/docker.sock
 provisioner:
   name: ansible
-  lint:
-    name: ansible-lint
-    enabled: True
-    options:
-      r: ["bug"]
   playbooks:
     prepare: ../prepare.yml
     converge: ../monitor/converge.yml
@@ -39,14 +28,11 @@ provisioner:
         syslog_server_port: "udp://127.0.0.1:514"
 verifier:
   name: testinfra
-  lint:
-    name: flake8
   additional_files_or_dirs:
     - ../../tests/*
 scenario:
   name: multiple
   test_sequence:
-    - lint
     - cleanup
     - destroy
     - dependency

deployment-e2e/molecule/oracle/molecule.yml

@@ -3,48 +3,35 @@ dependency:
   name: galaxy
 driver:
   name: docker
-lint:
-  name: yamllint
-  enabled: True
-  options:
-    config-data:
-      ignore: ../../hosts.yml
 platforms:
   - name: oracle-host
     groups:
       - example
     children:
-      - oracle
+      - oracle_swarm
     image: ubuntu:16.04
     privileged: true
     network_mode: host
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
 provisioner:
   name: ansible
-  lint:
-    name: ansible-lint
-    enabled: True
-    options:
-      r: ["bug"]
   playbooks:
     prepare: ../prepare.yml
     converge: ../../../deployment/site.yml
   inventory:
     host_vars:
       oracle-host:
-        ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY: "6c48435bd464a53ed66ed62127c4dba8af75cf1a99a8ebe2680599948fbfbc6d"
+        ORACLE_VALIDATOR_KEYSTORE_PATH: "../../../e2e-commons/keystore.json"
+        ORACLE_VALIDATOR_KEYSTORE_PASSWORD: "12345678"
         syslog_server_port: "udp://127.0.0.1:514"
 verifier:
   name: testinfra
-  lint:
-    name: flake8
   additional_files_or_dirs:
     - ../../tests/*
 scenario:
   name: oracle
   test_sequence:
-    - lint
     - cleanup
     - destroy
     - dependency

deployment-e2e/molecule/oracle/tests/test_oracle.py

@@ -3,22 +3,21 @@
 import testinfra.utils.ansible_runner
 
 testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
-    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('oracle')
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('oracle_swarm')
 
 
 @pytest.mark.parametrize("name", [
-    ("oracle_rabbit_1"),
-    ("oracle_redis_1"),
-    ("oracle_bridge_request_1"),
-    ("oracle_bridge_collected_1"),
-    ("oracle_bridge_affirmation_1"),
-    ("oracle_bridge_senderhome_1"),
-    ("oracle_bridge_senderforeign_1"),
-    ("oracle_bridge_shutdown_1"),
+    ("oracle_rabbit"),
+    ("oracle_redis"),
+    ("oracle_bridge_request"),
+    ("oracle_bridge_collected"),
+    ("oracle_bridge_affirmation"),
+    ("oracle_bridge_senderhome"),
+    ("oracle_bridge_senderforeign"),
+    ("oracle_bridge_shutdown"),
 ])
 def test_docker_containers(host, name):
-    container = host.docker(name)
-    assert container.is_running
+    assert host.docker(name) is not None
 
 
 @pytest.mark.parametrize("service", [

deployment-e2e/molecule/ultimate-amb/molecule.yml

@@ -25,8 +25,6 @@ provisioner:
         ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY: "8e829f695aed89a154550f30262f1529582cc49dc30eff74a6b491359e0230f9"
 verifier:
   name: testinfra
-  lint:
-    name: flake8
 scenario:
   name: ultimate-amb
   test_sequence:

deployment-e2e/molecule/ultimate-erc-to-native/molecule.yml

@@ -27,8 +27,6 @@ provisioner:
         ORACLE_FOREIGN_START_BLOCK: 1
 verifier:
   name: testinfra
-  lint:
-    name: flake8
 scenario:
   name: ultimate-erc-to-native
   test_sequence:

deployment/hosts.yml.example

@@ -7,6 +7,13 @@ sokol-kovan:
           ansible_user: ubuntu
           ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
           #syslog_server_port: "udp://127.0.0.1:514"
+    oracle_swarm:
+      hosts:
+        127.0.0.1:
+          ansible_user: ubuntu
+          ORACLE_VALIDATOR_KEYSTORE_PATH: "/path/to/keystore.json"
+          ORACLE_VALIDATOR_KEYSTORE_PASSWORD: "12345678"
+          #syslog_server_port: "udp://127.0.0.1:514"
     monitor:
       hosts:
         127.0.0.1:

deployment/roles/common/files/daemon.json

@@ -1,4 +1,4 @@
 {
-    "live-restore": true,
+    "live-restore": false,
     "no-new-privileges": true
 }

deployment/roles/common/tasks/dependencies.yml

@@ -31,6 +31,7 @@
     owner: "root"
     group: "root"
     mode: "0755"
+  when: skip_compose is undefined
 
 - name: Upgrade pip version
   shell: pip3 install --upgrade pip==19.3.1
@@ -45,6 +46,9 @@
     group: docker
     createhome: yes
 
+- name: reset ssh connection to allow user changes to affect ansible user
+  meta: reset_connection
+
 - name: Install auditd
   apt:
     name: auditd

deployment/roles/oracle_swarm/defaults/main.yml

@@ -0,0 +1,9 @@
+---
+bridge_path: "/home/{{ compose_service_user }}/bridge"
+bridge_data_path: "/home/{{ compose_service_user }}/bridge_data"
+ORACLE_ALLOW_HTTP_FOR_RPC: no
+ORACLE_QUEUE_URL: amqp://rabbit
+ORACLE_REDIS_URL: redis://redis
+keyfile_path: "/root/.key"
+keystore_path: "/root/.keystore.json"
+oracle_image: poanetwork/tokenbridge-oracle:latest

deployment/roles/oracle_swarm/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: common, skip_repo: true, skip_compose: true }

deployment/roles/oracle_swarm/tasks/jumpbox.yml

@@ -0,0 +1,8 @@
+---
+- name: Pull the containers images
+  community.docker.docker_image:
+    name: "{{ oracle_image }}"
+    source: pull
+  when: skip_pull is undefined
+  vars:
+    ansible_python_interpreter: /usr/bin/python3

deployment/roles/oracle_swarm/tasks/logging.yml

@@ -0,0 +1,41 @@
+---
+- name: Set the oracle's containers local logs configuration file
+  template:
+    src: 31-oracle-docker.conf.j2
+    dest: /etc/rsyslog.d/31-oracle-docker.conf
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Set the redis container local logs configuration file
+  template:
+    src: 32-redis-docker.conf.j2
+    dest: /etc/rsyslog.d/32-redis-docker.conf
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Set the rabbit MQ container local logs configuration file
+  template:
+    src: 33-rabbit-docker.conf.j2
+    dest: /etc/rsyslog.d/33-rabbit-docker.conf
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Set the log configuration file to send container logs to remote server
+  template:
+    src: 36-oracle-remote-logging.conf.j2
+    dest: /etc/rsyslog.d/36-oracle-remote-logging.conf
+    owner: root
+    group: root
+    mode: 0644
+  when: syslog_server_port is defined
+
+- name: Discarding unwanted messages in rsyslog
+  blockinfile:
+    path: /etc/rsyslog.conf
+    insertbefore: "# Where to place spool and state files"
+    marker: "#{mark} add string to discarding unwanted messages"
+    content: ':msg, contains, "ORACLE_VALIDATOR_ADDRESS_PRIVATE_KEY" ~'
+  notify: restart rsyslog

deployment/roles/oracle_swarm/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+- include_tasks: pre_config.yml
+- include_tasks: logging.yml
+- include_tasks: jumpbox.yml
+- include_tasks: post_config.yml
+- include_tasks: servinstall.yml

deployment/roles/oracle_swarm/tasks/post_config.yml

@@ -0,0 +1,46 @@
+---
+- name: Get blocks
+  become_user: "{{ compose_service_user }}"
+  shell: docker run --env-file .env --rm {{ oracle_image }} scripts/getValidatorStartBlocks.js
+  args:
+    chdir: "{{ bridge_path }}/oracle"
+  register: BLOCKS
+  when: (ORACLE_HOME_START_BLOCK is not defined) or (ORACLE_FOREIGN_START_BLOCK is not defined)
+
+- name: Write blocks
+  blockinfile:
+    path: "{{ bridge_path }}/oracle/.env"
+    marker: "## {mark} Calculated by scripts/getValidatorStartBlocks.js"
+    block: |
+      ORACLE_HOME_START_BLOCK={{ (BLOCKS.stdout | from_json).homeStartBlock }}
+      ORACLE_FOREIGN_START_BLOCK={{ (BLOCKS.stdout | from_json).foreignStartBlock }}
+  when: (ORACLE_HOME_START_BLOCK is not defined) or (ORACLE_FOREIGN_START_BLOCK is not defined)
+
+- name: Copy keystore file
+  copy:
+    src: "{{ ORACLE_VALIDATOR_KEYSTORE_PATH }}"
+    dest: "{{ keystore_path }}"
+    owner: root
+    group: root
+    mode: 0600
+
+- name: Create swarm secret
+  community.docker.docker_secret:
+    name: oracle_keystore
+    state: present
+    data_src: "{{ keystore_path }}"
+  vars:
+    ansible_python_interpreter: /usr/bin/python3
+
+- name: Remove unencrypted keystore file
+  file:
+    path: "{{ keystore_path }}"
+    state: absent
+
+- name: Install .key config
+  template:
+    src: key.j2
+    dest: "{{ keyfile_path }}"
+    owner: root
+    group: root
+    mode: 0600