diff --git a/ansible/consul-values.yml b/ansible/consul-values.yml index 5f226ffd..87405e35 100644 --- a/ansible/consul-values.yml +++ b/ansible/consul-values.yml @@ -17,6 +17,7 @@ with_items: "{{ consul_seed }}" - name: get values from consul + tags: consul_values run_once: true when: read_values is defined uri: @@ -26,6 +27,7 @@ register: values - name: print values to screen + tags: consul_values run_once: true when: read_values is defined debug: msg="{{ item.item.key }}" -> "{{ item.json[0].Value | b64decode }}" diff --git a/ansible/delta-hosts/hosts b/ansible/delta-hosts/hosts index c18acb2a..556368bf 100644 --- a/ansible/delta-hosts/hosts +++ b/ansible/delta-hosts/hosts @@ -1,6 +1,9 @@ [bastion] delta-bastion +[user-local] +127.0.0.1 + [hipache] delta-hipache httpsCheckForBackend80=false prependIncomingPort=true subDomainDepth=4 @@ -172,6 +175,7 @@ shiva socket-server socket-server-proxy swarm-manager +user-local userland web worker diff --git a/ansible/dock-generate-launch-config.yml b/ansible/dock-generate-launch-config.yml new file mode 100644 index 00000000..89603148 --- /dev/null +++ b/ansible/dock-generate-launch-config.yml @@ -0,0 +1,8 @@ +--- +- hosts: redis +- hosts: consul + +- hosts: user-local + connection: local + roles: + - { role: dock_launch_config } diff --git a/ansible/dock.yml b/ansible/dock.yml index 29e1f932..bc3daba9 100644 --- a/ansible/dock.yml +++ b/ansible/dock.yml @@ -11,9 +11,7 @@ name={{ dock }} groups=dock -- include: charon.yml git_branch="v5.0.1" -- include: dock-init.yml git_branch="v10.1.2" -- include: krain.yml git_branch="v0.3.0" +- include: image-builder.yml git_branch="v4.3.0" - hosts: "{{ dock }}" tasks: @@ -28,3 +26,7 @@ roles: - { role: install-ssm } - { role: dock-images } + +- include: charon.yml git_branch="v5.0.1" +- include: dock-init.yml git_branch="v10.1.2" +- include: krain.yml git_branch="v0.3.1" diff --git a/ansible/gamma-hosts/hosts b/ansible/gamma-hosts/hosts index 163577d6..9d569e3d 100644 --- a/ansible/gamma-hosts/hosts +++ b/ansible/gamma-hosts/hosts @@ -1,6 +1,9 @@ [bastion] gamma-bastion +[user-local] +127.0.0.1 + [hipache] gamma-hipache httpsCheckForBackend80=false prependIncomingPort=true subDomainDepth=4 @@ -171,6 +174,7 @@ socket-server socket-server-proxy swarm-manager userland +user-local web worker diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index dc6b33f4..1501730a 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -207,6 +207,9 @@ navi_mongo_port: 27017 npm_token: c76363e9-78e0-4667-82ac-e2ac01efcfe2 +# remote vault +vault_port: 8200 + # local-vault vault_local_port: 31836 vault_addr: http://127.0.0.1:{{ vault_local_port }} diff --git a/ansible/group_vars/alpha-docker-listener.yml b/ansible/group_vars/alpha-docker-listener.yml index 758c06e6..8ef1192e 100644 --- a/ansible/group_vars/alpha-docker-listener.yml +++ b/ansible/group_vars/alpha-docker-listener.yml @@ -13,6 +13,7 @@ container_envs: > -e DATADOG_HOST={{ datadog_host_address }} -e DATADOG_PORT={{ datadog_port }} -e DOCKER_CERT_PATH=/etc/ssl/docker + -e IMAGE_INSPECT_LIST=localhost,runnable/image-builder -e LOGGLY_TOKEN={{ loggly_token }} -e NODE_ENV={{ node_env }} -e RABBITMQ_HOSTNAME={{ rabbit_host_address }} diff --git a/ansible/group_vars/alpha-krain.yml b/ansible/group_vars/alpha-krain.yml index 285fb62d..454c1792 100644 --- a/ansible/group_vars/alpha-krain.yml +++ b/ansible/group_vars/alpha-krain.yml @@ -6,7 +6,6 @@ krain_env: default # upstart template variables app_name: krain app_repo: git@github.com:CodeNow/krain.git -node_env: "{{ krain_env }}" enviroment_vars: {} diff --git a/ansible/group_vars/alpha-vault.yml b/ansible/group_vars/alpha-vault.yml index 3d1f8c45..4d4faa4c 100644 --- a/ansible/group_vars/alpha-vault.yml +++ b/ansible/group_vars/alpha-vault.yml @@ -13,7 +13,7 @@ container_run_opts: > -v /opt/vault/client-consul:/opt/vault/client-consul:ro -v /opt/vault/server:/opt/vault/server:ro -v {{ app_log_dir }}:{{ app_log_dir }}:rw - -p {{ ansible_default_ipv4.address }}:8200:8200 + -p {{ ansible_default_ipv4.address }}:{{ vault_port }}:{{ vault_port }} -p {{ ansible_default_ipv4.address }}:8201:8201 --cap-add IPC_LOCK --restart=always diff --git a/ansible/roles/base_ubuntu/tasks/main.yml b/ansible/roles/base_ubuntu/tasks/main.yml index ee624a85..fbac16c8 100644 --- a/ansible/roles/base_ubuntu/tasks/main.yml +++ b/ansible/roles/base_ubuntu/tasks/main.yml @@ -1,6 +1,7 @@ --- - name: ensure registry.runnable in /etc/hosts become: true + tags: ensure_registry when: dock is not defined lineinfile: dest=/etc/hosts diff --git a/ansible/roles/consul_value/tasks/main.yml b/ansible/roles/consul_value/tasks/main.yml index e750812f..e92441f5 100644 --- a/ansible/roles/consul_value/tasks/main.yml +++ b/ansible/roles/consul_value/tasks/main.yml @@ -14,7 +14,7 @@ cache_valid_time=604800 - name: put values into consul - tags: deploy + tags: deploy, consul_values run_once: true when: consul_host_address is defined and consul_api_port is defined and consul_values is defined and dock is not defined uri: diff --git a/ansible/roles/dock-images/tasks/main.yml b/ansible/roles/dock-images/tasks/main.yml index 6a75f343..774a8f2e 100644 --- a/ansible/roles/dock-images/tasks/main.yml +++ b/ansible/roles/dock-images/tasks/main.yml @@ -5,19 +5,10 @@ name=docker state=started -- name: add runnable registry to /etc/hosts - become: true - blockinfile: - dest: /etc/hosts - insertafter: EOF - block: | - {{ registry_address }} registry.runnable.com - - name: pulling docker images become: true command: docker pull {{ item }} with_items: - - "registry.runnable.com/runnable/image-builder:v4.3.0" - "swarm:1.2.5" - "registry:2.3.1" - "google/cadvisor:v0.24.1" diff --git a/ansible/roles/dock-init/tasks/main.yml b/ansible/roles/dock-init/tasks/main.yml index fd0bc261..805169ca 100644 --- a/ansible/roles/dock-init/tasks/main.yml +++ b/ansible/roles/dock-init/tasks/main.yml @@ -1,5 +1,6 @@ --- - fail: msg="value tokens need to be defined for this role" + tags: vault_files when: vault_auth_token is not defined or vault_token_01 is not defined or vault_token_02 is not defined or vault_token_03 is not defined - name: create vault auth directory @@ -25,12 +26,10 @@ - { file_name: 'token-02', value: "{{ vault_token_02 }}" } - { file_name: 'token-03', value: "{{ vault_token_03 }}" } -- fail: msg="docks_rollbar_key needs to be defined for this role" - when: docks_rollbar_key is not defined - - name: copy rollbar token tags: rollbar become: true + when: docks_rollbar_key is defined lineinfile: dest="/opt/runnable/dock-init/key/rollbar.token" line="{{ docks_rollbar_key }}" diff --git a/ansible/roles/dock_launch_config/tasks/main.yml b/ansible/roles/dock_launch_config/tasks/main.yml new file mode 100644 index 00000000..c8276e79 --- /dev/null +++ b/ansible/roles/dock_launch_config/tasks/main.yml @@ -0,0 +1,72 @@ +--- +- name: load variables + include_vars: "group_vars/alpha-krain.yml" + +- name: create new config file for krain + template: + src=upstart.conf + dest=~/{{ app_name }}.conf + backup=yes + vars: + - app_name: "krain" + - enviroment_vars: enviroment_vars + +- name: encode krain config to base64 + shell: cat ~/{{ app_name }}.conf | base64 -w 0 + register: krain_base64 + vars: + - app_name: "krain" + +- name: load variables + include_vars: "group_vars/alpha-charon.yml" + +- name: create new config file for charon + template: + src=upstart.conf + dest=~/{{ app_name }}.conf + backup=yes + vars: + - app_name: "charon" + - enviroment_vars: enviroment_vars + +- name: encode krain config to base64 + shell: cat ~/{{ app_name }}.conf | base64 -w 0 + register: charon_base64 + vars: + - app_name: "charon" + +- name: register tokens + set_fact: + vault_tokens: + - { file_name: 'auth-token', value: "{{ vault_auth_token }}" } + - { file_name: 'token-01', value: "{{ vault_token_01 }}" } + - { file_name: 'token-02', value: "{{ vault_token_02 }}" } + - { file_name: 'token-03', value: "{{ vault_token_03 }}" } + +- name: encode ca.pem to base64 + shell: cat ./certs/ca.pem | base64 -w 0 + register: ca_pem_base64 + +- name: encode ca-key.pem to base64 + shell: cat ./certs/ca-key.pem | base64 -w 0 + register: ca_key_pem_base64 + +- name: encode pass to base64 + shell: cat ./certs/pass | base64 -w 0 + register: pass_base64 + +- name: Generate dock script + template: + src=init.tmpl + dest=~/dock.sh + vars: + tokens: "{{ vault_tokens }}" + is_dock_pool: false + +- name: Generate dock script + template: + src=init.tmpl + dest=~/dock-pool.sh + vars: + tokens: "{{ vault_tokens }}" + is_dock_pool: true diff --git a/ansible/roles/dock_launch_config/templates/init.tmpl b/ansible/roles/dock_launch_config/templates/init.tmpl new file mode 100644 index 00000000..550a9f83 --- /dev/null +++ b/ansible/roles/dock_launch_config/templates/init.tmpl @@ -0,0 +1,38 @@ +#!/bin/bash + +# Set ENV files +export CONSUL_PORT={{ consul_api_port }} +export CONSUL_HOSTNAME={{ consul_host_address }} +export VAULT_PORT={{ vault_port }} + +# Create directory for env +mkdir -p /opt/runnable/dock-init/consul-resources/vault/{{ node_env }} +chown ubuntu:ubuntu /opt/runnable/dock-init/consul-resources/vault/{{ node_env }} +chmod 0711 /opt/runnable/dock-init/consul-resources/vault/{{ node_env }} + +# Set Vault Tokens (Used for fetching templates) +{% for item in tokens %} +echo {{ item.value }} > /opt/runnable/dock-init/consul-resources/vault/{{ node_env }}/{{ item.file_name }} +{% endfor %} + +# Add upstart files for charon and krain +mkdir -p /docker/app-logs/ +echo {{ krain_base64['stdout'] }} | base64 --decode > /etc/init/krain.conf +echo {{ charon_base64['stdout'] }} | base64 --decode > /etc/init/charon.conf + +# Add Certs (Used for genereting Docker client keys + certs) +mkdir -p /etc/ssl/docker/ +echo {{ ca_pem_base64['stdout'] }} | base64 --decode > /etc/ssl/docker/ca.pem +echo {{ ca_key_pem_base64['stdout'] }} | base64 --decode > /etc/ssl/docker/ca-key.pem +echo {{ pass_base64['stdout'] }} | base64 --decode > /etc/ssl/docker/pass +chmod -R 0440 /etc/ssl/docker/ + +# Start services +{% if is_dock_pool %} +start amazon-ssm-agent +{% endif %} +service krain start +service charon start +{% if not is_dock_pool %} +CONSUL_HOSTNAME={{ consul_host_address }} CONSUL_PORT={{ consul_api_port }} VAULT_PORT={{ vault_port }} bash /opt/runnable/dock-init/init.sh | tee /var/log/user-script-dock-init.log +{% endif %} diff --git a/ansible/roles/dock_launch_config/templates/upstart.conf b/ansible/roles/dock_launch_config/templates/upstart.conf new file mode 100644 index 00000000..38e0e7ad --- /dev/null +++ b/ansible/roles/dock_launch_config/templates/upstart.conf @@ -0,0 +1,41 @@ +#!upstart +description "{{ app_name }}" +author "Jorge Silva" + +env NPM_BIN=/usr/local/bin/npm +env APP_DIR=/opt/runnable/{{ app_name }} +env LOG_FILE={{ app_log_dir }}/{{ app_name }}.log +env NODE_ENV={{ node_env }} + +{% if enviroment_vars is defined %} +{% for name, value in enviroment_vars.iteritems() %} +env {{ name }}={{ value }} +{% endfor %} +{% endif %} + +start on (local-filesystems and net-device-up IFACE=eth0) +stop on shutdown + +script + touch $LOG_FILE + chdir $APP_DIR + echo $$ > /var/run/{{ app_name }}.pid + exec $NPM_BIN start >> $LOG_FILE 2>&1 +end script + +pre-start script + # Date format same as (new Date()).toISOString() for consistency + echo "[`date -u +%Y-%m-%dT%T.%3NZ`] (sys) Starting" >> $LOG_FILE +end script + +pre-stop script + rm /var/run/{{ app_name }}.pid + echo "[`date -u +%Y-%m-%dT%T.%3NZ`] (sys) Stopping" >> $LOG_FILE +end script + +post-start script + echo "===== App restarted =====" >> $LOG_FILE +end script + +respawn +respawn limit 5 1 # give up restart after 5 respawns in 1 seconds diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml index e2e3fffd..427a7877 100644 --- a/ansible/roles/docker/tasks/main.yml +++ b/ansible/roles/docker/tasks/main.yml @@ -15,17 +15,6 @@ owner=root group=root -- name: copy docker certs - become: true - tags: docker_certs - when: dock is defined - copy: - src=certs/ - dest=/etc/ssl/docker - mode=0440 - owner=root - group=root - - name: create core file dir become: true when: docker_config == "runnable" and core_file_dir != "/var/log" diff --git a/ansible/roles/git_repo/tasks/main.yml b/ansible/roles/git_repo/tasks/main.yml index c30e2279..56ab69d7 100644 --- a/ansible/roles/git_repo/tasks/main.yml +++ b/ansible/roles/git_repo/tasks/main.yml @@ -9,6 +9,7 @@ msg: "application Installed: {{ app_name }}, branch : {{ git_branch }} " - name: create repository dir + become: true file: path=/opt/runnable/{{ app_name }} state=directory diff --git a/ansible/roles/image-builder/tasks/main.yml b/ansible/roles/image-builder/tasks/main.yml index ac71aa5e..1a29a3aa 100644 --- a/ansible/roles/image-builder/tasks/main.yml +++ b/ansible/roles/image-builder/tasks/main.yml @@ -7,10 +7,10 @@ - name: build the image-builder tags: deploy - command: sudo docker build --no-cache --tag="registry.runnable.com/{{ image_builder_docker_namespace }}:{{ git_branch }}" /opt/runnable/image-builder + become: true + command: docker build --no-cache --tag="{{ image_builder_docker_namespace }}:{{ git_branch }}" /opt/runnable/image-builder -- name: push image-builder +- name: tag the image-builder with registry.runnable.com tags: deploy - run_once: true - command: sudo docker push "registry.runnable.com/{{ image_builder_docker_namespace }}:{{ git_branch }}" - when: dock is not defined + become: true + command: docker tag {{ image_builder_docker_namespace }}:{{ git_branch }} registry.runnable.com/{{ image_builder_docker_namespace }}:{{ git_branch }} diff --git a/ansible/roles/loggly/tasks/main.yml b/ansible/roles/loggly/tasks/main.yml index a9556ae1..2d29fd14 100644 --- a/ansible/roles/loggly/tasks/main.yml +++ b/ansible/roles/loggly/tasks/main.yml @@ -122,7 +122,7 @@ state=present - name: runnable bin directory - tags: [ loggly, clean ] + tags: [ loggly, clean ] become: true file: path=/opt/runnable/bin @@ -141,7 +141,7 @@ mode=0700 - name: purge log files - tags: [ loggly, clean ] + tags: [ loggly, clean ] become: true cron: name="purge log files" diff --git a/ansible/roles/node_service/tasks/main.yml b/ansible/roles/node_service/tasks/main.yml index 8b454355..48c77851 100644 --- a/ansible/roles/node_service/tasks/main.yml +++ b/ansible/roles/node_service/tasks/main.yml @@ -22,7 +22,7 @@ create=yes - name: create new config file - tags: deploy + tags: deploy,render_node_service_config become: true template: src=upstart.conf @@ -30,7 +30,7 @@ backup=yes - name: restart service {{ app_name }} - tags: deploy + tags: deploy,render_node_service_config become: true when: dock is not defined service: diff --git a/ansible/vault-values.yml b/ansible/vault-values.yml index d856017e..5d8a063e 100644 --- a/ansible/vault-values.yml +++ b/ansible/vault-values.yml @@ -13,7 +13,7 @@ when: write_values is defined uri: method=PUT - url=http://{{ ansible_default_ipv4.address }}:8200/v1/{{ item.key }} + url=http://{{ ansible_default_ipv4.address }}:{{ vault_port }}/v1/{{ item.key }} HEADER_X-Vault-Token="{{ vault_auth_token }}" body_format=json body='{{ item.data | to_json }}' @@ -25,7 +25,7 @@ when: write_values is defined uri: method=GET - url=http://{{ ansible_default_ipv4.address }}:8200/v1/sys/mounts + url=http://{{ ansible_default_ipv4.address }}:{{ vault_port }}/v1/sys/mounts HEADER_X-Vault-Token="{{ vault_auth_token }}" return_content=yes register: mounts @@ -36,7 +36,7 @@ uri: method=POST follow_redirects=all - url=http://{{ ansible_default_ipv4.address }}:8200/v1/sys/mounts/aws_1h + url=http://{{ ansible_default_ipv4.address }}:{{ vault_port }}/v1/sys/mounts/aws_1h HEADER_X-Vault-Token="{{ vault_auth_token }}" body_format=json body='{{ item | to_json }}' @@ -53,7 +53,7 @@ uri: method=POST follow_redirects=all - url=http://{{ ansible_default_ipv4.address }}:8200/v1/sys/mounts/aws_1yr + url=http://{{ ansible_default_ipv4.address }}:{{ vault_port }}/v1/sys/mounts/aws_1yr HEADER_X-Vault-Token="{{ vault_auth_token }}" body_format=json body='{{ item | to_json }}' @@ -70,7 +70,7 @@ uri: method=POST follow_redirects=all - url=http://{{ ansible_default_ipv4.address }}:8200/v1/aws_1h/config/root + url=http://{{ ansible_default_ipv4.address }}:{{ vault_port }}/v1/aws_1h/config/root HEADER_X-Vault-Token="{{ vault_auth_token }}" body_format=json body='{{ item | to_json }}' @@ -87,7 +87,7 @@ uri: method=POST follow_redirects=all - url=http://{{ ansible_default_ipv4.address }}:8200/v1/aws_1yr/config/root + url=http://{{ ansible_default_ipv4.address }}:{{ vault_port }}/v1/aws_1yr/config/root HEADER_X-Vault-Token="{{ vault_auth_token }}" body_format=json body='{{ item | to_json }}' @@ -104,7 +104,7 @@ uri: method=GET follow_redirects=all - url=http://{{ ansible_default_ipv4.address }}:8200/v1/aws_1h/roles/dock-init + url=http://{{ ansible_default_ipv4.address }}:{{ vault_port }}/v1/aws_1h/roles/dock-init HEADER_X-Vault-Token="{{ vault_auth_token }}" status_code=200,404 register: role @@ -115,7 +115,7 @@ uri: method=POST follow_redirects=all - url=http://{{ ansible_default_ipv4.address }}:8200/v1/aws_1h/roles/dock-init + url=http://{{ ansible_default_ipv4.address }}:{{ vault_port }}/v1/aws_1h/roles/dock-init HEADER_X-Vault-Token="{{ vault_auth_token }}" body_format=json body='{{ item | to_json | replace("\\\\", "") }}' diff --git a/ansible/vault.yml b/ansible/vault.yml index fe9b3357..c0c5a4f2 100644 --- a/ansible/vault.yml +++ b/ansible/vault.yml @@ -13,7 +13,7 @@ tags: [ deploy ] uri: method=GET - url=http://{{ ansible_default_ipv4.address }}:8200/v1/sys/seal-status + url=http://{{ ansible_default_ipv4.address }}:{{ vault_port }}/v1/sys/seal-status HEADER_X-Vault-Token="{{ vault_auth_token }}" return_content=yes register: seal_status @@ -23,7 +23,7 @@ when: seal_status.json.sealed uri: method=PUT - url=http://{{ ansible_default_ipv4.address }}:8200/v1/sys/unseal + url=http://{{ ansible_default_ipv4.address }}:{{ vault_port }}/v1/sys/unseal HEADER_X-Vault-Token="{{ vault_auth_token }}" body_format=json body='{{ item | to_json }}'