diff --git a/ansible/agreeable-egret.yml b/ansible/agreeable-egret.yml new file mode 100644 index 00000000..db0f5215 --- /dev/null +++ b/ansible/agreeable-egret.yml @@ -0,0 +1,11 @@ +--- +- hosts: redis +- hosts: agreeable-egret + vars_files: + - group_vars/alpha-agreeable-egret.yml + roles: + - role: notify + tags: [ notify ] + - role: builder + tags: [ build ] + - role: container_kill_start diff --git a/ansible/epsilon-hosts/docks.js b/ansible/default-hosts/docks.js similarity index 88% rename from ansible/epsilon-hosts/docks.js rename to ansible/default-hosts/docks.js index 4049befa..b2804306 100755 --- a/ansible/epsilon-hosts/docks.js +++ b/ansible/default-hosts/docks.js @@ -4,9 +4,9 @@ var aws = require('aws-sdk'); var ec2 = new aws.EC2({ - accessKeyId: 'AKIAJ3RCYU6FCULAJP2Q', - secretAccessKey: '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', - region: 'us-west-2' + accessKeyId: '${AWS_ACCESS_KEY_ID_1}', + secretAccessKey: '${AWS_SECRET_ACCESS_KEY_1}', + region: '${AWS_REGION}' }); var params = { @@ -14,7 +14,7 @@ var params = { // Only search for docks in the cluster security group { Name: 'instance.group-id', - Values: ['sg-3322e454'] + Values: ['${AWS_DOCK_SG}'] // This script is the same for all environments }, // Only fetch instances that are tagged as docks { diff --git a/ansible/default-hosts/hosts b/ansible/default-hosts/hosts new file mode 100644 index 00000000..2ef19a71 --- /dev/null +++ b/ansible/default-hosts/hosts @@ -0,0 +1,159 @@ +[bastion] +dafault-bastion + +[hipache] +default-main httpsCheckForBackend80=false prependIncomingPort=true subDomainDepth=4 + +[userland] +default-main + +[mongodb] +default-main + +[api_group:children] +worker +api +socket-server + +[api] +default-main + +[big-poppa] +default-main + +[cream] +default-main + +[consul] +default-main + +[docker-listener] +default-main + +[vault] +default-main + +[worker] +default-main + +[navi] +default-main + +[ingress] +default-main + +[link] +default-main + +[mongo-navi] +default-main + +[charon] +default-main + +[khronos] +default-main + +[optimus] +default-main + +[detention] +default-main + +[palantiri] +default-main + +[rabbitmq] +default-main + +[web] +default-main + +[redis] +default-main + +[redis-slave] +default-main + +[sauron] +default-main + +[shiva] +default-main + +[socket-server] +default-main + +[socket-server-proxy] +default-main + +[registry] +default-main + +[swarm-manager] +default-main + +[metis] +default-main + +[drake] +default-main + +[pheidi] +default-main + +[github-varnish] +default-main + +[single-host-proxy] +default-main + +[docks] + +[dock] + +[prometheus] +default-main + +[bear-clone:children] +api +bastion +big-poppa +charon +consul +cream +dock +docker-listener +docks +drake +hipache +ingress +khronos +metis +mongodb +navi +optimus +pheidi +prometheus +rabbitmq +redis +redis-slave +registry +sauron +shiva +single-host-proxy +socket-server +socket-server-proxy +swarm-manager +userland +web +worker + +[local] +127.0.0.1 + +[ec2] +local + +[targets] +localhost ansible_connection=local bastion_name=default-bastion diff --git a/ansible/default-hosts/variables b/ansible/default-hosts/variables new file mode 100644 index 00000000..a5128c5d --- /dev/null +++ b/ansible/default-hosts/variables @@ -0,0 +1,134 @@ +[api_group:vars] +api_aws_access_key_id=${AWS_ACCESS_KEY_ID_1} +api_aws_secret_access_key=${AWS_SECRET_ACCESS_KEY_1} +api_github_client_id=${GITHUB_CLIEND_ID} +api_github_client_secret=${GITHUB_CLIENT_SECRET} +api_github_deploy_keys_bucket=runnable.deploykeys.${ENV} +api_mongo_auth=${MONGO_USERNAME}:${MONGO_PASSWORD} +api_mongo_database=${ENV} +api_mongo_replset_name=${ENV}-rs0 +api_s3_context_bucket=runnable.context.resources.${ENV} + +[big-poppa:vars] +big_poppa_pg_pass=${POSTGRES_PASSWORD} +big_poppa_pg_host=${POSTGRES_HOST}:${POSTGRES_PORT} +big_poppa_pg_port=${POSTGRES_PORT} +big_poppa_pg_user=big_poppa +big_poppa_github_token=${GITHUB_ACCESS_TOKEN} +big_poppa_mongo_auth=${MONGO_USERNAME}:${MONGO_PASSWORD} +big_poppa_mongo_database=${MONGO_DATABASE} +big_poppa_mongo_replset_name=${MONGO_DATABASE}-rs0 +big_poppa_pg_pool_min=10 +big_poppa_pg_pool_max=20 + +[cream:vars] +cream_hello_runnable_github_token=${GITHUB_ACCESS_TOKEN} +cream_stripe_secret_key=${STRIPE_SECRET_KEY} +cream_stripe_publishable_key=${STRIPE_PUBLISHABLE_KEY} + +[docks:vars] +docker_config=docks +docks_rollbar_key=${ROLLBAR_TOKEN_DOCKS} + +[dock:vars] +docks_rollbar_key=${ROLLBAR_TOKEN_DOCKS} + +[drake:vars] +drake_port=80 + +[khronos:vars] +khronos_mongo_auth=${MONGO_USER}:${MONGO_PASSWORD} +khronos_mongo_database=${MONGO_DATABASE} +khronos_mongo_replset_name=${MONGO_DATABASE} + +[metis:vars] + +[navi:vars] +navi_cookie_secret=${COOKIE_SECRET} +_navi_proxy_port=65100 +_navi_proxy_ssl_port=65101 + +[optimus:vars] +optimus_aws_access_id=${AWS_ACCESS_KEY_ID_1} +optimus_aws_secret_id=${AWS_SECRET_ACCESS_KEY_1} +optimus_github_deploy_keys_bucket=runnable.deploykeys.${ENV} + +[palantiri:vars] + +[pheidi:vars] +pheidi_mongo_auth=${MONGO_USER}:${MONGO_PASSWORD} +pheidi_mongo_database=${MONGO_DATABASE} +pheidi_mongo_replset_name=${MONGO_DATABASE} +pheidi_runnabot_tokens=${GITHUB_ACCESS_TOKEN} + +[sauron:vars] + +[registry:vars] +registry_s3_access_key=${AWS_ACCESS_KEY_ID_1} +registry_s3_secret_key=${AWS_SECRET_ACCESS_KEY_1} +registry_s3_bucket=runnableimages.${ENV} +registry_s3_region=${AWS_REGION} + +[shiva:vars] +aws_access_key_id=${AWS_ACCESS_KEY_ID_1} +aws_secret_access_key=${AWS_ACCESS_KEY_ID_1} +shiva_aws_region=${AWS_REGION} +shiva_dock_security_groups=${AWS_DOCK_SG} +shiva_ssh_key_name=${AWS_SSH_KEY_NAME} +shiva_aws_instance_image_id=${AWS_DOCK_AMI_ID} +shiva_aws_instance_image_name=${AWS_DOCK_AMI_NAME} +shiva_aws_instance_type=t2.medium +shiva_dock_pool_asg_name=${ENV}-asg-dock-pool +shiva_aws_launch_configuration_name=${ENV}-lc-${AWS_LC_VERSION} +shiva_aws_auto_scaling_group_subnets=${AWS_ASG_SUBNET} +shiva_aws_auto_scaling_group_max=29 +shiva_aws_auto_scaling_group_prefix=asg-${ENV}- + +[swarm-manager:vars] +aws_access_key=${AWS_ACCESS_KEY_ID_1} +aws_secret_key=${AWS_SECRET_ACCESS_KEY_1} +environment_name=${ENV} + +[vault:vars] +vault_hello_runnable_github_token=${GITHUB_ACCESS_TOKEN_HELLO_RUNNABLE} +vault_aws_access_key_id=${AWS_ACCESS_KEY_ID_1} +vault_aws_secret_key=${AWS_SECRET_ACCESS_KEY_1} +vault_aws_region=${AWS_REGION} +vault_root_token=${LOCAL_VAULT_ROOT_TOKEN} +vault_unseal_tokens={'one':'${LOCAL_VAULT_TOKEN_1}', 'two': '${LOCAL_VAULT_TOKEN_2}', 'three': '${LOCAL_VAULT_TOKEN_3}', 'four': '${LOCAL_VAULT_TOKEN_4}', 'five': '${LOCAL_VAULT_TOKEN_5}'} +_vault_port=65240 +_vault_ssl_port=65241 + +[${ENV}:vars] +bastion_sshd_port=60709 +datadog_tags=env:${ENV} +datadog_mongodb_user=datadog +datadog_mongodb_pwd= +domain=${DOMAIN} +mongo_port=27017 +node_env=${ENV} +pg_user=astral +pg_pass=${POSTGRES_PASSWORD} +pg_host=${POSTGRES_HOST}:${POSTGRES_PORT} +rabbit_password=${RABBIT_PASSWORD} +rabbit_username=${RABBIT_USERNAME} +_registry_port=65001 +_consul_api_port=65200 +_consul_https_port=65201 +_swarm_master_port=65250 +user_content_domain=${USER_CONTENT_DOMAIN} +max_navi_port=65000 +_redis_port=65075 +_redis_tls_port=65076 +api_hello_runnable_github_token=${GITHUB_ACCESS_TOKEN_HELLO_RUNNABLE} +vault_auth_token=${REMOTE_VAULT_ROOT_TOKEN} +vault_token_01=${REMOTE_VAULT_TOKEN_1} +vault_token_02=${REMOTE_VAULT_TOKEN_2} +vault_token_03=${REMOTE_VAULT_TOKEN_3} +vault_token_04=${REMOTE_VAULT_TOKEN_4} +vault_token_05=${REMOTE_VAULT_TOKEN_5} +github_domain=api.github.com +is_github_enterprise=false +github_protocol=https +proxy_container_image=runnable/sticky-nginx +proxy_container_image_version=v1.8.1 diff --git a/ansible/delta-hosts/hosts b/ansible/delta-hosts/hosts index 8fa2d00b..92536b47 100644 --- a/ansible/delta-hosts/hosts +++ b/ansible/delta-hosts/hosts @@ -59,6 +59,9 @@ delta-app-services [dock] +[agreeable-egret] +delta-app-services + [eru] delta-app-services @@ -92,6 +95,9 @@ delta-rabbit [web] delta-app-services +[marketing] +delta-app-services + [metabase] delta-metabase @@ -138,6 +144,7 @@ delta-app-services delta-prometheus [delta:children] +agreeable-egret api arithmancy bastion diff --git a/ansible/delta-hosts/variables b/ansible/delta-hosts/variables index 119fb76d..219f59f3 100644 --- a/ansible/delta-hosts/variables +++ b/ansible/delta-hosts/variables @@ -1,3 +1,11 @@ +[agreeable-egret:vars] +agreeable_egret_port=65520 +egret_pg_host=delta-big-poppa.cnksgdqarobf.us-west-2.rds.amazonaws.com +egret_pg_port=5432 +egret_pg_user=egret +egret_pg_pass=wwHQ5B4RfY9iKS3m +egret_pg_database=egret + [api_group:vars] api_aws_access_key_id=AKIAJWSSSJYUXKNW2ZDA api_aws_secret_access_key=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 @@ -140,6 +148,11 @@ vault_aws_access_key_id=AKIAJ7R4UIM45KH2WGWQ vault_aws_secret_key=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 vault_aws_region=us-east-1 +[marketing:vars] +marketing_bucket=runnable.com +marketing_aws_access_key=AKIAIPPPY2JIOHX7QVCA +marketing_aws_secret_key=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 + [delta:vars] ansible_ssh_private_key_file=~/.ssh/delta.pem api_hello_runnable_github_token=88ddc423c2312d02a8bbcaad76dd4c374a30e4af @@ -196,3 +209,4 @@ vpc_id=vpc-864c6be3 [web:vars] web_intercom_id=wqzm3rju web_sift_public_key=27e9da5c97 +web_aws_bucket_region=us-west-2 diff --git a/ansible/epsilon-hosts/hosts b/ansible/epsilon-hosts/hosts deleted file mode 100644 index a427f748..00000000 --- a/ansible/epsilon-hosts/hosts +++ /dev/null @@ -1,160 +0,0 @@ -[bastion] -epsilon-bastion - -[hipache] -epsilon-hipache httpsCheckForBackend80=false prependIncomingPort=true subDomainDepth=4 - -[userland] -epsilon-userland - -[mongodb] -epsilon-mongo - -[api_group:children] -worker -api -socket-server - -[api] -epsilon-api - -[big-poppa] -epsilon-app-services - -[cream] -epsilon-app-services - -[consul] -epsilon-consul-a -epsilon-consul-b -epsilon-consul-c - -[docker-listener] -epsilon-dock-services - -[vault] -epsilon-consul-a -epsilon-consul-b -epsilon-consul-c - -[worker] -epsilon-api-worker - -[socket-server] -epsilon-api-socket - -[socket-server-proxy] -epsilon-api-socket-proxy - -[docks] - -[dock] - -[eru] -epsilon-app-services - -[navi] -epsilon-navi - -[mongo-navi] -epsilon-navi - -[link] -epsilon-navi - -[charon] -epsilon-app-services - -[khronos] -epsilon-dock-services - -[optimus] -epsilon-dock-services - -[detention] -epsilon-app-services - -[palantiri] -epsilon-dock-services - -[rabbitmq] -epsilon-rabbit - -[web] -epsilon-app-services - -[redis] -epsilon-redis - -[shiva] -epsilon-app-services - -[registry] -epsilon-registry - -[sauron] -epsilon-dock-services - -[swarm-manager] -epsilon-dock-services - -[metis] -epsilon-app-services - -[drake] -epsilon-app-services - -[pheidi] -epsilon-app-services - -[github-varnish] -epsilon-app-services - -[arithmancy] -epsilon-app-services - -[prometheus] -epsilon-prometheus - -[epsilon:children] -api -arithmancy -bastion -big-poppa -charon -consul -cream -dock -docker-listener -docks -drake -eru -github-varnish -hipache -khronos -metis -mongodb -navi -optimus -rabbitmq -redis -pheidi -prometheus -registry -sauron -shiva -socket-server -socket-server-proxy -swarm-manager -userland -web -worker - -[local] -127.0.0.1 - -[ec2] -local - -[targets] -localhost ansible_connection=local bastion_name=epsilon-bastion diff --git a/ansible/gamma-hosts/hosts b/ansible/gamma-hosts/hosts index c344426c..7feb4a76 100644 --- a/ansible/gamma-hosts/hosts +++ b/ansible/gamma-hosts/hosts @@ -45,6 +45,9 @@ gamma-consul-c [worker] gamma-api-worker +[agreeable-egret] +gamma-app-services + [eru] gamma-app-services @@ -81,6 +84,9 @@ gamma-rabbit [web] gamma-app-services +[marketing] +gamma-app-services + [redis] gamma-redis @@ -128,6 +134,7 @@ gamma-app-services gamma-dock-services [gamma:children] +agreeable-egret api arithmancy bastion diff --git a/ansible/gamma-hosts/variables b/ansible/gamma-hosts/variables index d7a91619..90cc22ef 100644 --- a/ansible/gamma-hosts/variables +++ b/ansible/gamma-hosts/variables @@ -1,3 +1,11 @@ +[agreeable-egret:vars] +agreeable_egret_port=65520 +egret_pg_host=gamma-big-poppa.cnksgdqarobf.us-west-2.rds.amazonaws.com:32659 +egret_pg_port=32659 +egret_pg_user=egret +egret_pg_pass=b3UKjxbGblKZtG6c +egret_pg_database=egret + [api_group:vars] api_aws_access_key_id=AKIAIDC4WVMTCGV7KRVQ api_aws_secret_access_key=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 @@ -122,6 +130,11 @@ vault_aws_access_key_id=AKIAJ7R4UIM45KH2WGWQ vault_aws_secret_key=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 vault_aws_region=us-east-1 +[marketing:vars] +marketing_bucket=runnable-gamma.com +marketing_aws_access_key=AKIAICIWKIZEQCMDXLEA +marketing_aws_secret_key=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 + [gamma:vars] ansible_ssh_private_key_file=~/.ssh/gamma.pem api_hello_runnable_github_token=88ddc423c2312d02a8bbcaad76dd4c374a30e4af diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 1aae7121..fbe378d0 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -44,6 +44,9 @@ core_file_dir: /docker/app-cores ## shared application configs ## +# agreeable-egret +agreeable_egret_hostname: egret.{{ domain }} + #angular angular_url: https://app.{{ domain }} mixpanel_proxy_url: https://mixpanel.{{ domain }} diff --git a/ansible/group_vars/alpha-agreeable-egret.yml b/ansible/group_vars/alpha-agreeable-egret.yml new file mode 100644 index 00000000..5cf08169 --- /dev/null +++ b/ansible/group_vars/alpha-agreeable-egret.yml @@ -0,0 +1,27 @@ +name: "agreeable-egret" + +container_image: "registry.runnable.com/runnable/{{ name }}" +container_tag: "{{ git_branch }}" +hosted_ports: ["{{ agreeable_egret_port }}"] +repo: "git@github.com:CodeNow/{{ name }}.git" +node_version: "4.3.1" +npm_version: "3.7.5" + +# Exposes egret +redis_key: "frontend:{{ agreeable_egret_hostname }}" +is_redis_update_required: 'yes' + +# container settings +container_envs: > + -e HELLO_RUNNABLE_GITHUB_TOKEN={{ api_hello_runnable_github_token }} + -e NODE_ENV={{ node_env }} + -e RUNNABLE_API_URL={{ api_url }} + -e PORT={{ hosted_ports[0] }} + -e RUNNABLE_USER_CONTENT_DOMAIN={{ user_content_domain }} + -e POSTGRES_CONNECT_STRING=postgres://{{ egret_pg_user }}:{{ egret_pg_pass }}@{{ egret_pg_host }}/{{ egret_pg_database }} + +container_run_opts: > + -h {{ name }} + -d + -p {{ hosted_ports[0] }}:{{ hosted_ports[0] }} + {{ container_envs }} diff --git a/ansible/group_vars/alpha-api-base.yml b/ansible/group_vars/alpha-api-base.yml index d01c68fd..75cdb85a 100644 --- a/ansible/group_vars/alpha-api-base.yml +++ b/ansible/group_vars/alpha-api-base.yml @@ -51,7 +51,7 @@ api_base_container_envs: >- -e GITHUB_PROTOCOL=http -e HELLO_RUNNABLE_GITHUB_TOKEN={{ api_hello_runnable_github_token }} -e KRAIN_PORT={{ krain_port }} - -e MIXPANEL_APP_ID={{ api_mixpanel_app_id }} + {% if api_mixpanel_app_id is defined %} -e MIXPANEL_APP_ID={{ api_mixpanel_app_id }} {% endif %} -e MONGO_REPLSET_NAME={{ api_mongo_replset_name }} -e MONGO=mongodb://{{ api_mongo_auth }}@{{ mongo_hosts }}/{{ api_mongo_database }} -e NAVI_HOST=http://{{ navi_host_address }}:{{ navi_http_port }} diff --git a/ansible/group_vars/alpha-consul.yml b/ansible/group_vars/alpha-consul.yml index b94c19c5..2a9e2c18 100644 --- a/ansible/group_vars/alpha-consul.yml +++ b/ansible/group_vars/alpha-consul.yml @@ -40,3 +40,7 @@ consul_seed: value: "{{ api_hostname }}" - key: api/url value: "{{ api_url }}" + - key: s3/bucket + value: "{{ registry_s3_bucket }}" + - key: s3/region + value: "{{ registry_s3_region }}" diff --git a/ansible/group_vars/alpha-lets-encrypt-certs-generation.yml b/ansible/group_vars/alpha-lets-encrypt-certs-generation.yml new file mode 100644 index 00000000..69ea09ba --- /dev/null +++ b/ansible/group_vars/alpha-lets-encrypt-certs-generation.yml @@ -0,0 +1,19 @@ +--- +name: nginx + +# used by consul template updater +target_container_name: nginx +target_updater_file_path: /etc/nginx/sites-enabled + +# used by container_kill_start +container_image: "{{ name }}" +container_tag: "1.10" + +restart_policy: always + +container_run_opts: > + -d + --name {{ name }} + -p 0.0.0.0:443:443 + -p 0.0.0.0:80:80 + -v /etc/ssl/certs/{{ domain }}:/etc/ssl/certs/{{ domain }}:ro diff --git a/ansible/group_vars/alpha-marketing.yml b/ansible/group_vars/alpha-marketing.yml index 94e57d3b..a97aadca 100644 --- a/ansible/group_vars/alpha-marketing.yml +++ b/ansible/group_vars/alpha-marketing.yml @@ -9,10 +9,11 @@ do_not_push: yes dockerfile_enviroment: [ "API_URL https://{{ api_hostname }}", - "AWS_ACCESS_KEY {{ aws_access_key }}", - "AWS_SECRET_KEY {{ aws_secret_key }}", + "AWS_ACCESS_KEY {{ marketing_aws_access_key }}", + "AWS_SECRET_KEY {{ marketing_aws_secret_key }}", "ANGULAR_URL {{ angular_url }}", - "AWS_BUCKET {{ domain }}", + "AWS_BUCKET {{ marketing_bucket }}", + "AWS_REGION {{ web_aws_bucket_region | default('us-standard') }}", "NODE_ENV {{ node_env }}" ] diff --git a/ansible/group_vars/alpha-metis.yml b/ansible/group_vars/alpha-metis.yml index fe371c36..a0a4761f 100644 --- a/ansible/group_vars/alpha-metis.yml +++ b/ansible/group_vars/alpha-metis.yml @@ -27,7 +27,7 @@ container_envs: > -e REDIS_CACERT={{ redis_ca_cert_path }} -e REDIS_HOST={{ redis_host_address }} -e REDIS_PORT={{ redis_tls_port }} - -e REGISTRY_HOST={{ registry_host }} + -e REGISTRY_HOST={{ hostvars[groups['registry'][0]]['ansible_default_ipv4']['address'] }} -e ROLLBAR_KEY={{ metis_rollbar_key }} -e DOCKER_PORT={{ docker_port }} diff --git a/ansible/group_vars/alpha-shiva.yml b/ansible/group_vars/alpha-shiva.yml index 24f3daa5..244d5b39 100644 --- a/ansible/group_vars/alpha-shiva.yml +++ b/ansible/group_vars/alpha-shiva.yml @@ -24,10 +24,10 @@ container_envs: > -e REDIS_CACERT={{ redis_ca_cert_path }} -e REDIS_PORT={{ redis_tls_port }} -e REDIS_IPADDRESS={{ redis_host_address }} - -e REGISTRY_HOST={{ registry_host }} + -e REGISTRY_HOST={{ hostvars[groups['registry'][0]]['ansible_default_ipv4']['address'] }} {% if shiva_rollbar_token is defined %} -e ROLLBAR_KEY={{ shiva_rollbar_token }} {% endif %} -e DOCKER_PORT={{ docker_port }} - {% if shiva_consult_hostname is defined %} -e CONSUL_HOSTNAME={{ shiva_consult_hostname }} {% endif %} + {% if shiva_consult_hostname is defined %} -e CONSUL_HOSTNAME={{ hostvars[groups['dock'][0]]['ansible_default_ipv4']['address'] }} {% endif %} {% if shiva_aws_region is defined %} -e AWS_REGION={{ shiva_aws_region }} {% endif %} {% if shiva_dock_security_groups is defined %} -e AWS_DOCK_SECURITY_GROUPS={{ shiva_dock_security_groups }} {% endif %} {% if shiva_ssh_key_name is defined %} -e AWS_SSH_KEY_NAME={{ shiva_ssh_key_name }} {% endif %} diff --git a/ansible/group_vars/alpha-web.yml b/ansible/group_vars/alpha-web.yml index be7bf832..eb67c00e 100644 --- a/ansible/group_vars/alpha-web.yml +++ b/ansible/group_vars/alpha-web.yml @@ -14,6 +14,7 @@ dockerfile_enviroment: [ "MIXPANEL_PROXY_URL {{ mixpanel_proxy_url }}", "AWS_ACCESS_KEY {{ aws_access_key }}", "AWS_BUCKET app.{{ domain }}", + "AWS_REGION {{ web_aws_bucket_region | default('us-east-1') }}", "AWS_SECRET_KEY {{ aws_secret_key }}", "INTERCOM_APP_ID {{ web_intercom_id }}", "MARKETING_URL {{ marketing_url }}", diff --git a/ansible/lets-encrypt-certs-generation.yml b/ansible/lets-encrypt-certs-generation.yml new file mode 100644 index 00000000..eef0fe09 --- /dev/null +++ b/ansible/lets-encrypt-certs-generation.yml @@ -0,0 +1,11 @@ +--- +- hosts: userland + vars_files: + - group_vars/alpha-lets-encrypt-certs-generation.yml + roles: + - role: datadog + has_dd_integration: yes + + - role: lets-encrypt-certs-generation + + - role: container_kill_start diff --git a/ansible/marketing.yml b/ansible/marketing.yml index 3403b7d2..64a8a07b 100644 --- a/ansible/marketing.yml +++ b/ansible/marketing.yml @@ -1,5 +1,5 @@ --- -- hosts: web +- hosts: marketing vars_files: - "group_vars/alpha-marketing.yml" roles: diff --git a/ansible/roles/base_ubuntu/tasks/main.yml b/ansible/roles/base_ubuntu/tasks/main.yml index ee624a85..33f5ba32 100644 --- a/ansible/roles/base_ubuntu/tasks/main.yml +++ b/ansible/roles/base_ubuntu/tasks/main.yml @@ -4,6 +4,6 @@ when: dock is not defined lineinfile: dest=/etc/hosts - line="{{ registry_host }} registry.runnable.com" + line="{{ hostvars[groups['registry'][0]]['ansible_default_ipv4']['address'] }} registry.runnable.com" state=present regexp=".+ registry\.runnable\.com" diff --git a/ansible/roles/consul-services/tasks/main.yml b/ansible/roles/consul-services/tasks/main.yml index 26458199..dc76654b 100644 --- a/ansible/roles/consul-services/tasks/main.yml +++ b/ansible/roles/consul-services/tasks/main.yml @@ -29,7 +29,7 @@ tags: ['master'] port: '{{ redis_port }}' - name: 'registry' - host_address: '{{ registry_host }}' + host_address: "{{ hostvars[groups['registry'][0]]['ansible_default_ipv4']['address'] }}" tags: ['master'] port: '{{ registry_port }}' diff --git a/ansible/roles/dock-images/tasks/main.yml b/ansible/roles/dock-images/tasks/main.yml index 1c284621..6a75f343 100644 --- a/ansible/roles/dock-images/tasks/main.yml +++ b/ansible/roles/dock-images/tasks/main.yml @@ -17,7 +17,7 @@ become: true command: docker pull {{ item }} with_items: - - "registry.runnable.com/runnable/image-builder:v4.2.3" + - "registry.runnable.com/runnable/image-builder:v4.3.0" - "swarm:1.2.5" - "registry:2.3.1" - "google/cadvisor:v0.24.1" diff --git a/ansible/roles/docker_client/README.md b/ansible/roles/docker_client/README.md index 3b95b530..f6bf908d 100644 --- a/ansible/roles/docker_client/README.md +++ b/ansible/roles/docker_client/README.md @@ -7,7 +7,7 @@ Ansible Role to Install Docker Client Certs on Ubuntu Creating new docker client certs: 1. cd into this dir ```cd ``` 2. ensure you have ca-key.pem here `roles/docker_client/ca-key.pem` -3. run cert generator ```sudo ./scripts/genClientCert.sh ``` +3. run cert generator ```sudo ./scripts/genClientCert.sh ``` ## Author Information diff --git a/ansible/roles/docker_client/scripts/genClientCert.sh b/ansible/roles/docker_client/scripts/genClientCert.sh index a4737355..1d951211 100755 --- a/ansible/roles/docker_client/scripts/genClientCert.sh +++ b/ansible/roles/docker_client/scripts/genClientCert.sh @@ -7,10 +7,12 @@ fi CLIENT=./files/certs/$1 echo 'WARN: hard coded alpha-api-old gamma-services and beta-services for SWARM' -# if [[ $2 = '' ]]; then -# echo 'script requires a client ip address' -# exit 1 -# fi +if [[ $2 = '' ]]; then + echo 'script requires a client ip address' + exit 1 +fi + +MAIN_HOST_IP_ADDRESS=$2 mkdir $CLIENT @@ -28,7 +30,8 @@ openssl req \ chmod 400 "$CLIENT/client.csr" echo extendedKeyUsage=clientAuth,serverAuth > "$CLIENT/extfile.cnf" -echo subjectAltName=IP:10.4.0.148,IP:10.8.4.40,IP:10.12.12.136,IP:10.8.5.63,IP:10.8.6.59,IP:10.4.6.251,IP:127.0.0.1,DNS:localhost,DNS:swarm-staging-codenow.runnableapp.com >> "$CLIENT/extfile.cnf" +echo subjectAltName=IP:127.0.0.1,DNS:localhost >> "$CLIENT/extfile.cnf" +echo subjectAltName=IP:${MAIN_HOST_IP_ADDRESS},IP:10.4.0.148,IP:10.8.4.40,IP:10.12.12.136,IP:10.8.5.63,IP:10.8.6.59,IP:10.4.6.251,IP:127.0.0.1,DNS:localhost,DNS:swarm-staging-codenow.runnableapp.com >> "$CLIENT/extfile.cnf" # generate cert for client openssl x509 \ diff --git a/ansible/roles/lets-encrypt-certs-generation/tasks/main.yml b/ansible/roles/lets-encrypt-certs-generation/tasks/main.yml new file mode 100644 index 00000000..7357ebd5 --- /dev/null +++ b/ansible/roles/lets-encrypt-certs-generation/tasks/main.yml @@ -0,0 +1,35 @@ +--- +- name: make sure cert directory is in place + tags: [ configure_proxy, certs ] + become: true + file: + dest: /etc/ssl/certs/{{ domain }} + state: directory + +- name: make sure nginx directory is in place + tags: [ configure_proxy, configure_files ] + become: true + file: + dest: /etc/nginx + state: directory + +- name: put nginx configuration in place + tags: [ configure_proxy, configure_files ] + become: yes + template: + src: proxy-nginx.conf + dest: /etc/nginx/nginx.conf + +- name: assert nginx sites-enabled directory + tags: [ configure_proxy, configure_files ] + become: yes + file: + state: directory + dest: /etc/nginx/sites-enabled + +- name: put lets-encrypt conf in place + tags: [ configure_proxy, configure_files ] + become: yes + template: + src: lets-encrypt.tmpl + dest: /etc/nginx/sites-enabled/lets-encrypt.conf diff --git a/ansible/roles/lets-encrypt-certs-generation/templates/default b/ansible/roles/lets-encrypt-certs-generation/templates/default new file mode 100644 index 00000000..61d40e80 --- /dev/null +++ b/ansible/roles/lets-encrypt-certs-generation/templates/default @@ -0,0 +1,13 @@ +server { + listen [::]:80 default_server; + server_name {{ domain }} *.{{ domain }}; + root /var/www/html; + + location ~ /.well-known { + allow all; + } + + location /test/ { + return 200 "Its alive"; + } +} diff --git a/ansible/roles/lets-encrypt-certs-generation/templates/proxy-nginx.conf b/ansible/roles/lets-encrypt-certs-generation/templates/proxy-nginx.conf new file mode 100644 index 00000000..dc663d03 --- /dev/null +++ b/ansible/roles/lets-encrypt-certs-generation/templates/proxy-nginx.conf @@ -0,0 +1,29 @@ +user www-data; +worker_processes 4; +pid /run/nginx.pid; + +events { + worker_connections 5000; +} + +http { + ## + # Basic Settings + ## + tcp_nodelay on; + keepalive_timeout 65; + server_tokens off; + + ## + # Logging Settings + ## + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + ## + # Virtual Host Configs + ## + + include /etc/nginx/sites-enabled/*; +} diff --git a/ansible/roles/local-vault/handlers/main.yml b/ansible/roles/local-vault/handlers/main.yml index 572ad76c..7c055ec5 100644 --- a/ansible/roles/local-vault/handlers/main.yml +++ b/ansible/roles/local-vault/handlers/main.yml @@ -1,7 +1,7 @@ --- - name: stop vault local_action: - shell kill $(cat /tmp/vault.pid) + shell kill $(ps aux | grep "vault server" | grep -v grep | cut -d' ' -f3) - name: remove vault config local_action: diff --git a/ansible/roles/local-vault/tasks/main.yml b/ansible/roles/local-vault/tasks/main.yml index 1afb3dce..29befa21 100644 --- a/ansible/roles/local-vault/tasks/main.yml +++ b/ansible/roles/local-vault/tasks/main.yml @@ -11,13 +11,13 @@ - name: start vault daemon run_once: true local_action: - command daemon --pidfile=/tmp/vault.pid -- vault server --config=/tmp/vault.hcl + shell vault server --config=/tmp/vault.hcl > /tmp/log 2>&1 & notify: - stop vault - name: pause for start pause: - seconds: 1 + seconds: 5 - name: check vault seal tags: [ unseal ] diff --git a/ansible/roles/runnable-domain-proxy/templates/registry.tmpl b/ansible/roles/runnable-domain-proxy/templates/registry.tmpl index c0500a16..769850b8 100644 --- a/ansible/roles/runnable-domain-proxy/templates/registry.tmpl +++ b/ansible/roles/runnable-domain-proxy/templates/registry.tmpl @@ -1,5 +1,5 @@ upstream docker-registry { - server {{ registry_host }}:{{ registry_port }}; + server {{ hostvars[groups['registry'][0]]['ansible_default_ipv4']['address'] }}:{{ registry_port }}; } server { @@ -13,7 +13,7 @@ server { location / { auth_basic off; - proxy_pass http://{{ registry_host }}:{{ registry_port }}; + proxy_pass http://{{ hostvars[groups['registry'][0]]['ansible_default_ipv4']['address'] }}:{{ registry_port }}; proxy_set_header Host $http_host; # required for docker client's sake proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP } @@ -25,7 +25,7 @@ server { return 404; } - proxy_pass http://{{ registry_host }}:{{ registry_port }}; + proxy_pass http://{{ hostvars[groups['registry'][0]]['ansible_default_ipv4']['address'] }}:{{ registry_port }}; proxy_set_header Host $http_host; # required for docker client's sake proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/ansible/single-host-part-1-setup.yml b/ansible/single-host-part-1-setup.yml new file mode 100644 index 00000000..d088bc78 --- /dev/null +++ b/ansible/single-host-part-1-setup.yml @@ -0,0 +1,4 @@ +# Initial values (Only run the first time) +- include: consul-values.yml -e write_values="true" # Only run the first time +- include: consul-services.yml # Only run the first time +- include: vault-values.yml -e write_values="true" diff --git a/ansible/single-host-part-1.yml b/ansible/single-host-part-1.yml new file mode 100644 index 00000000..cf21cd4e --- /dev/null +++ b/ansible/single-host-part-1.yml @@ -0,0 +1,3 @@ +## Service Discovery: +- include: consul.yml +- include: vault.yml diff --git a/ansible/single-host-part-2.yml b/ansible/single-host-part-2.yml new file mode 100644 index 00000000..e8b1a51f --- /dev/null +++ b/ansible/single-host-part-2.yml @@ -0,0 +1,6 @@ +# Databases +- include: mongo.yml +- include: rabbitmq.yml +- include: redis.yml +- include: redis-tls.yml # Only used by navi and shiva +- include: registry.yml diff --git a/ansible/single-host-part-3.yml b/ansible/single-host-part-3.yml new file mode 100644 index 00000000..69ff80b5 --- /dev/null +++ b/ansible/single-host-part-3.yml @@ -0,0 +1,28 @@ +# Docks Services +- include: swarm-manager.yml +- include: palantiri.yml git_branch="{{ palantiri_branch }}" -t deploy +- include: sauron.yml git_branch="{{ sauron_branch }}" -t deploy +- include: shiva.yml git_branch="{{ astral_branch }}" -t deploy +- include: khronos.yml git_branch="{{ khronos_branch }}" -t deploy +- include: docker-listener.yml git_branch="{{ docker_listener_branch }}" -t deploy + +## Proxies +- include: registrator-api.yml # Only one of these is neededi, so registrator-navi is not needed +- include: single-host-proxy.yml # API depends on NGINX to be running +- include: github-varnish.yml git_branch="{{ github_varnish_branch }}" -t deploy + +# Main +- include: big-poppa.yml git_branch="{{ big_poppa_branch }}" -t deploy +- include: api.yml git_branch="{{ api_branch }}" -t deploy +- include: cream.yml git_branch="{{ cream_branch }}" -t deploy # CREAM fails if big-poppa or API is down +- include: web.yml git_branch="{{ angular_branch }}" -t deploy # fucked + +# Networking services +- include: detention.yml git_branch="{{ detention_branch }}" -t deploy +- include: link.yml git_branch="{{ link_branch }}" -t deploy +- include: navi.yml git_branch="{{ navi_branch }}" -t deploy # Connects to Redis over tls port + +# Other +- include: optimus.yml git_branch="{{ optimus_branch }}" -t deploy +- include: drake.yml git_branch="{{ drake_branch }}" -t deploy +- include: pheidi.yml git_branch="{{ pheidi_branch }}" -t deploy diff --git a/ansible/single-host.yml b/ansible/single-host.yml index 1888688c..351eabb0 100644 --- a/ansible/single-host.yml +++ b/ansible/single-host.yml @@ -1,49 +1,3 @@ -## configure security group policy -- include: sg_configure.yml - -## Install Datadog Agent -# - include: datadog.yml - -## begin with databases: -- include: consul.yml -- include: vault.yml - -# Initial values (Only run the first time) -- include: consul-values.yml -e write_values="true" # Only run the first time -- include: consul-services.yml # Only run the first time -- include: vault-values.yml -e write_values="true" - -# Databases -- include: rabbitmq.yml -- include: redis.yml -- include: redis-tls.yml # Only used by navi and shiva -- include: registry.yml - -# Docks Services -- include: swarm-manager.yml -- include: palantiri.yml git_branch="{{ palantiri_branch }}" -t deploy -- include: sauron.yml git_branch="{{ sauron_branch }}" -t deploy -- include: shiva.yml git_branch="{{ astral_branch }}" -t deploy -- include: khronos.yml git_branch="{{ khronos_branch }}" -t deploy -- include: docker-listener.yml git_branch="{{ docker_listener_branch }}" -t deploy - -## Proxies -- include: registrator-api.yml # Only one of these is neededi, so registrator-navi is not needed -- include: single-host-proxy.yml # API depends on NGINX to be running -- include: github-varnish.yml git_branch="{{ github_varnish_branch }}" -t deploy - -# Main -- include: big-poppa.yml git_branch="{{ big_poppa_branch }}" -t deploy -- include: api.yml git_branch="{{ api_branch }}" -t deploy -- include: cream.yml git_branch="{{ cream_branch }}" -t deploy # CREAM fails if big-poppa or API is down -- include: web.yml git_branch="{{ angular_branch }}" -t deploy # fucked - -# Networking services -- include: detention.yml git_branch="{{ detention_branch }}" -t deploy -- include: link.yml git_branch="{{ link_branch }}" -t deploy -- include: navi.yml git_branch="{{ navi_branch }}" -t deploy # Connects to Redis over tls port - -# Other -- include: optimus.yml git_branch="{{ optimus_branch }}" -t deploy -- include: drake.yml git_branch="{{ drake_branch }}" -t deploy -- include: pheidi.yml git_branch="{{ pheidi_branch }}" -t deploy +- include: single-host-part-1.yml +- include: single-host-part-2.yml +- include: single-host-part-3.yml