Enhanced quick start pr 2.x

install.sh

@@ -161,7 +161,9 @@ if [ ! -d "/home/$CONTROLLER_USER/ce-provision" ]; then
   /usr/bin/su - "$CONTROLLER_USER" -c "git clone --branch $CONFIG_REPO_BRANCH $CONFIG_REPO /home/$CONTROLLER_USER/ce-provision/config"
   /usr/bin/su - "$CONTROLLER_USER" -c "/usr/bin/ln -s /home/$CONTROLLER_USER/ce-provision/config/ansible.cfg /home/$CONTROLLER_USER/ce-provision/ansible.cfg"
 else
-  /usr/bin/echo "ce-provision directory at /home/$CONTROLLER_USER/ce-provision already exists. Skipping."
+  /usr/bin/echo "ce-provision directory at /home/$CONTROLLER_USER/ce-provision already exists. Updating."
+  /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && git pull origin $VERSION"
+  /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision/config && git pull origin $CONFIG_REPO_BRANCH"
   /usr/bin/echo "-------------------------------------------------"
 fi
 /usr/bin/mkdir -p "/home/$CONTROLLER_USER/ce-provision/galaxy/roles"

plays/aws_account/README.md

@@ -0,0 +1,3 @@
+# Base playbook for configuring an AWS account.
+
+@TODO provide example infra repo for use with the AWS EC2 inventory plugin.

plays/aws_account/aws_account.yml

@@ -0,0 +1,16 @@
+---
+# Global infra setup.
+- hosts: localhost
+  connection: local
+  become: false
+  vars:
+    _init:
+      vars_dirs:
+        - "{{ _ce_provision_build_dir }}/vars/_global"
+    # used for tagging
+    _profile: core
+    _env_type: core
+  roles:
+    - _init
+    - _meta/aws_account
+    - _exit

plays/aws_asg/README.md

@@ -0,0 +1,14 @@
+# Base playbooks for creating a new AWS ASG.
+For a standard ASG build just add `cluster.yml` to your environment play, like this:
+
+```yaml
+- import_playbook: "{{ _ce_provision_base_dir }}/plays/aws_asg/cluster.yml"
+  vars:
+    _aws_region: eu-west-1
+    _env_type: dev
+    _aws_resource_name: cluster-acme-com
+```
+
+If you have specific requirements for your AMIs you can copy these plays to your infra repository and alter them accordingly. Don't forget to copy/include `launch.yml` from the `_ec2_standalone` plays or orchestration of brand new clusters will fail.
+
+@TODO provide example infra repo for use with the AWS EC2 inventory plugin.

plays/aws_asg/ami.yml

@@ -0,0 +1,35 @@
+---
+# This is the provisioning for the AMI and will run inside a temporary instance using Packer.
+- hosts: default
+  become: true
+
+  vars:
+    _init:
+      vars_dirs:
+        - "{{ _ce_provision_build_dir }}/vars/_global"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/_common"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/{{ _env_type }}"
+        - "{{ _ce_provision_build_dir }}/vars/{{ _aws_resource_name }}"
+    _profile: asg
+
+  tasks:
+    - name: Upgrade the system and update cache
+      ansible.builtin.apt:
+        upgrade: dist
+        update_cache: true
+    - ansible.builtin.import_role:
+        name: _init
+    - ansible.builtin.import_role:
+        name: _meta/aws_client_instance
+    - ansible.builtin.import_role:
+        name: _meta/webserver
+    - ansible.builtin.import_role:
+        name: debian/aws_efs_client
+    - ansible.builtin.import_role:
+        name: debian/squashfs
+    - ansible.builtin.import_role:
+        name: debian/mount_sync
+    - ansible.builtin.import_role:
+        name: debian/swap
+    - ansible.builtin.import_role:
+        name: _exit

plays/aws_asg/asg.yml

@@ -0,0 +1,25 @@
+---
+# Common ASG infra.
+- hosts: "_{{ _aws_resource_name | regex_replace('-', '_') }}"
+  connection: local
+  become: false
+
+  vars:
+    _init:
+      vars_dirs:
+        - "{{ _ce_provision_build_dir }}/vars/_global"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/_common"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/{{ _env_type }}"
+        - "{{ _ce_provision_build_dir }}/vars/{{ _aws_resource_name }}"
+    _profile: asg
+
+  tasks:
+    - ansible.builtin.import_role:
+        name: _init
+      run_once: true
+    - ansible.builtin.import_role:
+        name: aws/aws_ec2_autoscale_cluster
+      run_once: true
+    - ansible.builtin.import_role:
+        name: _exit
+      run_once: true

plays/aws_asg/cluster.yml

@@ -0,0 +1,5 @@
+---
+# Creates hosts entry so play isn't skipped.
+- ansible.builtin.import_playbook: ../aws_ec2_standalone/launch.yml
+# Spins up the cluster.
+- ansible.builtin.import_playbook: asg.yml

plays/aws_ec2_standalone/README.md

@@ -0,0 +1,8 @@
+# Base playbook for setting up a standalone EC2 instance.
+IMPORTANT: these plays deliberately exclude the `_init._profile` variable because it usually needs to be set at runtime or in a separate infrastructure config repo.
+
+The `server.yml` file is the 'main' play, to customise we suggest this is copied to an infra repo and renamed as `hostname.yml` e.g. `acme-dev1.yml`.  You also need to copy `provision.yml` so you can control what is provisioned. The `provision.yml` file is intended only as a model.
+
+If you want a separate RDS instance to pair with your EC2 instance then uncomment the last two play import lines in `server.yml`, however note you do need to sort out outbound firewall ports in iptables and a Security Group for inbound traffic to the RDS instance - usually port `3306` outbound from the EC2 instance in `firewall_config` and an SG that allows `3306` inbound to RDS.
+
+@TODO provide example infra repo for use with the AWS EC2 inventory plugin.

plays/aws_ec2_standalone/ami.yml

@@ -0,0 +1,20 @@
+---
+# This is the bare provisioning for the AMI.
+- hosts: default
+  become: true
+
+  vars:
+    _init:
+      vars_dirs:
+        - "{{ _ce_provision_build_dir }}/vars/_global"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/_common"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/{{ _env_type }}"
+        - "{{ _ce_provision_build_dir }}/vars/{{ _aws_resource_name }}"
+
+  tasks:
+    - ansible.builtin.import_role:
+        name: _init
+    - ansible.builtin.import_role:
+        name: debian/user_provision
+    - ansible.builtin.import_role:
+        name: _exit

plays/aws_ec2_standalone/ec2.yml

@@ -0,0 +1,24 @@
+---
+# First step. Spin up a "blank" instance from a fresh AMI.
+- hosts: "_{{ _aws_resource_name | regex_replace('-', '_') }}"
+  connection: local
+  become: false
+
+  vars:
+    _init:
+      vars_dirs:
+        - "{{ _ce_provision_build_dir }}/vars/_global"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/_common"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/{{ _env_type }}"
+        - "{{ _ce_provision_build_dir }}/vars/{{ _aws_resource_name }}"
+
+  tasks:
+    - ansible.builtin.import_role:
+        name: _init
+    - ansible.builtin.import_role:
+        name: aws/aws_ami
+    - ansible.builtin.import_role:
+        name: aws/aws_ec2_with_eip
+    - ansible.builtin.import_role:
+        name: _exit
+    - ansible.builtin.meta: refresh_inventory

plays/aws_ec2_standalone/launch.yml

@@ -0,0 +1,38 @@
+---
+# Prepare the ground for a new EC2 machine
+- hosts: localhost
+  connection: local
+  become: false
+
+  vars:
+    _init:
+      vars_dirs:
+        - "{{ _ce_provision_build_dir }}/vars/_global"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/_common"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/{{ _env_type }}"
+        - "{{ _ce_provision_build_dir }}/vars/{{ _aws_resource_name }}"
+    # copied from aws_ami.yml in group_vars/all because we do not want to load aws_ami vars yet
+    ami_groups:
+      - "all"
+      - "_{{ _aws_resource_name | regex_replace('-', '_') }}"
+      - "_{{ _infra_name | regex_replace('-', '_') }}"
+      - "_{{ _env_type | regex_replace('-', '_') }}"
+
+  tasks:
+    - ansible.builtin.import_role:
+        name: _init
+    - name: Blank the _aws_hostname variable.
+      ansible.builtin.set_fact:
+        _aws_hostname: ""
+    - name: Check to see if an Ansible host exists.
+      ansible.builtin.set_fact:
+        _aws_hostname: "{{ item }}"
+      with_inventory_hostnames:
+        - "_{{ _aws_resource_name | regex_replace('-', '_') }}"
+    - name: If an Ansible host is not found, create it so we can execute EC2 orchestration.
+      ansible.builtin.add_host:
+        name: "_{{ _aws_resource_name | regex_replace('-', '_') }}"
+        groups: "{{ ami_groups }}"
+      when: _aws_hostname | length == 0
+    - ansible.builtin.import_role:
+        name: _exit

plays/aws_ec2_standalone/mysql_client.yml

@@ -0,0 +1,32 @@
+---
+- hosts: "_{{ _aws_resource_name | regex_replace('-', '_') }}"
+  become: true
+
+  vars:
+    _init:
+      vars_dirs:
+        - "{{ _ce_provision_build_dir }}/vars/_global"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/_common"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/{{ _env_type }}"
+        - "{{ _ce_provision_build_dir }}/vars/{{ _aws_resource_name }}"
+
+  tasks:
+    - ansible.builtin.import_role:
+        name: _init
+    - ansible.builtin.import_role:
+        name: debian/user_deploy
+    # Look up RDS hostname
+    - name: Get information about an instance
+      community.aws.rds_instance_info:
+        region: "{{ _aws_region }}"
+        profile: "{{ _aws_profile }}"
+        db_instance_identifier: "{{ _aws_resource_name }}"
+      become: true
+      become_user: "{{ user_provision.username }}"
+      delegate_to: localhost # needs to run on controller
+      register: _database_info
+    # Install MySQL client
+    - ansible.builtin.import_role:
+        name: debian/mysql_client
+    - ansible.builtin.import_role:
+        name: _exit

plays/aws_ec2_standalone/provision.yml

@@ -0,0 +1,29 @@
+---
+- hosts: "_{{ _aws_resource_name | regex_replace('-', '_') }}"
+  become: true
+
+  vars:
+    _init:
+      vars_dirs:
+        - "{{ _ce_provision_build_dir }}/vars/_global"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/_common"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/{{ _env_type }}"
+        - "{{ _ce_provision_build_dir }}/vars/{{ _aws_resource_name }}"
+
+  tasks:
+    - ansible.builtin.import_role:
+        name: _init
+    - ansible.builtin.import_role:
+        name: ce_ldap_safelist
+    - ansible.builtin.import_role:
+        name: _meta/common_base
+    - ansible.builtin.import_role:
+        name: _meta/aws_client_instance
+    - ansible.builtin.import_role:
+        name: debian/ssh_server
+    - ansible.builtin.import_role:
+        name: debian/firewall_config
+    - ansible.builtin.import_role:
+        name: debian/swap
+    - ansible.builtin.import_role:
+        name: _exit

plays/aws_ec2_standalone/rds.yml

@@ -0,0 +1,56 @@
+---
+# Create an RDS instance.
+- hosts: "_{{ _aws_resource_name | regex_replace('-', '_') }}"
+  connection: local
+  become: false
+
+  vars:
+    _init:
+      vars_dirs:
+        - "{{ _ce_provision_build_dir }}/vars/_global"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/_common"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/{{ _env_type }}"
+        - "{{ _ce_provision_build_dir }}/vars/{{ _aws_resource_name }}"
+
+  tasks:
+    - ansible.builtin.import_role:
+        name: _init
+
+    # Automate subnet fetching
+    - name: Create empty var to hold subnet IDs.
+      ansible.builtin.set_fact:
+        _aws_rds_vpc_subnet_ids: []
+
+    - name: Gather VPC information.
+      amazon.aws.ec2_vpc_net_info:
+        profile: "{{ aws_rds.aws_profile }}"
+        region: "{{ aws_rds.region }}"
+        filters:
+          "tag:Name": "{{ _infra_name }}"
+      register: _aws_rds_vpc
+
+    - name: Set the VPC id from name.
+      ansible.builtin.set_fact:
+        _aws_rds_vpc_id: "{{ _aws_rds_vpc.vpcs[0].vpc_id }}"
+
+    - name: Gather public subnet information.
+      amazon.aws.ec2_vpc_subnet_info:
+        profile: "{{ aws_rds.aws_profile }}"
+        region: "{{ aws_rds.region }}"
+        filters:
+          vpc-id: "{{ _aws_rds_vpc_id }}"
+          tag:Env: "{{ _env_type }}"
+          tag:Profile: "core"
+      register: _aws_rds_vpc_subnets
+
+    - name: Place subnet IDs in a list.
+      ansible.builtin.set_fact:
+        _aws_rds_vpc_subnet_ids: "{{ _aws_rds_vpc_subnet_ids + [item.subnet_id] }}"
+      loop: "{{ _aws_rds_vpc_subnets.subnets }}"
+
+    # Build the RDS instance.
+    - ansible.builtin.import_role:
+        name: aws/aws_rds
+
+    - ansible.builtin.import_role:
+        name: _exit

plays/aws_ec2_standalone/server.yml

@@ -0,0 +1,11 @@
+# Prepares a host entry so the ec2.yml play succeeds.
+- ansible.builtin.import_playbook: "{{ _ce_provision_base_dir }}/plays/aws_ec2_standalone/launch.yml"
+# Spins up the instance.
+# We use the central _deploy role to provision the EC2 instance to avoid duplication.
+- ansible.builtin.import_playbook: "{{ _ce_provision_base_dir }}/plays/aws_ec2_standalone/ec2.yml"
+# Actual provisioning
+- ansible.builtin.import_playbook: provision.yml
+# RDS instance
+#- ansible.builtin.import_playbook: "{{ _ce_provision_base_dir }}/plays/aws_ec2_standalone/rds.yml"
+# MySQL client - needs to happen after RDS instance is created
+#- ansible.builtin.import_playbook: "{{ _ce_provision_base_dir }}/plays/aws_ec2_standalone/mysql_client.yml"

plays/aws_region/README.md

@@ -0,0 +1,3 @@
+# Base playbook for configuring an AWS region.
+
+@TODO provide example infra repo for use with the AWS EC2 inventory plugin.

plays/aws_region/aws_region.yml

@@ -0,0 +1,17 @@
+---
+# Global infra setup.
+- hosts: localhost
+  connection: local
+  become: false
+  vars:
+    _init:
+      vars_dirs:
+        - "{{ _ce_provision_build_dir }}/vars/_global"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/_common"
+        - "{{ _ce_provision_build_dir }}/vars/_regions/{{ _aws_region }}/{{ _env_type }}"
+    # used for tagging
+    _profile: core
+  roles:
+    - _init
+    - _meta/aws_region
+    - _exit

plays/controller/README.md

@@ -0,0 +1,31 @@
+# Base playbook for setting up an infra controller.
+This playbook provides a model for managing an Ansible infra controller with ce-provision based at AWS.
+
+If your server is not in AWS or you are not using the AWS EC2 inventory plugin, you must ensure your server's hostname is in your Ansible hosts file (`config/hosts/hosts` or `hosts.yml`) and provide the same hostname in the `_provision_host` variable. Then call `provision.yml` directly, for example:
+
+```yaml
+---
+- name: Configure my controller server.
+  ansible.builtin.import_playbook: "{{ _ce_provision_base_dir }}/plays/controller/provision.yml"
+  vars:
+    _env_type: util
+    _provision_host: controller.acme.com
+    _profile: controller
+```
+
+If you are using the AWS EC2 inventory plugin and the Code Enigme recommended set-up, you must provide the `_aws_resource_name` variable - note, this is hyphenated, no dots - and call `aws_controller.yml`, for example:
+
+```yaml
+---
+- name: Configure my controller server at AWS.
+  ansible.builtin.import_playbook: "{{ _ce_provision_base_dir }}/plays/controller/aws_controller.yml"
+  vars:
+    _env_type: util
+    _aws_region: eu-west-1
+    _aws_resource_name: controller-acme-com
+    _profile: controller
+```
+
+This will create or find an EC2 instance with the AWS tag of `Name: controller-acme-com` which will be in an inventory group called `_controller_acme_com`.
+
+@TODO provide example infra repo for use with the AWS EC2 inventory plugin.