Ansible in init pr 2.x (#1554)

* Adding new Python pip package role.

* Updating NGINX and Boto3 roles to use the new Python pip role.

* Moving Ansible install to _init.

* Detecting connection type before installing Ansible.

* The _init role should not generate SSH keys and ce_provision should optionally install a new user.

* Updating documentation.

* Switching to using the user_provision role for controller user in _init.

* Allowing for ce_provision to install Ansible in another location.

* Passing vars to the core Ansible install.

* We will need linters if the system didn't install them.

* Duplicity role doesn't need to ensure permissions, it's done in python_pip_packages already.

* Allowing ce-provision to set a different UID from the system user.

Author: gregharvey

Diff for: docs/_Sidebar.md

@@ -70,6 +70,7 @@
        - [Process Manager](/roles/debian/process_manager)
        - [Python Boto](/roles/debian/python_boto)
        - [Python Common](/roles/debian/python_common)
+       - [Python Pip Packages](/roles/debian/python_pip_packages)
        - [rkhunter](/roles/debian/rkhunter)
        - [Rsyslog](/roles/debian/rsyslog)
        - [solr](/roles/debian/solr)

Diff for: docs/roles/_init.md

@@ -9,6 +9,12 @@ This is meant to ALWAYS be included as the first task of a play. If you include
 ## Default variables
 ```yaml
 ---
+_ce_provision_username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}"
+_venv_path: "/home/{{ _ce_provision_username }}/ansible"
+_venv_command: /usr/bin/python3 -m venv
+_venv_install_username: "{{ _ce_provision_username }}"
+_ce_ansible_timer_name: upgrade_ce_provision_ansible
+
 _init:
   # A list of var directories to include. We only support .yml extensions.
   # This is used to detect if the playbook must re-run or not.
@@ -18,11 +24,17 @@ _init:
 
   # Although these variables logically belong with ce_provision, the _init role needs to
   # gather the extra variables if there are any, so there are _init variables.
+
+  # ce-provision user creation
+  ce_provision_new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user
+  #ce_provision_uid # optionally hardcode the UID for this user
+
   # Extra config repo.
   ce_provision_extra_repository: ""
   ce_provision_extra_repository_branch: "master"
   ce_provision_extra_repository_skip_checkout: false
   ce_provision_extra_repository_vars_file: "custom.yml"
+
   # Whether to commit back changes to extra repo.
   ce_provision_extra_repository_push: false
   ce_provision_extra_repository_allowed_vars: []

Diff for: docs/roles/debian/ansible.md

@@ -11,7 +11,7 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip`
 ```yaml
 ---
 ce_ansible:
-  # These are usually set within another role using _venv_path, _venv_command and _install_username but can be overridden.
+  # These are usually set within another role using _venv_path, _venv_command and _venv_install_username but can be overridden.
   #venv_path: "/home/{{ ce_provision.username }}/ansible"
   #venv_command: /usr/bin/python3.11 -m venv
   #install_username: deploy # user to become when creating venv

Diff for: docs/roles/debian/ce_provision.md

@@ -1,32 +1,31 @@
 # ce-provision
-Installs Code Enigma's infrastructure management stack on a server.
+Installs Code Enigma's infrastructure management stack on a server. Note, the `_init` role creates the user and installs Ansible in a virtual environment, so that must be run prior to the `ce_provision` role.
+
 <!--TOC-->
 <!--ENDTOC-->
 
 <!--ROLEVARS-->
 ## Default variables
 ```yaml
 ---
-# See roles/_init/defaults/main.yml for extra variables repo settings.
-_ce_provision:
-  username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}"
-
+# See roles/_init/defaults/main.yml for Ansible installation, controller user creation and extra variables repo settings.
 ce_provision:
-  # Location of Ansible installation and components.
-  venv_path: "/home/{{ _ce_provision.username }}/ansible"
-  venv_command: /usr/bin/python3 -m venv
-  install_username: "{{ _ce_provision.username }}"
-  upgrade_timer_name: upgrade_ce_provision_ansible
+  # Optional venv overrides - if commented out, values taken from _init defaults.
+  #venv_path: "/home/{{ _ce_provision_username }}/ansible"
+  #venv_command: /usr/bin/python3 -m venv
+  #venv_install_username: "{{ _ce_provision_username }}"
+  #upgrade_timer_name: upgrade_ce_provision_ansible
   # Other ce-provision settings.
-  username: "{{ _ce_provision.username }}"
-  new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user
-  key_name: id_rsa.pub # existing users may have a key of a different name
+  new_user: "{{ _init.ce_provision_new_user }}" # see _init defaults, set to false if user already exists or is ephemeral, e.g. an LDAP user
+  username: "{{ _ce_provision_username }}" # see _init defaults
+  #uid: "{{ _init.ce_provision_uid }}" # see _init defaults, optionally hardcode the UID for this user
+  public_key_name: id_rsa.pub # existing users may have a key of a different name
   # Main repo.
   own_repository: "https://github.com/codeenigma/ce-provision.git"
   own_repository_branch: "master"
   own_repository_skip_checkout: false
   # Destination.
-  local_dir: "/home/{{ _ce_provision.username }}/ce-provision"
+  local_dir: "/home/{{ _ce_provision_username }}/ce-provision"
   # Private config repo.
   config_repository: ""
   config_repository_branch: "master"
@@ -43,10 +42,10 @@ ce_provision:
       branch: master
   # File containing default roles and collections to install via Ansible Galaxy.
   # Roles will be installed to $HOME/.ansible/roles for the provision user. This roles path should be added to your ansible.cfg file.
-  galaxy_custom_requirements_file: "/home/{{ _ce_provision.username }}/ce-provision/config/files/galaxy-requirements.yml"
+  galaxy_custom_requirements_file: "/home/{{ _ce_provision_username }}/ce-provision/config/files/galaxy-requirements.yml"
   upgrade_galaxy:
     enabled: true
-    command: "/home/{{ _ce_provision.username }}/ansible/bin/ansible-galaxy collection install --force" # must match venv_path
+    command: "/home/{{ _ce_provision_username }}/ansible/bin/ansible-galaxy collection install --force" # must match venv_path
     on_calendar: "Mon *-*-* 04:00:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events
 
 ```

Diff for: docs/roles/debian/nginx.md

@@ -72,6 +72,7 @@ nginx:
         # on_calendar: "Mon *-*-* 04:00:00"
       ratelimitingcrawlers: true
       is_default: true
+      is_behind_cloudfront: false # set to true to disable gzip.
       basic_auth:
         auth_enabled: false
         auth_file: "" # optionally provide the path on the deploy server to a htpasswd file - WARNING - it must be valid and will not be checked!

Diff for: docs/roles/debian/python_boto.md

@@ -9,7 +9,7 @@ Role to install the `boto3` library for Python integration with AWS services.
 ```yaml
 python_boto:
   boto3_version: "" # version string, e.g. "1.22.13" - empty string means latest
-  # These are usually set within another role using _venv_path and _venv_command but can be overridden.
+  # These are usually set within another role using _venv_path, _venv_command and _venv_install_username but can be overridden.
   #venv_path: /path/to/venv
   #venv_command: /usr/bin/python3.11 -m venv
   #install_username: deploy # user to become when creating venv

Diff for: docs/roles/debian/python_pip_packages.md

@@ -0,0 +1,22 @@
+# Python Pip Packages
+Role to install a list of Python packages in a specified Python virtual environment.
+
+<!--TOC-->
+<!--ENDTOC-->
+
+<!--ROLEVARS-->
+## Default variables
+```yaml
+---
+python_pip_packages:
+  packages: []
+  #  - name: pip
+  #    state: latest
+
+  # These are usually set within another role using _venv_path, _venv_command and _venv_install_username but can be overridden.
+  #venv_path: /path/to/venv
+  #venv_command: /usr/bin/python3.11 -m venv
+  #install_username: deploy # user to become when creating venv
+```
+
+<!--ENDROLEVARS-->

Diff for: roles/_init/README.md

@@ -9,6 +9,12 @@ This is meant to ALWAYS be included as the first task of a play. If you include
 ## Default variables
 ```yaml
 ---
+_ce_provision_username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}"
+_venv_path: "/home/{{ _ce_provision_username }}/ansible"
+_venv_command: /usr/bin/python3 -m venv
+_venv_install_username: "{{ _ce_provision_username }}"
+_ce_ansible_timer_name: upgrade_ce_provision_ansible
+
 _init:
   # A list of var directories to include. We only support .yml extensions.
   # This is used to detect if the playbook must re-run or not.
@@ -18,11 +24,17 @@ _init:
 
   # Although these variables logically belong with ce_provision, the _init role needs to
   # gather the extra variables if there are any, so there are _init variables.
+
+  # ce-provision user creation
+  ce_provision_new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user
+  #ce_provision_uid # optionally hardcode the UID for this user
+
   # Extra config repo.
   ce_provision_extra_repository: ""
   ce_provision_extra_repository_branch: "master"
   ce_provision_extra_repository_skip_checkout: false
   ce_provision_extra_repository_vars_file: "custom.yml"
+
   # Whether to commit back changes to extra repo.
   ce_provision_extra_repository_push: false
   ce_provision_extra_repository_allowed_vars: []

Diff for: roles/_init/defaults/main.yml

@@ -1,4 +1,10 @@
 ---
+_ce_provision_username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}"
+_venv_path: "/home/{{ _ce_provision_username }}/ansible"
+_venv_command: /usr/bin/python3 -m venv
+_venv_install_username: "{{ _ce_provision_username }}"
+_ce_ansible_timer_name: upgrade_ce_provision_ansible
+
 _init:
   # A list of var directories to include. We only support .yml extensions.
   # This is used to detect if the playbook must re-run or not.
@@ -8,11 +14,17 @@ _init:
 
   # Although these variables logically belong with ce_provision, the _init role needs to
   # gather the extra variables if there are any, so there are _init variables.
+
+  # ce-provision user creation
+  ce_provision_new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user
+  #ce_provision_uid # optionally hardcode the UID for this user
+
   # Extra config repo.
   ce_provision_extra_repository: ""
   ce_provision_extra_repository_branch: "master"
   ce_provision_extra_repository_skip_checkout: false
   ce_provision_extra_repository_vars_file: "custom.yml"
+
   # Whether to commit back changes to extra repo.
   ce_provision_extra_repository_push: false
   ce_provision_extra_repository_allowed_vars: []

Diff for: roles/_init/tasks/main.yml

@@ -98,3 +98,27 @@
     - _init.ce_provision_extra_repository
     - _init.ce_provision_extra_repository_vars_file
     - _init.ce_provision_extra_repository_allowed_vars
+
+# Install Ansible under the controller user for all servers
+# Ensure ansible_connection == 'ssh' (i.e. we are connecting to a server) before executing
+- name: Install Ansible.
+  ansible.builtin.include_role:
+    name: debian/user_provision
+  when: ansible_connection == 'ssh'
+
+- name: Install Ansible.
+  ansible.builtin.include_role:
+    name: debian/ansible
+  vars:
+    ce_ansible:
+      venv_path: "{{ _venv_path }}"
+      venv_command: "{{ _venv_command }}"
+      install_username: "{{ _venv_install_username }}"
+      upgrade:
+        enabled: true
+        command: "/home/{{ _venv_install_username }}/ansible/bin/python3 -m pip install --upgrade ansible"
+        on_calendar: "*-*-* 01:30:00"
+        timer_name: "{{ _ce_ansible_timer_name }}"
+      linters:
+        enabled: false
+  when: ansible_connection == 'ssh'

Diff for: roles/_meta/common_base/meta/main.yml

@@ -27,7 +27,6 @@ dependencies:
     when: is_local is not defined or not is_local
   - role: debian/locales
   - role: debian/user_root
-  - role: debian/user_provision
   - role: debian/apt_extra_packages
   - role: debian/apt_unattended_upgrades
     when: is_local is not defined or not is_local
@@ -42,15 +41,3 @@ dependencies:
     when: is_local is not defined or not is_local
   - role: debian/sudo_config
     when: is_local is not defined or not is_local
-  - role: debian/ansible
-    ce_ansible:
-      venv_path: "/home/{{ user_provision.username }}/ansible"
-      venv_command: "/usr/bin/python3 -m venv"
-      install_username: "{{ user_provision.username }}"
-      upgrade:
-        enabled: true
-        command: "/home/{{ user_provision.username }}/ansible/bin/python3 -m pip install --upgrade ansible"
-        on_calendar: "*-*-* 01:30:00"
-        timer_name: upgrade_ansible
-      linters:
-        enabled: false

Diff for: roles/debian/ansible/README.md

@@ -11,7 +11,7 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip`
 ```yaml
 ---
 ce_ansible:
-  # These are usually set within another role using _venv_path, _venv_command and _install_username but can be overridden.
+  # These are usually set within another role using _venv_path, _venv_command and _venv_install_username but can be overridden.
   #venv_path: "/home/{{ ce_provision.username }}/ansible"
   #venv_command: /usr/bin/python3.11 -m venv
   #install_username: deploy # user to become when creating venv

Diff for: roles/debian/ansible/defaults/main.yml

@@ -1,6 +1,6 @@
 ---
 ce_ansible:
-  # These are usually set within another role using _venv_path, _venv_command and _install_username but can be overridden.
+  # These are usually set within another role using _venv_path, _venv_command and _venv_install_username but can be overridden.
   #venv_path: "/home/{{ ce_provision.username }}/ansible"
   #venv_command: /usr/bin/python3.11 -m venv
   #install_username: deploy # user to become when creating venv

Diff for: roles/debian/ansible/tasks/main.yml

@@ -33,42 +33,26 @@
         - python3-venv
         - cloud-init # package can get removed with python3-yaml but we need it for auto-scale
 
-- name: Ensure pip is at latest version.
-  ansible.builtin.pip:
-    name:
-      - pip
-    state: latest
-    virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}"
-    virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}"
-  become_user: "{{ ce_ansible.install_username | default(_install_username) }}"
-  become: true
-
-- name: Install Ansible.
-  ansible.builtin.pip:
-    name:
-      - ansible
-    virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}"
-    virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}"
-  become_user: "{{ ce_ansible.install_username | default(_install_username) }}"
-  become: true
+- name: Install pip and Ansible.
+  ansible.builtin.include_role:
+    name: debian/python_pip_packages
+  vars:
+    python_pip_packages:
+      packages:
+        - name: pip
+          state: latest
+        - name: ansible
 
 - name: Install linters.
-  ansible.builtin.pip:
-    name:
-      - ansible-lint
-      - yamllint
-    virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}"
-    virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}"
-  become_user: "{{ ce_ansible.install_username | default(_install_username) }}"
-  become: true
+  ansible.builtin.include_role:
+    name: debian/python_pip_packages
+  vars:
+    python_pip_packages:
+      packages:
+        - name: ansible-lint
+        - name: yamllint
   when: ce_ansible.linters.enabled
 
-- name: Ensure Ansible venv permissions.
-  ansible.builtin.file:
-    path: "{{ ce_ansible.venv_path | default(_venv_path) }}"
-    owner: "{{ ce_ansible.install_username | default(_install_username) }}"
-    group: "{{ ce_ansible.install_username | default(_install_username) }}"
-
 - name: Add the venv to $PATH using profile.d.
   ansible.builtin.copy:
     content: "export PATH=$PATH:{{ ce_ansible.venv_path | default(_venv_path) }}/bin"

Diff for: roles/debian/ce_deploy/tasks/main.yml

@@ -32,7 +32,7 @@
     recurse: true
   when: ce_deploy.new_user
 
-- name: Fetch back the SSH pub key.
+- name: Place the public key in the ce-provision data directory on the controller server.
   ansible.builtin.fetch:
     dest: "{{ _ce_provision_data_dir }}"
     src: "/home/{{ ce_deploy.username }}/.ssh/{{ ce_deploy.key_name }}"
@@ -48,7 +48,6 @@
     dest: "{{ ce_deploy.local_dir }}"
     version: "{{ ce_deploy.own_repository_branch  | default('master') }}"
     update: true
-    # @todo?
     accept_hostkey: true
   become: true
   become_user: "{{ ce_deploy.username }}"
@@ -132,7 +131,7 @@
   ansible.builtin.set_fact:
     _venv_path: "{{ ce_deploy.venv_path }}"
     _venv_command: "{{ ce_deploy.venv_command }}"
-    _install_username: "{{ ce_deploy.install_username }}"
+    _venv_install_username: "{{ ce_deploy.install_username }}"
     _ce_ansible_timer_name: "{{ ce_deploy.upgrade_timer_name }}"
 
 - name: Install Ansible.
@@ -175,6 +174,14 @@
   become_user: "{{ ce_deploy.username }}"
   when: _ce_deploy_custom_galaxy_requirements.stat.exists
 
+- name: Ensure Ansible venv permissions.
+  ansible.builtin.file:
+    path: "{{ _venv_path }}"
+    state: directory
+    recurse: true
+    owner: "{{ ce_deploy.username }}"
+    group: "{{ ce_deploy.username }}"
+
 - name: Create systemd timer to upgrade mandatory ansible-galaxy collections.
   ansible.builtin.include_role:
     name: contrib/systemd_timers