From 90aeb2c5638643dad10808e36ec9e4b691f38591 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 22 Nov 2023 18:16:51 +0100 Subject: [PATCH 01/51] Improving AWS subnet docs. --- docs/roles/aws/aws_ec2_autoscale_cluster.md | 2 ++ docs/roles/aws/aws_rds.md | 1 + docs/roles/aws/aws_vpc_subnet.md | 1 + roles/aws/aws_ec2_autoscale_cluster/README.md | 2 ++ roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml | 2 ++ roles/aws/aws_rds/README.md | 1 + roles/aws/aws_rds/defaults/main.yml | 1 + roles/aws/aws_vpc_subnet/README.md | 1 + roles/aws/aws_vpc_subnet/defaults/main.yml | 1 + 9 files changed, 12 insertions(+) diff --git a/docs/roles/aws/aws_ec2_autoscale_cluster.md b/docs/roles/aws/aws_ec2_autoscale_cluster.md index 5e6909609..49f419391 100644 --- a/docs/roles/aws/aws_ec2_autoscale_cluster.md +++ b/docs/roles/aws/aws_ec2_autoscale_cluster.md @@ -36,6 +36,8 @@ aws_ec2_autoscale_cluster: vpc_id: vpc-XXXX # One of vpc_id or vpc_name is mandatory. # vpc_name: example-vpc subnets: + # If rds.rds.true is set and you are creating an RDS instance then you must use at least two AZs, this is hard requirement. + # This means your region and environment must also have a minimum of two subnets - see `aws_vpc_subnet` # - az: a # cidr: "10.0.3.0/26" - az: b diff --git a/docs/roles/aws/aws_rds.md b/docs/roles/aws/aws_rds.md index e89cd6e74..1e5287a15 100644 --- a/docs/roles/aws/aws_rds.md +++ b/docs/roles/aws/aws_rds.md @@ -19,6 +19,7 @@ aws_rds: aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" multi_az: true + # You must provide at least two subnets in two different AZs, even for single AZ deployments. subnets: - subnet-aaaaaaaa - subnet-bbbbbbbb diff --git a/docs/roles/aws/aws_vpc_subnet.md b/docs/roles/aws/aws_vpc_subnet.md index 5d97d303a..1b4ef012b 100644 --- a/docs/roles/aws/aws_vpc_subnet.md +++ b/docs/roles/aws/aws_vpc_subnet.md @@ -11,6 +11,7 @@ aws_vpc_subnet: # vpc_name: example-vpc aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" + # If you intend to have RDS instances you must provide at least two subnets. subnets: - cidr_block: "10.0.0.0/24" # ipv6_cidr_block: "1" # This will create something like xxxx:xxxx:xxxx:xxyy::/64 where yy is created using the ansible.utils.ipsubnet filter automatically - DO NOT DEFINE IF IPV6 IS NOT REQUIRED diff --git a/roles/aws/aws_ec2_autoscale_cluster/README.md b/roles/aws/aws_ec2_autoscale_cluster/README.md index 5e6909609..49f419391 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/README.md +++ b/roles/aws/aws_ec2_autoscale_cluster/README.md @@ -36,6 +36,8 @@ aws_ec2_autoscale_cluster: vpc_id: vpc-XXXX # One of vpc_id or vpc_name is mandatory. # vpc_name: example-vpc subnets: + # If rds.rds.true is set and you are creating an RDS instance then you must use at least two AZs, this is hard requirement. + # This means your region and environment must also have a minimum of two subnets - see `aws_vpc_subnet` # - az: a # cidr: "10.0.3.0/26" - az: b diff --git a/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml b/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml index 5bdd39aef..9af962f5c 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml +++ b/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml @@ -17,6 +17,8 @@ aws_ec2_autoscale_cluster: vpc_id: vpc-XXXX # One of vpc_id or vpc_name is mandatory. # vpc_name: example-vpc subnets: + # If rds.rds.true is set and you are creating an RDS instance then you must use at least two AZs, this is hard requirement. + # This means your region and environment must also have a minimum of two subnets - see `aws_vpc_subnet` # - az: a # cidr: "10.0.3.0/26" - az: b diff --git a/roles/aws/aws_rds/README.md b/roles/aws/aws_rds/README.md index e89cd6e74..1e5287a15 100644 --- a/roles/aws/aws_rds/README.md +++ b/roles/aws/aws_rds/README.md @@ -19,6 +19,7 @@ aws_rds: aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" multi_az: true + # You must provide at least two subnets in two different AZs, even for single AZ deployments. subnets: - subnet-aaaaaaaa - subnet-bbbbbbbb diff --git a/roles/aws/aws_rds/defaults/main.yml b/roles/aws/aws_rds/defaults/main.yml index 96b4d4914..faa3ee5bb 100644 --- a/roles/aws/aws_rds/defaults/main.yml +++ b/roles/aws/aws_rds/defaults/main.yml @@ -2,6 +2,7 @@ aws_rds: aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" multi_az: true + # You must provide at least two subnets in two different AZs, even for single AZ deployments. subnets: - subnet-aaaaaaaa - subnet-bbbbbbbb diff --git a/roles/aws/aws_vpc_subnet/README.md b/roles/aws/aws_vpc_subnet/README.md index 5d97d303a..1b4ef012b 100644 --- a/roles/aws/aws_vpc_subnet/README.md +++ b/roles/aws/aws_vpc_subnet/README.md @@ -11,6 +11,7 @@ aws_vpc_subnet: # vpc_name: example-vpc aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" + # If you intend to have RDS instances you must provide at least two subnets. subnets: - cidr_block: "10.0.0.0/24" # ipv6_cidr_block: "1" # This will create something like xxxx:xxxx:xxxx:xxyy::/64 where yy is created using the ansible.utils.ipsubnet filter automatically - DO NOT DEFINE IF IPV6 IS NOT REQUIRED diff --git a/roles/aws/aws_vpc_subnet/defaults/main.yml b/roles/aws/aws_vpc_subnet/defaults/main.yml index 1e29a7d01..e72b3ae64 100644 --- a/roles/aws/aws_vpc_subnet/defaults/main.yml +++ b/roles/aws/aws_vpc_subnet/defaults/main.yml @@ -3,6 +3,7 @@ aws_vpc_subnet: # vpc_name: example-vpc aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" + # If you intend to have RDS instances you must provide at least two subnets. subnets: - cidr_block: "10.0.0.0/24" # ipv6_cidr_block: "1" # This will create something like xxxx:xxxx:xxxx:xxyy::/64 where yy is created using the ansible.utils.ipsubnet filter automatically - DO NOT DEFINE IF IPV6 IS NOT REQUIRED From 7c31b40ef78d23c459b7c05ed01463edfe756255 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 12 Dec 2023 18:34:14 +0100 Subject: [PATCH 02/51] Error in timers structure in the SSL role. --- roles/debian/nginx/defaults/main.yml | 1 + roles/debian/ssl/tasks/letsencrypt.yml | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/debian/nginx/defaults/main.yml b/roles/debian/nginx/defaults/main.yml index 61f2be571..64dbc76fa 100644 --- a/roles/debian/nginx/defaults/main.yml +++ b/roles/debian/nginx/defaults/main.yml @@ -57,6 +57,7 @@ nginx: # reload_command: restart # reload: # - nginx + # on_calendar: "Mon *-*-* 04:00:00" ratelimitingcrawlers: true is_default: true basic_auth: diff --git a/roles/debian/ssl/tasks/letsencrypt.yml b/roles/debian/ssl/tasks/letsencrypt.yml index 616040ae1..65ff333b4 100644 --- a/roles/debian/ssl/tasks/letsencrypt.yml +++ b/roles/debian/ssl/tasks/letsencrypt.yml @@ -87,6 +87,5 @@ ansible.builtin.include_role: name: contrib/systemd_timers vars: - timers: - certbot_renewal: "{{ _certbot_renewal_timer }}" + timers: "{{ _certbot_renewal_timer }}" when: ssl.autorenew From b75fe03127d42033ed1b4927f2f89ac78d4a78d4 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 21 Dec 2023 16:00:54 +0100 Subject: [PATCH 03/51] Removing obsolete backports requirements. --- roles/debian/firewall_config/tasks/main.yml | 10 ------ roles/debian/ldap_server/tasks/main.yml | 11 ------- roles/debian/lhci/tasks/main.yml | 34 --------------------- 3 files changed, 55 deletions(-) diff --git a/roles/debian/firewall_config/tasks/main.yml b/roles/debian/firewall_config/tasks/main.yml index 66941b51a..cf770ab4e 100644 --- a/roles/debian/firewall_config/tasks/main.yml +++ b/roles/debian/firewall_config/tasks/main.yml @@ -1,14 +1,4 @@ --- -# @TODO remove when https://redmine.codeenigma.net/issues/58246 can be done -- name: Install iptables with backports. # we have to do this in ce-dev or the contrib role will fail - ansible.builtin.apt: - pkg: ["iptables"] - state: present - default_release: buster-backports - when: - - is_local is defined - - is_local - - name: Shift general firewall settings to expected variables. ansible.builtin.set_fact: firewall_state: "{{ firewall_config.firewall_state }}" diff --git a/roles/debian/ldap_server/tasks/main.yml b/roles/debian/ldap_server/tasks/main.yml index 52fb1ccf4..9a773da58 100644 --- a/roles/debian/ldap_server/tasks/main.yml +++ b/roles/debian/ldap_server/tasks/main.yml @@ -14,21 +14,10 @@ purge: true when: ldap_server.slapd.purge -# @TODO remove when https://redmine.codeenigma.net/issues/58246 can be done -- name: Ensure LDAP and dependencies are installed from backports. - ansible.builtin.apt: - pkg: ["slapd", "ldapscripts", "libldap2-dev"] # python-ldap needs libldap2-dev - state: present - default_release: buster-backports - when: - - is_local is defined - - is_local - - name: Ensure LDAP and python-ldap and dependencies are installed. ansible.builtin.apt: pkg: ["slapd", "ldapscripts", "libldap2-dev"] state: present - when: is_local is not defined - name: Ensure additional dependencies for python-ldap are installed. ansible.builtin.apt: diff --git a/roles/debian/lhci/tasks/main.yml b/roles/debian/lhci/tasks/main.yml index c92447d4b..56c1fd9c9 100644 --- a/roles/debian/lhci/tasks/main.yml +++ b/roles/debian/lhci/tasks/main.yml @@ -10,26 +10,6 @@ state: present filename: google-chrome -# @TODO remove when https://redmine.codeenigma.net/issues/58246 can be done -- name: Install Google Chrome "headful" mode dependencies from backports. - ansible.builtin.apt: - name: - - xorg - - xvfb - - gtk2-engines-pixbuf - - dbus-x11 - - xfonts-base - - xfonts-100dpi - - xfonts-75dpi - - xfonts-cyrillic - - xfonts-scalable - default_release: buster-backports - state: present - when: - - is_local - - ansible_distribution == "Debian" - - ansible_distribution_major_version == "10" - - name: Install Google Chrome "headful" mode dependencies. ansible.builtin.apt: name: @@ -43,19 +23,6 @@ - xfonts-cyrillic - xfonts-scalable state: present - when: is_local is not defined - -# @TODO remove when https://redmine.codeenigma.net/issues/58246 can be done -- name: Install VNC support from backports. - ansible.builtin.apt: - name: x11vnc - default_release: buster-backports - state: present - when: - - lhci.enable_vnc - - is_local - - ansible_distribution == "Debian" - - ansible_distribution_major_version == "10" - name: Install VNC support. ansible.builtin.apt: @@ -63,7 +30,6 @@ state: present when: - lhci.enable_vnc - - is_local is not defined - name: Configure Xvfb to start on boot. ansible.builtin.shell: | From 5b27a8c0f0d7a9f8f1fd8437c96fafe33f98a692 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 21 Dec 2023 16:46:16 +0100 Subject: [PATCH 04/51] Allow the billing role to access Sustainability information. --- roles/aws/aws_iam_saml/templates/access_billing_policy.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 b/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 index cf20372f8..714056642 100644 --- a/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 +++ b/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 @@ -9,7 +9,8 @@ "freetier:*", "ce:*", "cur:*", - "tax:*" + "tax:*", + "sustainability:*" ], "Effect": "Allow", "Resource": "*" From 9ca4148dddc42a0d5e102562902f2d017500b996 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 21 Dec 2023 18:27:48 +0100 Subject: [PATCH 05/51] Missing comma in IAM billing policy. --- roles/aws/aws_iam_saml/templates/access_billing_policy.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 b/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 index 714056642..b26f9e7c7 100644 --- a/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 +++ b/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 @@ -28,7 +28,7 @@ "tax:BatchPutTaxRegistration", "tax:DeleteTaxRegistration", "tax:PutTaxInheritance" - ] + ], "Effect": "Deny", "Resource": "*" } From c8199bef92edb321c8df6e7d11a361c98227941e Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 9 Jan 2024 19:30:01 +0100 Subject: [PATCH 06/51] Removing broken GitLab Runner code. --- roles/debian/gitlab_runner/tasks/main.yml | 25 +++++++++++-------- .../gitlab_runner/templates/fargate.toml.j2 | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/roles/debian/gitlab_runner/tasks/main.yml b/roles/debian/gitlab_runner/tasks/main.yml index 2601d8e4f..37a30227a 100644 --- a/roles/debian/gitlab_runner/tasks/main.yml +++ b/roles/debian/gitlab_runner/tasks/main.yml @@ -35,18 +35,21 @@ # gitlab-runner register --url https://gitlab.com/ --registration-token TOKEN_HERE --name fargate-test-runner --run-untagged --executor custom -n # This will automatically update /etc/gitlab-runner/config.toml +# @TODO this needs rethinking, we cannot delegate include_role! +# Replace security_group in template with _aws_security_group_list[0] when resolved + # Populates the _aws_security_group_list variable used in fargate.toml.j2 -- name: Generate security group information. - ansible.builtin.include_role: - name: aws/aws_security_groups - vars: - profile: "{{ aws_elasticache.aws_profile }}" - region: "{{ aws_elasticache.region }}" - group_names: - - "{{ gitlab_runner.fargate.security_group }}" - return_type: ids - when: gitlab_runner.install_fargate - delegate_to: localhost +#- name: Generate security group information. +# ansible.builtin.include_role: +# name: aws/aws_security_groups +# vars: +# profile: "{{ gitlab_runner.fargate.region }}" +# region: "{{ gitlab_runner.fargate.region }}" +# group_names: +# - "{{ gitlab_runner.fargate.security_group }}" +# return_type: ids +# when: gitlab_runner.install_fargate +# delegate_to: localhost - name: Create the Fargate driver directory if it does not exist. ansible.builtin.file: diff --git a/roles/debian/gitlab_runner/templates/fargate.toml.j2 b/roles/debian/gitlab_runner/templates/fargate.toml.j2 index 6af78fc5a..c32c3633a 100644 --- a/roles/debian/gitlab_runner/templates/fargate.toml.j2 +++ b/roles/debian/gitlab_runner/templates/fargate.toml.j2 @@ -5,7 +5,7 @@ LogFormat = "text" Cluster = "{{ gitlab_runner.fargate.cluster }}" Region = "{{ gitlab_runner.fargate.region }}" Subnet = "{{ gitlab_runner.fargate.subnet }}" - SecurityGroup = "{{ _aws_security_group_list[0] }}" + SecurityGroup = "{{ gitlab_runner.fargate.security_group }}" TaskDefinition = "{{ gitlab_runner.fargate.task_definition }}" EnablePublicIP = {{ gitlab_runner.fargate.public_ip }} PlatformVersion = "{{ gitlab_runner.fargate.version }}" From 0ba41388c95088c6befa309343268d6c21459f25 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 10:58:45 +0100 Subject: [PATCH 07/51] Fixed the include_role task in gitlab_runner. --- docs/roles/debian/gitlab_runner.md | 1 + roles/debian/gitlab_runner/README.md | 1 + roles/debian/gitlab_runner/defaults/main.yml | 1 + roles/debian/gitlab_runner/tasks/main.yml | 26 +++++++++---------- .../gitlab_runner/templates/fargate.toml.j2 | 2 +- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/roles/debian/gitlab_runner.md b/docs/roles/debian/gitlab_runner.md index 52554ced5..10829036f 100644 --- a/docs/roles/debian/gitlab_runner.md +++ b/docs/roles/debian/gitlab_runner.md @@ -48,6 +48,7 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name + profile: "{{ _aws_profile }}" region: "eu-west-1" # AWS region name subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name diff --git a/roles/debian/gitlab_runner/README.md b/roles/debian/gitlab_runner/README.md index 52554ced5..10829036f 100644 --- a/roles/debian/gitlab_runner/README.md +++ b/roles/debian/gitlab_runner/README.md @@ -48,6 +48,7 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name + profile: "{{ _aws_profile }}" region: "eu-west-1" # AWS region name subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name diff --git a/roles/debian/gitlab_runner/defaults/main.yml b/roles/debian/gitlab_runner/defaults/main.yml index 845371447..ba1c1d48e 100644 --- a/roles/debian/gitlab_runner/defaults/main.yml +++ b/roles/debian/gitlab_runner/defaults/main.yml @@ -36,6 +36,7 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name + profile: "{{ _aws_profile }}" region: "eu-west-1" # AWS region name subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name diff --git a/roles/debian/gitlab_runner/tasks/main.yml b/roles/debian/gitlab_runner/tasks/main.yml index 37a30227a..685b0bfc2 100644 --- a/roles/debian/gitlab_runner/tasks/main.yml +++ b/roles/debian/gitlab_runner/tasks/main.yml @@ -35,21 +35,19 @@ # gitlab-runner register --url https://gitlab.com/ --registration-token TOKEN_HERE --name fargate-test-runner --run-untagged --executor custom -n # This will automatically update /etc/gitlab-runner/config.toml -# @TODO this needs rethinking, we cannot delegate include_role! -# Replace security_group in template with _aws_security_group_list[0] when resolved - # Populates the _aws_security_group_list variable used in fargate.toml.j2 -#- name: Generate security group information. -# ansible.builtin.include_role: -# name: aws/aws_security_groups -# vars: -# profile: "{{ gitlab_runner.fargate.region }}" -# region: "{{ gitlab_runner.fargate.region }}" -# group_names: -# - "{{ gitlab_runner.fargate.security_group }}" -# return_type: ids -# when: gitlab_runner.install_fargate -# delegate_to: localhost +- name: Generate security group information. + ansible.builtin.include_role: + name: aws/aws_security_groups + apply: + delegate_to: localhost # this is how you delegate the include_role module + vars: + profile: "{{ gitlab_runner.fargate.profile }}" + region: "{{ gitlab_runner.fargate.region }}" + group_names: + - "{{ gitlab_runner.fargate.security_group }}" + return_type: ids + when: gitlab_runner.install_fargate - name: Create the Fargate driver directory if it does not exist. ansible.builtin.file: diff --git a/roles/debian/gitlab_runner/templates/fargate.toml.j2 b/roles/debian/gitlab_runner/templates/fargate.toml.j2 index c32c3633a..6af78fc5a 100644 --- a/roles/debian/gitlab_runner/templates/fargate.toml.j2 +++ b/roles/debian/gitlab_runner/templates/fargate.toml.j2 @@ -5,7 +5,7 @@ LogFormat = "text" Cluster = "{{ gitlab_runner.fargate.cluster }}" Region = "{{ gitlab_runner.fargate.region }}" Subnet = "{{ gitlab_runner.fargate.subnet }}" - SecurityGroup = "{{ gitlab_runner.fargate.security_group }}" + SecurityGroup = "{{ _aws_security_group_list[0] }}" TaskDefinition = "{{ gitlab_runner.fargate.task_definition }}" EnablePublicIP = {{ gitlab_runner.fargate.public_ip }} PlatformVersion = "{{ gitlab_runner.fargate.version }}" From aacf70beb8415c637571707c37c6dc5d59704ea0 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 11:00:16 +0100 Subject: [PATCH 08/51] Suppressing a failure if there is no system pip to call. --- roles/debian/ansible/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 722690a64..d5f414624 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -10,6 +10,7 @@ state: absent executable: pip3 when: ansible_distribution_major_version | int < 12 + failed_when: false # don't stop the build if there's no system pip - name: Set up Python packages. ansible.builtin.include_role: From c634ef5e61ced0403f781263549e731833c4121d Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 11:19:53 +0100 Subject: [PATCH 09/51] Logic error in Ansible installer username, needs to be set from calling role. --- docs/roles/debian/ansible.md | 3 ++- docs/roles/debian/ce_deploy.md | 1 + docs/roles/debian/ce_provision.md | 1 + roles/debian/ansible/README.md | 3 ++- roles/debian/ansible/defaults/main.yml | 3 ++- roles/debian/ansible/tasks/main.yml | 6 +++--- roles/debian/ce_deploy/README.md | 1 + roles/debian/ce_deploy/defaults/main.yml | 1 + roles/debian/ce_deploy/tasks/main.yml | 1 + roles/debian/ce_provision/README.md | 1 + roles/debian/ce_provision/defaults/main.yml | 1 + roles/debian/ce_provision/tasks/main.yml | 1 + 12 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/roles/debian/ansible.md b/docs/roles/debian/ansible.md index 7e0015fd9..e87cedd66 100644 --- a/docs/roles/debian/ansible.md +++ b/docs/roles/debian/ansible.md @@ -11,9 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path and _venv_command but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv + #ansible_user: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index 66d7402b2..d829d089f 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -14,6 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index 741ca5a6c..001d8fad6 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -15,6 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ansible/README.md b/roles/debian/ansible/README.md index 7e0015fd9..e87cedd66 100644 --- a/roles/debian/ansible/README.md +++ b/roles/debian/ansible/README.md @@ -11,9 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path and _venv_command but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv + #ansible_user: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/defaults/main.yml b/roles/debian/ansible/defaults/main.yml index 746251010..2c6108938 100644 --- a/roles/debian/ansible/defaults/main.yml +++ b/roles/debian/ansible/defaults/main.yml @@ -1,8 +1,9 @@ --- ce_ansible: - # These are usually set within another role using _venv_path and _venv_command but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv + #ansible_user: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index d5f414624..ead087450 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -24,7 +24,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_provision.username }}" + become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" - name: Install Ansible. ansible.builtin.pip: @@ -33,7 +33,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_provision.username }}" + become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" - name: Install linters. ansible.builtin.pip: @@ -44,7 +44,7 @@ virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" when: ce_ansible.linters.enabled become: true - become_user: "{{ ce_provision.username }}" + become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" - name: Add the venv to $PATH using profile.d. ansible.builtin.copy: diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index 66d7402b2..d829d089f 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -14,6 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 92fefc0ed..276780b3b 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -6,6 +6,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index 723ef9f57..f047afacc 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -132,6 +132,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_deploy.venv_path }}" _venv_command: "{{ ce_deploy.venv_command }}" + _ansible_user: "{{ ce_deploy.ansible_user }}" - name: Install Ansible. ansible.builtin.include_role: diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index 741ca5a6c..001d8fad6 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -15,6 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index f69f7c939..35da59995 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -7,6 +7,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index 36b38bc8b..62662a1f5 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -140,6 +140,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_provision.venv_path }}" _venv_command: "{{ ce_provision.venv_command }}" + _ansible_user: "{{ ce_provision.ansible_user }}" - name: Install Ansible. ansible.builtin.include_role: From 267365988cfdb39561781854e2243f7ad485e843 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 12:40:14 +0100 Subject: [PATCH 10/51] ansible_user is a reserved variable, seems to be causing issues. --- docs/roles/debian/ansible.md | 4 ++-- docs/roles/debian/ce_deploy.md | 2 +- docs/roles/debian/ce_provision.md | 2 +- roles/debian/ansible/README.md | 4 ++-- roles/debian/ansible/defaults/main.yml | 4 ++-- roles/debian/ansible/tasks/main.yml | 6 +++--- roles/debian/ce_deploy/README.md | 2 +- roles/debian/ce_deploy/defaults/main.yml | 2 +- roles/debian/ce_deploy/tasks/main.yml | 2 +- roles/debian/ce_provision/README.md | 2 +- roles/debian/ce_provision/defaults/main.yml | 2 +- roles/debian/ce_provision/tasks/main.yml | 2 +- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/roles/debian/ansible.md b/docs/roles/debian/ansible.md index e87cedd66..faad26b84 100644 --- a/docs/roles/debian/ansible.md +++ b/docs/roles/debian/ansible.md @@ -11,10 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_user: deploy # user to become when creating venv + #ansible_install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index d829d089f..b9f3e983d 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -14,7 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_deploy.username }}" + ansible_install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index 001d8fad6..305ec1894 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -15,7 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_provision.username }}" + ansible_install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ansible/README.md b/roles/debian/ansible/README.md index e87cedd66..faad26b84 100644 --- a/roles/debian/ansible/README.md +++ b/roles/debian/ansible/README.md @@ -11,10 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_user: deploy # user to become when creating venv + #ansible_install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/defaults/main.yml b/roles/debian/ansible/defaults/main.yml index 2c6108938..319bdfe5d 100644 --- a/roles/debian/ansible/defaults/main.yml +++ b/roles/debian/ansible/defaults/main.yml @@ -1,9 +1,9 @@ --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_user: deploy # user to become when creating venv + #ansible_install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index ead087450..5b419b787 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -24,7 +24,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" + become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" - name: Install Ansible. ansible.builtin.pip: @@ -33,7 +33,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" + become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" - name: Install linters. ansible.builtin.pip: @@ -44,7 +44,7 @@ virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" when: ce_ansible.linters.enabled become: true - become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" + become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" - name: Add the venv to $PATH using profile.d. ansible.builtin.copy: diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index d829d089f..b9f3e983d 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -14,7 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_deploy.username }}" + ansible_install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 276780b3b..53ee6f8ed 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -6,7 +6,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_deploy.username }}" + ansible_install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index f047afacc..90143cdb5 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -132,7 +132,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_deploy.venv_path }}" _venv_command: "{{ ce_deploy.venv_command }}" - _ansible_user: "{{ ce_deploy.ansible_user }}" + _ansible_install_username: "{{ ce_deploy.ansible_install_username }}" - name: Install Ansible. ansible.builtin.include_role: diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index 001d8fad6..305ec1894 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -15,7 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_provision.username }}" + ansible_install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index 35da59995..d5e33b279 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -7,7 +7,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_provision.username }}" + ansible_install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index 62662a1f5..ba1be5773 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -140,7 +140,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_provision.venv_path }}" _venv_command: "{{ ce_provision.venv_command }}" - _ansible_user: "{{ ce_provision.ansible_user }}" + _ansible_install_username: "{{ ce_provision.ansible_install_username }}" - name: Install Ansible. ansible.builtin.include_role: From c6af4c0f7ea36b38a3c0417dca1abd747da3c9dc Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 13:10:59 +0100 Subject: [PATCH 11/51] _ansible_ANYTHING is reserved, using _install_username instead. --- docs/roles/debian/ansible.md | 4 ++-- docs/roles/debian/ce_deploy.md | 2 +- docs/roles/debian/ce_provision.md | 2 +- roles/debian/ansible/README.md | 4 ++-- roles/debian/ansible/defaults/main.yml | 4 ++-- roles/debian/ansible/tasks/main.yml | 6 +++--- roles/debian/ce_deploy/README.md | 2 +- roles/debian/ce_deploy/defaults/main.yml | 2 +- roles/debian/ce_deploy/tasks/main.yml | 2 +- roles/debian/ce_provision/README.md | 2 +- roles/debian/ce_provision/defaults/main.yml | 2 +- roles/debian/ce_provision/tasks/main.yml | 2 +- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/roles/debian/ansible.md b/docs/roles/debian/ansible.md index faad26b84..b7dfe0dbc 100644 --- a/docs/roles/debian/ansible.md +++ b/docs/roles/debian/ansible.md @@ -11,10 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_install_username: deploy # user to become when creating venv + #install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index b9f3e983d..d020663ab 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -14,7 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_deploy.username }}" + install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index 305ec1894..e8f83ea13 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -15,7 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_provision.username }}" + install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ansible/README.md b/roles/debian/ansible/README.md index faad26b84..b7dfe0dbc 100644 --- a/roles/debian/ansible/README.md +++ b/roles/debian/ansible/README.md @@ -11,10 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_install_username: deploy # user to become when creating venv + #install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/defaults/main.yml b/roles/debian/ansible/defaults/main.yml index 319bdfe5d..f58a026b0 100644 --- a/roles/debian/ansible/defaults/main.yml +++ b/roles/debian/ansible/defaults/main.yml @@ -1,9 +1,9 @@ --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_install_username: deploy # user to become when creating venv + #install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 5b419b787..4aac433a9 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -24,7 +24,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" + become_user: "{{ ce_ansible.install_username | default(_install_username) }}" - name: Install Ansible. ansible.builtin.pip: @@ -33,7 +33,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" + become_user: "{{ ce_ansible.install_username | default(_install_username) }}" - name: Install linters. ansible.builtin.pip: @@ -44,7 +44,7 @@ virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" when: ce_ansible.linters.enabled become: true - become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" + become_user: "{{ ce_ansible.install_username | default(_install_username) }}" - name: Add the venv to $PATH using profile.d. ansible.builtin.copy: diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index b9f3e983d..d020663ab 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -14,7 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_deploy.username }}" + install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 53ee6f8ed..52bd86415 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -6,7 +6,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_deploy.username }}" + install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index 90143cdb5..a9d607723 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -132,7 +132,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_deploy.venv_path }}" _venv_command: "{{ ce_deploy.venv_command }}" - _ansible_install_username: "{{ ce_deploy.ansible_install_username }}" + _install_username: "{{ ce_deploy.install_username }}" - name: Install Ansible. ansible.builtin.include_role: diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index 305ec1894..e8f83ea13 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -15,7 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_provision.username }}" + install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index d5e33b279..4c266e3e0 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -7,7 +7,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_provision.username }}" + install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index ba1be5773..ca59d5928 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -140,7 +140,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_provision.venv_path }}" _venv_command: "{{ ce_provision.venv_command }}" - _ansible_install_username: "{{ ce_provision.ansible_install_username }}" + _install_username: "{{ ce_provision.install_username }}" - name: Install Ansible. ansible.builtin.include_role: From b3026765ac2516ef1df8c48783e43ff41bde278d Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 18:00:23 +0100 Subject: [PATCH 12/51] python_boto role also needs the username set in the calling role. --- roles/debian/python_boto/defaults/main.yml | 3 ++- roles/debian/python_boto/tasks/main.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/debian/python_boto/defaults/main.yml b/roles/debian/python_boto/defaults/main.yml index 489e33ef6..69daa3a62 100644 --- a/roles/debian/python_boto/defaults/main.yml +++ b/roles/debian/python_boto/defaults/main.yml @@ -2,4 +2,5 @@ python_boto: boto3_version: "" # version string, e.g. "1.22.13" - empty string means latest # These are usually set within another role using _venv_path and _venv_command but can be overridden. #venv_path: /path/to/venv - #venv_command: /usr/bin/python3.11 -m venv \ No newline at end of file + #venv_command: /usr/bin/python3.11 -m venv + #install_username: deploy # user to become when creating venv diff --git a/roles/debian/python_boto/tasks/main.yml b/roles/debian/python_boto/tasks/main.yml index 8b989f63c..c89cc37e6 100644 --- a/roles/debian/python_boto/tasks/main.yml +++ b/roles/debian/python_boto/tasks/main.yml @@ -15,4 +15,4 @@ virtualenv: "{{ python_boto.venv_path | default(_venv_path) }}" virtualenv_command: "{{ python_boto.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_provision.username }}" + become_user: "{{ python_boto.install_username | default(_install_username) }}" From 2fe29a1457002b496e3cf17b341154924d4b2d49 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 18:00:48 +0100 Subject: [PATCH 13/51] Updating python_boto docs. --- docs/roles/debian/python_boto.md | 2 ++ roles/debian/python_boto/README.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/roles/debian/python_boto.md b/docs/roles/debian/python_boto.md index ee3e99a52..268af9614 100644 --- a/docs/roles/debian/python_boto.md +++ b/docs/roles/debian/python_boto.md @@ -12,6 +12,8 @@ python_boto: # These are usually set within another role using _venv_path and _venv_command but can be overridden. #venv_path: /path/to/venv #venv_command: /usr/bin/python3.11 -m venv + #install_username: deploy # user to become when creating venv + ``` diff --git a/roles/debian/python_boto/README.md b/roles/debian/python_boto/README.md index ee3e99a52..268af9614 100644 --- a/roles/debian/python_boto/README.md +++ b/roles/debian/python_boto/README.md @@ -12,6 +12,8 @@ python_boto: # These are usually set within another role using _venv_path and _venv_command but can be overridden. #venv_path: /path/to/venv #venv_command: /usr/bin/python3.11 -m venv + #install_username: deploy # user to become when creating venv + ``` From 32f3ced402d3020afbc2d5aa3fae93a0bfca8729 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 Jan 2024 14:21:52 +0100 Subject: [PATCH 14/51] Making profile.d loading more robust. --- scripts/_common.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index 0da872d6f..a1aae6923 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -34,10 +34,12 @@ if [ ! -d "$ANSIBLE_DATA_DIR" ]; then mkdir "$ANSIBLE_DATA_DIR" fi # Load the contents of profile.d in case we added items to $PATH there. -for f in /etc/profile.d/*; do -# shellcheck source=/dev/null - . "$f" -done +if [ -n "$(ls -A /etc/profile.d)" ]; then + for f in /etc/profile.d/*; do + # shellcheck source=/dev/null + . "$f" + done +fi # Parse options arguments. parse_options(){ while [ "${1:-}" ]; do From 1f8d80e99ee28f40c41d60e976fbbfbf7c262698 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 Jan 2024 14:22:34 +0100 Subject: [PATCH 15/51] Also pip removing ansible-core and trying with pip and pip3 to cover all bases. --- roles/debian/ansible/tasks/main.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 4aac433a9..7ea4f100d 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -4,12 +4,17 @@ pkg: ansible state: absent -- name: Ensure Ansible is not installed globally with pip3 on older versions of Debian. +- name: Ensure Ansible is not installed globally with pip or pip3 on older versions of Debian. ansible.builtin.pip: - name: ansible + name: + - ansible + - ansible-core state: absent - executable: pip3 + executable: "{{ item }}" when: ansible_distribution_major_version | int < 12 + with_items: + - pip + - pip3 failed_when: false # don't stop the build if there's no system pip - name: Set up Python packages. From ff2803f77418a668031bff0b82c52214126dc506 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 Jan 2024 14:23:27 +0100 Subject: [PATCH 16/51] Updating bad AWS SG role var namespacing in other roles. --- .../aws_ec2_autoscale_cluster/tasks/main.yml | 18 ++++++++++-------- roles/aws/aws_ec2_with_eip/tasks/main.yml | 9 +++++---- roles/aws/aws_efs/tasks/main.yml | 9 +++++---- roles/aws/aws_elasticache/tasks/main.yml | 9 +++++---- roles/aws/aws_elb/tasks/main.yml | 9 +++++---- roles/aws/aws_opensearch/tasks/main.yml | 9 +++++---- roles/aws/aws_rds/tasks/main.yml | 9 +++++---- roles/debian/gitlab_runner/tasks/main.yml | 11 ++++++----- 8 files changed, 46 insertions(+), 37 deletions(-) diff --git a/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml b/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml index 70597d699..611ba383c 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml +++ b/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml @@ -379,10 +379,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_ec2_autoscale_cluster.aws_profile }}" - region: "{{ aws_ec2_autoscale_cluster.region }}" - group_names: "{{ aws_ec2_autoscale_cluster.alb_security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_ec2_autoscale_cluster.aws_profile }}" + region: "{{ aws_ec2_autoscale_cluster.region }}" + group_names: "{{ aws_ec2_autoscale_cluster.alb_security_groups }}" + return_type: ids when: - aws_ec2_autoscale_cluster.alb_security_groups | length > 0 - aws_ec2_autoscale_cluster.create_elb @@ -425,10 +426,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_ec2_autoscale_cluster.aws_profile }}" - region: "{{ aws_ec2_autoscale_cluster.region }}" - group_names: "{{ aws_ec2_autoscale_cluster.cluster_security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_ec2_autoscale_cluster.aws_profile }}" + region: "{{ aws_ec2_autoscale_cluster.region }}" + group_names: "{{ aws_ec2_autoscale_cluster.cluster_security_groups }}" + return_type: ids when: - aws_ec2_autoscale_cluster.cluster_security_groups | length > 0 - aws_ec2_autoscale_cluster.asg_refresh diff --git a/roles/aws/aws_ec2_with_eip/tasks/main.yml b/roles/aws/aws_ec2_with_eip/tasks/main.yml index 06aaec917..fea2c4ecb 100644 --- a/roles/aws/aws_ec2_with_eip/tasks/main.yml +++ b/roles/aws/aws_ec2_with_eip/tasks/main.yml @@ -95,10 +95,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_ec2_with_eip.aws_profile }}" - region: "{{ aws_ec2_with_eip.region }}" - group_names: "{{ aws_ec2_with_eip.security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_ec2_with_eip.aws_profile }}" + region: "{{ aws_ec2_with_eip.region }}" + group_names: "{{ aws_ec2_with_eip.security_groups }}" + return_type: ids when: aws_ec2_with_eip.security_groups | length > 0 # Do not create an instance if _aws_hostname is not an EC2 generated address unless `force: true` diff --git a/roles/aws/aws_efs/tasks/main.yml b/roles/aws/aws_efs/tasks/main.yml index 6e07019b5..b7351ebd8 100644 --- a/roles/aws/aws_efs/tasks/main.yml +++ b/roles/aws/aws_efs/tasks/main.yml @@ -3,10 +3,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_efs.aws_profile }}" - region: "{{ aws_efs.region }}" - group_names: "{{ aws_efs.security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_efs.aws_profile }}" + region: "{{ aws_efs.region }}" + group_names: "{{ aws_efs.security_groups }}" + return_type: ids when: aws_efs.security_groups | length > 0 - name: Reset subnet ids info diff --git a/roles/aws/aws_elasticache/tasks/main.yml b/roles/aws/aws_elasticache/tasks/main.yml index 83c64efd8..7c75d36eb 100644 --- a/roles/aws/aws_elasticache/tasks/main.yml +++ b/roles/aws/aws_elasticache/tasks/main.yml @@ -3,10 +3,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_elasticache.aws_profile }}" - region: "{{ aws_elasticache.region }}" - group_names: "{{ aws_elasticache.elasticache_security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_elasticache.aws_profile }}" + region: "{{ aws_elasticache.region }}" + group_names: "{{ aws_elasticache.elasticache_security_groups }}" + return_type: ids when: aws_elasticache.elasticache_security_groups | length > 0 - name: Create ElastiCache subnet group. diff --git a/roles/aws/aws_elb/tasks/main.yml b/roles/aws/aws_elb/tasks/main.yml index 4c4416338..a47f372ad 100644 --- a/roles/aws/aws_elb/tasks/main.yml +++ b/roles/aws/aws_elb/tasks/main.yml @@ -113,10 +113,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_elb.aws_profile }}" - region: "{{ aws_elb.region }}" - group_names: "{{ aws_elb.elb_security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_elb.aws_profile }}" + region: "{{ aws_elb.region }}" + group_names: "{{ aws_elb.elb_security_groups }}" + return_type: ids when: aws_elb.elb_security_groups | length > 0 - name: Create the ALB. diff --git a/roles/aws/aws_opensearch/tasks/main.yml b/roles/aws/aws_opensearch/tasks/main.yml index f43b79bb0..39368148a 100644 --- a/roles/aws/aws_opensearch/tasks/main.yml +++ b/roles/aws/aws_opensearch/tasks/main.yml @@ -4,10 +4,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_opensearch.aws_profile }}" - region: "{{ aws_opensearch.region }}" - group_names: "{{ aws_opensearch.security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_opensearch.aws_profile }}" + region: "{{ aws_opensearch.region }}" + group_names: "{{ aws_opensearch.security_groups }}" + return_type: ids when: aws_opensearch.security_groups | length > 0 - name: Get the current AWS account caller identity information diff --git a/roles/aws/aws_rds/tasks/main.yml b/roles/aws/aws_rds/tasks/main.yml index 416dd30b4..85301323d 100644 --- a/roles/aws/aws_rds/tasks/main.yml +++ b/roles/aws/aws_rds/tasks/main.yml @@ -67,10 +67,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_rds.aws_profile }}" - region: "{{ aws_rds.region }}" - group_names: "{{ aws_rds.security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_rds.aws_profile }}" + region: "{{ aws_rds.region }}" + group_names: "{{ aws_rds.security_groups }}" + return_type: ids when: aws_rds.security_groups | length > 0 - name: Create RDS instance diff --git a/roles/debian/gitlab_runner/tasks/main.yml b/roles/debian/gitlab_runner/tasks/main.yml index 685b0bfc2..47b32ac4c 100644 --- a/roles/debian/gitlab_runner/tasks/main.yml +++ b/roles/debian/gitlab_runner/tasks/main.yml @@ -42,11 +42,12 @@ apply: delegate_to: localhost # this is how you delegate the include_role module vars: - profile: "{{ gitlab_runner.fargate.profile }}" - region: "{{ gitlab_runner.fargate.region }}" - group_names: - - "{{ gitlab_runner.fargate.security_group }}" - return_type: ids + aws_security_groups: + profile: "{{ gitlab_runner.fargate.profile }}" + region: "{{ gitlab_runner.fargate.region }}" + group_names: + - "{{ gitlab_runner.fargate.security_group }}" + return_type: ids when: gitlab_runner.install_fargate - name: Create the Fargate driver directory if it does not exist. From 09663980f70358fe135ff16488218e0b16814da4 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 Jan 2024 20:37:14 +0100 Subject: [PATCH 17/51] Refactoring how we handle python3-pip. --- roles/debian/ldap_server/meta/main.yml | 3 --- roles/debian/ldap_server/tasks/main.yml | 13 +++++++++++++ roles/debian/mysql_client/meta/main.yml | 3 --- roles/debian/mysql_client/tasks/main.yml | 13 +++++++++++++ roles/debian/nginx/meta/main.yml | 3 --- roles/debian/nginx/tasks/main.yml | 13 +++++++++++++ roles/debian/python_common/defaults/main.yml | 1 - 7 files changed, 39 insertions(+), 10 deletions(-) delete mode 100644 roles/debian/ldap_server/meta/main.yml delete mode 100644 roles/debian/mysql_client/meta/main.yml delete mode 100644 roles/debian/nginx/meta/main.yml diff --git a/roles/debian/ldap_server/meta/main.yml b/roles/debian/ldap_server/meta/main.yml deleted file mode 100644 index 551567003..000000000 --- a/roles/debian/ldap_server/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: debian/python_common diff --git a/roles/debian/ldap_server/tasks/main.yml b/roles/debian/ldap_server/tasks/main.yml index 9a773da58..bf2c0d3d1 100644 --- a/roles/debian/ldap_server/tasks/main.yml +++ b/roles/debian/ldap_server/tasks/main.yml @@ -14,6 +14,19 @@ purge: true when: ldap_server.slapd.purge +- name: Set up Python packages. + ansible.builtin.include_role: + name: debian/python_common + vars: + python_common: + remove_packages: + - python-pip + - python3-yaml + - python-botocore + - python-urllib3 + install_packages: + - python3-pip + - name: Ensure LDAP and python-ldap and dependencies are installed. ansible.builtin.apt: pkg: ["slapd", "ldapscripts", "libldap2-dev"] diff --git a/roles/debian/mysql_client/meta/main.yml b/roles/debian/mysql_client/meta/main.yml deleted file mode 100644 index 551567003..000000000 --- a/roles/debian/mysql_client/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: debian/python_common diff --git a/roles/debian/mysql_client/tasks/main.yml b/roles/debian/mysql_client/tasks/main.yml index d17123205..2ecca067e 100644 --- a/roles/debian/mysql_client/tasks/main.yml +++ b/roles/debian/mysql_client/tasks/main.yml @@ -6,6 +6,19 @@ update_cache: true cache_valid_time: 240 +- name: Set up Python packages. + ansible.builtin.include_role: + name: debian/python_common + vars: + python_common: + remove_packages: + - python-pip + - python3-yaml + - python-botocore + - python-urllib3 + install_packages: + - python3-pip + - name: Ensure PyMySQL is installed with pip3. ansible.builtin.pip: name: pymysql diff --git a/roles/debian/nginx/meta/main.yml b/roles/debian/nginx/meta/main.yml deleted file mode 100644 index 551567003..000000000 --- a/roles/debian/nginx/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: debian/python_common diff --git a/roles/debian/nginx/tasks/main.yml b/roles/debian/nginx/tasks/main.yml index 60a8c4246..0388ed007 100644 --- a/roles/debian/nginx/tasks/main.yml +++ b/roles/debian/nginx/tasks/main.yml @@ -6,6 +6,19 @@ update_cache: true cache_valid_time: 240 +- name: Set up Python packages. + ansible.builtin.include_role: + name: debian/python_common + vars: + python_common: + remove_packages: + - python-pip + - python3-yaml + - python-botocore + - python-urllib3 + install_packages: + - python3-pip + - name: Ensure passlib is installed with pip3. ansible.builtin.pip: name: passlib diff --git a/roles/debian/python_common/defaults/main.yml b/roles/debian/python_common/defaults/main.yml index afb3840df..59c3c7e72 100644 --- a/roles/debian/python_common/defaults/main.yml +++ b/roles/debian/python_common/defaults/main.yml @@ -8,5 +8,4 @@ python_common: install_packages: - python3-distutils - python3-venv - - python3-pip - cloud-init # package can get removed with python3-yaml but we need it for auto-scale \ No newline at end of file From 6b2b69b137c08153e459f323e97f5cd75ba4455e Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 12 Jan 2024 09:42:08 +0100 Subject: [PATCH 18/51] Allow passing in of the Python interpreter to Ansible. --- scripts/_common.sh | 11 ++++++++++- scripts/provision.sh | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index a1aae6923..da5277ef9 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -12,6 +12,7 @@ TARGET_PROVISION_BRANCH="" ANSIBLE_EXTRA_VARS="" ANSIBLE_DEFAULT_EXTRA_VARS="" ANSIBLE_PATH="" +PYTHON_INTERPRETER="" BUILD_WORKSPACE="" BUILD_WORKSPACE_BASE="$OWN_DIR/build" BUILD_ID="" @@ -101,6 +102,10 @@ parse_options(){ shift ANSIBLE_PATH="$1" ;; + "--python-interpreter") + shift + PYTHON_INTERPRETER="$1" + ;; *) usage exit 1 @@ -123,7 +128,11 @@ get_build_workspace(){ # Common extra-vars to pass to Ansible. get_ansible_defaults_vars(){ get_build_id - ANSIBLE_DEFAULT_EXTRA_VARS="{_ce_provision_base_dir: $OWN_DIR, _ce_provision_build_dir: $BUILD_WORKSPACE, _ce_provision_build_tmp_dir: $BUILD_TMP_DIR, _ce_provision_data_dir: $ANSIBLE_DATA_DIR, _ce_provision_build_id: $BUILD_ID, _ce_provision_force_play: $FORCE_PLAY, target_branch: $TARGET_PROVISION_BRANCH}" + if [ -n "$PYTHON_INTERPRETER" ]; then + ANSIBLE_DEFAULT_EXTRA_VARS="{ansible_python_interpreter: $PYTHON_INTERPRETER, _ce_provision_base_dir: $OWN_DIR, _ce_provision_build_dir: $BUILD_WORKSPACE, _ce_provision_build_tmp_dir: $BUILD_TMP_DIR, _ce_provision_data_dir: $ANSIBLE_DATA_DIR, _ce_provision_build_id: $BUILD_ID, _ce_provision_force_play: $FORCE_PLAY, target_branch: $TARGET_PROVISION_BRANCH}" + else + ANSIBLE_DEFAULT_EXTRA_VARS="{_ce_provision_base_dir: $OWN_DIR, _ce_provision_build_dir: $BUILD_WORKSPACE, _ce_provision_build_tmp_dir: $BUILD_TMP_DIR, _ce_provision_data_dir: $ANSIBLE_DATA_DIR, _ce_provision_build_id: $BUILD_ID, _ce_provision_force_play: $FORCE_PLAY, target_branch: $TARGET_PROVISION_BRANCH}" + fi } # Clone our target repo. diff --git a/scripts/provision.sh b/scripts/provision.sh index 92d2e0265..fda465e05 100755 --- a/scripts/provision.sh +++ b/scripts/provision.sh @@ -14,6 +14,7 @@ usage(){ echo 'Available options:' echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.' echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.' + echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.' echo '--workspace: Local existing clone of the repo/branch (if your deployment tool already has one). This will skip the cloning/fetching of the repo.' echo '--user: Linux user executing the script (defaults to controller).' echo '--absolute-playbook-path: Prevent prepending of the workspace path to the playbook path when Ansible is called.' From 3c1d06c9cfdbc7e86a8125966cfc3107ac8e9e0b Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 15 Jan 2024 15:58:21 +0100 Subject: [PATCH 19/51] Updating the packages server for CE. --- roles/debian/pam_linotp/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/debian/pam_linotp/tasks/main.yml b/roles/debian/pam_linotp/tasks/main.yml index d4736a2e6..d7fe822a2 100644 --- a/roles/debian/pam_linotp/tasks/main.yml +++ b/roles/debian/pam_linotp/tasks/main.yml @@ -1,13 +1,13 @@ --- - name: Add key for LinOTP repository. ansible.builtin.apt_key: - keyserver: http://packages.codeenigma.com/debian/codeenigma.pub - id: A344A0826FD987C6 + keyserver: https://packages.codeenigma.net/debian/codeenigma.pub + id: 77AFCA1C45124FBB2EFCD7267DC594C08E0497D4 state: present - name: Add repository for LinOTP. ansible.builtin.apt_repository: - repo: "deb http://packages.codeenigma.com/debian buster main" + repo: "deb https://packages.codeenigma.net/debian {{ ansible_distribution_release }} main" state: present - name: Ensure libpam-linotp is installed. From 2b422a93772edbcf2aa8f5caa67481027ab694bd Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 16 Jan 2024 11:56:42 +0100 Subject: [PATCH 20/51] Installing Ansible in a venv on all machines. --- roles/_meta/common_base/meta/main.yml | 2 ++ roles/_meta/webserver/meta/main.yml | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 95fcebc29..1ff94fc17 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -4,6 +4,7 @@ dependencies: - { role: debian/apt_extra_packages, apt_extra_packages: ["dnsutils", "telnet", "atop", "sysstat"], when: ( is_local is not defined or not is_local ) } - role: debian/locales - role: debian/user_root + - role: debian/user_provision - role: debian/apt_extra_packages - { role: debian/apt_unattended_upgrades, when: ( is_local is not defined or not is_local ) } - { role: debian/hosts, when: ( is_local is not defined or not is_local ) } @@ -12,3 +13,4 @@ dependencies: - { role: debian/clamav, when: ( is_local is not defined or not is_local ) } - { role: debian/firewall_config, when: ( is_local is not defined or not is_local ) } - { role: debian/sudo_config, when: ( is_local is not defined or not is_local ) } + - { role: debian/ansible, ce_ansible: {venv_path: '/home/' + ce_provision.username + '/ansible', venv_command: '/usr/bin/python3.11 -m venv', install_username: ce_provision.username, upgrade: {enabled: true, command: '/home/' + ce_provision.username + '/ansible/bin/python3 -m pip install --upgrade ansible', on_calendar: '*-*-* 01:30:00'}, linters: {enabled: false}} } diff --git a/roles/_meta/webserver/meta/main.yml b/roles/_meta/webserver/meta/main.yml index a66530c03..00249c536 100644 --- a/roles/_meta/webserver/meta/main.yml +++ b/roles/_meta/webserver/meta/main.yml @@ -2,7 +2,6 @@ dependencies: - { role: debian/ssh_server, when: ( is_local is not defined or not is_local ) } - role: _meta/common_base - - role: debian/user_provision - role: debian/user_deploy - role: debian/mysql_client - role: debian/nodejs From 33abf0a173133b414a96f792c405e0d518a8bd72 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 16 Jan 2024 12:17:07 +0100 Subject: [PATCH 21/51] Changing common_base format for readability. --- roles/_meta/common_base/meta/main.yml | 57 ++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 1ff94fc17..140fcf1bb 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -1,16 +1,55 @@ --- dependencies: - - { role: debian/apt_extra_packages, apt_extra_packages: ["rsync", "python3-apt", "acl", "gnupg", "build-essential", "cron", "bsd-mailx", "vim", "unzip", "ca-certificates"] } - - { role: debian/apt_extra_packages, apt_extra_packages: ["dnsutils", "telnet", "atop", "sysstat"], when: ( is_local is not defined or not is_local ) } + - role: debian/apt_extra_packages + apt_extra_packages: + - rsync + - python3-apt + - acl + - gnupg + - build-essential + - cron + - bsd-mailx + - vim + - unzip + - ca-certificates + - role: debian/apt_extra_packages + apt_extra_packages: + - dnsutils + - telnet + - atop + - sysstat + - tcpdump + - mtr-tiny + - htop + - ntp + - screen + - iptraf-ng + when: is_local is not defined or not is_local - role: debian/locales - role: debian/user_root - role: debian/user_provision - role: debian/apt_extra_packages - - { role: debian/apt_unattended_upgrades, when: ( is_local is not defined or not is_local ) } - - { role: debian/hosts, when: ( is_local is not defined or not is_local ) } - - { role: debian/rkhunter, when: ( is_local is not defined or not is_local ) } + - role: debian/apt_unattended_upgrades + when: is_local is not defined or not is_local + - role: debian/hosts + when: is_local is not defined or not is_local + - role: debian/rkhunter + when: is_local is not defined or not is_local - role: debian/postfix - - { role: debian/clamav, when: ( is_local is not defined or not is_local ) } - - { role: debian/firewall_config, when: ( is_local is not defined or not is_local ) } - - { role: debian/sudo_config, when: ( is_local is not defined or not is_local ) } - - { role: debian/ansible, ce_ansible: {venv_path: '/home/' + ce_provision.username + '/ansible', venv_command: '/usr/bin/python3.11 -m venv', install_username: ce_provision.username, upgrade: {enabled: true, command: '/home/' + ce_provision.username + '/ansible/bin/python3 -m pip install --upgrade ansible', on_calendar: '*-*-* 01:30:00'}, linters: {enabled: false}} } + - role: debian/clamav + when: is_local is not defined or not is_local + - role: debian/firewall_config + when: is_local is not defined or not is_local + - role: debian/sudo_config + when: is_local is not defined or not is_local + - role: debian/ansible + ce_ansible: + venv_path: "/home/{{ ce_provision.username }}/ansible" + venv_command: "/usr/bin/python3.11 -m venv" + install_username: "{{ ce_provision.username }}" + upgrade: + enabled: true + command: "/home/{{ ce_provision.username }}/ansible/bin/python3 -m pip install --upgrade ansible" + on_calendar: "*-*-* 01:30:00" + linters: + enabled: false From 394ec01472e16adef55bf5321a16dd8217730ff7 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 17 Jan 2024 19:02:48 +0100 Subject: [PATCH 22/51] No need to specify Python to the point release. --- roles/_meta/common_base/meta/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 140fcf1bb..9e3635fa6 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -45,7 +45,7 @@ dependencies: - role: debian/ansible ce_ansible: venv_path: "/home/{{ ce_provision.username }}/ansible" - venv_command: "/usr/bin/python3.11 -m venv" + venv_command: "/usr/bin/python3 -m venv" install_username: "{{ ce_provision.username }}" upgrade: enabled: true From f906530099ea48eb6ce0149c5ac319800ffc42bd Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 17 Jan 2024 19:08:34 +0100 Subject: [PATCH 23/51] Docs update. --- docs/_Sidebar.md | 2 +- docs/roles/debian/python_common.md | 14 ++++++++------ docs/roles/debian/swap.md | 7 +++---- roles/debian/python_common/README.md | 14 ++++++++------ roles/debian/swap/README.md | 7 ++----- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md index aa59208c8..5a5213a0b 100644 --- a/docs/_Sidebar.md +++ b/docs/_Sidebar.md @@ -76,7 +76,7 @@ - [SSHD](/roles/debian/ssh_server) - [SSL](/roles/debian/ssl) - [sudo config](/roles/debian/sudo_config) - - [General swap role to add swap space](/roles/debian/swap) + - [Swap](/roles/debian/swap) - [varnish-config](/roles/debian/varnish_config) - [wazuh](/roles/debian/wazuh) - [Init role](/roles/_init) diff --git a/docs/roles/debian/python_common.md b/docs/roles/debian/python_common.md index 2c9f384f9..94573f816 100644 --- a/docs/roles/debian/python_common.md +++ b/docs/roles/debian/python_common.md @@ -9,15 +9,17 @@ Packages required by all Python scripts and applications. ```yaml --- python_common: - remove_packages: - - python-pip - - python3-yaml # linters need a newer version of PyYAML than the one that ships with Debian - - python-botocore - - python-urllib3 + cleanup: false # set to true to remove unwanted packages + remove_packages: [] # list of packages to remove if cleanup: true + # Example from ansible role + #- python-pip + #- python3-yaml # linters need a newer version of PyYAML than the one that ships with Debian + #- python-botocore + #- python-urllib3 + # Commonly required Python system packages install_packages: - python3-distutils - python3-venv - - python3-pip - cloud-init # package can get removed with python3-yaml but we need it for auto-scale ``` diff --git a/docs/roles/debian/swap.md b/docs/roles/debian/swap.md index 8bfeaa59c..ef6f5f208 100644 --- a/docs/roles/debian/swap.md +++ b/docs/roles/debian/swap.md @@ -1,6 +1,5 @@ -# General swap role to add swap space - -By default, role is set to add 2G of swap space. +# Swap +General role to add swap space to servers. By default, role is set to add 4G of swap space. As a general rule of thumb, swap should be set to 2.5 times the amount of RAM. So if a server has 2G RAM, we should set swap to 5G by setting `size: "5G"` in the variables for that server. @@ -11,7 +10,7 @@ By default, role is set to add 2G of swap space. --- swap: count: 2048 - size: "2G" + size: "4G" swap_space: swapfile ``` diff --git a/roles/debian/python_common/README.md b/roles/debian/python_common/README.md index 2c9f384f9..94573f816 100644 --- a/roles/debian/python_common/README.md +++ b/roles/debian/python_common/README.md @@ -9,15 +9,17 @@ Packages required by all Python scripts and applications. ```yaml --- python_common: - remove_packages: - - python-pip - - python3-yaml # linters need a newer version of PyYAML than the one that ships with Debian - - python-botocore - - python-urllib3 + cleanup: false # set to true to remove unwanted packages + remove_packages: [] # list of packages to remove if cleanup: true + # Example from ansible role + #- python-pip + #- python3-yaml # linters need a newer version of PyYAML than the one that ships with Debian + #- python-botocore + #- python-urllib3 + # Commonly required Python system packages install_packages: - python3-distutils - python3-venv - - python3-pip - cloud-init # package can get removed with python3-yaml but we need it for auto-scale ``` diff --git a/roles/debian/swap/README.md b/roles/debian/swap/README.md index ea66bc066..ef6f5f208 100644 --- a/roles/debian/swap/README.md +++ b/roles/debian/swap/README.md @@ -1,8 +1,5 @@ -# General swap role to add swap space - -By default, role is set to add 2G of swap space. -As a general rule of thumb, swap should be set to 2.5x of current RAM. -If server has 2G RAM, set RAM to 5G by creating swap.yml inside vars at local directory. +# Swap +General role to add swap space to servers. By default, role is set to add 4G of swap space. As a general rule of thumb, swap should be set to 2.5 times the amount of RAM. So if a server has 2G RAM, we should set swap to 5G by setting `size: "5G"` in the variables for that server. From 9b6395403bbe2bdbffa3566ba620ee97bad895e1 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 18 Jan 2024 19:39:50 +0100 Subject: [PATCH 24/51] Fixing LDAP SSL to use systemd timer. --- docs/roles/debian/ldap_server.md | 1 + roles/debian/ldap_server/README.md | 1 + roles/debian/ldap_server/defaults/main.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/docs/roles/debian/ldap_server.md b/docs/roles/debian/ldap_server.md index 7fb78adfb..6514e4779 100644 --- a/docs/roles/debian/ldap_server.md +++ b/docs/roles/debian/ldap_server.md @@ -107,6 +107,7 @@ ldap_server: # reload_command: restart # reload: # - slapd + # on_calendar: "Mon *-*-* 04:00:00" replication: host: "" # host must be present in config/hosts for ce-provision, leave empty if no replication is desired port: "636" diff --git a/roles/debian/ldap_server/README.md b/roles/debian/ldap_server/README.md index 7fb78adfb..6514e4779 100644 --- a/roles/debian/ldap_server/README.md +++ b/roles/debian/ldap_server/README.md @@ -107,6 +107,7 @@ ldap_server: # reload_command: restart # reload: # - slapd + # on_calendar: "Mon *-*-* 04:00:00" replication: host: "" # host must be present in config/hosts for ce-provision, leave empty if no replication is desired port: "636" diff --git a/roles/debian/ldap_server/defaults/main.yml b/roles/debian/ldap_server/defaults/main.yml index 981d6b995..8178688a0 100644 --- a/roles/debian/ldap_server/defaults/main.yml +++ b/roles/debian/ldap_server/defaults/main.yml @@ -41,6 +41,7 @@ ldap_server: # reload_command: restart # reload: # - slapd + # on_calendar: "Mon *-*-* 04:00:00" replication: host: "" # host must be present in config/hosts for ce-provision, leave empty if no replication is desired port: "636" From 516f61d9ea0dde476856e6b216f75b1140bfac4e Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 18 Jan 2024 19:40:07 +0100 Subject: [PATCH 25/51] Allowing different systemd timer names for different Ansible installs. --- docs/roles/debian/ansible.md | 1 + docs/roles/debian/ce_deploy.md | 1 + docs/roles/debian/ce_provision.md | 1 + roles/_meta/common_base/meta/main.yml | 1 + roles/debian/ansible/README.md | 1 + roles/debian/ansible/defaults/main.yml | 1 + roles/debian/ansible/tasks/main.yml | 9 +++++---- roles/debian/ce_deploy/README.md | 1 + roles/debian/ce_deploy/defaults/main.yml | 1 + roles/debian/ce_deploy/tasks/main.yml | 1 + roles/debian/ce_provision/README.md | 1 + roles/debian/ce_provision/defaults/main.yml | 1 + roles/debian/ce_provision/tasks/main.yml | 1 + 13 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/roles/debian/ansible.md b/docs/roles/debian/ansible.md index b7dfe0dbc..f50de4929 100644 --- a/docs/roles/debian/ansible.md +++ b/docs/roles/debian/ansible.md @@ -19,6 +19,7 @@ ce_ansible: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too on_calendar: "*-*-* 01:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + #timer_name: upgrade_ansible linters: enabled: true # will not install linters if false, installing linters breaks cloud-init diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index d020663ab..f9e55b15f 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -15,6 +15,7 @@ ce_deploy: venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_deploy.username }}" + upgrade_timer_name: upgrade_ce_deploy_ansible # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index e8f83ea13..dba324120 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -16,6 +16,7 @@ ce_provision: venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_provision.username }}" + upgrade_timer_name: upgrade_ce_provision_ansible # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 9e3635fa6..7a79993e8 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -51,5 +51,6 @@ dependencies: enabled: true command: "/home/{{ ce_provision.username }}/ansible/bin/python3 -m pip install --upgrade ansible" on_calendar: "*-*-* 01:30:00" + timer_name: upgrade_ansible linters: enabled: false diff --git a/roles/debian/ansible/README.md b/roles/debian/ansible/README.md index b7dfe0dbc..f50de4929 100644 --- a/roles/debian/ansible/README.md +++ b/roles/debian/ansible/README.md @@ -19,6 +19,7 @@ ce_ansible: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too on_calendar: "*-*-* 01:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + #timer_name: upgrade_ansible linters: enabled: true # will not install linters if false, installing linters breaks cloud-init diff --git a/roles/debian/ansible/defaults/main.yml b/roles/debian/ansible/defaults/main.yml index f58a026b0..949bb48b7 100644 --- a/roles/debian/ansible/defaults/main.yml +++ b/roles/debian/ansible/defaults/main.yml @@ -8,5 +8,6 @@ ce_ansible: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too on_calendar: "*-*-* 01:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + #timer_name: upgrade_ansible linters: enabled: true # will not install linters if false, installing linters breaks cloud-init diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 56d1d94f2..b1aa47a7f 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -68,12 +68,13 @@ content: "export PATH=$PATH:{{ ce_ansible.venv_path | default(_venv_path) }}/bin" dest: "/etc/profile.d/ansible-path.sh" +- name: Build systemd timer variables. + ansible.builtin.set_fact: + _timer: {"{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}": { timer_command: "{{ ce_ansible.upgrade.command }}", timer_OnCalendar: "{{ ce_ansible.upgrade.on_calendar }}" }} + - name: Create systemd timer to upgrade Ansible. ansible.builtin.include_role: name: contrib/systemd_timers vars: - timers: - upgrade_ansible: - timer_command: "{{ ce_ansible.upgrade.command }}" - timer_OnCalendar: "{{ ce_ansible.upgrade.on_calendar }}" + timers: "{{ _timer }}" when: ce_ansible.upgrade.enabled diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index d020663ab..f9e55b15f 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -15,6 +15,7 @@ ce_deploy: venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_deploy.username }}" + upgrade_timer_name: upgrade_ce_deploy_ansible # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 52bd86415..1dc9ba5b3 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -7,6 +7,7 @@ ce_deploy: venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_deploy.username }}" + upgrade_timer_name: upgrade_ce_deploy_ansible # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index a9d607723..ae5395780 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -133,6 +133,7 @@ _venv_path: "{{ ce_deploy.venv_path }}" _venv_command: "{{ ce_deploy.venv_command }}" _install_username: "{{ ce_deploy.install_username }}" + _ce_ansible_timer_name: "{{ ce_deploy.upgrade_timer_name }}" - name: Install Ansible. ansible.builtin.include_role: diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index e8f83ea13..dba324120 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -16,6 +16,7 @@ ce_provision: venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_provision.username }}" + upgrade_timer_name: upgrade_ce_provision_ansible # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index 4c266e3e0..8f8d508cf 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -8,6 +8,7 @@ ce_provision: venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_provision.username }}" + upgrade_timer_name: upgrade_ce_provision_ansible # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index ca59d5928..0f4edeab1 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -141,6 +141,7 @@ _venv_path: "{{ ce_provision.venv_path }}" _venv_command: "{{ ce_provision.venv_command }}" _install_username: "{{ ce_provision.install_username }}" + _ce_ansible_timer_name: "{{ ce_provision.upgrade_timer_name }}" - name: Install Ansible. ansible.builtin.include_role: From 17f7804c427e1b3785b30a72c2f9a1762f3ebae9 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 19 Jan 2024 10:29:40 +0100 Subject: [PATCH 26/51] Fixing dynamic key name in ansible role. --- roles/debian/ansible/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index b1aa47a7f..8cbdc9241 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -70,7 +70,7 @@ - name: Build systemd timer variables. ansible.builtin.set_fact: - _timer: {"{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}": { timer_command: "{{ ce_ansible.upgrade.command }}", timer_OnCalendar: "{{ ce_ansible.upgrade.on_calendar }}" }} + _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" - name: Create systemd timer to upgrade Ansible. ansible.builtin.include_role: From 3f9f67e2ecb40cabb45fdabd7f3be5af5cdbe15a Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 19 Jan 2024 10:49:26 +0100 Subject: [PATCH 27/51] Trying to debug missing timer_command var. --- roles/debian/ansible/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 8cbdc9241..5ed23549e 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -72,6 +72,9 @@ ansible.builtin.set_fact: _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" +- debug: + msg: "{{ _timer }}" + - name: Create systemd timer to upgrade Ansible. ansible.builtin.include_role: name: contrib/systemd_timers From e34d247f22be1540e08717b05131def007bcf776 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 19 Jan 2024 11:07:08 +0100 Subject: [PATCH 28/51] Treating the timer string so it becomes a dict. --- roles/debian/ansible/tasks/main.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 5ed23549e..a34b6d0fe 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -68,12 +68,13 @@ content: "export PATH=$PATH:{{ ce_ansible.venv_path | default(_venv_path) }}/bin" dest: "/etc/profile.d/ansible-path.sh" -- name: Build systemd timer variables. +- name: Build systemd timer variables string. ansible.builtin.set_fact: _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" -- debug: - msg: "{{ _timer }}" +- name: Turn the timer string into a dictionary. + ansible.builtin.set_fact: + _timer: "{{ _timer.splitlines()|map('from_yaml')|list }}" - name: Create systemd timer to upgrade Ansible. ansible.builtin.include_role: From 170c782caf1f483f28bacd8f5f5184cacf16a831 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 22 Jan 2024 12:22:45 +0100 Subject: [PATCH 29/51] Moving default log location for clamav. --- roles/debian/clamav/defaults/main.yml | 3 ++- roles/debian/clamav/templates/clamscan.j2 | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/debian/clamav/defaults/main.yml b/roles/debian/clamav/defaults/main.yml index 651cafde3..6185d71fa 100644 --- a/roles/debian/clamav/defaults/main.yml +++ b/roles/debian/clamav/defaults/main.yml @@ -4,7 +4,8 @@ clamav: server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. on_calendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events scripts_location: /usr/local/clamav/script - log_location: /usr/local/clamav/log + log_location: /var/log/clamav + log_name: clamav.log send_mail: false # Important: will not send any emails by default. send_on_fail: true # Only sends emails on scan failure, will not email for successful scans. report_recipient_email: mail@example.com diff --git a/roles/debian/clamav/templates/clamscan.j2 b/roles/debian/clamav/templates/clamscan.j2 index 0142a1b8b..c2b54e2cb 100644 --- a/roles/debian/clamav/templates/clamscan.j2 +++ b/roles/debian/clamav/templates/clamscan.j2 @@ -1,7 +1,7 @@ #!/bin/bash SUBJECT="{{ clamav.server_name }} PASSED SCAN" EMAIL="{{ clamav.report_recipient_email }}" -LOG={{ clamav.log_location }}/clamav.log +LOG={{ clamav.log_location }}/{{ clamav.log_name }} TMP_LOG=/tmp/clam.daily rm -rf ${TMP_LOG} From 21f882b8376ee1af0f77247ed9121648c62e7e6d Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 22 Jan 2024 12:23:47 +0100 Subject: [PATCH 30/51] Updating ClamAV docs. --- docs/roles/debian/clamav.md | 3 ++- roles/debian/clamav/README.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/roles/debian/clamav.md b/docs/roles/debian/clamav.md index 81a888707..8ce643b7e 100644 --- a/docs/roles/debian/clamav.md +++ b/docs/roles/debian/clamav.md @@ -15,7 +15,8 @@ clamav: server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. on_calendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events scripts_location: /usr/local/clamav/script - log_location: /usr/local/clamav/log + log_location: /var/log/clamav + log_name: clamav.log send_mail: false # Important: will not send any emails by default. send_on_fail: true # Only sends emails on scan failure, will not email for successful scans. report_recipient_email: mail@example.com diff --git a/roles/debian/clamav/README.md b/roles/debian/clamav/README.md index 81a888707..8ce643b7e 100644 --- a/roles/debian/clamav/README.md +++ b/roles/debian/clamav/README.md @@ -15,7 +15,8 @@ clamav: server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. on_calendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events scripts_location: /usr/local/clamav/script - log_location: /usr/local/clamav/log + log_location: /var/log/clamav + log_name: clamav.log send_mail: false # Important: will not send any emails by default. send_on_fail: true # Only sends emails on scan failure, will not email for successful scans. report_recipient_email: mail@example.com From a57c5ffa6862ec3daa56e3cca95ce639d7ee9b4e Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 25 Jan 2024 13:32:37 +0100 Subject: [PATCH 31/51] Grouping systemd timer tasks together. --- roles/debian/ansible/tasks/main.yml | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index b845f2567..f14197d28 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -68,17 +68,19 @@ content: "export PATH=$PATH:{{ ce_ansible.venv_path | default(_venv_path) }}/bin" dest: "/etc/profile.d/ansible-path.sh" -- name: Build systemd timer variables string. - ansible.builtin.set_fact: - _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" +- name: Install systemd timer. + when: ce_ansible.upgrade.enabled + block: + - name: Build systemd timer variables string. + ansible.builtin.set_fact: + _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" -- name: Turn the timer string into a dictionary. - ansible.builtin.set_fact: - _timer: "{{ _timer.splitlines()|map('from_yaml')|list }}" + - name: Turn the timer string into a dictionary. + ansible.builtin.set_fact: + _timer: "{{ _timer.splitlines()|map('from_yaml')|list }}" -- name: Create systemd timer to upgrade Ansible. - ansible.builtin.include_role: - name: contrib/systemd_timers - vars: - timers: "{{ _timer }}" - when: ce_ansible.upgrade.enabled + - name: Create systemd timer to upgrade Ansible. + ansible.builtin.include_role: + name: contrib/systemd_timers + vars: + timers: "{{ _timer }}" From 36a52eeb2cbea9e18d66a43b1d88e692c8dfd842 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 21 Feb 2024 16:47:33 +0100 Subject: [PATCH 32/51] Exposing ce-provision version in build output. --- roles/_init/defaults/main.yml | 1 + roles/_init/tasks/main.yml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 01249906a..7b7c06560 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -4,6 +4,7 @@ _init: # This is used to detect if the playbook must re-run or not. vars_dirs: [] force_play: false + ce_provision_version: 2.x # Outputted by the _init role at the start of plays. # Although these variables logically belong with ce_provision, the _init role needs to # gather the extra variables if there are any, so there are _init variables. diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index 39c368378..357f94c5e 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -1,4 +1,8 @@ --- +- name: Version check. + ansible.builtin.debug: + msg: "Using ce-provision {{ _init.ce_provision_version }}" + - name: Check OS family. when: ansible_os_family != "Debian" block: From a02e9e5fbade7ddf56ec3d8d73a3df062fb08e84 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 23 Feb 2024 18:24:49 +0100 Subject: [PATCH 33/51] Wrong variable in meta role for controller username. --- roles/_meta/common_base/meta/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 7a79993e8..70c522c53 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -44,12 +44,12 @@ dependencies: when: is_local is not defined or not is_local - role: debian/ansible ce_ansible: - venv_path: "/home/{{ ce_provision.username }}/ansible" + venv_path: "/home/{{ user_provision.username }}/ansible" venv_command: "/usr/bin/python3 -m venv" - install_username: "{{ ce_provision.username }}" + install_username: "{{ user_provision.username }}" upgrade: enabled: true - command: "/home/{{ ce_provision.username }}/ansible/bin/python3 -m pip install --upgrade ansible" + command: "/home/{{ user_provision.username }}/ansible/bin/python3 -m pip install --upgrade ansible" on_calendar: "*-*-* 01:30:00" timer_name: upgrade_ansible linters: From 206b36e1758dcb8f06195e8ac514e6bc67edccab Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 13 May 2024 17:17:17 +0200 Subject: [PATCH 34/51] Removing any reference to _aws variables in debian role defaults. --- docs/roles/_init.md | 5 +++++ docs/roles/debian/aws_cloudwatch_agent.md | 6 +++--- docs/roles/debian/aws_efs_client.md | 4 ++-- docs/roles/debian/duplicity.md | 2 +- docs/roles/debian/gitlab_runner.md | 4 ++-- roles/_init/README.md | 5 +++++ roles/_init/defaults/main.yml | 5 +++++ roles/debian/aws_cloudwatch_agent/README.md | 6 +++--- roles/debian/aws_cloudwatch_agent/defaults/main.yml | 6 +++--- roles/debian/aws_efs_client/README.md | 4 ++-- roles/debian/aws_efs_client/defaults/main.yml | 4 ++-- roles/debian/duplicity/README.md | 2 +- roles/debian/duplicity/defaults/main.yml | 2 +- roles/debian/gitlab_runner/README.md | 4 ++-- roles/debian/gitlab_runner/defaults/main.yml | 4 ++-- 15 files changed, 39 insertions(+), 24 deletions(-) diff --git a/docs/roles/_init.md b/docs/roles/_init.md index 77feacb1f..4abd58990 100644 --- a/docs/roles/_init.md +++ b/docs/roles/_init.md @@ -15,6 +15,11 @@ _venv_command: /usr/bin/python3 -m venv _venv_install_username: "{{ _ce_provision_username }}" _ce_ansible_timer_name: upgrade_ce_provision_ansible +# AWS variables - if you are using an AWS account, you can preset certain variables +# Generally it is recommended to place these in your ce-provision-config repository under hosts/group_vars/all +#_aws_profile: example # boto profile name +#_aws_region: eu-west-1 + _init: # A list of var directories to include. We only support .yml extensions. # This is used to detect if the playbook must re-run or not. diff --git a/docs/roles/debian/aws_cloudwatch_agent.md b/docs/roles/debian/aws_cloudwatch_agent.md index ee37572c7..cdda9f9b5 100644 --- a/docs/roles/debian/aws_cloudwatch_agent.md +++ b/docs/roles/debian/aws_cloudwatch_agent.md @@ -14,9 +14,9 @@ module in your playbooks. --- aws_cloudwatch_agent: # Stream name. - log_stream_name: "example" + log_stream_name: example # Namespace for metrics. Leave empty to use the default CWAgent. - metrics_namespace: "example" + metrics_namespace: example # Group prefix. Useful for grouping by environments. # Eg. instead of "syslog", you can have "dev syslog", "prod syslog", etc. log_group_prefix: "" @@ -26,7 +26,7 @@ aws_cloudwatch_agent: credentials: aws_access_key_id: XXX aws_secret_access_key: XXX - region: "{{ _aws_region }}" + region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set ``` diff --git a/docs/roles/debian/aws_efs_client.md b/docs/roles/debian/aws_efs_client.md index 6d3cd392d..856fe71ca 100644 --- a/docs/roles/debian/aws_efs_client.md +++ b/docs/roles/debian/aws_efs_client.md @@ -44,8 +44,8 @@ In the output of listed file systems, for each one you will find an entry like t _mount_opts: "_netdev,noresvport" # _netdev tells OS to wait for network before attempting to mount _mount_state: present aws_efs_client: - aws_profile: "{{ _aws_profile }}" - region: "{{ _aws_region }}" + aws_profile: example # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set + region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set version: 1.35.0 # version of AWS EFS utils to use build_suffix: "-1_all" # sometimes there is a suffix appended to the package name, e.g. `amazon-efs-utils-1.35.0-1_all.deb` deb_url: "" # provide an alternative location for the .deb package diff --git a/docs/roles/debian/duplicity.md b/docs/roles/debian/duplicity.md index 7e4911527..c9140007e 100644 --- a/docs/roles/debian/duplicity.md +++ b/docs/roles/debian/duplicity.md @@ -17,7 +17,7 @@ duplicity: backend: s3 # currently also support b2 for Backblaze access_key_id: "somekey" secret_access_key: "somesecret" - backend_url: "s3-eu-west-1.amazonaws.com" + backend_url: "s3-eu-west-1.amazonaws.com" # "eu-west-1" can be substituted for "{{ _aws_region }}" if set s3_options: "--s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options bucketname: "somebucket" dirs: diff --git a/docs/roles/debian/gitlab_runner.md b/docs/roles/debian/gitlab_runner.md index 2804320b5..7311c5bf0 100644 --- a/docs/roles/debian/gitlab_runner.md +++ b/docs/roles/debian/gitlab_runner.md @@ -49,8 +49,8 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name - profile: "{{ _aws_profile }}" - region: "eu-west-1" # AWS region name + profile: "example" # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set + region: "eu-west-1" # AWS region name - can be substituted for "{{ _aws_region }}" if set subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name task_definition: "my-task:1" # task definition in format name:revision, if revision is not provided ECS will use latest diff --git a/roles/_init/README.md b/roles/_init/README.md index 77feacb1f..4abd58990 100644 --- a/roles/_init/README.md +++ b/roles/_init/README.md @@ -15,6 +15,11 @@ _venv_command: /usr/bin/python3 -m venv _venv_install_username: "{{ _ce_provision_username }}" _ce_ansible_timer_name: upgrade_ce_provision_ansible +# AWS variables - if you are using an AWS account, you can preset certain variables +# Generally it is recommended to place these in your ce-provision-config repository under hosts/group_vars/all +#_aws_profile: example # boto profile name +#_aws_region: eu-west-1 + _init: # A list of var directories to include. We only support .yml extensions. # This is used to detect if the playbook must re-run or not. diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index d05a6fc78..5e38419ea 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -5,6 +5,11 @@ _venv_command: /usr/bin/python3 -m venv _venv_install_username: "{{ _ce_provision_username }}" _ce_ansible_timer_name: upgrade_ce_provision_ansible +# AWS variables - if you are using an AWS account, you can preset certain variables +# Generally it is recommended to place these in your ce-provision-config repository under hosts/group_vars/all +#_aws_profile: example # boto profile name +#_aws_region: eu-west-1 + _init: # A list of var directories to include. We only support .yml extensions. # This is used to detect if the playbook must re-run or not. diff --git a/roles/debian/aws_cloudwatch_agent/README.md b/roles/debian/aws_cloudwatch_agent/README.md index ee37572c7..cdda9f9b5 100644 --- a/roles/debian/aws_cloudwatch_agent/README.md +++ b/roles/debian/aws_cloudwatch_agent/README.md @@ -14,9 +14,9 @@ module in your playbooks. --- aws_cloudwatch_agent: # Stream name. - log_stream_name: "example" + log_stream_name: example # Namespace for metrics. Leave empty to use the default CWAgent. - metrics_namespace: "example" + metrics_namespace: example # Group prefix. Useful for grouping by environments. # Eg. instead of "syslog", you can have "dev syslog", "prod syslog", etc. log_group_prefix: "" @@ -26,7 +26,7 @@ aws_cloudwatch_agent: credentials: aws_access_key_id: XXX aws_secret_access_key: XXX - region: "{{ _aws_region }}" + region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set ``` diff --git a/roles/debian/aws_cloudwatch_agent/defaults/main.yml b/roles/debian/aws_cloudwatch_agent/defaults/main.yml index f73385114..36d3d3751 100644 --- a/roles/debian/aws_cloudwatch_agent/defaults/main.yml +++ b/roles/debian/aws_cloudwatch_agent/defaults/main.yml @@ -1,9 +1,9 @@ --- aws_cloudwatch_agent: # Stream name. - log_stream_name: "example" + log_stream_name: example # Namespace for metrics. Leave empty to use the default CWAgent. - metrics_namespace: "example" + metrics_namespace: example # Group prefix. Useful for grouping by environments. # Eg. instead of "syslog", you can have "dev syslog", "prod syslog", etc. log_group_prefix: "" @@ -13,4 +13,4 @@ aws_cloudwatch_agent: credentials: aws_access_key_id: XXX aws_secret_access_key: XXX - region: "{{ _aws_region }}" + region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set diff --git a/roles/debian/aws_efs_client/README.md b/roles/debian/aws_efs_client/README.md index 6d3cd392d..856fe71ca 100644 --- a/roles/debian/aws_efs_client/README.md +++ b/roles/debian/aws_efs_client/README.md @@ -44,8 +44,8 @@ In the output of listed file systems, for each one you will find an entry like t _mount_opts: "_netdev,noresvport" # _netdev tells OS to wait for network before attempting to mount _mount_state: present aws_efs_client: - aws_profile: "{{ _aws_profile }}" - region: "{{ _aws_region }}" + aws_profile: example # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set + region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set version: 1.35.0 # version of AWS EFS utils to use build_suffix: "-1_all" # sometimes there is a suffix appended to the package name, e.g. `amazon-efs-utils-1.35.0-1_all.deb` deb_url: "" # provide an alternative location for the .deb package diff --git a/roles/debian/aws_efs_client/defaults/main.yml b/roles/debian/aws_efs_client/defaults/main.yml index ba81a7629..7758d8820 100644 --- a/roles/debian/aws_efs_client/defaults/main.yml +++ b/roles/debian/aws_efs_client/defaults/main.yml @@ -2,8 +2,8 @@ _mount_opts: "_netdev,noresvport" # _netdev tells OS to wait for network before attempting to mount _mount_state: present aws_efs_client: - aws_profile: "{{ _aws_profile }}" - region: "{{ _aws_region }}" + aws_profile: example # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set + region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set version: 1.35.0 # version of AWS EFS utils to use build_suffix: "-1_all" # sometimes there is a suffix appended to the package name, e.g. `amazon-efs-utils-1.35.0-1_all.deb` deb_url: "" # provide an alternative location for the .deb package diff --git a/roles/debian/duplicity/README.md b/roles/debian/duplicity/README.md index 7e4911527..c9140007e 100644 --- a/roles/debian/duplicity/README.md +++ b/roles/debian/duplicity/README.md @@ -17,7 +17,7 @@ duplicity: backend: s3 # currently also support b2 for Backblaze access_key_id: "somekey" secret_access_key: "somesecret" - backend_url: "s3-eu-west-1.amazonaws.com" + backend_url: "s3-eu-west-1.amazonaws.com" # "eu-west-1" can be substituted for "{{ _aws_region }}" if set s3_options: "--s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options bucketname: "somebucket" dirs: diff --git a/roles/debian/duplicity/defaults/main.yml b/roles/debian/duplicity/defaults/main.yml index 6d7a6f3d6..33cecfec2 100644 --- a/roles/debian/duplicity/defaults/main.yml +++ b/roles/debian/duplicity/defaults/main.yml @@ -8,7 +8,7 @@ duplicity: backend: s3 # currently also support b2 for Backblaze access_key_id: "somekey" secret_access_key: "somesecret" - backend_url: "s3-eu-west-1.amazonaws.com" + backend_url: "s3-eu-west-1.amazonaws.com" # "eu-west-1" can be substituted for "{{ _aws_region }}" if set s3_options: "--s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options bucketname: "somebucket" dirs: diff --git a/roles/debian/gitlab_runner/README.md b/roles/debian/gitlab_runner/README.md index 2804320b5..7311c5bf0 100644 --- a/roles/debian/gitlab_runner/README.md +++ b/roles/debian/gitlab_runner/README.md @@ -49,8 +49,8 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name - profile: "{{ _aws_profile }}" - region: "eu-west-1" # AWS region name + profile: "example" # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set + region: "eu-west-1" # AWS region name - can be substituted for "{{ _aws_region }}" if set subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name task_definition: "my-task:1" # task definition in format name:revision, if revision is not provided ECS will use latest diff --git a/roles/debian/gitlab_runner/defaults/main.yml b/roles/debian/gitlab_runner/defaults/main.yml index af891d799..1813aa89b 100644 --- a/roles/debian/gitlab_runner/defaults/main.yml +++ b/roles/debian/gitlab_runner/defaults/main.yml @@ -37,8 +37,8 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name - profile: "{{ _aws_profile }}" - region: "eu-west-1" # AWS region name + profile: "example" # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set + region: "eu-west-1" # AWS region name - can be substituted for "{{ _aws_region }}" if set subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name task_definition: "my-task:1" # task definition in format name:revision, if revision is not provided ECS will use latest From 8ee9e69dba20735d34cba2052df4d4fa22a112dd Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 14 May 2024 16:27:06 +0200 Subject: [PATCH 35/51] Setting more sane ASG defaults. --- docs/roles/aws/aws_ec2_autoscale_cluster.md | 12 ++++++------ roles/aws/aws_ec2_autoscale_cluster/README.md | 12 ++++++------ .../aws/aws_ec2_autoscale_cluster/defaults/main.yml | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/roles/aws/aws_ec2_autoscale_cluster.md b/docs/roles/aws/aws_ec2_autoscale_cluster.md index 6278a17d0..ebda2b7a0 100644 --- a/docs/roles/aws/aws_ec2_autoscale_cluster.md +++ b/docs/roles/aws/aws_ec2_autoscale_cluster.md @@ -106,15 +106,15 @@ aws_ec2_autoscale_cluster: - name: "{{ _env_type }}-scale-up-policy" policy_type: "SimpleScaling" adjustment_type: "ChangeInCapacity" - adjustment: 2 + adjustment: 2 # Add two servers per scaling event adjustment_step: 1 # Only used when adjustment_type is PercentChangeInCapacity. - cooldown: 300 + cooldown: 120 - name: "{{ _env_type }}-scale-down-policy" policy_type: "SimpleScaling" adjustment_type: "ChangeInCapacity" - adjustment: -2 + adjustment: -1 # Reduce by one server at a time adjustment_step: -1 # Only used when adjustment_type is PercentChangeInCapacity. - cooldown: 300 + cooldown: 120 asg_cloudwatch_alarm_scale_up_name: "{{ _env_type }}-cloudwatch-metric-alarm-cpu-scale-up" asg_cloudwatch_alarm_scale_down_name: "{{ _env_type }}-cloudwatch-metric-alarm-cpu-scale-down" asg_cloudwatch_alarms: @@ -126,8 +126,8 @@ aws_ec2_autoscale_cluster: threshold: 80 unit: "Percent" comparison: "GreaterThanOrEqualToThreshold" - period: 120 - evaluation_periods: 5 + period: 30 + evaluation_periods: 3 - scale_direction: "down" description: "CPU under 40% so scale down." metric: "CPUUtilization" diff --git a/roles/aws/aws_ec2_autoscale_cluster/README.md b/roles/aws/aws_ec2_autoscale_cluster/README.md index 6278a17d0..ebda2b7a0 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/README.md +++ b/roles/aws/aws_ec2_autoscale_cluster/README.md @@ -106,15 +106,15 @@ aws_ec2_autoscale_cluster: - name: "{{ _env_type }}-scale-up-policy" policy_type: "SimpleScaling" adjustment_type: "ChangeInCapacity" - adjustment: 2 + adjustment: 2 # Add two servers per scaling event adjustment_step: 1 # Only used when adjustment_type is PercentChangeInCapacity. - cooldown: 300 + cooldown: 120 - name: "{{ _env_type }}-scale-down-policy" policy_type: "SimpleScaling" adjustment_type: "ChangeInCapacity" - adjustment: -2 + adjustment: -1 # Reduce by one server at a time adjustment_step: -1 # Only used when adjustment_type is PercentChangeInCapacity. - cooldown: 300 + cooldown: 120 asg_cloudwatch_alarm_scale_up_name: "{{ _env_type }}-cloudwatch-metric-alarm-cpu-scale-up" asg_cloudwatch_alarm_scale_down_name: "{{ _env_type }}-cloudwatch-metric-alarm-cpu-scale-down" asg_cloudwatch_alarms: @@ -126,8 +126,8 @@ aws_ec2_autoscale_cluster: threshold: 80 unit: "Percent" comparison: "GreaterThanOrEqualToThreshold" - period: 120 - evaluation_periods: 5 + period: 30 + evaluation_periods: 3 - scale_direction: "down" description: "CPU under 40% so scale down." metric: "CPUUtilization" diff --git a/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml b/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml index 03e99461c..b6aedd0b2 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml +++ b/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml @@ -87,15 +87,15 @@ aws_ec2_autoscale_cluster: - name: "{{ _env_type }}-scale-up-policy" policy_type: "SimpleScaling" adjustment_type: "ChangeInCapacity" - adjustment: 2 + adjustment: 2 # Add two servers per scaling event adjustment_step: 1 # Only used when adjustment_type is PercentChangeInCapacity. - cooldown: 300 + cooldown: 120 - name: "{{ _env_type }}-scale-down-policy" policy_type: "SimpleScaling" adjustment_type: "ChangeInCapacity" - adjustment: -2 + adjustment: -1 # Reduce by one server at a time adjustment_step: -1 # Only used when adjustment_type is PercentChangeInCapacity. - cooldown: 300 + cooldown: 120 asg_cloudwatch_alarm_scale_up_name: "{{ _env_type }}-cloudwatch-metric-alarm-cpu-scale-up" asg_cloudwatch_alarm_scale_down_name: "{{ _env_type }}-cloudwatch-metric-alarm-cpu-scale-down" asg_cloudwatch_alarms: @@ -107,8 +107,8 @@ aws_ec2_autoscale_cluster: threshold: 80 unit: "Percent" comparison: "GreaterThanOrEqualToThreshold" - period: 120 - evaluation_periods: 5 + period: 30 + evaluation_periods: 3 - scale_direction: "down" description: "CPU under 40% so scale down." metric: "CPUUtilization" From aff0054631dc3a67aeaf6f1fa175fc1bafe633cf Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 14 May 2024 17:14:18 +0200 Subject: [PATCH 36/51] Making ClamAV timers a list so they can be entirely replaced. --- docs/roles/debian/clamav.md | 8 ++++---- roles/debian/clamav/README.md | 8 ++++---- roles/debian/clamav/defaults/main.yml | 8 ++++---- roles/debian/clamav/tasks/main.yml | 8 +++++++- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/roles/debian/clamav.md b/docs/roles/debian/clamav.md index 4529ab043..0a35265c2 100644 --- a/docs/roles/debian/clamav.md +++ b/docs/roles/debian/clamav.md @@ -19,11 +19,11 @@ clamav: - /sys/ scan_location: / log_name: clamav_daily.log - # scheduled scans, set to an empty dictionary for no timers + # scheduled scans, set to an empty list for no timers timers: - clamscan_daily: - timer_command: /usr/local/clamav/script/clamscan_daily # path to clamscan wrapper script, ensure it is defined in clamav.scripts - timer_OnCalendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + - clamscan_daily: + timer_command: /usr/local/clamav/script/clamscan_daily # path to clamscan wrapper script, ensure it is defined in clamav.scripts + timer_OnCalendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. log_location: /var/log/clamav send_mail: false # Important: will not send any emails by default. diff --git a/roles/debian/clamav/README.md b/roles/debian/clamav/README.md index 4529ab043..0a35265c2 100644 --- a/roles/debian/clamav/README.md +++ b/roles/debian/clamav/README.md @@ -19,11 +19,11 @@ clamav: - /sys/ scan_location: / log_name: clamav_daily.log - # scheduled scans, set to an empty dictionary for no timers + # scheduled scans, set to an empty list for no timers timers: - clamscan_daily: - timer_command: /usr/local/clamav/script/clamscan_daily # path to clamscan wrapper script, ensure it is defined in clamav.scripts - timer_OnCalendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + - clamscan_daily: + timer_command: /usr/local/clamav/script/clamscan_daily # path to clamscan wrapper script, ensure it is defined in clamav.scripts + timer_OnCalendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. log_location: /var/log/clamav send_mail: false # Important: will not send any emails by default. diff --git a/roles/debian/clamav/defaults/main.yml b/roles/debian/clamav/defaults/main.yml index 648ddd934..85af99d42 100644 --- a/roles/debian/clamav/defaults/main.yml +++ b/roles/debian/clamav/defaults/main.yml @@ -8,11 +8,11 @@ clamav: - /sys/ scan_location: / log_name: clamav_daily.log - # scheduled scans, set to an empty dictionary for no timers + # scheduled scans, set to an empty list for no timers timers: - clamscan_daily: - timer_command: /usr/local/clamav/script/clamscan_daily # path to clamscan wrapper script, ensure it is defined in clamav.scripts - timer_OnCalendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + - clamscan_daily: + timer_command: /usr/local/clamav/script/clamscan_daily # path to clamscan wrapper script, ensure it is defined in clamav.scripts + timer_OnCalendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. log_location: /var/log/clamav send_mail: false # Important: will not send any emails by default. diff --git a/roles/debian/clamav/tasks/main.yml b/roles/debian/clamav/tasks/main.yml index ebf032e74..02e643781 100644 --- a/roles/debian/clamav/tasks/main.yml +++ b/roles/debian/clamav/tasks/main.yml @@ -23,11 +23,17 @@ state: directory mode: '0755' +- name: Build a dictionary of timers for the systemd timers role. + ansible.builtin.set_fact: + _clamav_timers: "{{ _clamav_timers | default({}) | combine( item ) }}" + with_items: "{{ clamav.timers }}" + when: clamav.timers | length > 0 + - name: Set up a ClamAV systemd timers. ansible.builtin.include_role: name: contrib/systemd_timers vars: - timers: "{{ clamav.timers }}" + timers: "{{ _clamav_timers }}" when: clamav.timers | length > 0 - name: Install clamdscan. From cb4f6eb175eef4578d52b092fa37ebcd3f367f75 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 16 May 2024 17:32:27 +0200 Subject: [PATCH 37/51] Spacing fix for linting. --- roles/debian/clamav/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/debian/clamav/tasks/main.yml b/roles/debian/clamav/tasks/main.yml index 02e643781..eeaf31013 100644 --- a/roles/debian/clamav/tasks/main.yml +++ b/roles/debian/clamav/tasks/main.yml @@ -25,7 +25,7 @@ - name: Build a dictionary of timers for the systemd timers role. ansible.builtin.set_fact: - _clamav_timers: "{{ _clamav_timers | default({}) | combine( item ) }}" + _clamav_timers: "{{ _clamav_timers | default({}) | combine(item) }}" with_items: "{{ clamav.timers }}" when: clamav.timers | length > 0 From ad2038555530a783d2c00f112cfe2ad2d1d85adf Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 22 May 2024 13:13:56 +0200 Subject: [PATCH 38/51] Renaming npm module. --- roles/debian/lhci/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/debian/lhci/tasks/main.yml b/roles/debian/lhci/tasks/main.yml index a2a24d7f8..6c7a3a038 100644 --- a/roles/debian/lhci/tasks/main.yml +++ b/roles/debian/lhci/tasks/main.yml @@ -52,7 +52,7 @@ state: present - name: Install "lhci/cli" node.js package. - community.general.packaging.language.npm: + community.general.npm: name: "@lhci/cli" version: "{{ lhci.cli_version | default(omit) }}" state: "{{ lhci.cli_state | default(omit) }}" @@ -60,7 +60,7 @@ global: true - name: Install "lighthouse" node.js package. - community.general.packaging.language.npm: + community.general.npm: name: "lighthouse" version: "{{ lhci.lighthouse_version | default(omit) }}" state: "{{ lhci.lighthouse_state | default(omit) }}" From cf6d60c9643a25881395dd195b41aa356c35b992 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 27 May 2024 16:23:37 +0200 Subject: [PATCH 39/51] Removing NGINX installation as part of phpMyAdmin role by default. --- docs/roles/debian/phpmyadmin.md | 6 ++++-- roles/debian/phpmyadmin/README.md | 6 ++++-- roles/debian/phpmyadmin/defaults/main.yml | 3 ++- roles/debian/phpmyadmin/meta/main.yml | 1 - roles/debian/phpmyadmin/tasks/main.yml | 5 +++++ 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/roles/debian/phpmyadmin.md b/docs/roles/debian/phpmyadmin.md index 3c329c981..7eeb9bcca 100644 --- a/docs/roles/debian/phpmyadmin.md +++ b/docs/roles/debian/phpmyadmin.md @@ -1,7 +1,7 @@ # phpMyAdmin -This role only installs phpMyAdmin ready for configuration, it does not do any active configuration, nor does it create the necessary NGINX vhost. It will install `debian/php-fpm` and `debian/nginx` so do review their variables and set what you need prior to running a first build with `phpmyadmin`. +This role only installs phpMyAdmin ready for configuration, it does not do any active configuration, nor does it create the necessary NGINX vhost. It will install `debian/php-fpm` so do review the variables for PHP and set what you need prior to running a first build with `phpmyadmin`. -Here is an example NGINX vhost config you can copy to your `nginx.yml` file and adjust as required: +Similarly, it can optionally install `debian/nginx` if you set `phpmyadmin.install_nginx: true`, so if you do that be sure to provide a sensible NGINX config. Here is an example NGINX vhost config you can copy to your `nginx.yml` file and adjust as required: ```yaml nginx: @@ -54,6 +54,8 @@ phpmyadmin: # Assuming user and group should match php-fpm by default www_user: "{{ php.fpm.pool_user }}" www_group: "{{ php.fpm.pool_group }}" + install_nginx: false + ``` diff --git a/roles/debian/phpmyadmin/README.md b/roles/debian/phpmyadmin/README.md index 3c329c981..7eeb9bcca 100644 --- a/roles/debian/phpmyadmin/README.md +++ b/roles/debian/phpmyadmin/README.md @@ -1,7 +1,7 @@ # phpMyAdmin -This role only installs phpMyAdmin ready for configuration, it does not do any active configuration, nor does it create the necessary NGINX vhost. It will install `debian/php-fpm` and `debian/nginx` so do review their variables and set what you need prior to running a first build with `phpmyadmin`. +This role only installs phpMyAdmin ready for configuration, it does not do any active configuration, nor does it create the necessary NGINX vhost. It will install `debian/php-fpm` so do review the variables for PHP and set what you need prior to running a first build with `phpmyadmin`. -Here is an example NGINX vhost config you can copy to your `nginx.yml` file and adjust as required: +Similarly, it can optionally install `debian/nginx` if you set `phpmyadmin.install_nginx: true`, so if you do that be sure to provide a sensible NGINX config. Here is an example NGINX vhost config you can copy to your `nginx.yml` file and adjust as required: ```yaml nginx: @@ -54,6 +54,8 @@ phpmyadmin: # Assuming user and group should match php-fpm by default www_user: "{{ php.fpm.pool_user }}" www_group: "{{ php.fpm.pool_group }}" + install_nginx: false + ``` diff --git a/roles/debian/phpmyadmin/defaults/main.yml b/roles/debian/phpmyadmin/defaults/main.yml index 00960891a..24fbc46b3 100644 --- a/roles/debian/phpmyadmin/defaults/main.yml +++ b/roles/debian/phpmyadmin/defaults/main.yml @@ -5,4 +5,5 @@ phpmyadmin: install_path: "/home/{{ ce_deploy.username }}/deploy" # Assuming user and group should match php-fpm by default www_user: "{{ php.fpm.pool_user }}" - www_group: "{{ php.fpm.pool_group }}" \ No newline at end of file + www_group: "{{ php.fpm.pool_group }}" + install_nginx: false diff --git a/roles/debian/phpmyadmin/meta/main.yml b/roles/debian/phpmyadmin/meta/main.yml index 54ffd52ed..d0cf08b3d 100644 --- a/roles/debian/phpmyadmin/meta/main.yml +++ b/roles/debian/phpmyadmin/meta/main.yml @@ -1,4 +1,3 @@ --- dependencies: - - role: debian/nginx - role: debian/php-fpm diff --git a/roles/debian/phpmyadmin/tasks/main.yml b/roles/debian/phpmyadmin/tasks/main.yml index 474e867fb..b3fac7e2d 100644 --- a/roles/debian/phpmyadmin/tasks/main.yml +++ b/roles/debian/phpmyadmin/tasks/main.yml @@ -17,3 +17,8 @@ ansible.builtin.command: cmd: "/usr/local/bin/phpmyadmin.sh -u -v {{ phpmyadmin.version }}" when: phpmyadmin.method == 'upgrade' + +- name: Install the NGINX web server. + ansible.builtin.include_role: + name: debian/nginx + when: phpmyadmin.install_nginx From 3de5bf65348c1a82bfd4691bf7dea59f883b381e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 27 May 2024 16:32:10 +0200 Subject: [PATCH 40/51] Fixing Varnish handler names. --- roles/debian/varnish_config/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/debian/varnish_config/tasks/main.yml b/roles/debian/varnish_config/tasks/main.yml index 362d33824..ad6d2836e 100644 --- a/roles/debian/varnish_config/tasks/main.yml +++ b/roles/debian/varnish_config/tasks/main.yml @@ -11,5 +11,5 @@ - "{{ _ce_provision_base_dir }}/config/files/templates/{{ varnish_config.template_filename }}.j2" - "default.vcl.j2" notify: - - Reload systemd - - Restart varnish + - reload systemd # handlers in https://github.com/geerlingguy/ansible-role-varnish/blob/master/handlers/main.yml + - restart varnish From cda46421faca0782333a786a2f1c1f5be4a55c1d Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 27 May 2024 16:40:13 +0200 Subject: [PATCH 41/51] Excluding name[casing] rule from linting due to false positives. --- .ansible-lint | 1 + 1 file changed, 1 insertion(+) diff --git a/.ansible-lint b/.ansible-lint index 71d7648cc..a68f44f4c 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -14,4 +14,5 @@ skip_list: - template-instead-of-copy # to skip over roles/ssl/tasks/copy.yml errors, temporarily. - name[template] # it doesn't like Jinja templates being in the middle of a task name, which seems silly to me. exclude_paths: + - name[casing] # sometimes included Galaxy roles break linting rules and cause failures - roles/debian/wazuh/ # imported role uses yes/no instead of true/false so always fails From 3a25d9b27d87e62e82f00db3b20267bf2f339e79 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 27 May 2024 16:42:12 +0200 Subject: [PATCH 42/51] Put rule in wrong place! --- .ansible-lint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ansible-lint b/.ansible-lint index a68f44f4c..390760f80 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -13,6 +13,6 @@ skip_list: - fqcn-builtins - template-instead-of-copy # to skip over roles/ssl/tasks/copy.yml errors, temporarily. - name[template] # it doesn't like Jinja templates being in the middle of a task name, which seems silly to me. -exclude_paths: - name[casing] # sometimes included Galaxy roles break linting rules and cause failures +exclude_paths: - roles/debian/wazuh/ # imported role uses yes/no instead of true/false so always fails From 31b7e90279011a85927ee33e7d3cb249f6196430 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 29 May 2024 10:52:07 +0200 Subject: [PATCH 43/51] Removing lock file behaviour from ASGs as it cannot work unless controller and ASG are in the same VPC. --- .../aws_ec2_autoscale_cluster/tasks/main.yml | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml b/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml index e6abb61fd..77b0e0a11 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml +++ b/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml @@ -1,30 +1,4 @@ --- -# With ASGs the lock behaviour in _init will not work, we need to do it here. -- name: Check for a ce-deploy lock file. - ansible.builtin.stat: - path: "{{ _init.deploy_lock_file }}" - register: _ce_deploy_lock - connection: ssh - run_once: true - when: aws_ec2_autoscale_cluster.type == "ec2" - -- name: Abort if ce-deploy lock file exists. - when: - - _ce_deploy_lock.stat.exists is defined and _ce_deploy_lock.stat.exists - block: - - name: Abort if ce-deploy lock file is found. - ansible.builtin.debug: - msg: "ce-deploy lock file discovered, a deployment is in progress! If this is not the case, login to the affected server and delete the file at {{ _init.deploy_lock_file }}." - - ansible.builtin.meta: end_play - -- name: Set a lock file on each ASG machine. - ansible.builtin.file: - path: "{{ _init.lock_file }}" - state: touch - mode: 0644 - connection: ssh - when: aws_ec2_autoscale_cluster.type == "ec2" - # @todo Do we still need that? Used by AMI and LC, # but probably could simply use the base name. - name: Define unique name for resources. @@ -782,13 +756,3 @@ when: - aws_ec2_autoscale_cluster.route_53.zone is defined - aws_ec2_autoscale_cluster.route_53.zone | length > 0 - -# We only need to remove the lock file if the machines will not be replaced. -- name: Remove the lock file on each ASG machine. - ansible.builtin.file: - path: "{{ _init.lock_file }}" - state: absent - connection: ssh - when: - - aws_ec2_autoscale_cluster.type == "ec2" - - not aws_ec2_autoscale_cluster.asg_refresh From 34d295d2f64f9cdda257a6af20fe45ef8bdaa73e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 29 May 2024 11:04:05 +0200 Subject: [PATCH 44/51] Capturing lock file limitations in comment. --- roles/_init/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index 485fdfa5f..2a7e3add7 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -3,7 +3,8 @@ ansible.builtin.debug: msg: "Using ce-provision {{ _init.ce_provision_version }}" -# See roles/aws/aws_ec2_autoscale_cluster for lock handling in ASGs. +# @TODO This cannot work for ASGs. +# We will need something central for lock files and it cannot be controller servers because they may be containers. - name: Check for a ce-deploy lock file. ansible.builtin.stat: path: "{{ _init.deploy_lock_file }}" From af108d735926dc32cf425ee313ee6164ab08bcad Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 31 May 2024 10:49:11 +0200 Subject: [PATCH 45/51] Updating documentation for LE. --- docs/roles/debian/apache.md | 4 ++-- docs/roles/debian/ldap_server.md | 4 ++-- docs/roles/debian/mailpit.md | 4 ++-- docs/roles/debian/nginx.md | 4 ++-- docs/roles/debian/phpmyadmin.md | 4 ++-- docs/roles/debian/ssl.md | 13 ++++++++++--- 6 files changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/roles/debian/apache.md b/docs/roles/debian/apache.md index 3902bfdab..d1da36405 100644 --- a/docs/roles/debian/apache.md +++ b/docs/roles/debian/apache.md @@ -62,8 +62,8 @@ apache: # services: # - apache2 # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - apache2 diff --git a/docs/roles/debian/ldap_server.md b/docs/roles/debian/ldap_server.md index 62421b9be..d3a234d4e 100644 --- a/docs/roles/debian/ldap_server.md +++ b/docs/roles/debian/ldap_server.md @@ -105,8 +105,8 @@ ldap_server: # email: sysadm@codeenigma.com # services: [] # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - slapd diff --git a/docs/roles/debian/mailpit.md b/docs/roles/debian/mailpit.md index 5431d0cf2..2cd4b5157 100644 --- a/docs/roles/debian/mailpit.md +++ b/docs/roles/debian/mailpit.md @@ -51,8 +51,8 @@ mailpit: #services: # - nginx #web_server: standalone - #certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - #certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + #certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + #certbot_renew_command: "certonly --agree-tos --force-renew" #reload_command: restart #reload: # - mailpit diff --git a/docs/roles/debian/nginx.md b/docs/roles/debian/nginx.md index 3b63015da..2f7eade5d 100644 --- a/docs/roles/debian/nginx.md +++ b/docs/roles/debian/nginx.md @@ -91,8 +91,8 @@ nginx: # email: sysadm@codeenigma.com # services: [] # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - nginx diff --git a/docs/roles/debian/phpmyadmin.md b/docs/roles/debian/phpmyadmin.md index 7eeb9bcca..54f24414d 100644 --- a/docs/roles/debian/phpmyadmin.md +++ b/docs/roles/debian/phpmyadmin.md @@ -21,8 +21,8 @@ nginx: email: sysadm@codeenigma.com services: [] web_server: standalone - certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + certbot_renew_command: "certonly --agree-tos --force-renew" reload_command: reload reload: - nginx diff --git a/docs/roles/debian/ssl.md b/docs/roles/debian/ssl.md index ecf91fc72..d9743ca77 100644 --- a/docs/roles/debian/ssl.md +++ b/docs/roles/debian/ssl.md @@ -36,11 +36,18 @@ nginx: autorenew: true email: administrator@example.com services: [] - certbot_register_command: "/usr/bin/certbot certonly --standalone --agree-tos --preferred-challenges http -n" - certbot_renew_command: "/usr/bin/certbot certonly --standalone --agree-tos --force-renew" + web_server: standalone + certbot_register_command: "certonly --standalone --agree-tos --preferred-challenges http -n" + certbot_renew_command: "certonly --standalone --agree-tos --force-renew" + reload_command: reload + reload: + - nginx + renewal_minute: "0" # minute to run renewal cron job + renewal_hour: "0" # hour to run renewal cron job + renewal_weekday: "0" # day of week to run renewal ``` -You need to include *all* variables required by the `letsencrypt` SSL handler because defaults will not load from the `ssl` role in this context. +As in the example above, you need to include *all* variables required by the `letsencrypt` SSL handler because defaults will not load from the `ssl` role in this context. If you are using Nginx or Apache you can set the `ssl.web_server` for each domain to either `nginx` or `apache` to have the necessary plugin installed for `certbot` to do automatic handling of LetsEncrypt requests. Be aware, it does this by temporarily altering your web server config and reloading - use this option at your own risk. This is *not* intended to be used with but *instead of* `ssl.http_01_port`. From b14da199adfa3066be4b6e3ea453493875dad1a9 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 31 May 2024 10:49:52 +0200 Subject: [PATCH 46/51] Using pip to install certbot plugins. --- roles/debian/ssl/tasks/letsencrypt.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/roles/debian/ssl/tasks/letsencrypt.yml b/roles/debian/ssl/tasks/letsencrypt.yml index 90e469949..8d3f26634 100644 --- a/roles/debian/ssl/tasks/letsencrypt.yml +++ b/roles/debian/ssl/tasks/letsencrypt.yml @@ -53,11 +53,13 @@ # @todo - add support for Route 53 DNS plugin - name: "Ensure certbot-{{ _ssl_web_server }} plugin is installed." - ansible.builtin.apt: - pkg: "python3-certbot-{{ _ssl_web_server }}" - state: present - when: - - _ssl_web_server != 'standalone' + ansible.builtin.include_role: + name: debian/python_pip_packages + vars: + python_pip_packages: + packages: + - name: "certbot-{{ _ssl_web_server }}" + when: _ssl_web_server != "standalone" # there is no certbot-standalone package in PyPi - name: Define SSL base path. ansible.builtin.set_fact: From 0620270834a0193614877e0555bbc9be593ac839 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 31 May 2024 10:50:04 +0200 Subject: [PATCH 47/51] Updating README docs. --- roles/debian/apache/README.md | 4 ++-- roles/debian/apache/defaults/main.yml | 4 ++-- roles/debian/ldap_server/README.md | 4 ++-- roles/debian/ldap_server/defaults/main.yml | 4 ++-- roles/debian/mailpit/README.md | 4 ++-- roles/debian/mailpit/defaults/main.yml | 4 ++-- roles/debian/nginx/README.md | 4 ++-- roles/debian/nginx/defaults/main.yml | 4 ++-- roles/debian/phpmyadmin/README.md | 4 ++-- roles/debian/ssl/README.md | 13 ++++++++++--- 10 files changed, 28 insertions(+), 21 deletions(-) diff --git a/roles/debian/apache/README.md b/roles/debian/apache/README.md index 3902bfdab..d1da36405 100644 --- a/roles/debian/apache/README.md +++ b/roles/debian/apache/README.md @@ -62,8 +62,8 @@ apache: # services: # - apache2 # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - apache2 diff --git a/roles/debian/apache/defaults/main.yml b/roles/debian/apache/defaults/main.yml index b6705c0e4..c76ddc0b0 100644 --- a/roles/debian/apache/defaults/main.yml +++ b/roles/debian/apache/defaults/main.yml @@ -52,8 +52,8 @@ apache: # services: # - apache2 # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - apache2 diff --git a/roles/debian/ldap_server/README.md b/roles/debian/ldap_server/README.md index 62421b9be..d3a234d4e 100644 --- a/roles/debian/ldap_server/README.md +++ b/roles/debian/ldap_server/README.md @@ -105,8 +105,8 @@ ldap_server: # email: sysadm@codeenigma.com # services: [] # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - slapd diff --git a/roles/debian/ldap_server/defaults/main.yml b/roles/debian/ldap_server/defaults/main.yml index 6924e7741..93956ffc5 100644 --- a/roles/debian/ldap_server/defaults/main.yml +++ b/roles/debian/ldap_server/defaults/main.yml @@ -39,8 +39,8 @@ ldap_server: # email: sysadm@codeenigma.com # services: [] # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - slapd diff --git a/roles/debian/mailpit/README.md b/roles/debian/mailpit/README.md index 5431d0cf2..2cd4b5157 100644 --- a/roles/debian/mailpit/README.md +++ b/roles/debian/mailpit/README.md @@ -51,8 +51,8 @@ mailpit: #services: # - nginx #web_server: standalone - #certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - #certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + #certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + #certbot_renew_command: "certonly --agree-tos --force-renew" #reload_command: restart #reload: # - mailpit diff --git a/roles/debian/mailpit/defaults/main.yml b/roles/debian/mailpit/defaults/main.yml index 5a0666593..a883589f8 100644 --- a/roles/debian/mailpit/defaults/main.yml +++ b/roles/debian/mailpit/defaults/main.yml @@ -34,8 +34,8 @@ mailpit: #services: # - nginx #web_server: standalone - #certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - #certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + #certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + #certbot_renew_command: "certonly --agree-tos --force-renew" #reload_command: restart #reload: # - mailpit diff --git a/roles/debian/nginx/README.md b/roles/debian/nginx/README.md index 3b63015da..2f7eade5d 100644 --- a/roles/debian/nginx/README.md +++ b/roles/debian/nginx/README.md @@ -91,8 +91,8 @@ nginx: # email: sysadm@codeenigma.com # services: [] # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - nginx diff --git a/roles/debian/nginx/defaults/main.yml b/roles/debian/nginx/defaults/main.yml index 5f6e7c7fe..5f66e981f 100644 --- a/roles/debian/nginx/defaults/main.yml +++ b/roles/debian/nginx/defaults/main.yml @@ -79,8 +79,8 @@ nginx: # email: sysadm@codeenigma.com # services: [] # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - nginx diff --git a/roles/debian/phpmyadmin/README.md b/roles/debian/phpmyadmin/README.md index 7eeb9bcca..54f24414d 100644 --- a/roles/debian/phpmyadmin/README.md +++ b/roles/debian/phpmyadmin/README.md @@ -21,8 +21,8 @@ nginx: email: sysadm@codeenigma.com services: [] web_server: standalone - certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + certbot_renew_command: "certonly --agree-tos --force-renew" reload_command: reload reload: - nginx diff --git a/roles/debian/ssl/README.md b/roles/debian/ssl/README.md index ecf91fc72..d9743ca77 100644 --- a/roles/debian/ssl/README.md +++ b/roles/debian/ssl/README.md @@ -36,11 +36,18 @@ nginx: autorenew: true email: administrator@example.com services: [] - certbot_register_command: "/usr/bin/certbot certonly --standalone --agree-tos --preferred-challenges http -n" - certbot_renew_command: "/usr/bin/certbot certonly --standalone --agree-tos --force-renew" + web_server: standalone + certbot_register_command: "certonly --standalone --agree-tos --preferred-challenges http -n" + certbot_renew_command: "certonly --standalone --agree-tos --force-renew" + reload_command: reload + reload: + - nginx + renewal_minute: "0" # minute to run renewal cron job + renewal_hour: "0" # hour to run renewal cron job + renewal_weekday: "0" # day of week to run renewal ``` -You need to include *all* variables required by the `letsencrypt` SSL handler because defaults will not load from the `ssl` role in this context. +As in the example above, you need to include *all* variables required by the `letsencrypt` SSL handler because defaults will not load from the `ssl` role in this context. If you are using Nginx or Apache you can set the `ssl.web_server` for each domain to either `nginx` or `apache` to have the necessary plugin installed for `certbot` to do automatic handling of LetsEncrypt requests. Be aware, it does this by temporarily altering your web server config and reloading - use this option at your own risk. This is *not* intended to be used with but *instead of* `ssl.http_01_port`. From ebda10c935cb0f0c1476edc25890d15f6a25702c Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 31 May 2024 11:00:03 +0200 Subject: [PATCH 48/51] Docs error corrected. --- docs/roles/debian/phpmyadmin.md | 4 +--- docs/roles/debian/ssl.md | 6 ++---- roles/debian/phpmyadmin/README.md | 4 +--- roles/debian/ssl/README.md | 6 ++---- roles/debian/ssl/defaults/main.yml | 2 +- 5 files changed, 7 insertions(+), 15 deletions(-) diff --git a/docs/roles/debian/phpmyadmin.md b/docs/roles/debian/phpmyadmin.md index 54f24414d..4f83dd3ce 100644 --- a/docs/roles/debian/phpmyadmin.md +++ b/docs/roles/debian/phpmyadmin.md @@ -26,9 +26,7 @@ nginx: reload_command: reload reload: - nginx - renewal_minute: "0" # minute to run renewal cron job - renewal_hour: "0" # hour to run renewal cron job - renewal_weekday: "0" # day of week to run renewal + on_calendar: "Mon *-*-* 04:00:00" ratelimitingcrawlers: true is_default: false servers: diff --git a/docs/roles/debian/ssl.md b/docs/roles/debian/ssl.md index d9743ca77..2ead29797 100644 --- a/docs/roles/debian/ssl.md +++ b/docs/roles/debian/ssl.md @@ -42,9 +42,7 @@ nginx: reload_command: reload reload: - nginx - renewal_minute: "0" # minute to run renewal cron job - renewal_hour: "0" # hour to run renewal cron job - renewal_weekday: "0" # day of week to run renewal + on_calendar: "Mon *-*-* 04:00:00" ``` As in the example above, you need to include *all* variables required by the `letsencrypt` SSL handler because defaults will not load from the `ssl` role in this context. @@ -91,7 +89,7 @@ ssl: autorenew: false # set to true to create a systemd timer to renew LE certs certbot_renew_command: "certonly --agree-tos --force-renew" # root of the command used in the systemd timer # See systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events - #on_calendar: "Mon *-*-* 04:00:00" + on_calendar: "Mon *-*-* 04:00:00" web_server: standalone # values are standalone, nginx or apache - warning, nginx and apache will attempt to manipulate your vhosts! # For "letsencrypt" handling, a list of service to stop while creating the certificate. diff --git a/roles/debian/phpmyadmin/README.md b/roles/debian/phpmyadmin/README.md index 54f24414d..4f83dd3ce 100644 --- a/roles/debian/phpmyadmin/README.md +++ b/roles/debian/phpmyadmin/README.md @@ -26,9 +26,7 @@ nginx: reload_command: reload reload: - nginx - renewal_minute: "0" # minute to run renewal cron job - renewal_hour: "0" # hour to run renewal cron job - renewal_weekday: "0" # day of week to run renewal + on_calendar: "Mon *-*-* 04:00:00" ratelimitingcrawlers: true is_default: false servers: diff --git a/roles/debian/ssl/README.md b/roles/debian/ssl/README.md index d9743ca77..2ead29797 100644 --- a/roles/debian/ssl/README.md +++ b/roles/debian/ssl/README.md @@ -42,9 +42,7 @@ nginx: reload_command: reload reload: - nginx - renewal_minute: "0" # minute to run renewal cron job - renewal_hour: "0" # hour to run renewal cron job - renewal_weekday: "0" # day of week to run renewal + on_calendar: "Mon *-*-* 04:00:00" ``` As in the example above, you need to include *all* variables required by the `letsencrypt` SSL handler because defaults will not load from the `ssl` role in this context. @@ -91,7 +89,7 @@ ssl: autorenew: false # set to true to create a systemd timer to renew LE certs certbot_renew_command: "certonly --agree-tos --force-renew" # root of the command used in the systemd timer # See systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events - #on_calendar: "Mon *-*-* 04:00:00" + on_calendar: "Mon *-*-* 04:00:00" web_server: standalone # values are standalone, nginx or apache - warning, nginx and apache will attempt to manipulate your vhosts! # For "letsencrypt" handling, a list of service to stop while creating the certificate. diff --git a/roles/debian/ssl/defaults/main.yml b/roles/debian/ssl/defaults/main.yml index 2c862cda9..f1191de89 100644 --- a/roles/debian/ssl/defaults/main.yml +++ b/roles/debian/ssl/defaults/main.yml @@ -35,7 +35,7 @@ ssl: autorenew: false # set to true to create a systemd timer to renew LE certs certbot_renew_command: "certonly --agree-tos --force-renew" # root of the command used in the systemd timer # See systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events - #on_calendar: "Mon *-*-* 04:00:00" + on_calendar: "Mon *-*-* 04:00:00" web_server: standalone # values are standalone, nginx or apache - warning, nginx and apache will attempt to manipulate your vhosts! # For "letsencrypt" handling, a list of service to stop while creating the certificate. From 9acaddc25d079f37db53b8e9875e2831913a6bae Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 31 May 2024 15:36:05 +0200 Subject: [PATCH 49/51] Working around deprecated SSH algorithms. --- roles/aws/aws_ami/templates/packer.json.j2 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/roles/aws/aws_ami/templates/packer.json.j2 b/roles/aws/aws_ami/templates/packer.json.j2 index 5efd118b6..7f04c6b10 100755 --- a/roles/aws/aws_ami/templates/packer.json.j2 +++ b/roles/aws/aws_ami/templates/packer.json.j2 @@ -50,6 +50,7 @@ ], "provisioners": [{ "type": "ansible", + "use_proxy": false, "playbook_file": "{{ aws_ami.playbook_file }}", "inventory_directory": "{{ _ce_provision_base_dir }}/hosts", "ssh_authorized_key_file": "/home/{{ user_provision.username }}/.ssh/id_rsa.pub", @@ -63,7 +64,8 @@ "ANSIBLE_CONFIG={{ _ce_provision_base_dir }}/ansible.cfg" ], "extra_arguments": - [ + [ + "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa", {% if ansible_verbosity >= 1 %} "-vvvv", {% endif %} @@ -71,8 +73,8 @@ "--extra-vars", "{{ _aws_ami_extra_vars }}", {% endif %} - "--extra-vars", - "{ _ce_provision_base_dir: {{ _ce_provision_base_dir }}, _ce_provision_build_dir: {{ _ce_provision_build_dir }}, _ce_provision_build_tmp_dir: {{ _ce_provision_build_tmp_dir }}, _ce_provision_data_dir: {{ _ce_provision_data_dir }}, _ce_provision_build_id: {{ _ce_provision_build_id }}, _ce_provision_force_play: true }" - ] + "--extra-vars", + "{ _ce_provision_base_dir: {{ _ce_provision_base_dir }}, _ce_provision_build_dir: {{ _ce_provision_build_dir }}, _ce_provision_build_tmp_dir: {{ _ce_provision_build_tmp_dir }}, _ce_provision_data_dir: {{ _ce_provision_data_dir }}, _ce_provision_build_id: {{ _ce_provision_build_id }}, _ce_provision_force_play: true }" + ] }] } From 581e2afa599ae84285fd9096838ca63c6e76275c Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 3 Jun 2024 14:32:02 +0200 Subject: [PATCH 50/51] Upgrading SSH key type standard for controller and deploy users. --- docs/roles/aws/aws_ami.md | 1 + docs/roles/debian/ce_deploy.md | 4 +++- docs/roles/debian/ce_provision.md | 4 +++- roles/aws/aws_ami/README.md | 1 + roles/aws/aws_ami/defaults/main.yml | 1 + roles/aws/aws_ami/templates/packer.json.j2 | 10 ++++------ roles/debian/ce_deploy/README.md | 4 +++- roles/debian/ce_deploy/defaults/main.yml | 4 +++- roles/debian/ce_deploy/tasks/main.yml | 5 +++-- roles/debian/ce_provision/README.md | 4 +++- roles/debian/ce_provision/defaults/main.yml | 4 +++- roles/debian/ce_provision/tasks/main.yml | 3 ++- 12 files changed, 30 insertions(+), 15 deletions(-) diff --git a/docs/roles/aws/aws_ami.md b/docs/roles/aws/aws_ami.md index f4f3bd2be..acc2f10e4 100644 --- a/docs/roles/aws/aws_ami.md +++ b/docs/roles/aws/aws_ami.md @@ -29,6 +29,7 @@ aws_ami: ami_name: "example" owner: "136693071363" # Global AWS account ID of owner, defaults to Debian official ssh_username: "admin" + public_key_name: id_ecdsa.pub # from Debian 12 (Bookworm) onwards RSA keys, i.e. id_rsa.pub, are deprecated encrypt_boot: false # EBS volume options device_name: /dev/xvda # default for Debian AMIs diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index 856e548eb..d3c939fd2 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -20,7 +20,9 @@ ce_deploy: # Other ce-deploy settings. aws_support: true # installs boto3 new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user - key_name: id_rsa.pub # existing users may have a key of a different name + ssh_key_bits: "521" # recommended to use 4096 for RSA keys, 521 is the maximum for ECDSA keys + ssh_key_type: ecdsa # set to rsa to create an RSA key + public_key_name: id_ecdsa.pub # this might be id_rsa.pub for RSA keys, existing users may have a key of a different name username: "{{ _ce_deploy.username }}" own_repository: "https://github.com/codeenigma/ce-deploy.git" own_repository_branch: "master" diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index 8611d76b4..e3816975d 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -21,7 +21,9 @@ ce_provision: new_user: "{{ _init.ce_provision_new_user }}" # see _init defaults, set to false if user already exists or is ephemeral, e.g. an LDAP user username: "{{ _ce_provision_username }}" # see _init defaults #uid: "{{ _init.ce_provision_uid }}" # see _init defaults, optionally hardcode the UID for this user - public_key_name: id_rsa.pub # existing users may have a key of a different name + ssh_key_bits: "521" # recommended to use 4096 for RSA keys, 521 is the maximum for ECDSA keys + ssh_key_type: ecdsa # set to rsa to create an RSA key + public_key_name: id_ecdsa.pub # this might be id_rsa.pub for RSA keys, existing users may have a key of a different name # Main repo. own_repository: "https://github.com/codeenigma/ce-provision.git" own_repository_branch: "master" diff --git a/roles/aws/aws_ami/README.md b/roles/aws/aws_ami/README.md index f4f3bd2be..acc2f10e4 100644 --- a/roles/aws/aws_ami/README.md +++ b/roles/aws/aws_ami/README.md @@ -29,6 +29,7 @@ aws_ami: ami_name: "example" owner: "136693071363" # Global AWS account ID of owner, defaults to Debian official ssh_username: "admin" + public_key_name: id_ecdsa.pub # from Debian 12 (Bookworm) onwards RSA keys, i.e. id_rsa.pub, are deprecated encrypt_boot: false # EBS volume options device_name: /dev/xvda # default for Debian AMIs diff --git a/roles/aws/aws_ami/defaults/main.yml b/roles/aws/aws_ami/defaults/main.yml index 4b9bcbf1c..c828adaec 100644 --- a/roles/aws/aws_ami/defaults/main.yml +++ b/roles/aws/aws_ami/defaults/main.yml @@ -9,6 +9,7 @@ aws_ami: ami_name: "example" owner: "136693071363" # Global AWS account ID of owner, defaults to Debian official ssh_username: "admin" + public_key_name: id_ecdsa.pub # from Debian 12 (Bookworm) onwards RSA keys, i.e. id_rsa.pub, are deprecated encrypt_boot: false # EBS volume options device_name: /dev/xvda # default for Debian AMIs diff --git a/roles/aws/aws_ami/templates/packer.json.j2 b/roles/aws/aws_ami/templates/packer.json.j2 index 7f04c6b10..ae3c72ef5 100755 --- a/roles/aws/aws_ami/templates/packer.json.j2 +++ b/roles/aws/aws_ami/templates/packer.json.j2 @@ -50,27 +50,25 @@ ], "provisioners": [{ "type": "ansible", - "use_proxy": false, "playbook_file": "{{ aws_ami.playbook_file }}", "inventory_directory": "{{ _ce_provision_base_dir }}/hosts", - "ssh_authorized_key_file": "/home/{{ user_provision.username }}/.ssh/id_rsa.pub", + "ssh_authorized_key_file": "/home/{{ user_provision.username }}/.ssh/{{ aws_ami.public_key_name }}", {% if aws_ami.groups is defined and aws_ami.groups | length %} "groups": {{ aws_ami.groups | to_json }}, {% endif %} "ansible_env_vars": [ - "ANSIBLE_FORCE_COLOR=True", - "ALLOW_WORLD_READABLE_TMPFILES=True", + "ANSIBLE_FORCE_COLOR=True", + "ALLOW_WORLD_READABLE_TMPFILES=True", "ANSIBLE_CONFIG={{ _ce_provision_base_dir }}/ansible.cfg" ], "extra_arguments": [ - "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa", {% if ansible_verbosity >= 1 %} "-vvvv", {% endif %} {% if _aws_ami_extra_vars is defined and _aws_ami_extra_vars | length %} - "--extra-vars", + "--extra-vars", "{{ _aws_ami_extra_vars }}", {% endif %} "--extra-vars", diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index 856e548eb..d3c939fd2 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -20,7 +20,9 @@ ce_deploy: # Other ce-deploy settings. aws_support: true # installs boto3 new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user - key_name: id_rsa.pub # existing users may have a key of a different name + ssh_key_bits: "521" # recommended to use 4096 for RSA keys, 521 is the maximum for ECDSA keys + ssh_key_type: ecdsa # set to rsa to create an RSA key + public_key_name: id_ecdsa.pub # this might be id_rsa.pub for RSA keys, existing users may have a key of a different name username: "{{ _ce_deploy.username }}" own_repository: "https://github.com/codeenigma/ce-deploy.git" own_repository_branch: "master" diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index d43bde3e6..dfccea372 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -12,7 +12,9 @@ ce_deploy: # Other ce-deploy settings. aws_support: true # installs boto3 new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user - key_name: id_rsa.pub # existing users may have a key of a different name + ssh_key_bits: "521" # recommended to use 4096 for RSA keys, 521 is the maximum for ECDSA keys + ssh_key_type: ecdsa # set to rsa to create an RSA key + public_key_name: id_ecdsa.pub # this might be id_rsa.pub for RSA keys, existing users may have a key of a different name username: "{{ _ce_deploy.username }}" own_repository: "https://github.com/codeenigma/ce-deploy.git" own_repository_branch: "master" diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index d5bbb3761..447cd99c1 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -16,7 +16,8 @@ generate_ssh_key: true home: "/home/{{ ce_deploy.username }}" password: "*" - ssh_key_bits: 4096 + ssh_key_bits: "{{ ce_deploy.ssh_key_bits }}" + ssh_key_type: "{{ ce_deploy.ssh_key_type }}" ssh_key_comment: "{{ ce_deploy.username }}@{{ ansible_host }}" system: true groups: "{{ ce_deploy.groups }}" @@ -35,7 +36,7 @@ - name: Place the public key in the ce-provision data directory on the controller server. ansible.builtin.fetch: dest: "{{ _ce_provision_data_dir }}" - src: "/home/{{ ce_deploy.username }}/.ssh/{{ ce_deploy.key_name }}" + src: "/home/{{ ce_deploy.username }}/.ssh/{{ ce_deploy.public_key_name }}" - name: Ensure Git is installed. ansible.builtin.apt: diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index 8611d76b4..e3816975d 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -21,7 +21,9 @@ ce_provision: new_user: "{{ _init.ce_provision_new_user }}" # see _init defaults, set to false if user already exists or is ephemeral, e.g. an LDAP user username: "{{ _ce_provision_username }}" # see _init defaults #uid: "{{ _init.ce_provision_uid }}" # see _init defaults, optionally hardcode the UID for this user - public_key_name: id_rsa.pub # existing users may have a key of a different name + ssh_key_bits: "521" # recommended to use 4096 for RSA keys, 521 is the maximum for ECDSA keys + ssh_key_type: ecdsa # set to rsa to create an RSA key + public_key_name: id_ecdsa.pub # this might be id_rsa.pub for RSA keys, existing users may have a key of a different name # Main repo. own_repository: "https://github.com/codeenigma/ce-provision.git" own_repository_branch: "master" diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index 62c92b9e9..863be9952 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -12,7 +12,9 @@ ce_provision: new_user: "{{ _init.ce_provision_new_user }}" # see _init defaults, set to false if user already exists or is ephemeral, e.g. an LDAP user username: "{{ _ce_provision_username }}" # see _init defaults #uid: "{{ _init.ce_provision_uid }}" # see _init defaults, optionally hardcode the UID for this user - public_key_name: id_rsa.pub # existing users may have a key of a different name + ssh_key_bits: "521" # recommended to use 4096 for RSA keys, 521 is the maximum for ECDSA keys + ssh_key_type: ecdsa # set to rsa to create an RSA key + public_key_name: id_ecdsa.pub # this might be id_rsa.pub for RSA keys, existing users may have a key of a different name # Main repo. own_repository: "https://github.com/codeenigma/ce-provision.git" own_repository_branch: "master" diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index d19b458cc..7a531051d 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -30,7 +30,8 @@ generate_ssh_key: true home: "/home/{{ ce_provision.username }}" password: "*" - ssh_key_bits: 4096 + ssh_key_bits: "{{ ce_provision.ssh_key_bits }}" + ssh_key_type: "{{ ce_provision.ssh_key_type }}" ssh_key_comment: "{{ ce_provision.username }}@{{ ansible_host }}" system: true groups: "{{ ce_provision.groups }}" From f6ed39fafbb9d387bf36c213b23126dfcc80ef3d Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 3 Jun 2024 17:40:59 +0200 Subject: [PATCH 51/51] Adding SCP args for legacy mode needed by Packer. --- roles/aws/aws_ami/templates/packer.json.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/aws/aws_ami/templates/packer.json.j2 b/roles/aws/aws_ami/templates/packer.json.j2 index ae3c72ef5..d2abad8af 100755 --- a/roles/aws/aws_ami/templates/packer.json.j2 +++ b/roles/aws/aws_ami/templates/packer.json.j2 @@ -64,6 +64,7 @@ ], "extra_arguments": [ + "--scp-extra-args", "'-O'", {% if ansible_verbosity >= 1 %} "-vvvv", {% endif %}