From 90aeb2c5638643dad10808e36ec9e4b691f38591 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 22 Nov 2023 18:16:51 +0100 Subject: [PATCH 001/206] Improving AWS subnet docs. --- docs/roles/aws/aws_ec2_autoscale_cluster.md | 2 ++ docs/roles/aws/aws_rds.md | 1 + docs/roles/aws/aws_vpc_subnet.md | 1 + roles/aws/aws_ec2_autoscale_cluster/README.md | 2 ++ roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml | 2 ++ roles/aws/aws_rds/README.md | 1 + roles/aws/aws_rds/defaults/main.yml | 1 + roles/aws/aws_vpc_subnet/README.md | 1 + roles/aws/aws_vpc_subnet/defaults/main.yml | 1 + 9 files changed, 12 insertions(+) diff --git a/docs/roles/aws/aws_ec2_autoscale_cluster.md b/docs/roles/aws/aws_ec2_autoscale_cluster.md index 5e6909609..49f419391 100644 --- a/docs/roles/aws/aws_ec2_autoscale_cluster.md +++ b/docs/roles/aws/aws_ec2_autoscale_cluster.md @@ -36,6 +36,8 @@ aws_ec2_autoscale_cluster: vpc_id: vpc-XXXX # One of vpc_id or vpc_name is mandatory. # vpc_name: example-vpc subnets: + # If rds.rds.true is set and you are creating an RDS instance then you must use at least two AZs, this is hard requirement. + # This means your region and environment must also have a minimum of two subnets - see `aws_vpc_subnet` # - az: a # cidr: "10.0.3.0/26" - az: b diff --git a/docs/roles/aws/aws_rds.md b/docs/roles/aws/aws_rds.md index e89cd6e74..1e5287a15 100644 --- a/docs/roles/aws/aws_rds.md +++ b/docs/roles/aws/aws_rds.md @@ -19,6 +19,7 @@ aws_rds: aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" multi_az: true + # You must provide at least two subnets in two different AZs, even for single AZ deployments. subnets: - subnet-aaaaaaaa - subnet-bbbbbbbb diff --git a/docs/roles/aws/aws_vpc_subnet.md b/docs/roles/aws/aws_vpc_subnet.md index 5d97d303a..1b4ef012b 100644 --- a/docs/roles/aws/aws_vpc_subnet.md +++ b/docs/roles/aws/aws_vpc_subnet.md @@ -11,6 +11,7 @@ aws_vpc_subnet: # vpc_name: example-vpc aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" + # If you intend to have RDS instances you must provide at least two subnets. subnets: - cidr_block: "10.0.0.0/24" # ipv6_cidr_block: "1" # This will create something like xxxx:xxxx:xxxx:xxyy::/64 where yy is created using the ansible.utils.ipsubnet filter automatically - DO NOT DEFINE IF IPV6 IS NOT REQUIRED diff --git a/roles/aws/aws_ec2_autoscale_cluster/README.md b/roles/aws/aws_ec2_autoscale_cluster/README.md index 5e6909609..49f419391 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/README.md +++ b/roles/aws/aws_ec2_autoscale_cluster/README.md @@ -36,6 +36,8 @@ aws_ec2_autoscale_cluster: vpc_id: vpc-XXXX # One of vpc_id or vpc_name is mandatory. # vpc_name: example-vpc subnets: + # If rds.rds.true is set and you are creating an RDS instance then you must use at least two AZs, this is hard requirement. + # This means your region and environment must also have a minimum of two subnets - see `aws_vpc_subnet` # - az: a # cidr: "10.0.3.0/26" - az: b diff --git a/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml b/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml index 5bdd39aef..9af962f5c 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml +++ b/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml @@ -17,6 +17,8 @@ aws_ec2_autoscale_cluster: vpc_id: vpc-XXXX # One of vpc_id or vpc_name is mandatory. # vpc_name: example-vpc subnets: + # If rds.rds.true is set and you are creating an RDS instance then you must use at least two AZs, this is hard requirement. + # This means your region and environment must also have a minimum of two subnets - see `aws_vpc_subnet` # - az: a # cidr: "10.0.3.0/26" - az: b diff --git a/roles/aws/aws_rds/README.md b/roles/aws/aws_rds/README.md index e89cd6e74..1e5287a15 100644 --- a/roles/aws/aws_rds/README.md +++ b/roles/aws/aws_rds/README.md @@ -19,6 +19,7 @@ aws_rds: aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" multi_az: true + # You must provide at least two subnets in two different AZs, even for single AZ deployments. subnets: - subnet-aaaaaaaa - subnet-bbbbbbbb diff --git a/roles/aws/aws_rds/defaults/main.yml b/roles/aws/aws_rds/defaults/main.yml index 96b4d4914..faa3ee5bb 100644 --- a/roles/aws/aws_rds/defaults/main.yml +++ b/roles/aws/aws_rds/defaults/main.yml @@ -2,6 +2,7 @@ aws_rds: aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" multi_az: true + # You must provide at least two subnets in two different AZs, even for single AZ deployments. subnets: - subnet-aaaaaaaa - subnet-bbbbbbbb diff --git a/roles/aws/aws_vpc_subnet/README.md b/roles/aws/aws_vpc_subnet/README.md index 5d97d303a..1b4ef012b 100644 --- a/roles/aws/aws_vpc_subnet/README.md +++ b/roles/aws/aws_vpc_subnet/README.md @@ -11,6 +11,7 @@ aws_vpc_subnet: # vpc_name: example-vpc aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" + # If you intend to have RDS instances you must provide at least two subnets. subnets: - cidr_block: "10.0.0.0/24" # ipv6_cidr_block: "1" # This will create something like xxxx:xxxx:xxxx:xxyy::/64 where yy is created using the ansible.utils.ipsubnet filter automatically - DO NOT DEFINE IF IPV6 IS NOT REQUIRED diff --git a/roles/aws/aws_vpc_subnet/defaults/main.yml b/roles/aws/aws_vpc_subnet/defaults/main.yml index 1e29a7d01..e72b3ae64 100644 --- a/roles/aws/aws_vpc_subnet/defaults/main.yml +++ b/roles/aws/aws_vpc_subnet/defaults/main.yml @@ -3,6 +3,7 @@ aws_vpc_subnet: # vpc_name: example-vpc aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" + # If you intend to have RDS instances you must provide at least two subnets. subnets: - cidr_block: "10.0.0.0/24" # ipv6_cidr_block: "1" # This will create something like xxxx:xxxx:xxxx:xxyy::/64 where yy is created using the ansible.utils.ipsubnet filter automatically - DO NOT DEFINE IF IPV6 IS NOT REQUIRED From 7c31b40ef78d23c459b7c05ed01463edfe756255 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 12 Dec 2023 18:34:14 +0100 Subject: [PATCH 002/206] Error in timers structure in the SSL role. --- roles/debian/nginx/defaults/main.yml | 1 + roles/debian/ssl/tasks/letsencrypt.yml | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/debian/nginx/defaults/main.yml b/roles/debian/nginx/defaults/main.yml index 61f2be571..64dbc76fa 100644 --- a/roles/debian/nginx/defaults/main.yml +++ b/roles/debian/nginx/defaults/main.yml @@ -57,6 +57,7 @@ nginx: # reload_command: restart # reload: # - nginx + # on_calendar: "Mon *-*-* 04:00:00" ratelimitingcrawlers: true is_default: true basic_auth: diff --git a/roles/debian/ssl/tasks/letsencrypt.yml b/roles/debian/ssl/tasks/letsencrypt.yml index 616040ae1..65ff333b4 100644 --- a/roles/debian/ssl/tasks/letsencrypt.yml +++ b/roles/debian/ssl/tasks/letsencrypt.yml @@ -87,6 +87,5 @@ ansible.builtin.include_role: name: contrib/systemd_timers vars: - timers: - certbot_renewal: "{{ _certbot_renewal_timer }}" + timers: "{{ _certbot_renewal_timer }}" when: ssl.autorenew From b75fe03127d42033ed1b4927f2f89ac78d4a78d4 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 21 Dec 2023 16:00:54 +0100 Subject: [PATCH 003/206] Removing obsolete backports requirements. --- roles/debian/firewall_config/tasks/main.yml | 10 ------ roles/debian/ldap_server/tasks/main.yml | 11 ------- roles/debian/lhci/tasks/main.yml | 34 --------------------- 3 files changed, 55 deletions(-) diff --git a/roles/debian/firewall_config/tasks/main.yml b/roles/debian/firewall_config/tasks/main.yml index 66941b51a..cf770ab4e 100644 --- a/roles/debian/firewall_config/tasks/main.yml +++ b/roles/debian/firewall_config/tasks/main.yml @@ -1,14 +1,4 @@ --- -# @TODO remove when https://redmine.codeenigma.net/issues/58246 can be done -- name: Install iptables with backports. # we have to do this in ce-dev or the contrib role will fail - ansible.builtin.apt: - pkg: ["iptables"] - state: present - default_release: buster-backports - when: - - is_local is defined - - is_local - - name: Shift general firewall settings to expected variables. ansible.builtin.set_fact: firewall_state: "{{ firewall_config.firewall_state }}" diff --git a/roles/debian/ldap_server/tasks/main.yml b/roles/debian/ldap_server/tasks/main.yml index 52fb1ccf4..9a773da58 100644 --- a/roles/debian/ldap_server/tasks/main.yml +++ b/roles/debian/ldap_server/tasks/main.yml @@ -14,21 +14,10 @@ purge: true when: ldap_server.slapd.purge -# @TODO remove when https://redmine.codeenigma.net/issues/58246 can be done -- name: Ensure LDAP and dependencies are installed from backports. - ansible.builtin.apt: - pkg: ["slapd", "ldapscripts", "libldap2-dev"] # python-ldap needs libldap2-dev - state: present - default_release: buster-backports - when: - - is_local is defined - - is_local - - name: Ensure LDAP and python-ldap and dependencies are installed. ansible.builtin.apt: pkg: ["slapd", "ldapscripts", "libldap2-dev"] state: present - when: is_local is not defined - name: Ensure additional dependencies for python-ldap are installed. ansible.builtin.apt: diff --git a/roles/debian/lhci/tasks/main.yml b/roles/debian/lhci/tasks/main.yml index c92447d4b..56c1fd9c9 100644 --- a/roles/debian/lhci/tasks/main.yml +++ b/roles/debian/lhci/tasks/main.yml @@ -10,26 +10,6 @@ state: present filename: google-chrome -# @TODO remove when https://redmine.codeenigma.net/issues/58246 can be done -- name: Install Google Chrome "headful" mode dependencies from backports. - ansible.builtin.apt: - name: - - xorg - - xvfb - - gtk2-engines-pixbuf - - dbus-x11 - - xfonts-base - - xfonts-100dpi - - xfonts-75dpi - - xfonts-cyrillic - - xfonts-scalable - default_release: buster-backports - state: present - when: - - is_local - - ansible_distribution == "Debian" - - ansible_distribution_major_version == "10" - - name: Install Google Chrome "headful" mode dependencies. ansible.builtin.apt: name: @@ -43,19 +23,6 @@ - xfonts-cyrillic - xfonts-scalable state: present - when: is_local is not defined - -# @TODO remove when https://redmine.codeenigma.net/issues/58246 can be done -- name: Install VNC support from backports. - ansible.builtin.apt: - name: x11vnc - default_release: buster-backports - state: present - when: - - lhci.enable_vnc - - is_local - - ansible_distribution == "Debian" - - ansible_distribution_major_version == "10" - name: Install VNC support. ansible.builtin.apt: @@ -63,7 +30,6 @@ state: present when: - lhci.enable_vnc - - is_local is not defined - name: Configure Xvfb to start on boot. ansible.builtin.shell: | From 5b27a8c0f0d7a9f8f1fd8437c96fafe33f98a692 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 21 Dec 2023 16:46:16 +0100 Subject: [PATCH 004/206] Allow the billing role to access Sustainability information. --- roles/aws/aws_iam_saml/templates/access_billing_policy.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 b/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 index cf20372f8..714056642 100644 --- a/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 +++ b/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 @@ -9,7 +9,8 @@ "freetier:*", "ce:*", "cur:*", - "tax:*" + "tax:*", + "sustainability:*" ], "Effect": "Allow", "Resource": "*" From 9ca4148dddc42a0d5e102562902f2d017500b996 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 21 Dec 2023 18:27:48 +0100 Subject: [PATCH 005/206] Missing comma in IAM billing policy. --- roles/aws/aws_iam_saml/templates/access_billing_policy.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 b/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 index 714056642..b26f9e7c7 100644 --- a/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 +++ b/roles/aws/aws_iam_saml/templates/access_billing_policy.j2 @@ -28,7 +28,7 @@ "tax:BatchPutTaxRegistration", "tax:DeleteTaxRegistration", "tax:PutTaxInheritance" - ] + ], "Effect": "Deny", "Resource": "*" } From c8199bef92edb321c8df6e7d11a361c98227941e Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 9 Jan 2024 19:30:01 +0100 Subject: [PATCH 006/206] Removing broken GitLab Runner code. --- roles/debian/gitlab_runner/tasks/main.yml | 25 +++++++++++-------- .../gitlab_runner/templates/fargate.toml.j2 | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/roles/debian/gitlab_runner/tasks/main.yml b/roles/debian/gitlab_runner/tasks/main.yml index 2601d8e4f..37a30227a 100644 --- a/roles/debian/gitlab_runner/tasks/main.yml +++ b/roles/debian/gitlab_runner/tasks/main.yml @@ -35,18 +35,21 @@ # gitlab-runner register --url https://gitlab.com/ --registration-token TOKEN_HERE --name fargate-test-runner --run-untagged --executor custom -n # This will automatically update /etc/gitlab-runner/config.toml +# @TODO this needs rethinking, we cannot delegate include_role! +# Replace security_group in template with _aws_security_group_list[0] when resolved + # Populates the _aws_security_group_list variable used in fargate.toml.j2 -- name: Generate security group information. - ansible.builtin.include_role: - name: aws/aws_security_groups - vars: - profile: "{{ aws_elasticache.aws_profile }}" - region: "{{ aws_elasticache.region }}" - group_names: - - "{{ gitlab_runner.fargate.security_group }}" - return_type: ids - when: gitlab_runner.install_fargate - delegate_to: localhost +#- name: Generate security group information. +# ansible.builtin.include_role: +# name: aws/aws_security_groups +# vars: +# profile: "{{ gitlab_runner.fargate.region }}" +# region: "{{ gitlab_runner.fargate.region }}" +# group_names: +# - "{{ gitlab_runner.fargate.security_group }}" +# return_type: ids +# when: gitlab_runner.install_fargate +# delegate_to: localhost - name: Create the Fargate driver directory if it does not exist. ansible.builtin.file: diff --git a/roles/debian/gitlab_runner/templates/fargate.toml.j2 b/roles/debian/gitlab_runner/templates/fargate.toml.j2 index 6af78fc5a..c32c3633a 100644 --- a/roles/debian/gitlab_runner/templates/fargate.toml.j2 +++ b/roles/debian/gitlab_runner/templates/fargate.toml.j2 @@ -5,7 +5,7 @@ LogFormat = "text" Cluster = "{{ gitlab_runner.fargate.cluster }}" Region = "{{ gitlab_runner.fargate.region }}" Subnet = "{{ gitlab_runner.fargate.subnet }}" - SecurityGroup = "{{ _aws_security_group_list[0] }}" + SecurityGroup = "{{ gitlab_runner.fargate.security_group }}" TaskDefinition = "{{ gitlab_runner.fargate.task_definition }}" EnablePublicIP = {{ gitlab_runner.fargate.public_ip }} PlatformVersion = "{{ gitlab_runner.fargate.version }}" From 0ba41388c95088c6befa309343268d6c21459f25 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 10:58:45 +0100 Subject: [PATCH 007/206] Fixed the include_role task in gitlab_runner. --- docs/roles/debian/gitlab_runner.md | 1 + roles/debian/gitlab_runner/README.md | 1 + roles/debian/gitlab_runner/defaults/main.yml | 1 + roles/debian/gitlab_runner/tasks/main.yml | 26 +++++++++---------- .../gitlab_runner/templates/fargate.toml.j2 | 2 +- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/roles/debian/gitlab_runner.md b/docs/roles/debian/gitlab_runner.md index 52554ced5..10829036f 100644 --- a/docs/roles/debian/gitlab_runner.md +++ b/docs/roles/debian/gitlab_runner.md @@ -48,6 +48,7 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name + profile: "{{ _aws_profile }}" region: "eu-west-1" # AWS region name subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name diff --git a/roles/debian/gitlab_runner/README.md b/roles/debian/gitlab_runner/README.md index 52554ced5..10829036f 100644 --- a/roles/debian/gitlab_runner/README.md +++ b/roles/debian/gitlab_runner/README.md @@ -48,6 +48,7 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name + profile: "{{ _aws_profile }}" region: "eu-west-1" # AWS region name subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name diff --git a/roles/debian/gitlab_runner/defaults/main.yml b/roles/debian/gitlab_runner/defaults/main.yml index 845371447..ba1c1d48e 100644 --- a/roles/debian/gitlab_runner/defaults/main.yml +++ b/roles/debian/gitlab_runner/defaults/main.yml @@ -36,6 +36,7 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name + profile: "{{ _aws_profile }}" region: "eu-west-1" # AWS region name subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name diff --git a/roles/debian/gitlab_runner/tasks/main.yml b/roles/debian/gitlab_runner/tasks/main.yml index 37a30227a..685b0bfc2 100644 --- a/roles/debian/gitlab_runner/tasks/main.yml +++ b/roles/debian/gitlab_runner/tasks/main.yml @@ -35,21 +35,19 @@ # gitlab-runner register --url https://gitlab.com/ --registration-token TOKEN_HERE --name fargate-test-runner --run-untagged --executor custom -n # This will automatically update /etc/gitlab-runner/config.toml -# @TODO this needs rethinking, we cannot delegate include_role! -# Replace security_group in template with _aws_security_group_list[0] when resolved - # Populates the _aws_security_group_list variable used in fargate.toml.j2 -#- name: Generate security group information. -# ansible.builtin.include_role: -# name: aws/aws_security_groups -# vars: -# profile: "{{ gitlab_runner.fargate.region }}" -# region: "{{ gitlab_runner.fargate.region }}" -# group_names: -# - "{{ gitlab_runner.fargate.security_group }}" -# return_type: ids -# when: gitlab_runner.install_fargate -# delegate_to: localhost +- name: Generate security group information. + ansible.builtin.include_role: + name: aws/aws_security_groups + apply: + delegate_to: localhost # this is how you delegate the include_role module + vars: + profile: "{{ gitlab_runner.fargate.profile }}" + region: "{{ gitlab_runner.fargate.region }}" + group_names: + - "{{ gitlab_runner.fargate.security_group }}" + return_type: ids + when: gitlab_runner.install_fargate - name: Create the Fargate driver directory if it does not exist. ansible.builtin.file: diff --git a/roles/debian/gitlab_runner/templates/fargate.toml.j2 b/roles/debian/gitlab_runner/templates/fargate.toml.j2 index c32c3633a..6af78fc5a 100644 --- a/roles/debian/gitlab_runner/templates/fargate.toml.j2 +++ b/roles/debian/gitlab_runner/templates/fargate.toml.j2 @@ -5,7 +5,7 @@ LogFormat = "text" Cluster = "{{ gitlab_runner.fargate.cluster }}" Region = "{{ gitlab_runner.fargate.region }}" Subnet = "{{ gitlab_runner.fargate.subnet }}" - SecurityGroup = "{{ gitlab_runner.fargate.security_group }}" + SecurityGroup = "{{ _aws_security_group_list[0] }}" TaskDefinition = "{{ gitlab_runner.fargate.task_definition }}" EnablePublicIP = {{ gitlab_runner.fargate.public_ip }} PlatformVersion = "{{ gitlab_runner.fargate.version }}" From aacf70beb8415c637571707c37c6dc5d59704ea0 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 11:00:16 +0100 Subject: [PATCH 008/206] Suppressing a failure if there is no system pip to call. --- roles/debian/ansible/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 722690a64..d5f414624 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -10,6 +10,7 @@ state: absent executable: pip3 when: ansible_distribution_major_version | int < 12 + failed_when: false # don't stop the build if there's no system pip - name: Set up Python packages. ansible.builtin.include_role: From c634ef5e61ced0403f781263549e731833c4121d Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 11:19:53 +0100 Subject: [PATCH 009/206] Logic error in Ansible installer username, needs to be set from calling role. --- docs/roles/debian/ansible.md | 3 ++- docs/roles/debian/ce_deploy.md | 1 + docs/roles/debian/ce_provision.md | 1 + roles/debian/ansible/README.md | 3 ++- roles/debian/ansible/defaults/main.yml | 3 ++- roles/debian/ansible/tasks/main.yml | 6 +++--- roles/debian/ce_deploy/README.md | 1 + roles/debian/ce_deploy/defaults/main.yml | 1 + roles/debian/ce_deploy/tasks/main.yml | 1 + roles/debian/ce_provision/README.md | 1 + roles/debian/ce_provision/defaults/main.yml | 1 + roles/debian/ce_provision/tasks/main.yml | 1 + 12 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/roles/debian/ansible.md b/docs/roles/debian/ansible.md index 7e0015fd9..e87cedd66 100644 --- a/docs/roles/debian/ansible.md +++ b/docs/roles/debian/ansible.md @@ -11,9 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path and _venv_command but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv + #ansible_user: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index 66d7402b2..d829d089f 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -14,6 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index 741ca5a6c..001d8fad6 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -15,6 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ansible/README.md b/roles/debian/ansible/README.md index 7e0015fd9..e87cedd66 100644 --- a/roles/debian/ansible/README.md +++ b/roles/debian/ansible/README.md @@ -11,9 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path and _venv_command but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv + #ansible_user: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/defaults/main.yml b/roles/debian/ansible/defaults/main.yml index 746251010..2c6108938 100644 --- a/roles/debian/ansible/defaults/main.yml +++ b/roles/debian/ansible/defaults/main.yml @@ -1,8 +1,9 @@ --- ce_ansible: - # These are usually set within another role using _venv_path and _venv_command but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv + #ansible_user: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index d5f414624..ead087450 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -24,7 +24,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_provision.username }}" + become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" - name: Install Ansible. ansible.builtin.pip: @@ -33,7 +33,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_provision.username }}" + become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" - name: Install linters. ansible.builtin.pip: @@ -44,7 +44,7 @@ virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" when: ce_ansible.linters.enabled become: true - become_user: "{{ ce_provision.username }}" + become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" - name: Add the venv to $PATH using profile.d. ansible.builtin.copy: diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index 66d7402b2..d829d089f 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -14,6 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 92fefc0ed..276780b3b 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -6,6 +6,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index 723ef9f57..f047afacc 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -132,6 +132,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_deploy.venv_path }}" _venv_command: "{{ ce_deploy.venv_command }}" + _ansible_user: "{{ ce_deploy.ansible_user }}" - name: Install Ansible. ansible.builtin.include_role: diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index 741ca5a6c..001d8fad6 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -15,6 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index f69f7c939..35da59995 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -7,6 +7,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv + ansible_user: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index 36b38bc8b..62662a1f5 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -140,6 +140,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_provision.venv_path }}" _venv_command: "{{ ce_provision.venv_command }}" + _ansible_user: "{{ ce_provision.ansible_user }}" - name: Install Ansible. ansible.builtin.include_role: From 267365988cfdb39561781854e2243f7ad485e843 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 12:40:14 +0100 Subject: [PATCH 010/206] ansible_user is a reserved variable, seems to be causing issues. --- docs/roles/debian/ansible.md | 4 ++-- docs/roles/debian/ce_deploy.md | 2 +- docs/roles/debian/ce_provision.md | 2 +- roles/debian/ansible/README.md | 4 ++-- roles/debian/ansible/defaults/main.yml | 4 ++-- roles/debian/ansible/tasks/main.yml | 6 +++--- roles/debian/ce_deploy/README.md | 2 +- roles/debian/ce_deploy/defaults/main.yml | 2 +- roles/debian/ce_deploy/tasks/main.yml | 2 +- roles/debian/ce_provision/README.md | 2 +- roles/debian/ce_provision/defaults/main.yml | 2 +- roles/debian/ce_provision/tasks/main.yml | 2 +- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/roles/debian/ansible.md b/docs/roles/debian/ansible.md index e87cedd66..faad26b84 100644 --- a/docs/roles/debian/ansible.md +++ b/docs/roles/debian/ansible.md @@ -11,10 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_user: deploy # user to become when creating venv + #ansible_install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index d829d089f..b9f3e983d 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -14,7 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_deploy.username }}" + ansible_install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index 001d8fad6..305ec1894 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -15,7 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_provision.username }}" + ansible_install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ansible/README.md b/roles/debian/ansible/README.md index e87cedd66..faad26b84 100644 --- a/roles/debian/ansible/README.md +++ b/roles/debian/ansible/README.md @@ -11,10 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_user: deploy # user to become when creating venv + #ansible_install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/defaults/main.yml b/roles/debian/ansible/defaults/main.yml index 2c6108938..319bdfe5d 100644 --- a/roles/debian/ansible/defaults/main.yml +++ b/roles/debian/ansible/defaults/main.yml @@ -1,9 +1,9 @@ --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_user but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_user: deploy # user to become when creating venv + #ansible_install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index ead087450..5b419b787 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -24,7 +24,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" + become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" - name: Install Ansible. ansible.builtin.pip: @@ -33,7 +33,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" + become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" - name: Install linters. ansible.builtin.pip: @@ -44,7 +44,7 @@ virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" when: ce_ansible.linters.enabled become: true - become_user: "{{ ce_ansible.ansible_user | default(_ansible_user) }}" + become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" - name: Add the venv to $PATH using profile.d. ansible.builtin.copy: diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index d829d089f..b9f3e983d 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -14,7 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_deploy.username }}" + ansible_install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 276780b3b..53ee6f8ed 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -6,7 +6,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_deploy.username }}" + ansible_install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index f047afacc..90143cdb5 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -132,7 +132,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_deploy.venv_path }}" _venv_command: "{{ ce_deploy.venv_command }}" - _ansible_user: "{{ ce_deploy.ansible_user }}" + _ansible_install_username: "{{ ce_deploy.ansible_install_username }}" - name: Install Ansible. ansible.builtin.include_role: diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index 001d8fad6..305ec1894 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -15,7 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_provision.username }}" + ansible_install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index 35da59995..d5e33b279 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -7,7 +7,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_user: "{{ _ce_provision.username }}" + ansible_install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index 62662a1f5..ba1be5773 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -140,7 +140,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_provision.venv_path }}" _venv_command: "{{ ce_provision.venv_command }}" - _ansible_user: "{{ ce_provision.ansible_user }}" + _ansible_install_username: "{{ ce_provision.ansible_install_username }}" - name: Install Ansible. ansible.builtin.include_role: From c6af4c0f7ea36b38a3c0417dca1abd747da3c9dc Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 13:10:59 +0100 Subject: [PATCH 011/206] _ansible_ANYTHING is reserved, using _install_username instead. --- docs/roles/debian/ansible.md | 4 ++-- docs/roles/debian/ce_deploy.md | 2 +- docs/roles/debian/ce_provision.md | 2 +- roles/debian/ansible/README.md | 4 ++-- roles/debian/ansible/defaults/main.yml | 4 ++-- roles/debian/ansible/tasks/main.yml | 6 +++--- roles/debian/ce_deploy/README.md | 2 +- roles/debian/ce_deploy/defaults/main.yml | 2 +- roles/debian/ce_deploy/tasks/main.yml | 2 +- roles/debian/ce_provision/README.md | 2 +- roles/debian/ce_provision/defaults/main.yml | 2 +- roles/debian/ce_provision/tasks/main.yml | 2 +- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/roles/debian/ansible.md b/docs/roles/debian/ansible.md index faad26b84..b7dfe0dbc 100644 --- a/docs/roles/debian/ansible.md +++ b/docs/roles/debian/ansible.md @@ -11,10 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_install_username: deploy # user to become when creating venv + #install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index b9f3e983d..d020663ab 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -14,7 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_deploy.username }}" + install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index 305ec1894..e8f83ea13 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -15,7 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_provision.username }}" + install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ansible/README.md b/roles/debian/ansible/README.md index faad26b84..b7dfe0dbc 100644 --- a/roles/debian/ansible/README.md +++ b/roles/debian/ansible/README.md @@ -11,10 +11,10 @@ Note, it is vitally important that Ansible is *not* installed via `apt` or `pip` ```yaml --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_install_username: deploy # user to become when creating venv + #install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/defaults/main.yml b/roles/debian/ansible/defaults/main.yml index 319bdfe5d..f58a026b0 100644 --- a/roles/debian/ansible/defaults/main.yml +++ b/roles/debian/ansible/defaults/main.yml @@ -1,9 +1,9 @@ --- ce_ansible: - # These are usually set within another role using _venv_path, _venv_command and _ansible_install_username but can be overridden. + # These are usually set within another role using _venv_path, _venv_command and _install_username but can be overridden. #venv_path: "/home/{{ ce_provision.username }}/ansible" #venv_command: /usr/bin/python3.11 -m venv - #ansible_install_username: deploy # user to become when creating venv + #install_username: deploy # user to become when creating venv upgrade: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 5b419b787..4aac433a9 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -24,7 +24,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" + become_user: "{{ ce_ansible.install_username | default(_install_username) }}" - name: Install Ansible. ansible.builtin.pip: @@ -33,7 +33,7 @@ virtualenv: "{{ ce_ansible.venv_path | default(_venv_path) }}" virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" + become_user: "{{ ce_ansible.install_username | default(_install_username) }}" - name: Install linters. ansible.builtin.pip: @@ -44,7 +44,7 @@ virtualenv_command: "{{ ce_ansible.venv_command | default(_venv_command) }}" when: ce_ansible.linters.enabled become: true - become_user: "{{ ce_ansible.ansible_install_username | default(_ansible_install_username) }}" + become_user: "{{ ce_ansible.install_username | default(_install_username) }}" - name: Add the venv to $PATH using profile.d. ansible.builtin.copy: diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index b9f3e983d..d020663ab 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -14,7 +14,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_deploy.username }}" + install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 53ee6f8ed..52bd86415 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -6,7 +6,7 @@ ce_deploy: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_deploy.username }}" + install_username: "{{ _ce_deploy.username }}" # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index 90143cdb5..a9d607723 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -132,7 +132,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_deploy.venv_path }}" _venv_command: "{{ ce_deploy.venv_command }}" - _ansible_install_username: "{{ ce_deploy.ansible_install_username }}" + _install_username: "{{ ce_deploy.install_username }}" - name: Install Ansible. ansible.builtin.include_role: diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index 305ec1894..e8f83ea13 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -15,7 +15,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_provision.username }}" + install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index d5e33b279..4c266e3e0 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -7,7 +7,7 @@ ce_provision: # Location of Ansible installation and components. venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv - ansible_install_username: "{{ _ce_provision.username }}" + install_username: "{{ _ce_provision.username }}" # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index ba1be5773..ca59d5928 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -140,7 +140,7 @@ ansible.builtin.set_fact: _venv_path: "{{ ce_provision.venv_path }}" _venv_command: "{{ ce_provision.venv_command }}" - _ansible_install_username: "{{ ce_provision.ansible_install_username }}" + _install_username: "{{ ce_provision.install_username }}" - name: Install Ansible. ansible.builtin.include_role: From b3026765ac2516ef1df8c48783e43ff41bde278d Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 18:00:23 +0100 Subject: [PATCH 012/206] python_boto role also needs the username set in the calling role. --- roles/debian/python_boto/defaults/main.yml | 3 ++- roles/debian/python_boto/tasks/main.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/debian/python_boto/defaults/main.yml b/roles/debian/python_boto/defaults/main.yml index 489e33ef6..69daa3a62 100644 --- a/roles/debian/python_boto/defaults/main.yml +++ b/roles/debian/python_boto/defaults/main.yml @@ -2,4 +2,5 @@ python_boto: boto3_version: "" # version string, e.g. "1.22.13" - empty string means latest # These are usually set within another role using _venv_path and _venv_command but can be overridden. #venv_path: /path/to/venv - #venv_command: /usr/bin/python3.11 -m venv \ No newline at end of file + #venv_command: /usr/bin/python3.11 -m venv + #install_username: deploy # user to become when creating venv diff --git a/roles/debian/python_boto/tasks/main.yml b/roles/debian/python_boto/tasks/main.yml index 8b989f63c..c89cc37e6 100644 --- a/roles/debian/python_boto/tasks/main.yml +++ b/roles/debian/python_boto/tasks/main.yml @@ -15,4 +15,4 @@ virtualenv: "{{ python_boto.venv_path | default(_venv_path) }}" virtualenv_command: "{{ python_boto.venv_command | default(_venv_command) }}" become: true - become_user: "{{ ce_provision.username }}" + become_user: "{{ python_boto.install_username | default(_install_username) }}" From 2fe29a1457002b496e3cf17b341154924d4b2d49 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 10 Jan 2024 18:00:48 +0100 Subject: [PATCH 013/206] Updating python_boto docs. --- docs/roles/debian/python_boto.md | 2 ++ roles/debian/python_boto/README.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/roles/debian/python_boto.md b/docs/roles/debian/python_boto.md index ee3e99a52..268af9614 100644 --- a/docs/roles/debian/python_boto.md +++ b/docs/roles/debian/python_boto.md @@ -12,6 +12,8 @@ python_boto: # These are usually set within another role using _venv_path and _venv_command but can be overridden. #venv_path: /path/to/venv #venv_command: /usr/bin/python3.11 -m venv + #install_username: deploy # user to become when creating venv + ``` diff --git a/roles/debian/python_boto/README.md b/roles/debian/python_boto/README.md index ee3e99a52..268af9614 100644 --- a/roles/debian/python_boto/README.md +++ b/roles/debian/python_boto/README.md @@ -12,6 +12,8 @@ python_boto: # These are usually set within another role using _venv_path and _venv_command but can be overridden. #venv_path: /path/to/venv #venv_command: /usr/bin/python3.11 -m venv + #install_username: deploy # user to become when creating venv + ``` From 32f3ced402d3020afbc2d5aa3fae93a0bfca8729 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 Jan 2024 14:21:52 +0100 Subject: [PATCH 014/206] Making profile.d loading more robust. --- scripts/_common.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index 0da872d6f..a1aae6923 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -34,10 +34,12 @@ if [ ! -d "$ANSIBLE_DATA_DIR" ]; then mkdir "$ANSIBLE_DATA_DIR" fi # Load the contents of profile.d in case we added items to $PATH there. -for f in /etc/profile.d/*; do -# shellcheck source=/dev/null - . "$f" -done +if [ -n "$(ls -A /etc/profile.d)" ]; then + for f in /etc/profile.d/*; do + # shellcheck source=/dev/null + . "$f" + done +fi # Parse options arguments. parse_options(){ while [ "${1:-}" ]; do From 1f8d80e99ee28f40c41d60e976fbbfbf7c262698 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 Jan 2024 14:22:34 +0100 Subject: [PATCH 015/206] Also pip removing ansible-core and trying with pip and pip3 to cover all bases. --- roles/debian/ansible/tasks/main.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 4aac433a9..7ea4f100d 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -4,12 +4,17 @@ pkg: ansible state: absent -- name: Ensure Ansible is not installed globally with pip3 on older versions of Debian. +- name: Ensure Ansible is not installed globally with pip or pip3 on older versions of Debian. ansible.builtin.pip: - name: ansible + name: + - ansible + - ansible-core state: absent - executable: pip3 + executable: "{{ item }}" when: ansible_distribution_major_version | int < 12 + with_items: + - pip + - pip3 failed_when: false # don't stop the build if there's no system pip - name: Set up Python packages. From ff2803f77418a668031bff0b82c52214126dc506 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 Jan 2024 14:23:27 +0100 Subject: [PATCH 016/206] Updating bad AWS SG role var namespacing in other roles. --- .../aws_ec2_autoscale_cluster/tasks/main.yml | 18 ++++++++++-------- roles/aws/aws_ec2_with_eip/tasks/main.yml | 9 +++++---- roles/aws/aws_efs/tasks/main.yml | 9 +++++---- roles/aws/aws_elasticache/tasks/main.yml | 9 +++++---- roles/aws/aws_elb/tasks/main.yml | 9 +++++---- roles/aws/aws_opensearch/tasks/main.yml | 9 +++++---- roles/aws/aws_rds/tasks/main.yml | 9 +++++---- roles/debian/gitlab_runner/tasks/main.yml | 11 ++++++----- 8 files changed, 46 insertions(+), 37 deletions(-) diff --git a/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml b/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml index 70597d699..611ba383c 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml +++ b/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml @@ -379,10 +379,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_ec2_autoscale_cluster.aws_profile }}" - region: "{{ aws_ec2_autoscale_cluster.region }}" - group_names: "{{ aws_ec2_autoscale_cluster.alb_security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_ec2_autoscale_cluster.aws_profile }}" + region: "{{ aws_ec2_autoscale_cluster.region }}" + group_names: "{{ aws_ec2_autoscale_cluster.alb_security_groups }}" + return_type: ids when: - aws_ec2_autoscale_cluster.alb_security_groups | length > 0 - aws_ec2_autoscale_cluster.create_elb @@ -425,10 +426,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_ec2_autoscale_cluster.aws_profile }}" - region: "{{ aws_ec2_autoscale_cluster.region }}" - group_names: "{{ aws_ec2_autoscale_cluster.cluster_security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_ec2_autoscale_cluster.aws_profile }}" + region: "{{ aws_ec2_autoscale_cluster.region }}" + group_names: "{{ aws_ec2_autoscale_cluster.cluster_security_groups }}" + return_type: ids when: - aws_ec2_autoscale_cluster.cluster_security_groups | length > 0 - aws_ec2_autoscale_cluster.asg_refresh diff --git a/roles/aws/aws_ec2_with_eip/tasks/main.yml b/roles/aws/aws_ec2_with_eip/tasks/main.yml index 06aaec917..fea2c4ecb 100644 --- a/roles/aws/aws_ec2_with_eip/tasks/main.yml +++ b/roles/aws/aws_ec2_with_eip/tasks/main.yml @@ -95,10 +95,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_ec2_with_eip.aws_profile }}" - region: "{{ aws_ec2_with_eip.region }}" - group_names: "{{ aws_ec2_with_eip.security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_ec2_with_eip.aws_profile }}" + region: "{{ aws_ec2_with_eip.region }}" + group_names: "{{ aws_ec2_with_eip.security_groups }}" + return_type: ids when: aws_ec2_with_eip.security_groups | length > 0 # Do not create an instance if _aws_hostname is not an EC2 generated address unless `force: true` diff --git a/roles/aws/aws_efs/tasks/main.yml b/roles/aws/aws_efs/tasks/main.yml index 6e07019b5..b7351ebd8 100644 --- a/roles/aws/aws_efs/tasks/main.yml +++ b/roles/aws/aws_efs/tasks/main.yml @@ -3,10 +3,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_efs.aws_profile }}" - region: "{{ aws_efs.region }}" - group_names: "{{ aws_efs.security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_efs.aws_profile }}" + region: "{{ aws_efs.region }}" + group_names: "{{ aws_efs.security_groups }}" + return_type: ids when: aws_efs.security_groups | length > 0 - name: Reset subnet ids info diff --git a/roles/aws/aws_elasticache/tasks/main.yml b/roles/aws/aws_elasticache/tasks/main.yml index 83c64efd8..7c75d36eb 100644 --- a/roles/aws/aws_elasticache/tasks/main.yml +++ b/roles/aws/aws_elasticache/tasks/main.yml @@ -3,10 +3,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_elasticache.aws_profile }}" - region: "{{ aws_elasticache.region }}" - group_names: "{{ aws_elasticache.elasticache_security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_elasticache.aws_profile }}" + region: "{{ aws_elasticache.region }}" + group_names: "{{ aws_elasticache.elasticache_security_groups }}" + return_type: ids when: aws_elasticache.elasticache_security_groups | length > 0 - name: Create ElastiCache subnet group. diff --git a/roles/aws/aws_elb/tasks/main.yml b/roles/aws/aws_elb/tasks/main.yml index 4c4416338..a47f372ad 100644 --- a/roles/aws/aws_elb/tasks/main.yml +++ b/roles/aws/aws_elb/tasks/main.yml @@ -113,10 +113,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_elb.aws_profile }}" - region: "{{ aws_elb.region }}" - group_names: "{{ aws_elb.elb_security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_elb.aws_profile }}" + region: "{{ aws_elb.region }}" + group_names: "{{ aws_elb.elb_security_groups }}" + return_type: ids when: aws_elb.elb_security_groups | length > 0 - name: Create the ALB. diff --git a/roles/aws/aws_opensearch/tasks/main.yml b/roles/aws/aws_opensearch/tasks/main.yml index f43b79bb0..39368148a 100644 --- a/roles/aws/aws_opensearch/tasks/main.yml +++ b/roles/aws/aws_opensearch/tasks/main.yml @@ -4,10 +4,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_opensearch.aws_profile }}" - region: "{{ aws_opensearch.region }}" - group_names: "{{ aws_opensearch.security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_opensearch.aws_profile }}" + region: "{{ aws_opensearch.region }}" + group_names: "{{ aws_opensearch.security_groups }}" + return_type: ids when: aws_opensearch.security_groups | length > 0 - name: Get the current AWS account caller identity information diff --git a/roles/aws/aws_rds/tasks/main.yml b/roles/aws/aws_rds/tasks/main.yml index 416dd30b4..85301323d 100644 --- a/roles/aws/aws_rds/tasks/main.yml +++ b/roles/aws/aws_rds/tasks/main.yml @@ -67,10 +67,11 @@ ansible.builtin.include_role: name: aws/aws_security_groups vars: - profile: "{{ aws_rds.aws_profile }}" - region: "{{ aws_rds.region }}" - group_names: "{{ aws_rds.security_groups }}" - return_type: ids + aws_security_groups: + profile: "{{ aws_rds.aws_profile }}" + region: "{{ aws_rds.region }}" + group_names: "{{ aws_rds.security_groups }}" + return_type: ids when: aws_rds.security_groups | length > 0 - name: Create RDS instance diff --git a/roles/debian/gitlab_runner/tasks/main.yml b/roles/debian/gitlab_runner/tasks/main.yml index 685b0bfc2..47b32ac4c 100644 --- a/roles/debian/gitlab_runner/tasks/main.yml +++ b/roles/debian/gitlab_runner/tasks/main.yml @@ -42,11 +42,12 @@ apply: delegate_to: localhost # this is how you delegate the include_role module vars: - profile: "{{ gitlab_runner.fargate.profile }}" - region: "{{ gitlab_runner.fargate.region }}" - group_names: - - "{{ gitlab_runner.fargate.security_group }}" - return_type: ids + aws_security_groups: + profile: "{{ gitlab_runner.fargate.profile }}" + region: "{{ gitlab_runner.fargate.region }}" + group_names: + - "{{ gitlab_runner.fargate.security_group }}" + return_type: ids when: gitlab_runner.install_fargate - name: Create the Fargate driver directory if it does not exist. From 09663980f70358fe135ff16488218e0b16814da4 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 11 Jan 2024 20:37:14 +0100 Subject: [PATCH 017/206] Refactoring how we handle python3-pip. --- roles/debian/ldap_server/meta/main.yml | 3 --- roles/debian/ldap_server/tasks/main.yml | 13 +++++++++++++ roles/debian/mysql_client/meta/main.yml | 3 --- roles/debian/mysql_client/tasks/main.yml | 13 +++++++++++++ roles/debian/nginx/meta/main.yml | 3 --- roles/debian/nginx/tasks/main.yml | 13 +++++++++++++ roles/debian/python_common/defaults/main.yml | 1 - 7 files changed, 39 insertions(+), 10 deletions(-) delete mode 100644 roles/debian/ldap_server/meta/main.yml delete mode 100644 roles/debian/mysql_client/meta/main.yml delete mode 100644 roles/debian/nginx/meta/main.yml diff --git a/roles/debian/ldap_server/meta/main.yml b/roles/debian/ldap_server/meta/main.yml deleted file mode 100644 index 551567003..000000000 --- a/roles/debian/ldap_server/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: debian/python_common diff --git a/roles/debian/ldap_server/tasks/main.yml b/roles/debian/ldap_server/tasks/main.yml index 9a773da58..bf2c0d3d1 100644 --- a/roles/debian/ldap_server/tasks/main.yml +++ b/roles/debian/ldap_server/tasks/main.yml @@ -14,6 +14,19 @@ purge: true when: ldap_server.slapd.purge +- name: Set up Python packages. + ansible.builtin.include_role: + name: debian/python_common + vars: + python_common: + remove_packages: + - python-pip + - python3-yaml + - python-botocore + - python-urllib3 + install_packages: + - python3-pip + - name: Ensure LDAP and python-ldap and dependencies are installed. ansible.builtin.apt: pkg: ["slapd", "ldapscripts", "libldap2-dev"] diff --git a/roles/debian/mysql_client/meta/main.yml b/roles/debian/mysql_client/meta/main.yml deleted file mode 100644 index 551567003..000000000 --- a/roles/debian/mysql_client/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: debian/python_common diff --git a/roles/debian/mysql_client/tasks/main.yml b/roles/debian/mysql_client/tasks/main.yml index d17123205..2ecca067e 100644 --- a/roles/debian/mysql_client/tasks/main.yml +++ b/roles/debian/mysql_client/tasks/main.yml @@ -6,6 +6,19 @@ update_cache: true cache_valid_time: 240 +- name: Set up Python packages. + ansible.builtin.include_role: + name: debian/python_common + vars: + python_common: + remove_packages: + - python-pip + - python3-yaml + - python-botocore + - python-urllib3 + install_packages: + - python3-pip + - name: Ensure PyMySQL is installed with pip3. ansible.builtin.pip: name: pymysql diff --git a/roles/debian/nginx/meta/main.yml b/roles/debian/nginx/meta/main.yml deleted file mode 100644 index 551567003..000000000 --- a/roles/debian/nginx/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - role: debian/python_common diff --git a/roles/debian/nginx/tasks/main.yml b/roles/debian/nginx/tasks/main.yml index 60a8c4246..0388ed007 100644 --- a/roles/debian/nginx/tasks/main.yml +++ b/roles/debian/nginx/tasks/main.yml @@ -6,6 +6,19 @@ update_cache: true cache_valid_time: 240 +- name: Set up Python packages. + ansible.builtin.include_role: + name: debian/python_common + vars: + python_common: + remove_packages: + - python-pip + - python3-yaml + - python-botocore + - python-urllib3 + install_packages: + - python3-pip + - name: Ensure passlib is installed with pip3. ansible.builtin.pip: name: passlib diff --git a/roles/debian/python_common/defaults/main.yml b/roles/debian/python_common/defaults/main.yml index afb3840df..59c3c7e72 100644 --- a/roles/debian/python_common/defaults/main.yml +++ b/roles/debian/python_common/defaults/main.yml @@ -8,5 +8,4 @@ python_common: install_packages: - python3-distutils - python3-venv - - python3-pip - cloud-init # package can get removed with python3-yaml but we need it for auto-scale \ No newline at end of file From 6b2b69b137c08153e459f323e97f5cd75ba4455e Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 12 Jan 2024 09:42:08 +0100 Subject: [PATCH 018/206] Allow passing in of the Python interpreter to Ansible. --- scripts/_common.sh | 11 ++++++++++- scripts/provision.sh | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index a1aae6923..da5277ef9 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -12,6 +12,7 @@ TARGET_PROVISION_BRANCH="" ANSIBLE_EXTRA_VARS="" ANSIBLE_DEFAULT_EXTRA_VARS="" ANSIBLE_PATH="" +PYTHON_INTERPRETER="" BUILD_WORKSPACE="" BUILD_WORKSPACE_BASE="$OWN_DIR/build" BUILD_ID="" @@ -101,6 +102,10 @@ parse_options(){ shift ANSIBLE_PATH="$1" ;; + "--python-interpreter") + shift + PYTHON_INTERPRETER="$1" + ;; *) usage exit 1 @@ -123,7 +128,11 @@ get_build_workspace(){ # Common extra-vars to pass to Ansible. get_ansible_defaults_vars(){ get_build_id - ANSIBLE_DEFAULT_EXTRA_VARS="{_ce_provision_base_dir: $OWN_DIR, _ce_provision_build_dir: $BUILD_WORKSPACE, _ce_provision_build_tmp_dir: $BUILD_TMP_DIR, _ce_provision_data_dir: $ANSIBLE_DATA_DIR, _ce_provision_build_id: $BUILD_ID, _ce_provision_force_play: $FORCE_PLAY, target_branch: $TARGET_PROVISION_BRANCH}" + if [ -n "$PYTHON_INTERPRETER" ]; then + ANSIBLE_DEFAULT_EXTRA_VARS="{ansible_python_interpreter: $PYTHON_INTERPRETER, _ce_provision_base_dir: $OWN_DIR, _ce_provision_build_dir: $BUILD_WORKSPACE, _ce_provision_build_tmp_dir: $BUILD_TMP_DIR, _ce_provision_data_dir: $ANSIBLE_DATA_DIR, _ce_provision_build_id: $BUILD_ID, _ce_provision_force_play: $FORCE_PLAY, target_branch: $TARGET_PROVISION_BRANCH}" + else + ANSIBLE_DEFAULT_EXTRA_VARS="{_ce_provision_base_dir: $OWN_DIR, _ce_provision_build_dir: $BUILD_WORKSPACE, _ce_provision_build_tmp_dir: $BUILD_TMP_DIR, _ce_provision_data_dir: $ANSIBLE_DATA_DIR, _ce_provision_build_id: $BUILD_ID, _ce_provision_force_play: $FORCE_PLAY, target_branch: $TARGET_PROVISION_BRANCH}" + fi } # Clone our target repo. diff --git a/scripts/provision.sh b/scripts/provision.sh index 92d2e0265..fda465e05 100755 --- a/scripts/provision.sh +++ b/scripts/provision.sh @@ -14,6 +14,7 @@ usage(){ echo 'Available options:' echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.' echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.' + echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.' echo '--workspace: Local existing clone of the repo/branch (if your deployment tool already has one). This will skip the cloning/fetching of the repo.' echo '--user: Linux user executing the script (defaults to controller).' echo '--absolute-playbook-path: Prevent prepending of the workspace path to the playbook path when Ansible is called.' From 3c1d06c9cfdbc7e86a8125966cfc3107ac8e9e0b Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 15 Jan 2024 15:58:21 +0100 Subject: [PATCH 019/206] Updating the packages server for CE. --- roles/debian/pam_linotp/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/debian/pam_linotp/tasks/main.yml b/roles/debian/pam_linotp/tasks/main.yml index d4736a2e6..d7fe822a2 100644 --- a/roles/debian/pam_linotp/tasks/main.yml +++ b/roles/debian/pam_linotp/tasks/main.yml @@ -1,13 +1,13 @@ --- - name: Add key for LinOTP repository. ansible.builtin.apt_key: - keyserver: http://packages.codeenigma.com/debian/codeenigma.pub - id: A344A0826FD987C6 + keyserver: https://packages.codeenigma.net/debian/codeenigma.pub + id: 77AFCA1C45124FBB2EFCD7267DC594C08E0497D4 state: present - name: Add repository for LinOTP. ansible.builtin.apt_repository: - repo: "deb http://packages.codeenigma.com/debian buster main" + repo: "deb https://packages.codeenigma.net/debian {{ ansible_distribution_release }} main" state: present - name: Ensure libpam-linotp is installed. From 2b422a93772edbcf2aa8f5caa67481027ab694bd Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 16 Jan 2024 11:56:42 +0100 Subject: [PATCH 020/206] Installing Ansible in a venv on all machines. --- roles/_meta/common_base/meta/main.yml | 2 ++ roles/_meta/webserver/meta/main.yml | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 95fcebc29..1ff94fc17 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -4,6 +4,7 @@ dependencies: - { role: debian/apt_extra_packages, apt_extra_packages: ["dnsutils", "telnet", "atop", "sysstat"], when: ( is_local is not defined or not is_local ) } - role: debian/locales - role: debian/user_root + - role: debian/user_provision - role: debian/apt_extra_packages - { role: debian/apt_unattended_upgrades, when: ( is_local is not defined or not is_local ) } - { role: debian/hosts, when: ( is_local is not defined or not is_local ) } @@ -12,3 +13,4 @@ dependencies: - { role: debian/clamav, when: ( is_local is not defined or not is_local ) } - { role: debian/firewall_config, when: ( is_local is not defined or not is_local ) } - { role: debian/sudo_config, when: ( is_local is not defined or not is_local ) } + - { role: debian/ansible, ce_ansible: {venv_path: '/home/' + ce_provision.username + '/ansible', venv_command: '/usr/bin/python3.11 -m venv', install_username: ce_provision.username, upgrade: {enabled: true, command: '/home/' + ce_provision.username + '/ansible/bin/python3 -m pip install --upgrade ansible', on_calendar: '*-*-* 01:30:00'}, linters: {enabled: false}} } diff --git a/roles/_meta/webserver/meta/main.yml b/roles/_meta/webserver/meta/main.yml index a66530c03..00249c536 100644 --- a/roles/_meta/webserver/meta/main.yml +++ b/roles/_meta/webserver/meta/main.yml @@ -2,7 +2,6 @@ dependencies: - { role: debian/ssh_server, when: ( is_local is not defined or not is_local ) } - role: _meta/common_base - - role: debian/user_provision - role: debian/user_deploy - role: debian/mysql_client - role: debian/nodejs From 33abf0a173133b414a96f792c405e0d518a8bd72 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 16 Jan 2024 12:17:07 +0100 Subject: [PATCH 021/206] Changing common_base format for readability. --- roles/_meta/common_base/meta/main.yml | 57 ++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 1ff94fc17..140fcf1bb 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -1,16 +1,55 @@ --- dependencies: - - { role: debian/apt_extra_packages, apt_extra_packages: ["rsync", "python3-apt", "acl", "gnupg", "build-essential", "cron", "bsd-mailx", "vim", "unzip", "ca-certificates"] } - - { role: debian/apt_extra_packages, apt_extra_packages: ["dnsutils", "telnet", "atop", "sysstat"], when: ( is_local is not defined or not is_local ) } + - role: debian/apt_extra_packages + apt_extra_packages: + - rsync + - python3-apt + - acl + - gnupg + - build-essential + - cron + - bsd-mailx + - vim + - unzip + - ca-certificates + - role: debian/apt_extra_packages + apt_extra_packages: + - dnsutils + - telnet + - atop + - sysstat + - tcpdump + - mtr-tiny + - htop + - ntp + - screen + - iptraf-ng + when: is_local is not defined or not is_local - role: debian/locales - role: debian/user_root - role: debian/user_provision - role: debian/apt_extra_packages - - { role: debian/apt_unattended_upgrades, when: ( is_local is not defined or not is_local ) } - - { role: debian/hosts, when: ( is_local is not defined or not is_local ) } - - { role: debian/rkhunter, when: ( is_local is not defined or not is_local ) } + - role: debian/apt_unattended_upgrades + when: is_local is not defined or not is_local + - role: debian/hosts + when: is_local is not defined or not is_local + - role: debian/rkhunter + when: is_local is not defined or not is_local - role: debian/postfix - - { role: debian/clamav, when: ( is_local is not defined or not is_local ) } - - { role: debian/firewall_config, when: ( is_local is not defined or not is_local ) } - - { role: debian/sudo_config, when: ( is_local is not defined or not is_local ) } - - { role: debian/ansible, ce_ansible: {venv_path: '/home/' + ce_provision.username + '/ansible', venv_command: '/usr/bin/python3.11 -m venv', install_username: ce_provision.username, upgrade: {enabled: true, command: '/home/' + ce_provision.username + '/ansible/bin/python3 -m pip install --upgrade ansible', on_calendar: '*-*-* 01:30:00'}, linters: {enabled: false}} } + - role: debian/clamav + when: is_local is not defined or not is_local + - role: debian/firewall_config + when: is_local is not defined or not is_local + - role: debian/sudo_config + when: is_local is not defined or not is_local + - role: debian/ansible + ce_ansible: + venv_path: "/home/{{ ce_provision.username }}/ansible" + venv_command: "/usr/bin/python3.11 -m venv" + install_username: "{{ ce_provision.username }}" + upgrade: + enabled: true + command: "/home/{{ ce_provision.username }}/ansible/bin/python3 -m pip install --upgrade ansible" + on_calendar: "*-*-* 01:30:00" + linters: + enabled: false From 394ec01472e16adef55bf5321a16dd8217730ff7 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 17 Jan 2024 19:02:48 +0100 Subject: [PATCH 022/206] No need to specify Python to the point release. --- roles/_meta/common_base/meta/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 140fcf1bb..9e3635fa6 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -45,7 +45,7 @@ dependencies: - role: debian/ansible ce_ansible: venv_path: "/home/{{ ce_provision.username }}/ansible" - venv_command: "/usr/bin/python3.11 -m venv" + venv_command: "/usr/bin/python3 -m venv" install_username: "{{ ce_provision.username }}" upgrade: enabled: true From f906530099ea48eb6ce0149c5ac319800ffc42bd Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 17 Jan 2024 19:08:34 +0100 Subject: [PATCH 023/206] Docs update. --- docs/_Sidebar.md | 2 +- docs/roles/debian/python_common.md | 14 ++++++++------ docs/roles/debian/swap.md | 7 +++---- roles/debian/python_common/README.md | 14 ++++++++------ roles/debian/swap/README.md | 7 ++----- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md index aa59208c8..5a5213a0b 100644 --- a/docs/_Sidebar.md +++ b/docs/_Sidebar.md @@ -76,7 +76,7 @@ - [SSHD](/roles/debian/ssh_server) - [SSL](/roles/debian/ssl) - [sudo config](/roles/debian/sudo_config) - - [General swap role to add swap space](/roles/debian/swap) + - [Swap](/roles/debian/swap) - [varnish-config](/roles/debian/varnish_config) - [wazuh](/roles/debian/wazuh) - [Init role](/roles/_init) diff --git a/docs/roles/debian/python_common.md b/docs/roles/debian/python_common.md index 2c9f384f9..94573f816 100644 --- a/docs/roles/debian/python_common.md +++ b/docs/roles/debian/python_common.md @@ -9,15 +9,17 @@ Packages required by all Python scripts and applications. ```yaml --- python_common: - remove_packages: - - python-pip - - python3-yaml # linters need a newer version of PyYAML than the one that ships with Debian - - python-botocore - - python-urllib3 + cleanup: false # set to true to remove unwanted packages + remove_packages: [] # list of packages to remove if cleanup: true + # Example from ansible role + #- python-pip + #- python3-yaml # linters need a newer version of PyYAML than the one that ships with Debian + #- python-botocore + #- python-urllib3 + # Commonly required Python system packages install_packages: - python3-distutils - python3-venv - - python3-pip - cloud-init # package can get removed with python3-yaml but we need it for auto-scale ``` diff --git a/docs/roles/debian/swap.md b/docs/roles/debian/swap.md index 8bfeaa59c..ef6f5f208 100644 --- a/docs/roles/debian/swap.md +++ b/docs/roles/debian/swap.md @@ -1,6 +1,5 @@ -# General swap role to add swap space - -By default, role is set to add 2G of swap space. +# Swap +General role to add swap space to servers. By default, role is set to add 4G of swap space. As a general rule of thumb, swap should be set to 2.5 times the amount of RAM. So if a server has 2G RAM, we should set swap to 5G by setting `size: "5G"` in the variables for that server. @@ -11,7 +10,7 @@ By default, role is set to add 2G of swap space. --- swap: count: 2048 - size: "2G" + size: "4G" swap_space: swapfile ``` diff --git a/roles/debian/python_common/README.md b/roles/debian/python_common/README.md index 2c9f384f9..94573f816 100644 --- a/roles/debian/python_common/README.md +++ b/roles/debian/python_common/README.md @@ -9,15 +9,17 @@ Packages required by all Python scripts and applications. ```yaml --- python_common: - remove_packages: - - python-pip - - python3-yaml # linters need a newer version of PyYAML than the one that ships with Debian - - python-botocore - - python-urllib3 + cleanup: false # set to true to remove unwanted packages + remove_packages: [] # list of packages to remove if cleanup: true + # Example from ansible role + #- python-pip + #- python3-yaml # linters need a newer version of PyYAML than the one that ships with Debian + #- python-botocore + #- python-urllib3 + # Commonly required Python system packages install_packages: - python3-distutils - python3-venv - - python3-pip - cloud-init # package can get removed with python3-yaml but we need it for auto-scale ``` diff --git a/roles/debian/swap/README.md b/roles/debian/swap/README.md index ea66bc066..ef6f5f208 100644 --- a/roles/debian/swap/README.md +++ b/roles/debian/swap/README.md @@ -1,8 +1,5 @@ -# General swap role to add swap space - -By default, role is set to add 2G of swap space. -As a general rule of thumb, swap should be set to 2.5x of current RAM. -If server has 2G RAM, set RAM to 5G by creating swap.yml inside vars at local directory. +# Swap +General role to add swap space to servers. By default, role is set to add 4G of swap space. As a general rule of thumb, swap should be set to 2.5 times the amount of RAM. So if a server has 2G RAM, we should set swap to 5G by setting `size: "5G"` in the variables for that server. From 9b6395403bbe2bdbffa3566ba620ee97bad895e1 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 18 Jan 2024 19:39:50 +0100 Subject: [PATCH 024/206] Fixing LDAP SSL to use systemd timer. --- docs/roles/debian/ldap_server.md | 1 + roles/debian/ldap_server/README.md | 1 + roles/debian/ldap_server/defaults/main.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/docs/roles/debian/ldap_server.md b/docs/roles/debian/ldap_server.md index 7fb78adfb..6514e4779 100644 --- a/docs/roles/debian/ldap_server.md +++ b/docs/roles/debian/ldap_server.md @@ -107,6 +107,7 @@ ldap_server: # reload_command: restart # reload: # - slapd + # on_calendar: "Mon *-*-* 04:00:00" replication: host: "" # host must be present in config/hosts for ce-provision, leave empty if no replication is desired port: "636" diff --git a/roles/debian/ldap_server/README.md b/roles/debian/ldap_server/README.md index 7fb78adfb..6514e4779 100644 --- a/roles/debian/ldap_server/README.md +++ b/roles/debian/ldap_server/README.md @@ -107,6 +107,7 @@ ldap_server: # reload_command: restart # reload: # - slapd + # on_calendar: "Mon *-*-* 04:00:00" replication: host: "" # host must be present in config/hosts for ce-provision, leave empty if no replication is desired port: "636" diff --git a/roles/debian/ldap_server/defaults/main.yml b/roles/debian/ldap_server/defaults/main.yml index 981d6b995..8178688a0 100644 --- a/roles/debian/ldap_server/defaults/main.yml +++ b/roles/debian/ldap_server/defaults/main.yml @@ -41,6 +41,7 @@ ldap_server: # reload_command: restart # reload: # - slapd + # on_calendar: "Mon *-*-* 04:00:00" replication: host: "" # host must be present in config/hosts for ce-provision, leave empty if no replication is desired port: "636" From 516f61d9ea0dde476856e6b216f75b1140bfac4e Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 18 Jan 2024 19:40:07 +0100 Subject: [PATCH 025/206] Allowing different systemd timer names for different Ansible installs. --- docs/roles/debian/ansible.md | 1 + docs/roles/debian/ce_deploy.md | 1 + docs/roles/debian/ce_provision.md | 1 + roles/_meta/common_base/meta/main.yml | 1 + roles/debian/ansible/README.md | 1 + roles/debian/ansible/defaults/main.yml | 1 + roles/debian/ansible/tasks/main.yml | 9 +++++---- roles/debian/ce_deploy/README.md | 1 + roles/debian/ce_deploy/defaults/main.yml | 1 + roles/debian/ce_deploy/tasks/main.yml | 1 + roles/debian/ce_provision/README.md | 1 + roles/debian/ce_provision/defaults/main.yml | 1 + roles/debian/ce_provision/tasks/main.yml | 1 + 13 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/roles/debian/ansible.md b/docs/roles/debian/ansible.md index b7dfe0dbc..f50de4929 100644 --- a/docs/roles/debian/ansible.md +++ b/docs/roles/debian/ansible.md @@ -19,6 +19,7 @@ ce_ansible: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too on_calendar: "*-*-* 01:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + #timer_name: upgrade_ansible linters: enabled: true # will not install linters if false, installing linters breaks cloud-init diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index d020663ab..f9e55b15f 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -15,6 +15,7 @@ ce_deploy: venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_deploy.username }}" + upgrade_timer_name: upgrade_ce_deploy_ansible # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index e8f83ea13..dba324120 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -16,6 +16,7 @@ ce_provision: venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_provision.username }}" + upgrade_timer_name: upgrade_ce_provision_ansible # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 9e3635fa6..7a79993e8 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -51,5 +51,6 @@ dependencies: enabled: true command: "/home/{{ ce_provision.username }}/ansible/bin/python3 -m pip install --upgrade ansible" on_calendar: "*-*-* 01:30:00" + timer_name: upgrade_ansible linters: enabled: false diff --git a/roles/debian/ansible/README.md b/roles/debian/ansible/README.md index b7dfe0dbc..f50de4929 100644 --- a/roles/debian/ansible/README.md +++ b/roles/debian/ansible/README.md @@ -19,6 +19,7 @@ ce_ansible: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too on_calendar: "*-*-* 01:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + #timer_name: upgrade_ansible linters: enabled: true # will not install linters if false, installing linters breaks cloud-init diff --git a/roles/debian/ansible/defaults/main.yml b/roles/debian/ansible/defaults/main.yml index f58a026b0..949bb48b7 100644 --- a/roles/debian/ansible/defaults/main.yml +++ b/roles/debian/ansible/defaults/main.yml @@ -8,5 +8,6 @@ ce_ansible: enabled: true # create systemd timer to auto-upgrade Ansible command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" # if you set venv_path above then set it here too on_calendar: "*-*-* 01:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + #timer_name: upgrade_ansible linters: enabled: true # will not install linters if false, installing linters breaks cloud-init diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 56d1d94f2..b1aa47a7f 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -68,12 +68,13 @@ content: "export PATH=$PATH:{{ ce_ansible.venv_path | default(_venv_path) }}/bin" dest: "/etc/profile.d/ansible-path.sh" +- name: Build systemd timer variables. + ansible.builtin.set_fact: + _timer: {"{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}": { timer_command: "{{ ce_ansible.upgrade.command }}", timer_OnCalendar: "{{ ce_ansible.upgrade.on_calendar }}" }} + - name: Create systemd timer to upgrade Ansible. ansible.builtin.include_role: name: contrib/systemd_timers vars: - timers: - upgrade_ansible: - timer_command: "{{ ce_ansible.upgrade.command }}" - timer_OnCalendar: "{{ ce_ansible.upgrade.on_calendar }}" + timers: "{{ _timer }}" when: ce_ansible.upgrade.enabled diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index d020663ab..f9e55b15f 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -15,6 +15,7 @@ ce_deploy: venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_deploy.username }}" + upgrade_timer_name: upgrade_ce_deploy_ansible # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 52bd86415..1dc9ba5b3 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -7,6 +7,7 @@ ce_deploy: venv_path: "/home/{{ _ce_deploy.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_deploy.username }}" + upgrade_timer_name: upgrade_ce_deploy_ansible # Other ce-deploy settings. new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user key_name: id_rsa.pub # existing users may have a key of a different name diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index a9d607723..ae5395780 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -133,6 +133,7 @@ _venv_path: "{{ ce_deploy.venv_path }}" _venv_command: "{{ ce_deploy.venv_command }}" _install_username: "{{ ce_deploy.install_username }}" + _ce_ansible_timer_name: "{{ ce_deploy.upgrade_timer_name }}" - name: Install Ansible. ansible.builtin.include_role: diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index e8f83ea13..dba324120 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -16,6 +16,7 @@ ce_provision: venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_provision.username }}" + upgrade_timer_name: upgrade_ce_provision_ansible # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index 4c266e3e0..8f8d508cf 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -8,6 +8,7 @@ ce_provision: venv_path: "/home/{{ _ce_provision.username }}/ansible" venv_command: /usr/bin/python3 -m venv install_username: "{{ _ce_provision.username }}" + upgrade_timer_name: upgrade_ce_provision_ansible # Other ce-provision settings. username: "{{ _ce_provision.username }}" new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index ca59d5928..0f4edeab1 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -141,6 +141,7 @@ _venv_path: "{{ ce_provision.venv_path }}" _venv_command: "{{ ce_provision.venv_command }}" _install_username: "{{ ce_provision.install_username }}" + _ce_ansible_timer_name: "{{ ce_provision.upgrade_timer_name }}" - name: Install Ansible. ansible.builtin.include_role: From 17f7804c427e1b3785b30a72c2f9a1762f3ebae9 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 19 Jan 2024 10:29:40 +0100 Subject: [PATCH 026/206] Fixing dynamic key name in ansible role. --- roles/debian/ansible/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index b1aa47a7f..8cbdc9241 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -70,7 +70,7 @@ - name: Build systemd timer variables. ansible.builtin.set_fact: - _timer: {"{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}": { timer_command: "{{ ce_ansible.upgrade.command }}", timer_OnCalendar: "{{ ce_ansible.upgrade.on_calendar }}" }} + _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" - name: Create systemd timer to upgrade Ansible. ansible.builtin.include_role: From 3f9f67e2ecb40cabb45fdabd7f3be5af5cdbe15a Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 19 Jan 2024 10:49:26 +0100 Subject: [PATCH 027/206] Trying to debug missing timer_command var. --- roles/debian/ansible/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 8cbdc9241..5ed23549e 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -72,6 +72,9 @@ ansible.builtin.set_fact: _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" +- debug: + msg: "{{ _timer }}" + - name: Create systemd timer to upgrade Ansible. ansible.builtin.include_role: name: contrib/systemd_timers From e34d247f22be1540e08717b05131def007bcf776 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 19 Jan 2024 11:07:08 +0100 Subject: [PATCH 028/206] Treating the timer string so it becomes a dict. --- roles/debian/ansible/tasks/main.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 5ed23549e..a34b6d0fe 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -68,12 +68,13 @@ content: "export PATH=$PATH:{{ ce_ansible.venv_path | default(_venv_path) }}/bin" dest: "/etc/profile.d/ansible-path.sh" -- name: Build systemd timer variables. +- name: Build systemd timer variables string. ansible.builtin.set_fact: _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" -- debug: - msg: "{{ _timer }}" +- name: Turn the timer string into a dictionary. + ansible.builtin.set_fact: + _timer: "{{ _timer.splitlines()|map('from_yaml')|list }}" - name: Create systemd timer to upgrade Ansible. ansible.builtin.include_role: From 170c782caf1f483f28bacd8f5f5184cacf16a831 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 22 Jan 2024 12:22:45 +0100 Subject: [PATCH 029/206] Moving default log location for clamav. --- roles/debian/clamav/defaults/main.yml | 3 ++- roles/debian/clamav/templates/clamscan.j2 | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/debian/clamav/defaults/main.yml b/roles/debian/clamav/defaults/main.yml index 651cafde3..6185d71fa 100644 --- a/roles/debian/clamav/defaults/main.yml +++ b/roles/debian/clamav/defaults/main.yml @@ -4,7 +4,8 @@ clamav: server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. on_calendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events scripts_location: /usr/local/clamav/script - log_location: /usr/local/clamav/log + log_location: /var/log/clamav + log_name: clamav.log send_mail: false # Important: will not send any emails by default. send_on_fail: true # Only sends emails on scan failure, will not email for successful scans. report_recipient_email: mail@example.com diff --git a/roles/debian/clamav/templates/clamscan.j2 b/roles/debian/clamav/templates/clamscan.j2 index 0142a1b8b..c2b54e2cb 100644 --- a/roles/debian/clamav/templates/clamscan.j2 +++ b/roles/debian/clamav/templates/clamscan.j2 @@ -1,7 +1,7 @@ #!/bin/bash SUBJECT="{{ clamav.server_name }} PASSED SCAN" EMAIL="{{ clamav.report_recipient_email }}" -LOG={{ clamav.log_location }}/clamav.log +LOG={{ clamav.log_location }}/{{ clamav.log_name }} TMP_LOG=/tmp/clam.daily rm -rf ${TMP_LOG} From 21f882b8376ee1af0f77247ed9121648c62e7e6d Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 22 Jan 2024 12:23:47 +0100 Subject: [PATCH 030/206] Updating ClamAV docs. --- docs/roles/debian/clamav.md | 3 ++- roles/debian/clamav/README.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/roles/debian/clamav.md b/docs/roles/debian/clamav.md index 81a888707..8ce643b7e 100644 --- a/docs/roles/debian/clamav.md +++ b/docs/roles/debian/clamav.md @@ -15,7 +15,8 @@ clamav: server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. on_calendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events scripts_location: /usr/local/clamav/script - log_location: /usr/local/clamav/log + log_location: /var/log/clamav + log_name: clamav.log send_mail: false # Important: will not send any emails by default. send_on_fail: true # Only sends emails on scan failure, will not email for successful scans. report_recipient_email: mail@example.com diff --git a/roles/debian/clamav/README.md b/roles/debian/clamav/README.md index 81a888707..8ce643b7e 100644 --- a/roles/debian/clamav/README.md +++ b/roles/debian/clamav/README.md @@ -15,7 +15,8 @@ clamav: server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. on_calendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events scripts_location: /usr/local/clamav/script - log_location: /usr/local/clamav/log + log_location: /var/log/clamav + log_name: clamav.log send_mail: false # Important: will not send any emails by default. send_on_fail: true # Only sends emails on scan failure, will not email for successful scans. report_recipient_email: mail@example.com From a57c5ffa6862ec3daa56e3cca95ce639d7ee9b4e Mon Sep 17 00:00:00 2001 From: gregharvey Date: Thu, 25 Jan 2024 13:32:37 +0100 Subject: [PATCH 031/206] Grouping systemd timer tasks together. --- roles/debian/ansible/tasks/main.yml | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index b845f2567..f14197d28 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -68,17 +68,19 @@ content: "export PATH=$PATH:{{ ce_ansible.venv_path | default(_venv_path) }}/bin" dest: "/etc/profile.d/ansible-path.sh" -- name: Build systemd timer variables string. - ansible.builtin.set_fact: - _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" +- name: Install systemd timer. + when: ce_ansible.upgrade.enabled + block: + - name: Build systemd timer variables string. + ansible.builtin.set_fact: + _timer: "{'{{ ce_ansible.upgrade.timer_name | default(_ce_ansible_timer_name) }}': { timer_command: '{{ ce_ansible.upgrade.command }}', timer_OnCalendar: '{{ ce_ansible.upgrade.on_calendar }}' }}" -- name: Turn the timer string into a dictionary. - ansible.builtin.set_fact: - _timer: "{{ _timer.splitlines()|map('from_yaml')|list }}" + - name: Turn the timer string into a dictionary. + ansible.builtin.set_fact: + _timer: "{{ _timer.splitlines()|map('from_yaml')|list }}" -- name: Create systemd timer to upgrade Ansible. - ansible.builtin.include_role: - name: contrib/systemd_timers - vars: - timers: "{{ _timer }}" - when: ce_ansible.upgrade.enabled + - name: Create systemd timer to upgrade Ansible. + ansible.builtin.include_role: + name: contrib/systemd_timers + vars: + timers: "{{ _timer }}" From 36a52eeb2cbea9e18d66a43b1d88e692c8dfd842 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Wed, 21 Feb 2024 16:47:33 +0100 Subject: [PATCH 032/206] Exposing ce-provision version in build output. --- roles/_init/defaults/main.yml | 1 + roles/_init/tasks/main.yml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 01249906a..7b7c06560 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -4,6 +4,7 @@ _init: # This is used to detect if the playbook must re-run or not. vars_dirs: [] force_play: false + ce_provision_version: 2.x # Outputted by the _init role at the start of plays. # Although these variables logically belong with ce_provision, the _init role needs to # gather the extra variables if there are any, so there are _init variables. diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index 39c368378..357f94c5e 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -1,4 +1,8 @@ --- +- name: Version check. + ansible.builtin.debug: + msg: "Using ce-provision {{ _init.ce_provision_version }}" + - name: Check OS family. when: ansible_os_family != "Debian" block: From a02e9e5fbade7ddf56ec3d8d73a3df062fb08e84 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 23 Feb 2024 18:24:49 +0100 Subject: [PATCH 033/206] Wrong variable in meta role for controller username. --- roles/_meta/common_base/meta/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 7a79993e8..70c522c53 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -44,12 +44,12 @@ dependencies: when: is_local is not defined or not is_local - role: debian/ansible ce_ansible: - venv_path: "/home/{{ ce_provision.username }}/ansible" + venv_path: "/home/{{ user_provision.username }}/ansible" venv_command: "/usr/bin/python3 -m venv" - install_username: "{{ ce_provision.username }}" + install_username: "{{ user_provision.username }}" upgrade: enabled: true - command: "/home/{{ ce_provision.username }}/ansible/bin/python3 -m pip install --upgrade ansible" + command: "/home/{{ user_provision.username }}/ansible/bin/python3 -m pip install --upgrade ansible" on_calendar: "*-*-* 01:30:00" timer_name: upgrade_ansible linters: From 206b36e1758dcb8f06195e8ac514e6bc67edccab Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 13 May 2024 17:17:17 +0200 Subject: [PATCH 034/206] Removing any reference to _aws variables in debian role defaults. --- docs/roles/_init.md | 5 +++++ docs/roles/debian/aws_cloudwatch_agent.md | 6 +++--- docs/roles/debian/aws_efs_client.md | 4 ++-- docs/roles/debian/duplicity.md | 2 +- docs/roles/debian/gitlab_runner.md | 4 ++-- roles/_init/README.md | 5 +++++ roles/_init/defaults/main.yml | 5 +++++ roles/debian/aws_cloudwatch_agent/README.md | 6 +++--- roles/debian/aws_cloudwatch_agent/defaults/main.yml | 6 +++--- roles/debian/aws_efs_client/README.md | 4 ++-- roles/debian/aws_efs_client/defaults/main.yml | 4 ++-- roles/debian/duplicity/README.md | 2 +- roles/debian/duplicity/defaults/main.yml | 2 +- roles/debian/gitlab_runner/README.md | 4 ++-- roles/debian/gitlab_runner/defaults/main.yml | 4 ++-- 15 files changed, 39 insertions(+), 24 deletions(-) diff --git a/docs/roles/_init.md b/docs/roles/_init.md index 77feacb1f..4abd58990 100644 --- a/docs/roles/_init.md +++ b/docs/roles/_init.md @@ -15,6 +15,11 @@ _venv_command: /usr/bin/python3 -m venv _venv_install_username: "{{ _ce_provision_username }}" _ce_ansible_timer_name: upgrade_ce_provision_ansible +# AWS variables - if you are using an AWS account, you can preset certain variables +# Generally it is recommended to place these in your ce-provision-config repository under hosts/group_vars/all +#_aws_profile: example # boto profile name +#_aws_region: eu-west-1 + _init: # A list of var directories to include. We only support .yml extensions. # This is used to detect if the playbook must re-run or not. diff --git a/docs/roles/debian/aws_cloudwatch_agent.md b/docs/roles/debian/aws_cloudwatch_agent.md index ee37572c7..cdda9f9b5 100644 --- a/docs/roles/debian/aws_cloudwatch_agent.md +++ b/docs/roles/debian/aws_cloudwatch_agent.md @@ -14,9 +14,9 @@ module in your playbooks. --- aws_cloudwatch_agent: # Stream name. - log_stream_name: "example" + log_stream_name: example # Namespace for metrics. Leave empty to use the default CWAgent. - metrics_namespace: "example" + metrics_namespace: example # Group prefix. Useful for grouping by environments. # Eg. instead of "syslog", you can have "dev syslog", "prod syslog", etc. log_group_prefix: "" @@ -26,7 +26,7 @@ aws_cloudwatch_agent: credentials: aws_access_key_id: XXX aws_secret_access_key: XXX - region: "{{ _aws_region }}" + region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set ``` diff --git a/docs/roles/debian/aws_efs_client.md b/docs/roles/debian/aws_efs_client.md index 6d3cd392d..856fe71ca 100644 --- a/docs/roles/debian/aws_efs_client.md +++ b/docs/roles/debian/aws_efs_client.md @@ -44,8 +44,8 @@ In the output of listed file systems, for each one you will find an entry like t _mount_opts: "_netdev,noresvport" # _netdev tells OS to wait for network before attempting to mount _mount_state: present aws_efs_client: - aws_profile: "{{ _aws_profile }}" - region: "{{ _aws_region }}" + aws_profile: example # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set + region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set version: 1.35.0 # version of AWS EFS utils to use build_suffix: "-1_all" # sometimes there is a suffix appended to the package name, e.g. `amazon-efs-utils-1.35.0-1_all.deb` deb_url: "" # provide an alternative location for the .deb package diff --git a/docs/roles/debian/duplicity.md b/docs/roles/debian/duplicity.md index 7e4911527..c9140007e 100644 --- a/docs/roles/debian/duplicity.md +++ b/docs/roles/debian/duplicity.md @@ -17,7 +17,7 @@ duplicity: backend: s3 # currently also support b2 for Backblaze access_key_id: "somekey" secret_access_key: "somesecret" - backend_url: "s3-eu-west-1.amazonaws.com" + backend_url: "s3-eu-west-1.amazonaws.com" # "eu-west-1" can be substituted for "{{ _aws_region }}" if set s3_options: "--s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options bucketname: "somebucket" dirs: diff --git a/docs/roles/debian/gitlab_runner.md b/docs/roles/debian/gitlab_runner.md index 2804320b5..7311c5bf0 100644 --- a/docs/roles/debian/gitlab_runner.md +++ b/docs/roles/debian/gitlab_runner.md @@ -49,8 +49,8 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name - profile: "{{ _aws_profile }}" - region: "eu-west-1" # AWS region name + profile: "example" # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set + region: "eu-west-1" # AWS region name - can be substituted for "{{ _aws_region }}" if set subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name task_definition: "my-task:1" # task definition in format name:revision, if revision is not provided ECS will use latest diff --git a/roles/_init/README.md b/roles/_init/README.md index 77feacb1f..4abd58990 100644 --- a/roles/_init/README.md +++ b/roles/_init/README.md @@ -15,6 +15,11 @@ _venv_command: /usr/bin/python3 -m venv _venv_install_username: "{{ _ce_provision_username }}" _ce_ansible_timer_name: upgrade_ce_provision_ansible +# AWS variables - if you are using an AWS account, you can preset certain variables +# Generally it is recommended to place these in your ce-provision-config repository under hosts/group_vars/all +#_aws_profile: example # boto profile name +#_aws_region: eu-west-1 + _init: # A list of var directories to include. We only support .yml extensions. # This is used to detect if the playbook must re-run or not. diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index d05a6fc78..5e38419ea 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -5,6 +5,11 @@ _venv_command: /usr/bin/python3 -m venv _venv_install_username: "{{ _ce_provision_username }}" _ce_ansible_timer_name: upgrade_ce_provision_ansible +# AWS variables - if you are using an AWS account, you can preset certain variables +# Generally it is recommended to place these in your ce-provision-config repository under hosts/group_vars/all +#_aws_profile: example # boto profile name +#_aws_region: eu-west-1 + _init: # A list of var directories to include. We only support .yml extensions. # This is used to detect if the playbook must re-run or not. diff --git a/roles/debian/aws_cloudwatch_agent/README.md b/roles/debian/aws_cloudwatch_agent/README.md index ee37572c7..cdda9f9b5 100644 --- a/roles/debian/aws_cloudwatch_agent/README.md +++ b/roles/debian/aws_cloudwatch_agent/README.md @@ -14,9 +14,9 @@ module in your playbooks. --- aws_cloudwatch_agent: # Stream name. - log_stream_name: "example" + log_stream_name: example # Namespace for metrics. Leave empty to use the default CWAgent. - metrics_namespace: "example" + metrics_namespace: example # Group prefix. Useful for grouping by environments. # Eg. instead of "syslog", you can have "dev syslog", "prod syslog", etc. log_group_prefix: "" @@ -26,7 +26,7 @@ aws_cloudwatch_agent: credentials: aws_access_key_id: XXX aws_secret_access_key: XXX - region: "{{ _aws_region }}" + region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set ``` diff --git a/roles/debian/aws_cloudwatch_agent/defaults/main.yml b/roles/debian/aws_cloudwatch_agent/defaults/main.yml index f73385114..36d3d3751 100644 --- a/roles/debian/aws_cloudwatch_agent/defaults/main.yml +++ b/roles/debian/aws_cloudwatch_agent/defaults/main.yml @@ -1,9 +1,9 @@ --- aws_cloudwatch_agent: # Stream name. - log_stream_name: "example" + log_stream_name: example # Namespace for metrics. Leave empty to use the default CWAgent. - metrics_namespace: "example" + metrics_namespace: example # Group prefix. Useful for grouping by environments. # Eg. instead of "syslog", you can have "dev syslog", "prod syslog", etc. log_group_prefix: "" @@ -13,4 +13,4 @@ aws_cloudwatch_agent: credentials: aws_access_key_id: XXX aws_secret_access_key: XXX - region: "{{ _aws_region }}" + region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set diff --git a/roles/debian/aws_efs_client/README.md b/roles/debian/aws_efs_client/README.md index 6d3cd392d..856fe71ca 100644 --- a/roles/debian/aws_efs_client/README.md +++ b/roles/debian/aws_efs_client/README.md @@ -44,8 +44,8 @@ In the output of listed file systems, for each one you will find an entry like t _mount_opts: "_netdev,noresvport" # _netdev tells OS to wait for network before attempting to mount _mount_state: present aws_efs_client: - aws_profile: "{{ _aws_profile }}" - region: "{{ _aws_region }}" + aws_profile: example # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set + region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set version: 1.35.0 # version of AWS EFS utils to use build_suffix: "-1_all" # sometimes there is a suffix appended to the package name, e.g. `amazon-efs-utils-1.35.0-1_all.deb` deb_url: "" # provide an alternative location for the .deb package diff --git a/roles/debian/aws_efs_client/defaults/main.yml b/roles/debian/aws_efs_client/defaults/main.yml index ba81a7629..7758d8820 100644 --- a/roles/debian/aws_efs_client/defaults/main.yml +++ b/roles/debian/aws_efs_client/defaults/main.yml @@ -2,8 +2,8 @@ _mount_opts: "_netdev,noresvport" # _netdev tells OS to wait for network before attempting to mount _mount_state: present aws_efs_client: - aws_profile: "{{ _aws_profile }}" - region: "{{ _aws_region }}" + aws_profile: example # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set + region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set version: 1.35.0 # version of AWS EFS utils to use build_suffix: "-1_all" # sometimes there is a suffix appended to the package name, e.g. `amazon-efs-utils-1.35.0-1_all.deb` deb_url: "" # provide an alternative location for the .deb package diff --git a/roles/debian/duplicity/README.md b/roles/debian/duplicity/README.md index 7e4911527..c9140007e 100644 --- a/roles/debian/duplicity/README.md +++ b/roles/debian/duplicity/README.md @@ -17,7 +17,7 @@ duplicity: backend: s3 # currently also support b2 for Backblaze access_key_id: "somekey" secret_access_key: "somesecret" - backend_url: "s3-eu-west-1.amazonaws.com" + backend_url: "s3-eu-west-1.amazonaws.com" # "eu-west-1" can be substituted for "{{ _aws_region }}" if set s3_options: "--s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options bucketname: "somebucket" dirs: diff --git a/roles/debian/duplicity/defaults/main.yml b/roles/debian/duplicity/defaults/main.yml index 6d7a6f3d6..33cecfec2 100644 --- a/roles/debian/duplicity/defaults/main.yml +++ b/roles/debian/duplicity/defaults/main.yml @@ -8,7 +8,7 @@ duplicity: backend: s3 # currently also support b2 for Backblaze access_key_id: "somekey" secret_access_key: "somesecret" - backend_url: "s3-eu-west-1.amazonaws.com" + backend_url: "s3-eu-west-1.amazonaws.com" # "eu-west-1" can be substituted for "{{ _aws_region }}" if set s3_options: "--s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options bucketname: "somebucket" dirs: diff --git a/roles/debian/gitlab_runner/README.md b/roles/debian/gitlab_runner/README.md index 2804320b5..7311c5bf0 100644 --- a/roles/debian/gitlab_runner/README.md +++ b/roles/debian/gitlab_runner/README.md @@ -49,8 +49,8 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name - profile: "{{ _aws_profile }}" - region: "eu-west-1" # AWS region name + profile: "example" # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set + region: "eu-west-1" # AWS region name - can be substituted for "{{ _aws_region }}" if set subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name task_definition: "my-task:1" # task definition in format name:revision, if revision is not provided ECS will use latest diff --git a/roles/debian/gitlab_runner/defaults/main.yml b/roles/debian/gitlab_runner/defaults/main.yml index af891d799..1813aa89b 100644 --- a/roles/debian/gitlab_runner/defaults/main.yml +++ b/roles/debian/gitlab_runner/defaults/main.yml @@ -37,8 +37,8 @@ gitlab_runner: # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs fargate: cluster: "my-cluster" # ECS cluster name - profile: "{{ _aws_profile }}" - region: "eu-west-1" # AWS region name + profile: "example" # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set + region: "eu-west-1" # AWS region name - can be substituted for "{{ _aws_region }}" if set subnet: "subnet-abcdef123456" # subnet ID security_group: "my-security-group" # SG name task_definition: "my-task:1" # task definition in format name:revision, if revision is not provided ECS will use latest From 8ee9e69dba20735d34cba2052df4d4fa22a112dd Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 14 May 2024 16:27:06 +0200 Subject: [PATCH 035/206] Setting more sane ASG defaults. --- docs/roles/aws/aws_ec2_autoscale_cluster.md | 12 ++++++------ roles/aws/aws_ec2_autoscale_cluster/README.md | 12 ++++++------ .../aws/aws_ec2_autoscale_cluster/defaults/main.yml | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/roles/aws/aws_ec2_autoscale_cluster.md b/docs/roles/aws/aws_ec2_autoscale_cluster.md index 6278a17d0..ebda2b7a0 100644 --- a/docs/roles/aws/aws_ec2_autoscale_cluster.md +++ b/docs/roles/aws/aws_ec2_autoscale_cluster.md @@ -106,15 +106,15 @@ aws_ec2_autoscale_cluster: - name: "{{ _env_type }}-scale-up-policy" policy_type: "SimpleScaling" adjustment_type: "ChangeInCapacity" - adjustment: 2 + adjustment: 2 # Add two servers per scaling event adjustment_step: 1 # Only used when adjustment_type is PercentChangeInCapacity. - cooldown: 300 + cooldown: 120 - name: "{{ _env_type }}-scale-down-policy" policy_type: "SimpleScaling" adjustment_type: "ChangeInCapacity" - adjustment: -2 + adjustment: -1 # Reduce by one server at a time adjustment_step: -1 # Only used when adjustment_type is PercentChangeInCapacity. - cooldown: 300 + cooldown: 120 asg_cloudwatch_alarm_scale_up_name: "{{ _env_type }}-cloudwatch-metric-alarm-cpu-scale-up" asg_cloudwatch_alarm_scale_down_name: "{{ _env_type }}-cloudwatch-metric-alarm-cpu-scale-down" asg_cloudwatch_alarms: @@ -126,8 +126,8 @@ aws_ec2_autoscale_cluster: threshold: 80 unit: "Percent" comparison: "GreaterThanOrEqualToThreshold" - period: 120 - evaluation_periods: 5 + period: 30 + evaluation_periods: 3 - scale_direction: "down" description: "CPU under 40% so scale down." metric: "CPUUtilization" diff --git a/roles/aws/aws_ec2_autoscale_cluster/README.md b/roles/aws/aws_ec2_autoscale_cluster/README.md index 6278a17d0..ebda2b7a0 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/README.md +++ b/roles/aws/aws_ec2_autoscale_cluster/README.md @@ -106,15 +106,15 @@ aws_ec2_autoscale_cluster: - name: "{{ _env_type }}-scale-up-policy" policy_type: "SimpleScaling" adjustment_type: "ChangeInCapacity" - adjustment: 2 + adjustment: 2 # Add two servers per scaling event adjustment_step: 1 # Only used when adjustment_type is PercentChangeInCapacity. - cooldown: 300 + cooldown: 120 - name: "{{ _env_type }}-scale-down-policy" policy_type: "SimpleScaling" adjustment_type: "ChangeInCapacity" - adjustment: -2 + adjustment: -1 # Reduce by one server at a time adjustment_step: -1 # Only used when adjustment_type is PercentChangeInCapacity. - cooldown: 300 + cooldown: 120 asg_cloudwatch_alarm_scale_up_name: "{{ _env_type }}-cloudwatch-metric-alarm-cpu-scale-up" asg_cloudwatch_alarm_scale_down_name: "{{ _env_type }}-cloudwatch-metric-alarm-cpu-scale-down" asg_cloudwatch_alarms: @@ -126,8 +126,8 @@ aws_ec2_autoscale_cluster: threshold: 80 unit: "Percent" comparison: "GreaterThanOrEqualToThreshold" - period: 120 - evaluation_periods: 5 + period: 30 + evaluation_periods: 3 - scale_direction: "down" description: "CPU under 40% so scale down." metric: "CPUUtilization" diff --git a/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml b/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml index 03e99461c..b6aedd0b2 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml +++ b/roles/aws/aws_ec2_autoscale_cluster/defaults/main.yml @@ -87,15 +87,15 @@ aws_ec2_autoscale_cluster: - name: "{{ _env_type }}-scale-up-policy" policy_type: "SimpleScaling" adjustment_type: "ChangeInCapacity" - adjustment: 2 + adjustment: 2 # Add two servers per scaling event adjustment_step: 1 # Only used when adjustment_type is PercentChangeInCapacity. - cooldown: 300 + cooldown: 120 - name: "{{ _env_type }}-scale-down-policy" policy_type: "SimpleScaling" adjustment_type: "ChangeInCapacity" - adjustment: -2 + adjustment: -1 # Reduce by one server at a time adjustment_step: -1 # Only used when adjustment_type is PercentChangeInCapacity. - cooldown: 300 + cooldown: 120 asg_cloudwatch_alarm_scale_up_name: "{{ _env_type }}-cloudwatch-metric-alarm-cpu-scale-up" asg_cloudwatch_alarm_scale_down_name: "{{ _env_type }}-cloudwatch-metric-alarm-cpu-scale-down" asg_cloudwatch_alarms: @@ -107,8 +107,8 @@ aws_ec2_autoscale_cluster: threshold: 80 unit: "Percent" comparison: "GreaterThanOrEqualToThreshold" - period: 120 - evaluation_periods: 5 + period: 30 + evaluation_periods: 3 - scale_direction: "down" description: "CPU under 40% so scale down." metric: "CPUUtilization" From aff0054631dc3a67aeaf6f1fa175fc1bafe633cf Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 14 May 2024 17:14:18 +0200 Subject: [PATCH 036/206] Making ClamAV timers a list so they can be entirely replaced. --- docs/roles/debian/clamav.md | 8 ++++---- roles/debian/clamav/README.md | 8 ++++---- roles/debian/clamav/defaults/main.yml | 8 ++++---- roles/debian/clamav/tasks/main.yml | 8 +++++++- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/roles/debian/clamav.md b/docs/roles/debian/clamav.md index 4529ab043..0a35265c2 100644 --- a/docs/roles/debian/clamav.md +++ b/docs/roles/debian/clamav.md @@ -19,11 +19,11 @@ clamav: - /sys/ scan_location: / log_name: clamav_daily.log - # scheduled scans, set to an empty dictionary for no timers + # scheduled scans, set to an empty list for no timers timers: - clamscan_daily: - timer_command: /usr/local/clamav/script/clamscan_daily # path to clamscan wrapper script, ensure it is defined in clamav.scripts - timer_OnCalendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + - clamscan_daily: + timer_command: /usr/local/clamav/script/clamscan_daily # path to clamscan wrapper script, ensure it is defined in clamav.scripts + timer_OnCalendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. log_location: /var/log/clamav send_mail: false # Important: will not send any emails by default. diff --git a/roles/debian/clamav/README.md b/roles/debian/clamav/README.md index 4529ab043..0a35265c2 100644 --- a/roles/debian/clamav/README.md +++ b/roles/debian/clamav/README.md @@ -19,11 +19,11 @@ clamav: - /sys/ scan_location: / log_name: clamav_daily.log - # scheduled scans, set to an empty dictionary for no timers + # scheduled scans, set to an empty list for no timers timers: - clamscan_daily: - timer_command: /usr/local/clamav/script/clamscan_daily # path to clamscan wrapper script, ensure it is defined in clamav.scripts - timer_OnCalendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + - clamscan_daily: + timer_command: /usr/local/clamav/script/clamscan_daily # path to clamscan wrapper script, ensure it is defined in clamav.scripts + timer_OnCalendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. log_location: /var/log/clamav send_mail: false # Important: will not send any emails by default. diff --git a/roles/debian/clamav/defaults/main.yml b/roles/debian/clamav/defaults/main.yml index 648ddd934..85af99d42 100644 --- a/roles/debian/clamav/defaults/main.yml +++ b/roles/debian/clamav/defaults/main.yml @@ -8,11 +8,11 @@ clamav: - /sys/ scan_location: / log_name: clamav_daily.log - # scheduled scans, set to an empty dictionary for no timers + # scheduled scans, set to an empty list for no timers timers: - clamscan_daily: - timer_command: /usr/local/clamav/script/clamscan_daily # path to clamscan wrapper script, ensure it is defined in clamav.scripts - timer_OnCalendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + - clamscan_daily: + timer_command: /usr/local/clamav/script/clamscan_daily # path to clamscan wrapper script, ensure it is defined in clamav.scripts + timer_OnCalendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. log_location: /var/log/clamav send_mail: false # Important: will not send any emails by default. diff --git a/roles/debian/clamav/tasks/main.yml b/roles/debian/clamav/tasks/main.yml index ebf032e74..02e643781 100644 --- a/roles/debian/clamav/tasks/main.yml +++ b/roles/debian/clamav/tasks/main.yml @@ -23,11 +23,17 @@ state: directory mode: '0755' +- name: Build a dictionary of timers for the systemd timers role. + ansible.builtin.set_fact: + _clamav_timers: "{{ _clamav_timers | default({}) | combine( item ) }}" + with_items: "{{ clamav.timers }}" + when: clamav.timers | length > 0 + - name: Set up a ClamAV systemd timers. ansible.builtin.include_role: name: contrib/systemd_timers vars: - timers: "{{ clamav.timers }}" + timers: "{{ _clamav_timers }}" when: clamav.timers | length > 0 - name: Install clamdscan. From cb4f6eb175eef4578d52b092fa37ebcd3f367f75 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 16 May 2024 17:32:27 +0200 Subject: [PATCH 037/206] Spacing fix for linting. --- roles/debian/clamav/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/debian/clamav/tasks/main.yml b/roles/debian/clamav/tasks/main.yml index 02e643781..eeaf31013 100644 --- a/roles/debian/clamav/tasks/main.yml +++ b/roles/debian/clamav/tasks/main.yml @@ -25,7 +25,7 @@ - name: Build a dictionary of timers for the systemd timers role. ansible.builtin.set_fact: - _clamav_timers: "{{ _clamav_timers | default({}) | combine( item ) }}" + _clamav_timers: "{{ _clamav_timers | default({}) | combine(item) }}" with_items: "{{ clamav.timers }}" when: clamav.timers | length > 0 From ad2038555530a783d2c00f112cfe2ad2d1d85adf Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 22 May 2024 13:13:56 +0200 Subject: [PATCH 038/206] Renaming npm module. --- roles/debian/lhci/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/debian/lhci/tasks/main.yml b/roles/debian/lhci/tasks/main.yml index a2a24d7f8..6c7a3a038 100644 --- a/roles/debian/lhci/tasks/main.yml +++ b/roles/debian/lhci/tasks/main.yml @@ -52,7 +52,7 @@ state: present - name: Install "lhci/cli" node.js package. - community.general.packaging.language.npm: + community.general.npm: name: "@lhci/cli" version: "{{ lhci.cli_version | default(omit) }}" state: "{{ lhci.cli_state | default(omit) }}" @@ -60,7 +60,7 @@ global: true - name: Install "lighthouse" node.js package. - community.general.packaging.language.npm: + community.general.npm: name: "lighthouse" version: "{{ lhci.lighthouse_version | default(omit) }}" state: "{{ lhci.lighthouse_state | default(omit) }}" From cf6d60c9643a25881395dd195b41aa356c35b992 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 27 May 2024 16:23:37 +0200 Subject: [PATCH 039/206] Removing NGINX installation as part of phpMyAdmin role by default. --- docs/roles/debian/phpmyadmin.md | 6 ++++-- roles/debian/phpmyadmin/README.md | 6 ++++-- roles/debian/phpmyadmin/defaults/main.yml | 3 ++- roles/debian/phpmyadmin/meta/main.yml | 1 - roles/debian/phpmyadmin/tasks/main.yml | 5 +++++ 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/roles/debian/phpmyadmin.md b/docs/roles/debian/phpmyadmin.md index 3c329c981..7eeb9bcca 100644 --- a/docs/roles/debian/phpmyadmin.md +++ b/docs/roles/debian/phpmyadmin.md @@ -1,7 +1,7 @@ # phpMyAdmin -This role only installs phpMyAdmin ready for configuration, it does not do any active configuration, nor does it create the necessary NGINX vhost. It will install `debian/php-fpm` and `debian/nginx` so do review their variables and set what you need prior to running a first build with `phpmyadmin`. +This role only installs phpMyAdmin ready for configuration, it does not do any active configuration, nor does it create the necessary NGINX vhost. It will install `debian/php-fpm` so do review the variables for PHP and set what you need prior to running a first build with `phpmyadmin`. -Here is an example NGINX vhost config you can copy to your `nginx.yml` file and adjust as required: +Similarly, it can optionally install `debian/nginx` if you set `phpmyadmin.install_nginx: true`, so if you do that be sure to provide a sensible NGINX config. Here is an example NGINX vhost config you can copy to your `nginx.yml` file and adjust as required: ```yaml nginx: @@ -54,6 +54,8 @@ phpmyadmin: # Assuming user and group should match php-fpm by default www_user: "{{ php.fpm.pool_user }}" www_group: "{{ php.fpm.pool_group }}" + install_nginx: false + ``` diff --git a/roles/debian/phpmyadmin/README.md b/roles/debian/phpmyadmin/README.md index 3c329c981..7eeb9bcca 100644 --- a/roles/debian/phpmyadmin/README.md +++ b/roles/debian/phpmyadmin/README.md @@ -1,7 +1,7 @@ # phpMyAdmin -This role only installs phpMyAdmin ready for configuration, it does not do any active configuration, nor does it create the necessary NGINX vhost. It will install `debian/php-fpm` and `debian/nginx` so do review their variables and set what you need prior to running a first build with `phpmyadmin`. +This role only installs phpMyAdmin ready for configuration, it does not do any active configuration, nor does it create the necessary NGINX vhost. It will install `debian/php-fpm` so do review the variables for PHP and set what you need prior to running a first build with `phpmyadmin`. -Here is an example NGINX vhost config you can copy to your `nginx.yml` file and adjust as required: +Similarly, it can optionally install `debian/nginx` if you set `phpmyadmin.install_nginx: true`, so if you do that be sure to provide a sensible NGINX config. Here is an example NGINX vhost config you can copy to your `nginx.yml` file and adjust as required: ```yaml nginx: @@ -54,6 +54,8 @@ phpmyadmin: # Assuming user and group should match php-fpm by default www_user: "{{ php.fpm.pool_user }}" www_group: "{{ php.fpm.pool_group }}" + install_nginx: false + ``` diff --git a/roles/debian/phpmyadmin/defaults/main.yml b/roles/debian/phpmyadmin/defaults/main.yml index 00960891a..24fbc46b3 100644 --- a/roles/debian/phpmyadmin/defaults/main.yml +++ b/roles/debian/phpmyadmin/defaults/main.yml @@ -5,4 +5,5 @@ phpmyadmin: install_path: "/home/{{ ce_deploy.username }}/deploy" # Assuming user and group should match php-fpm by default www_user: "{{ php.fpm.pool_user }}" - www_group: "{{ php.fpm.pool_group }}" \ No newline at end of file + www_group: "{{ php.fpm.pool_group }}" + install_nginx: false diff --git a/roles/debian/phpmyadmin/meta/main.yml b/roles/debian/phpmyadmin/meta/main.yml index 54ffd52ed..d0cf08b3d 100644 --- a/roles/debian/phpmyadmin/meta/main.yml +++ b/roles/debian/phpmyadmin/meta/main.yml @@ -1,4 +1,3 @@ --- dependencies: - - role: debian/nginx - role: debian/php-fpm diff --git a/roles/debian/phpmyadmin/tasks/main.yml b/roles/debian/phpmyadmin/tasks/main.yml index 474e867fb..b3fac7e2d 100644 --- a/roles/debian/phpmyadmin/tasks/main.yml +++ b/roles/debian/phpmyadmin/tasks/main.yml @@ -17,3 +17,8 @@ ansible.builtin.command: cmd: "/usr/local/bin/phpmyadmin.sh -u -v {{ phpmyadmin.version }}" when: phpmyadmin.method == 'upgrade' + +- name: Install the NGINX web server. + ansible.builtin.include_role: + name: debian/nginx + when: phpmyadmin.install_nginx From 3de5bf65348c1a82bfd4691bf7dea59f883b381e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 27 May 2024 16:32:10 +0200 Subject: [PATCH 040/206] Fixing Varnish handler names. --- roles/debian/varnish_config/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/debian/varnish_config/tasks/main.yml b/roles/debian/varnish_config/tasks/main.yml index 362d33824..ad6d2836e 100644 --- a/roles/debian/varnish_config/tasks/main.yml +++ b/roles/debian/varnish_config/tasks/main.yml @@ -11,5 +11,5 @@ - "{{ _ce_provision_base_dir }}/config/files/templates/{{ varnish_config.template_filename }}.j2" - "default.vcl.j2" notify: - - Reload systemd - - Restart varnish + - reload systemd # handlers in https://github.com/geerlingguy/ansible-role-varnish/blob/master/handlers/main.yml + - restart varnish From cda46421faca0782333a786a2f1c1f5be4a55c1d Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 27 May 2024 16:40:13 +0200 Subject: [PATCH 041/206] Excluding name[casing] rule from linting due to false positives. --- .ansible-lint | 1 + 1 file changed, 1 insertion(+) diff --git a/.ansible-lint b/.ansible-lint index 71d7648cc..a68f44f4c 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -14,4 +14,5 @@ skip_list: - template-instead-of-copy # to skip over roles/ssl/tasks/copy.yml errors, temporarily. - name[template] # it doesn't like Jinja templates being in the middle of a task name, which seems silly to me. exclude_paths: + - name[casing] # sometimes included Galaxy roles break linting rules and cause failures - roles/debian/wazuh/ # imported role uses yes/no instead of true/false so always fails From 3a25d9b27d87e62e82f00db3b20267bf2f339e79 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 27 May 2024 16:42:12 +0200 Subject: [PATCH 042/206] Put rule in wrong place! --- .ansible-lint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ansible-lint b/.ansible-lint index a68f44f4c..390760f80 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -13,6 +13,6 @@ skip_list: - fqcn-builtins - template-instead-of-copy # to skip over roles/ssl/tasks/copy.yml errors, temporarily. - name[template] # it doesn't like Jinja templates being in the middle of a task name, which seems silly to me. -exclude_paths: - name[casing] # sometimes included Galaxy roles break linting rules and cause failures +exclude_paths: - roles/debian/wazuh/ # imported role uses yes/no instead of true/false so always fails From 31b7e90279011a85927ee33e7d3cb249f6196430 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 29 May 2024 10:52:07 +0200 Subject: [PATCH 043/206] Removing lock file behaviour from ASGs as it cannot work unless controller and ASG are in the same VPC. --- .../aws_ec2_autoscale_cluster/tasks/main.yml | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml b/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml index e6abb61fd..77b0e0a11 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml +++ b/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml @@ -1,30 +1,4 @@ --- -# With ASGs the lock behaviour in _init will not work, we need to do it here. -- name: Check for a ce-deploy lock file. - ansible.builtin.stat: - path: "{{ _init.deploy_lock_file }}" - register: _ce_deploy_lock - connection: ssh - run_once: true - when: aws_ec2_autoscale_cluster.type == "ec2" - -- name: Abort if ce-deploy lock file exists. - when: - - _ce_deploy_lock.stat.exists is defined and _ce_deploy_lock.stat.exists - block: - - name: Abort if ce-deploy lock file is found. - ansible.builtin.debug: - msg: "ce-deploy lock file discovered, a deployment is in progress! If this is not the case, login to the affected server and delete the file at {{ _init.deploy_lock_file }}." - - ansible.builtin.meta: end_play - -- name: Set a lock file on each ASG machine. - ansible.builtin.file: - path: "{{ _init.lock_file }}" - state: touch - mode: 0644 - connection: ssh - when: aws_ec2_autoscale_cluster.type == "ec2" - # @todo Do we still need that? Used by AMI and LC, # but probably could simply use the base name. - name: Define unique name for resources. @@ -782,13 +756,3 @@ when: - aws_ec2_autoscale_cluster.route_53.zone is defined - aws_ec2_autoscale_cluster.route_53.zone | length > 0 - -# We only need to remove the lock file if the machines will not be replaced. -- name: Remove the lock file on each ASG machine. - ansible.builtin.file: - path: "{{ _init.lock_file }}" - state: absent - connection: ssh - when: - - aws_ec2_autoscale_cluster.type == "ec2" - - not aws_ec2_autoscale_cluster.asg_refresh From 34d295d2f64f9cdda257a6af20fe45ef8bdaa73e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 29 May 2024 11:04:05 +0200 Subject: [PATCH 044/206] Capturing lock file limitations in comment. --- roles/_init/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index 485fdfa5f..2a7e3add7 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -3,7 +3,8 @@ ansible.builtin.debug: msg: "Using ce-provision {{ _init.ce_provision_version }}" -# See roles/aws/aws_ec2_autoscale_cluster for lock handling in ASGs. +# @TODO This cannot work for ASGs. +# We will need something central for lock files and it cannot be controller servers because they may be containers. - name: Check for a ce-deploy lock file. ansible.builtin.stat: path: "{{ _init.deploy_lock_file }}" From af108d735926dc32cf425ee313ee6164ab08bcad Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 31 May 2024 10:49:11 +0200 Subject: [PATCH 045/206] Updating documentation for LE. --- docs/roles/debian/apache.md | 4 ++-- docs/roles/debian/ldap_server.md | 4 ++-- docs/roles/debian/mailpit.md | 4 ++-- docs/roles/debian/nginx.md | 4 ++-- docs/roles/debian/phpmyadmin.md | 4 ++-- docs/roles/debian/ssl.md | 13 ++++++++++--- 6 files changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/roles/debian/apache.md b/docs/roles/debian/apache.md index 3902bfdab..d1da36405 100644 --- a/docs/roles/debian/apache.md +++ b/docs/roles/debian/apache.md @@ -62,8 +62,8 @@ apache: # services: # - apache2 # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - apache2 diff --git a/docs/roles/debian/ldap_server.md b/docs/roles/debian/ldap_server.md index 62421b9be..d3a234d4e 100644 --- a/docs/roles/debian/ldap_server.md +++ b/docs/roles/debian/ldap_server.md @@ -105,8 +105,8 @@ ldap_server: # email: sysadm@codeenigma.com # services: [] # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - slapd diff --git a/docs/roles/debian/mailpit.md b/docs/roles/debian/mailpit.md index 5431d0cf2..2cd4b5157 100644 --- a/docs/roles/debian/mailpit.md +++ b/docs/roles/debian/mailpit.md @@ -51,8 +51,8 @@ mailpit: #services: # - nginx #web_server: standalone - #certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - #certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + #certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + #certbot_renew_command: "certonly --agree-tos --force-renew" #reload_command: restart #reload: # - mailpit diff --git a/docs/roles/debian/nginx.md b/docs/roles/debian/nginx.md index 3b63015da..2f7eade5d 100644 --- a/docs/roles/debian/nginx.md +++ b/docs/roles/debian/nginx.md @@ -91,8 +91,8 @@ nginx: # email: sysadm@codeenigma.com # services: [] # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - nginx diff --git a/docs/roles/debian/phpmyadmin.md b/docs/roles/debian/phpmyadmin.md index 7eeb9bcca..54f24414d 100644 --- a/docs/roles/debian/phpmyadmin.md +++ b/docs/roles/debian/phpmyadmin.md @@ -21,8 +21,8 @@ nginx: email: sysadm@codeenigma.com services: [] web_server: standalone - certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + certbot_renew_command: "certonly --agree-tos --force-renew" reload_command: reload reload: - nginx diff --git a/docs/roles/debian/ssl.md b/docs/roles/debian/ssl.md index ecf91fc72..d9743ca77 100644 --- a/docs/roles/debian/ssl.md +++ b/docs/roles/debian/ssl.md @@ -36,11 +36,18 @@ nginx: autorenew: true email: administrator@example.com services: [] - certbot_register_command: "/usr/bin/certbot certonly --standalone --agree-tos --preferred-challenges http -n" - certbot_renew_command: "/usr/bin/certbot certonly --standalone --agree-tos --force-renew" + web_server: standalone + certbot_register_command: "certonly --standalone --agree-tos --preferred-challenges http -n" + certbot_renew_command: "certonly --standalone --agree-tos --force-renew" + reload_command: reload + reload: + - nginx + renewal_minute: "0" # minute to run renewal cron job + renewal_hour: "0" # hour to run renewal cron job + renewal_weekday: "0" # day of week to run renewal ``` -You need to include *all* variables required by the `letsencrypt` SSL handler because defaults will not load from the `ssl` role in this context. +As in the example above, you need to include *all* variables required by the `letsencrypt` SSL handler because defaults will not load from the `ssl` role in this context. If you are using Nginx or Apache you can set the `ssl.web_server` for each domain to either `nginx` or `apache` to have the necessary plugin installed for `certbot` to do automatic handling of LetsEncrypt requests. Be aware, it does this by temporarily altering your web server config and reloading - use this option at your own risk. This is *not* intended to be used with but *instead of* `ssl.http_01_port`. From b14da199adfa3066be4b6e3ea453493875dad1a9 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 31 May 2024 10:49:52 +0200 Subject: [PATCH 046/206] Using pip to install certbot plugins. --- roles/debian/ssl/tasks/letsencrypt.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/roles/debian/ssl/tasks/letsencrypt.yml b/roles/debian/ssl/tasks/letsencrypt.yml index 90e469949..8d3f26634 100644 --- a/roles/debian/ssl/tasks/letsencrypt.yml +++ b/roles/debian/ssl/tasks/letsencrypt.yml @@ -53,11 +53,13 @@ # @todo - add support for Route 53 DNS plugin - name: "Ensure certbot-{{ _ssl_web_server }} plugin is installed." - ansible.builtin.apt: - pkg: "python3-certbot-{{ _ssl_web_server }}" - state: present - when: - - _ssl_web_server != 'standalone' + ansible.builtin.include_role: + name: debian/python_pip_packages + vars: + python_pip_packages: + packages: + - name: "certbot-{{ _ssl_web_server }}" + when: _ssl_web_server != "standalone" # there is no certbot-standalone package in PyPi - name: Define SSL base path. ansible.builtin.set_fact: From 0620270834a0193614877e0555bbc9be593ac839 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 31 May 2024 10:50:04 +0200 Subject: [PATCH 047/206] Updating README docs. --- roles/debian/apache/README.md | 4 ++-- roles/debian/apache/defaults/main.yml | 4 ++-- roles/debian/ldap_server/README.md | 4 ++-- roles/debian/ldap_server/defaults/main.yml | 4 ++-- roles/debian/mailpit/README.md | 4 ++-- roles/debian/mailpit/defaults/main.yml | 4 ++-- roles/debian/nginx/README.md | 4 ++-- roles/debian/nginx/defaults/main.yml | 4 ++-- roles/debian/phpmyadmin/README.md | 4 ++-- roles/debian/ssl/README.md | 13 ++++++++++--- 10 files changed, 28 insertions(+), 21 deletions(-) diff --git a/roles/debian/apache/README.md b/roles/debian/apache/README.md index 3902bfdab..d1da36405 100644 --- a/roles/debian/apache/README.md +++ b/roles/debian/apache/README.md @@ -62,8 +62,8 @@ apache: # services: # - apache2 # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - apache2 diff --git a/roles/debian/apache/defaults/main.yml b/roles/debian/apache/defaults/main.yml index b6705c0e4..c76ddc0b0 100644 --- a/roles/debian/apache/defaults/main.yml +++ b/roles/debian/apache/defaults/main.yml @@ -52,8 +52,8 @@ apache: # services: # - apache2 # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - apache2 diff --git a/roles/debian/ldap_server/README.md b/roles/debian/ldap_server/README.md index 62421b9be..d3a234d4e 100644 --- a/roles/debian/ldap_server/README.md +++ b/roles/debian/ldap_server/README.md @@ -105,8 +105,8 @@ ldap_server: # email: sysadm@codeenigma.com # services: [] # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - slapd diff --git a/roles/debian/ldap_server/defaults/main.yml b/roles/debian/ldap_server/defaults/main.yml index 6924e7741..93956ffc5 100644 --- a/roles/debian/ldap_server/defaults/main.yml +++ b/roles/debian/ldap_server/defaults/main.yml @@ -39,8 +39,8 @@ ldap_server: # email: sysadm@codeenigma.com # services: [] # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - slapd diff --git a/roles/debian/mailpit/README.md b/roles/debian/mailpit/README.md index 5431d0cf2..2cd4b5157 100644 --- a/roles/debian/mailpit/README.md +++ b/roles/debian/mailpit/README.md @@ -51,8 +51,8 @@ mailpit: #services: # - nginx #web_server: standalone - #certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - #certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + #certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + #certbot_renew_command: "certonly --agree-tos --force-renew" #reload_command: restart #reload: # - mailpit diff --git a/roles/debian/mailpit/defaults/main.yml b/roles/debian/mailpit/defaults/main.yml index 5a0666593..a883589f8 100644 --- a/roles/debian/mailpit/defaults/main.yml +++ b/roles/debian/mailpit/defaults/main.yml @@ -34,8 +34,8 @@ mailpit: #services: # - nginx #web_server: standalone - #certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - #certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + #certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + #certbot_renew_command: "certonly --agree-tos --force-renew" #reload_command: restart #reload: # - mailpit diff --git a/roles/debian/nginx/README.md b/roles/debian/nginx/README.md index 3b63015da..2f7eade5d 100644 --- a/roles/debian/nginx/README.md +++ b/roles/debian/nginx/README.md @@ -91,8 +91,8 @@ nginx: # email: sysadm@codeenigma.com # services: [] # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - nginx diff --git a/roles/debian/nginx/defaults/main.yml b/roles/debian/nginx/defaults/main.yml index 5f6e7c7fe..5f66e981f 100644 --- a/roles/debian/nginx/defaults/main.yml +++ b/roles/debian/nginx/defaults/main.yml @@ -79,8 +79,8 @@ nginx: # email: sysadm@codeenigma.com # services: [] # web_server: standalone - # certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - # certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + # certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + # certbot_renew_command: "certonly --agree-tos --force-renew" # reload_command: restart # reload: # - nginx diff --git a/roles/debian/phpmyadmin/README.md b/roles/debian/phpmyadmin/README.md index 7eeb9bcca..54f24414d 100644 --- a/roles/debian/phpmyadmin/README.md +++ b/roles/debian/phpmyadmin/README.md @@ -21,8 +21,8 @@ nginx: email: sysadm@codeenigma.com services: [] web_server: standalone - certbot_register_command: "/usr/bin/certbot certonly --agree-tos --preferred-challenges http -n" - certbot_renew_command: "/usr/bin/certbot certonly --agree-tos --force-renew" + certbot_register_command: "certonly --agree-tos --preferred-challenges http -n" + certbot_renew_command: "certonly --agree-tos --force-renew" reload_command: reload reload: - nginx diff --git a/roles/debian/ssl/README.md b/roles/debian/ssl/README.md index ecf91fc72..d9743ca77 100644 --- a/roles/debian/ssl/README.md +++ b/roles/debian/ssl/README.md @@ -36,11 +36,18 @@ nginx: autorenew: true email: administrator@example.com services: [] - certbot_register_command: "/usr/bin/certbot certonly --standalone --agree-tos --preferred-challenges http -n" - certbot_renew_command: "/usr/bin/certbot certonly --standalone --agree-tos --force-renew" + web_server: standalone + certbot_register_command: "certonly --standalone --agree-tos --preferred-challenges http -n" + certbot_renew_command: "certonly --standalone --agree-tos --force-renew" + reload_command: reload + reload: + - nginx + renewal_minute: "0" # minute to run renewal cron job + renewal_hour: "0" # hour to run renewal cron job + renewal_weekday: "0" # day of week to run renewal ``` -You need to include *all* variables required by the `letsencrypt` SSL handler because defaults will not load from the `ssl` role in this context. +As in the example above, you need to include *all* variables required by the `letsencrypt` SSL handler because defaults will not load from the `ssl` role in this context. If you are using Nginx or Apache you can set the `ssl.web_server` for each domain to either `nginx` or `apache` to have the necessary plugin installed for `certbot` to do automatic handling of LetsEncrypt requests. Be aware, it does this by temporarily altering your web server config and reloading - use this option at your own risk. This is *not* intended to be used with but *instead of* `ssl.http_01_port`. From ebda10c935cb0f0c1476edc25890d15f6a25702c Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 31 May 2024 11:00:03 +0200 Subject: [PATCH 048/206] Docs error corrected. --- docs/roles/debian/phpmyadmin.md | 4 +--- docs/roles/debian/ssl.md | 6 ++---- roles/debian/phpmyadmin/README.md | 4 +--- roles/debian/ssl/README.md | 6 ++---- roles/debian/ssl/defaults/main.yml | 2 +- 5 files changed, 7 insertions(+), 15 deletions(-) diff --git a/docs/roles/debian/phpmyadmin.md b/docs/roles/debian/phpmyadmin.md index 54f24414d..4f83dd3ce 100644 --- a/docs/roles/debian/phpmyadmin.md +++ b/docs/roles/debian/phpmyadmin.md @@ -26,9 +26,7 @@ nginx: reload_command: reload reload: - nginx - renewal_minute: "0" # minute to run renewal cron job - renewal_hour: "0" # hour to run renewal cron job - renewal_weekday: "0" # day of week to run renewal + on_calendar: "Mon *-*-* 04:00:00" ratelimitingcrawlers: true is_default: false servers: diff --git a/docs/roles/debian/ssl.md b/docs/roles/debian/ssl.md index d9743ca77..2ead29797 100644 --- a/docs/roles/debian/ssl.md +++ b/docs/roles/debian/ssl.md @@ -42,9 +42,7 @@ nginx: reload_command: reload reload: - nginx - renewal_minute: "0" # minute to run renewal cron job - renewal_hour: "0" # hour to run renewal cron job - renewal_weekday: "0" # day of week to run renewal + on_calendar: "Mon *-*-* 04:00:00" ``` As in the example above, you need to include *all* variables required by the `letsencrypt` SSL handler because defaults will not load from the `ssl` role in this context. @@ -91,7 +89,7 @@ ssl: autorenew: false # set to true to create a systemd timer to renew LE certs certbot_renew_command: "certonly --agree-tos --force-renew" # root of the command used in the systemd timer # See systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events - #on_calendar: "Mon *-*-* 04:00:00" + on_calendar: "Mon *-*-* 04:00:00" web_server: standalone # values are standalone, nginx or apache - warning, nginx and apache will attempt to manipulate your vhosts! # For "letsencrypt" handling, a list of service to stop while creating the certificate. diff --git a/roles/debian/phpmyadmin/README.md b/roles/debian/phpmyadmin/README.md index 54f24414d..4f83dd3ce 100644 --- a/roles/debian/phpmyadmin/README.md +++ b/roles/debian/phpmyadmin/README.md @@ -26,9 +26,7 @@ nginx: reload_command: reload reload: - nginx - renewal_minute: "0" # minute to run renewal cron job - renewal_hour: "0" # hour to run renewal cron job - renewal_weekday: "0" # day of week to run renewal + on_calendar: "Mon *-*-* 04:00:00" ratelimitingcrawlers: true is_default: false servers: diff --git a/roles/debian/ssl/README.md b/roles/debian/ssl/README.md index d9743ca77..2ead29797 100644 --- a/roles/debian/ssl/README.md +++ b/roles/debian/ssl/README.md @@ -42,9 +42,7 @@ nginx: reload_command: reload reload: - nginx - renewal_minute: "0" # minute to run renewal cron job - renewal_hour: "0" # hour to run renewal cron job - renewal_weekday: "0" # day of week to run renewal + on_calendar: "Mon *-*-* 04:00:00" ``` As in the example above, you need to include *all* variables required by the `letsencrypt` SSL handler because defaults will not load from the `ssl` role in this context. @@ -91,7 +89,7 @@ ssl: autorenew: false # set to true to create a systemd timer to renew LE certs certbot_renew_command: "certonly --agree-tos --force-renew" # root of the command used in the systemd timer # See systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events - #on_calendar: "Mon *-*-* 04:00:00" + on_calendar: "Mon *-*-* 04:00:00" web_server: standalone # values are standalone, nginx or apache - warning, nginx and apache will attempt to manipulate your vhosts! # For "letsencrypt" handling, a list of service to stop while creating the certificate. diff --git a/roles/debian/ssl/defaults/main.yml b/roles/debian/ssl/defaults/main.yml index 2c862cda9..f1191de89 100644 --- a/roles/debian/ssl/defaults/main.yml +++ b/roles/debian/ssl/defaults/main.yml @@ -35,7 +35,7 @@ ssl: autorenew: false # set to true to create a systemd timer to renew LE certs certbot_renew_command: "certonly --agree-tos --force-renew" # root of the command used in the systemd timer # See systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events - #on_calendar: "Mon *-*-* 04:00:00" + on_calendar: "Mon *-*-* 04:00:00" web_server: standalone # values are standalone, nginx or apache - warning, nginx and apache will attempt to manipulate your vhosts! # For "letsencrypt" handling, a list of service to stop while creating the certificate. From 9acaddc25d079f37db53b8e9875e2831913a6bae Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 31 May 2024 15:36:05 +0200 Subject: [PATCH 049/206] Working around deprecated SSH algorithms. --- roles/aws/aws_ami/templates/packer.json.j2 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/roles/aws/aws_ami/templates/packer.json.j2 b/roles/aws/aws_ami/templates/packer.json.j2 index 5efd118b6..7f04c6b10 100755 --- a/roles/aws/aws_ami/templates/packer.json.j2 +++ b/roles/aws/aws_ami/templates/packer.json.j2 @@ -50,6 +50,7 @@ ], "provisioners": [{ "type": "ansible", + "use_proxy": false, "playbook_file": "{{ aws_ami.playbook_file }}", "inventory_directory": "{{ _ce_provision_base_dir }}/hosts", "ssh_authorized_key_file": "/home/{{ user_provision.username }}/.ssh/id_rsa.pub", @@ -63,7 +64,8 @@ "ANSIBLE_CONFIG={{ _ce_provision_base_dir }}/ansible.cfg" ], "extra_arguments": - [ + [ + "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa", {% if ansible_verbosity >= 1 %} "-vvvv", {% endif %} @@ -71,8 +73,8 @@ "--extra-vars", "{{ _aws_ami_extra_vars }}", {% endif %} - "--extra-vars", - "{ _ce_provision_base_dir: {{ _ce_provision_base_dir }}, _ce_provision_build_dir: {{ _ce_provision_build_dir }}, _ce_provision_build_tmp_dir: {{ _ce_provision_build_tmp_dir }}, _ce_provision_data_dir: {{ _ce_provision_data_dir }}, _ce_provision_build_id: {{ _ce_provision_build_id }}, _ce_provision_force_play: true }" - ] + "--extra-vars", + "{ _ce_provision_base_dir: {{ _ce_provision_base_dir }}, _ce_provision_build_dir: {{ _ce_provision_build_dir }}, _ce_provision_build_tmp_dir: {{ _ce_provision_build_tmp_dir }}, _ce_provision_data_dir: {{ _ce_provision_data_dir }}, _ce_provision_build_id: {{ _ce_provision_build_id }}, _ce_provision_force_play: true }" + ] }] } From 581e2afa599ae84285fd9096838ca63c6e76275c Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 3 Jun 2024 14:32:02 +0200 Subject: [PATCH 050/206] Upgrading SSH key type standard for controller and deploy users. --- docs/roles/aws/aws_ami.md | 1 + docs/roles/debian/ce_deploy.md | 4 +++- docs/roles/debian/ce_provision.md | 4 +++- roles/aws/aws_ami/README.md | 1 + roles/aws/aws_ami/defaults/main.yml | 1 + roles/aws/aws_ami/templates/packer.json.j2 | 10 ++++------ roles/debian/ce_deploy/README.md | 4 +++- roles/debian/ce_deploy/defaults/main.yml | 4 +++- roles/debian/ce_deploy/tasks/main.yml | 5 +++-- roles/debian/ce_provision/README.md | 4 +++- roles/debian/ce_provision/defaults/main.yml | 4 +++- roles/debian/ce_provision/tasks/main.yml | 3 ++- 12 files changed, 30 insertions(+), 15 deletions(-) diff --git a/docs/roles/aws/aws_ami.md b/docs/roles/aws/aws_ami.md index f4f3bd2be..acc2f10e4 100644 --- a/docs/roles/aws/aws_ami.md +++ b/docs/roles/aws/aws_ami.md @@ -29,6 +29,7 @@ aws_ami: ami_name: "example" owner: "136693071363" # Global AWS account ID of owner, defaults to Debian official ssh_username: "admin" + public_key_name: id_ecdsa.pub # from Debian 12 (Bookworm) onwards RSA keys, i.e. id_rsa.pub, are deprecated encrypt_boot: false # EBS volume options device_name: /dev/xvda # default for Debian AMIs diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index 856e548eb..d3c939fd2 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -20,7 +20,9 @@ ce_deploy: # Other ce-deploy settings. aws_support: true # installs boto3 new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user - key_name: id_rsa.pub # existing users may have a key of a different name + ssh_key_bits: "521" # recommended to use 4096 for RSA keys, 521 is the maximum for ECDSA keys + ssh_key_type: ecdsa # set to rsa to create an RSA key + public_key_name: id_ecdsa.pub # this might be id_rsa.pub for RSA keys, existing users may have a key of a different name username: "{{ _ce_deploy.username }}" own_repository: "https://github.com/codeenigma/ce-deploy.git" own_repository_branch: "master" diff --git a/docs/roles/debian/ce_provision.md b/docs/roles/debian/ce_provision.md index 8611d76b4..e3816975d 100644 --- a/docs/roles/debian/ce_provision.md +++ b/docs/roles/debian/ce_provision.md @@ -21,7 +21,9 @@ ce_provision: new_user: "{{ _init.ce_provision_new_user }}" # see _init defaults, set to false if user already exists or is ephemeral, e.g. an LDAP user username: "{{ _ce_provision_username }}" # see _init defaults #uid: "{{ _init.ce_provision_uid }}" # see _init defaults, optionally hardcode the UID for this user - public_key_name: id_rsa.pub # existing users may have a key of a different name + ssh_key_bits: "521" # recommended to use 4096 for RSA keys, 521 is the maximum for ECDSA keys + ssh_key_type: ecdsa # set to rsa to create an RSA key + public_key_name: id_ecdsa.pub # this might be id_rsa.pub for RSA keys, existing users may have a key of a different name # Main repo. own_repository: "https://github.com/codeenigma/ce-provision.git" own_repository_branch: "master" diff --git a/roles/aws/aws_ami/README.md b/roles/aws/aws_ami/README.md index f4f3bd2be..acc2f10e4 100644 --- a/roles/aws/aws_ami/README.md +++ b/roles/aws/aws_ami/README.md @@ -29,6 +29,7 @@ aws_ami: ami_name: "example" owner: "136693071363" # Global AWS account ID of owner, defaults to Debian official ssh_username: "admin" + public_key_name: id_ecdsa.pub # from Debian 12 (Bookworm) onwards RSA keys, i.e. id_rsa.pub, are deprecated encrypt_boot: false # EBS volume options device_name: /dev/xvda # default for Debian AMIs diff --git a/roles/aws/aws_ami/defaults/main.yml b/roles/aws/aws_ami/defaults/main.yml index 4b9bcbf1c..c828adaec 100644 --- a/roles/aws/aws_ami/defaults/main.yml +++ b/roles/aws/aws_ami/defaults/main.yml @@ -9,6 +9,7 @@ aws_ami: ami_name: "example" owner: "136693071363" # Global AWS account ID of owner, defaults to Debian official ssh_username: "admin" + public_key_name: id_ecdsa.pub # from Debian 12 (Bookworm) onwards RSA keys, i.e. id_rsa.pub, are deprecated encrypt_boot: false # EBS volume options device_name: /dev/xvda # default for Debian AMIs diff --git a/roles/aws/aws_ami/templates/packer.json.j2 b/roles/aws/aws_ami/templates/packer.json.j2 index 7f04c6b10..ae3c72ef5 100755 --- a/roles/aws/aws_ami/templates/packer.json.j2 +++ b/roles/aws/aws_ami/templates/packer.json.j2 @@ -50,27 +50,25 @@ ], "provisioners": [{ "type": "ansible", - "use_proxy": false, "playbook_file": "{{ aws_ami.playbook_file }}", "inventory_directory": "{{ _ce_provision_base_dir }}/hosts", - "ssh_authorized_key_file": "/home/{{ user_provision.username }}/.ssh/id_rsa.pub", + "ssh_authorized_key_file": "/home/{{ user_provision.username }}/.ssh/{{ aws_ami.public_key_name }}", {% if aws_ami.groups is defined and aws_ami.groups | length %} "groups": {{ aws_ami.groups | to_json }}, {% endif %} "ansible_env_vars": [ - "ANSIBLE_FORCE_COLOR=True", - "ALLOW_WORLD_READABLE_TMPFILES=True", + "ANSIBLE_FORCE_COLOR=True", + "ALLOW_WORLD_READABLE_TMPFILES=True", "ANSIBLE_CONFIG={{ _ce_provision_base_dir }}/ansible.cfg" ], "extra_arguments": [ - "--ssh-extra-args", "-o IdentitiesOnly=yes -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa", {% if ansible_verbosity >= 1 %} "-vvvv", {% endif %} {% if _aws_ami_extra_vars is defined and _aws_ami_extra_vars | length %} - "--extra-vars", + "--extra-vars", "{{ _aws_ami_extra_vars }}", {% endif %} "--extra-vars", diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index 856e548eb..d3c939fd2 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -20,7 +20,9 @@ ce_deploy: # Other ce-deploy settings. aws_support: true # installs boto3 new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user - key_name: id_rsa.pub # existing users may have a key of a different name + ssh_key_bits: "521" # recommended to use 4096 for RSA keys, 521 is the maximum for ECDSA keys + ssh_key_type: ecdsa # set to rsa to create an RSA key + public_key_name: id_ecdsa.pub # this might be id_rsa.pub for RSA keys, existing users may have a key of a different name username: "{{ _ce_deploy.username }}" own_repository: "https://github.com/codeenigma/ce-deploy.git" own_repository_branch: "master" diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index d43bde3e6..dfccea372 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -12,7 +12,9 @@ ce_deploy: # Other ce-deploy settings. aws_support: true # installs boto3 new_user: true # set to false if user already exists or is ephemeral, e.g. an LDAP user - key_name: id_rsa.pub # existing users may have a key of a different name + ssh_key_bits: "521" # recommended to use 4096 for RSA keys, 521 is the maximum for ECDSA keys + ssh_key_type: ecdsa # set to rsa to create an RSA key + public_key_name: id_ecdsa.pub # this might be id_rsa.pub for RSA keys, existing users may have a key of a different name username: "{{ _ce_deploy.username }}" own_repository: "https://github.com/codeenigma/ce-deploy.git" own_repository_branch: "master" diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index d5bbb3761..447cd99c1 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -16,7 +16,8 @@ generate_ssh_key: true home: "/home/{{ ce_deploy.username }}" password: "*" - ssh_key_bits: 4096 + ssh_key_bits: "{{ ce_deploy.ssh_key_bits }}" + ssh_key_type: "{{ ce_deploy.ssh_key_type }}" ssh_key_comment: "{{ ce_deploy.username }}@{{ ansible_host }}" system: true groups: "{{ ce_deploy.groups }}" @@ -35,7 +36,7 @@ - name: Place the public key in the ce-provision data directory on the controller server. ansible.builtin.fetch: dest: "{{ _ce_provision_data_dir }}" - src: "/home/{{ ce_deploy.username }}/.ssh/{{ ce_deploy.key_name }}" + src: "/home/{{ ce_deploy.username }}/.ssh/{{ ce_deploy.public_key_name }}" - name: Ensure Git is installed. ansible.builtin.apt: diff --git a/roles/debian/ce_provision/README.md b/roles/debian/ce_provision/README.md index 8611d76b4..e3816975d 100644 --- a/roles/debian/ce_provision/README.md +++ b/roles/debian/ce_provision/README.md @@ -21,7 +21,9 @@ ce_provision: new_user: "{{ _init.ce_provision_new_user }}" # see _init defaults, set to false if user already exists or is ephemeral, e.g. an LDAP user username: "{{ _ce_provision_username }}" # see _init defaults #uid: "{{ _init.ce_provision_uid }}" # see _init defaults, optionally hardcode the UID for this user - public_key_name: id_rsa.pub # existing users may have a key of a different name + ssh_key_bits: "521" # recommended to use 4096 for RSA keys, 521 is the maximum for ECDSA keys + ssh_key_type: ecdsa # set to rsa to create an RSA key + public_key_name: id_ecdsa.pub # this might be id_rsa.pub for RSA keys, existing users may have a key of a different name # Main repo. own_repository: "https://github.com/codeenigma/ce-provision.git" own_repository_branch: "master" diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index 62c92b9e9..863be9952 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -12,7 +12,9 @@ ce_provision: new_user: "{{ _init.ce_provision_new_user }}" # see _init defaults, set to false if user already exists or is ephemeral, e.g. an LDAP user username: "{{ _ce_provision_username }}" # see _init defaults #uid: "{{ _init.ce_provision_uid }}" # see _init defaults, optionally hardcode the UID for this user - public_key_name: id_rsa.pub # existing users may have a key of a different name + ssh_key_bits: "521" # recommended to use 4096 for RSA keys, 521 is the maximum for ECDSA keys + ssh_key_type: ecdsa # set to rsa to create an RSA key + public_key_name: id_ecdsa.pub # this might be id_rsa.pub for RSA keys, existing users may have a key of a different name # Main repo. own_repository: "https://github.com/codeenigma/ce-provision.git" own_repository_branch: "master" diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index d19b458cc..7a531051d 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -30,7 +30,8 @@ generate_ssh_key: true home: "/home/{{ ce_provision.username }}" password: "*" - ssh_key_bits: 4096 + ssh_key_bits: "{{ ce_provision.ssh_key_bits }}" + ssh_key_type: "{{ ce_provision.ssh_key_type }}" ssh_key_comment: "{{ ce_provision.username }}@{{ ansible_host }}" system: true groups: "{{ ce_provision.groups }}" From f6ed39fafbb9d387bf36c213b23126dfcc80ef3d Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 3 Jun 2024 17:40:59 +0200 Subject: [PATCH 051/206] Adding SCP args for legacy mode needed by Packer. --- roles/aws/aws_ami/templates/packer.json.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/aws/aws_ami/templates/packer.json.j2 b/roles/aws/aws_ami/templates/packer.json.j2 index ae3c72ef5..d2abad8af 100755 --- a/roles/aws/aws_ami/templates/packer.json.j2 +++ b/roles/aws/aws_ami/templates/packer.json.j2 @@ -64,6 +64,7 @@ ], "extra_arguments": [ + "--scp-extra-args", "'-O'", {% if ansible_verbosity >= 1 %} "-vvvv", {% endif %} From f43419c5b2c553c5df1eaaae580b4b54ec1a1a59 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 4 Jun 2024 13:42:09 +0200 Subject: [PATCH 052/206] Adding an extra when clause to ACM SAN cert check. --- roles/aws/aws_acm/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/aws/aws_acm/tasks/main.yml b/roles/aws/aws_acm/tasks/main.yml index 30e49ea9d..944a1aaa5 100644 --- a/roles/aws/aws_acm/tasks/main.yml +++ b/roles/aws/aws_acm/tasks/main.yml @@ -75,6 +75,7 @@ _aws_acm_extra_domains_in_cert_list: false with_items: "{{ _acm_san_domains }}" when: + - aws_acm_certificate | length > 0 - _acm_san_domains is defined - item not in aws_acm_certificate.SubjectAlternativeNameSummaries - aws_acm.extra_domains | length > 0 From 511ac8dafe8d2ae94f02d8f7baeb8538e35b4911 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 4 Jun 2024 14:09:05 +0200 Subject: [PATCH 053/206] Trying different approach to ACM SAN cert check. --- roles/aws/aws_acm/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/aws/aws_acm/tasks/main.yml b/roles/aws/aws_acm/tasks/main.yml index 94932ace8..7b717b257 100644 --- a/roles/aws/aws_acm/tasks/main.yml +++ b/roles/aws/aws_acm/tasks/main.yml @@ -23,7 +23,8 @@ aws_acm_certificate_arn: "" _aws_acm_domain_in_cert_list: false _aws_acm_extra_domains_in_cert_list: true - aws_acm_certificate: {} + aws_acm_certificate: + SubjectAlternativeNameSummaries: [] - name: Iterate through the certificate list checking for domain. ansible.builtin.set_fact: @@ -76,7 +77,6 @@ _aws_acm_extra_domains_in_cert_list: false with_items: "{{ _acm_san_domains }}" when: - - aws_acm_certificate | length > 0 - _acm_san_domains is defined - item not in aws_acm_certificate.SubjectAlternativeNameSummaries - aws_acm.extra_domains | length > 0 From f480f7475c2566f07bb374affd5d8c74275fea03 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 5 Jun 2024 14:20:51 +0200 Subject: [PATCH 054/206] Removing /bin/which from rkhunter defaults, it isn't present in Debian 11. --- docs/roles/debian/rkhunter.md | 1 - roles/debian/rkhunter/README.md | 1 - roles/debian/rkhunter/defaults/main.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/docs/roles/debian/rkhunter.md b/docs/roles/debian/rkhunter.md index 8609a6315..e91854742 100644 --- a/docs/roles/debian/rkhunter.md +++ b/docs/roles/debian/rkhunter.md @@ -25,7 +25,6 @@ rkhunter: scriptwhitelist: - /bin/egrep - /bin/fgrep - - /bin/which - /usr/bin/ldd # - /usr/bin/lwp-request - /usr/sbin/adduser diff --git a/roles/debian/rkhunter/README.md b/roles/debian/rkhunter/README.md index 8609a6315..e91854742 100644 --- a/roles/debian/rkhunter/README.md +++ b/roles/debian/rkhunter/README.md @@ -25,7 +25,6 @@ rkhunter: scriptwhitelist: - /bin/egrep - /bin/fgrep - - /bin/which - /usr/bin/ldd # - /usr/bin/lwp-request - /usr/sbin/adduser diff --git a/roles/debian/rkhunter/defaults/main.yml b/roles/debian/rkhunter/defaults/main.yml index 9afbdd084..a6a958cb6 100644 --- a/roles/debian/rkhunter/defaults/main.yml +++ b/roles/debian/rkhunter/defaults/main.yml @@ -16,7 +16,6 @@ rkhunter: scriptwhitelist: - /bin/egrep - /bin/fgrep - - /bin/which - /usr/bin/ldd # - /usr/bin/lwp-request - /usr/sbin/adduser From 5eae4cf3070d1c0d027e47dfaf14b3c3221a8884 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 5 Jun 2024 14:25:22 +0200 Subject: [PATCH 055/206] RDS param group module has changed name. --- roles/aws/aws_rds/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/aws/aws_rds/tasks/main.yml b/roles/aws/aws_rds/tasks/main.yml index bf4720f5e..7b39674b2 100644 --- a/roles/aws/aws_rds/tasks/main.yml +++ b/roles/aws/aws_rds/tasks/main.yml @@ -10,7 +10,7 @@ register: _aws_rds_rds_subnet_group - name: Create RDS parameter group - amazon.aws.rds_param_group: + amazon.aws.rds_instance_param_group: state: present name: "{{ aws_rds.db_parameter_group_name }}" profile: "{{ aws_rds.aws_profile }}" From 14516cb5a66f7c94dcd5474b70adbeaa6a4b8f53 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 5 Jun 2024 15:47:46 +0200 Subject: [PATCH 056/206] Adding passlib to libraries installed for ce-provision. --- roles/debian/ce_provision/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index 7a531051d..d81941702 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -202,7 +202,7 @@ name: debian/python_boto when: ce_provision.aws_support -- name: Ensure dnspython, certifi and pyopenssl are installed. +- name: Ensure dnspython, certifi, pyopenssl and passlib are installed. ansible.builtin.include_role: name: debian/python_pip_packages vars: @@ -211,6 +211,7 @@ - name: dnspython - name: certifi - name: pyopenssl + - name: passlib - name: Add Python virtual environment to $PATH. ansible.builtin.copy: From 69ce89497e90441fea40507d6185a1c570628ce2 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 5 Jun 2024 15:48:06 +0200 Subject: [PATCH 057/206] Adding in valid path for 'which' to rkhunter. --- docs/roles/debian/rkhunter.md | 1 + roles/debian/rkhunter/README.md | 1 + roles/debian/rkhunter/defaults/main.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/docs/roles/debian/rkhunter.md b/docs/roles/debian/rkhunter.md index e91854742..cb452f702 100644 --- a/docs/roles/debian/rkhunter.md +++ b/docs/roles/debian/rkhunter.md @@ -30,6 +30,7 @@ rkhunter: - /usr/sbin/adduser # - /usr/sbin/prelink - /usr/sbin/unhide.rb + - /usr/bin/which allowhiddendir: - /etc/.java - /tmp/.ce-deploy # see https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L7 diff --git a/roles/debian/rkhunter/README.md b/roles/debian/rkhunter/README.md index e91854742..cb452f702 100644 --- a/roles/debian/rkhunter/README.md +++ b/roles/debian/rkhunter/README.md @@ -30,6 +30,7 @@ rkhunter: - /usr/sbin/adduser # - /usr/sbin/prelink - /usr/sbin/unhide.rb + - /usr/bin/which allowhiddendir: - /etc/.java - /tmp/.ce-deploy # see https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L7 diff --git a/roles/debian/rkhunter/defaults/main.yml b/roles/debian/rkhunter/defaults/main.yml index a6a958cb6..30e20d59a 100644 --- a/roles/debian/rkhunter/defaults/main.yml +++ b/roles/debian/rkhunter/defaults/main.yml @@ -21,6 +21,7 @@ rkhunter: - /usr/sbin/adduser # - /usr/sbin/prelink - /usr/sbin/unhide.rb + - /usr/bin/which allowhiddendir: - /etc/.java - /tmp/.ce-deploy # see https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L7 From 1a0241a4b8f4d556326db6a0c76064e1f869a364 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 13 Jun 2024 16:03:13 +0200 Subject: [PATCH 058/206] Catching up documentation. --- docs/roles/aws/aws_acl.md | 89 ++++++++++++++++++++++++++++++++------ docs/roles/debian/wazuh.md | 21 ++++----- 2 files changed, 86 insertions(+), 24 deletions(-) diff --git a/docs/roles/aws/aws_acl.md b/docs/roles/aws/aws_acl.md index a318bad7b..fb2c8efb5 100644 --- a/docs/roles/aws/aws_acl.md +++ b/docs/roles/aws/aws_acl.md @@ -15,32 +15,93 @@ aws_acl: region: "us-east-1" tags: {} rules: - rate_limit: 200 # set to 0 to skip rate limit rule, set to a value to set how many requests to allow in period before blocking - botControl: "COMMON" # or set to TARGETED inspection level (comment out to avoid addign rule) - + rate_limit: + value: 200 # set to 0 to skip rate limit rule, set to a value to set how many requests to allow in period before blocking + priority: 2 ip_sets: - - name: "Allowed-ips-example" + - name: "Allowed-ips" action: allow - list: [] + priority: 1 + list: [] # If the list is empty, ip set won't be recreated #- 1.1.1.1/32 # list of ip ranges #- 2.2.2.2/32 #- 3.3.3.3/32 - - name: "Blocked-ips-example" + - name: "Blocked-ips" action: block - list: [] + priority: 0 + list: [] # If the list is empty, ip set won't be recreated #- 4.4.4.4/32 # list of ip ranges #- 5.5.5.5/32 #- 6.6.6.6/32 - - cc_block_list: [] - + country_codes: + - name: "allowed-countries" + action: allow + priority: 7 + list: + - GB + - HR + - name: "blocked-countries" + action: block + priority: 8 + list: + - RU + - CN regular_rules: - name: allow_panels action: allow - string: "panels/ajax" - position: "CONTAINS" - -# cyber_sec: #Need to implement task + statements_type: "single" # supported "and", "or" and "not" ("and" and "or" supports multiple statements) + priority: 4 + statements: + - inspect: "UriPath" # Aslo supported: "SingleHeader" + position: "CONTAINS" + string: "panels/ajax" + text_trans: "NONE" + - name: block_bots + action: block + statements_type: "or" # supported "and", "or" and "not" ("and" and "or" supports multiple statements) + priority: 5 + statements: + - inspect: "SingleHeader" # Aslo supported: "SingleHeader" + position: "CONTAINS" + string: "spider" + text_trans: "LOWERCASE" + - inspect: "SingleHeader" # Aslo supported: "SingleHeader" + position: "CONTAINS" + string: "bot" + text_trans: "LOWERCASE" + - inspect: "SingleHeader" # Aslo supported: "SingleHeader" + position: "CONTAINS" + string: "crawl" + text_trans: "LOWERCASE" + # Managed rules list + bot_control: + enabled: false + target: "COMMON" # or set to TARGETED inspection level (comment out to avoid addign rule) + priority: 3 + cyber_sec: + enabled: false # Need to subscribe first in AWS + rule_list: [] + priority: 6 + amazon_ip_reputation: + enabled: false + rule_list: [] + priority: 9 + common_rule_set: + enabled: false + rule_list: [] + priority: 10 + php_rule_set: + enabled: false + rule_list: [] + priority: 11 + known_bad_inputs: + enabled: false + rule_list: [] + priority: 12 + anonymous_ip_list: + enabled: false + rule_list: [] + priority: 13 ``` diff --git a/docs/roles/debian/wazuh.md b/docs/roles/debian/wazuh.md index bd4afb89b..5bb89b58c 100644 --- a/docs/roles/debian/wazuh.md +++ b/docs/roles/debian/wazuh.md @@ -120,29 +120,29 @@ wazuh: wazuh_manager_api: bind_addr: 0.0.0.0 port: 55000 - behind_proxy_server: no - https: yes + behind_proxy_server: "no" + https: "yes" https_key: "api/configuration/ssl/server.key" https_cert: "api/configuration/ssl/server.crt" - https_use_ca: False + https_use_ca: false https_ca: "api/configuration/ssl/ca.crt" logging_level: "info" logging_path: "logs/api.log" - cors: no + cors: "no" cors_source_route: "*" cors_expose_headers: "*" cors_allow_headers: "*" - cors_allow_credentials: no - cache: yes + cors_allow_credentials: "no" + cache: "yes" cache_time: 0.750 access_max_login_attempts: 5 access_block_time: 300 access_max_request_per_minute: 300 - drop_privileges: yes - experimental_features: no - remote_commands_localfile: yes + drop_privileges: "yes" + experimental_features: "no" + remote_commands_localfile: "yes" remote_commands_localfile_exceptions: [] - remote_commands_wodle: yes + remote_commands_wodle: "yes" remote_commands_wodle_exceptions: [] #wazuh_api_users: # - username: custom-user @@ -165,6 +165,7 @@ wazuh: dashboard_security: true dashboard_user: kibanaserver dashboard_password: changeme + ``` From 5001c61ad724d2e6e4ae96f945b50e817993171b Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 13 Jun 2024 16:03:37 +0200 Subject: [PATCH 059/206] Catching up documentation. --- roles/aws/aws_acl/README.md | 25 +++++++++---------------- roles/debian/wazuh/README.md | 21 +++++++++++---------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/roles/aws/aws_acl/README.md b/roles/aws/aws_acl/README.md index 86ccbd839..fb2c8efb5 100644 --- a/roles/aws/aws_acl/README.md +++ b/roles/aws/aws_acl/README.md @@ -5,7 +5,7 @@ Creates an ACL to be attached to a CloudFront distribution or an Application Loa -## Default variables for creation of ACL (pass it as a list) +## Default variables ```yaml --- aws_acl: @@ -73,43 +73,36 @@ aws_acl: position: "CONTAINS" string: "crawl" text_trans: "LOWERCASE" - # AWS Managed rules + # Managed rules list bot_control: + enabled: false target: "COMMON" # or set to TARGETED inspection level (comment out to avoid addign rule) priority: 3 cyber_sec: - enabled: true # Need to subscribe first in AWS + enabled: false # Need to subscribe first in AWS rule_list: [] priority: 6 amazon_ip_reputation: - enabled: true + enabled: false rule_list: [] priority: 9 common_rule_set: - enabled: true + enabled: false rule_list: [] priority: 10 php_rule_set: - enabled: true + enabled: false rule_list: [] priority: 11 known_bad_inputs: - enabled: true + enabled: false rule_list: [] priority: 12 anonymous_ip_list: - enabled: true + enabled: false rule_list: [] priority: 13 -``` -## Default variables for assigning ACL to CF or ALB -```yaml ---- -aws_acl: - name: example_master_acl # Name of the ACL to apply - scope: CLOUDFRONT # Can be REGIONAL for ALBs - region: "us-east-1" ``` diff --git a/roles/debian/wazuh/README.md b/roles/debian/wazuh/README.md index bd4afb89b..5bb89b58c 100644 --- a/roles/debian/wazuh/README.md +++ b/roles/debian/wazuh/README.md @@ -120,29 +120,29 @@ wazuh: wazuh_manager_api: bind_addr: 0.0.0.0 port: 55000 - behind_proxy_server: no - https: yes + behind_proxy_server: "no" + https: "yes" https_key: "api/configuration/ssl/server.key" https_cert: "api/configuration/ssl/server.crt" - https_use_ca: False + https_use_ca: false https_ca: "api/configuration/ssl/ca.crt" logging_level: "info" logging_path: "logs/api.log" - cors: no + cors: "no" cors_source_route: "*" cors_expose_headers: "*" cors_allow_headers: "*" - cors_allow_credentials: no - cache: yes + cors_allow_credentials: "no" + cache: "yes" cache_time: 0.750 access_max_login_attempts: 5 access_block_time: 300 access_max_request_per_minute: 300 - drop_privileges: yes - experimental_features: no - remote_commands_localfile: yes + drop_privileges: "yes" + experimental_features: "no" + remote_commands_localfile: "yes" remote_commands_localfile_exceptions: [] - remote_commands_wodle: yes + remote_commands_wodle: "yes" remote_commands_wodle_exceptions: [] #wazuh_api_users: # - username: custom-user @@ -165,6 +165,7 @@ wazuh: dashboard_security: true dashboard_user: kibanaserver dashboard_password: changeme + ``` From e1c15c041786357611526cc5417f4a99a3a2bbb8 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 13 Jun 2024 16:05:58 +0200 Subject: [PATCH 060/206] Making user creation optional and home directories a variable. --- docs/roles/debian/user_ansible.md | 39 +++++++++++++++++++ roles/debian/user_ansible/README.md | 39 +++++++++++++++++++ roles/debian/user_ansible/defaults/main.yml | 9 +++-- roles/debian/user_ansible/tasks/main.yml | 1 + roles/debian/user_deploy/defaults/main.yml | 5 ++- roles/debian/user_provision/defaults/main.yml | 5 ++- 6 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 docs/roles/debian/user_ansible.md create mode 100644 roles/debian/user_ansible/README.md diff --git a/docs/roles/debian/user_ansible.md b/docs/roles/debian/user_ansible.md new file mode 100644 index 000000000..d53d75323 --- /dev/null +++ b/docs/roles/debian/user_ansible.md @@ -0,0 +1,39 @@ +# User Ansible +General role to create Linux users and corresponding keys, groups, home directory, etc. You may call this role directly or it can be imported, such as when called by the `user_provision` and `user_deploy` roles for handling our system users. + + + + + +## Default variables +```yaml +--- +_user_ansible_username: "" +user_ansible: + # This sets both username and main group. + username: "{{ _user_ansible_username }}" + home: "/home/{{ _user_ansible_username }}" + create: true # if you know the user already exists, set this to false to not create the user. + # Optional Linux uid and gid for user + # uid: 999 + # gid: 999 + # Local username of the deploy user. + utility_host: "localhost" + utility_username: "" + sudoer: + false + # List of additional groups to add the user to. + groups: [] + # List of SSH pub keys to authorize. These must be provided as strings (content of the pub key). + ssh_keys: [] + # List of SSH private keys to add to server. These must be provided as strings (content of the private key). + # Be sure to store securely using SOPS or similar. + ssh_private_keys: [] + # List of hostnames to add to known_hosts. + known_hosts: [] + # Whether or not to hash any provided hosts for known_hosts. + known_hosts_hash: true + +``` + + diff --git a/roles/debian/user_ansible/README.md b/roles/debian/user_ansible/README.md new file mode 100644 index 000000000..d53d75323 --- /dev/null +++ b/roles/debian/user_ansible/README.md @@ -0,0 +1,39 @@ +# User Ansible +General role to create Linux users and corresponding keys, groups, home directory, etc. You may call this role directly or it can be imported, such as when called by the `user_provision` and `user_deploy` roles for handling our system users. + + + + + +## Default variables +```yaml +--- +_user_ansible_username: "" +user_ansible: + # This sets both username and main group. + username: "{{ _user_ansible_username }}" + home: "/home/{{ _user_ansible_username }}" + create: true # if you know the user already exists, set this to false to not create the user. + # Optional Linux uid and gid for user + # uid: 999 + # gid: 999 + # Local username of the deploy user. + utility_host: "localhost" + utility_username: "" + sudoer: + false + # List of additional groups to add the user to. + groups: [] + # List of SSH pub keys to authorize. These must be provided as strings (content of the pub key). + ssh_keys: [] + # List of SSH private keys to add to server. These must be provided as strings (content of the private key). + # Be sure to store securely using SOPS or similar. + ssh_private_keys: [] + # List of hostnames to add to known_hosts. + known_hosts: [] + # Whether or not to hash any provided hosts for known_hosts. + known_hosts_hash: true + +``` + + diff --git a/roles/debian/user_ansible/defaults/main.yml b/roles/debian/user_ansible/defaults/main.yml index bcccd873c..472018871 100644 --- a/roles/debian/user_ansible/defaults/main.yml +++ b/roles/debian/user_ansible/defaults/main.yml @@ -1,7 +1,10 @@ --- +_user_ansible_username: "" user_ansible: # This sets both username and main group. - username: "" + username: "{{ _user_ansible_username }}" + home: "/home/{{ _user_ansible_username }}" + create: true # if you know the user already exists, set this to false to not create the user. # Optional Linux uid and gid for user # uid: 999 # gid: 999 @@ -10,8 +13,8 @@ user_ansible: utility_username: "" sudoer: false - # List of additional groups to add the user to. - groups: "" + # List of additional groups to add the user to. + groups: [] # List of SSH pub keys to authorize. These must be provided as strings (content of the pub key). ssh_keys: [] # List of SSH private keys to add to server. These must be provided as strings (content of the private key). diff --git a/roles/debian/user_ansible/tasks/main.yml b/roles/debian/user_ansible/tasks/main.yml index c40bbed11..a59acf5d3 100644 --- a/roles/debian/user_ansible/tasks/main.yml +++ b/roles/debian/user_ansible/tasks/main.yml @@ -26,6 +26,7 @@ home: "/home/{{ user_ansible.username }}" password: "*" groups: "{{ user_ansible.groups }}" + when: user_ansible.create - name: Add user to sudoers. ansible.builtin.include_role: diff --git a/roles/debian/user_deploy/defaults/main.yml b/roles/debian/user_deploy/defaults/main.yml index 983c0f137..61edbde90 100644 --- a/roles/debian/user_deploy/defaults/main.yml +++ b/roles/debian/user_deploy/defaults/main.yml @@ -1,8 +1,11 @@ --- +_user_deploy_username: "{% if is_local is defined and is_local %}ce-dev{% else %}deploy{% endif %}" user_deploy: # This sets both username and main group. # If you are using ce-deploy to deploy code this must match the `deploy_user` variable - username: "{% if is_local is defined and is_local %}ce-dev{% else %}deploy{% endif %}" + username: "{{ _user_deploy_username }}}" + home: "/home/{{ _user_deploy_username }}}" + create: true # if you know the user already exists, set this to false to not create the user. # It is often desirable to fix POSIX IDs for system users so they are consistent across your fleet # This is shown for documentation, you should do this in your config repo # uid: 989 diff --git a/roles/debian/user_provision/defaults/main.yml b/roles/debian/user_provision/defaults/main.yml index 3f53b4d71..9656919bd 100644 --- a/roles/debian/user_provision/defaults/main.yml +++ b/roles/debian/user_provision/defaults/main.yml @@ -1,7 +1,10 @@ --- +_user_provision_username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}" user_provision: # This sets both username and main group. - username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}" + username: "{{ _user_provision_username }}" + home: "/home/{{ _user_provision_username }}}" + create: true # if you know the user already exists, set this to false to not create the user. # It is often desirable to fix POSIX IDs for system users so they are consistent across your fleet # This is shown for documentation, you should do this in your config repo # uid: 988 From fff110e7452e036400555a8b641b46d5c794783f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 13 Jun 2024 16:11:58 +0200 Subject: [PATCH 061/206] Missed passing new home var to task. --- docs/_Sidebar.md | 1 + docs/roles/debian/user_ansible.md | 1 + roles/debian/user_ansible/README.md | 1 + roles/debian/user_ansible/defaults/main.yml | 1 + roles/debian/user_ansible/tasks/main.yml | 4 ++-- roles/debian/user_deploy/defaults/main.yml | 1 + roles/debian/user_provision/defaults/main.yml | 1 + 7 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md index 6386f333b..90b0fc558 100644 --- a/docs/_Sidebar.md +++ b/docs/_Sidebar.md @@ -87,6 +87,7 @@ - [SSL](/roles/debian/ssl) - [sudo config](/roles/debian/sudo_config) - [Swap](/roles/debian/swap) + - [User Ansible](/roles/debian/user_ansible) - [varnish_config](/roles/debian/varnish_config) - [wazuh](/roles/debian/wazuh) - [Init role](/roles/_init) diff --git a/docs/roles/debian/user_ansible.md b/docs/roles/debian/user_ansible.md index d53d75323..397d4318f 100644 --- a/docs/roles/debian/user_ansible.md +++ b/docs/roles/debian/user_ansible.md @@ -14,6 +14,7 @@ user_ansible: username: "{{ _user_ansible_username }}" home: "/home/{{ _user_ansible_username }}" create: true # if you know the user already exists, set this to false to not create the user. + create_home: true # Optional Linux uid and gid for user # uid: 999 # gid: 999 diff --git a/roles/debian/user_ansible/README.md b/roles/debian/user_ansible/README.md index d53d75323..397d4318f 100644 --- a/roles/debian/user_ansible/README.md +++ b/roles/debian/user_ansible/README.md @@ -14,6 +14,7 @@ user_ansible: username: "{{ _user_ansible_username }}" home: "/home/{{ _user_ansible_username }}" create: true # if you know the user already exists, set this to false to not create the user. + create_home: true # Optional Linux uid and gid for user # uid: 999 # gid: 999 diff --git a/roles/debian/user_ansible/defaults/main.yml b/roles/debian/user_ansible/defaults/main.yml index 472018871..a0b64b3de 100644 --- a/roles/debian/user_ansible/defaults/main.yml +++ b/roles/debian/user_ansible/defaults/main.yml @@ -5,6 +5,7 @@ user_ansible: username: "{{ _user_ansible_username }}" home: "/home/{{ _user_ansible_username }}" create: true # if you know the user already exists, set this to false to not create the user. + create_home: true # Optional Linux uid and gid for user # uid: 999 # gid: 999 diff --git a/roles/debian/user_ansible/tasks/main.yml b/roles/debian/user_ansible/tasks/main.yml index a59acf5d3..e77e698b0 100644 --- a/roles/debian/user_ansible/tasks/main.yml +++ b/roles/debian/user_ansible/tasks/main.yml @@ -22,8 +22,8 @@ group: "{{ user_ansible.username }}" uid: "{{ user_ansible.uid | default(omit) }}" system: true - create_home: true - home: "/home/{{ user_ansible.username }}" + create_home: "{{ user_ansible.create_home }}" + home: "{{ user_ansible.home }}" password: "*" groups: "{{ user_ansible.groups }}" when: user_ansible.create diff --git a/roles/debian/user_deploy/defaults/main.yml b/roles/debian/user_deploy/defaults/main.yml index 61edbde90..9d45adf3f 100644 --- a/roles/debian/user_deploy/defaults/main.yml +++ b/roles/debian/user_deploy/defaults/main.yml @@ -6,6 +6,7 @@ user_deploy: username: "{{ _user_deploy_username }}}" home: "/home/{{ _user_deploy_username }}}" create: true # if you know the user already exists, set this to false to not create the user. + create_home: true # It is often desirable to fix POSIX IDs for system users so they are consistent across your fleet # This is shown for documentation, you should do this in your config repo # uid: 989 diff --git a/roles/debian/user_provision/defaults/main.yml b/roles/debian/user_provision/defaults/main.yml index 9656919bd..a6a08284c 100644 --- a/roles/debian/user_provision/defaults/main.yml +++ b/roles/debian/user_provision/defaults/main.yml @@ -5,6 +5,7 @@ user_provision: username: "{{ _user_provision_username }}" home: "/home/{{ _user_provision_username }}}" create: true # if you know the user already exists, set this to false to not create the user. + create_home: true # It is often desirable to fix POSIX IDs for system users so they are consistent across your fleet # This is shown for documentation, you should do this in your config repo # uid: 988 From 9f4e35939570c4825ca0d10acc4c70545f8c5d31 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 13 Jun 2024 16:38:35 +0200 Subject: [PATCH 062/206] Fixing firewall.bash deletion issues. --- roles/debian/firewall_config/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/debian/firewall_config/tasks/main.yml b/roles/debian/firewall_config/tasks/main.yml index cf770ab4e..8d64de18a 100644 --- a/roles/debian/firewall_config/tasks/main.yml +++ b/roles/debian/firewall_config/tasks/main.yml @@ -96,6 +96,11 @@ state: absent when: firewall_config.purge +- name: Touch firewall.bash to ensure it exists and avoid a crash. + ansible.builtin.file: + path: /etc/firewall.bash + state: touch + - name: Apply firewall rules. ansible.builtin.import_role: name: geerlingguy.firewall From f751953d666071b125df8941847756385432c190 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 13 Jun 2024 17:29:22 +0200 Subject: [PATCH 063/206] Getting rid of accidental extra braces. --- roles/debian/user_deploy/defaults/main.yml | 4 ++-- roles/debian/user_provision/defaults/main.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/debian/user_deploy/defaults/main.yml b/roles/debian/user_deploy/defaults/main.yml index 9d45adf3f..084fbb153 100644 --- a/roles/debian/user_deploy/defaults/main.yml +++ b/roles/debian/user_deploy/defaults/main.yml @@ -3,8 +3,8 @@ _user_deploy_username: "{% if is_local is defined and is_local %}ce-dev{% else % user_deploy: # This sets both username and main group. # If you are using ce-deploy to deploy code this must match the `deploy_user` variable - username: "{{ _user_deploy_username }}}" - home: "/home/{{ _user_deploy_username }}}" + username: "{{ _user_deploy_username }}" + home: "/home/{{ _user_deploy_username }}" create: true # if you know the user already exists, set this to false to not create the user. create_home: true # It is often desirable to fix POSIX IDs for system users so they are consistent across your fleet diff --git a/roles/debian/user_provision/defaults/main.yml b/roles/debian/user_provision/defaults/main.yml index a6a08284c..a7a5a49d2 100644 --- a/roles/debian/user_provision/defaults/main.yml +++ b/roles/debian/user_provision/defaults/main.yml @@ -3,7 +3,7 @@ _user_provision_username: "{% if is_local is defined and is_local %}ce-dev{% els user_provision: # This sets both username and main group. username: "{{ _user_provision_username }}" - home: "/home/{{ _user_provision_username }}}" + home: "/home/{{ _user_provision_username }}" create: true # if you know the user already exists, set this to false to not create the user. create_home: true # It is often desirable to fix POSIX IDs for system users so they are consistent across your fleet From 5fef05f4c4b19ce11f99324819f53c623f126e16 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 13 Jun 2024 17:34:26 +0200 Subject: [PATCH 064/206] Simplifying usernames so you only need to set one var. --- roles/debian/user_ansible/defaults/main.yml | 2 +- roles/debian/user_deploy/defaults/main.yml | 2 +- roles/debian/user_provision/defaults/main.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/debian/user_ansible/defaults/main.yml b/roles/debian/user_ansible/defaults/main.yml index a0b64b3de..c3cb758e8 100644 --- a/roles/debian/user_ansible/defaults/main.yml +++ b/roles/debian/user_ansible/defaults/main.yml @@ -11,7 +11,7 @@ user_ansible: # gid: 999 # Local username of the deploy user. utility_host: "localhost" - utility_username: "" + utility_username: "{{ _user_ansible_username }}" sudoer: false # List of additional groups to add the user to. diff --git a/roles/debian/user_deploy/defaults/main.yml b/roles/debian/user_deploy/defaults/main.yml index 084fbb153..711f53c89 100644 --- a/roles/debian/user_deploy/defaults/main.yml +++ b/roles/debian/user_deploy/defaults/main.yml @@ -13,7 +13,7 @@ user_deploy: # gid: 989 # Local username of the deploy user. utility_host: "localhost" - utility_username: "{% if is_local is defined and is_local %}ce-dev{% else %}deploy{% endif %}" + utility_username: "{{ _user_deploy_username }}" sudoer: false # List of additional groups to add the user to. groups: diff --git a/roles/debian/user_provision/defaults/main.yml b/roles/debian/user_provision/defaults/main.yml index a7a5a49d2..a1c8799cf 100644 --- a/roles/debian/user_provision/defaults/main.yml +++ b/roles/debian/user_provision/defaults/main.yml @@ -12,7 +12,7 @@ user_provision: # gid: 988 # Local username of the system user. utility_host: "localhost" - utility_username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}" + utility_username: "{{ _user_provision_username }}" sudoer: true groups: [] ssh_keys: [] From 6fcda09e11cd2648b9cb78d3b8640fb884aa3b44 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 18 Jul 2024 14:29:51 +0200 Subject: [PATCH 065/206] Docs update and making Ansible installation via _init an option. --- docs/roles/_init.md | 3 ++- docs/roles/aws/aws_opensearch.md | 2 ++ roles/_init/README.md | 3 ++- roles/_init/defaults/main.yml | 3 ++- roles/_init/tasks/main.yml | 43 ++++++++++++++++-------------- roles/aws/aws_opensearch/README.md | 2 ++ 6 files changed, 33 insertions(+), 23 deletions(-) diff --git a/docs/roles/_init.md b/docs/roles/_init.md index 27d3629e3..0f40180e3 100644 --- a/docs/roles/_init.md +++ b/docs/roles/_init.md @@ -27,7 +27,8 @@ _init: force_play: false lock_file: /tmp/ce-provision-lock deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy - ce_provision_version: 2.x # Outputted by the _init role at the start of plays. + ce_provision_version: 2.x # Outputted by the _init role at the start of plays + install_ansible: true # set to false to not install Ansible in a venv # Although these variables logically belong with ce_provision, the _init role needs to # gather the extra variables if there are any, so there are _init variables. diff --git a/docs/roles/aws/aws_opensearch.md b/docs/roles/aws/aws_opensearch.md index c27085bfa..f952e3e08 100644 --- a/docs/roles/aws/aws_opensearch.md +++ b/docs/roles/aws/aws_opensearch.md @@ -35,6 +35,8 @@ aws_opensearch: # value: 2 # unit: "HOURS" # cron_expression_for_recurrence: "cron(0 2 * * *)" + wait: true + wait_timeout: 1800 # Default is 300 seconds, but in fact the domain becomes available in about 30 minutes. ``` diff --git a/roles/_init/README.md b/roles/_init/README.md index 27d3629e3..0f40180e3 100644 --- a/roles/_init/README.md +++ b/roles/_init/README.md @@ -27,7 +27,8 @@ _init: force_play: false lock_file: /tmp/ce-provision-lock deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy - ce_provision_version: 2.x # Outputted by the _init role at the start of plays. + ce_provision_version: 2.x # Outputted by the _init role at the start of plays + install_ansible: true # set to false to not install Ansible in a venv # Although these variables logically belong with ce_provision, the _init role needs to # gather the extra variables if there are any, so there are _init variables. diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 8c2646bd8..97a5ad1bf 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -17,7 +17,8 @@ _init: force_play: false lock_file: /tmp/ce-provision-lock deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy - ce_provision_version: 2.x # Outputted by the _init role at the start of plays. + ce_provision_version: 2.x # Outputted by the _init role at the start of plays + install_ansible: true # set to false to not install Ansible in a venv # Although these variables logically belong with ce_provision, the _init role needs to # gather the extra variables if there are any, so there are _init variables. diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index 4abe95653..457afa494 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -123,23 +123,26 @@ # Install Ansible under the controller user for all servers # Ensure ansible_connection == 'ssh' (i.e. we are connecting to a server) before executing - name: Install Ansible. - ansible.builtin.include_role: - name: debian/user_provision - when: ansible_connection == 'ssh' - -- name: Install Ansible. - ansible.builtin.include_role: - name: debian/ansible - vars: - ce_ansible: - venv_path: "{{ _venv_path }}" - venv_command: "{{ _venv_command }}" - install_username: "{{ _venv_install_username }}" - upgrade: - enabled: true - command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" - on_calendar: "*-*-* 01:30:00" - timer_name: "{{ _ce_ansible_timer_name }}" - linters: - enabled: false - when: ansible_connection == 'ssh' + when: + - ansible_connection == 'ssh' + - install_ansible + block: + - name: Install ce-provision user. + ansible.builtin.include_role: + name: debian/user_provision + + - name: Install Ansible. + ansible.builtin.include_role: + name: debian/ansible + vars: + ce_ansible: + venv_path: "{{ _venv_path }}" + venv_command: "{{ _venv_command }}" + install_username: "{{ _venv_install_username }}" + upgrade: + enabled: true + command: "{{ _venv_path }}/bin/python3 -m pip install --upgrade ansible" + on_calendar: "*-*-* 01:30:00" + timer_name: "{{ _ce_ansible_timer_name }}" + linters: + enabled: false diff --git a/roles/aws/aws_opensearch/README.md b/roles/aws/aws_opensearch/README.md index c27085bfa..f952e3e08 100644 --- a/roles/aws/aws_opensearch/README.md +++ b/roles/aws/aws_opensearch/README.md @@ -35,6 +35,8 @@ aws_opensearch: # value: 2 # unit: "HOURS" # cron_expression_for_recurrence: "cron(0 2 * * *)" + wait: true + wait_timeout: 1800 # Default is 300 seconds, but in fact the domain becomes available in about 30 minutes. ``` From 87f82aa1406bca91dcf4dcd086dff5f8c419d5e9 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 18 Jul 2024 15:14:39 +0200 Subject: [PATCH 066/206] Variable path error. --- roles/_init/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index 457afa494..3207c13f4 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -125,7 +125,7 @@ - name: Install Ansible. when: - ansible_connection == 'ssh' - - install_ansible + - _init.install_ansible block: - name: Install ce-provision user. ansible.builtin.include_role: From 06f9c8dda4f3540c18b9d9f90cd17b50f8a9f0cc Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 18 Jul 2024 15:17:25 +0200 Subject: [PATCH 067/206] Updating linter ignore paths. --- .ansible-lint | 2 +- .yamllint | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.ansible-lint b/.ansible-lint index 390760f80..02d2d1c31 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -15,4 +15,4 @@ skip_list: - name[template] # it doesn't like Jinja templates being in the middle of a task name, which seems silly to me. - name[casing] # sometimes included Galaxy roles break linting rules and cause failures exclude_paths: - - roles/debian/wazuh/ # imported role uses yes/no instead of true/false so always fails + - roles/contrib/ # we don't control these roles diff --git a/.yamllint b/.yamllint index 8287b967d..0247745fe 100644 --- a/.yamllint +++ b/.yamllint @@ -18,3 +18,6 @@ rules: new-line-at-end-of-file: disable new-lines: {type: unix} trailing-spaces: enable + +ignore: + - roles/contrib/ From 3fc282b56349cb3e060cc37d7cd1345fd12a9773 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 19 Jul 2024 11:46:58 +0200 Subject: [PATCH 068/206] Making the NGINX test result var private. --- roles/debian/nginx/tasks/main.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/roles/debian/nginx/tasks/main.yml b/roles/debian/nginx/tasks/main.yml index 85c720068..35aa6fe5c 100644 --- a/roles/debian/nginx/tasks/main.yml +++ b/roles/debian/nginx/tasks/main.yml @@ -34,12 +34,12 @@ - name: Test NGINX configuration. ansible.builtin.command: nginx -t - register: nginx_test_result + register: _nginx_test_result failed_when: false - name: Display current NGINX config test result. ansible.builtin.debug: - msg: "{{ nginx_test_result.stderr }}" + msg: "{{ _nginx_test_result.stderr }}" - name: Copy main NGINX config. ansible.builtin.template: @@ -142,27 +142,27 @@ - name: Test NGINX configuration. ansible.builtin.command: nginx -t - register: nginx_test_result + register: _nginx_test_result failed_when: false - name: Display NGINX test result. ansible.builtin.debug: - msg: "{{ nginx_test_result.stderr }}" + msg: "{{ _nginx_test_result.stderr }}" - name: Ensure NGINX is restarted. ansible.builtin.service: name: nginx state: restarted - when: nginx_test_result.rc == 0 + when: _nginx_test_result.rc == 0 - name: Restore previous NGINX config from backup if test fails. ansible.builtin.unarchive: src: /home/{{ _ce_provision_username }}/nginx_backup.tar.gz dest: / remote_src: true - when: nginx_test_result.rc != 0 + when: _nginx_test_result.rc != 0 - name: Fail the playbook after restore. ansible.builtin.fail: msg: "NGINX configuration test failed. Previous configuration was restored. You need to check it manually." - when: nginx_test_result.rc != 0 + when: _nginx_test_result.rc != 0 From cc3519520f2f5eb92665573b92a4b40536a9521c Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 11 Sep 2024 13:18:16 +0200 Subject: [PATCH 069/206] Documentation update. --- docs/roles/debian/rkhunter.md | 2 ++ docs/roles/debian/user_ansible.md | 7 ++++--- docs/roles/debian/wazuh.md | 2 ++ roles/debian/rkhunter/README.md | 2 ++ roles/debian/user_ansible/README.md | 7 ++++--- roles/debian/wazuh/README.md | 2 ++ 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/roles/debian/rkhunter.md b/docs/roles/debian/rkhunter.md index cb452f702..30cb22852 100644 --- a/docs/roles/debian/rkhunter.md +++ b/docs/roles/debian/rkhunter.md @@ -25,6 +25,8 @@ rkhunter: scriptwhitelist: - /bin/egrep - /bin/fgrep + - /usr/bin/egrep + - /usr/bin/fgrep - /usr/bin/ldd # - /usr/bin/lwp-request - /usr/sbin/adduser diff --git a/docs/roles/debian/user_ansible.md b/docs/roles/debian/user_ansible.md index 117adcd08..5693b08e5 100644 --- a/docs/roles/debian/user_ansible.md +++ b/docs/roles/debian/user_ansible.md @@ -15,14 +15,15 @@ user_ansible: home: "/home/{{ _user_ansible_username }}" create: true # if you know the user already exists, set this to false to not create the user. create_home: true - # Optional Linux uid and gid for user + update_password: "always" + # It is often desirable to fix POSIX IDs for system users so they are consistent across your fleet + # This is shown for documentation, you should do this in your config repo # uid: 999 # gid: 999 # Local username of the deploy user. utility_host: "localhost" utility_username: "{{ _user_ansible_username }}" - sudoer: - false + sudoer: false # List of additional groups to add the user to. groups: [] # List of SSH pub keys to authorize. These must be provided as strings (content of the pub key). diff --git a/docs/roles/debian/wazuh.md b/docs/roles/debian/wazuh.md index 1ead6377f..c11fa68fe 100644 --- a/docs/roles/debian/wazuh.md +++ b/docs/roles/debian/wazuh.md @@ -10,6 +10,8 @@ Supports standalone managers, scaled out services and agent installation (defaul ## Default variables ```yaml --- +wazuh_agent_sources_installation: + enabled: true wazuh: path: wazuh #roles_directory: "/path/to/roles" # defaults to /home/controller/.ansible/roles/wazuh-ansible diff --git a/roles/debian/rkhunter/README.md b/roles/debian/rkhunter/README.md index cb452f702..30cb22852 100644 --- a/roles/debian/rkhunter/README.md +++ b/roles/debian/rkhunter/README.md @@ -25,6 +25,8 @@ rkhunter: scriptwhitelist: - /bin/egrep - /bin/fgrep + - /usr/bin/egrep + - /usr/bin/fgrep - /usr/bin/ldd # - /usr/bin/lwp-request - /usr/sbin/adduser diff --git a/roles/debian/user_ansible/README.md b/roles/debian/user_ansible/README.md index 117adcd08..5693b08e5 100644 --- a/roles/debian/user_ansible/README.md +++ b/roles/debian/user_ansible/README.md @@ -15,14 +15,15 @@ user_ansible: home: "/home/{{ _user_ansible_username }}" create: true # if you know the user already exists, set this to false to not create the user. create_home: true - # Optional Linux uid and gid for user + update_password: "always" + # It is often desirable to fix POSIX IDs for system users so they are consistent across your fleet + # This is shown for documentation, you should do this in your config repo # uid: 999 # gid: 999 # Local username of the deploy user. utility_host: "localhost" utility_username: "{{ _user_ansible_username }}" - sudoer: - false + sudoer: false # List of additional groups to add the user to. groups: [] # List of SSH pub keys to authorize. These must be provided as strings (content of the pub key). diff --git a/roles/debian/wazuh/README.md b/roles/debian/wazuh/README.md index 1ead6377f..c11fa68fe 100644 --- a/roles/debian/wazuh/README.md +++ b/roles/debian/wazuh/README.md @@ -10,6 +10,8 @@ Supports standalone managers, scaled out services and agent installation (defaul ## Default variables ```yaml --- +wazuh_agent_sources_installation: + enabled: true wazuh: path: wazuh #roles_directory: "/path/to/roles" # defaults to /home/controller/.ansible/roles/wazuh-ansible From 9c1e2cbf30af06b598ff36983b41252d333702ea Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 11 Sep 2024 13:18:37 +0200 Subject: [PATCH 070/206] Fixing role dependency in NGINX role. --- docs/roles/debian/nginx.md | 4 ++++ roles/debian/nginx/README.md | 1 + roles/debian/nginx/defaults/main.yml | 1 + roles/debian/nginx/tasks/main.yml | 4 ++-- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/roles/debian/nginx.md b/docs/roles/debian/nginx.md index 87b1a76dd..5a1131c5b 100644 --- a/docs/roles/debian/nginx.md +++ b/docs/roles/debian/nginx.md @@ -141,7 +141,11 @@ nginx: client_max_body_size: "700M" fastcgi_read_timeout: 60 recreate_vhosts: true # handle vhosts with ansible, if 'true' then clean up 'sites-enabled' dir and run domain.yml. + vhost_backup_location: "/home/{{ _ce_provision_username }}" # see _init for _ce_provision_username overrides: [] # See the '_overrides' role. + # Nginx ships a default vhost config that can clash with other services + # running on port 80. Set this to false to remove it after Nginx installation. + keep_default_vhost: true domains: - server_name: "{{ _domain_name }}" access_log: "/var/log/nginx/access.log" diff --git a/roles/debian/nginx/README.md b/roles/debian/nginx/README.md index 40d997b6c..5a1131c5b 100644 --- a/roles/debian/nginx/README.md +++ b/roles/debian/nginx/README.md @@ -141,6 +141,7 @@ nginx: client_max_body_size: "700M" fastcgi_read_timeout: 60 recreate_vhosts: true # handle vhosts with ansible, if 'true' then clean up 'sites-enabled' dir and run domain.yml. + vhost_backup_location: "/home/{{ _ce_provision_username }}" # see _init for _ce_provision_username overrides: [] # See the '_overrides' role. # Nginx ships a default vhost config that can clash with other services # running on port 80. Set this to false to remove it after Nginx installation. diff --git a/roles/debian/nginx/defaults/main.yml b/roles/debian/nginx/defaults/main.yml index ce9de03d2..56d69572d 100644 --- a/roles/debian/nginx/defaults/main.yml +++ b/roles/debian/nginx/defaults/main.yml @@ -129,6 +129,7 @@ nginx: client_max_body_size: "700M" fastcgi_read_timeout: 60 recreate_vhosts: true # handle vhosts with ansible, if 'true' then clean up 'sites-enabled' dir and run domain.yml. + vhost_backup_location: "/home/{{ _ce_provision_username }}" # see _init for _ce_provision_username overrides: [] # See the '_overrides' role. # Nginx ships a default vhost config that can clash with other services # running on port 80. Set this to false to remove it after Nginx installation. diff --git a/roles/debian/nginx/tasks/main.yml b/roles/debian/nginx/tasks/main.yml index eeef5b873..74aa7942d 100644 --- a/roles/debian/nginx/tasks/main.yml +++ b/roles/debian/nginx/tasks/main.yml @@ -40,7 +40,7 @@ - name: passlib - name: Backup the whole NGINX config with vhosts. - ansible.builtin.command: "tar -czf /home/{{ ce_provision.username }}/nginx_backup.tar.gz /etc/nginx/" + ansible.builtin.command: "tar -czf {{ nginx.vhost_backup_location }}/nginx_backup.tar.gz /etc/nginx/" - name: Test NGINX configuration. ansible.builtin.command: nginx -t @@ -167,7 +167,7 @@ - name: Restore previous NGINX config from backup if test fails. ansible.builtin.unarchive: - src: /home/{{ ce_provision.username }}/nginx_backup.tar.gz + src: "{{ nginx.vhost_backup_location }}/nginx_backup.tar.gz" dest: / remote_src: true when: _nginx_test_result.rc != 0 From d221f9b21024659086bd382987625f357b2010d8 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Sep 2024 18:04:28 +0200 Subject: [PATCH 071/206] Adding installation path handling for Galaxy collections. --- ce-dev/ansible/vars/provision/ce_provision.yml | 1 + install.sh | 3 ++- roles/debian/ce_deploy/ansible.cfg | 5 ++++- roles/debian/ce_deploy/defaults/main.yml | 2 ++ roles/debian/ce_deploy/tasks/main.yml | 8 ++++---- roles/debian/ce_provision/ansible.cfg | 5 ++++- roles/debian/ce_provision/defaults/main.yml | 2 ++ roles/debian/ce_provision/tasks/main.yml | 8 ++++---- 8 files changed, 23 insertions(+), 11 deletions(-) diff --git a/ce-dev/ansible/vars/provision/ce_provision.yml b/ce-dev/ansible/vars/provision/ce_provision.yml index d2476fe0f..4819f04c5 100644 --- a/ce-dev/ansible/vars/provision/ce_provision.yml +++ b/ce-dev/ansible/vars/provision/ce_provision.yml @@ -1,4 +1,5 @@ # Common vars for local development. +_ce_provision_username: "ce-dev" ce_provision: username: "ce-dev" own_repository: "https://github.com/codeenigma/ce-provision.git" diff --git a/install.sh b/install.sh index c47d96562..30f57ebc2 100755 --- a/install.sh +++ b/install.sh @@ -136,7 +136,7 @@ echo "-------------------------------------------------" su - "$CONTROLLER_USER" -c "/usr/bin/python3 -m venv /home/$CONTROLLER_USER/ce-python" su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/python3 -m pip install --upgrade pip" su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/pip install ansible netaddr python-debian" -su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection install ansible.posix --force" +su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy -p /home/$CONTROLLER_USER/.ansible/collections/ansible_collections collection install ansible.posix --force" if [ "$AWS_SUPPORT" = "true" ]; then su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/pip install boto3" fi @@ -193,6 +193,7 @@ ce_provision: repo: https://github.com/vlcty/ansible-systemd-timers.git branch: master galaxy_custom_requirements_file: "" + galaxy_installation_path: "/home/{{ _ce_provision_username }}/.ansible/collections/ansible_collections" upgrade_galaxy: enabled: true command: "/home/${CONTROLLER_USER}/ce-python/bin/ansible-galaxy collection install --force" diff --git a/roles/debian/ce_deploy/ansible.cfg b/roles/debian/ce_deploy/ansible.cfg index d1191e163..f3117748d 100644 --- a/roles/debian/ce_deploy/ansible.cfg +++ b/roles/debian/ce_deploy/ansible.cfg @@ -65,7 +65,10 @@ inventory = hosts # inject_facts_as_vars = True # additional paths to search for roles in, colon separated -roles_path = roles +roles_path = roles:/home/controller/.ansible/roles:/home/controller/ce-provision/roles + +# additional paths to search for collections in, colon separated +collections_path = /home/controller/.ansible/collections # uncomment this to disable SSH key host checking # @todo We probably want to accept new keys, but fail on changes. diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index dfccea372..061e08fc6 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -26,7 +26,9 @@ ce_deploy: groups: [] # File containing default roles and collections to install via Ansible Galaxy. # Roles will be installed to $HOME/.ansible/roles for the provision user. This roles path should be added to your ansible.cfg file. + # Collections will be installed to $HOME/.ansible/collections for the provision user. This collections path should also be added to your ansible.cfg file. galaxy_custom_requirements_file: "/home/{{ _ce_deploy.username }}/ce-deploy/config/files/galaxy-requirements.yml" + galaxy_installation_path: "/home/{{ _ce_deploy.username }}/.ansible/collections/ansible_collections" upgrade_galaxy: enabled: true command: "{{ _venv_path }}/bin/ansible-galaxy collection install --force" # _venv_path in the _init role - must match ce_deploy.venv_path if overridden diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index 447cd99c1..1edcc7007 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -183,13 +183,13 @@ - name: Install mandatory ansible-galaxy roles and collections. ansible.builtin.command: - cmd: "{{ _venv_path }}/bin/ansible-galaxy install -r {{ ce_deploy.local_dir }}/requirements-{{ ansible_distribution_major_version }}.yml --force" + cmd: "{{ _venv_path }}/bin/ansible-galaxy install -p {{ ce_deploy.galaxy_installation_path }} -r {{ ce_deploy.local_dir }}/requirements-{{ ansible_distribution_major_version }}.yml --force" become: true become_user: "{{ ce_deploy.username }}" - name: Install custom ansible-galaxy roles and collections. ansible.builtin.command: - cmd: "{{ _venv_path }}/bin/ansible-galaxy install -r {{ ce_deploy.galaxy_custom_requirements_file }} --force" + cmd: "{{ _venv_path }}/bin/ansible-galaxy install -p {{ ce_deploy.galaxy_installation_path }} -r {{ ce_deploy.galaxy_custom_requirements_file }} --force" become: true become_user: "{{ ce_deploy.username }}" when: _ce_deploy_custom_galaxy_requirements.stat.exists @@ -208,7 +208,7 @@ vars: timers: upgrade_ce_deploy_ansible_galaxy_mandatory: - timer_command: "{{ ce_deploy.upgrade_galaxy.command }} -r {{ ce_deploy.local_dir }}/requirements-{{ ansible_distribution_major_version }}.yml" + timer_command: "{{ ce_deploy.upgrade_galaxy.command }} -p {{ ce_deploy.galaxy_installation_path }} -r {{ ce_deploy.local_dir }}/requirements-{{ ansible_distribution_major_version }}.yml" timer_user: "{{ ce_deploy.username }}" timer_OnCalendar: "{{ ce_deploy.upgrade_galaxy.on_calendar }}" when: ce_deploy.upgrade_galaxy.enabled @@ -219,7 +219,7 @@ vars: timers: upgrade_ce_deploy_ansible_galaxy_custom: - timer_command: "{{ ce_deploy.upgrade_galaxy.command }} -r {{ ce_deploy.galaxy_custom_requirements_file }}" + timer_command: "{{ ce_deploy.upgrade_galaxy.command }} -p {{ ce_deploy.galaxy_installation_path }} -r {{ ce_deploy.galaxy_custom_requirements_file }}" timer_user: "{{ ce_deploy.username }}" timer_OnCalendar: "{{ ce_deploy.upgrade_galaxy.on_calendar }}" when: diff --git a/roles/debian/ce_provision/ansible.cfg b/roles/debian/ce_provision/ansible.cfg index d73ed95ac..932994edd 100644 --- a/roles/debian/ce_provision/ansible.cfg +++ b/roles/debian/ce_provision/ansible.cfg @@ -65,7 +65,10 @@ library = modules # inject_facts_as_vars = True # additional paths to search for roles in, colon separated -roles_path = roles +roles_path = roles:/home/controller/.ansible/roles:/home/controller/ce-provision/roles + +# additional paths to search for collections in, colon separated +collections_path = /home/controller/.ansible/collections # uncomment this to disable SSH key host checking # @todo We probably want to accept new keys, but fail on changes. diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index 863be9952..59ff8bc78 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -37,7 +37,9 @@ ce_provision: branch: master # File containing default roles and collections to install via Ansible Galaxy. # Roles will be installed to $HOME/.ansible/roles for the provision user. This roles path should be added to your ansible.cfg file. + # Collections will be installed to $HOME/.ansible/collections for the provision user. This collections path should also be added to your ansible.cfg file. galaxy_custom_requirements_file: "/home/{{ _ce_provision_username }}/ce-provision/config/files/galaxy-requirements.yml" + galaxy_installation_path: "/home/{{ _ce_provision_username }}/.ansible/collections/ansible_collections" upgrade_galaxy: enabled: true command: "{{ _venv_path }}/bin/ansible-galaxy collection install --force" # _venv_path in the _init role - must match ce_provision.venv_path if overridden diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index d81941702..9dab0d778 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -226,13 +226,13 @@ - name: Install mandatory ansible-galaxy roles and collections. ansible.builtin.command: - cmd: "{{ _venv_path }}/bin/ansible-galaxy install -r {{ ce_provision.local_dir }}/roles/debian/ce_provision/meta/requirements-{{ ansible_distribution_major_version }}.yml --force-with-deps" + cmd: "{{ _venv_path }}/bin/ansible-galaxy install -p {{ ce_provision.galaxy_installation_path }} -r {{ ce_provision.local_dir }}/roles/debian/ce_provision/meta/requirements-{{ ansible_distribution_major_version }}.yml --force-with-deps" become: true become_user: "{{ ce_provision.username }}" - name: Install custom ansible-galaxy roles and collections. ansible.builtin.command: - cmd: "{{ _venv_path }}/bin/ansible-galaxy install -r {{ ce_provision.galaxy_custom_requirements_file }}" + cmd: "{{ _venv_path }}/bin/ansible-galaxy install -p {{ ce_provision.galaxy_installation_path }} -r {{ ce_provision.galaxy_custom_requirements_file }}" become: true become_user: "{{ ce_provision.username }}" when: _ce_provision_custom_galaxy_requirements.stat.exists @@ -243,7 +243,7 @@ vars: timers: upgrade_ce_provision_ansible_galaxy_mandatory: - timer_command: "{{ ce_provision.upgrade_galaxy.command }} -r {{ ce_provision.local_dir }}/roles/debian/ce_provision/meta/requirements-{{ ansible_distribution_major_version }}.yml" + timer_command: "{{ ce_provision.upgrade_galaxy.command }} -p {{ ce_provision.galaxy_installation_path }} -r {{ ce_provision.local_dir }}/roles/debian/ce_provision/meta/requirements-{{ ansible_distribution_major_version }}.yml" timer_user: "{{ ce_provision.username }}" timer_OnCalendar: "{{ ce_provision.upgrade_galaxy.on_calendar }}" when: ce_provision.upgrade_galaxy.enabled @@ -254,7 +254,7 @@ vars: timers: upgrade_ce_provision_ansible_galaxy_custom: - timer_command: "{{ ce_provision.upgrade_galaxy.command }} -r {{ ce_provision.galaxy_custom_requirements_file }}" + timer_command: "{{ ce_provision.upgrade_galaxy.command }} -p {{ ce_provision.galaxy_installation_path }} -r {{ ce_provision.galaxy_custom_requirements_file }}" timer_user: "{{ ce_provision.username }}" timer_OnCalendar: "{{ ce_provision.upgrade_galaxy.on_calendar }}" when: From ca0a702ccd1a1a877e6af812e3ce1ece9d32004d Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Sep 2024 19:29:10 +0200 Subject: [PATCH 072/206] Removing -p option due to unexpected ill effects for role paths. --- install.sh | 1 - roles/debian/ce_deploy/defaults/main.yml | 2 -- roles/debian/ce_deploy/tasks/main.yml | 8 ++++---- roles/debian/ce_provision/defaults/main.yml | 2 -- roles/debian/ce_provision/tasks/main.yml | 8 ++++---- 5 files changed, 8 insertions(+), 13 deletions(-) diff --git a/install.sh b/install.sh index 30f57ebc2..5ff21ac82 100755 --- a/install.sh +++ b/install.sh @@ -193,7 +193,6 @@ ce_provision: repo: https://github.com/vlcty/ansible-systemd-timers.git branch: master galaxy_custom_requirements_file: "" - galaxy_installation_path: "/home/{{ _ce_provision_username }}/.ansible/collections/ansible_collections" upgrade_galaxy: enabled: true command: "/home/${CONTROLLER_USER}/ce-python/bin/ansible-galaxy collection install --force" diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 061e08fc6..dfccea372 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -26,9 +26,7 @@ ce_deploy: groups: [] # File containing default roles and collections to install via Ansible Galaxy. # Roles will be installed to $HOME/.ansible/roles for the provision user. This roles path should be added to your ansible.cfg file. - # Collections will be installed to $HOME/.ansible/collections for the provision user. This collections path should also be added to your ansible.cfg file. galaxy_custom_requirements_file: "/home/{{ _ce_deploy.username }}/ce-deploy/config/files/galaxy-requirements.yml" - galaxy_installation_path: "/home/{{ _ce_deploy.username }}/.ansible/collections/ansible_collections" upgrade_galaxy: enabled: true command: "{{ _venv_path }}/bin/ansible-galaxy collection install --force" # _venv_path in the _init role - must match ce_deploy.venv_path if overridden diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index 1edcc7007..447cd99c1 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -183,13 +183,13 @@ - name: Install mandatory ansible-galaxy roles and collections. ansible.builtin.command: - cmd: "{{ _venv_path }}/bin/ansible-galaxy install -p {{ ce_deploy.galaxy_installation_path }} -r {{ ce_deploy.local_dir }}/requirements-{{ ansible_distribution_major_version }}.yml --force" + cmd: "{{ _venv_path }}/bin/ansible-galaxy install -r {{ ce_deploy.local_dir }}/requirements-{{ ansible_distribution_major_version }}.yml --force" become: true become_user: "{{ ce_deploy.username }}" - name: Install custom ansible-galaxy roles and collections. ansible.builtin.command: - cmd: "{{ _venv_path }}/bin/ansible-galaxy install -p {{ ce_deploy.galaxy_installation_path }} -r {{ ce_deploy.galaxy_custom_requirements_file }} --force" + cmd: "{{ _venv_path }}/bin/ansible-galaxy install -r {{ ce_deploy.galaxy_custom_requirements_file }} --force" become: true become_user: "{{ ce_deploy.username }}" when: _ce_deploy_custom_galaxy_requirements.stat.exists @@ -208,7 +208,7 @@ vars: timers: upgrade_ce_deploy_ansible_galaxy_mandatory: - timer_command: "{{ ce_deploy.upgrade_galaxy.command }} -p {{ ce_deploy.galaxy_installation_path }} -r {{ ce_deploy.local_dir }}/requirements-{{ ansible_distribution_major_version }}.yml" + timer_command: "{{ ce_deploy.upgrade_galaxy.command }} -r {{ ce_deploy.local_dir }}/requirements-{{ ansible_distribution_major_version }}.yml" timer_user: "{{ ce_deploy.username }}" timer_OnCalendar: "{{ ce_deploy.upgrade_galaxy.on_calendar }}" when: ce_deploy.upgrade_galaxy.enabled @@ -219,7 +219,7 @@ vars: timers: upgrade_ce_deploy_ansible_galaxy_custom: - timer_command: "{{ ce_deploy.upgrade_galaxy.command }} -p {{ ce_deploy.galaxy_installation_path }} -r {{ ce_deploy.galaxy_custom_requirements_file }}" + timer_command: "{{ ce_deploy.upgrade_galaxy.command }} -r {{ ce_deploy.galaxy_custom_requirements_file }}" timer_user: "{{ ce_deploy.username }}" timer_OnCalendar: "{{ ce_deploy.upgrade_galaxy.on_calendar }}" when: diff --git a/roles/debian/ce_provision/defaults/main.yml b/roles/debian/ce_provision/defaults/main.yml index 59ff8bc78..863be9952 100644 --- a/roles/debian/ce_provision/defaults/main.yml +++ b/roles/debian/ce_provision/defaults/main.yml @@ -37,9 +37,7 @@ ce_provision: branch: master # File containing default roles and collections to install via Ansible Galaxy. # Roles will be installed to $HOME/.ansible/roles for the provision user. This roles path should be added to your ansible.cfg file. - # Collections will be installed to $HOME/.ansible/collections for the provision user. This collections path should also be added to your ansible.cfg file. galaxy_custom_requirements_file: "/home/{{ _ce_provision_username }}/ce-provision/config/files/galaxy-requirements.yml" - galaxy_installation_path: "/home/{{ _ce_provision_username }}/.ansible/collections/ansible_collections" upgrade_galaxy: enabled: true command: "{{ _venv_path }}/bin/ansible-galaxy collection install --force" # _venv_path in the _init role - must match ce_provision.venv_path if overridden diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index 9dab0d778..d81941702 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -226,13 +226,13 @@ - name: Install mandatory ansible-galaxy roles and collections. ansible.builtin.command: - cmd: "{{ _venv_path }}/bin/ansible-galaxy install -p {{ ce_provision.galaxy_installation_path }} -r {{ ce_provision.local_dir }}/roles/debian/ce_provision/meta/requirements-{{ ansible_distribution_major_version }}.yml --force-with-deps" + cmd: "{{ _venv_path }}/bin/ansible-galaxy install -r {{ ce_provision.local_dir }}/roles/debian/ce_provision/meta/requirements-{{ ansible_distribution_major_version }}.yml --force-with-deps" become: true become_user: "{{ ce_provision.username }}" - name: Install custom ansible-galaxy roles and collections. ansible.builtin.command: - cmd: "{{ _venv_path }}/bin/ansible-galaxy install -p {{ ce_provision.galaxy_installation_path }} -r {{ ce_provision.galaxy_custom_requirements_file }}" + cmd: "{{ _venv_path }}/bin/ansible-galaxy install -r {{ ce_provision.galaxy_custom_requirements_file }}" become: true become_user: "{{ ce_provision.username }}" when: _ce_provision_custom_galaxy_requirements.stat.exists @@ -243,7 +243,7 @@ vars: timers: upgrade_ce_provision_ansible_galaxy_mandatory: - timer_command: "{{ ce_provision.upgrade_galaxy.command }} -p {{ ce_provision.galaxy_installation_path }} -r {{ ce_provision.local_dir }}/roles/debian/ce_provision/meta/requirements-{{ ansible_distribution_major_version }}.yml" + timer_command: "{{ ce_provision.upgrade_galaxy.command }} -r {{ ce_provision.local_dir }}/roles/debian/ce_provision/meta/requirements-{{ ansible_distribution_major_version }}.yml" timer_user: "{{ ce_provision.username }}" timer_OnCalendar: "{{ ce_provision.upgrade_galaxy.on_calendar }}" when: ce_provision.upgrade_galaxy.enabled @@ -254,7 +254,7 @@ vars: timers: upgrade_ce_provision_ansible_galaxy_custom: - timer_command: "{{ ce_provision.upgrade_galaxy.command }} -p {{ ce_provision.galaxy_installation_path }} -r {{ ce_provision.galaxy_custom_requirements_file }}" + timer_command: "{{ ce_provision.upgrade_galaxy.command }} -r {{ ce_provision.galaxy_custom_requirements_file }}" timer_user: "{{ ce_provision.username }}" timer_OnCalendar: "{{ ce_provision.upgrade_galaxy.on_calendar }}" when: From 670745405ce2ce5eeb6c0e493192d83194676dce Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 19 Sep 2024 15:59:56 +0200 Subject: [PATCH 073/206] Moving X-Content-Type-Options header to project type templates. --- roles/debian/nginx/templates/_common.j2 | 5 +++-- roles/debian/nginx/templates/blank.j2 | 3 +++ roles/debian/nginx/templates/cloudwatch-main.json.j2 | 4 +++- roles/debian/nginx/templates/cloudwatch-vhost.json.j2 | 4 +++- roles/debian/nginx/templates/custom.j2 | 7 ++++++- roles/debian/nginx/templates/docker_registry.j2 | 6 +++++- roles/debian/nginx/templates/drupal10.j2 | 2 ++ roles/debian/nginx/templates/drupal7.j2 | 5 +++++ roles/debian/nginx/templates/drupal8.j2 | 5 +++++ roles/debian/nginx/templates/drupal_common.j2 | 2 ++ roles/debian/nginx/templates/flat.j2 | 5 +++++ roles/debian/nginx/templates/matomo.j2 | 5 +++-- roles/debian/nginx/templates/mattermost.j2 | 5 +++++ roles/debian/nginx/templates/mautic.j2 | 8 ++++++-- roles/debian/nginx/templates/mime.types.j2 | 2 ++ roles/debian/nginx/templates/nginx.conf.j2 | 2 ++ roles/debian/nginx/templates/proxy.j2 | 5 +++++ roles/debian/nginx/templates/simplesamlphp.j2 | 5 +++++ roles/debian/nginx/templates/symfony3.j2 | 5 +++++ roles/debian/nginx/templates/symfony3_prod.j2 | 5 +++++ roles/debian/nginx/templates/symfony4.j2 | 5 +++++ roles/debian/nginx/templates/wordpress.j2 | 6 +++++- 22 files changed, 90 insertions(+), 11 deletions(-) diff --git a/roles/debian/nginx/templates/_common.j2 b/roles/debian/nginx/templates/_common.j2 index 6129a309d..a0a22b778 100644 --- a/roles/debian/nginx/templates/_common.j2 +++ b/roles/debian/nginx/templates/_common.j2 @@ -1,5 +1,8 @@ ### {{ ansible_managed }} +# Because Drupal 10.1 onwards sets X-Content-Type-Options in PHP, that header is no longer set here. +# See project type templates. + gzip on; gzip_proxied any; gzip_static on; @@ -20,5 +23,3 @@ client_body_buffer_size {{ nginx.http.client_body_buffer_size }}; fastcgi_buffer_size {{ nginx.http.fastcgi_buffer_size }} ; fastcgi_buffers {{ nginx.http.fastcgi_buffers }} ; client_max_body_size {{ nginx.client_max_body_size }} ; -# Disable content sniffing, since it's an attack vector. -add_header X-Content-Type-Options nosniff; \ No newline at end of file diff --git a/roles/debian/nginx/templates/blank.j2 b/roles/debian/nginx/templates/blank.j2 index 47768dd92..b717bb1c2 100644 --- a/roles/debian/nginx/templates/blank.j2 +++ b/roles/debian/nginx/templates/blank.j2 @@ -1,3 +1,6 @@ #location / { #do something here #} + +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; diff --git a/roles/debian/nginx/templates/cloudwatch-main.json.j2 b/roles/debian/nginx/templates/cloudwatch-main.json.j2 index 9396f9ab2..f71865390 100644 --- a/roles/debian/nginx/templates/cloudwatch-main.json.j2 +++ b/roles/debian/nginx/templates/cloudwatch-main.json.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + { "logs": { "logs_collected": { @@ -25,4 +27,4 @@ } } } -} \ No newline at end of file +} diff --git a/roles/debian/nginx/templates/cloudwatch-vhost.json.j2 b/roles/debian/nginx/templates/cloudwatch-vhost.json.j2 index a7e3f9576..04463d0fd 100644 --- a/roles/debian/nginx/templates/cloudwatch-vhost.json.j2 +++ b/roles/debian/nginx/templates/cloudwatch-vhost.json.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + { "logs": { "logs_collected": { @@ -33,4 +35,4 @@ } } } -} \ No newline at end of file +} diff --git a/roles/debian/nginx/templates/custom.j2 b/roles/debian/nginx/templates/custom.j2 index 44616f926..4c687c1e1 100644 --- a/roles/debian/nginx/templates/custom.j2 +++ b/roles/debian/nginx/templates/custom.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + location / { try_files $uri /index.php$is_args$args; } @@ -8,4 +10,7 @@ location ~ \.php(/|$) { fastcgi_param SCRIPT_FILENAME $realpath_root$fastcgi_script_name; fastcgi_param DOCUMENT_ROOT $realpath_root; fastcgi_pass {{ nginx.php_fastcgi_backend }}; -} \ No newline at end of file +} + +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; diff --git a/roles/debian/nginx/templates/docker_registry.j2 b/roles/debian/nginx/templates/docker_registry.j2 index bbc3ef14c..ea204d08c 100644 --- a/roles/debian/nginx/templates/docker_registry.j2 +++ b/roles/debian/nginx/templates/docker_registry.j2 @@ -1,4 +1,5 @@ ### {{ ansible_managed }} + # required to avoid HTTP 411: see Issue #1486 (https://github.com/moby/moby/issues/1486) chunked_transfer_encoding on; location /v2/ { @@ -17,4 +18,7 @@ location /v2/ { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 900; -} \ No newline at end of file +} + +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; diff --git a/roles/debian/nginx/templates/drupal10.j2 b/roles/debian/nginx/templates/drupal10.j2 index c74dc42d0..3278dd8e6 100644 --- a/roles/debian/nginx/templates/drupal10.j2 +++ b/roles/debian/nginx/templates/drupal10.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + ######### Default block. # Generic rewrite rule. diff --git a/roles/debian/nginx/templates/drupal7.j2 b/roles/debian/nginx/templates/drupal7.j2 index daa00a795..31bc0d2b8 100644 --- a/roles/debian/nginx/templates/drupal7.j2 +++ b/roles/debian/nginx/templates/drupal7.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + ######### Default block. # Generic rewrite rule. @@ -6,3 +8,6 @@ location @rewrite { } include "/etc/nginx/conf.d/drupal_common"; + +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; diff --git a/roles/debian/nginx/templates/drupal8.j2 b/roles/debian/nginx/templates/drupal8.j2 index 1ea78977d..d4f3e2901 100644 --- a/roles/debian/nginx/templates/drupal8.j2 +++ b/roles/debian/nginx/templates/drupal8.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + ######### Default block. # Generic rewrite rule. @@ -6,3 +8,6 @@ location @rewrite { } include "/etc/nginx/conf.d/drupal_common"; + +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; diff --git a/roles/debian/nginx/templates/drupal_common.j2 b/roles/debian/nginx/templates/drupal_common.j2 index 8e3bf96f6..770203509 100644 --- a/roles/debian/nginx/templates/drupal_common.j2 +++ b/roles/debian/nginx/templates/drupal_common.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + ######### Default block. # Generic rewrite rule is in project_type config which includes this one. diff --git a/roles/debian/nginx/templates/flat.j2 b/roles/debian/nginx/templates/flat.j2 index 2e3726b59..31138d440 100644 --- a/roles/debian/nginx/templates/flat.j2 +++ b/roles/debian/nginx/templates/flat.j2 @@ -1,5 +1,10 @@ +### {{ ansible_managed }} + index index.html index.htm index.nginx-debian.html; location / { try_files $uri $uri/ =404; } + +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; diff --git a/roles/debian/nginx/templates/matomo.j2 b/roles/debian/nginx/templates/matomo.j2 index 7630e5fc7..ab7d76b22 100644 --- a/roles/debian/nginx/templates/matomo.j2 +++ b/roles/debian/nginx/templates/matomo.j2 @@ -1,7 +1,8 @@ ### {{ ansible_managed }} add_header Referrer-Policy origin always; -add_header X-Content-Type-Options "nosniff" always; +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; add_header X-XSS-Protection "1; mode=block" always; # Generic rewrite rule. @@ -25,7 +26,7 @@ location / { # @todo # limit_req zone=bots burst=5 nodelay; {% endif %} - + index index.php; try_files @rewrite /index.php; } diff --git a/roles/debian/nginx/templates/mattermost.j2 b/roles/debian/nginx/templates/mattermost.j2 index 0942bb28c..893fccb3b 100644 --- a/roles/debian/nginx/templates/mattermost.j2 +++ b/roles/debian/nginx/templates/mattermost.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + ## SSL ssl_session_timeout 1d; # Enable TLS versions (TLSv1.3 is required upcoming HTTP/3 QUIC). @@ -52,3 +54,6 @@ location / { proxy_http_version 1.1; proxy_pass http://mattermost; } + +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; diff --git a/roles/debian/nginx/templates/mautic.j2 b/roles/debian/nginx/templates/mautic.j2 index 6af79196d..92845e808 100644 --- a/roles/debian/nginx/templates/mautic.j2 +++ b/roles/debian/nginx/templates/mautic.j2 @@ -1,4 +1,5 @@ ### {{ ansible_managed }} + ######### Default block. # Generic rewrite rule. @@ -9,13 +10,13 @@ location @rewrite { # PHP fastcgi pass. location @phpprocess { include fastcgi_params; - fastcgi_split_path_info ^(.+\.php)(/.+)$; + fastcgi_split_path_info ^(.+\.php)(/.+)$; fastcgi_param SCRIPT_FILENAME $realpath_root$fastcgi_script_name; fastcgi_param DOCUMENT_ROOT $realpath_root; fastcgi_pass {{ nginx.php_fastcgi_backend }}; } -# Rewrite any request not allowed below, +# Rewrite any request not allowed below, # so they get processed through the app level (and 404). location / { try_files @rewrite @rewrite; @@ -74,3 +75,6 @@ location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|eot|woff2|ttf)$ { location = /robots.txt { try_files $uri @rewrite; } + +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; diff --git a/roles/debian/nginx/templates/mime.types.j2 b/roles/debian/nginx/templates/mime.types.j2 index f3f27da19..d8e992918 100644 --- a/roles/debian/nginx/templates/mime.types.j2 +++ b/roles/debian/nginx/templates/mime.types.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + # This file is generated from Ansible template types { {% for mime_type, extensions in nginx.http.mime_types.items() %} diff --git a/roles/debian/nginx/templates/nginx.conf.j2 b/roles/debian/nginx/templates/nginx.conf.j2 index 7d608906e..9b3dbd5f0 100644 --- a/roles/debian/nginx/templates/nginx.conf.j2 +++ b/roles/debian/nginx/templates/nginx.conf.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + user {{ nginx.user }}; worker_processes {{ nginx.worker_processes }}; pid /run/nginx.pid; diff --git a/roles/debian/nginx/templates/proxy.j2 b/roles/debian/nginx/templates/proxy.j2 index 1b2888efe..efad0a6a7 100644 --- a/roles/debian/nginx/templates/proxy.j2 +++ b/roles/debian/nginx/templates/proxy.j2 @@ -1,6 +1,11 @@ +### {{ ansible_managed }} + location / { proxy_pass https://{{ nginx.http.proxy_host }}:{{ nginx.http.proxy_port }}/; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_redirect http:// https://; } + +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; diff --git a/roles/debian/nginx/templates/simplesamlphp.j2 b/roles/debian/nginx/templates/simplesamlphp.j2 index 9ac4dac1f..9d4da235e 100644 --- a/roles/debian/nginx/templates/simplesamlphp.j2 +++ b/roles/debian/nginx/templates/simplesamlphp.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + location / { try_files $uri /index.php$is_args$args; } @@ -12,3 +14,6 @@ location ~ \.php(/|$) { #fastcgi_param SIMPLESAMLPHP_CONFIG_DIR ""; fastcgi_pass {{ nginx.php_fastcgi_backend }}; } + +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; diff --git a/roles/debian/nginx/templates/symfony3.j2 b/roles/debian/nginx/templates/symfony3.j2 index 3b1d51b41..d4270677a 100644 --- a/roles/debian/nginx/templates/symfony3.j2 +++ b/roles/debian/nginx/templates/symfony3.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + location / { try_files $uri /app_{{ symfony_env }}.php$is_args$args; } @@ -10,3 +12,6 @@ location ~ \.php(/|$) { fastcgi_param SCRIPT_FILENAME $realpath_root$fastcgi_script_name; fastcgi_param DOCUMENT_ROOT $realpath_root; } + +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; diff --git a/roles/debian/nginx/templates/symfony3_prod.j2 b/roles/debian/nginx/templates/symfony3_prod.j2 index acf6f1fec..558c04c23 100644 --- a/roles/debian/nginx/templates/symfony3_prod.j2 +++ b/roles/debian/nginx/templates/symfony3_prod.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + location / { try_files $uri /app_{{ symfony_env }}.php$is_args$args; } @@ -19,3 +21,6 @@ location ~ \.php(/|$) { fastcgi_param SCRIPT_FILENAME $realpath_root$fastcgi_script_name; fastcgi_param DOCUMENT_ROOT $realpath_root; } + +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; diff --git a/roles/debian/nginx/templates/symfony4.j2 b/roles/debian/nginx/templates/symfony4.j2 index b01fb1a8e..7d6c4a17e 100644 --- a/roles/debian/nginx/templates/symfony4.j2 +++ b/roles/debian/nginx/templates/symfony4.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + location / { try_files $uri /index.php$is_args$args; } @@ -10,3 +12,6 @@ location ~ \.php(/|$) { fastcgi_param SCRIPT_FILENAME $realpath_root$fastcgi_script_name; fastcgi_param DOCUMENT_ROOT $realpath_root; } + +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; diff --git a/roles/debian/nginx/templates/wordpress.j2 b/roles/debian/nginx/templates/wordpress.j2 index a6bf9f460..b273cba79 100644 --- a/roles/debian/nginx/templates/wordpress.j2 +++ b/roles/debian/nginx/templates/wordpress.j2 @@ -1,3 +1,4 @@ +### {{ ansible_managed }} location = /favicon.ico { log_not_found off; @@ -46,4 +47,7 @@ location ~ \.php$ { fastcgi_index index.php; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; fastcgi_pass {{ nginx.php_fastcgi_backend }}; -} \ No newline at end of file +} + +# Disable content sniffing, since it's an attack vector. +add_header X-Content-Type-Options nosniff; From 2f15b327a4837ce82b20f5847faa7fe3229be0d7 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 19 Sep 2024 16:00:11 +0200 Subject: [PATCH 074/206] Adding some inline documentation. --- roles/debian/nginx/templates/_common.j2 | 4 ++-- roles/debian/nginx/templates/blank.j2 | 2 ++ roles/debian/nginx/templates/drupal10.j2 | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/debian/nginx/templates/_common.j2 b/roles/debian/nginx/templates/_common.j2 index a0a22b778..874bd266d 100644 --- a/roles/debian/nginx/templates/_common.j2 +++ b/roles/debian/nginx/templates/_common.j2 @@ -1,7 +1,7 @@ ### {{ ansible_managed }} -# Because Drupal 10.1 onwards sets X-Content-Type-Options in PHP, that header is no longer set here. -# See project type templates. +# Drupal 10.1 onwards sets X-Content-Type-Options in PHP, see: https://www.drupal.org/node/3334163 +# Consequently that header is no longer set here, it has been moved to project type templates as necessary. gzip on; gzip_proxied any; diff --git a/roles/debian/nginx/templates/blank.j2 b/roles/debian/nginx/templates/blank.j2 index b717bb1c2..c2aa187f2 100644 --- a/roles/debian/nginx/templates/blank.j2 +++ b/roles/debian/nginx/templates/blank.j2 @@ -1,3 +1,5 @@ +### {{ ansible_managed }} + #location / { #do something here #} diff --git a/roles/debian/nginx/templates/drupal10.j2 b/roles/debian/nginx/templates/drupal10.j2 index 3278dd8e6..fee75d6c5 100644 --- a/roles/debian/nginx/templates/drupal10.j2 +++ b/roles/debian/nginx/templates/drupal10.j2 @@ -1,6 +1,7 @@ ### {{ ansible_managed }} ######### Default block. +# Drupal 10.1 onwards sets X-Content-Type-Options in PHP, see: https://www.drupal.org/node/3334163 # Generic rewrite rule. location @rewrite { From e26ce5a68749945253051febc1c35cb78b66c810 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 19 Sep 2024 16:47:08 +0200 Subject: [PATCH 075/206] Fixing Postfix template to allow external relays. --- docs/roles/debian/postfix.md | 2 +- roles/debian/postfix/README.md | 2 +- roles/debian/postfix/defaults/main.yml | 2 +- roles/debian/postfix/templates/transport.j2 | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/roles/debian/postfix.md b/docs/roles/debian/postfix.md index e63f3b70a..afd84b496 100644 --- a/docs/roles/debian/postfix.md +++ b/docs/roles/debian/postfix.md @@ -22,7 +22,7 @@ postfix: networks: "[::1]/128 [::ffff:127.0.0.0]/104 127.0.0.0/8" protocols: all relayhost: "" - transport_maps: + transport_maps: # see Postfix manual examples - https://www.postfix.org/transport.5.html - "* discard :" use_dkim: false # AWS SES config - to authenticate with an IAM user see https://docs.aws.amazon.com/ses/latest/dg/smtp-credentials.html diff --git a/roles/debian/postfix/README.md b/roles/debian/postfix/README.md index e63f3b70a..afd84b496 100644 --- a/roles/debian/postfix/README.md +++ b/roles/debian/postfix/README.md @@ -22,7 +22,7 @@ postfix: networks: "[::1]/128 [::ffff:127.0.0.0]/104 127.0.0.0/8" protocols: all relayhost: "" - transport_maps: + transport_maps: # see Postfix manual examples - https://www.postfix.org/transport.5.html - "* discard :" use_dkim: false # AWS SES config - to authenticate with an IAM user see https://docs.aws.amazon.com/ses/latest/dg/smtp-credentials.html diff --git a/roles/debian/postfix/defaults/main.yml b/roles/debian/postfix/defaults/main.yml index 10e373b83..fb00787d3 100644 --- a/roles/debian/postfix/defaults/main.yml +++ b/roles/debian/postfix/defaults/main.yml @@ -11,7 +11,7 @@ postfix: networks: "[::1]/128 [::ffff:127.0.0.0]/104 127.0.0.0/8" protocols: all relayhost: "" - transport_maps: + transport_maps: # see Postfix manual examples - https://www.postfix.org/transport.5.html - "* discard :" use_dkim: false # AWS SES config - to authenticate with an IAM user see https://docs.aws.amazon.com/ses/latest/dg/smtp-credentials.html diff --git a/roles/debian/postfix/templates/transport.j2 b/roles/debian/postfix/templates/transport.j2 index 46cfabbdf..eb6e69b27 100644 --- a/roles/debian/postfix/templates/transport.j2 +++ b/roles/debian/postfix/templates/transport.j2 @@ -1,6 +1,6 @@ {{ ansible_hostname }} : {% for transport in postfix.transport_maps %} {% if transport|length %} -{{ transport }} : +{{ transport }} {% endif %} -{% endfor %} \ No newline at end of file +{% endfor %} From ee38316eafee5e4f9e4fd76b3e2f2b6cdc55763f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 19 Sep 2024 17:46:09 +0200 Subject: [PATCH 076/206] Adding a FQDN postfix transport map. --- roles/debian/postfix/templates/transport.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/debian/postfix/templates/transport.j2 b/roles/debian/postfix/templates/transport.j2 index eb6e69b27..098bf5265 100644 --- a/roles/debian/postfix/templates/transport.j2 +++ b/roles/debian/postfix/templates/transport.j2 @@ -1,4 +1,5 @@ {{ ansible_hostname }} : +{{ ansible_fqdn }} : {% for transport in postfix.transport_maps %} {% if transport|length %} {{ transport }} From 3f3643fe146b787f271b04223b81f857b124044e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 3 Dec 2024 16:09:06 +0100 Subject: [PATCH 077/206] Updating CI to 2.x. --- .github/workflows/ce-provision-test-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 55a6942f4..50ec80b74 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -31,7 +31,7 @@ jobs: sudo mv ./mkcert /usr/local/bin && cd ../ sudo chmod +x /usr/local/bin/mkcert rm -Rf mkcert - curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux + curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/${{ github.event.pull_request.base.ref }}/install.sh | /bin/sh -s -- --platform linux # Uses the ce-dev stack to run a test provision - name: Run a test provision From bb57d35cbd73c2e18f1cff62e01a56459d9fd201 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 3 Dec 2024 17:14:32 +0100 Subject: [PATCH 078/206] Defending against missing Ansible. --- scripts/_common.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index 8ab48b9ad..2eb8d4197 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -154,14 +154,25 @@ cleanup_build_tmp_dir(){ rm -rf "$BUILD_TMP_DIR" fi } + # Trigger actual Ansible job. ansible_play(){ if [ -z "$ANSIBLE_PATH" ]; then if [ "$LINT" = "yes" ]; then # apt repo installed - ANSIBLE_BIN=$(command -v ansible-lint) + if ! command -v ansible-lint; then + echo "### Could not find ansible-lint - Exiting! ###" + exit 1 + else + ANSIBLE_BIN=$(command -v ansible-lint) + fi else - ANSIBLE_BIN=$(command -v ansible-playbook) + if ! command -v ansible-playbook; then + echo "### Could not find ansible-playbook - Exiting! ###" + exit 1 + else + ANSIBLE_BIN=$(command -v ansible-playbook) + fi fi else if [ "$LINT" = "yes" ]; then From c42be9ef62f212d844470dc324487dffd71457b0 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 3 Dec 2024 17:28:35 +0100 Subject: [PATCH 079/206] Making the ce-provision-config branch in CI dynamic. --- .github/workflows/ce-provision-test-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 50ec80b74..275af558e 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -37,6 +37,6 @@ jobs: - name: Run a test provision if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - git clone --branch 1.x https://github.com/codeenigma/ce-dev-ce-provision-config.git config + git clone --branch ${{ github.event.pull_request.base.ref }} https://github.com/codeenigma/ce-dev-ce-provision-config.git config /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} shell: bash From f120ac9f0ed3483cdf69f419c83d6674f25d0ce3 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 3 Dec 2024 18:05:56 +0100 Subject: [PATCH 080/206] We do not want a 'ce-dev provision' because it breaks our controller. --- .github/workflows/ce-provision-test-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 275af558e..92d49ba6e 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -38,5 +38,5 @@ jobs: if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | git clone --branch ${{ github.event.pull_request.base.ref }} https://github.com/codeenigma/ce-dev-ce-provision-config.git config - /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --no-provision shell: bash From 3820afcac3aef3602219a3d778b8e234c85faaa6 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 3 Dec 2024 18:17:49 +0100 Subject: [PATCH 081/206] Reverting 'ce-dev provision' change. --- .github/workflows/ce-provision-test-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 92d49ba6e..275af558e 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -38,5 +38,5 @@ jobs: if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | git clone --branch ${{ github.event.pull_request.base.ref }} https://github.com/codeenigma/ce-dev-ce-provision-config.git config - /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --no-provision + /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} shell: bash From 3533e183d72697e520858d811f2c940090f40f87 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 4 Dec 2024 10:28:11 +0100 Subject: [PATCH 082/206] Trying a different ansible_facts var. --- roles/debian/php-common/templates/sury.pref.j2 | 2 +- roles/debian/php-fpm/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/debian/php-common/templates/sury.pref.j2 b/roles/debian/php-common/templates/sury.pref.j2 index 582556999..c42729d7e 100644 --- a/roles/debian/php-common/templates/sury.pref.j2 +++ b/roles/debian/php-common/templates/sury.pref.j2 @@ -1,3 +1,3 @@ Package: * Pin: origin packages.sury.org/deb/ -Pin-Priority: 1101 \ No newline at end of file +Pin-Priority: 1101 diff --git a/roles/debian/php-fpm/defaults/main.yml b/roles/debian/php-fpm/defaults/main.yml index bd11bd594..120525f84 100644 --- a/roles/debian/php-fpm/defaults/main.yml +++ b/roles/debian/php-fpm/defaults/main.yml @@ -13,7 +13,7 @@ php: # It is important to scale up processes on bigger servers, so that more # requests can be handled. Double the number of vCPUs is a good default. # Can be between 5 and 64. - max_children: "{{ [5, [ansible_facts.processor_vcpus * 2, 64] | min] | max }}" + max_children: "{{ [5, [ansible_facts.ansible_processor_nproc * 2, 64] | min] | max }}" start_servers: 2 min_spare_servers: 1 max_spare_servers: 3 From 585277b6cde703218acce9000059af4ab9aea713 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 4 Dec 2024 10:51:18 +0100 Subject: [PATCH 083/206] Testing using the source branch in ce-dev. --- .github/workflows/ce-provision-test-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 275af558e..5dcf73fae 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -38,5 +38,5 @@ jobs: if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | git clone --branch ${{ github.event.pull_request.base.ref }} https://github.com/codeenigma/ce-dev-ce-provision-config.git config - /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} shell: bash From e6b875019c24f7471a347ee47e26913eb9e6f566 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 4 Dec 2024 14:52:30 +0100 Subject: [PATCH 084/206] Setting max_childen to an integer to avoid CI issues. --- ce-dev/ansible/vars/web/php.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ce-dev/ansible/vars/web/php.yml b/ce-dev/ansible/vars/web/php.yml index e77d64b22..6c41d0016 100644 --- a/ce-dev/ansible/vars/web/php.yml +++ b/ce-dev/ansible/vars/web/php.yml @@ -1,3 +1,5 @@ php: version: - - 8.1 \ No newline at end of file + - 8.1 + fpm: + max_children: 5 From 0d0509b278dbb95a067b34abccc182403dca6e0a Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 4 Dec 2024 16:04:25 +0100 Subject: [PATCH 085/206] Trying to change the python interpreter used. --- ce-dev/ansible/test.sh | 2 +- ce-dev/ansible/vars/provision/ce_provision.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ce-dev/ansible/test.sh b/ce-dev/ansible/test.sh index 61124de0a..0144cf452 100755 --- a/ce-dev/ansible/test.sh +++ b/ce-dev/ansible/test.sh @@ -93,7 +93,7 @@ provision-controller provision-target provision-privileged EOT - PROVISION_CMD="/bin/sh /home/ce-dev/ce-provision/scripts/provision.sh" + PROVISION_CMD="/bin/sh /home/ce-dev/ce-provision/scripts/provision.sh --python-interpreter /home/ce-dev/ce-python/bin/python3" echo "# Executing $1 project" PROVISION_CMD="$PROVISION_CMD --repo dummy --branch dummy --workspace /home/ce-dev/ce-provision/ce-dev/ansible --playbook plays/$1/$1.yml --own-branch $2 --config-branch $3 --force" if [ $VERBOSE = true ]; then diff --git a/ce-dev/ansible/vars/provision/ce_provision.yml b/ce-dev/ansible/vars/provision/ce_provision.yml index 4819f04c5..15dcb2705 100644 --- a/ce-dev/ansible/vars/provision/ce_provision.yml +++ b/ce-dev/ansible/vars/provision/ce_provision.yml @@ -6,7 +6,7 @@ ce_provision: own_repository_branch: "devel-2.x" own_repository_skip_checkout: true config_repository: "https://github.com/codeenigma/ce-dev-ce-provision-config.git" - config_repository_branch: "1.x" + config_repository_branch: "devel-2.x" config_repository_skip_checkout: true local_dir: "/home/ce-dev/ce-provision" galaxy_custom_requirements_file: "/home/ce-dev/ce-provision/ce-dev/ansible/vars/provision/galaxy-requirements.yml" From 67c6e4f74c3f4924cef1fd35b3616769a81bfb14 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 4 Dec 2024 16:54:00 +0100 Subject: [PATCH 086/206] Adding platform and cgroup values to ce-dev compose template. --- ce-dev/ce-dev.compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ce-dev/ce-dev.compose.yml b/ce-dev/ce-dev.compose.yml index 78a4ac0f8..f512e6176 100644 --- a/ce-dev/ce-dev.compose.yml +++ b/ce-dev/ce-dev.compose.yml @@ -11,6 +11,8 @@ services: x-ce_dev: {} volumes: - ../:/home/ce-dev/ce-provision + platform: linux/amd64 + cgroup: host target: image: codeenigma/ce-dev-1.x:devel x-ce_dev: @@ -21,6 +23,8 @@ services: - 443 cap_add: - NET_ADMIN + platform: linux/amd64 + cgroup: host privileged: image: codeenigma/ce-dev-1.x:devel x-ce_dev: @@ -32,3 +36,5 @@ services: - 443 cap_add: - NET_ADMIN + platform: linux/amd64 + cgroup: host From 018584ca1290a11e9c846ff807d887ef89fcb203 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 4 Dec 2024 17:37:16 +0100 Subject: [PATCH 087/206] Trying latest ubuntu containers in GitHub Actions. --- .github/workflows/ce-provision-test-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 5dcf73fae..e78a140d0 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -10,7 +10,7 @@ jobs: # Name the Job name: Run tests against Ansible code base # Set the type of machine to run on - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: # Checks out a copy of your repository on the ubuntu-latest machine From 35b6def9b9b92aae7e9b286881ada3ad0efc5d8f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 4 Dec 2024 18:44:54 +0100 Subject: [PATCH 088/206] Fixing the test.sh script to work with venvs. --- ce-dev/ansible/plays/_common/setup.yml | 20 ++++++++++++++++++++ ce-dev/ansible/test.sh | 14 +++++++++----- 2 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 ce-dev/ansible/plays/_common/setup.yml diff --git a/ce-dev/ansible/plays/_common/setup.yml b/ce-dev/ansible/plays/_common/setup.yml new file mode 100644 index 000000000..59135624b --- /dev/null +++ b/ce-dev/ansible/plays/_common/setup.yml @@ -0,0 +1,20 @@ +--- +# Run the _init role to correctly install Ansible first. +- hosts: provision-target + become: true + + vars: + project_name: blank + is_local: true + _ce_provision_base_dir: /home/ce-dev/ce-provision + _init: + force_play: true + vars_dirs: + - "{{ _ce_provision_base_dir }}/ce-dev/ansible/vars/_common" + - "{{ _ce_provision_base_dir }}/ce-dev/ansible/vars/{{ project_name }}" + + tasks: + - ansible.builtin.import_role: + name: _init + - ansible.builtin.import_role: + name: _exit diff --git a/ce-dev/ansible/test.sh b/ce-dev/ansible/test.sh index 0144cf452..ebf637089 100755 --- a/ce-dev/ansible/test.sh +++ b/ce-dev/ansible/test.sh @@ -93,16 +93,20 @@ provision-controller provision-target provision-privileged EOT - PROVISION_CMD="/bin/sh /home/ce-dev/ce-provision/scripts/provision.sh --python-interpreter /home/ce-dev/ce-python/bin/python3" - echo "# Executing $1 project" - PROVISION_CMD="$PROVISION_CMD --repo dummy --branch dummy --workspace /home/ce-dev/ce-provision/ce-dev/ansible --playbook plays/$1/$1.yml --own-branch $2 --config-branch $3 --force" + PROVISION_CMD="/bin/sh /home/ce-dev/ce-provision/scripts/provision.sh" if [ $VERBOSE = true ]; then echo "# In verbose mode" PROVISION_CMD="$PROVISION_CMD --verbose" fi - echo "# Running command: $PROVISION_CMD" + echo "# Executing $1 project" + PROVISION_EXEC="$PROVISION_CMD --repo dummy --branch dummy --workspace /home/ce-dev/ce-provision/ce-dev/ansible --playbook plays/_common/setup.yml --own-branch $2 --config-branch $3 --force" + echo "# Installing Ansible in a venv: $PROVISION_EXEC" + # shellcheck disable=SC2086 + sudo docker exec -t --workdir /home/ce-dev/ce-provision --user ce-dev provision-controller $PROVISION_EXEC + PROVISION_EXEC="$PROVISION_CMD --python-interpreter /home/ce-dev/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/ce-dev/ce-provision/ce-dev/ansible --playbook plays/$1/$1.yml --own-branch $2 --config-branch $3 --force" + echo "# Running command: $PROVISION_EXEC" # shellcheck disable=SC2086 - sudo docker exec -t --workdir /home/ce-dev/ce-provision --user ce-dev provision-controller $PROVISION_CMD + sudo docker exec -t --workdir /home/ce-dev/ce-provision --user ce-dev provision-controller $PROVISION_EXEC echo "### $1 project completed ###" } From 251e9bb58c250f77450577cf57cca536265a493e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 4 Dec 2024 19:08:44 +0100 Subject: [PATCH 089/206] Documentation for PHP in CI. --- ce-dev/ansible/vars/web/php.yml | 1 + docs/roles/debian/php-fpm.md | 2 +- roles/debian/php-fpm/README.md | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ce-dev/ansible/vars/web/php.yml b/ce-dev/ansible/vars/web/php.yml index 6c41d0016..e6f4aeeb2 100644 --- a/ce-dev/ansible/vars/web/php.yml +++ b/ce-dev/ansible/vars/web/php.yml @@ -1,5 +1,6 @@ php: version: - 8.1 + # GitHub Actions doesn't make ansible_facts for CPU info available fpm: max_children: 5 diff --git a/docs/roles/debian/php-fpm.md b/docs/roles/debian/php-fpm.md index 1c6484680..063e97276 100644 --- a/docs/roles/debian/php-fpm.md +++ b/docs/roles/debian/php-fpm.md @@ -23,7 +23,7 @@ php: # It is important to scale up processes on bigger servers, so that more # requests can be handled. Double the number of vCPUs is a good default. # Can be between 5 and 64. - max_children: "{{ [5, [ansible_facts.processor_vcpus * 2, 64] | min] | max }}" + max_children: "{{ [5, [ansible_facts.ansible_processor_nproc * 2, 64] | min] | max }}" start_servers: 2 min_spare_servers: 1 max_spare_servers: 3 diff --git a/roles/debian/php-fpm/README.md b/roles/debian/php-fpm/README.md index 1c6484680..063e97276 100644 --- a/roles/debian/php-fpm/README.md +++ b/roles/debian/php-fpm/README.md @@ -23,7 +23,7 @@ php: # It is important to scale up processes on bigger servers, so that more # requests can be handled. Double the number of vCPUs is a good default. # Can be between 5 and 64. - max_children: "{{ [5, [ansible_facts.processor_vcpus * 2, 64] | min] | max }}" + max_children: "{{ [5, [ansible_facts.ansible_processor_nproc * 2, 64] | min] | max }}" start_servers: 2 min_spare_servers: 1 max_spare_servers: 3 From 6f7aa28168b6d18bcba76fddc08f9bcaa2a91d05 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 4 Dec 2024 19:32:02 +0100 Subject: [PATCH 090/206] Adding GitLab test back in. --- ...on-test-gitlab.yml.NOPE => ce-provision-test-gitlab.yml} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename .github/workflows/{ce-provision-test-gitlab.yml.NOPE => ce-provision-test-gitlab.yml} (84%) diff --git a/.github/workflows/ce-provision-test-gitlab.yml.NOPE b/.github/workflows/ce-provision-test-gitlab.yml similarity index 84% rename from .github/workflows/ce-provision-test-gitlab.yml.NOPE rename to .github/workflows/ce-provision-test-gitlab.yml index 8b171062e..4d7f226d9 100644 --- a/.github/workflows/ce-provision-test-gitlab.yml.NOPE +++ b/.github/workflows/ce-provision-test-gitlab.yml @@ -31,12 +31,12 @@ jobs: sudo mv ./mkcert /usr/local/bin && cd ../ sudo chmod +x /usr/local/bin/mkcert rm -Rf mkcert - curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux + curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/${{ github.event.pull_request.base.ref }}/install.sh | /bin/sh -s -- --platform linux # Uses the ce-dev stack to run a test provision - name: Run a test provision if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - git clone --branch 1.x https://github.com/codeenigma/ce-dev-ce-provision-config.git config - /bin/bash ce-dev/ansible/test.sh --examples gitlab --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + git clone --branch ${{ github.event.pull_request.base.ref }} https://github.com/codeenigma/ce-dev-ce-provision-config.git config + /bin/bash ce-dev/ansible/test.sh --examples gitlab --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} shell: bash From 40c687ca9947e2bc93ce528f745bda65d8ab7ff8 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 4 Dec 2024 22:11:10 +0100 Subject: [PATCH 091/206] Fixing role namespaces. --- ce-dev/ansible/plays/gitlab/gitlab.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ce-dev/ansible/plays/gitlab/gitlab.yml b/ce-dev/ansible/plays/gitlab/gitlab.yml index 262a16f6c..020c4502f 100644 --- a/ce-dev/ansible/plays/gitlab/gitlab.yml +++ b/ce-dev/ansible/plays/gitlab/gitlab.yml @@ -16,24 +16,24 @@ - ansible.builtin.import_role: name: _init - ansible.builtin.import_role: - name: user_provision + name: debian/user_provision - ansible.builtin.import_role: name: _meta/common_base - ansible.builtin.import_role: - name: ce_deploy + name: debian/ce_deploy - ansible.builtin.import_role: name: aws/aws_credentials - ansible.builtin.import_role: - name: gitlab + name: debian/gitlab - ansible.builtin.import_role: - name: gitlab_runner + name: debian/gitlab_runner - ansible.builtin.import_role: - name: ssh_server + name: debian/ssh_server - ansible.builtin.import_role: - name: sops + name: debian/sops - ansible.builtin.import_role: - name: gpg_key + name: debian/gpg_key - ansible.builtin.import_role: - name: firewall_config + name: debian/firewall_config - ansible.builtin.import_role: name: _exit From 616ad284d8ecb63a8fd464845e0fd3211f0b0c99 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 6 Dec 2024 16:13:20 +0100 Subject: [PATCH 092/206] Minor bug fixes to ce-provision installer. --- install.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/install.sh b/install.sh index 5ff21ac82..2c1638215 100755 --- a/install.sh +++ b/install.sh @@ -83,7 +83,7 @@ if [ "$(id -u)" -ne 0 ] then echo "Please run this script as root or using sudo!" exit fi - + # Check we are using a compatible Linux distribution. if [ "$ID" != "debian" ]; then if [ "$ID_LIKE" != "debian" ]; then @@ -136,7 +136,7 @@ echo "-------------------------------------------------" su - "$CONTROLLER_USER" -c "/usr/bin/python3 -m venv /home/$CONTROLLER_USER/ce-python" su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/python3 -m pip install --upgrade pip" su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/pip install ansible netaddr python-debian" -su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy -p /home/$CONTROLLER_USER/.ansible/collections/ansible_collections collection install ansible.posix --force" +su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection install ansible.posix -p /home/$CONTROLLER_USER/.ansible/collections/ansible_collections --force" if [ "$AWS_SUPPORT" = "true" ]; then su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/pip install boto3" fi @@ -152,7 +152,7 @@ else echo "-------------------------------------------------" fi # Create playbook for ce-provision. -/usr/bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL +/bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL --- - hosts: "localhost" become: true @@ -164,7 +164,7 @@ fi name: debian/ce_provision EOL # Create vars file. -/usr/bin/cat >"/home/$CONTROLLER_USER/ce-provision/vars.yml" << EOL +/bin/cat >"/home/$CONTROLLER_USER/ce-provision/vars.yml" << EOL _domain_name: ${SERVER_HOSTNAME} _ce_provision_data_dir: /home/${CONTROLLER_USER}/ce-provision/data _ce_provision_username: ${CONTROLLER_USER} @@ -176,6 +176,8 @@ ce_provision: aws_support: ${AWS_SUPPORT} new_user: ${CONTROLLER_USER} username: ${CONTROLLER_USER} + ssh_key_bits: "521" + ssh_key_type: ecdsa public_key_name: id_rsa.pub own_repository: "https://github.com/codeenigma/ce-provision.git" own_repository_branch: "${VERSION}" @@ -222,7 +224,7 @@ rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" echo "-------------------------------------------------" echo "Install firewall." echo "-------------------------------------------------" -/usr/bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL +/bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL --- - hosts: "localhost" become: true @@ -233,7 +235,7 @@ echo "-------------------------------------------------" ansible.builtin.import_role: name: debian/firewall_config EOL -su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" +su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" echo "-------------------------------------------------" # Install GitLab @@ -241,7 +243,7 @@ if [ "$GITLAB_URL" != "no" ]; then echo "Install GitLab." echo "-------------------------------------------------" # Create playbook. - /usr/bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL + /bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL --- - hosts: "localhost" become: true @@ -256,7 +258,7 @@ if [ "$GITLAB_URL" != "no" ]; then name: debian/gitlab EOL # Create vars file. - /usr/bin/cat >"/home/$CONTROLLER_USER/ce-provision/vars.yml" << EOL + /bin/cat >"/home/$CONTROLLER_USER/ce-provision/vars.yml" << EOL gitlab_runner: apt_origin: "origin=packages.gitlab.com/runner/gitlab-runner,codename=\${distro_codename},label=gitlab-runner" # used by apt_unattended_upgrades apt_signed_by: https://packages.gitlab.com/runner/gitlab-runner/gpgkey @@ -344,7 +346,7 @@ EOT EOT echo "-------------------------------------------------" fi - su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" + su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" echo "-------------------------------------------------" else echo "GitLab not requested. Skipping." From c12de39b45a4329f4773acdb20c67f0df36890a8 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 6 Dec 2024 16:13:41 +0100 Subject: [PATCH 093/206] Testing installing ce-provision in the GitHub Actions container directly. --- .github/workflows/ce-provision-test-web.yml | 44 ++++++++++++--------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index e78a140d0..40adcfa4f 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -18,25 +18,33 @@ jobs: if: ${{ github.event.pull_request.head.ref != 'documentation' }} uses: actions/checkout@v2 - # Installs the ce-dev stack - - name: Install ce-dev + # Installs ce-provision + - name: Install ce-provision if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - cd /tmp - wget https://golang.org/dl/go1.15.8.linux-amd64.tar.gz - sudo tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz - export PATH=$PATH:/usr/local/go/bin - git clone https://github.com/FiloSottile/mkcert && cd mkcert - go build -ldflags "-X main.Version=$(git describe --tags)" - sudo mv ./mkcert /usr/local/bin && cd ../ - sudo chmod +x /usr/local/bin/mkcert - rm -Rf mkcert - curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/${{ github.event.pull_request.base.ref }}/install.sh | /bin/sh -s -- --platform linux + curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.base.ref }}/install.sh + chmod +x ./install.sh + sudo ./install.sh --user ce-dev --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + + # Installs the ce-dev stack + #- name: Install ce-dev + # if: ${{ github.event.pull_request.head.ref != 'documentation' }} + # run: | + # cd /tmp + # wget https://golang.org/dl/go1.15.8.linux-amd64.tar.gz + # sudo tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz + # export PATH=$PATH:/usr/local/go/bin + # git clone https://github.com/FiloSottile/mkcert && cd mkcert + # go build -ldflags "-X main.Version=$(git describe --tags)" + # sudo mv ./mkcert /usr/local/bin && cd ../ + # sudo chmod +x /usr/local/bin/mkcert + # rm -Rf mkcert + # curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/${{ github.event.pull_request.base.ref }}/install.sh | /bin/sh -s -- --platform linux # Uses the ce-dev stack to run a test provision - - name: Run a test provision - if: ${{ github.event.pull_request.head.ref != 'documentation' }} - run: | - git clone --branch ${{ github.event.pull_request.base.ref }} https://github.com/codeenigma/ce-dev-ce-provision-config.git config - /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} - shell: bash + #- name: Run a test provision + # if: ${{ github.event.pull_request.head.ref != 'documentation' }} + # run: | + # git clone --branch ${{ github.event.pull_request.base.ref }} https://github.com/codeenigma/ce-dev-ce-provision-config.git config + # /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + # shell: bash From dee1878a3abfd9618d69fdfb820de24d51e73ea2 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 6 Dec 2024 16:18:46 +0100 Subject: [PATCH 094/206] Using the submitted install script as well. --- .github/workflows/ce-provision-test-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 40adcfa4f..eefd6c1c6 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -22,7 +22,7 @@ jobs: - name: Install ce-provision if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.base.ref }}/install.sh + curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh chmod +x ./install.sh sudo ./install.sh --user ce-dev --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} From 6619ecba94e16411ad2232457f8578597048d998 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 6 Dec 2024 16:35:01 +0100 Subject: [PATCH 095/206] Trying as runner user. --- .github/workflows/ce-provision-test-web.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index eefd6c1c6..86f801336 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -14,9 +14,9 @@ jobs: steps: # Checks out a copy of your repository on the ubuntu-latest machine - - name: Checkout code - if: ${{ github.event.pull_request.head.ref != 'documentation' }} - uses: actions/checkout@v2 + #- name: Checkout code + # if: ${{ github.event.pull_request.head.ref != 'documentation' }} + # uses: actions/checkout@v2 # Installs ce-provision - name: Install ce-provision @@ -24,7 +24,13 @@ jobs: run: | curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh chmod +x ./install.sh - sudo ./install.sh --user ce-dev --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + sudo ./install.sh --user runner --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + + # Run a web server provision + - name: Provision a test web server + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + run: | + /bin/sh /home/runner/ce-provision/scripts/provision.sh --python-interpreter /home/runner/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/runner/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force # Installs the ce-dev stack #- name: Install ce-dev From e1d415f45a8995052d80cf5db1e94f6d5605d1aa Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 6 Dec 2024 16:55:58 +0100 Subject: [PATCH 096/206] Trying to use the ce-dev base container. --- .github/workflows/ce-provision-test-web.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 86f801336..9d8004e22 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -12,6 +12,10 @@ jobs: # Set the type of machine to run on runs-on: ubuntu-latest + # Use our ce-dev Debian base container + container: + image: codeenigma/ce-dev:2.x + steps: # Checks out a copy of your repository on the ubuntu-latest machine #- name: Checkout code From 76155450e496294698faea29061a9a04e4431e81 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 15:53:38 +0100 Subject: [PATCH 097/206] Updating key name. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 2c1638215..d43d442bd 100755 --- a/install.sh +++ b/install.sh @@ -178,7 +178,7 @@ ce_provision: username: ${CONTROLLER_USER} ssh_key_bits: "521" ssh_key_type: ecdsa - public_key_name: id_rsa.pub + public_key_name: id_ecdsa.pub own_repository: "https://github.com/codeenigma/ce-provision.git" own_repository_branch: "${VERSION}" own_repository_skip_checkout: false From e46c1b77e9800afa0980c7f4345264a6295d3dfd Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 16:10:50 +0100 Subject: [PATCH 098/206] Suppressing systemd actions in Docker. --- .github/workflows/ce-provision-test-web.yml | 2 +- install.sh | 12 +++++++++++- roles/debian/ansible/tasks/main.yml | 4 +++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 9d8004e22..01b7973bd 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -28,7 +28,7 @@ jobs: run: | curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh chmod +x ./install.sh - sudo ./install.sh --user runner --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + sudo ./install.sh --user runner --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker # Run a web server provision - name: Provision a test web server diff --git a/install.sh b/install.sh index d43d442bd..6bee5d0ad 100755 --- a/install.sh +++ b/install.sh @@ -18,6 +18,7 @@ usage(){ echo '--gitlab: install GitLab CE on this server (default: no, set to desired GitLab address to install, e.g. gitlab.example.com)' echo '--letsencrypt: try to create an SSL certificate with LetsEncrypt (requires DNS pointing at this server for provided GitLab URL)' echo '--aws: enable AWS support' + echo '--docker: script is running in a Docker container' echo '' } @@ -51,6 +52,9 @@ parse_options(){ "--aws") AWS_SUPPORT="true" ;; + "--docker") + IS_LOCAL="true" + ;; *) usage exit 1 @@ -68,6 +72,7 @@ CONFIG_REPO_BRANCH="1.x" GITLAB_URL="no" LE_SUPPORT="no" AWS_SUPPORT="false" +IS_LOCAL="false" SERVER_HOSTNAME=$(hostname) # Parse options. @@ -218,7 +223,12 @@ firewall_config: - "80" - "443" EOL -su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" +# Tell Ansible this is a Docker container +if [ "$IS_LOCAL" = "true" ]; then + su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml --extra-vars {is_local: $IS_LOCAL}" +else + su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" +fi rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" # Create playbook for firewall. echo "-------------------------------------------------" diff --git a/roles/debian/ansible/tasks/main.yml b/roles/debian/ansible/tasks/main.yml index 39ccb984b..f876b55cb 100644 --- a/roles/debian/ansible/tasks/main.yml +++ b/roles/debian/ansible/tasks/main.yml @@ -79,7 +79,9 @@ dest: "/etc/profile.d/ansible-path.sh" - name: Install systemd timer. - when: ce_ansible.upgrade.enabled + when: + - ce_ansible.upgrade.enabled + - not is_local block: - name: Build systemd timer variables string. ansible.builtin.set_fact: From eaac4f5dc017b0c94db9f3e08aacf7c9e3a59d4d Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 16:14:25 +0100 Subject: [PATCH 099/206] Seems Ansible flags have changed. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 6bee5d0ad..d625e8809 100755 --- a/install.sh +++ b/install.sh @@ -225,7 +225,7 @@ firewall_config: EOL # Tell Ansible this is a Docker container if [ "$IS_LOCAL" = "true" ]; then - su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml --extra-vars {is_local: $IS_LOCAL}" + su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml -e {is_local: $IS_LOCAL}" else su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" fi From cf7e3629a46aff29773c8b86e7df45b6351d1a20 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 16:18:44 +0100 Subject: [PATCH 100/206] Still trying to get --extra-vars right! --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index d625e8809..6ba4fad51 100755 --- a/install.sh +++ b/install.sh @@ -225,7 +225,7 @@ firewall_config: EOL # Tell Ansible this is a Docker container if [ "$IS_LOCAL" = "true" ]; then - su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml -e {is_local: $IS_LOCAL}" + su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" else su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" fi From 78f5e8139fcb0d3b15c4ef8e35e7c234d80fe3ac Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 16:23:55 +0100 Subject: [PATCH 101/206] Catching Ansible Galaxy upgrade timers for docker containers. --- roles/debian/ansible_galaxy/tasks/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/debian/ansible_galaxy/tasks/main.yml b/roles/debian/ansible_galaxy/tasks/main.yml index b267a73fe..5dde4a6df 100644 --- a/roles/debian/ansible_galaxy/tasks/main.yml +++ b/roles/debian/ansible_galaxy/tasks/main.yml @@ -11,7 +11,9 @@ register: _galaxy_requirements - name: Install ansible-galaxy roles and/or collections. - when: _galaxy_requirements.stat.exists + when: + - _galaxy_requirements.stat.exists + - not is_local block: - name: Set up the ansible-galaxy command. ansible.builtin.set_fact: @@ -32,6 +34,7 @@ when: - ansible_galaxy.upgrade_galaxy.enabled - _galaxy_requirements.stat.exists + - not is_local block: - name: Build timers variable. ansible.builtin.set_fact: From d4625001f9f9de1ec9d1c3b165dedd955fce712c Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 16:55:53 +0100 Subject: [PATCH 102/206] Trying to force --roles-path for Galaxy. --- install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 6ba4fad51..29e1181ea 100755 --- a/install.sh +++ b/install.sh @@ -225,9 +225,9 @@ firewall_config: EOL # Tell Ansible this is a Docker container if [ "$IS_LOCAL" = "true" ]; then - su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" + su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL, ansible_galaxy.extra_params: \"--force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles\"}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" else - su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" + su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{ansible_galaxy.extra_params: \"--force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles\"}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" fi rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" # Create playbook for firewall. From 34b8bcb2098bafd54f53b44a2e5b47f6052b5b34 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 17:03:43 +0100 Subject: [PATCH 103/206] Trying different quotes. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 29e1181ea..e1a533fb9 100755 --- a/install.sh +++ b/install.sh @@ -227,7 +227,7 @@ EOL if [ "$IS_LOCAL" = "true" ]; then su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL, ansible_galaxy.extra_params: \"--force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles\"}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" else - su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{ansible_galaxy.extra_params: \"--force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles\"}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" + su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{ansible_galaxy.extra_params: '--force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles'}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" fi rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" # Create playbook for firewall. From 14d4a46cb4b68895008dae1419e66b2cb8aec794 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 17:07:29 +0100 Subject: [PATCH 104/206] Missed a line. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index e1a533fb9..5ba57a188 100755 --- a/install.sh +++ b/install.sh @@ -225,7 +225,7 @@ firewall_config: EOL # Tell Ansible this is a Docker container if [ "$IS_LOCAL" = "true" ]; then - su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL, ansible_galaxy.extra_params: \"--force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles\"}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" + su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL, ansible_galaxy.extra_params: '--force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles'}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" else su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{ansible_galaxy.extra_params: '--force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles'}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" fi From 247cc29510b0ec2c52a159de6646d28ae03f77f8 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 17:11:52 +0100 Subject: [PATCH 105/206] Trying a different approach to passing vars. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 5ba57a188..35c951cec 100755 --- a/install.sh +++ b/install.sh @@ -225,7 +225,7 @@ firewall_config: EOL # Tell Ansible this is a Docker container if [ "$IS_LOCAL" = "true" ]; then - su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL, ansible_galaxy.extra_params: '--force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles'}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" + su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL, ansible_galaxy.extra_params: --force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" else su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{ansible_galaxy.extra_params: '--force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles'}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" fi From cb0e7fb34a7cf3db1848d8a7946bc6a2367f2e05 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 17:17:59 +0100 Subject: [PATCH 106/206] Adding some debug. --- install.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 35c951cec..7c8788e70 100755 --- a/install.sh +++ b/install.sh @@ -227,9 +227,17 @@ EOL if [ "$IS_LOCAL" = "true" ]; then su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL, ansible_galaxy.extra_params: --force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" else - su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{ansible_galaxy.extra_params: '--force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles'}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" + su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{ansible_galaxy.extra_params: --force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" fi rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" +# TEST +echo "-------------------------------------------------" +echo "### system python" +su - "$CONTROLLER_USER" -c "ansible-galaxy collection list" +su - "$CONTROLLER_USER" -c "ansible-galaxy role list" +echo "### ce-python" +su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection list" +su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy role list" # Create playbook for firewall. echo "-------------------------------------------------" echo "Install firewall." From 2d63cf6b07b01fed931600db4a849b51f3d3a8f6 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 17:27:07 +0100 Subject: [PATCH 107/206] Running ce-python debug first. --- install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install.sh b/install.sh index 7c8788e70..1b5d8f6b6 100755 --- a/install.sh +++ b/install.sh @@ -232,12 +232,12 @@ fi rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" # TEST echo "-------------------------------------------------" -echo "### system python" -su - "$CONTROLLER_USER" -c "ansible-galaxy collection list" -su - "$CONTROLLER_USER" -c "ansible-galaxy role list" echo "### ce-python" su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection list" su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy role list" +echo "### system python" +su - "$CONTROLLER_USER" -c "ansible-galaxy collection list" +su - "$CONTROLLER_USER" -c "ansible-galaxy role list" # Create playbook for firewall. echo "-------------------------------------------------" echo "Install firewall." From 3f546edb8f7c5b9d5df35861a795863eac8bcddf Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 17:30:36 +0100 Subject: [PATCH 108/206] Trying moving to the ce-provision directory. --- install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install.sh b/install.sh index 1b5d8f6b6..a295d64d7 100755 --- a/install.sh +++ b/install.sh @@ -233,11 +233,11 @@ rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" # TEST echo "-------------------------------------------------" echo "### ce-python" -su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection list" -su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy role list" +su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection list" +su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy role list" echo "### system python" -su - "$CONTROLLER_USER" -c "ansible-galaxy collection list" -su - "$CONTROLLER_USER" -c "ansible-galaxy role list" +su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && ansible-galaxy collection list" +su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && ansible-galaxy role list" # Create playbook for firewall. echo "-------------------------------------------------" echo "Install firewall." From 7d1a0c52718ceab89e34f5707fbfb041b1a83d14 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 17:34:34 +0100 Subject: [PATCH 109/206] Checking the specific path to galaxy roles in ce-provision. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index a295d64d7..da18567c6 100755 --- a/install.sh +++ b/install.sh @@ -234,7 +234,7 @@ rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" echo "-------------------------------------------------" echo "### ce-python" su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection list" -su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy role list" +su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy role list --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles" echo "### system python" su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && ansible-galaxy collection list" su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && ansible-galaxy role list" From 46656f08553d71394a46904926185281322d6bf0 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 17:42:00 +0100 Subject: [PATCH 110/206] Trying as controller user again. --- .github/workflows/ce-provision-test-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 01b7973bd..9b1c15b32 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -28,7 +28,7 @@ jobs: run: | curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh chmod +x ./install.sh - sudo ./install.sh --user runner --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker + sudo ./install.sh --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker # Run a web server provision - name: Provision a test web server From b2392a031fd307a3178bdb0cc947f1281080cb22 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 17:46:20 +0100 Subject: [PATCH 111/206] Trying to make the roles dir. --- install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install.sh b/install.sh index da18567c6..cddcf271f 100755 --- a/install.sh +++ b/install.sh @@ -156,6 +156,7 @@ else echo "ce-provision directory at /home/$CONTROLLER_USER/ce-provision already exists. Skipping." echo "-------------------------------------------------" fi +mkdir -p "/home/$CONTROLLER_USER/ce-provision/galaxy/roles" # Create playbook for ce-provision. /bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL --- From cd7cbcb0b87343ad4473206e79ce3a89148f513e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 17:54:54 +0100 Subject: [PATCH 112/206] Being consistent about paths in bash. --- install.sh | 168 ++++++++++++++++++++++++++--------------------------- 1 file changed, 84 insertions(+), 84 deletions(-) diff --git a/install.sh b/install.sh index cddcf271f..3dbee223f 100755 --- a/install.sh +++ b/install.sh @@ -6,20 +6,20 @@ set -e . /etc/os-release usage(){ - echo 'install.sh [OPTIONS]' - echo 'Install the latest ce-provision version, or the version specified as option.' - echo 'Please ensure you are using Debian Linux or similar and at least Bullseye (11) or higher.' - echo '' - echo 'Available options:' - echo '--version: ce-provision version to use (default: 2.x)' - echo '--user: Ansible controller user (default: controller)' - echo '--config: Git URL to your ce-provision Ansible config repository (default: https://github.com/codeenigma/ce-provision-config-example.git)' - echo '--config-branch: branch of your Ansible config repository to use (default: 1.x)' - echo '--gitlab: install GitLab CE on this server (default: no, set to desired GitLab address to install, e.g. gitlab.example.com)' - echo '--letsencrypt: try to create an SSL certificate with LetsEncrypt (requires DNS pointing at this server for provided GitLab URL)' - echo '--aws: enable AWS support' - echo '--docker: script is running in a Docker container' - echo '' + /usr/bin/echo 'install.sh [OPTIONS]' + /usr/bin/echo 'Install the latest ce-provision version, or the version specified as option.' + /usr/bin/echo 'Please ensure you are using Debian Linux or similar and at least Bullseye (11) or higher.' + /usr/bin/echo '' + /usr/bin/echo 'Available options:' + /usr/bin/echo '--version: ce-provision version to use (default: 2.x)' + /usr/bin/echo '--user: Ansible controller user (default: controller)' + /usr/bin/echo '--config: Git URL to your ce-provision Ansible config repository (default: https://github.com/codeenigma/ce-provision-config-example.git)' + /usr/bin/echo '--config-branch: branch of your Ansible config repository to use (default: 1.x)' + /usr/bin/echo '--gitlab: install GitLab CE on this server (default: no, set to desired GitLab address to install, e.g. gitlab.example.com)' + /usr/bin/echo '--letsencrypt: try to create an SSL certificate with LetsEncrypt (requires DNS pointing at this server for provided GitLab URL)' + /usr/bin/echo '--aws: enable AWS support' + /usr/bin/echo '--docker: script is running in a Docker container' + /usr/bin/echo '' } # Parse options arguments. @@ -92,40 +92,40 @@ fi # Check we are using a compatible Linux distribution. if [ "$ID" != "debian" ]; then if [ "$ID_LIKE" != "debian" ]; then - echo "ce-provision only supports Debian Linux and derivatives." + /usr/bin/echo "ce-provision only supports Debian Linux and derivatives." exit 0 else - echo "ce-provision works best with Debian Linux, it may work with this distro but no promises!" - echo "-------------------------------------------------" - echo "Carrying on regardless..." - echo "-------------------------------------------------" + /usr/bin/echo "ce-provision works best with Debian Linux, it may work with this distro but no promises!" + /usr/bin/echo "-------------------------------------------------" + /usr/bin/echo "Carrying on regardless..." + /usr/bin/echo "-------------------------------------------------" fi fi -echo "Beginning ce-provision installation." -echo "-------------------------------------------------" +/usr/bin/echo "Beginning ce-provision installation." +/usr/bin/echo "-------------------------------------------------" # Create required user. -echo "Check if user named $CONTROLLER_USER exists." +/usr/bin/echo "Check if user named $CONTROLLER_USER exists." # Check if user exists -if id "$CONTROLLER_USER" >/dev/null 2>&1; then - echo "The user named $CONTROLLER_USER already exists. Skipping." +if /usr/bin/id "$CONTROLLER_USER" >/dev/null 2>&1; then + /usr/bin/echo "The user named $CONTROLLER_USER already exists. Skipping." else # User not found so let's create them. - echo "Create user named $CONTROLLER_USER." + /usr/bin/echo "Create user named $CONTROLLER_USER." /usr/sbin/useradd -s /bin/bash "$CONTROLLER_USER" - echo "$CONTROLLER_USER":"$CONTROLLER_USER" | chpasswd -m - install -m 755 -o "$CONTROLLER_USER" -g "$CONTROLLER_USER" -d /home/"$CONTROLLER_USER" - install -m 700 -o "$CONTROLLER_USER" -g "$CONTROLLER_USER" -d /home/"$CONTROLLER_USER"/.ssh - echo root:"$CONTROLLER_USER" | chpasswd -m - echo "$CONTROLLER_USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/"$CONTROLLER_USER" - chmod 0440 /etc/sudoers.d/"$CONTROLLER_USER" + /usr/bin/echo "$CONTROLLER_USER":"$CONTROLLER_USER" | chpasswd -m + /usr/bin/install -m 755 -o "$CONTROLLER_USER" -g "$CONTROLLER_USER" -d /home/"$CONTROLLER_USER" + /usr/bin/install -m 700 -o "$CONTROLLER_USER" -g "$CONTROLLER_USER" -d /home/"$CONTROLLER_USER"/.ssh + /usr/bin/echo root:"$CONTROLLER_USER" | chpasswd -m + /usr/bin/echo "$CONTROLLER_USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/"$CONTROLLER_USER" + /usr/bin/chmod 0440 /etc/sudoers.d/"$CONTROLLER_USER" fi -echo "-------------------------------------------------" +/usr/bin/echo "-------------------------------------------------" # Install APT packages. -echo "Install required packages." -echo "-------------------------------------------------" +/usr/bin/echo "Install required packages." +/usr/bin/echo "-------------------------------------------------" /usr/bin/apt-get update /usr/bin/apt-get dist-upgrade -y -o Dpkg::Options::="--force-confnew" /usr/bin/apt-get install -y -o Dpkg::Options::="--force-confnew" \ @@ -133,30 +133,30 @@ echo "-------------------------------------------------" openssh-client nfs-common stunnel4 \ python3-venv python3-debian \ zip unzip gzip tar dnsutils -echo "-------------------------------------------------" +/usr/bin/echo "-------------------------------------------------" # Install Ansible in a Python virtual environment. -echo "Install Ansible and dependencies." -echo "-------------------------------------------------" -su - "$CONTROLLER_USER" -c "/usr/bin/python3 -m venv /home/$CONTROLLER_USER/ce-python" -su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/python3 -m pip install --upgrade pip" -su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/pip install ansible netaddr python-debian" -su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection install ansible.posix -p /home/$CONTROLLER_USER/.ansible/collections/ansible_collections --force" +/usr/bin/echo "Install Ansible and dependencies." +/usr/bin/echo "-------------------------------------------------" +/usr/bin/su - "$CONTROLLER_USER" -c "/usr/bin/python3 -m venv /home/$CONTROLLER_USER/ce-python" +/usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/python3 -m pip install --upgrade pip" +/usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/pip install ansible netaddr python-debian" +/usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection install ansible.posix -p /home/$CONTROLLER_USER/.ansible/collections/ansible_collections --force" if [ "$AWS_SUPPORT" = "true" ]; then - su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/pip install boto3" + /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/pip install boto3" fi -echo "-------------------------------------------------" +/usr/bin/echo "-------------------------------------------------" # Install ce-provision. -echo "Install ce-provision." -echo "-------------------------------------------------" +/usr/bin/echo "Install ce-provision." +/usr/bin/echo "-------------------------------------------------" if [ ! -d "/home/$CONTROLLER_USER/ce-provision" ]; then - su - "$CONTROLLER_USER" -c "git clone --branch $VERSION https://github.com/codeenigma/ce-provision.git /home/$CONTROLLER_USER/ce-provision" + /usr/bin/su - "$CONTROLLER_USER" -c "git clone --branch $VERSION https://github.com/codeenigma/ce-provision.git /home/$CONTROLLER_USER/ce-provision" else - echo "ce-provision directory at /home/$CONTROLLER_USER/ce-provision already exists. Skipping." - echo "-------------------------------------------------" + /usr/bin/echo "ce-provision directory at /home/$CONTROLLER_USER/ce-provision already exists. Skipping." + /usr/bin/echo "-------------------------------------------------" fi -mkdir -p "/home/$CONTROLLER_USER/ce-provision/galaxy/roles" +/usr/bin/mkdir -p "/home/$CONTROLLER_USER/ce-provision/galaxy/roles" # Create playbook for ce-provision. /bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL --- @@ -226,23 +226,23 @@ firewall_config: EOL # Tell Ansible this is a Docker container if [ "$IS_LOCAL" = "true" ]; then - su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL, ansible_galaxy.extra_params: --force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" + /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL, ansible_galaxy.extra_params: --force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" else - su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{ansible_galaxy.extra_params: --force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" + /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{ansible_galaxy.extra_params: --force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" fi -rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" +/usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" # TEST -echo "-------------------------------------------------" -echo "### ce-python" -su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection list" -su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy role list --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles" -echo "### system python" -su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && ansible-galaxy collection list" -su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && ansible-galaxy role list" +/usr/bin/echo "-------------------------------------------------" +/usr/bin/echo "### ce-python" +/usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection list" +/usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy role list --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles" +/usr/bin/echo "### system python" +/usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && ansible-galaxy collection list" +/usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && ansible-galaxy role list" # Create playbook for firewall. -echo "-------------------------------------------------" -echo "Install firewall." -echo "-------------------------------------------------" +/usr/bin/echo "-------------------------------------------------" +/usr/bin/echo "Install firewall." +/usr/bin/echo "-------------------------------------------------" /bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL --- - hosts: "localhost" @@ -254,13 +254,13 @@ echo "-------------------------------------------------" ansible.builtin.import_role: name: debian/firewall_config EOL -su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" -echo "-------------------------------------------------" +/usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" +/usr/bin/echo "-------------------------------------------------" # Install GitLab if [ "$GITLAB_URL" != "no" ]; then - echo "Install GitLab." - echo "-------------------------------------------------" + /usr/bin/echo "Install GitLab." + /usr/bin/echo "-------------------------------------------------" # Create playbook. /bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL --- @@ -330,47 +330,47 @@ gitlab: custom_nginx_config: "" EOL if [ "$LE_SUPPORT" = "yes" ]; then - echo "Will try to create an SSL certificate with LetsEncrypt." - echo "*** THIS STEP WILL FAIL IF YOUR DNS IS NOT CORRECT! ***" + /usr/bin/echo "Will try to create an SSL certificate with LetsEncrypt." + /usr/bin/echo "*** THIS STEP WILL FAIL IF YOUR DNS IS NOT CORRECT! ***" if [ -n "$(dig +short "$GITLAB_URL".)" ]; then - echo "DNS record found, attempting LetsEncrypt request..." + /usr/bin/echo "DNS record found, attempting LetsEncrypt request..." # Write GitLab vars with LE for SSL - cat <> "/home/$CONTROLLER_USER/ce-provision/vars.yml" + /bin/cat <> "/home/$CONTROLLER_USER/ce-provision/vars.yml" letsencrypt: "true" ssl: enabled: false EOT - echo "-------------------------------------------------" + /usr/bin/echo "-------------------------------------------------" else - echo "No DNS found for provided URL, will create a self-signed certificate instead." + /usr/bin/echo "No DNS found for provided URL, will create a self-signed certificate instead." # Write GitLab vars with self-signed SSL - cat <> "/home/$CONTROLLER_USER/ce-provision/vars.yml" + /bin/cat <> "/home/$CONTROLLER_USER/ce-provision/vars.yml" letsencrypt: "false" ssl: enabled: true handling: selfsigned replace_existing: false EOT - echo "-------------------------------------------------" + /usr/bin/echo "-------------------------------------------------" fi else # Write GitLab vars with self-signed SSL - echo "Create a self-signed SSL certificate." - cat <> "/home/$CONTROLLER_USER/ce-provision/vars.yml" + /usr/bin/echo "Create a self-signed SSL certificate." + /bin/cat <> "/home/$CONTROLLER_USER/ce-provision/vars.yml" letsencrypt: "false" ssl: enabled: true handling: selfsigned replace_existing: false EOT - echo "-------------------------------------------------" + /usr/bin/echo "-------------------------------------------------" fi - su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" - echo "-------------------------------------------------" + /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" + /usr/bin/echo "-------------------------------------------------" else - echo "GitLab not requested. Skipping." - echo "-------------------------------------------------" + /usr/bin/echo "GitLab not requested. Skipping." + /usr/bin/echo "-------------------------------------------------" fi -rm "/home/$CONTROLLER_USER/ce-provision/vars.yml" -rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" -echo "DONE." +/usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/vars.yml" +/usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" +/usr/bin/echo "DONE." From 14847df85570a36d01571413950737b6866c62a7 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 17:55:22 +0100 Subject: [PATCH 113/206] Removing debug lines for now. --- install.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/install.sh b/install.sh index 3dbee223f..1de23eae4 100755 --- a/install.sh +++ b/install.sh @@ -231,14 +231,6 @@ else /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{ansible_galaxy.extra_params: --force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" fi /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" -# TEST -/usr/bin/echo "-------------------------------------------------" -/usr/bin/echo "### ce-python" -/usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection list" -/usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy role list --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles" -/usr/bin/echo "### system python" -/usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && ansible-galaxy collection list" -/usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && ansible-galaxy role list" # Create playbook for firewall. /usr/bin/echo "-------------------------------------------------" /usr/bin/echo "Install firewall." From de9b496b04da0b6110574e56e305dbb3c3ea41a8 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 18:08:45 +0100 Subject: [PATCH 114/206] Allowing script to skip iptables. --- .github/workflows/ce-provision-test-web.yml | 2 +- install.sh | 30 +++++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 9b1c15b32..f35781dcb 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -28,7 +28,7 @@ jobs: run: | curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh chmod +x ./install.sh - sudo ./install.sh --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker + sudo ./install.sh --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker --no-firewall # Run a web server provision - name: Provision a test web server diff --git a/install.sh b/install.sh index 1de23eae4..d454288db 100755 --- a/install.sh +++ b/install.sh @@ -15,6 +15,7 @@ usage(){ /usr/bin/echo '--user: Ansible controller user (default: controller)' /usr/bin/echo '--config: Git URL to your ce-provision Ansible config repository (default: https://github.com/codeenigma/ce-provision-config-example.git)' /usr/bin/echo '--config-branch: branch of your Ansible config repository to use (default: 1.x)' + /usr/bin/echo '--firewall: install and configure iptables with ports 22, 80 and 443 open' /usr/bin/echo '--gitlab: install GitLab CE on this server (default: no, set to desired GitLab address to install, e.g. gitlab.example.com)' /usr/bin/echo '--letsencrypt: try to create an SSL certificate with LetsEncrypt (requires DNS pointing at this server for provided GitLab URL)' /usr/bin/echo '--aws: enable AWS support' @@ -49,6 +50,9 @@ parse_options(){ "--letsencrypt") LE_SUPPORT="yes" ;; + "--no-firewall") + FIREWALL="false" + ;; "--aws") AWS_SUPPORT="true" ;; @@ -71,6 +75,7 @@ CONFIG_REPO="https://github.com/codeenigma/ce-provision-config-example.git" CONFIG_REPO_BRANCH="1.x" GITLAB_URL="no" LE_SUPPORT="no" +FIREWALL="true" AWS_SUPPORT="false" IS_LOCAL="false" SERVER_HOSTNAME=$(hostname) @@ -224,18 +229,22 @@ firewall_config: - "80" - "443" EOL + # Tell Ansible this is a Docker container if [ "$IS_LOCAL" = "true" ]; then - /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL, ansible_galaxy.extra_params: --force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" + /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" else - /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{ansible_galaxy.extra_params: --force --roles-path /home/$CONTROLLER_USER/ce-provision/galaxy/roles}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" + /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" fi /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" + +# Install firewall +if [ "$FIREWALL" = "true" ]; then # Create playbook for firewall. -/usr/bin/echo "-------------------------------------------------" -/usr/bin/echo "Install firewall." -/usr/bin/echo "-------------------------------------------------" -/bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL + /usr/bin/echo "-------------------------------------------------" + /usr/bin/echo "Install firewall." + /usr/bin/echo "-------------------------------------------------" + /bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL --- - hosts: "localhost" become: true @@ -246,8 +255,13 @@ fi ansible.builtin.import_role: name: debian/firewall_config EOL -/usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" -/usr/bin/echo "-------------------------------------------------" + /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" + /usr/bin/echo "-------------------------------------------------" +else + /usr/bin/echo "-------------------------------------------------" + /usr/bin/echo "Skipping firewall." + /usr/bin/echo "-------------------------------------------------" +fi # Install GitLab if [ "$GITLAB_URL" != "no" ]; then From 451011757bedb096f88fd6aa4d90b4d0d298ff7e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 18:10:22 +0100 Subject: [PATCH 115/206] Misnamed flag. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index d454288db..32b856e30 100755 --- a/install.sh +++ b/install.sh @@ -15,7 +15,7 @@ usage(){ /usr/bin/echo '--user: Ansible controller user (default: controller)' /usr/bin/echo '--config: Git URL to your ce-provision Ansible config repository (default: https://github.com/codeenigma/ce-provision-config-example.git)' /usr/bin/echo '--config-branch: branch of your Ansible config repository to use (default: 1.x)' - /usr/bin/echo '--firewall: install and configure iptables with ports 22, 80 and 443 open' + /usr/bin/echo '--no-firewall: skip installing iptables with ports 22, 80 and 443 open' /usr/bin/echo '--gitlab: install GitLab CE on this server (default: no, set to desired GitLab address to install, e.g. gitlab.example.com)' /usr/bin/echo '--letsencrypt: try to create an SSL certificate with LetsEncrypt (requires DNS pointing at this server for provided GitLab URL)' /usr/bin/echo '--aws: enable AWS support' From 78d63187f2a517517bfb48b89396dc916783049a Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 18:27:31 +0100 Subject: [PATCH 116/206] Adding user_provision role to configure controller user. --- install.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/install.sh b/install.sh index 32b856e30..680099f0c 100755 --- a/install.sh +++ b/install.sh @@ -173,6 +173,9 @@ fi - name: Install ce-provision. ansible.builtin.import_role: name: debian/ce_provision + - name: Configure controller user. + ansible.builtin.import_role: + name: debian/user_provision EOL # Create vars file. /bin/cat >"/home/$CONTROLLER_USER/ce-provision/vars.yml" << EOL @@ -210,6 +213,14 @@ ce_provision: enabled: true command: "/home/${CONTROLLER_USER}/ce-python/bin/ansible-galaxy collection install --force" on_calendar: "Mon *-*-* 04:00:00" +user_provision: + username: controller + utility_username: controller + utility_host: localhost + groups: + - bypass2fa + ssh_keys: + - "{{ lookup('file', '/home/${CONTROLLER_USER}/ce-provision/data/localhost/home/${CONTROLLER_USER}/.ssh/id_ecdsa.pub') }}" firewall_config: purge: true firewall_state: started From 1343c7b3a598aaf83ffa35c97e8d27642639a005 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 18:29:14 +0100 Subject: [PATCH 117/206] Wrapping cleanup so it doesn't break GitHub Actions. --- install.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 680099f0c..2a1d960dc 100755 --- a/install.sh +++ b/install.sh @@ -388,6 +388,9 @@ else /usr/bin/echo "GitLab not requested. Skipping." /usr/bin/echo "-------------------------------------------------" fi -/usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/vars.yml" -/usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" +# Tidy up if not a container +if [ "$IS_LOCAL" = "true" ]; then + /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/vars.yml" + /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" +fi /usr/bin/echo "DONE." From 5d5b95e551113d0d9be369ddd1adba50e201f986 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 18:35:21 +0100 Subject: [PATCH 118/206] Completing variables for user_provisin. --- install.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 2a1d960dc..e7a25b4ba 100755 --- a/install.sh +++ b/install.sh @@ -214,13 +214,20 @@ ce_provision: command: "/home/${CONTROLLER_USER}/ce-python/bin/ansible-galaxy collection install --force" on_calendar: "Mon *-*-* 04:00:00" user_provision: - username: controller - utility_username: controller + username: ${CONTROLLER_USER} + home: /home/${CONTROLLER_USER} + create: false + create_home: false + update_password: always + utility_username: ${CONTROLLER_USER} utility_host: localhost groups: - bypass2fa ssh_keys: - "{{ lookup('file', '/home/${CONTROLLER_USER}/ce-provision/data/localhost/home/${CONTROLLER_USER}/.ssh/id_ecdsa.pub') }}" + ssh_private_keys: [] + known_hosts: [] + known_hosts_hash: true firewall_config: purge: true firewall_state: started From 3184d96f1f57fb68d77ee5e345467b064c349fc5 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 18:39:09 +0100 Subject: [PATCH 119/206] Missed the sudoers var. --- install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install.sh b/install.sh index e7a25b4ba..92430f881 100755 --- a/install.sh +++ b/install.sh @@ -221,6 +221,7 @@ user_provision: update_password: always utility_username: ${CONTROLLER_USER} utility_host: localhost + sudoer: true groups: - bypass2fa ssh_keys: From d56d1becd25f788e0e351665c8d67e5cb0e09875 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 18:45:01 +0100 Subject: [PATCH 120/206] Quoting vars. --- install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install.sh b/install.sh index 92430f881..1c7b5a905 100755 --- a/install.sh +++ b/install.sh @@ -214,12 +214,12 @@ ce_provision: command: "/home/${CONTROLLER_USER}/ce-python/bin/ansible-galaxy collection install --force" on_calendar: "Mon *-*-* 04:00:00" user_provision: - username: ${CONTROLLER_USER} - home: /home/${CONTROLLER_USER} + username: "${CONTROLLER_USER}" + home: "/home/${CONTROLLER_USER}" create: false create_home: false update_password: always - utility_username: ${CONTROLLER_USER} + utility_username: "${CONTROLLER_USER}" utility_host: localhost sudoer: true groups: From 953a925c3a00c608ede2c476997a3cfb3d013caf Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 18:49:40 +0100 Subject: [PATCH 121/206] GitLab installer needs _domain_name. --- install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install.sh b/install.sh index 1c7b5a905..d31c1668f 100755 --- a/install.sh +++ b/install.sh @@ -179,6 +179,7 @@ fi EOL # Create vars file. /bin/cat >"/home/$CONTROLLER_USER/ce-provision/vars.yml" << EOL +--- _domain_name: ${SERVER_HOSTNAME} _ce_provision_data_dir: /home/${CONTROLLER_USER}/ce-provision/data _ce_provision_username: ${CONTROLLER_USER} @@ -303,6 +304,8 @@ if [ "$GITLAB_URL" != "no" ]; then EOL # Create vars file. /bin/cat >"/home/$CONTROLLER_USER/ce-provision/vars.yml" << EOL +--- +_domain_name: ${SERVER_HOSTNAME} gitlab_runner: apt_origin: "origin=packages.gitlab.com/runner/gitlab-runner,codename=\${distro_codename},label=gitlab-runner" # used by apt_unattended_upgrades apt_signed_by: https://packages.gitlab.com/runner/gitlab-runner/gpgkey From 49643d38c4efe2b1fc9e93a67d4b22c02f71a12b Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 18:53:19 +0100 Subject: [PATCH 122/206] Logic error in clean-up script. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index d31c1668f..5af3ab91f 100755 --- a/install.sh +++ b/install.sh @@ -400,7 +400,7 @@ else /usr/bin/echo "-------------------------------------------------" fi # Tidy up if not a container -if [ "$IS_LOCAL" = "true" ]; then +if [ "$IS_LOCAL" = "false" ]; then /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/vars.yml" /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" fi From b0f7731c5a2eb6babde79573ded6bfeae1fa48c3 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 18:58:35 +0100 Subject: [PATCH 123/206] Fixing paths to ce-provision in container. --- .github/workflows/ce-provision-test-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index f35781dcb..0fecfe2ee 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -34,7 +34,7 @@ jobs: - name: Provision a test web server if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - /bin/sh /home/runner/ce-provision/scripts/provision.sh --python-interpreter /home/runner/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/runner/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force + /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force # Installs the ce-dev stack #- name: Install ce-dev From 73fbf874afdcfd67555c47108aab9843c4028104 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 19:03:08 +0100 Subject: [PATCH 124/206] Trying to fix CI perms issues. --- .github/workflows/ce-provision-test-web.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 0fecfe2ee..d9e26484c 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -34,6 +34,7 @@ jobs: - name: Provision a test web server if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | + chown -Rf controller:controller /home/controller/ce-provision /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force # Installs the ce-dev stack From dcd04c839205ba370f0d30c96fde815328026be4 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 19:06:22 +0100 Subject: [PATCH 125/206] Git dubious ownership error. --- .github/workflows/ce-provision-test-web.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index d9e26484c..a02a0997b 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -35,6 +35,7 @@ jobs: if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | chown -Rf controller:controller /home/controller/ce-provision + git config --global --add safe.directory /home/controller/ce-provision /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force # Installs the ce-dev stack From cccda4ccd808694331eddd29e81e399804db5af0 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 19:12:56 +0100 Subject: [PATCH 126/206] Git dubious ownership error. --- .github/workflows/ce-provision-test-web.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index a02a0997b..444ffb450 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -36,6 +36,7 @@ jobs: run: | chown -Rf controller:controller /home/controller/ce-provision git config --global --add safe.directory /home/controller/ce-provision + git config --global --add safe.directory /home/controller/ce-provision/config /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force # Installs the ce-dev stack From f41b60db6def00e68d24b9ad73bb1443fa78099a Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 19:18:52 +0100 Subject: [PATCH 127/206] Running the web server test as the controller user. --- .github/workflows/ce-provision-test-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 444ffb450..c00823989 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -37,7 +37,7 @@ jobs: chown -Rf controller:controller /home/controller/ce-provision git config --global --add safe.directory /home/controller/ce-provision git config --global --add safe.directory /home/controller/ce-provision/config - /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force + su - controller -c "cd /home/$CONTROLLER_USER/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" # Installs the ce-dev stack #- name: Install ce-dev From 0c20f29b303eecb33e2070590db3f2d6e5ac94fd Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 19:25:10 +0100 Subject: [PATCH 128/206] Missed a controller var. --- .github/workflows/ce-provision-test-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index c00823989..bdff644d6 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -37,7 +37,7 @@ jobs: chown -Rf controller:controller /home/controller/ce-provision git config --global --add safe.directory /home/controller/ce-provision git config --global --add safe.directory /home/controller/ce-provision/config - su - controller -c "cd /home/$CONTROLLER_USER/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" + su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" # Installs the ce-dev stack #- name: Install ce-dev From 2401969e9325984739b27937561153be62d7fe89 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 19:30:43 +0100 Subject: [PATCH 129/206] Commenting out the CE container to test. --- .github/workflows/ce-provision-test-web.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index bdff644d6..d0f226fb5 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -13,8 +13,8 @@ jobs: runs-on: ubuntu-latest # Use our ce-dev Debian base container - container: - image: codeenigma/ce-dev:2.x + #container: + # image: codeenigma/ce-dev:2.x steps: # Checks out a copy of your repository on the ubuntu-latest machine From 6674206a0ac1212b555ac58ed10793540c52cc85 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 19:47:08 +0100 Subject: [PATCH 130/206] Adding a separate step for Git actions. --- .github/workflows/ce-provision-test-web.yml | 8 +++++--- install.sh | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index d0f226fb5..586598969 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -31,13 +31,15 @@ jobs: sudo ./install.sh --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker --no-firewall # Run a web server provision - - name: Provision a test web server + - name: Prepare Git repos on disk if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - chown -Rf controller:controller /home/controller/ce-provision git config --global --add safe.directory /home/controller/ce-provision git config --global --add safe.directory /home/controller/ce-provision/config - su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" + + - name: Provision a test web server + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + run: su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" # Installs the ce-dev stack #- name: Install ce-dev diff --git a/install.sh b/install.sh index 5af3ab91f..08eb8c5e4 100755 --- a/install.sh +++ b/install.sh @@ -95,6 +95,7 @@ if [ "$(id -u)" -ne 0 ] fi # Check we are using a compatible Linux distribution. +/usr/bin/echo "-------------------------------------------------" if [ "$ID" != "debian" ]; then if [ "$ID_LIKE" != "debian" ]; then /usr/bin/echo "ce-provision only supports Debian Linux and derivatives." From 4d2e9f5cb0fcb6b31fd120503ceea2f7e9001fe0 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 16 Dec 2024 19:52:11 +0100 Subject: [PATCH 131/206] Need sudo for Ubuntu. --- .github/workflows/ce-provision-test-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 586598969..51d55d52d 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -39,7 +39,7 @@ jobs: - name: Provision a test web server if: ${{ github.event.pull_request.head.ref != 'documentation' }} - run: su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" + run: sudo su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" # Installs the ce-dev stack #- name: Install ce-dev From 74342744f0795485b807c422232dfdb5da6f3df0 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 12:26:19 +0100 Subject: [PATCH 132/206] Using a volume to persist data between steps. --- .github/workflows/ce-provision-test-web.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 51d55d52d..011740dfe 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -13,8 +13,10 @@ jobs: runs-on: ubuntu-latest # Use our ce-dev Debian base container - #container: - # image: codeenigma/ce-dev:2.x + container: + image: codeenigma/ce-dev:2.x + volumes: + - /home/controller steps: # Checks out a copy of your repository on the ubuntu-latest machine @@ -39,7 +41,7 @@ jobs: - name: Provision a test web server if: ${{ github.event.pull_request.head.ref != 'documentation' }} - run: sudo su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" + run: su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" # Installs the ce-dev stack #- name: Install ce-dev From aa4bed835b916ba226239b1804d1ca018ed55dc1 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 12:38:35 +0100 Subject: [PATCH 133/206] Adding debug commands to test volumes. --- .github/workflows/ce-provision-test-web.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 011740dfe..dea4fc789 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -17,6 +17,7 @@ jobs: image: codeenigma/ce-dev:2.x volumes: - /home/controller + - ${{ github.workspace }}:/home/controller/ce-provision steps: # Checks out a copy of your repository on the ubuntu-latest machine @@ -24,6 +25,12 @@ jobs: # if: ${{ github.event.pull_request.head.ref != 'documentation' }} # uses: actions/checkout@v2 + - name: Check volumes. + run: | + ls -la /home/controller + ls -la /home/controller/ce-provision + ls -la /home/runner + # Installs ce-provision - name: Install ce-provision if: ${{ github.event.pull_request.head.ref != 'documentation' }} @@ -32,6 +39,12 @@ jobs: chmod +x ./install.sh sudo ./install.sh --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker --no-firewall + - name: Check volumes. + run: | + ls -la /home/controller + ls -la /home/controller/ce-provision + ls -la /home/runner + # Run a web server provision - name: Prepare Git repos on disk if: ${{ github.event.pull_request.head.ref != 'documentation' }} @@ -39,9 +52,9 @@ jobs: git config --global --add safe.directory /home/controller/ce-provision git config --global --add safe.directory /home/controller/ce-provision/config - - name: Provision a test web server - if: ${{ github.event.pull_request.head.ref != 'documentation' }} - run: su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" + #- name: Provision a test web server + # if: ${{ github.event.pull_request.head.ref != 'documentation' }} + # run: su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" # Installs the ce-dev stack #- name: Install ce-dev From 75e27c3cb03cfbea84684d192758d35642b9bdd7 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 12:43:23 +0100 Subject: [PATCH 134/206] Tweaking volumes. --- .github/workflows/ce-provision-test-web.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index dea4fc789..d776e798e 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -29,7 +29,6 @@ jobs: run: | ls -la /home/controller ls -la /home/controller/ce-provision - ls -la /home/runner # Installs ce-provision - name: Install ce-provision @@ -43,7 +42,6 @@ jobs: run: | ls -la /home/controller ls -la /home/controller/ce-provision - ls -la /home/runner # Run a web server provision - name: Prepare Git repos on disk From c63f4407b4b1ec4f76127ac6dddd2efd8cd2fde1 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 12:45:30 +0100 Subject: [PATCH 135/206] Adding the checkout command back in. --- .github/workflows/ce-provision-test-web.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index d776e798e..a6742a9e1 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -21,9 +21,9 @@ jobs: steps: # Checks out a copy of your repository on the ubuntu-latest machine - #- name: Checkout code - # if: ${{ github.event.pull_request.head.ref != 'documentation' }} - # uses: actions/checkout@v2 + - name: Checkout code + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + uses: actions/checkout@v2 - name: Check volumes. run: | From d9af7c351eba158de28dc6f8656818d69a893726 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 12:49:37 +0100 Subject: [PATCH 136/206] Trying a different approach. --- .github/workflows/ce-provision-test-web.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index a6742a9e1..ff83c1c86 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -16,19 +16,17 @@ jobs: container: image: codeenigma/ce-dev:2.x volumes: - - /home/controller - - ${{ github.workspace }}:/home/controller/ce-provision + - ${{ github.workspace }}:/home/controller steps: # Checks out a copy of your repository on the ubuntu-latest machine - - name: Checkout code - if: ${{ github.event.pull_request.head.ref != 'documentation' }} - uses: actions/checkout@v2 + #- name: Checkout code + # if: ${{ github.event.pull_request.head.ref != 'documentation' }} + # uses: actions/checkout@v2 - name: Check volumes. run: | ls -la /home/controller - ls -la /home/controller/ce-provision # Installs ce-provision - name: Install ce-provision From cec85fb6df1aae025bba95bd8defaa2abbcef1d0 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 13:08:28 +0100 Subject: [PATCH 137/206] ls command looks good, so putting web build back in. --- .github/workflows/ce-provision-test-web.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index ff83c1c86..742b67611 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -48,9 +48,9 @@ jobs: git config --global --add safe.directory /home/controller/ce-provision git config --global --add safe.directory /home/controller/ce-provision/config - #- name: Provision a test web server - # if: ${{ github.event.pull_request.head.ref != 'documentation' }} - # run: su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" + - name: Provision a test web server + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + run: su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" # Installs the ce-dev stack #- name: Install ce-dev From 8865958e9413472de1073fea19743fe783630120 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 13:13:29 +0100 Subject: [PATCH 138/206] More Ansible Galaxy debug. --- .github/workflows/ce-provision-test-web.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 742b67611..df9e4ed41 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -50,7 +50,11 @@ jobs: - name: Provision a test web server if: ${{ github.event.pull_request.head.ref != 'documentation' }} - run: su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" + run: | + ls -la /home/controller/ce-provision + ls -la /home/controller/ce-provision/galaxy + /home/controller/ce-python/bin/ansible-galaxy role list + su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" # Installs the ce-dev stack #- name: Install ce-dev From f5d3c450cfb5a984720f944704b69024743b036b Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 13:29:45 +0100 Subject: [PATCH 139/206] Trying to make ansible-galaxy detect installed roles. --- .github/workflows/ce-provision-test-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index df9e4ed41..a8be0de1d 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -53,7 +53,7 @@ jobs: run: | ls -la /home/controller/ce-provision ls -la /home/controller/ce-provision/galaxy - /home/controller/ce-python/bin/ansible-galaxy role list + cd /home/controller/ce-provision && /home/controller/ce-python/bin/ansible-galaxy role list su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" # Installs the ce-dev stack From 8985c3dae6d2bc7422f9b6f69cae06bfd7255811 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 13:33:05 +0100 Subject: [PATCH 140/206] Run galaxy command as controller. --- .github/workflows/ce-provision-test-web.yml | 40 +-------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index a8be0de1d..2aad74299 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -19,16 +19,6 @@ jobs: - ${{ github.workspace }}:/home/controller steps: - # Checks out a copy of your repository on the ubuntu-latest machine - #- name: Checkout code - # if: ${{ github.event.pull_request.head.ref != 'documentation' }} - # uses: actions/checkout@v2 - - - name: Check volumes. - run: | - ls -la /home/controller - - # Installs ce-provision - name: Install ce-provision if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | @@ -36,11 +26,6 @@ jobs: chmod +x ./install.sh sudo ./install.sh --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker --no-firewall - - name: Check volumes. - run: | - ls -la /home/controller - ls -la /home/controller/ce-provision - # Run a web server provision - name: Prepare Git repos on disk if: ${{ github.event.pull_request.head.ref != 'documentation' }} @@ -53,28 +38,5 @@ jobs: run: | ls -la /home/controller/ce-provision ls -la /home/controller/ce-provision/galaxy - cd /home/controller/ce-provision && /home/controller/ce-python/bin/ansible-galaxy role list + cd /home/controller/ce-provision && su - controller -c "/home/controller/ce-python/bin/ansible-galaxy role list" su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" - - # Installs the ce-dev stack - #- name: Install ce-dev - # if: ${{ github.event.pull_request.head.ref != 'documentation' }} - # run: | - # cd /tmp - # wget https://golang.org/dl/go1.15.8.linux-amd64.tar.gz - # sudo tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz - # export PATH=$PATH:/usr/local/go/bin - # git clone https://github.com/FiloSottile/mkcert && cd mkcert - # go build -ldflags "-X main.Version=$(git describe --tags)" - # sudo mv ./mkcert /usr/local/bin && cd ../ - # sudo chmod +x /usr/local/bin/mkcert - # rm -Rf mkcert - # curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/${{ github.event.pull_request.base.ref }}/install.sh | /bin/sh -s -- --platform linux - - # Uses the ce-dev stack to run a test provision - #- name: Run a test provision - # if: ${{ github.event.pull_request.head.ref != 'documentation' }} - # run: | - # git clone --branch ${{ github.event.pull_request.base.ref }} https://github.com/codeenigma/ce-dev-ce-provision-config.git config - # /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} - # shell: bash From 85d091f3f1ab9d5a65a5a6d967ccd1dbaf87c37e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 13:36:58 +0100 Subject: [PATCH 141/206] Trying galaxy command and cd wrapped in su. --- .github/workflows/ce-provision-test-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 2aad74299..b75b48efc 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -38,5 +38,5 @@ jobs: run: | ls -la /home/controller/ce-provision ls -la /home/controller/ce-provision/galaxy - cd /home/controller/ce-provision && su - controller -c "/home/controller/ce-python/bin/ansible-galaxy role list" + su - controller -c "cd /home/controller/ce-provision && /home/controller/ce-python/bin/ansible-galaxy role list" su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" From 2f56b6a311d0bf9085746d9d1fa95d1f4da79d57 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 13:40:27 +0100 Subject: [PATCH 142/206] Specifically checking the contents of galaxy/roles. --- .github/workflows/ce-provision-test-web.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index b75b48efc..9a712dafb 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -25,6 +25,7 @@ jobs: curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh chmod +x ./install.sh sudo ./install.sh --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker --no-firewall + ls -la /home/controller/ce-provision/galaxy/roles # Run a web server provision - name: Prepare Git repos on disk @@ -38,5 +39,6 @@ jobs: run: | ls -la /home/controller/ce-provision ls -la /home/controller/ce-provision/galaxy + ls -la /home/controller/ce-provision/galaxy/roles su - controller -c "cd /home/controller/ce-provision && /home/controller/ce-python/bin/ansible-galaxy role list" su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" From f3fff365b15cf0bbce6336e1475d180ea3d44bf7 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 13:52:12 +0100 Subject: [PATCH 143/206] Trying a double-tap install process. --- install.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/install.sh b/install.sh index 08eb8c5e4..2561e6875 100755 --- a/install.sh +++ b/install.sh @@ -254,8 +254,12 @@ EOL # Tell Ansible this is a Docker container if [ "$IS_LOCAL" = "true" ]; then /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" + # Run a second time to install Ansible Galaxy roles in the right place + /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" else /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" + # Run a second time to install Ansible Galaxy roles in the right place + /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" fi /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" From d16e16994b11a5eabccfb28c1aa3361ee387454c Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 13:57:54 +0100 Subject: [PATCH 144/206] Quick refactor and debug of SSH. --- install.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/install.sh b/install.sh index 2561e6875..9c8acbbd3 100755 --- a/install.sh +++ b/install.sh @@ -79,6 +79,7 @@ FIREWALL="true" AWS_SUPPORT="false" IS_LOCAL="false" SERVER_HOSTNAME=$(hostname) +ANSIBLE_COMMAND="" # Parse options. parse_options "$@" @@ -253,14 +254,14 @@ EOL # Tell Ansible this is a Docker container if [ "$IS_LOCAL" = "true" ]; then - /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" - # Run a second time to install Ansible Galaxy roles in the right place - /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-playbook --extra-vars \"{is_local: $IS_LOCAL}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" + ANSIBLE_COMMAND="ansible-playbook --extra-vars \"{is_local: $IS_LOCAL}\" /home/$CONTROLLER_USER/ce-provision/provision.yml" else - /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" - # Run a second time to install Ansible Galaxy roles in the right place - /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" + ANSIBLE_COMMAND="ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" fi +/usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/$ANSIBLE_COMMAND" +ls -la /home/$CONTROLLER_USER/ce-provision/data +# Run a second time to install Ansible Galaxy roles in the right place +/usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/$ANSIBLE_COMMAND" /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" # Install firewall From 29c82cfeafb83dc3d4cd642e870b5439ff060950 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 14:05:31 +0100 Subject: [PATCH 145/206] Adding OpenSSH server package. --- install.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 9c8acbbd3..13b445793 100755 --- a/install.sh +++ b/install.sh @@ -139,7 +139,7 @@ fi git ca-certificates git-lfs \ openssh-client nfs-common stunnel4 \ python3-venv python3-debian \ - zip unzip gzip tar dnsutils + zip unzip gzip tar dnsutils openssh-server /usr/bin/echo "-------------------------------------------------" # Install Ansible in a Python virtual environment. @@ -259,7 +259,8 @@ else ANSIBLE_COMMAND="ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" fi /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/$ANSIBLE_COMMAND" -ls -la /home/$CONTROLLER_USER/ce-provision/data +ls -la "/home/$CONTROLLER_USER/ce-provision/data" +ls -la "/home/$CONTROLLER_USER/.ssh" # Run a second time to install Ansible Galaxy roles in the right place /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/$ANSIBLE_COMMAND" /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" From d3e72f11bce260c06eaecaf3c1106fd4e229633d Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 14:09:32 +0100 Subject: [PATCH 146/206] Checking for a firewall. --- install.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/install.sh b/install.sh index 13b445793..e600f3d70 100755 --- a/install.sh +++ b/install.sh @@ -130,6 +130,10 @@ else fi /usr/bin/echo "-------------------------------------------------" +# Debug +sudo iptables -L INPUT +telnet localhost 22 + # Install APT packages. /usr/bin/echo "Install required packages." /usr/bin/echo "-------------------------------------------------" From 031eeeaf44386d17e566c895efaa6b32cf7667c6 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 14:16:56 +0100 Subject: [PATCH 147/206] Checking listening packages. --- install.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/install.sh b/install.sh index e600f3d70..e7a24666d 100755 --- a/install.sh +++ b/install.sh @@ -130,10 +130,6 @@ else fi /usr/bin/echo "-------------------------------------------------" -# Debug -sudo iptables -L INPUT -telnet localhost 22 - # Install APT packages. /usr/bin/echo "Install required packages." /usr/bin/echo "-------------------------------------------------" @@ -143,9 +139,12 @@ telnet localhost 22 git ca-certificates git-lfs \ openssh-client nfs-common stunnel4 \ python3-venv python3-debian \ - zip unzip gzip tar dnsutils openssh-server + zip unzip gzip tar dnsutils net-tools /usr/bin/echo "-------------------------------------------------" +/usr/bin/echo "Listening apps" +/usr/bin/netstat -tulpn | grep LISTEN + # Install Ansible in a Python virtual environment. /usr/bin/echo "Install Ansible and dependencies." /usr/bin/echo "-------------------------------------------------" From d75c8efc951e66bd12669a340db505baa9944890 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 14:18:46 +0100 Subject: [PATCH 148/206] Starting SSHD especially. --- install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install.sh b/install.sh index e7a24666d..375759f3f 100755 --- a/install.sh +++ b/install.sh @@ -265,6 +265,9 @@ fi ls -la "/home/$CONTROLLER_USER/ce-provision/data" ls -la "/home/$CONTROLLER_USER/.ssh" # Run a second time to install Ansible Galaxy roles in the right place +/usr/bin/systemctl start sshd +/usr/bin/echo "Listening apps" +/usr/bin/netstat -tulpn | grep LISTEN /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/$ANSIBLE_COMMAND" /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" From 3d9f51e00f4402123f204a68c1493bdfaa698c5f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 14:25:23 +0100 Subject: [PATCH 149/206] Starting SSHD without systemd. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 375759f3f..1e41e4961 100755 --- a/install.sh +++ b/install.sh @@ -265,7 +265,7 @@ fi ls -la "/home/$CONTROLLER_USER/ce-provision/data" ls -la "/home/$CONTROLLER_USER/.ssh" # Run a second time to install Ansible Galaxy roles in the right place -/usr/bin/systemctl start sshd +/usr/sbin/sshd& /usr/bin/echo "Listening apps" /usr/bin/netstat -tulpn | grep LISTEN /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/$ANSIBLE_COMMAND" From db2dde66d8824f9849199932ac26eb3483f31754 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 14:33:28 +0100 Subject: [PATCH 150/206] Pre-empting config a bit more. --- install.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/install.sh b/install.sh index 1e41e4961..821f92288 100755 --- a/install.sh +++ b/install.sh @@ -162,6 +162,8 @@ fi /usr/bin/echo "-------------------------------------------------" if [ ! -d "/home/$CONTROLLER_USER/ce-provision" ]; then /usr/bin/su - "$CONTROLLER_USER" -c "git clone --branch $VERSION https://github.com/codeenigma/ce-provision.git /home/$CONTROLLER_USER/ce-provision" + /usr/bin/su - "$CONTROLLER_USER" -c "git clone --branch $CONFIG_REPO_BRANCH $CONFIG_REPO /home/$CONTROLLER_USER/ce-provision/config" + /usr/bin/su - "$CONTROLLER_USER" -c "/usr/bin/ln -s /home/$CONTROLLER_USER/ce-provision/config/ansible.cfg /home/$CONTROLLER_USER/ce-provision/ansible.cfg" else /usr/bin/echo "ce-provision directory at /home/$CONTROLLER_USER/ce-provision already exists. Skipping." /usr/bin/echo "-------------------------------------------------" @@ -261,14 +263,8 @@ if [ "$IS_LOCAL" = "true" ]; then else ANSIBLE_COMMAND="ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" fi -/usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/$ANSIBLE_COMMAND" -ls -la "/home/$CONTROLLER_USER/ce-provision/data" -ls -la "/home/$CONTROLLER_USER/.ssh" -# Run a second time to install Ansible Galaxy roles in the right place -/usr/sbin/sshd& -/usr/bin/echo "Listening apps" -/usr/bin/netstat -tulpn | grep LISTEN /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/$ANSIBLE_COMMAND" +ls -la "/home/$CONTROLLER_USER/ce-provision/galaxy/roles" /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" # Install firewall From 53c17cd03e55ab7f72d1dc7c6bce4bc30e193b8d Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 14:38:59 +0100 Subject: [PATCH 151/206] More galaxy path debug. --- install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install.sh b/install.sh index 821f92288..7595ead3b 100755 --- a/install.sh +++ b/install.sh @@ -265,6 +265,9 @@ else fi /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/$ANSIBLE_COMMAND" ls -la "/home/$CONTROLLER_USER/ce-provision/galaxy/roles" +/usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER" +/usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy role list" +/usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection list" /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" # Install firewall From e77584faa4a8cbf3707cf163e8f7e83a67a11b16 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 18:00:21 +0100 Subject: [PATCH 152/206] Running a find to see if we can find the missing roles. --- install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install.sh b/install.sh index 7595ead3b..d81f50a94 100755 --- a/install.sh +++ b/install.sh @@ -265,6 +265,8 @@ else fi /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/$ANSIBLE_COMMAND" ls -la "/home/$CONTROLLER_USER/ce-provision/galaxy/roles" +find /home/controller/ -name 'geerlingguy.*' + /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER" /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy role list" /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection list" From 73f06826f6a01cadd4c866a0aaaba4728c7daeb4 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 18:03:57 +0100 Subject: [PATCH 153/206] More verbosity. --- install.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index d81f50a94..313be4d40 100755 --- a/install.sh +++ b/install.sh @@ -265,7 +265,10 @@ else fi /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/$ANSIBLE_COMMAND" ls -la "/home/$CONTROLLER_USER/ce-provision/galaxy/roles" -find /home/controller/ -name 'geerlingguy.*' +/usr/bin/echo "-------------------------------------------------" +/usr/bin/echo "Search for missing roles" +find / -name 'geerlingguy.*' +/usr/bin/echo "-------------------------------------------------" /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER" /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy role list" From b2f3fd4836bd50a3a579fc745661ea2b7535dc12 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 18:13:34 +0100 Subject: [PATCH 154/206] Checking for missing requirements file. --- install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install.sh b/install.sh index 313be4d40..286ba0f78 100755 --- a/install.sh +++ b/install.sh @@ -267,6 +267,7 @@ fi ls -la "/home/$CONTROLLER_USER/ce-provision/galaxy/roles" /usr/bin/echo "-------------------------------------------------" /usr/bin/echo "Search for missing roles" +cat "/home/$CONTROLLER_USER/ce-provision/roles/debian/ce_provision/meta/requirements-12.yml" find / -name 'geerlingguy.*' /usr/bin/echo "-------------------------------------------------" From 77709be88ea3c729748ecdbecf85c56969903b2f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 18:18:50 +0100 Subject: [PATCH 155/206] Removing eroneous when clause. --- install.sh | 3 --- roles/debian/ansible_galaxy/tasks/main.yml | 1 - 2 files changed, 4 deletions(-) diff --git a/install.sh b/install.sh index 286ba0f78..fb5ce2d19 100755 --- a/install.sh +++ b/install.sh @@ -142,9 +142,6 @@ fi zip unzip gzip tar dnsutils net-tools /usr/bin/echo "-------------------------------------------------" -/usr/bin/echo "Listening apps" -/usr/bin/netstat -tulpn | grep LISTEN - # Install Ansible in a Python virtual environment. /usr/bin/echo "Install Ansible and dependencies." /usr/bin/echo "-------------------------------------------------" diff --git a/roles/debian/ansible_galaxy/tasks/main.yml b/roles/debian/ansible_galaxy/tasks/main.yml index 5dde4a6df..106943f1d 100644 --- a/roles/debian/ansible_galaxy/tasks/main.yml +++ b/roles/debian/ansible_galaxy/tasks/main.yml @@ -13,7 +13,6 @@ - name: Install ansible-galaxy roles and/or collections. when: - _galaxy_requirements.stat.exists - - not is_local block: - name: Set up the ansible-galaxy command. ansible.builtin.set_fact: From e0cbd165c155b18c86bf33b9b9de7a87695a809a Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 18:26:54 +0100 Subject: [PATCH 156/206] Tidying up redundant debug lines. --- .github/workflows/ce-provision-test-web.yml | 4 ---- install.sh | 11 +---------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 9a712dafb..89c157f06 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -25,7 +25,6 @@ jobs: curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh chmod +x ./install.sh sudo ./install.sh --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker --no-firewall - ls -la /home/controller/ce-provision/galaxy/roles # Run a web server provision - name: Prepare Git repos on disk @@ -37,8 +36,5 @@ jobs: - name: Provision a test web server if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - ls -la /home/controller/ce-provision - ls -la /home/controller/ce-provision/galaxy - ls -la /home/controller/ce-provision/galaxy/roles su - controller -c "cd /home/controller/ce-provision && /home/controller/ce-python/bin/ansible-galaxy role list" su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" diff --git a/install.sh b/install.sh index fb5ce2d19..7ca8822d0 100755 --- a/install.sh +++ b/install.sh @@ -260,17 +260,8 @@ if [ "$IS_LOCAL" = "true" ]; then else ANSIBLE_COMMAND="ansible-playbook /home/$CONTROLLER_USER/ce-provision/provision.yml" fi +# Configure ce-provision /usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/$ANSIBLE_COMMAND" -ls -la "/home/$CONTROLLER_USER/ce-provision/galaxy/roles" -/usr/bin/echo "-------------------------------------------------" -/usr/bin/echo "Search for missing roles" -cat "/home/$CONTROLLER_USER/ce-provision/roles/debian/ce_provision/meta/requirements-12.yml" -find / -name 'geerlingguy.*' -/usr/bin/echo "-------------------------------------------------" - -/usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER" -/usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy role list" -/usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection list" /usr/bin/rm "/home/$CONTROLLER_USER/ce-provision/provision.yml" # Install firewall From 704c912dbd5010de6eb478a2b7c1b642b237f528 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 18:34:07 +0100 Subject: [PATCH 157/206] Creating a separate ci.yml play targeting localhost. --- .github/workflows/ce-provision-test-web.yml | 3 +-- ce-dev/ansible/plays/web/ci.yml | 22 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 ce-dev/ansible/plays/web/ci.yml diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 89c157f06..a04ea5100 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -36,5 +36,4 @@ jobs: - name: Provision a test web server if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - su - controller -c "cd /home/controller/ce-provision && /home/controller/ce-python/bin/ansible-galaxy role list" - su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/web.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" + su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/ci.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" diff --git a/ce-dev/ansible/plays/web/ci.yml b/ce-dev/ansible/plays/web/ci.yml new file mode 100644 index 000000000..94bd2f49f --- /dev/null +++ b/ce-dev/ansible/plays/web/ci.yml @@ -0,0 +1,22 @@ +--- +# Spin up a "web" instance. +- hosts: localhost + become: true + + vars: + project_name: web + is_local: true + _ce_provision_base_dir: /home/ce-dev/ce-provision + _init: + force_play: true + vars_dirs: + - "{{ _ce_provision_base_dir }}/ce-dev/ansible/vars/_common" + - "{{ _ce_provision_base_dir }}/ce-dev/ansible/vars/{{ project_name }}" + + tasks: + - ansible.builtin.import_role: + name: _init + - ansible.builtin.import_role: + name: _meta/webserver + - ansible.builtin.import_role: + name: _exit From 3ce84d543e8eefbd86c60ec54500fdf5abc110c5 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 18:35:21 +0100 Subject: [PATCH 158/206] Making sure sshd is running. --- .github/workflows/ce-provision-test-web.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index a04ea5100..c0eaa2545 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -22,18 +22,19 @@ jobs: - name: Install ce-provision if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh - chmod +x ./install.sh - sudo ./install.sh --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker --no-firewall + /usr/bin/curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh + /usr/bin/chmod +x ./install.sh + /usr/bin/sudo ./install.sh --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker --no-firewall # Run a web server provision - name: Prepare Git repos on disk if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - git config --global --add safe.directory /home/controller/ce-provision - git config --global --add safe.directory /home/controller/ce-provision/config + /usr/bin/git config --global --add safe.directory /home/controller/ce-provision + /usr/bin/git config --global --add safe.directory /home/controller/ce-provision/config - name: Provision a test web server if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/ci.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" + /usr/bin/sshd& + /usr/bin/su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/ci.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" From ebbb829340e14acb926d7dae63be85f450e300cb Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 18:40:14 +0100 Subject: [PATCH 159/206] Tidying up GitLab CI file and installing SSHD. --- .../workflows/ce-provision-test-gitlab.yml | 45 +++++++++---------- .github/workflows/ce-provision-test-web.yml | 3 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ce-provision-test-gitlab.yml b/.github/workflows/ce-provision-test-gitlab.yml index 4d7f226d9..266e8952c 100644 --- a/.github/workflows/ce-provision-test-gitlab.yml +++ b/.github/workflows/ce-provision-test-gitlab.yml @@ -8,35 +8,34 @@ jobs: # when a job name is not provided test-gitlab: # Name the Job - name: Run tests against Ansible code base + name: Build a GitLab server with ce-provision # Set the type of machine to run on runs-on: ubuntu-latest - steps: - # Checks out a copy of your repository on the ubuntu-latest machine - - name: Checkout code - if: ${{ github.event.pull_request.head.ref != 'documentation' }} - uses: actions/checkout@v2 + # Use our ce-dev Debian base container + container: + image: codeenigma/ce-dev:2.x + volumes: + - ${{ github.workspace }}:/home/controller - # Installs the ce-dev stack - - name: Install ce-dev + steps: + - name: Install ce-provision if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - cd /tmp - wget https://golang.org/dl/go1.15.8.linux-amd64.tar.gz - sudo tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz - export PATH=$PATH:/usr/local/go/bin - git clone https://github.com/FiloSottile/mkcert && cd mkcert - go build -ldflags "-X main.Version=$(git describe --tags)" - sudo mv ./mkcert /usr/local/bin && cd ../ - sudo chmod +x /usr/local/bin/mkcert - rm -Rf mkcert - curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/${{ github.event.pull_request.base.ref }}/install.sh | /bin/sh -s -- --platform linux + /usr/bin/curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh + /usr/bin/chmod +x ./install.sh + /usr/bin/sudo ./install.sh --version ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --docker --no-firewall - # Uses the ce-dev stack to run a test provision - - name: Run a test provision + # Run a GitLab server provision + - name: Prepare Git repos on disk if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | - git clone --branch ${{ github.event.pull_request.base.ref }} https://github.com/codeenigma/ce-dev-ce-provision-config.git config - /bin/bash ce-dev/ansible/test.sh --examples gitlab --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} - shell: bash + /usr/bin/git config --global --add safe.directory /home/controller/ce-provision + /usr/bin/git config --global --add safe.directory /home/controller/ce-provision/config + + #- name: Run a test provision + # if: ${{ github.event.pull_request.head.ref != 'documentation' }} + # run: | + # git clone --branch ${{ github.event.pull_request.base.ref }} https://github.com/codeenigma/ce-dev-ce-provision-config.git config + # /bin/bash ce-dev/ansible/test.sh --examples gitlab --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + # shell: bash diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index c0eaa2545..eacbabb01 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -8,7 +8,7 @@ jobs: # when a job name is not provided test-web: # Name the Job - name: Run tests against Ansible code base + name: Build a web server with ce-provision # Set the type of machine to run on runs-on: ubuntu-latest @@ -36,5 +36,6 @@ jobs: - name: Provision a test web server if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | + /usr/bin/apt-get install -y -o Dpkg::Options::="--force-confnew" openssh-server /usr/bin/sshd& /usr/bin/su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/ci.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" From 6ddd20865270218d3878c8ef364df4cfa13aead9 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 18:46:07 +0100 Subject: [PATCH 160/206] Installing SSHD as a separate step. --- .github/workflows/ce-provision-test-web.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index eacbabb01..0b30ffce9 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -33,9 +33,10 @@ jobs: /usr/bin/git config --global --add safe.directory /home/controller/ce-provision /usr/bin/git config --global --add safe.directory /home/controller/ce-provision/config + - name: Install SSHD + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + run: /usr/bin/apt-get install -y -o Dpkg::Options::="--force-confnew" openssh-server + - name: Provision a test web server if: ${{ github.event.pull_request.head.ref != 'documentation' }} - run: | - /usr/bin/apt-get install -y -o Dpkg::Options::="--force-confnew" openssh-server - /usr/bin/sshd& - /usr/bin/su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/ci.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" + run: /usr/bin/su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/ci.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" From 9f616ad8f46326a3106f6a0c8b829ef49be755e5 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 18:50:24 +0100 Subject: [PATCH 161/206] SSHD already installed, starting it instead. --- .github/workflows/ce-provision-test-web.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 0b30ffce9..2996caae4 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -33,9 +33,9 @@ jobs: /usr/bin/git config --global --add safe.directory /home/controller/ce-provision /usr/bin/git config --global --add safe.directory /home/controller/ce-provision/config - - name: Install SSHD + - name: Start SSHD if: ${{ github.event.pull_request.head.ref != 'documentation' }} - run: /usr/bin/apt-get install -y -o Dpkg::Options::="--force-confnew" openssh-server + run: /usr/sbin/sshd& - name: Provision a test web server if: ${{ github.event.pull_request.head.ref != 'documentation' }} From 4d819186435490131d3710d185db8a6c36f1f4d3 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 18:57:20 +0100 Subject: [PATCH 162/206] Don't create systemd timers in containers. --- roles/debian/apt_repository/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/debian/apt_repository/tasks/main.yml b/roles/debian/apt_repository/tasks/main.yml index df016422b..cef05098e 100644 --- a/roles/debian/apt_repository/tasks/main.yml +++ b/roles/debian/apt_repository/tasks/main.yml @@ -66,6 +66,7 @@ - apt_repository.signed_by is defined - apt_repository.signed_by | length > 0 - apt_repository.signed_by is url # https://docs.ansible.com/ansible/latest/collections/ansible/builtin/url_test.html + - not is_local block: - name: Create script to refresh APT repository key. ansible.builtin.template: From cfe0e2dadc767db17069c3540d063fde83742d4e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 19:01:06 +0100 Subject: [PATCH 163/206] Preparing a test GitLab build. --- .../workflows/ce-provision-test-gitlab.yml | 13 ++++--- ce-dev/ansible/plays/gitlab/ci.yml | 39 +++++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 ce-dev/ansible/plays/gitlab/ci.yml diff --git a/.github/workflows/ce-provision-test-gitlab.yml b/.github/workflows/ce-provision-test-gitlab.yml index 266e8952c..c44361d00 100644 --- a/.github/workflows/ce-provision-test-gitlab.yml +++ b/.github/workflows/ce-provision-test-gitlab.yml @@ -33,9 +33,10 @@ jobs: /usr/bin/git config --global --add safe.directory /home/controller/ce-provision /usr/bin/git config --global --add safe.directory /home/controller/ce-provision/config - #- name: Run a test provision - # if: ${{ github.event.pull_request.head.ref != 'documentation' }} - # run: | - # git clone --branch ${{ github.event.pull_request.base.ref }} https://github.com/codeenigma/ce-dev-ce-provision-config.git config - # /bin/bash ce-dev/ansible/test.sh --examples gitlab --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} - # shell: bash + - name: Start SSHD + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + run: /usr/sbin/sshd& + + - name: Provision a test GitLab server + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + run: /usr/bin/su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/gitlab/ci.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" diff --git a/ce-dev/ansible/plays/gitlab/ci.yml b/ce-dev/ansible/plays/gitlab/ci.yml new file mode 100644 index 000000000..1ad4a2561 --- /dev/null +++ b/ce-dev/ansible/plays/gitlab/ci.yml @@ -0,0 +1,39 @@ +--- +- hosts: localhost + become: true + + vars: + project_name: gitlab + is_local: true + _ce_provision_base_dir: /home/ce-dev/ce-provision + _init: + force_play: true + vars_dirs: + - "{{ _ce_provision_base_dir }}/ce-dev/ansible/vars/_common" + - "{{ _ce_provision_base_dir }}/ce-dev/ansible/vars/{{ project_name }}" + + tasks: + - ansible.builtin.import_role: + name: _init + - ansible.builtin.import_role: + name: debian/user_provision + - ansible.builtin.import_role: + name: _meta/common_base + - ansible.builtin.import_role: + name: debian/ce_deploy + - ansible.builtin.import_role: + name: aws/aws_credentials + - ansible.builtin.import_role: + name: debian/gitlab + - ansible.builtin.import_role: + name: debian/gitlab_runner + - ansible.builtin.import_role: + name: debian/ssh_server + - ansible.builtin.import_role: + name: debian/sops + - ansible.builtin.import_role: + name: debian/gpg_key + - ansible.builtin.import_role: + name: debian/firewall_config + - ansible.builtin.import_role: + name: _exit From 62c2bb15a1de798d604cae1bc6af0bbb9326dfaf Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 17 Dec 2024 19:11:19 +0100 Subject: [PATCH 164/206] Making builds nightly and fixing GitLab role bug. --- .github/workflows/ce-provision-test-gitlab.yml | 7 +++++-- .github/workflows/ce-provision-test-web.yml | 7 +++++-- roles/debian/gitlab/defaults/main.yml | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ce-provision-test-gitlab.yml b/.github/workflows/ce-provision-test-gitlab.yml index c44361d00..3fffbf263 100644 --- a/.github/workflows/ce-provision-test-gitlab.yml +++ b/.github/workflows/ce-provision-test-gitlab.yml @@ -1,7 +1,10 @@ name: Run GitLab server test build -# Run this workflow every time a new commit pushed to your repository -on: pull_request +# Run this workflow nightly and every time a new commit pushed to your repository +on: + schedule: + - cron: '30 4 * * *' + pull_request: jobs: # Set the job key. The key is displayed as the job name diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index 2996caae4..ab39b9e29 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -1,7 +1,10 @@ name: Run web server test build -# Run this workflow every time a new commit pushed to your repository -on: pull_request +# Run this workflow nightly and every time a new commit pushed to your repository +on: + schedule: + - cron: '30 4 * * *' + pull_request: jobs: # Set the job key. The key is displayed as the job name diff --git a/roles/debian/gitlab/defaults/main.yml b/roles/debian/gitlab/defaults/main.yml index f09d68f18..e2ed2ff05 100644 --- a/roles/debian/gitlab/defaults/main.yml +++ b/roles/debian/gitlab/defaults/main.yml @@ -25,6 +25,7 @@ gitlab: ssl: # @see the 'ssl' role. Note that domain is autopopulated from server_name above. enabled: false # manual SSL handling disabled by default handling: selfsigned + replace_existing: false # Linux setup linux_user: git linux_group: git From eeb5f7425117a4b8ec5dcadf804b2ed16094d490 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 10:50:13 +0100 Subject: [PATCH 165/206] Ensuring is_local var exists and making lock behaviour optional. --- roles/_exit/tasks/main.yml | 1 + roles/_init/defaults/main.yml | 9 ++++++--- roles/_init/tasks/main.yml | 6 +++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/roles/_exit/tasks/main.yml b/roles/_exit/tasks/main.yml index e6a844f4d..51d676278 100644 --- a/roles/_exit/tasks/main.yml +++ b/roles/_exit/tasks/main.yml @@ -53,3 +53,4 @@ ansible.builtin.file: path: "{{ _init.lock_file }}" state: absent + when: _init.lock_file | length > 0 diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 97a5ad1bf..5c2d85d42 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -1,5 +1,8 @@ --- -_ce_provision_username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}" +# Set this variable to true to tell ce-provision it is running in a container. +is_local: false + +_ce_provision_username: "{% if is_local %}ce-dev{% else %}controller{% endif %}" _venv_path: "/home/{{ _ce_provision_username }}/ce-python" _venv_command: /usr/bin/python3 -m venv _venv_install_username: "{{ _ce_provision_username }}" @@ -15,8 +18,8 @@ _init: # This is used to detect if the playbook must re-run or not. vars_dirs: [] force_play: false - lock_file: /tmp/ce-provision-lock - deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy + lock_file: /tmp/ce-provision-lock # set to an empty string to disable locking behaviour + deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy, set to an empty string to disable locking behaviour ce_provision_version: 2.x # Outputted by the _init role at the start of plays install_ansible: true # set to false to not install Ansible in a venv diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index 3207c13f4..ac996f250 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -8,10 +8,13 @@ - name: Check for a ce-deploy lock file. ansible.builtin.stat: path: "{{ _init.deploy_lock_file }}" + when: _init.deploy_lock_file | length > 0 register: _ce_deploy_lock - name: Abort if ce-deploy lock file exists. - when: _ce_deploy_lock.stat.exists is defined and _ce_deploy_lock.stat.exists + when: + - _init.deploy_lock_file | length > 0 + - _ce_deploy_lock.stat.exists is defined and _ce_deploy_lock.stat.exists block: - name: Abort if ce-deploy lock file is found. ansible.builtin.debug: @@ -31,6 +34,7 @@ path: "{{ _init.lock_file }}" state: touch mode: 0644 + when: _init.lock_file | length > 0 # Load Linux services into ansible_facts.services. - name: Populate service facts From d8f4984c20bc542f774836111923e12321f69b87 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 10:51:56 +0100 Subject: [PATCH 166/206] Fixing location and owner of Blackfire config so it is configurable. --- roles/debian/php_blackfire/defaults/main.yml | 2 ++ roles/debian/php_blackfire/tasks/main.yml | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/roles/debian/php_blackfire/defaults/main.yml b/roles/debian/php_blackfire/defaults/main.yml index 323e94a55..1fc9c0bba 100644 --- a/roles/debian/php_blackfire/defaults/main.yml +++ b/roles/debian/php_blackfire/defaults/main.yml @@ -6,3 +6,5 @@ blackfire: agent_server_token: "" agent_client_id: "" agent_client_token: "" + agent_cli_config_path: /home/controller/.blackfire.ini + agent_cli_config_owner: controller diff --git a/roles/debian/php_blackfire/tasks/main.yml b/roles/debian/php_blackfire/tasks/main.yml index d7e513f0f..e093d607c 100644 --- a/roles/debian/php_blackfire/tasks/main.yml +++ b/roles/debian/php_blackfire/tasks/main.yml @@ -67,13 +67,13 @@ - name: Generate agent CLI configuration. ansible.builtin.template: src: blackfire.ini.j2 - dest: "/home/vagrant/.blackfire.ini" - owner: vagrant - group: vagrant + dest: "{{ blackfire.agent_cli_config_path }}" + owner: "{{ blackfire.agent_cli_config_owner }}" + group: "{{ blackfire.agent_cli_config_owner }}" mode: 0644 when: - blackfire.enable - - is_local is defined and is_local + - is_local - name: Trigger overrides ansible.builtin.include_role: From 66bd1e4e1d5b29bc24b2193db9e53018e8de9b6c Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 10:54:05 +0100 Subject: [PATCH 167/206] Documentation update. --- docs/roles/_init.md | 9 ++++++--- docs/roles/debian/aws_efs_client.md | 2 +- docs/roles/debian/ce_deploy.md | 2 +- docs/roles/debian/gitlab.md | 1 + docs/roles/debian/php-fpm.md | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/roles/_init.md b/docs/roles/_init.md index 0f40180e3..e619e5656 100644 --- a/docs/roles/_init.md +++ b/docs/roles/_init.md @@ -9,7 +9,10 @@ This is meant to ALWAYS be included as the first task of a play. If you include ## Default variables ```yaml --- -_ce_provision_username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}" +# Set this variable to true to tell ce-provision it is running in a container. +is_local: false + +_ce_provision_username: "{% if is_local %}ce-dev{% else %}controller{% endif %}" _venv_path: "/home/{{ _ce_provision_username }}/ce-python" _venv_command: /usr/bin/python3 -m venv _venv_install_username: "{{ _ce_provision_username }}" @@ -25,8 +28,8 @@ _init: # This is used to detect if the playbook must re-run or not. vars_dirs: [] force_play: false - lock_file: /tmp/ce-provision-lock - deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy + lock_file: /tmp/ce-provision-lock # set to an empty string to disable locking behaviour + deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy, set to an empty string to disable locking behaviour ce_provision_version: 2.x # Outputted by the _init role at the start of plays install_ansible: true # set to false to not install Ansible in a venv diff --git a/docs/roles/debian/aws_efs_client.md b/docs/roles/debian/aws_efs_client.md index 0711f04d9..5fc392261 100644 --- a/docs/roles/debian/aws_efs_client.md +++ b/docs/roles/debian/aws_efs_client.md @@ -46,7 +46,7 @@ _mount_state: present aws_efs_client: aws_profile: example # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set - version: 2.1.0 # version of AWS EFS utils to use + version: "{{ '1.35.0' if ansible_distribution_major_version | int < 12 else '2.1.0' }}" # 2.1.0 requires libssl v3 which is absent on Debian < 12 by default. build_suffix: "-1_all" # sometimes there is a suffix appended to the package name, e.g. `amazon-efs-utils-1.35.0-1_all.deb` deb_url: "" # provide an alternative location for the .deb package # See https://docs.ansible.com/ansible/latest/modules/mount_module.html diff --git a/docs/roles/debian/ce_deploy.md b/docs/roles/debian/ce_deploy.md index 90aa38931..743cbf8cd 100644 --- a/docs/roles/debian/ce_deploy.md +++ b/docs/roles/debian/ce_deploy.md @@ -8,7 +8,7 @@ Installs Code Enigma's deploy stack on a server. ```yaml --- _ce_deploy: - username: "{% if is_local is defined and is_local %}ce-dev{% else %}deploy{% endif %}" + username: "{% if is_local %}ce-dev{% else %}deploy{% endif %}" ce_deploy: # These are usually set in the _init role using _venv_path, _venv_command and _venv_install_username but can be overridden. diff --git a/docs/roles/debian/gitlab.md b/docs/roles/debian/gitlab.md index c02282fd2..f4b11638b 100644 --- a/docs/roles/debian/gitlab.md +++ b/docs/roles/debian/gitlab.md @@ -38,6 +38,7 @@ gitlab: ssl: # @see the 'ssl' role. Note that domain is autopopulated from server_name above. enabled: false # manual SSL handling disabled by default handling: selfsigned + replace_existing: false # Linux setup linux_user: git linux_group: git diff --git a/docs/roles/debian/php-fpm.md b/docs/roles/debian/php-fpm.md index 063e97276..443ffbf87 100644 --- a/docs/roles/debian/php-fpm.md +++ b/docs/roles/debian/php-fpm.md @@ -23,7 +23,7 @@ php: # It is important to scale up processes on bigger servers, so that more # requests can be handled. Double the number of vCPUs is a good default. # Can be between 5 and 64. - max_children: "{{ [5, [ansible_facts.ansible_processor_nproc * 2, 64] | min] | max }}" + max_children: "{{ [5, [(ansible_facts.ansible_processor_nproc | default(1)) * 2, 64] | min] | max }}" # Fallback in case ansible_processor_nproc is not gathered before tasks start_servers: 2 min_spare_servers: 1 max_spare_servers: 3 From c6a410623fb70feb7edbdaaa8e689ceb6f3eeacb Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 10:54:27 +0100 Subject: [PATCH 168/206] Removing all is defined checks for is_local since it is now always defined. --- roles/_init/README.md | 9 ++++++--- roles/debian/aws_efs_client/README.md | 2 +- roles/debian/ce_deploy/README.md | 2 +- roles/debian/ce_deploy/defaults/main.yml | 2 +- roles/debian/gitlab/README.md | 1 + roles/debian/gitlab/tasks/main.yml | 12 +++--------- roles/debian/locales/tasks/main.yml | 2 +- roles/debian/mysql_server_mariadb/tasks/main.yml | 4 ++-- roles/debian/mysql_server_oracle_ce/tasks/main.yml | 4 ++-- roles/debian/nginx/templates/drupal10.j2 | 2 +- roles/debian/nginx/templates/drupal_common.j2 | 2 +- roles/debian/nginx/templates/mautic.j2 | 2 +- roles/debian/php-fpm/README.md | 2 +- roles/debian/postfix/tasks/main.yml | 7 +------ roles/debian/postfix/templates/main.cf.j2 | 2 +- roles/debian/user_deploy/defaults/main.yml | 2 +- roles/debian/user_provision/defaults/main.yml | 2 +- 17 files changed, 26 insertions(+), 33 deletions(-) diff --git a/roles/_init/README.md b/roles/_init/README.md index 0f40180e3..e619e5656 100644 --- a/roles/_init/README.md +++ b/roles/_init/README.md @@ -9,7 +9,10 @@ This is meant to ALWAYS be included as the first task of a play. If you include ## Default variables ```yaml --- -_ce_provision_username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}" +# Set this variable to true to tell ce-provision it is running in a container. +is_local: false + +_ce_provision_username: "{% if is_local %}ce-dev{% else %}controller{% endif %}" _venv_path: "/home/{{ _ce_provision_username }}/ce-python" _venv_command: /usr/bin/python3 -m venv _venv_install_username: "{{ _ce_provision_username }}" @@ -25,8 +28,8 @@ _init: # This is used to detect if the playbook must re-run or not. vars_dirs: [] force_play: false - lock_file: /tmp/ce-provision-lock - deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy + lock_file: /tmp/ce-provision-lock # set to an empty string to disable locking behaviour + deploy_lock_file: /tmp/ce-deploy-lock # must match lock_file in ce-deploy, set to an empty string to disable locking behaviour ce_provision_version: 2.x # Outputted by the _init role at the start of plays install_ansible: true # set to false to not install Ansible in a venv diff --git a/roles/debian/aws_efs_client/README.md b/roles/debian/aws_efs_client/README.md index 0711f04d9..5fc392261 100644 --- a/roles/debian/aws_efs_client/README.md +++ b/roles/debian/aws_efs_client/README.md @@ -46,7 +46,7 @@ _mount_state: present aws_efs_client: aws_profile: example # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set region: eu-west-1 # AWS region name - can be substituted for "{{ _aws_region }}" if set - version: 2.1.0 # version of AWS EFS utils to use + version: "{{ '1.35.0' if ansible_distribution_major_version | int < 12 else '2.1.0' }}" # 2.1.0 requires libssl v3 which is absent on Debian < 12 by default. build_suffix: "-1_all" # sometimes there is a suffix appended to the package name, e.g. `amazon-efs-utils-1.35.0-1_all.deb` deb_url: "" # provide an alternative location for the .deb package # See https://docs.ansible.com/ansible/latest/modules/mount_module.html diff --git a/roles/debian/ce_deploy/README.md b/roles/debian/ce_deploy/README.md index 90aa38931..743cbf8cd 100644 --- a/roles/debian/ce_deploy/README.md +++ b/roles/debian/ce_deploy/README.md @@ -8,7 +8,7 @@ Installs Code Enigma's deploy stack on a server. ```yaml --- _ce_deploy: - username: "{% if is_local is defined and is_local %}ce-dev{% else %}deploy{% endif %}" + username: "{% if is_local %}ce-dev{% else %}deploy{% endif %}" ce_deploy: # These are usually set in the _init role using _venv_path, _venv_command and _venv_install_username but can be overridden. diff --git a/roles/debian/ce_deploy/defaults/main.yml b/roles/debian/ce_deploy/defaults/main.yml index 2999edae9..b4af9748b 100644 --- a/roles/debian/ce_deploy/defaults/main.yml +++ b/roles/debian/ce_deploy/defaults/main.yml @@ -1,6 +1,6 @@ --- _ce_deploy: - username: "{% if is_local is defined and is_local %}ce-dev{% else %}deploy{% endif %}" + username: "{% if is_local %}ce-dev{% else %}deploy{% endif %}" ce_deploy: # These are usually set in the _init role using _venv_path, _venv_command and _venv_install_username but can be overridden. diff --git a/roles/debian/gitlab/README.md b/roles/debian/gitlab/README.md index c02282fd2..f4b11638b 100644 --- a/roles/debian/gitlab/README.md +++ b/roles/debian/gitlab/README.md @@ -38,6 +38,7 @@ gitlab: ssl: # @see the 'ssl' role. Note that domain is autopopulated from server_name above. enabled: false # manual SSL handling disabled by default handling: selfsigned + replace_existing: false # Linux setup linux_user: git linux_group: git diff --git a/roles/debian/gitlab/tasks/main.yml b/roles/debian/gitlab/tasks/main.yml index f98820592..3d4a962c7 100644 --- a/roles/debian/gitlab/tasks/main.yml +++ b/roles/debian/gitlab/tasks/main.yml @@ -98,9 +98,7 @@ content: "gitlab-docker" dest: "/opt/gitlab/embedded/service/gitlab-rails/INSTALLATION_TYPE" mode: "0666" - when: - - is_local is defined - - is_local + when: is_local - name: Copy startup script in place. ansible.builtin.template: @@ -110,9 +108,7 @@ group: root mode: "0555" force: true - when: - - is_local is defined - - is_local + when: is_local - name: Trigger overrides ansible.builtin.include_role: @@ -125,9 +121,7 @@ - name: Manually restart Gitlab/Docker. ansible.builtin.command: "/bin/sh /opt/gitlab-init.sh" - when: - - is_local is defined - - is_local + when: is_local - name: Ensure GitLab is started. ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl start diff --git a/roles/debian/locales/tasks/main.yml b/roles/debian/locales/tasks/main.yml index 7cdf73af4..e6e8af3bf 100644 --- a/roles/debian/locales/tasks/main.yml +++ b/roles/debian/locales/tasks/main.yml @@ -33,4 +33,4 @@ - name: Set timezone. community.general.timezone: name: "{{ locales.timezone }}" - when: not is_local is defined or not is_local + when: not is_local diff --git a/roles/debian/mysql_server_mariadb/tasks/main.yml b/roles/debian/mysql_server_mariadb/tasks/main.yml index f76d88ca7..f463d95e8 100644 --- a/roles/debian/mysql_server_mariadb/tasks/main.yml +++ b/roles/debian/mysql_server_mariadb/tasks/main.yml @@ -33,12 +33,12 @@ - name: Update MySQL root password. ansible.builtin.shell: > mysql -NBe 'GRANT ALL ON *.* TO "root"@"%" IDENTIFIED BY "root" WITH GRANT OPTION;' - when: is_local is defined and is_local + when: is_local - name: Update MySQL ce-dev password. ansible.builtin.shell: > mysql -NBe 'GRANT ALL ON *.* TO "ce-dev"@"%" IDENTIFIED BY "ce-dev" WITH GRANT OPTION;' - when: is_local is defined and is_local + when: is_local - name: Ensure mysql can write data. ansible.builtin.file: diff --git a/roles/debian/mysql_server_oracle_ce/tasks/main.yml b/roles/debian/mysql_server_oracle_ce/tasks/main.yml index b427af88c..f42b70f5b 100644 --- a/roles/debian/mysql_server_oracle_ce/tasks/main.yml +++ b/roles/debian/mysql_server_oracle_ce/tasks/main.yml @@ -77,12 +77,12 @@ - name: Update MySQL root password. ansible.builtin.shell: > mysql -NBe 'GRANT ALL ON *.* TO "root"@"%" IDENTIFIED BY "root" WITH GRANT OPTION;' - when: is_local is defined and is_local + when: is_local - name: Update MySQL ce-dev password. ansible.builtin.shell: > mysql -NBe 'GRANT ALL ON *.* TO "ce-dev"@"%" IDENTIFIED BY "ce-dev" WITH GRANT OPTION;' - when: is_local is defined and is_local + when: is_local - name: Ensure MySQL can write data. ansible.builtin.file: diff --git a/roles/debian/nginx/templates/drupal10.j2 b/roles/debian/nginx/templates/drupal10.j2 index fee75d6c5..a4dfa5449 100644 --- a/roles/debian/nginx/templates/drupal10.j2 +++ b/roles/debian/nginx/templates/drupal10.j2 @@ -81,7 +81,7 @@ location ~ ^/(index|cron|modules\/statistics\/statistics|core\/modules\/statisti try_files @phpprocess @phpprocess; } # Allow install/update for local stack. -{% if is_local is defined and is_local %} +{% if is_local %} location ~ ^/(install|update)\.php$ { try_files @phpprocess @phpprocess; } diff --git a/roles/debian/nginx/templates/drupal_common.j2 b/roles/debian/nginx/templates/drupal_common.j2 index 770203509..3e406dd6d 100644 --- a/roles/debian/nginx/templates/drupal_common.j2 +++ b/roles/debian/nginx/templates/drupal_common.j2 @@ -72,7 +72,7 @@ location ~ ^/(index|cron|modules\/statistics\/statistics|core\/modules\/statisti try_files @phpprocess @phpprocess; } # Allow install/update for local stack. -{% if is_local is defined and is_local %} +{% if is_local %} location ~ ^/(install|update)\.php$ { try_files @phpprocess @phpprocess; } diff --git a/roles/debian/nginx/templates/mautic.j2 b/roles/debian/nginx/templates/mautic.j2 index 92845e808..76de23a5c 100644 --- a/roles/debian/nginx/templates/mautic.j2 +++ b/roles/debian/nginx/templates/mautic.j2 @@ -60,7 +60,7 @@ location ~ /(addons|plugins)/.*/Assets/ { } # Allow index_dev/upgrade for local stack. -{% if is_local is defined and is_local %} +{% if is_local %} location ~ ^/(index_dev|upgrade)\.php$ { try_files @phpprocess @phpprocess; } diff --git a/roles/debian/php-fpm/README.md b/roles/debian/php-fpm/README.md index 063e97276..443ffbf87 100644 --- a/roles/debian/php-fpm/README.md +++ b/roles/debian/php-fpm/README.md @@ -23,7 +23,7 @@ php: # It is important to scale up processes on bigger servers, so that more # requests can be handled. Double the number of vCPUs is a good default. # Can be between 5 and 64. - max_children: "{{ [5, [ansible_facts.ansible_processor_nproc * 2, 64] | min] | max }}" + max_children: "{{ [5, [(ansible_facts.ansible_processor_nproc | default(1)) * 2, 64] | min] | max }}" # Fallback in case ansible_processor_nproc is not gathered before tasks start_servers: 2 min_spare_servers: 1 max_spare_servers: 3 diff --git a/roles/debian/postfix/tasks/main.yml b/roles/debian/postfix/tasks/main.yml index 83c7ecec6..94d433c28 100644 --- a/roles/debian/postfix/tasks/main.yml +++ b/roles/debian/postfix/tasks/main.yml @@ -99,7 +99,6 @@ state: directory when: - postfix.ce_dev_delivery_mode == "host" - - is_local is defined - is_local - name: Configure procmail to NULL. @@ -110,9 +109,7 @@ group: root mode: "0644" force: true - when: - - is_local is defined - - is_local + when: is_local - name: Configure procmail to host directory. ansible.builtin.template: @@ -124,7 +121,6 @@ force: true when: - postfix.ce_dev_delivery_mode == "host" - - is_local is defined - is_local - name: Configure procmail to local. @@ -133,7 +129,6 @@ state: absent when: - postfix.ce_dev_delivery_mode == "local" - - is_local is defined - is_local # Needed for Docker. diff --git a/roles/debian/postfix/templates/main.cf.j2 b/roles/debian/postfix/templates/main.cf.j2 index e79389db4..6addd3c23 100644 --- a/roles/debian/postfix/templates/main.cf.j2 +++ b/roles/debian/postfix/templates/main.cf.j2 @@ -52,7 +52,7 @@ smtpd_tls_key_file = {{ postfix.ssl.smtp_tls_key_file }} smtpd_tls_CApath = {{ postfix.ssl.smtp_tls_CApath }} smtpd_tls_CAfile = {{ postfix.ssl.smtp_tls_CAfile }} {% endif %} -{% if (is_local is defined) and is_local %} +{% if is_local %} # Force all mail to ce-dev user. virtual_alias_domains = "" virtual_alias_maps = static:ce-dev diff --git a/roles/debian/user_deploy/defaults/main.yml b/roles/debian/user_deploy/defaults/main.yml index dc22f8164..1d83a8cd7 100644 --- a/roles/debian/user_deploy/defaults/main.yml +++ b/roles/debian/user_deploy/defaults/main.yml @@ -1,5 +1,5 @@ --- -_user_deploy_username: "{% if is_local is defined and is_local %}ce-dev{% else %}deploy{% endif %}" +_user_deploy_username: "{% if is_local %}ce-dev{% else %}deploy{% endif %}" user_deploy: # This sets both username and main group. # If you are using ce-deploy to deploy code this must match the `deploy_user` variable diff --git a/roles/debian/user_provision/defaults/main.yml b/roles/debian/user_provision/defaults/main.yml index 7d07a8cc2..f10896c32 100644 --- a/roles/debian/user_provision/defaults/main.yml +++ b/roles/debian/user_provision/defaults/main.yml @@ -1,5 +1,5 @@ --- -_user_provision_username: "{% if is_local is defined and is_local %}ce-dev{% else %}controller{% endif %}" +_user_provision_username: "{% if is_local %}ce-dev{% else %}controller{% endif %}" user_provision: # This sets both username and main group. username: "{{ _user_provision_username }}" From 9a67b87d63c1573e92e383d95b5447041d45d512 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 10:55:59 +0100 Subject: [PATCH 169/206] Letting GitLab know it's on Docker earlier. --- roles/debian/gitlab/tasks/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/debian/gitlab/tasks/main.yml b/roles/debian/gitlab/tasks/main.yml index 3d4a962c7..b8f1a475c 100644 --- a/roles/debian/gitlab/tasks/main.yml +++ b/roles/debian/gitlab/tasks/main.yml @@ -87,12 +87,6 @@ src: gitlab-config.rb.j2 dest: /etc/gitlab/gitlab-config.rb -- name: Stop Gitlab. - ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl stop - -- name: Reconfigure Gitlab. - ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl reconfigure - - name: Let Gitlab know it's on Docker. ansible.builtin.copy: content: "gitlab-docker" @@ -100,6 +94,12 @@ mode: "0666" when: is_local +- name: Stop Gitlab. + ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl stop + +- name: Reconfigure Gitlab. + ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl reconfigure + - name: Copy startup script in place. ansible.builtin.template: src: docker-init.sh.j2 From a66eaccb462348dbf383c8062fdcf093c9542101 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 11:29:50 +0100 Subject: [PATCH 170/206] Trying to run runsvdir-start to avoid container freezing. --- roles/debian/gitlab/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/debian/gitlab/tasks/main.yml b/roles/debian/gitlab/tasks/main.yml index b8f1a475c..0345a06e7 100644 --- a/roles/debian/gitlab/tasks/main.yml +++ b/roles/debian/gitlab/tasks/main.yml @@ -97,6 +97,10 @@ - name: Stop Gitlab. ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl stop +- name: Start runit properly to avoid freezes in containers. + ansible.builtin.command: /opt/gitlab/embedded/bin/runsvdir-start + when: is_local + - name: Reconfigure Gitlab. ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl reconfigure From 94d118a629529f33d4e8e6308e8512c927a9a7be Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 11:50:31 +0100 Subject: [PATCH 171/206] Temporarily skipping reconfigure of GitLab to test the rest. --- roles/debian/gitlab/tasks/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/debian/gitlab/tasks/main.yml b/roles/debian/gitlab/tasks/main.yml index 0345a06e7..d7f2c6d83 100644 --- a/roles/debian/gitlab/tasks/main.yml +++ b/roles/debian/gitlab/tasks/main.yml @@ -97,12 +97,12 @@ - name: Stop Gitlab. ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl stop -- name: Start runit properly to avoid freezes in containers. - ansible.builtin.command: /opt/gitlab/embedded/bin/runsvdir-start - when: is_local +#- name: Start runit properly to avoid freezes in containers. +# ansible.builtin.command: /opt/gitlab/embedded/bin/runsvdir-start +# when: is_local -- name: Reconfigure Gitlab. - ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl reconfigure +#- name: Reconfigure Gitlab. +# ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl reconfigure - name: Copy startup script in place. ansible.builtin.template: From e570972f3978edab20bd9ff6b17e1a25af79214e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 12:14:05 +0100 Subject: [PATCH 172/206] Trying to move GitLab reconfigure commands to CI. --- .github/workflows/ce-provision-test-gitlab.yml | 14 ++++++++++---- .github/workflows/ce-provision-test-web.yml | 5 +---- roles/debian/gitlab/tasks/main.yml | 4 ++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ce-provision-test-gitlab.yml b/.github/workflows/ce-provision-test-gitlab.yml index 3fffbf263..2f626db1d 100644 --- a/.github/workflows/ce-provision-test-gitlab.yml +++ b/.github/workflows/ce-provision-test-gitlab.yml @@ -10,6 +10,7 @@ jobs: # Set the job key. The key is displayed as the job name # when a job name is not provided test-gitlab: + if: ${{ github.event.pull_request.head.ref != 'documentation' }} # Name the Job name: Build a GitLab server with ce-provision # Set the type of machine to run on @@ -23,7 +24,6 @@ jobs: steps: - name: Install ce-provision - if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | /usr/bin/curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh /usr/bin/chmod +x ./install.sh @@ -31,15 +31,21 @@ jobs: # Run a GitLab server provision - name: Prepare Git repos on disk - if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | /usr/bin/git config --global --add safe.directory /home/controller/ce-provision /usr/bin/git config --global --add safe.directory /home/controller/ce-provision/config - name: Start SSHD - if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: /usr/sbin/sshd& - name: Provision a test GitLab server - if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: /usr/bin/su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/gitlab/ci.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" + + - name: Start runit + run: /opt/gitlab/embedded/bin/runsvdir-start + + - name: Reconfigure GitLab + run: /opt/gitlab/bin/gitlab-ctl reconfigure + + - name: Run configuration script + run: /opt/gitlab/bin/gitlab-rails runner /etc/gitlab/gitlab-config.rb diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml index ab39b9e29..54421cd40 100644 --- a/.github/workflows/ce-provision-test-web.yml +++ b/.github/workflows/ce-provision-test-web.yml @@ -10,6 +10,7 @@ jobs: # Set the job key. The key is displayed as the job name # when a job name is not provided test-web: + if: ${{ github.event.pull_request.head.ref != 'documentation' }} # Name the Job name: Build a web server with ce-provision # Set the type of machine to run on @@ -23,7 +24,6 @@ jobs: steps: - name: Install ce-provision - if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | /usr/bin/curl -LO https://raw.githubusercontent.com/codeenigma/ce-provision/${{ github.event.pull_request.head.ref }}/install.sh /usr/bin/chmod +x ./install.sh @@ -31,15 +31,12 @@ jobs: # Run a web server provision - name: Prepare Git repos on disk - if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | /usr/bin/git config --global --add safe.directory /home/controller/ce-provision /usr/bin/git config --global --add safe.directory /home/controller/ce-provision/config - name: Start SSHD - if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: /usr/sbin/sshd& - name: Provision a test web server - if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: /usr/bin/su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/web/ci.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" diff --git a/roles/debian/gitlab/tasks/main.yml b/roles/debian/gitlab/tasks/main.yml index d7f2c6d83..059002b3f 100644 --- a/roles/debian/gitlab/tasks/main.yml +++ b/roles/debian/gitlab/tasks/main.yml @@ -130,5 +130,5 @@ - name: Ensure GitLab is started. ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl start -- name: Run the GitLab configuration script for config that cannot be set in gitlab.rb. - ansible.builtin.command: "gitlab-rails runner /etc/gitlab/gitlab-config.rb" +#- name: Run the GitLab configuration script for config that cannot be set in gitlab.rb. +# ansible.builtin.command: "gitlab-rails runner /etc/gitlab/gitlab-config.rb" From 8b101e6e169b9c3da3c4a1e0f913710111bb135c Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 12:28:25 +0100 Subject: [PATCH 173/206] Fixing service namespace for runner and reinstating GitLab tasks. --- roles/debian/gitlab/tasks/main.yml | 14 ++++++-------- roles/debian/gitlab_runner/tasks/main.yml | 2 +- roles/debian/jitsi/tasks/main.yml | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/roles/debian/gitlab/tasks/main.yml b/roles/debian/gitlab/tasks/main.yml index 059002b3f..39bc9781c 100644 --- a/roles/debian/gitlab/tasks/main.yml +++ b/roles/debian/gitlab/tasks/main.yml @@ -97,12 +97,9 @@ - name: Stop Gitlab. ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl stop -#- name: Start runit properly to avoid freezes in containers. -# ansible.builtin.command: /opt/gitlab/embedded/bin/runsvdir-start -# when: is_local - -#- name: Reconfigure Gitlab. -# ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl reconfigure +- name: Reconfigure Gitlab. + ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl reconfigure + when: not is_local - name: Copy startup script in place. ansible.builtin.template: @@ -129,6 +126,7 @@ - name: Ensure GitLab is started. ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl start + when: not is_local -#- name: Run the GitLab configuration script for config that cannot be set in gitlab.rb. -# ansible.builtin.command: "gitlab-rails runner /etc/gitlab/gitlab-config.rb" +- name: Run the GitLab configuration script for config that cannot be set in gitlab.rb. + ansible.builtin.command: "gitlab-rails runner /etc/gitlab/gitlab-config.rb" diff --git a/roles/debian/gitlab_runner/tasks/main.yml b/roles/debian/gitlab_runner/tasks/main.yml index 745daadc9..017dd4afc 100644 --- a/roles/debian/gitlab_runner/tasks/main.yml +++ b/roles/debian/gitlab_runner/tasks/main.yml @@ -138,7 +138,7 @@ become: true - name: Restart gitlab-runner service. - ansible.builtin.service: + ansible.builtin.systemd_service: name: gitlab-runner state: restarted daemon_reload: true diff --git a/roles/debian/jitsi/tasks/main.yml b/roles/debian/jitsi/tasks/main.yml index ec485f6ba..cef38260b 100644 --- a/roles/debian/jitsi/tasks/main.yml +++ b/roles/debian/jitsi/tasks/main.yml @@ -27,7 +27,7 @@ DefaultTasksMax=65000 - name: Force systemd to reread configs. - ansible.builtin.systemd: + ansible.builtin.systemd_service: daemon_reload: true - name: Write interactive hostname value for automated installation. From 3f2f0788fdada0fd9267450721887afcabed88d9 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 12:46:50 +0100 Subject: [PATCH 174/206] Trying to get config script working for GitLab in CI. --- .github/workflows/ce-provision-test-gitlab.yml | 6 ------ roles/debian/gitlab/tasks/main.yml | 5 +++-- roles/debian/gitlab/templates/gitlab-config.rb.j2 | 4 ---- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ce-provision-test-gitlab.yml b/.github/workflows/ce-provision-test-gitlab.yml index 2f626db1d..87d242578 100644 --- a/.github/workflows/ce-provision-test-gitlab.yml +++ b/.github/workflows/ce-provision-test-gitlab.yml @@ -41,11 +41,5 @@ jobs: - name: Provision a test GitLab server run: /usr/bin/su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/gitlab/ci.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" - - name: Start runit - run: /opt/gitlab/embedded/bin/runsvdir-start - - - name: Reconfigure GitLab - run: /opt/gitlab/bin/gitlab-ctl reconfigure - - name: Run configuration script run: /opt/gitlab/bin/gitlab-rails runner /etc/gitlab/gitlab-config.rb diff --git a/roles/debian/gitlab/tasks/main.yml b/roles/debian/gitlab/tasks/main.yml index 39bc9781c..8dabf7d2f 100644 --- a/roles/debian/gitlab/tasks/main.yml +++ b/roles/debian/gitlab/tasks/main.yml @@ -126,7 +126,8 @@ - name: Ensure GitLab is started. ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl start - when: not is_local +# @TODO - this task fails in CI with - name: Run the GitLab configuration script for config that cannot be set in gitlab.rb. - ansible.builtin.command: "gitlab-rails runner /etc/gitlab/gitlab-config.rb" + ansible.builtin.command: /opt/gitlab/bin/gitlab-rails runner /etc/gitlab/gitlab-config.rb + when: not is_local diff --git a/roles/debian/gitlab/templates/gitlab-config.rb.j2 b/roles/debian/gitlab/templates/gitlab-config.rb.j2 index d3e1690f1..4a0619be9 100644 --- a/roles/debian/gitlab/templates/gitlab-config.rb.j2 +++ b/roles/debian/gitlab/templates/gitlab-config.rb.j2 @@ -1,9 +1,6 @@ # Disable DSA keys ApplicationSetting.last.update(dsa_key_restriction: -1) -# Disable ECDSA keys -ApplicationSetting.last.update(ecdsa_key_restriction: -1) - # Enforce at least 2048 bits for RSA keys ApplicationSetting.last.update(rsa_key_restriction: 2048) @@ -24,4 +21,3 @@ ApplicationSetting.last.update(signup_enabled: false) # Disable standard sign-in dialogue Gitlab::CurrentSettings.update!(password_authentication_enabled_for_web: false) {% endif %} - From 39e7e70d158a7187ad3174175703a92f28272245 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 13:02:59 +0100 Subject: [PATCH 175/206] No systemd, do not try to restart gitlab-runner. --- ce-dev/ansible/vars/gitlab/gitlab_runner.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 ce-dev/ansible/vars/gitlab/gitlab_runner.yml diff --git a/ce-dev/ansible/vars/gitlab/gitlab_runner.yml b/ce-dev/ansible/vars/gitlab/gitlab_runner.yml new file mode 100644 index 000000000..721b65ab0 --- /dev/null +++ b/ce-dev/ansible/vars/gitlab/gitlab_runner.yml @@ -0,0 +1,2 @@ +gitlab_runner: + restart: false # no systemd in CI containers From f3842a16c26f10a89cfc8b44c2ddb840f1944e3e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 13:14:27 +0100 Subject: [PATCH 176/206] Removing firewall role from CI GitLab test, don't need it and it breaks CI. --- ce-dev/ansible/plays/gitlab/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ce-dev/ansible/plays/gitlab/ci.yml b/ce-dev/ansible/plays/gitlab/ci.yml index 1ad4a2561..eb61f611f 100644 --- a/ce-dev/ansible/plays/gitlab/ci.yml +++ b/ce-dev/ansible/plays/gitlab/ci.yml @@ -33,7 +33,5 @@ name: debian/sops - ansible.builtin.import_role: name: debian/gpg_key - - ansible.builtin.import_role: - name: debian/firewall_config - ansible.builtin.import_role: name: _exit From bea012f52f84cb05185172309d64e19606c58dec Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 13:31:31 +0100 Subject: [PATCH 177/206] Outputting PostGreSQL logs to see if there are errors. --- .github/workflows/ce-provision-test-gitlab.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-gitlab.yml b/.github/workflows/ce-provision-test-gitlab.yml index 87d242578..189a69407 100644 --- a/.github/workflows/ce-provision-test-gitlab.yml +++ b/.github/workflows/ce-provision-test-gitlab.yml @@ -42,4 +42,6 @@ jobs: run: /usr/bin/su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/gitlab/ci.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" - name: Run configuration script - run: /opt/gitlab/bin/gitlab-rails runner /etc/gitlab/gitlab-config.rb + run: | + /usr/bin/cat /var/log/gitlab/postgresql/current + /opt/gitlab/bin/gitlab-rails runner /etc/gitlab/gitlab-config.rb From 9bafd07524627ccf841a6ceaeeb990f2a9ca0c20 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 13:32:59 +0100 Subject: [PATCH 178/206] Outputting PostGreSQL logs to see if there are errors. --- .github/workflows/ce-provision-test-gitlab.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-test-gitlab.yml b/.github/workflows/ce-provision-test-gitlab.yml index 189a69407..d557e395d 100644 --- a/.github/workflows/ce-provision-test-gitlab.yml +++ b/.github/workflows/ce-provision-test-gitlab.yml @@ -44,4 +44,4 @@ jobs: - name: Run configuration script run: | /usr/bin/cat /var/log/gitlab/postgresql/current - /opt/gitlab/bin/gitlab-rails runner /etc/gitlab/gitlab-config.rb + #/opt/gitlab/bin/gitlab-rails runner /etc/gitlab/gitlab-config.rb From dd7e685f58451b4ea167a578be0f42b1b269b3bb Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 13:45:02 +0100 Subject: [PATCH 179/206] Trying the config script for GitLab again. --- .github/workflows/ce-provision-test-gitlab.yml | 5 ----- roles/debian/gitlab/tasks/main.yml | 1 - 2 files changed, 6 deletions(-) diff --git a/.github/workflows/ce-provision-test-gitlab.yml b/.github/workflows/ce-provision-test-gitlab.yml index d557e395d..bd50e7233 100644 --- a/.github/workflows/ce-provision-test-gitlab.yml +++ b/.github/workflows/ce-provision-test-gitlab.yml @@ -40,8 +40,3 @@ jobs: - name: Provision a test GitLab server run: /usr/bin/su - controller -c "cd /home/controller/ce-provision && /bin/sh /home/controller/ce-provision/scripts/provision.sh --python-interpreter /home/controller/ce-python/bin/python3 --repo dummy --branch dummy --workspace /home/controller/ce-provision/ce-dev/ansible --playbook plays/gitlab/ci.yml --own-branch ${{ github.event.pull_request.head.ref }} --config-branch ${{ github.event.pull_request.base.ref }} --force" - - - name: Run configuration script - run: | - /usr/bin/cat /var/log/gitlab/postgresql/current - #/opt/gitlab/bin/gitlab-rails runner /etc/gitlab/gitlab-config.rb diff --git a/roles/debian/gitlab/tasks/main.yml b/roles/debian/gitlab/tasks/main.yml index 8dabf7d2f..da0461f84 100644 --- a/roles/debian/gitlab/tasks/main.yml +++ b/roles/debian/gitlab/tasks/main.yml @@ -130,4 +130,3 @@ # @TODO - this task fails in CI with - name: Run the GitLab configuration script for config that cannot be set in gitlab.rb. ansible.builtin.command: /opt/gitlab/bin/gitlab-rails runner /etc/gitlab/gitlab-config.rb - when: not is_local From bfdefea7489b313ea9d550c19460cfa9deb28d1f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 13:54:23 +0100 Subject: [PATCH 180/206] Suppressing extra GitLab config for CI runs. --- roles/debian/gitlab/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/debian/gitlab/tasks/main.yml b/roles/debian/gitlab/tasks/main.yml index da0461f84..00b55e162 100644 --- a/roles/debian/gitlab/tasks/main.yml +++ b/roles/debian/gitlab/tasks/main.yml @@ -127,6 +127,7 @@ - name: Ensure GitLab is started. ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl start -# @TODO - this task fails in CI with +# @TODO - this task fails in CI with GitHub Actions because PostGreSQL isn't running - name: Run the GitLab configuration script for config that cannot be set in gitlab.rb. ansible.builtin.command: /opt/gitlab/bin/gitlab-rails runner /etc/gitlab/gitlab-config.rb + when: not is_local From e1dc0ac5f90bfdcfa0441b2c2403dc60860de3a8 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 18 Dec 2024 13:59:10 +0100 Subject: [PATCH 181/206] Setting Blackfire CLI defaults to use ce-dev user. --- roles/debian/php_blackfire/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/debian/php_blackfire/defaults/main.yml b/roles/debian/php_blackfire/defaults/main.yml index 1fc9c0bba..0e0e5c8a9 100644 --- a/roles/debian/php_blackfire/defaults/main.yml +++ b/roles/debian/php_blackfire/defaults/main.yml @@ -6,5 +6,5 @@ blackfire: agent_server_token: "" agent_client_id: "" agent_client_token: "" - agent_cli_config_path: /home/controller/.blackfire.ini - agent_cli_config_owner: controller + agent_cli_config_path: /home/ce-dev/.blackfire.ini + agent_cli_config_owner: ce-dev From 7133d726c375978d9d555247fa25460e8ba7ed07 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 14 Jan 2025 19:12:19 +0100 Subject: [PATCH 182/206] Improving GitLab vars and adding force stop feature. --- roles/debian/gitlab/defaults/main.yml | 11 ++++++----- roles/debian/gitlab/tasks/main.yml | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/roles/debian/gitlab/defaults/main.yml b/roles/debian/gitlab/defaults/main.yml index e2ed2ff05..ba7d1b4e5 100644 --- a/roles/debian/gitlab/defaults/main.yml +++ b/roles/debian/gitlab/defaults/main.yml @@ -11,14 +11,15 @@ gitlab: apt_origin: "origin=packages.gitlab.com/gitlab/gitlab-ce,codename=${distro_codename},label=gitlab-ce" # used by apt_unattended_upgrades apt_signed_by: https://packages.gitlab.com/gitlab/gitlab-ce/gpgkey server_name: "gitlab.{{ _domain_name }}" + force_stop: true # whether to stop GitLab to reconfigure or not # Add a record for GitLab in AWS Route 53 # If you use the aws_ec2_with_eip role to create your server this will not be necessary gitlab_route_53: state: present zone: "" # empty zone skips DNS creation - record: "{{ _domain_name }}" - type: A # change to CNAME if required - value: 1.2.3.4 # set IP if type: A and target hostname if type: CNAME + record: "gitlab.{{ _domain_name }}" + type: CNAME # change to A if required + value: "{{ _domain_name }}" # set IP if type: A and target hostname if type: CNAME aws_profile: another # Not necessarily the same as the "target" one for the server wildcard: true # Creates a matching wildcard CNAME letsencrypt: "true" # use built-in GitLab LetsEncrypt support by default letsencrypt: "true" # GitLab's built in SSL handling enabled by default @@ -77,10 +78,10 @@ gitlab: omniauth_auto_link_saml_user: "false" omniauth_block_auto_created_users: "true" omniauth_login_button_label: "Login with SAML" - omniauth_consumer_service_url: "https://{{ _domain_name }}/users/auth/saml/callback" + omniauth_consumer_service_url: "https://gitlab.{{ _domain_name }}/users/auth/saml/callback" omniauth_saml_cert_fingerprint: "00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00" # fingerprint of the SAML server's certificate omniauth_saml_endpoint_url: https://login.example.com/simplesaml/saml2/idp/SSOService.php # typical endpoint if you followed the SimpleSAMLphp QuickStart - https://simplesamlphp.org/docs/stable/simplesamlphp-idp.html - omniauth_saml_entity_id: "{{ _domain_name }}" # can be any string, typically just the domain name + omniauth_saml_entity_id: "gitlab.{{ _domain_name }}" # can be any string, typically just the domain name omniauth_saml_attribute_statements: "uid: ['uid']" # typical basic set-up if your SAML authsource is OpenLDAP # Other services prometheus: "true" # enable/disable built-in Prometheus diff --git a/roles/debian/gitlab/tasks/main.yml b/roles/debian/gitlab/tasks/main.yml index 00b55e162..766ec8097 100644 --- a/roles/debian/gitlab/tasks/main.yml +++ b/roles/debian/gitlab/tasks/main.yml @@ -96,6 +96,7 @@ - name: Stop Gitlab. ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl stop + when: gitlab.force_stop - name: Reconfigure Gitlab. ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl reconfigure @@ -126,6 +127,7 @@ - name: Ensure GitLab is started. ansible.builtin.command: /opt/gitlab/bin/gitlab-ctl start + when: gitlab.force_stop # @TODO - this task fails in CI with GitHub Actions because PostGreSQL isn't running - name: Run the GitLab configuration script for config that cannot be set in gitlab.rb. From fcbad7e39948d5bc914136c3792df5e7fff24f77 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 14 Jan 2025 19:12:41 +0100 Subject: [PATCH 183/206] Not installed aws_credentials in meta roles if AWS support disabled. --- roles/_meta/controller/meta/main.yml | 2 +- roles/_meta/deploy/meta/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/_meta/controller/meta/main.yml b/roles/_meta/controller/meta/main.yml index 955a23613..5eb04377e 100644 --- a/roles/_meta/controller/meta/main.yml +++ b/roles/_meta/controller/meta/main.yml @@ -3,7 +3,7 @@ dependencies: - role: debian/user_provision - { role: debian/ssh_server, when: ( is_local is not defined or not is_local ) } - role: _meta/common_base - - role: aws/aws_credentials + - { role: aws/aws_credentials, when ce_provision.aws_support } - role: debian/ce_provision - role: debian/gitlab - role: debian/gitlab_runner diff --git a/roles/_meta/deploy/meta/main.yml b/roles/_meta/deploy/meta/main.yml index 4409e5701..9747fd195 100644 --- a/roles/_meta/deploy/meta/main.yml +++ b/roles/_meta/deploy/meta/main.yml @@ -4,7 +4,7 @@ dependencies: - { role: debian/ssh_server, when: ( is_local is not defined or not is_local ) } - role: _meta/common_base - role: debian/ce_deploy - - role: aws/aws_credentials + - { role: aws/aws_credentials, when: ce_deploy.aws_support } - role: debian/gitlab - role: debian/gitlab_runner - role: debian/sops From a701c4a6a981271838e6f681cc6e8870ff7b4a89 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 14 Jan 2025 19:39:49 +0100 Subject: [PATCH 184/206] Most people will not want pam_ldap or pam_linotp, should not be in meta. --- roles/_meta/controller/meta/main.yml | 2 -- roles/_meta/deploy/meta/main.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/roles/_meta/controller/meta/main.yml b/roles/_meta/controller/meta/main.yml index 5eb04377e..a876c8c89 100644 --- a/roles/_meta/controller/meta/main.yml +++ b/roles/_meta/controller/meta/main.yml @@ -9,5 +9,3 @@ dependencies: - role: debian/gitlab_runner - role: debian/sops - role: debian/gpg_key - - role: debian/pam_ldap - - role: debian/pam_linotp diff --git a/roles/_meta/deploy/meta/main.yml b/roles/_meta/deploy/meta/main.yml index 9747fd195..cc6ce64c2 100644 --- a/roles/_meta/deploy/meta/main.yml +++ b/roles/_meta/deploy/meta/main.yml @@ -9,5 +9,3 @@ dependencies: - role: debian/gitlab_runner - role: debian/sops - role: debian/gpg_key - - role: debian/pam_ldap - - role: debian/pam_linotp From 5cdaf0fd36c7dccf6a52479112c39705ebdb203f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 14 Jan 2025 19:41:32 +0100 Subject: [PATCH 185/206] If you don't create LDAP SSL certs you might not have a /etc/ldap directory. --- roles/debian/pam_ldap/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/debian/pam_ldap/tasks/main.yml b/roles/debian/pam_ldap/tasks/main.yml index c66ba85f7..b2a4de250 100644 --- a/roles/debian/pam_ldap/tasks/main.yml +++ b/roles/debian/pam_ldap/tasks/main.yml @@ -5,6 +5,11 @@ state: present update_cache: true +- name: Create LDAP config directory. + ansible.builtin.file: + path: /etc/ldap + state: directory + - name: Create certificate directory. ansible.builtin.file: path: /etc/ldap/ssl From 253330aec7f2efd485764e3ca9f58e8f64127ae7 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Jan 2025 10:52:44 +0100 Subject: [PATCH 186/206] Adding a merge of the main branch into the docs branch to CI. --- .github/workflows/ce-provision-publish-docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ce-provision-publish-docs.yml b/.github/workflows/ce-provision-publish-docs.yml index b87f9afe5..89a446756 100644 --- a/.github/workflows/ce-provision-publish-docs.yml +++ b/.github/workflows/ce-provision-publish-docs.yml @@ -58,6 +58,7 @@ jobs: # First build and publish the markdown docs - name: Build and commit table of contents and README files back to the repo run: | + /usr/bin/git merge origin/${{ github.event.pull_request.base.ref }} /bin/sh contribute/toc.sh /usr/bin/find . -name "*.md" | xargs git add /usr/bin/git diff --staged --quiet || /usr/bin/git commit -am "GitHub Actions - updating markdown docs - ${{ github.event.repository.updated_at }}" From 9e417831fd9e10bb3f86ec710aa1ac61395cd75c Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Jan 2025 12:05:14 +0100 Subject: [PATCH 187/206] Adding comment to not use hyphens in boto profile names. --- roles/aws/aws_credentials/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/aws/aws_credentials/defaults/main.yml b/roles/aws/aws_credentials/defaults/main.yml index 8e58eb78d..80a5430fe 100644 --- a/roles/aws/aws_credentials/defaults/main.yml +++ b/roles/aws/aws_credentials/defaults/main.yml @@ -2,7 +2,7 @@ aws_credentials: - user: ce-dev profiles: - - name: profile1 + - name: profile1 # profiles should never contain hyphens access_key_id: XXX secret_access_key: XXXX - name: example From e091be91dc0ce2ee5a269f577c4ab03985ad0daa Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Jan 2025 12:22:18 +0100 Subject: [PATCH 188/206] Supporting different key types to publish to AWS. --- roles/aws/aws_provision_ec2_keypair/defaults/main.yml | 1 + roles/aws/aws_provision_ec2_keypair/tasks/main.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/aws/aws_provision_ec2_keypair/defaults/main.yml b/roles/aws/aws_provision_ec2_keypair/defaults/main.yml index 51e053f91..62300898b 100644 --- a/roles/aws/aws_provision_ec2_keypair/defaults/main.yml +++ b/roles/aws/aws_provision_ec2_keypair/defaults/main.yml @@ -3,3 +3,4 @@ aws_provision_ec2_keypair: aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" key_name: "{{ ce_provision.username }}@{{ ansible_hostname }}" + key_type: "{{ ce_provision.ssh_key_type }}" # defaults to ed25519 in the ce_provision role, set to rsa to or ecdsa as necessary diff --git a/roles/aws/aws_provision_ec2_keypair/tasks/main.yml b/roles/aws/aws_provision_ec2_keypair/tasks/main.yml index a0c5124d7..81de64d72 100644 --- a/roles/aws/aws_provision_ec2_keypair/tasks/main.yml +++ b/roles/aws/aws_provision_ec2_keypair/tasks/main.yml @@ -2,6 +2,6 @@ - name: Create EC2 key pair. amazon.aws.ec2_key: name: "{{ aws_provision_ec2_keypair.key_name }}" - key_material: "{{ lookup('file', '/home/{{ ce_provision.username }}/.ssh/id_rsa.pub') }}" + key_material: "{{ lookup('file', '/home/{{ ce_provision.username }}/.ssh/id_{{ aws_provision_ec2_keypair.key_type }}.pub') }}" profile: "{{ aws_provision_ec2_keypair.aws_profile }}" region: "{{ aws_provision_ec2_keypair.region }}" From 9319fac7d632d3a02e2c54b195a0596eb0ffe106 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Jan 2025 12:29:50 +0100 Subject: [PATCH 189/206] Adding a git fetch before the merge in docs publishing. --- .github/workflows/ce-provision-publish-docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ce-provision-publish-docs.yml b/.github/workflows/ce-provision-publish-docs.yml index 89a446756..f2c56fd7b 100644 --- a/.github/workflows/ce-provision-publish-docs.yml +++ b/.github/workflows/ce-provision-publish-docs.yml @@ -58,6 +58,7 @@ jobs: # First build and publish the markdown docs - name: Build and commit table of contents and README files back to the repo run: | + /usr/bin/git fetch origin 2.x /usr/bin/git merge origin/${{ github.event.pull_request.base.ref }} /bin/sh contribute/toc.sh /usr/bin/find . -name "*.md" | xargs git add From 11db615f43a2ef0e3bc15c96635377f6233e928f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Jan 2025 13:05:29 +0100 Subject: [PATCH 190/206] Using the safer _ce_provision_username var in AWS key role. --- roles/aws/aws_provision_ec2_keypair/defaults/main.yml | 4 ++-- roles/aws/aws_provision_ec2_keypair/tasks/main.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/aws/aws_provision_ec2_keypair/defaults/main.yml b/roles/aws/aws_provision_ec2_keypair/defaults/main.yml index 62300898b..efd4af9c0 100644 --- a/roles/aws/aws_provision_ec2_keypair/defaults/main.yml +++ b/roles/aws/aws_provision_ec2_keypair/defaults/main.yml @@ -2,5 +2,5 @@ aws_provision_ec2_keypair: aws_profile: "{{ _aws_profile }}" region: "{{ _aws_region }}" - key_name: "{{ ce_provision.username }}@{{ ansible_hostname }}" - key_type: "{{ ce_provision.ssh_key_type }}" # defaults to ed25519 in the ce_provision role, set to rsa to or ecdsa as necessary + key_name: "{{ _ce_provision_username }}@{{ ansible_hostname }}" + key_type: ed25519 # defaults to ed25519 as used in the ce_provision role, set to rsa to or ecdsa as necessary diff --git a/roles/aws/aws_provision_ec2_keypair/tasks/main.yml b/roles/aws/aws_provision_ec2_keypair/tasks/main.yml index 81de64d72..4eb8f2c98 100644 --- a/roles/aws/aws_provision_ec2_keypair/tasks/main.yml +++ b/roles/aws/aws_provision_ec2_keypair/tasks/main.yml @@ -2,6 +2,6 @@ - name: Create EC2 key pair. amazon.aws.ec2_key: name: "{{ aws_provision_ec2_keypair.key_name }}" - key_material: "{{ lookup('file', '/home/{{ ce_provision.username }}/.ssh/id_{{ aws_provision_ec2_keypair.key_type }}.pub') }}" + key_material: "{{ lookup('file', '/home/{{ _ce_provision_username }}/.ssh/id_{{ aws_provision_ec2_keypair.key_type }}.pub') }}" profile: "{{ aws_provision_ec2_keypair.aws_profile }}" region: "{{ aws_provision_ec2_keypair.region }}" From a6ec32f45339c38b7665962f59b1cda512a03d0d Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Jan 2025 13:16:47 +0100 Subject: [PATCH 191/206] Adding the --allow-unrelated-histories flag to git merge in CI. --- .github/workflows/ce-provision-publish-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-publish-docs.yml b/.github/workflows/ce-provision-publish-docs.yml index f2c56fd7b..232a71d99 100644 --- a/.github/workflows/ce-provision-publish-docs.yml +++ b/.github/workflows/ce-provision-publish-docs.yml @@ -59,7 +59,7 @@ jobs: - name: Build and commit table of contents and README files back to the repo run: | /usr/bin/git fetch origin 2.x - /usr/bin/git merge origin/${{ github.event.pull_request.base.ref }} + /usr/bin/git merge origin/${{ github.event.pull_request.base.ref }} --allow-unrelated-histories /bin/sh contribute/toc.sh /usr/bin/find . -name "*.md" | xargs git add /usr/bin/git diff --staged --quiet || /usr/bin/git commit -am "GitHub Actions - updating markdown docs - ${{ github.event.repository.updated_at }}" From ad125d825163abb9b8f26dda4450d37fab917493 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Jan 2025 13:47:41 +0100 Subject: [PATCH 192/206] Commenting out some of the AWS ACL rulesets to leave them as examples. --- roles/aws/aws_acl/defaults/main.yml | 52 +++++++++++++++++------------ 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/roles/aws/aws_acl/defaults/main.yml b/roles/aws/aws_acl/defaults/main.yml index 2757cffc6..d74402f89 100644 --- a/roles/aws/aws_acl/defaults/main.yml +++ b/roles/aws/aws_acl/defaults/main.yml @@ -10,29 +10,37 @@ aws_acl: rate_limit: value: 600 # set to 0 to skip rate limit rule, set to a value to set how many requests to allow in period before blocking priority: 2 # can be float with 1 decimal place - ip_sets: - - rule_name: "Allowed-IPs-rule" - set_name: "Allowed-IPs-set" - description: "List of IPs to whitelist - Ansible managed" - action: allow - priority: 1 - list: - - 1.1.1.1/32 - - 2.2.2.2/32 - country_codes: - - name: "allowed-countries" - action: allow - priority: 0.2 - list: - - GB - - HR - - name: "blocked-countries" - action: block - priority: 8 - list: - - RU - - CN + ip_sets: [] +# Example IP set to allow a list of safe IPs +# - rule_name: "Allowed-IPs-rule" +# set_name: "Allowed-IPs-set" +# description: "List of IPs to safelist - Ansible managed" +# action: allow +# priority: 1 +# list: +# - 1.1.1.1/32 +# - 2.2.2.2/32 +# - 30.30.30.0/24 +# Example country code ruleset allowing one set of countries and blocking another +# country_codes: +# - name: "allowed-countries" +# action: allow +# priority: 0.2 +# list: +# - GB +# - HR +# - FR +# - ES +# - UY +# - JP +# - name: "blocked-countries" +# action: block +# priority: 8 +# list: +# - RU +# - CN regular_rules: + # Commonly required Drupal rule to allow Panels to function - name: allow_panels action: allow statements_type: "single" # supported "single", "and", "or" and "not" ("and" and "or" supports multiple statements) From 2469388df2bf1f9eeec510efde332a2689718e32 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Jan 2025 14:39:52 +0100 Subject: [PATCH 193/206] Trying to pull the docs branch to ensure it is up to date. --- .github/workflows/ce-provision-publish-docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ce-provision-publish-docs.yml b/.github/workflows/ce-provision-publish-docs.yml index 232a71d99..1c9653eee 100644 --- a/.github/workflows/ce-provision-publish-docs.yml +++ b/.github/workflows/ce-provision-publish-docs.yml @@ -58,6 +58,7 @@ jobs: # First build and publish the markdown docs - name: Build and commit table of contents and README files back to the repo run: | + /usr/bin/git pull origin docs-2.x /usr/bin/git fetch origin 2.x /usr/bin/git merge origin/${{ github.event.pull_request.base.ref }} --allow-unrelated-histories /bin/sh contribute/toc.sh From c73b7a982e9dee3b12599b0291c4ad7391c3421c Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Jan 2025 14:40:11 +0100 Subject: [PATCH 194/206] Adding keyserver.ubuntu.com to the key servers we publish to by default in gpg_key. --- roles/debian/gpg_key/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/debian/gpg_key/defaults/main.yml b/roles/debian/gpg_key/defaults/main.yml index dccbcf101..cd3453d7a 100644 --- a/roles/debian/gpg_key/defaults/main.yml +++ b/roles/debian/gpg_key/defaults/main.yml @@ -1,5 +1,6 @@ --- gpg_key_servers: + - hkps://keyserver.ubuntu.com - hkps://pgp.mit.edu - hkps://keys.openpgp.org gpg_key: From d03a03031929e3b3fb0cf39c696e101938403945 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Jan 2025 19:17:07 +0100 Subject: [PATCH 195/206] CI updates and improving SOPS role. --- .github/workflows/ce-provision-publish-docs.yml | 3 +++ roles/debian/sops/defaults/main.yml | 2 +- roles/debian/sops/tasks/main.yml | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ce-provision-publish-docs.yml b/.github/workflows/ce-provision-publish-docs.yml index 1c9653eee..ef88357b4 100644 --- a/.github/workflows/ce-provision-publish-docs.yml +++ b/.github/workflows/ce-provision-publish-docs.yml @@ -60,6 +60,9 @@ jobs: run: | /usr/bin/git pull origin docs-2.x /usr/bin/git fetch origin 2.x + /usr/bin/git checkout -b 2.x + /usr/bin/git pull origin 2.x + /usr/bin/git checkout docs-2.x /usr/bin/git merge origin/${{ github.event.pull_request.base.ref }} --allow-unrelated-histories /bin/sh contribute/toc.sh /usr/bin/find . -name "*.md" | xargs git add diff --git a/roles/debian/sops/defaults/main.yml b/roles/debian/sops/defaults/main.yml index 969712ff6..ea24d0ff6 100644 --- a/roles/debian/sops/defaults/main.yml +++ b/roles/debian/sops/defaults/main.yml @@ -1,3 +1,3 @@ --- sops: - version: 3.7.3 # see https://github.com/mozilla/sops/releases + version: 3.9.3 # see https://github.com/mozilla/sops/releases diff --git a/roles/debian/sops/tasks/main.yml b/roles/debian/sops/tasks/main.yml index f290d2746..a1de2f71b 100644 --- a/roles/debian/sops/tasks/main.yml +++ b/roles/debian/sops/tasks/main.yml @@ -3,3 +3,4 @@ ansible.builtin.apt: deb: "https://github.com/mozilla/sops/releases/download/v{{ sops.version }}/sops_{{ sops.version }}_amd64.deb" state: present + force: true # avoids errors if newer version installed From 6a2b262bd70a16771502a9e6963c9ca632f39628 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Jan 2025 19:20:43 +0100 Subject: [PATCH 196/206] Swapping hard coded branch names for vars in CI. --- .github/workflows/ce-provision-publish-docs.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ce-provision-publish-docs.yml b/.github/workflows/ce-provision-publish-docs.yml index ef88357b4..b7532a67b 100644 --- a/.github/workflows/ce-provision-publish-docs.yml +++ b/.github/workflows/ce-provision-publish-docs.yml @@ -58,11 +58,11 @@ jobs: # First build and publish the markdown docs - name: Build and commit table of contents and README files back to the repo run: | - /usr/bin/git pull origin docs-2.x - /usr/bin/git fetch origin 2.x - /usr/bin/git checkout -b 2.x - /usr/bin/git pull origin 2.x - /usr/bin/git checkout docs-2.x + /usr/bin/git pull origin docs-${{ github.event.pull_request.base.ref }} + /usr/bin/git fetch origin ${{ github.event.pull_request.base.ref }} + /usr/bin/git checkout -b ${{ github.event.pull_request.base.ref }} + /usr/bin/git pull origin ${{ github.event.pull_request.base.ref }} + /usr/bin/git checkout docs-${{ github.event.pull_request.base.ref }} /usr/bin/git merge origin/${{ github.event.pull_request.base.ref }} --allow-unrelated-histories /bin/sh contribute/toc.sh /usr/bin/find . -name "*.md" | xargs git add From d2bf05d214eed4e047a11dc3da02e90edfc7c6fc Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Jan 2025 19:37:18 +0100 Subject: [PATCH 197/206] Giving up on merging 2.x - will have to try another approach. --- .github/workflows/ce-provision-publish-docs.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/ce-provision-publish-docs.yml b/.github/workflows/ce-provision-publish-docs.yml index b7532a67b..b87f9afe5 100644 --- a/.github/workflows/ce-provision-publish-docs.yml +++ b/.github/workflows/ce-provision-publish-docs.yml @@ -58,12 +58,6 @@ jobs: # First build and publish the markdown docs - name: Build and commit table of contents and README files back to the repo run: | - /usr/bin/git pull origin docs-${{ github.event.pull_request.base.ref }} - /usr/bin/git fetch origin ${{ github.event.pull_request.base.ref }} - /usr/bin/git checkout -b ${{ github.event.pull_request.base.ref }} - /usr/bin/git pull origin ${{ github.event.pull_request.base.ref }} - /usr/bin/git checkout docs-${{ github.event.pull_request.base.ref }} - /usr/bin/git merge origin/${{ github.event.pull_request.base.ref }} --allow-unrelated-histories /bin/sh contribute/toc.sh /usr/bin/find . -name "*.md" | xargs git add /usr/bin/git diff --staged --quiet || /usr/bin/git commit -am "GitHub Actions - updating markdown docs - ${{ github.event.repository.updated_at }}" From ca8ba7f63d2d0abc1d7daadd894cdc5f5f7d2c71 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 16 Jan 2025 18:19:12 +0100 Subject: [PATCH 198/206] Change of approach to Packer plugin management. --- roles/aws/aws_ami/templates/packer.json.j2 | 13 ------------- roles/debian/packer/defaults/main.yml | 3 +++ roles/debian/packer/tasks/main.yml | 6 ++++++ 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/roles/aws/aws_ami/templates/packer.json.j2 b/roles/aws/aws_ami/templates/packer.json.j2 index 36f8ee901..faa3074a1 100755 --- a/roles/aws/aws_ami/templates/packer.json.j2 +++ b/roles/aws/aws_ami/templates/packer.json.j2 @@ -1,17 +1,4 @@ { - "required_plugins": - { - "ansible": - { - "version": "~> 1", - "source": "github.com/hashicorp/ansible" - }, - "amazon": - { - "version": "~> 1", - "source": "github.com/hashicorp/amazon" - } - }, "variables": { "aws_profile": "{{ aws_ami.aws_profile }}" diff --git a/roles/debian/packer/defaults/main.yml b/roles/debian/packer/defaults/main.yml index e3892e914..32243a82b 100644 --- a/roles/debian/packer/defaults/main.yml +++ b/roles/debian/packer/defaults/main.yml @@ -1,3 +1,6 @@ --- packer: version: "1.11.2" # see https://releases.hashicorp.com/packer/ + plugins: + - github.com/hashicorp/amazon + - github.com/hashicorp/aws diff --git a/roles/debian/packer/tasks/main.yml b/roles/debian/packer/tasks/main.yml index b03a7256e..a181c9100 100644 --- a/roles/debian/packer/tasks/main.yml +++ b/roles/debian/packer/tasks/main.yml @@ -5,3 +5,9 @@ dest: /usr/local/bin/ remote_src: true mode: 0755 + +- name: Install the AWS EC2 plugin for Packer. + command: "packer plugins install {{ item }}" + with_items: "{{ packer.plugins }}" + become: true + become_user: "{{ ce_provision.username }}" From e64148a8bc2f42f0b78a4c52f2ece56ad137555f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 16 Jan 2025 18:24:28 +0100 Subject: [PATCH 199/206] Renaming task. --- roles/debian/packer/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/debian/packer/tasks/main.yml b/roles/debian/packer/tasks/main.yml index a181c9100..af93e7c45 100644 --- a/roles/debian/packer/tasks/main.yml +++ b/roles/debian/packer/tasks/main.yml @@ -6,7 +6,7 @@ remote_src: true mode: 0755 -- name: Install the AWS EC2 plugin for Packer. +- name: Install additional Packer plugins. command: "packer plugins install {{ item }}" with_items: "{{ packer.plugins }}" become: true From 3bd6d8f1ecde961827e6d13c41efcfc8df29dc1d Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 17 Jan 2025 13:33:44 +0100 Subject: [PATCH 200/206] Reorganising the GitLab Runner role into blocks. --- roles/debian/gitlab_runner/defaults/main.yml | 45 ++++--- roles/debian/gitlab_runner/tasks/main.yml | 130 +++++++++---------- 2 files changed, 91 insertions(+), 84 deletions(-) diff --git a/roles/debian/gitlab_runner/defaults/main.yml b/roles/debian/gitlab_runner/defaults/main.yml index 1813aa89b..5ed9f4cd5 100644 --- a/roles/debian/gitlab_runner/defaults/main.yml +++ b/roles/debian/gitlab_runner/defaults/main.yml @@ -2,6 +2,29 @@ gitlab_runner: apt_origin: "origin=packages.gitlab.com/runner/gitlab-runner,codename=${distro_codename},label=gitlab-runner" # used by apt_unattended_upgrades apt_signed_by: https://packages.gitlab.com/runner/gitlab-runner/gpgkey + use_docker: false # set to true to install Docker and use the 'docker' executor + install_fargate: false + restart: true # set to false if you're applying settings to a server responsible for its own runners + username: "{{ ce_deploy.username }}" + docker_group: "docker" + runner_workingdir: "/home/{{ ce_deploy.username }}/build" + runner_config: "/etc/gitlab-runner/config.toml" + # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs + fargate: + cluster: "my-cluster" # ECS cluster name + profile: "example" # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set + region: "eu-west-1" # AWS region name - can be substituted for "{{ _aws_region }}" if set + subnet: "subnet-abcdef123456" # subnet ID + security_group: "my-security-group" # SG name + task_definition: "my-task:1" # task definition in format name:revision, if revision is not provided ECS will use latest + public_ip: "false" # if your containers need a public IP assigning + version: "1.4.0" # Fargate platform version + metadata_dir: "/opt/gitlab-runner/metadata" + ssh_user: "root" + ssh_port: 22 + ################################################## + # @TODO - these variables are not currently used # + ################################################## concurrent_jobs: 10 check_interval: 0 session_timeout: 1800 @@ -28,22 +51,6 @@ gitlab_runner: # run_args: '["--config", "/etc/gitlab-runner/fargate.toml", "custom", "run"]' # cleanup_exec: "/opt/gitlab-runner/fargate" # cleanup_args: '["--config", "/etc/gitlab-runner/fargate.toml", "custom", "cleanup"]' - install_fargate: false - restart: true # set to false if you're applying settings to a server responsible for its own runners - username: "{{ ce_deploy.username }}" - docker_group: "docker" - runner_workingdir: "/home/{{ ce_deploy.username }}/build" - runner_config: "/etc/gitlab-runner/config.toml" - # see https://gitlab.com/gitlab-org/ci-cd/custom-executor-drivers/fargate/-/tree/master/docs - fargate: - cluster: "my-cluster" # ECS cluster name - profile: "example" # AWS boto profile name - can be substituted for "{{ _aws_profile }}" if set - region: "eu-west-1" # AWS region name - can be substituted for "{{ _aws_region }}" if set - subnet: "subnet-abcdef123456" # subnet ID - security_group: "my-security-group" # SG name - task_definition: "my-task:1" # task definition in format name:revision, if revision is not provided ECS will use latest - public_ip: "false" # if your containers need a public IP assigning - version: "1.4.0" # Fargate platform version - metadata_dir: "/opt/gitlab-runner/metadata" - ssh_user: "root" - ssh_port: 22 + ################################################## + # End @TODO # + ################################################## diff --git a/roles/debian/gitlab_runner/tasks/main.yml b/roles/debian/gitlab_runner/tasks/main.yml index 017dd4afc..f987e9b8d 100644 --- a/roles/debian/gitlab_runner/tasks/main.yml +++ b/roles/debian/gitlab_runner/tasks/main.yml @@ -45,61 +45,73 @@ # gitlab-runner register --url https://gitlab.com/ --registration-token TOKEN_HERE --name fargate-test-runner --run-untagged --executor custom -n # This will automatically update /etc/gitlab-runner/config.toml -# Populates the _aws_security_group_list variable used in fargate.toml.j2 -- name: Generate security group information. - ansible.builtin.include_role: - name: aws/aws_security_groups - apply: - delegate_to: localhost # this is how you delegate the include_role module - vars: - aws_security_groups: - profile: "{{ gitlab_runner.fargate.profile }}" - region: "{{ gitlab_runner.fargate.region }}" - group_names: - - "{{ gitlab_runner.fargate.security_group }}" - return_type: ids - when: gitlab_runner.install_fargate +# @TODO support different runner types - MPV custom (Fargate) and shell -- name: Create the Fargate driver directory if it does not exist. - ansible.builtin.file: - path: /opt/gitlab-runner - state: directory - mode: '0755' - owner: root - group: root - become: true - when: gitlab_runner.install_fargate - -- name: Install the Fargate driver. - ansible.builtin.get_url: - url: https://gitlab-runner-custom-fargate-downloads.s3.amazonaws.com/latest/fargate-linux-amd64 - dest: /opt/gitlab-runner/fargate - mode: 0755 - become: true +# Populates the _aws_security_group_list variable used in fargate.toml.j2 +- name: Install Fargate runner. when: gitlab_runner.install_fargate - -- name: Place the Fargate configuration file. - ansible.builtin.template: - src: fargate.toml.j2 - dest: /etc/gitlab-runner/fargate.toml - owner: root - group: root - force: true - become: true - when: - - gitlab_runner.install_fargate - - gitlab_runner.fargate | length > 0 - -- name: Ensure the Fargate metadata directory exists. - ansible.builtin.file: - path: /opt/gitlab-runner/metadata - state: directory - mode: '0755' - owner: root - group: root - become: true - when: - - gitlab_runner.install_fargate + block: + - name: Generate security group information. + ansible.builtin.include_role: + name: aws/aws_security_groups + apply: + delegate_to: localhost # this is how you delegate the include_role module + vars: + aws_security_groups: + profile: "{{ gitlab_runner.fargate.profile }}" + region: "{{ gitlab_runner.fargate.region }}" + group_names: + - "{{ gitlab_runner.fargate.security_group }}" + return_type: ids + + - name: Create the Fargate driver directory if it does not exist. + ansible.builtin.file: + path: /opt/gitlab-runner + state: directory + mode: '0755' + owner: root + group: root + become: true + + - name: Install the Fargate driver. + ansible.builtin.get_url: + url: https://gitlab-runner-custom-fargate-downloads.s3.amazonaws.com/latest/fargate-linux-amd64 + dest: /opt/gitlab-runner/fargate + mode: 0755 + become: true + + - name: Place the Fargate configuration file. + ansible.builtin.template: + src: fargate.toml.j2 + dest: /etc/gitlab-runner/fargate.toml + owner: root + group: root + force: true + become: true + when: + - gitlab_runner.fargate | length > 0 + + - name: Ensure the Fargate metadata directory exists. + ansible.builtin.file: + path: /opt/gitlab-runner/metadata + state: directory + mode: '0755' + owner: root + group: root + become: true + +- name: Install Docker. + when: gitlab_runner.install_fargate or gitlab_runner.use_docker + block: + - name: Run docker_ce role to install Docker. + ansible.builtin.include_role: + name: debian/docker_ce + + - name: Add the GitLab Runner user (usually the controller) to the docker group. + ansible.builtin.user: + name: "{{ gitlab_runner.username }}" + groups: "{{ gitlab_runner.docker_group }}" + append: true # @TODO inject the executor lines after any Fargate services in config.toml @@ -145,18 +157,6 @@ become: true when: gitlab_runner.restart -- name: Run docker_ce role when Fargate driver to be installed. - ansible.builtin.include_role: - name: debian/docker_ce - when: gitlab_runner.install_fargate - -- name: Add the ce-provision user to the docker group. - ansible.builtin.user: - name: "{{ gitlab_runner.username }}" - groups: "{{ gitlab_runner.docker_group }}" - append: true - when: gitlab_runner.install_fargate - # See https://docs.gitlab.com/runner/shells/index.html#shell-profile-loading - name: Delete $HOME/.bash_logout for the runner user if it exists. ansible.builtin.file: From f1e5603d373c936886affe245827e74120271bf8 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 23 Jan 2025 16:44:57 +0100 Subject: [PATCH 201/206] Updating amazon.aws collection for Ansible. --- roles/debian/ce_deploy/meta/requirements-11.yml | 1 - roles/debian/ce_deploy/meta/requirements-12.yml | 1 - roles/debian/ce_provision/meta/requirements-11.yml | 1 - roles/debian/ce_provision/meta/requirements-12.yml | 1 - roles/debian/clamav/defaults/main.yml | 2 +- roles/debian/gitlab_runner/tasks/main.yml | 2 +- 6 files changed, 2 insertions(+), 6 deletions(-) diff --git a/roles/debian/ce_deploy/meta/requirements-11.yml b/roles/debian/ce_deploy/meta/requirements-11.yml index a15dd2943..af33d8492 100644 --- a/roles/debian/ce_deploy/meta/requirements-11.yml +++ b/roles/debian/ce_deploy/meta/requirements-11.yml @@ -6,4 +6,3 @@ collections: - name: community.aws - name: ansible.posix - name: amazon.aws - version: "8.0.1" # ELB plugin broken since June diff --git a/roles/debian/ce_deploy/meta/requirements-12.yml b/roles/debian/ce_deploy/meta/requirements-12.yml index a15dd2943..af33d8492 100644 --- a/roles/debian/ce_deploy/meta/requirements-12.yml +++ b/roles/debian/ce_deploy/meta/requirements-12.yml @@ -6,4 +6,3 @@ collections: - name: community.aws - name: ansible.posix - name: amazon.aws - version: "8.0.1" # ELB plugin broken since June diff --git a/roles/debian/ce_provision/meta/requirements-11.yml b/roles/debian/ce_provision/meta/requirements-11.yml index fabdb2ff2..1e968332c 100644 --- a/roles/debian/ce_provision/meta/requirements-11.yml +++ b/roles/debian/ce_provision/meta/requirements-11.yml @@ -8,7 +8,6 @@ collections: - name: community.general - name: community.postgresql - name: amazon.aws - version: "8.0.1" # ELB plugin broken since June roles: - name: geerlingguy.varnish diff --git a/roles/debian/ce_provision/meta/requirements-12.yml b/roles/debian/ce_provision/meta/requirements-12.yml index 3bf1f5073..55df2f9d4 100644 --- a/roles/debian/ce_provision/meta/requirements-12.yml +++ b/roles/debian/ce_provision/meta/requirements-12.yml @@ -8,7 +8,6 @@ collections: - name: community.general - name: community.postgresql - name: amazon.aws - version: "8.0.1" # ELB plugin broken since June roles: - name: geerlingguy.varnish diff --git a/roles/debian/clamav/defaults/main.yml b/roles/debian/clamav/defaults/main.yml index 0e811ce8a..4706b665f 100644 --- a/roles/debian/clamav/defaults/main.yml +++ b/roles/debian/clamav/defaults/main.yml @@ -16,7 +16,7 @@ clamav: timer_OnCalendar: "*-*-* 02:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events server_name: "{{ inventory_hostname }}" # for identification via email, defaults to Ansible inventory name. log_location: /var/log/clamav - send_mail: false # Important: will not send any emails by default. + send_mail: false # Important - will not send any emails by default. send_on_fail: true # Only sends emails on scan failure, will not email for successful scans. report_recipient_email: mail@example.com report_sender_email: admin@server.example.com diff --git a/roles/debian/gitlab_runner/tasks/main.yml b/roles/debian/gitlab_runner/tasks/main.yml index f987e9b8d..e42627c74 100644 --- a/roles/debian/gitlab_runner/tasks/main.yml +++ b/roles/debian/gitlab_runner/tasks/main.yml @@ -45,7 +45,7 @@ # gitlab-runner register --url https://gitlab.com/ --registration-token TOKEN_HERE --name fargate-test-runner --run-untagged --executor custom -n # This will automatically update /etc/gitlab-runner/config.toml -# @TODO support different runner types - MPV custom (Fargate) and shell +# @TODO support different runner types - MVP custom (Fargate) and shell # Populates the _aws_security_group_list variable used in fargate.toml.j2 - name: Install Fargate runner. From 1e99d57816601f1cc89f87ec6726f32e5cd9a602 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 29 Jan 2025 15:58:56 +0100 Subject: [PATCH 202/206] Merging 2.x. --- ce-dev/ansible/vars/_common/user_deploy.yml | 2 -- ce-dev/ansible/vars/_common/user_provision.yml | 2 -- ce-dev/ansible/vars/gitlab/ce_deploy.yml | 3 +-- install.sh | 5 ++--- plays/launcher/configure.yml | 12 ++++++++++++ roles/debian/user_ansible/defaults/main.yml | 3 --- roles/debian/user_deploy/defaults/main.yml | 3 --- roles/debian/user_provision/defaults/main.yml | 3 --- 8 files changed, 15 insertions(+), 18 deletions(-) create mode 100644 plays/launcher/configure.yml diff --git a/ce-dev/ansible/vars/_common/user_deploy.yml b/ce-dev/ansible/vars/_common/user_deploy.yml index 0130b82cb..41ac2bc34 100644 --- a/ce-dev/ansible/vars/_common/user_deploy.yml +++ b/ce-dev/ansible/vars/_common/user_deploy.yml @@ -1,8 +1,6 @@ _user_deploy_username: ce-dev user_deploy: username: "{{ _user_deploy_username }}" - utility_host: "localhost" - utility_username: "{{ _user_deploy_username }}" sudo_config: entity_name: "{{ _user_deploy_username }}" hosts: "ALL" diff --git a/ce-dev/ansible/vars/_common/user_provision.yml b/ce-dev/ansible/vars/_common/user_provision.yml index 3fdc118c4..984f8a103 100644 --- a/ce-dev/ansible/vars/_common/user_provision.yml +++ b/ce-dev/ansible/vars/_common/user_provision.yml @@ -1,8 +1,6 @@ _user_provision_username: ce-dev user_provision: username: "{{ _user_provision_username }}" - utility_host: "localhost" - utility_username: "{{ _user_provision_username }}" sudo_config: entity_name: "{{ _user_provision_username }}" hosts: "ALL" diff --git a/ce-dev/ansible/vars/gitlab/ce_deploy.yml b/ce-dev/ansible/vars/gitlab/ce_deploy.yml index 5c5ae4499..3e30b2306 100644 --- a/ce-dev/ansible/vars/gitlab/ce_deploy.yml +++ b/ce-dev/ansible/vars/gitlab/ce_deploy.yml @@ -2,6 +2,5 @@ ce_deploy: own_repository: https://github.com/codeenigma/ce-deploy.git own_repository_branch: 1.x username: deploy - utility_username: deploy local_dir: /home/deploy/ce-deploy - ce_provision_dir: /home/ce-dev/ce-provision \ No newline at end of file + ce_provision_dir: /home/ce-dev/ce-provision diff --git a/install.sh b/install.sh index eddc31133..f8b7202ca 100755 --- a/install.sh +++ b/install.sh @@ -147,7 +147,6 @@ fi /usr/bin/su - "$CONTROLLER_USER" -c "/usr/bin/python3 -m venv /home/$CONTROLLER_USER/ce-python" /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/python3 -m pip install --upgrade pip" /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/pip install ansible netaddr python-debian" -/usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection install ansible.posix -p /home/$CONTROLLER_USER/.ansible/collections/ansible_collections --force" if [ "$AWS_SUPPORT" = "true" ]; then /usr/bin/su - "$CONTROLLER_USER" -c "/home/$CONTROLLER_USER/ce-python/bin/pip install boto3" fi @@ -167,6 +166,8 @@ else /usr/bin/echo "-------------------------------------------------" fi /usr/bin/mkdir -p "/home/$CONTROLLER_USER/ce-provision/galaxy/roles" +/usr/bin/su - "$CONTROLLER_USER" -c "cd /home/$CONTROLLER_USER/ce-provision && /home/$CONTROLLER_USER/ce-python/bin/ansible-galaxy collection install ansible.posix -p /home/$CONTROLLER_USER/ce-provision/galaxy/ansible_collections --force" + # Create playbook for ce-provision. /bin/cat >"/home/$CONTROLLER_USER/ce-provision/provision.yml" << EOL --- @@ -233,8 +234,6 @@ user_provision: create: false create_home: false update_password: always - utility_username: "${CONTROLLER_USER}" - utility_host: localhost sudo_config: entity_name: "${CONTROLLER_USER}" hosts: "ALL" diff --git a/plays/launcher/configure.yml b/plays/launcher/configure.yml new file mode 100644 index 000000000..39eba1563 --- /dev/null +++ b/plays/launcher/configure.yml @@ -0,0 +1,12 @@ +--- +# Bare provisioning of a new server. +# Use `ansible -i X.X.X.X, --user admin` to run against a server. +- hosts: default + become: true + tasks: + - ansible.builtin.import_role: + name: _init + - ansible.builtin.import_role: + name: debian/user_provision + - ansible.builtin.import_role: + name: _exit diff --git a/roles/debian/user_ansible/defaults/main.yml b/roles/debian/user_ansible/defaults/main.yml index 7f42053a6..5bec6c2c8 100644 --- a/roles/debian/user_ansible/defaults/main.yml +++ b/roles/debian/user_ansible/defaults/main.yml @@ -11,9 +11,6 @@ user_ansible: # This is shown for documentation, you should do this in your config repo # uid: 999 # gid: 999 - # Local username of the deploy user. - utility_host: "localhost" - utility_username: "{{ _user_ansible_username }}" sudo_config: {} # an empty dictionary will skip creating a sudo config # Example sudo config allowing full sudo permissions - see the debian/sudo_config role for more details. # entity_name: "{{ _user_ansible_username }}" diff --git a/roles/debian/user_deploy/defaults/main.yml b/roles/debian/user_deploy/defaults/main.yml index 31934a720..718004a07 100644 --- a/roles/debian/user_deploy/defaults/main.yml +++ b/roles/debian/user_deploy/defaults/main.yml @@ -12,9 +12,6 @@ user_deploy: # This is shown for documentation, you should do this in your config repo # uid: 989 # gid: 989 - # Local username of the deploy user. - utility_host: "localhost" - utility_username: "{{ _user_deploy_username }}" sudo_config: {} # Example config allowing for feature branching. Allows manipulation of NGINX vhosts and cron.d files. # Uncomment to use. diff --git a/roles/debian/user_provision/defaults/main.yml b/roles/debian/user_provision/defaults/main.yml index 3bd885e27..6039727a2 100644 --- a/roles/debian/user_provision/defaults/main.yml +++ b/roles/debian/user_provision/defaults/main.yml @@ -11,9 +11,6 @@ user_provision: # This is shown for documentation, you should do this in your config repo # uid: 988 # gid: 988 - # Local username of the system user. - utility_host: "localhost" - utility_username: "{{ _user_provision_username }}" # Sudo configuration for full passwordless admin privileges. sudo_config: entity_name: "{{ _user_provision_username }}" From 19179552849feeeed65754a6b96360c5a5d71f3e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 29 Jan 2025 18:23:55 +0100 Subject: [PATCH 203/206] Adding a generic launcher playbook. --- plays/launcher/configure.yml | 26 ++++++++++++++++++++++++-- roles/_init/defaults/main.yml | 1 + 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/plays/launcher/configure.yml b/plays/launcher/configure.yml index 39eba1563..2a490f090 100644 --- a/plays/launcher/configure.yml +++ b/plays/launcher/configure.yml @@ -1,7 +1,29 @@ --- # Bare provisioning of a new server. -# Use `ansible -i X.X.X.X, --user admin` to run against a server. -- hosts: default +# Use this command to execute: +# +# cd /home/controller/ce-provision && \ +# ./scripts/provision.sh --workspace /home/controller/ce-provision \ +# --repo none --branch none \ +# --playbook /home/controller/ce-provision/plays/launcher/configure.yml \ +# --ansible-extra-vars "_provision_host=X.X.X.X" +# +# Replace X.X.X.X with your hostname or IP address. +- hosts: localhost + tasks: + - name: Add the server to Ansible hosts in memory. + ansible.builtin.add_host: + hostname: "{{ _provision_host }}" + - name: Add the server to hosts.yml. + ansible.builtin.lineinfile: + path: "/home/{{ _ce_provision_username }}/ce-provision/config/hosts/hosts.yml" + line: "{{ _provision_host }}:" + create: true + +- hosts: "{{ _provision_host }}" + vars: + ansible_user: admin + _ce_provision_build_id: 0 become: true tasks: - ansible.builtin.import_role: diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 017f953ff..d086d6a81 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -7,6 +7,7 @@ _venv_path: "/home/{{ _ce_provision_username }}/ce-python" _venv_command: /usr/bin/python3 -m venv _venv_install_username: "{{ _ce_provision_username }}" _ce_ansible_timer_name: upgrade_ansible +_env_type: unspecified # AWS variables - if you are using an AWS account, you can preset certain variables # Generally it is recommended to place these in your ce-provision-config repository under hosts/group_vars/all From fe4b76c1fb4807f1a52e084b689e27b9951b8984 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 29 Jan 2025 18:38:24 +0100 Subject: [PATCH 204/206] Updating default branch names to 'main'. --- roles/debian/ce_deploy/tasks/main.yml | 4 ++-- roles/debian/ce_provision/tasks/main.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/debian/ce_deploy/tasks/main.yml b/roles/debian/ce_deploy/tasks/main.yml index 88e540b52..d67271386 100644 --- a/roles/debian/ce_deploy/tasks/main.yml +++ b/roles/debian/ce_deploy/tasks/main.yml @@ -47,7 +47,7 @@ ansible.builtin.git: repo: "{{ ce_deploy.own_repository | default('https://github.com/codeenigma/ce-deploy.git') }}" dest: "{{ ce_deploy.local_dir }}" - version: "{{ ce_deploy.own_repository_branch | default('master') }}" + version: "{{ ce_deploy.own_repository_branch | default('main') }}" update: true accept_hostkey: true become: true @@ -58,7 +58,7 @@ repo: "{{ ce_deploy.config_repository }}" accept_hostkey: true dest: "{{ _ce_provision_build_tmp_dir }}/config" - version: "{{ ce_deploy.config_repository_branch | default('master') }}" + version: "{{ ce_deploy.config_repository_branch | default('main') }}" become: false delegate_to: localhost when: ce_deploy.config_repository is defined and ce_deploy.config_repository diff --git a/roles/debian/ce_provision/tasks/main.yml b/roles/debian/ce_provision/tasks/main.yml index 30c3aeec3..ff7481f73 100644 --- a/roles/debian/ce_provision/tasks/main.yml +++ b/roles/debian/ce_provision/tasks/main.yml @@ -85,7 +85,7 @@ ansible.builtin.git: repo: "{{ ce_provision.own_repository | default('https://github.com/codeenigma/ce-provision.git') }}" dest: "{{ ce_provision.local_dir }}" - version: "{{ ce_provision.own_repository_branch | default('master') }}" + version: "{{ ce_provision.own_repository_branch | default('main') }}" update: true accept_hostkey: true #@todo? become: true @@ -98,7 +98,7 @@ repo: "{{ ce_provision.config_repository }}" accept_hostkey: true dest: "{{ ce_provision.local_dir }}/config" - version: "{{ ce_provision.config_repository_branch | default('master') }}" + version: "{{ ce_provision.config_repository_branch | default('main') }}" become: true become_user: "{{ ce_provision.username }}" when: From 8a1059a05e07c7b8cb430393d011d17f85e9928b Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 29 Jan 2025 18:50:37 +0100 Subject: [PATCH 205/206] Adding code to launcher playbook that adds new server to hosts.yml. --- plays/launcher/configure.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/plays/launcher/configure.yml b/plays/launcher/configure.yml index 2a490f090..15bd3f411 100644 --- a/plays/launcher/configure.yml +++ b/plays/launcher/configure.yml @@ -10,16 +10,35 @@ # # Replace X.X.X.X with your hostname or IP address. - hosts: localhost + vars: + _ce_provision_username: controller + # _ce_provision_config_branch: 2.x # uncomment this if your config branch is not 'main' tasks: - name: Add the server to Ansible hosts in memory. ansible.builtin.add_host: hostname: "{{ _provision_host }}" + - name: Add the server to hosts.yml. ansible.builtin.lineinfile: path: "/home/{{ _ce_provision_username }}/ce-provision/config/hosts/hosts.yml" line: "{{ _provision_host }}:" create: true + - name: Add hosts.yml changes. + ansible.builtin.command: git add . + args: + chdir: "/home/{{ _ce_provision_username }}/ce-provision/config" + + - name: Commit hosts.yml changes. + ansible.builtin.shell: "git diff --staged --quiet || git commit -m 'Ansible autogenerated - host {{ _provision_host }} added.'" + args: + chdir: "/home/{{ _ce_provision_username }}/ce-provision/config" + + - name: Push the hosts file change to the config repository. + ansible.builtin.command: "git push origin {{ _ce_provision_config_branch | default('main') }}" + args: + chdir: "/home/{{ _ce_provision_username }}/ce-provision/config" + - hosts: "{{ _provision_host }}" vars: ansible_user: admin From 655f61d1645f0931d9ccbe04e2a6903d43fbdc3a Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 29 Jan 2025 19:00:45 +0100 Subject: [PATCH 206/206] Adding hosts.yml file handling to launcher. --- plays/launcher/configure.yml | 38 ++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/plays/launcher/configure.yml b/plays/launcher/configure.yml index 15bd3f411..4c802bb72 100644 --- a/plays/launcher/configure.yml +++ b/plays/launcher/configure.yml @@ -12,32 +12,36 @@ - hosts: localhost vars: _ce_provision_username: controller - # _ce_provision_config_branch: 2.x # uncomment this if your config branch is not 'main' + _add_host: true # set to false if you do not want to add the new server to hosts.yml + _ce_provision_config_branch: 2.x # change this if you didn't use our example config and have a different main branch name tasks: - name: Add the server to Ansible hosts in memory. ansible.builtin.add_host: hostname: "{{ _provision_host }}" - name: Add the server to hosts.yml. - ansible.builtin.lineinfile: - path: "/home/{{ _ce_provision_username }}/ce-provision/config/hosts/hosts.yml" - line: "{{ _provision_host }}:" - create: true + when: _add_host + block: + - name: Edit hosts.yml file. + ansible.builtin.lineinfile: + path: "/home/{{ _ce_provision_username }}/ce-provision/config/hosts/hosts.yml" + line: "{{ _provision_host }}:" + create: true - - name: Add hosts.yml changes. - ansible.builtin.command: git add . - args: - chdir: "/home/{{ _ce_provision_username }}/ce-provision/config" + - name: Add hosts.yml changes. + ansible.builtin.command: git add . + args: + chdir: "/home/{{ _ce_provision_username }}/ce-provision/config" - - name: Commit hosts.yml changes. - ansible.builtin.shell: "git diff --staged --quiet || git commit -m 'Ansible autogenerated - host {{ _provision_host }} added.'" - args: - chdir: "/home/{{ _ce_provision_username }}/ce-provision/config" + - name: Commit hosts.yml changes. + ansible.builtin.shell: "git diff --staged --quiet || git commit -m 'Ansible autogenerated - host {{ _provision_host }} added.'" + args: + chdir: "/home/{{ _ce_provision_username }}/ce-provision/config" - - name: Push the hosts file change to the config repository. - ansible.builtin.command: "git push origin {{ _ce_provision_config_branch | default('main') }}" - args: - chdir: "/home/{{ _ce_provision_username }}/ce-provision/config" + - name: Push the hosts file change to the config repository. + ansible.builtin.command: "git push origin {{ _ce_provision_config_branch | default('main') }}" + args: + chdir: "/home/{{ _ce_provision_username }}/ce-provision/config" - hosts: "{{ _provision_host }}" vars: