From 195ff54953910c50f2d62a83d8f42b5eebce975a Mon Sep 17 00:00:00 2001 From: EmlynK Date: Wed, 5 Jan 2022 15:30:21 +0000 Subject: [PATCH 01/22] Need to check if is_local is defined in webserver meta dependencies. (#522) --- roles/_meta/webserver/meta/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/_meta/webserver/meta/main.yml b/roles/_meta/webserver/meta/main.yml index 838c86baa..17d8d92ce 100644 --- a/roles/_meta/webserver/meta/main.yml +++ b/roles/_meta/webserver/meta/main.yml @@ -2,7 +2,7 @@ dependencies: - role: _meta/common_base - role: firewall_config - - { role: ssh_server, when: not is_local } + - { role: ssh_server, when: ( is_local is not defined or not is_local ) } - role: user_provision - role: user_deploy - role: mysql_client From a21e8d1796b0c20a4ddc230de756b342796c081c Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 5 Jan 2022 16:58:54 +0100 Subject: [PATCH 02/22] Ce dev refactor pr 1.x (#518) * Making it easier to test with provision-target and ce-dev. * Moving the provision forcing var back to plays so _init has it. * Adding defaults vars and test script extra options. * Adding a web server test to CI. * examples string needs to be in quotes. * Making sure is_local and _ce_provision_force_play are available to the _init role. * Adding SSH keys to the provision user. * Adding a --force to the test script. * Explicitly adding vars to role. * Fixing _init behaviour and adding SSH key for web role. * Setting default PHP version to 7.4. * Looking up the generated ce-dev SSH key instead of hard-coding one. * We cannot run the ssh_server role locally, so excluding for tests of webserver role. * Trying to remove user_root.yml in case it's breaking CI. * Adding a verbose mode to the test script. * Exposing the command in the test script. * Trying hard-coded keys again. * Changing location of data dir for test containers. * Putting vars back and restricting CI to the 'web' example. --- .github/workflows/ce-provision-test.yml | 2 +- ce-dev/ansible/test.sh | 10 ++++++++++ ce-dev/ansible/vars/_common/user_deploy.yml | 3 ++- ce-dev/ansible/vars/_common/user_provision.yml | 3 ++- ce-dev/ansible/vars/_common/user_root.yml | 3 --- 5 files changed, 15 insertions(+), 6 deletions(-) delete mode 100644 ce-dev/ansible/vars/_common/user_root.yml diff --git a/.github/workflows/ce-provision-test.yml b/.github/workflows/ce-provision-test.yml index c9631ec71..0dfbf714e 100644 --- a/.github/workflows/ce-provision-test.yml +++ b/.github/workflows/ce-provision-test.yml @@ -44,7 +44,7 @@ jobs: if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | git clone --branch 1.x https://github.com/codeenigma/ce-dev-ce-provision-config.git config - /bin/bash ce-dev/ansible/test.sh --examples "web gitlab" --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} shell: bash # Builds the docs diff --git a/ce-dev/ansible/test.sh b/ce-dev/ansible/test.sh index 13918e4d6..61124de0a 100755 --- a/ce-dev/ansible/test.sh +++ b/ce-dev/ansible/test.sh @@ -12,6 +12,7 @@ usage(){ echo '--config-branch: Branch to use for the main stack config repository' echo '--no-rebuild: Do not tear down an existing ce-dev stack' echo '--no-provision: Do not run ce-provision against the ce-dev stack' + echo '--verbose: Run ce-provision and Ansible in verbose mode' } # Set defaults @@ -20,6 +21,7 @@ OWN_BRANCH="1.x" CONFIG_BRANCH="1.x" NO_REBUILD=false NO_PROVISION=false +VERBOSE=false # Parse options arguments. parse_options(){ @@ -43,6 +45,9 @@ parse_options(){ "--no-provision") NO_PROVISION=true ;; + "--verbose") + VERBOSE=true + ;; *) usage exit 1 @@ -91,6 +96,11 @@ EOT PROVISION_CMD="/bin/sh /home/ce-dev/ce-provision/scripts/provision.sh" echo "# Executing $1 project" PROVISION_CMD="$PROVISION_CMD --repo dummy --branch dummy --workspace /home/ce-dev/ce-provision/ce-dev/ansible --playbook plays/$1/$1.yml --own-branch $2 --config-branch $3 --force" + if [ $VERBOSE = true ]; then + echo "# In verbose mode" + PROVISION_CMD="$PROVISION_CMD --verbose" + fi + echo "# Running command: $PROVISION_CMD" # shellcheck disable=SC2086 sudo docker exec -t --workdir /home/ce-dev/ce-provision --user ce-dev provision-controller $PROVISION_CMD echo "### $1 project completed ###" diff --git a/ce-dev/ansible/vars/_common/user_deploy.yml b/ce-dev/ansible/vars/_common/user_deploy.yml index 58bbbcee5..bb18c1fde 100644 --- a/ce-dev/ansible/vars/_common/user_deploy.yml +++ b/ce-dev/ansible/vars/_common/user_deploy.yml @@ -5,4 +5,5 @@ user_deploy: sudoer: false groups: [] ssh_keys: - - "{{ lookup('file', '{{ _ce_provision_data_dir }}/provision-controller/home/ce-dev/.ssh/id_rsa.pub') }}" + - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCbzq9srh/W23MBCx/GmlTO3aJckDjZnCcczAAGtx3rAzGFEtvFgDaZbbHK+ya2SSazzQbfSXmwWAYABtFu6qRmZtDCBKlHhR7rI9MysiACmSX6X6fVIDbCZoVXIZIVGR0wdv52+kMArMktO9Y456jlman5whd3b7VD5LOt1xKQRLVQ7KjsxgQ/hpolSo6ZmsJuJe2xF9NMSfpQetmprTEFoCbXbVOl6XDUVqYsNF/e55A2jXBng2UNluztWQDnaP2sDODPk5uw3Fy6Znk0auYe9CmImUZX5x+BLulJIJWmsDa2+Ls3MCpF9VKPqV/UInETJ5P3ge1ieC8a1dboPL4/T9XERPXuGxeZtISNHwFZ3mGSjrbqFpMpJHZ7G41oGqBOq9dsw4eY3G0YlvMtv8Nqu2MtOQz/nGFSniES6e7Q4aHtBOoCbpJt7tDcpIDy9YfEhgQHxNtdAmt0n7GS71O0HS+15y89EzJ46139vZeuvSUkdgUxhQ5ZWhp1u3/6XyaCKt+SHKH5CbbrXQrfP/g98wm4W6kVAAjtkEXZ6rwsc71WdRLxZr17ArOpG+YVTJ67+iMEgsaEw9bi2DNDVeYl51NmNc4d873iH7d/0gsRq4ECzZwoNB0X9N6n0F5cVjQvx2802FzwHUMA2V8gS64UBuoC2OIv/r3YSkwuplzggw== controller@127.0.0.1" + #- "{{ lookup('file', '{{ _ce_provision_data_dir }}/provision-controller/home/ce-dev/.ssh/id_rsa.pub') }}" # works locally diff --git a/ce-dev/ansible/vars/_common/user_provision.yml b/ce-dev/ansible/vars/_common/user_provision.yml index a1b811429..fb9c97b97 100644 --- a/ce-dev/ansible/vars/_common/user_provision.yml +++ b/ce-dev/ansible/vars/_common/user_provision.yml @@ -5,4 +5,5 @@ user_provision: sudoer: true groups: [] ssh_keys: - - "{{ lookup('file', '{{ _ce_provision_data_dir }}/provision-controller/home/ce-dev/.ssh/id_rsa.pub') }}" + - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCbzq9srh/W23MBCx/GmlTO3aJckDjZnCcczAAGtx3rAzGFEtvFgDaZbbHK+ya2SSazzQbfSXmwWAYABtFu6qRmZtDCBKlHhR7rI9MysiACmSX6X6fVIDbCZoVXIZIVGR0wdv52+kMArMktO9Y456jlman5whd3b7VD5LOt1xKQRLVQ7KjsxgQ/hpolSo6ZmsJuJe2xF9NMSfpQetmprTEFoCbXbVOl6XDUVqYsNF/e55A2jXBng2UNluztWQDnaP2sDODPk5uw3Fy6Znk0auYe9CmImUZX5x+BLulJIJWmsDa2+Ls3MCpF9VKPqV/UInETJ5P3ge1ieC8a1dboPL4/T9XERPXuGxeZtISNHwFZ3mGSjrbqFpMpJHZ7G41oGqBOq9dsw4eY3G0YlvMtv8Nqu2MtOQz/nGFSniES6e7Q4aHtBOoCbpJt7tDcpIDy9YfEhgQHxNtdAmt0n7GS71O0HS+15y89EzJ46139vZeuvSUkdgUxhQ5ZWhp1u3/6XyaCKt+SHKH5CbbrXQrfP/g98wm4W6kVAAjtkEXZ6rwsc71WdRLxZr17ArOpG+YVTJ67+iMEgsaEw9bi2DNDVeYl51NmNc4d873iH7d/0gsRq4ECzZwoNB0X9N6n0F5cVjQvx2802FzwHUMA2V8gS64UBuoC2OIv/r3YSkwuplzggw== controller@127.0.0.1" + #- "{{ lookup('file', '{{ _ce_provision_data_dir }}/provision-controller/home/ce-dev/.ssh/id_rsa.pub') }}" # works locally diff --git a/ce-dev/ansible/vars/_common/user_root.yml b/ce-dev/ansible/vars/_common/user_root.yml deleted file mode 100644 index 3dda2aed2..000000000 --- a/ce-dev/ansible/vars/_common/user_root.yml +++ /dev/null @@ -1,3 +0,0 @@ -user_root: - authorized_keys: - - "{{ lookup('file', '{{ _ce_provision_data_dir }}/provision-controller/home/ce-dev/.ssh/id_rsa.pub') }}" From cb681f8ae83535cc14a78ab75221e277b4546ed0 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 10 Jan 2022 14:45:56 +0100 Subject: [PATCH 03/22] Adding backup handling to ldap_server. (#525) * Adding backup handling to ldap_server. * Improving SSL docs and handling perms for openldap and letsencrypt. * Cron user must be specified with file. * Running as root, do not need a 'sudo' in this cron. --- roles/ldap_server/defaults/main.yml | 12 +++++++- roles/ldap_server/tasks/main.yml | 38 +++++++++++++++++++++++++ roles/ldap_server/templates/slap-bak.j2 | 11 +++++++ 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 roles/ldap_server/templates/slap-bak.j2 diff --git a/roles/ldap_server/defaults/main.yml b/roles/ldap_server/defaults/main.yml index 13cbfda10..f6a04b345 100644 --- a/roles/ldap_server/defaults/main.yml +++ b/roles/ldap_server/defaults/main.yml @@ -14,9 +14,19 @@ ldap_server: # Execution order can be important, so ensure your files are named in alphabetical order. path: "{{ _ce_provision_base_dir }}/config/files/ldap_server/config" purge: false + backup: false # set to true to create local backups of LDAP + backup_path: /opt/slap-bak + backup_minute: "45" + backup_hour: "23" + # TLS settings in LDAP are not separately handled, you need to manually set it up or use the config import feature. + # If you use 'manual' SSL handling you need to provide a separate CA certificate. + # If you use 'letsencrypt' SSL handling then the LDAP TLS settings in your imported config should be as follows: + # olcTLSCACertificateFile: /etc/letsencrypt/live/{{ _domain_name }}/chain.pem + # olcTLSCertificateFile: /etc/letsencrypt/live/{{ _domain_name }}/cert.pem + # olcTLSCertificateKeyFile: /etc/letsencrypt/live/{{ _domain_name }}/privkey.pem ssl: # @see the 'ssl' role - does nothing by default. domain: "{{ _domain_name }}" - handling: "unmanaged" # LDAP needs to use 'manual' handling for SSL because a separate CA certificate is required. + handling: "unmanaged" key: "" cert: "" ca_cert: "" diff --git a/roles/ldap_server/tasks/main.yml b/roles/ldap_server/tasks/main.yml index 07938b012..a9e48b532 100644 --- a/roles/ldap_server/tasks/main.yml +++ b/roles/ldap_server/tasks/main.yml @@ -129,6 +129,25 @@ vars: ssl: "{{ ldap_server.ssl }}" +# LetsEncrypt handling - see https://serverfault.com/a/864655 +- name: Create user for LetsEncrypt. + ansible.builtin.user: + name: letsencrypt + group: letsencrypt + system: true + create_home: false + password: "*" + groups: openldap + when: ldap_server.ssl.handling == 'letsencrypt' + +- name: Allow OpenLDAP to view the LetsEncrypt directory. + ansible.builtin.file: + path: /etc/letsencrypt + state: directory + owner: openldap + group: letsencrypt + when: ldap_server.ssl.handling == 'letsencrypt' + - name: Start slapd service. ansible.builtin.service: name: slapd @@ -140,3 +159,22 @@ when: - ldap_server.replication.host - ldap_server.config.import + +- name: Create the slapd backup script. + ansible.builtin.template: + src: slap-bak.j2 + dest: /usr/local/bin/slap-bak + owner: root + group: root + mode: 0755 + when: ldap_server.config.backup + +- name: Create cron job for slapd backups. + ansible.builtin.cron: + name: "ldap backup" + minute: "{{ ldap_server.config.backup_minute }}" + hour: "{{ ldap_server.config.backup_hour }}" + job: "/usr/local/bin/slap-bak && find /opt/slap-bak/ -type f -mtime +60 -print0 | xargs -0 -r rm" + cron_file: slapd-backup + user: root + when: ldap_server.config.backup diff --git a/roles/ldap_server/templates/slap-bak.j2 b/roles/ldap_server/templates/slap-bak.j2 new file mode 100644 index 000000000..6ab8b38f2 --- /dev/null +++ b/roles/ldap_server/templates/slap-bak.j2 @@ -0,0 +1,11 @@ +#!/bin/sh +BACKUPDIR="{{ ldap_server.config.backup_path }}" +mkdir -p "$BACKUPDIR" + +CONFIG_LDAPBK="ldap-config-$( date +%y%m%d-%H%M ).ldif" +slapcat -n 0 > "$BACKUPDIR/$CONFIG_LDAPBK" +gzip -9 "$BACKUPDIR/$CONFIG_LDAPBK" + +DATA_LDAPBK="ldap-data-$( date +%y%m%d-%H%M ).ldif" +slapcat -n 1 > "$BACKUPDIR/$DATA_LDAPBK" +gzip -9 "$BACKUPDIR/$DATA_LDAPBK" From f134be0219aace7f6d5fd0b2ac09aee8f17168d7 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 17 Jan 2022 13:12:09 +0100 Subject: [PATCH 04/22] Allowing 'gitLab' to disable Prometheus. (#530) * Allowing 'gitLab' to disable Prometheus. * Booleans to use in jinja2 as strings must be cast as strings. --- ce-dev/ansible/vars/provision/ce_provision.yml | 2 +- ce-dev/ansible/vars/provision/galaxy-requirements.yml | 8 ++++++++ roles/ce_provision/meta/requirements.yml | 1 - roles/gitlab/defaults/main.yml | 5 ++++- roles/gitlab/templates/gitlab.rb.j2 | 6 +++--- 5 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 ce-dev/ansible/vars/provision/galaxy-requirements.yml diff --git a/ce-dev/ansible/vars/provision/ce_provision.yml b/ce-dev/ansible/vars/provision/ce_provision.yml index a93890652..6818f88bb 100644 --- a/ce-dev/ansible/vars/provision/ce_provision.yml +++ b/ce-dev/ansible/vars/provision/ce_provision.yml @@ -8,7 +8,7 @@ ce_provision: config_repository_branch: "1.x" config_repository_skip_checkout: true local_dir: "/home/ce-dev/ce-provision" - galaxy_custom_requirements_file: "" + galaxy_custom_requirements_file: "/home/ce-dev/ce-provision/ce-dev/ansible/vars/provision/galaxy-requirements.yml" _ce_provision_base_dir: /home/ce-dev/ce-provision _ce_provision_build_dir: /home/ce-dev/ce-provision/ce-dev/ansible/local _ce_provision_build_tmp_dir: /tmp diff --git a/ce-dev/ansible/vars/provision/galaxy-requirements.yml b/ce-dev/ansible/vars/provision/galaxy-requirements.yml new file mode 100644 index 000000000..eeab8eed2 --- /dev/null +++ b/ce-dev/ansible/vars/provision/galaxy-requirements.yml @@ -0,0 +1,8 @@ +--- +roles: + - name: geerlingguy.solr + - name: geerlingguy.java + - name: cloudalchemy.prometheus + - name: cloudalchemy.node_exporter + - name: cloudalchemy.alertmanager + - name: cloudalchemy.process_exporter diff --git a/roles/ce_provision/meta/requirements.yml b/roles/ce_provision/meta/requirements.yml index 0d87ad9c7..38b0d3b6a 100644 --- a/roles/ce_provision/meta/requirements.yml +++ b/roles/ce_provision/meta/requirements.yml @@ -5,7 +5,6 @@ collections: - name: amazon.aws roles: - - name: cloudalchemy.process_exporter - name: geerlingguy.varnish - name: geerlingguy.firewall - name: geerlingguy.composer diff --git a/roles/gitlab/defaults/main.yml b/roles/gitlab/defaults/main.yml index 1fa32e471..dd004aeb5 100644 --- a/roles/gitlab/defaults/main.yml +++ b/roles/gitlab/defaults/main.yml @@ -14,11 +14,14 @@ gitlab: unicorn_worker_processes: 2 puma_worker_processes: 2 initial_root_password: "" - ldap: false + ldap: false # enable/disable LDAP integration ldap_endpoint: "{{ ldap_client.endpoints[0] }}" ldap_lookup_base: "{{ ldap_client.lookup_base }}" ldap_binddn: "{{ ldap_client.binddn }}" ldap_bindpw: "{{ ldap_client.bindpw }}" + prometheus: "true" # enable/disable built-in Prometheus + node_exporter: "true" # enable/disable built-in Prometheus Node Exporter + alertmanager: "true" # enable/disable built-in Prometheus Alertmanager nginx: listen_port: 8881 listen_https: nil diff --git a/roles/gitlab/templates/gitlab.rb.j2 b/roles/gitlab/templates/gitlab.rb.j2 index 29588f0c1..b5ffaa8bd 100644 --- a/roles/gitlab/templates/gitlab.rb.j2 +++ b/roles/gitlab/templates/gitlab.rb.j2 @@ -1345,7 +1345,7 @@ nginx['listen_https'] = {{ gitlab.nginx.listen_https }} ##! Docs: https://docs.gitlab.com/ce/administration/monitoring/prometheus/ ################################################################################ -# prometheus['enable'] = true +prometheus['enable'] = {{ gitlab.prometheus }} # prometheus['monitor_kubernetes'] = true # prometheus['username'] = 'gitlab-prometheus' # prometheus['group'] = 'gitlab-prometheus' @@ -1427,7 +1427,7 @@ nginx['listen_https'] = {{ gitlab.nginx.listen_https }} ##! Docs: https://docs.gitlab.com/ce/administration/monitoring/prometheus/alertmanager.html ################################################################################ -# alertmanager['enable'] = true +alertmanager['enable'] = {{ gitlab.alertmanager }} # alertmanager['home'] = '/var/opt/gitlab/alertmanager' # alertmanager['log_directory'] = '/var/log/gitlab/alertmanager' # alertmanager['admin_email'] = 'admin@example.com' @@ -1449,7 +1449,7 @@ nginx['listen_https'] = {{ gitlab.nginx.listen_https }} ##! Docs: https://docs.gitlab.com/ce/administration/monitoring/prometheus/node_exporter.html ################################################################################ -# node_exporter['enable'] = true +node_exporter['enable'] = {{ gitlab.node_exporter }} # node_exporter['home'] = '/var/opt/gitlab/node-exporter' # node_exporter['log_directory'] = '/var/log/gitlab/node-exporter' # node_exporter['flags'] = { From 450d4090d3002053a3be1398b6e574f0fb14dc7b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:32:13 +0100 Subject: [PATCH 05/22] GitHub Actions - Rebuilt documentation. (#526) Co-authored-by: Code Enigma CI From 965d7b2c1169351638530f4c22b18d23bfc03fe8 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 17 Jan 2022 14:14:51 +0100 Subject: [PATCH 06/22] Prometheus pr 1.x (#533) * Allowing 'gitLab' to disable Prometheus. * Booleans to use in jinja2 as strings must be cast as strings. * Tidying up CI and adding a GitLab test. * Fixing CI job description. --- ...n-test.yml => ce-provision-build-docs.yml} | 29 ++----------- .../workflows/ce-provision-test-gitlab.yml | 42 +++++++++++++++++++ .github/workflows/ce-provision-test-web.yml | 42 +++++++++++++++++++ 3 files changed, 87 insertions(+), 26 deletions(-) rename .github/workflows/{ce-provision-test.yml => ce-provision-build-docs.yml} (59%) create mode 100644 .github/workflows/ce-provision-test-gitlab.yml create mode 100644 .github/workflows/ce-provision-test-web.yml diff --git a/.github/workflows/ce-provision-test.yml b/.github/workflows/ce-provision-build-docs.yml similarity index 59% rename from .github/workflows/ce-provision-test.yml rename to .github/workflows/ce-provision-build-docs.yml index 0dfbf714e..bbd1c649f 100644 --- a/.github/workflows/ce-provision-test.yml +++ b/.github/workflows/ce-provision-build-docs.yml @@ -1,4 +1,4 @@ -name: Run tests and build docs +name: Build docs # Run this workflow every time a new commit pushed to your repository on: pull_request @@ -6,9 +6,9 @@ on: pull_request jobs: # Set the job key. The key is displayed as the job name # when a job name is not provided - run-tests: + build-docs: # Name the Job - name: Run tests against Ansible code base + name: Build the documentation # Set the type of machine to run on runs-on: ubuntu-20.04 @@ -18,35 +18,12 @@ jobs: if: ${{ github.event.pull_request.head.ref != 'documentation' }} uses: actions/checkout@v2 - # Installs the ce-dev stack - - name: Install ce-dev - if: ${{ github.event.pull_request.head.ref != 'documentation' }} - run: | - cd /tmp - wget https://golang.org/dl/go1.15.8.linux-amd64.tar.gz - sudo tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz - export PATH=$PATH:/usr/local/go/bin - git clone https://github.com/FiloSottile/mkcert && cd mkcert - go build -ldflags "-X main.Version=$(git describe --tags)" - sudo mv ./mkcert /usr/local/bin && cd ../ - sudo chmod +x /usr/local/bin/mkcert - rm -Rf mkcert - curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux - # Configures global Git variables for committing - name: Configure Git run: | git config --global user.email "sysadm@codeenigma.com" git config --global user.name "Code Enigma CI" - # Uses the ce-dev stack to run a test provision - - name: Run a test provision - if: ${{ github.event.pull_request.head.ref != 'documentation' }} - run: | - git clone --branch 1.x https://github.com/codeenigma/ce-dev-ce-provision-config.git config - /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} - shell: bash - # Builds the docs - name: Build documentation if: ${{ github.event.pull_request.head.ref != 'documentation' }} diff --git a/.github/workflows/ce-provision-test-gitlab.yml b/.github/workflows/ce-provision-test-gitlab.yml new file mode 100644 index 000000000..1357e9cc2 --- /dev/null +++ b/.github/workflows/ce-provision-test-gitlab.yml @@ -0,0 +1,42 @@ +name: Run GitLab server test build + +# Run this workflow every time a new commit pushed to your repository +on: pull_request + +jobs: + # Set the job key. The key is displayed as the job name + # when a job name is not provided + test-gitlab: + # Name the Job + name: Run tests against Ansible code base + # Set the type of machine to run on + runs-on: ubuntu-20.04 + + steps: + # Checks out a copy of your repository on the ubuntu-latest machine + - name: Checkout code + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + uses: actions/checkout@v2 + + # Installs the ce-dev stack + - name: Install ce-dev + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + run: | + cd /tmp + wget https://golang.org/dl/go1.15.8.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz + export PATH=$PATH:/usr/local/go/bin + git clone https://github.com/FiloSottile/mkcert && cd mkcert + go build -ldflags "-X main.Version=$(git describe --tags)" + sudo mv ./mkcert /usr/local/bin && cd ../ + sudo chmod +x /usr/local/bin/mkcert + rm -Rf mkcert + curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux + + # Uses the ce-dev stack to run a test provision + - name: Run a test provision + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + run: | + git clone --branch 1.x https://github.com/codeenigma/ce-dev-ce-provision-config.git config + /bin/bash ce-dev/ansible/test.sh --examples gitlab --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + shell: bash diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml new file mode 100644 index 000000000..55a6942f4 --- /dev/null +++ b/.github/workflows/ce-provision-test-web.yml @@ -0,0 +1,42 @@ +name: Run web server test build + +# Run this workflow every time a new commit pushed to your repository +on: pull_request + +jobs: + # Set the job key. The key is displayed as the job name + # when a job name is not provided + test-web: + # Name the Job + name: Run tests against Ansible code base + # Set the type of machine to run on + runs-on: ubuntu-20.04 + + steps: + # Checks out a copy of your repository on the ubuntu-latest machine + - name: Checkout code + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + uses: actions/checkout@v2 + + # Installs the ce-dev stack + - name: Install ce-dev + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + run: | + cd /tmp + wget https://golang.org/dl/go1.15.8.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz + export PATH=$PATH:/usr/local/go/bin + git clone https://github.com/FiloSottile/mkcert && cd mkcert + go build -ldflags "-X main.Version=$(git describe --tags)" + sudo mv ./mkcert /usr/local/bin && cd ../ + sudo chmod +x /usr/local/bin/mkcert + rm -Rf mkcert + curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux + + # Uses the ce-dev stack to run a test provision + - name: Run a test provision + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + run: | + git clone --branch 1.x https://github.com/codeenigma/ce-dev-ce-provision-config.git config + /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + shell: bash From cbb386d21fed5cad5748effbf7728b8ff9889a8b Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 18 Jan 2022 13:47:45 +0000 Subject: [PATCH 07/22] Add private files support for Drupal in Nginx. (#535) --- roles/nginx/templates/drupal_common.j2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/roles/nginx/templates/drupal_common.j2 b/roles/nginx/templates/drupal_common.j2 index 19db7b1e9..2c6a5064a 100644 --- a/roles/nginx/templates/drupal_common.j2 +++ b/roles/nginx/templates/drupal_common.j2 @@ -89,6 +89,13 @@ location ~ ^/sites/.*/files/.* { try_files $uri @rewrite; } +# Allow private files support +location ~* ^/system/files/.*\.(png|jpg|jpeg|gif|ico|svg)$ { # If the image does not exist, it must be a private file. + try_files $uri @rewrite; + expires 7d; + log_not_found off; +} + # Core and contrib assets can be pretty much anywhere. location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|eot|woff2|ttf|otf)$ { try_files $uri @rewrite =404; From 8904ba8b515957954c90c6450ade2965c9221dec Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 18 Jan 2022 16:10:33 +0100 Subject: [PATCH 08/22] Prometheus pr 1.x (#539) * Allowing 'gitLab' to disable Prometheus. * Booleans to use in jinja2 as strings must be cast as strings. * Tidying up CI and adding a GitLab test. * Fixing CI job description. * Adding a firewall config preset to open port 80 for LetsEncrypt. --- ce-dev/.gitignore | 4 +++- roles/firewall_config/defaults/main.yml | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ce-dev/.gitignore b/ce-dev/.gitignore index 990b2ccfc..3b8ced0db 100644 --- a/ce-dev/.gitignore +++ b/ce-dev/.gitignore @@ -3,4 +3,6 @@ docker-compose.yml ansible/local/* !ansible/local/.gitkeep ansible/vars/provision-target -ansible/plays/provision-target/*.yml \ No newline at end of file +ansible/plays/provision-target/*.yml +ansible/vars/provision-privileged +ansible/plays/provision-privileged/*.yml \ No newline at end of file diff --git a/roles/firewall_config/defaults/main.yml b/roles/firewall_config/defaults/main.yml index 6625b9181..29e143853 100644 --- a/roles/firewall_config/defaults/main.yml +++ b/roles/firewall_config/defaults/main.yml @@ -36,6 +36,9 @@ firewall_config: firewall_allowed_tcp_ports: - "989" - "990" + letsencrypt: + firewall_allowed_tcp_ports: + - "80" ossec: firewall_allowed_udp_ports: - "1514" From df86ca38db6418829029cde4bfacce461c2e210a Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 25 Jan 2022 13:33:56 +0100 Subject: [PATCH 09/22] Removing our unused ClamAV roles and adding a Galaxy role to common base. (#541) --- roles/_meta/common_base/meta/main.yml | 1 + roles/ce_provision/meta/requirements.yml | 1 + roles/clamav_clamscan/README.md | 22 - roles/clamav_clamscan/defaults/main.yml | 7 - roles/clamav_clamscan/tasks/main.yml | 25 - .../templates/clamscan-cron.j2 | 2 - .../templates/clamscan-report.j2 | 79 --- roles/clamav_daemon/README.md | 20 - roles/clamav_daemon/defaults/main.yml | 6 - roles/clamav_daemon/handlers/main.yml | 7 - roles/clamav_daemon/tasks/main.yml | 20 - roles/clamav_daemon/templates/clamd.conf.j2 | 605 ------------------ 12 files changed, 2 insertions(+), 793 deletions(-) delete mode 100644 roles/clamav_clamscan/README.md delete mode 100644 roles/clamav_clamscan/defaults/main.yml delete mode 100644 roles/clamav_clamscan/tasks/main.yml delete mode 100644 roles/clamav_clamscan/templates/clamscan-cron.j2 delete mode 100644 roles/clamav_clamscan/templates/clamscan-report.j2 delete mode 100644 roles/clamav_daemon/README.md delete mode 100644 roles/clamav_daemon/defaults/main.yml delete mode 100644 roles/clamav_daemon/handlers/main.yml delete mode 100644 roles/clamav_daemon/tasks/main.yml delete mode 100644 roles/clamav_daemon/templates/clamd.conf.j2 diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 86f97952c..f23097474 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -7,3 +7,4 @@ dependencies: - { role: hosts, when: ( is_local is not defined or not is_local ) } - role: rkhunter - role: postfix + - role: geerlingguy.clamav diff --git a/roles/ce_provision/meta/requirements.yml b/roles/ce_provision/meta/requirements.yml index 38b0d3b6a..1a062f40e 100644 --- a/roles/ce_provision/meta/requirements.yml +++ b/roles/ce_provision/meta/requirements.yml @@ -8,4 +8,5 @@ roles: - name: geerlingguy.varnish - name: geerlingguy.firewall - name: geerlingguy.composer + - name: geerlingguy.clamav - name: robertdebock.openvpn diff --git a/roles/clamav_clamscan/README.md b/roles/clamav_clamscan/README.md deleted file mode 100644 index adbac2052..000000000 --- a/roles/clamav_clamscan/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# ClamAV Clamscan - - - -## Configuration -This role will install the ClamAV base package which will allow us to run clamscan on demand and email scan reports to an email address. -NOTE: This approach will not install clamd. Check the clamav_daemon role if this is what you are looking for. - - -## Default variables -```yaml ---- -# defaults file for clamav - -clamav_clamscan: - email: "admins@example.com" - schedule: "0 0 * * *" - path: "/var/www" - -``` - - diff --git a/roles/clamav_clamscan/defaults/main.yml b/roles/clamav_clamscan/defaults/main.yml deleted file mode 100644 index 4531b0fa4..000000000 --- a/roles/clamav_clamscan/defaults/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# defaults file for clamav - -clamav_clamscan: - email: "admins@example.com" - schedule: "0 0 * * *" - path: "/var/www" diff --git a/roles/clamav_clamscan/tasks/main.yml b/roles/clamav_clamscan/tasks/main.yml deleted file mode 100644 index 07be70701..000000000 --- a/roles/clamav_clamscan/tasks/main.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# tasks file for clamav - -- name: Ensure ClamAV packages are installed - ansible.builtin.apt: - pkg: ["clamav"] - state: present - update_cache: true - cache_valid_time: 240 - -- name: Copy clamscan report config - ansible.builtin.template: - src: "clamscan-report.j2" - dest: "/usr/local/bin/clamscan-report" - owner: root - group: root - mode: 0700 - -- name: Copy clamscan cron config - ansible.builtin.template: - src: "clamscan-cron.j2" - dest: "/etc/cron.d/clamscan_cron" - owner: root - group: root - mode: 0644 diff --git a/roles/clamav_clamscan/templates/clamscan-cron.j2 b/roles/clamav_clamscan/templates/clamscan-cron.j2 deleted file mode 100644 index b5d68185e..000000000 --- a/roles/clamav_clamscan/templates/clamscan-cron.j2 +++ /dev/null @@ -1,2 +0,0 @@ -MAILTO="" -{{ clamav_clamscan.schedule }} root /usr/local/bin/clamscan-report -d {{ clamav_clamscan.path }} diff --git a/roles/clamav_clamscan/templates/clamscan-report.j2 b/roles/clamav_clamscan/templates/clamscan-report.j2 deleted file mode 100644 index 25d808e16..000000000 --- a/roles/clamav_clamscan/templates/clamscan-report.j2 +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -# Variables -TODAY=$(date +%y%m%d) -EMAIL="{{ clamav_clamscan.email }}" -SERVER=`hostname -f` - -# A simple color function to report errors in red -color() { - printf '\033[%sm%s\033[m\n' "$@" - # usage color "31;5" "error message" - # 0 default - # 5 blink, 1 strong, 4 underlined - # fg: 31 red, 32 green, 33 yellow, 34 blue, 35 purple, 36 cyan, 37 white - # bg: 40 black, 41 red, 44 blue, 45 purple -} - -usage() -{ -cat << EOF -usage: $0 ARGUMENT OPTIONS - -This script scans a directory with clamscan and e-mails -a report if it finds any infected files. - -ARGUMENTS: - -d Directory to scan - -h This help message. -EOF -} - -# Parse the args -while getopts ":d:h" OPTION -do - case $OPTION in - h) - usage - exit - ;; - d) - DIRECTORY=$OPTARG - ;; - ?) - usage - exit - ;; - esac -done - -which clamscan > /dev/null -if [ $? -eq 1 ]; then - echo "clamscan doesn't seem to be installed!" - exit 1 -fi - -# Check for appropriate arguments and options - -# No task provided -if [[ -z $DIRECTORY ]]; then - color '31;1' "You didn't provide a directory to scan! Pass one as an argument with -d" - usage - exit 1 -fi - -if [[ ! -d $DIRECTORY ]]; then - echo "That directory $DIRECTORY doesn't exist!" - exit -fi - -# Scan the directory and log to a file -clamscan -r -i --cross-fs=no --log=/var/log/clamscan-report-${TODAY}.log --quiet $DIRECTORY - -# Check the log to see if there were any infected files -grep -q "Infected files: 0" /var/log/clamscan-report-${TODAY}.log - -# If we found infected files, send an e-mail -if [ $? -eq 1 ]; then - cat /var/log/clamscan-report-${TODAY}.log | mail -s "ClamScan report for ${SERVER}" $EMAIL -fi diff --git a/roles/clamav_daemon/README.md b/roles/clamav_daemon/README.md deleted file mode 100644 index 00f9582d1..000000000 --- a/roles/clamav_daemon/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# ClamAV Daemon - - - -## Configuration -This role will install the ClamAV daemon. If you want to install clamscan and generate reports check the clamav_clamscan role. - - -## Default variables -```yaml ---- -# defaults file for clamav - -clamav_daemon: - host: "127.0.0.1" - port: "3310" - -``` - - diff --git a/roles/clamav_daemon/defaults/main.yml b/roles/clamav_daemon/defaults/main.yml deleted file mode 100644 index 113aa4363..000000000 --- a/roles/clamav_daemon/defaults/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# defaults file for clamav - -clamav_daemon: - host: "127.0.0.1" - port: "3310" diff --git a/roles/clamav_daemon/handlers/main.yml b/roles/clamav_daemon/handlers/main.yml deleted file mode 100644 index 75c6d105f..000000000 --- a/roles/clamav_daemon/handlers/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# handlers file for clamav - -- name: restart clamav - service: - name: "clamav-daemon" - state: restarted diff --git a/roles/clamav_daemon/tasks/main.yml b/roles/clamav_daemon/tasks/main.yml deleted file mode 100644 index d23e49050..000000000 --- a/roles/clamav_daemon/tasks/main.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- name: Ensure ClamAV packages are installed - ansible.builtin.apt: - pkg: ["clamav-daemon"] - state: present - update_cache: true - cache_valid_time: 240 - -- name: Copy main ClamAV config - ansible.builtin.template: - src: "clamd.conf.j2" - dest: "/etc/clamav/clamd.conf" - owner: root - group: root - mode: "0644" - notify: - - restart clamav - -- pause: - minutes: 1 diff --git a/roles/clamav_daemon/templates/clamd.conf.j2 b/roles/clamav_daemon/templates/clamd.conf.j2 deleted file mode 100644 index 55ccfc991..000000000 --- a/roles/clamav_daemon/templates/clamd.conf.j2 +++ /dev/null @@ -1,605 +0,0 @@ -## -## Example config file for the Clam AV daemon -## Please read the clamd.conf(5) manual before editing this file. -## - - -# Comment or remove the line below. -# Example - -# Uncomment this option to enable logging. -# LogFile must be writable for the user running daemon. -# A full path is required. -# Default: disabled -#LogFile /tmp/clamd.log - -# By default the log file is locked for writing - the lock protects against -# running clamd multiple times (if want to run another clamd, please -# copy the configuration file, change the LogFile variable, and run -# the daemon with --config-file option). -# This option disables log file locking. -# Default: no -#LogFileUnlock yes - -# Maximum size of the log file. -# Value of 0 disables the limit. -# You may use 'M' or 'm' for megabytes (1M = 1m = 1048576 bytes) -# and 'K' or 'k' for kilobytes (1K = 1k = 1024 bytes). To specify the size -# in bytes just don't use modifiers. If LogFileMaxSize is enabled, log -# rotation (the LogRotate option) will always be enabled. -# Default: 1M -#LogFileMaxSize 2M - -# Log time with each message. -# Default: no -LogTime yes - -# Also log clean files. Useful in debugging but drastically increases the -# log size. -# Default: no -#LogClean yes - -# Use system logger (can work together with LogFile). -# Default: no -LogSyslog yes - -# Specify the type of syslog messages - please refer to 'man syslog' -# for facility names. -# Default: LOG_LOCAL6 -#LogFacility LOG_MAIL - -# Enable verbose logging. -# Default: no -#LogVerbose yes - -# Enable log rotation. Always enabled when LogFileMaxSize is enabled. -# Default: no -#LogRotate yes - -# Log additional information about the infected file, such as its -# size and hash, together with the virus name. -#ExtendedDetectionInfo yes - -# This option allows you to save a process identifier of the listening -# daemon (main thread). -# Default: disabled -# PidFile /var/run/clamd.pid - -# Optional path to the global temporary directory. -# Default: system specific (usually /tmp or /var/tmp). -#TemporaryDirectory /var/tmp - -# Path to the database directory. -# Default: hardcoded (depends on installation options) -#DatabaseDirectory /var/lib/clamav - -# Only load the official signatures published by the ClamAV project. -# Default: no -#OfficialDatabaseOnly no - -# The daemon can work in local mode, network mode or both. -# Due to security reasons we recommend the local mode. - -# Path to a local socket file the daemon will listen on. -# Default: disabled (must be specified by a user) -#LocalSocket /tmp/clamd.socket - -# Sets the group ownership on the unix socket. -# Default: disabled (the primary group of the user running clamd) -#LocalSocketGroup virusgroup - -# Sets the permissions on the unix socket to the specified mode. -# Default: disabled (socket is world accessible) -#LocalSocketMode 660 - -# Remove stale socket after unclean shutdown. -# Default: true -#FixStaleSocket yes - -# TCP port address. -# Default: no -TCPSocket {{ clamav_daemon.port }} - -# TCP address. -# By default we bind to INADDR_ANY, probably not wise. -# Enable the following to provide some degree of protection -# from the outside world. This option can be specified multiple -# times if you want to listen on multiple IPs. IPv6 is now supported. -# Default: no - -TCPAddr {{ clamav_daemon.host }} - -# Maximum length the queue of pending connections may grow to. -# Default: 200 -#MaxConnectionQueueLength 30 - -# Clamd uses FTP-like protocol to receive data from remote clients. -# If you are using clamav-milter to balance load between remote clamd daemons -# on firewall servers you may need to tune the options below. - -# Close the connection when the data size limit is exceeded. -# The value should match your MTA's limit for a maximum attachment size. -# Default: 25M -#StreamMaxLength 10M - -# Limit port range. -# Default: 1024 -#StreamMinPort 30000 -# Default: 2048 -#StreamMaxPort 32000 - -# Maximum number of threads running at the same time. -# Default: 10 -#MaxThreads 20 - -# Waiting for data from a client socket will timeout after this time (seconds). -# Default: 120 -#ReadTimeout 300 - -# This option specifies the time (in seconds) after which clamd should -# timeout if a client doesn't provide any initial command after connecting. -# Default: 5 -#CommandReadTimeout 5 - -# This option specifies how long to wait (in miliseconds) if the send buffer is full. -# Keep this value low to prevent clamd hanging -# -# Default: 500 -#SendBufTimeout 200 - -# Maximum number of queued items (including those being processed by MaxThreads threads) -# It is recommended to have this value at least twice MaxThreads if possible. -# WARNING: you shouldn't increase this too much to avoid running out of file descriptors, -# the following condition should hold: -# MaxThreads*MaxRecursion + (MaxQueue - MaxThreads) + 6< RLIMIT_NOFILE (usual max is 1024) -# -# Default: 100 -#MaxQueue 200 - -# Waiting for a new job will timeout after this time (seconds). -# Default: 30 -#IdleTimeout 60 - -# Don't scan files and directories matching regex -# This directive can be used multiple times -# Default: scan all -#ExcludePath ^/proc/ -#ExcludePath ^/sys/ - -# Maximum depth directories are scanned at. -# Default: 15 -#MaxDirectoryRecursion 20 - -# Follow directory symlinks. -# Default: no -#FollowDirectorySymlinks yes - -# Follow regular file symlinks. -# Default: no -#FollowFileSymlinks yes - -# Scan files and directories on other filesystems. -# Default: true -#CrossFilesystems yes - -# Perform a database check. -# Default: 600 (10 min) -#SelfCheck 600 - -# Execute a command when virus is found. In the command string %v will -# be replaced with the virus name. -# Default: no -#VirusEvent /usr/local/bin/send_sms 123456789 "VIRUS ALERT: %v" - -# Run as another user (clamd must be started by root for this option to work) -# Default: don't drop privileges -User clamav - -# Initialize supplementary group access (clamd must be started by root). -# Default: no -#AllowSupplementaryGroups no - -# Stop daemon when libclamav reports out of memory condition. -#ExitOnOOM yes - -# Don't fork into background. -# Default: no -#Foreground yes - -# Enable debug messages in libclamav. -# Default: no -#Debug yes - -# Do not remove temporary files (for debug purposes). -# Default: no -#LeaveTemporaryFiles yes - -# Permit use of the ALLMATCHSCAN command. If set to no, clamd will reject -# any ALLMATCHSCAN command as invalid. -# Default: true -#AllowAllMatchScan no - -# Detect Possibly Unwanted Applications. -# Default: no -#DetectPUA yes - -# Exclude a specific PUA category. This directive can be used multiple times. -# See https://github.com/vrtadmin/clamav-faq/blob/master/faq/faq-pua.md for -# the complete list of PUA categories. -# Default: Load all categories (if DetectPUA is activated) -#ExcludePUA NetTool -#ExcludePUA PWTool - -# Only include a specific PUA category. This directive can be used multiple -# times. -# Default: Load all categories (if DetectPUA is activated) -#IncludePUA Spy -#IncludePUA Scanner -#IncludePUA RAT - -# In some cases (eg. complex malware, exploits in graphic files, and others), -# ClamAV uses special algorithms to provide accurate detection. This option -# controls the algorithmic detection. -# Default: true -#AlgorithmicDetection yes - -# This option causes memory or nested map scans to dump the content to disk. -# If you turn on this option, more data is written to disk and is available -# when the LeaveTemporaryFiles option is enabled. -#ForceToDisk yes - -# This option allows you to disable the caching feature of the engine. By -# default, the engine will store an MD5 in a cache of any files that are -# not flagged as virus or that hit limits checks. Disabling the cache will -# have a negative performance impact on large scans. -# Default: no -#DisableCache yes - -## -## Executable files -## - -# PE stands for Portable Executable - it's an executable file format used -# in all 32 and 64-bit versions of Windows operating systems. This option allows -# ClamAV to perform a deeper analysis of executable files and it's also -# required for decompression of popular executable packers such as UPX, FSG, -# and Petite. If you turn off this option, the original files will still be -# scanned, but without additional processing. -# Default: true -#ScanPE yes - -# Certain PE files contain an authenticode signature. By default, we check -# the signature chain in the PE file against a database of trusted and -# revoked certificates if the file being scanned is marked as a virus. -# If any certificate in the chain validates against any trusted root, but -# does not match any revoked certificate, the file is marked as whitelisted. -# If the file does match a revoked certificate, the file is marked as virus. -# The following setting completely turns off authenticode verification. -# Default: no -#DisableCertCheck yes - -# Executable and Linking Format is a standard format for UN*X executables. -# This option allows you to control the scanning of ELF files. -# If you turn off this option, the original files will still be scanned, but -# without additional processing. -# Default: true -#ScanELF yes - -# With this option clamav will try to detect broken executables (both PE and -# ELF) and mark them as Broken.Executable. -# Default: no -#DetectBrokenExecutables yes - - -## -## Documents -## - -# This option enables scanning of OLE2 files, such as Microsoft Office -# documents and .msi files. -# If you turn off this option, the original files will still be scanned, but -# without additional processing. -# Default: true -#ScanOLE2 yes - -# With this option enabled OLE2 files with VBA macros, which were not -# detected by signatures will be marked as "Heuristics.OLE2.ContainsMacros". -# Default: no -#OLE2BlockMacros no - -# This option enables scanning within PDF files. -# If you turn off this option, the original files will still be scanned, but -# without decoding and additional processing. -# Default: true -#ScanPDF yes - -# This option enables scanning within SWF files. -# If you turn off this option, the original files will still be scanned, but -# without decoding and additional processing. -# Default: true -#ScanSWF yes - - -## -## Mail files -## - -# Enable internal e-mail scanner. -# If you turn off this option, the original files will still be scanned, but -# without parsing individual messages/attachments. -# Default: true -#ScanMail yes - -# Scan RFC1341 messages split over many emails. -# You will need to periodically clean up $TemporaryDirectory/clamav-partial directory. -# WARNING: This option may open your system to a DoS attack. -# Never use it on loaded servers. -# Default: no -#ScanPartialMessages yes - -# With this option enabled ClamAV will try to detect phishing attempts by using -# signatures. -# Default: true -#PhishingSignatures yes - -# Scan URLs found in mails for phishing attempts using heuristics. -# Default: true -#PhishingScanURLs yes - -# Always block SSL mismatches in URLs, even if the URL isn't in the database. -# This can lead to false positives. -# -# Default: no -#PhishingAlwaysBlockSSLMismatch no - -# Always block cloaked URLs, even if URL isn't in database. -# This can lead to false positives. -# -# Default: no -#PhishingAlwaysBlockCloak no - -# Detect partition intersections in raw disk images using heuristics. -# Default: no -#PartitionIntersection no - -# Allow heuristic match to take precedence. -# When enabled, if a heuristic scan (such as phishingScan) detects -# a possible virus/phish it will stop scan immediately. Recommended, saves CPU -# scan-time. -# When disabled, virus/phish detected by heuristic scans will be reported only at -# the end of a scan. If an archive contains both a heuristically detected -# virus/phish, and a real malware, the real malware will be reported -# -# Keep this disabled if you intend to handle "*.Heuristics.*" viruses -# differently from "real" malware. -# If a non-heuristically-detected virus (signature-based) is found first, -# the scan is interrupted immediately, regardless of this config option. -# -# Default: no -#HeuristicScanPrecedence yes - - -## -## Data Loss Prevention (DLP) -## - -# Enable the DLP module -# Default: No -#StructuredDataDetection yes - -# This option sets the lowest number of Credit Card numbers found in a file -# to generate a detect. -# Default: 3 -#StructuredMinCreditCardCount 5 - -# This option sets the lowest number of Social Security Numbers found -# in a file to generate a detect. -# Default: 3 -#StructuredMinSSNCount 5 - -# With this option enabled the DLP module will search for valid -# SSNs formatted as xxx-yy-zzzz -# Default: true -#StructuredSSNFormatNormal yes - -# With this option enabled the DLP module will search for valid -# SSNs formatted as xxxyyzzzz -# Default: no -#StructuredSSNFormatStripped yes - - -## -## HTML -## - -# Perform HTML normalisation and decryption of MS Script Encoder code. -# Default: true -# If you turn off this option, the original files will still be scanned, but -# without additional processing. -#ScanHTML yes - - -## -## Archives -## - -# ClamAV can scan within archives and compressed files. -# If you turn off this option, the original files will still be scanned, but -# without unpacking and additional processing. -# Default: true -#ScanArchive yes - -# Mark encrypted archives as viruses (Encrypted.Zip, Encrypted.RAR). -# Default: no -#ArchiveBlockEncrypted no - - -## -## Limits -## - -# The options below protect your system against Denial of Service attacks -# using archive bombs. - -# This option sets the maximum amount of data to be scanned for each input file. -# Archives and other containers are recursively extracted and scanned up to this -# value. -# Value of 0 disables the limit -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 100M -#MaxScanSize 150M - -# Files larger than this limit won't be scanned. Affects the input file itself -# as well as files contained inside it (when the input file is an archive, a -# document or some other kind of container). -# Value of 0 disables the limit. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 25M -#MaxFileSize 30M - -# Nested archives are scanned recursively, e.g. if a Zip archive contains a RAR -# file, all files within it will also be scanned. This options specifies how -# deeply the process should be continued. -# Note: setting this limit too high may result in severe damage to the system. -# Default: 16 -#MaxRecursion 10 - -# Number of files to be scanned within an archive, a document, or any other -# container file. -# Value of 0 disables the limit. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 10000 -#MaxFiles 15000 - -# Maximum size of a file to check for embedded PE. Files larger than this value -# will skip the additional analysis step. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 10M -#MaxEmbeddedPE 10M - -# Maximum size of a HTML file to normalize. HTML files larger than this value -# will not be normalized or scanned. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 10M -#MaxHTMLNormalize 10M - -# Maximum size of a normalized HTML file to scan. HTML files larger than this -# value after normalization will not be scanned. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 2M -#MaxHTMLNoTags 2M - -# Maximum size of a script file to normalize. Script content larger than this -# value will not be normalized or scanned. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 5M -#MaxScriptNormalize 5M - -# Maximum size of a ZIP file to reanalyze type recognition. ZIP files larger -# than this value will skip the step to potentially reanalyze as PE. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 1M -#MaxZipTypeRcg 1M - -# This option sets the maximum number of partitions of a raw disk image to be scanned. -# Raw disk images with more partitions than this value will have up to the value number -# partitions scanned. Negative values are not allowed. -# Note: setting this limit too high may result in severe damage or impact performance. -# Default: 50 -#MaxPartitions 128 - -# This option sets the maximum number of icons within a PE to be scanned. -# PE files with more icons than this value will have up to the value number icons scanned. -# Negative values are not allowed. -# WARNING: setting this limit too high may result in severe damage or impact performance. -# Default: 100 -#MaxIconsPE 200 - -## -## On-access Scan Settings -## - -# Enable on-access scanning. Currently, this is supported via fanotify. -# Clamuko/Dazuko support has been deprecated. -# Default: no -#ScanOnAccess yes - -# Don't scan files larger than OnAccessMaxFileSize -# Value of 0 disables the limit. -# Default: 5M -#OnAccessMaxFileSize 10M - -# Set the include paths (all files inside them will be scanned). You can have -# multiple OnAccessIncludePath directives but each directory must be added -# in a separate line. (On-access scan only) -# Default: disabled -#OnAccessIncludePath /home -#OnAccessIncludePath /students - -# Set the exclude paths. All subdirectories are also excluded. -# (On-access scan only) -# Default: disabled -#OnAccessExcludePath /home/bofh - -# With this option you can whitelist specific UIDs. Processes with these UIDs -# will be able to access all files. -# This option can be used multiple times (one per line). -# Default: disabled -#OnAccessExcludeUID 0 - - -## -## Bytecode -## - -# With this option enabled ClamAV will load bytecode from the database. -# It is highly recommended you keep this option on, otherwise you'll miss detections for many new viruses. -# Default: true -#Bytecode yes - -# Set bytecode security level. -# Possible values: -# None - no security at all, meant for debugging. DO NOT USE THIS ON PRODUCTION SYSTEMS -# This value is only available if clamav was built with --enable-debug! -# TrustSigned - trust bytecode loaded from signed .c[lv]d files, -# insert runtime safety checks for bytecode loaded from other sources -# Paranoid - don't trust any bytecode, insert runtime checks for all -# Recommended: TrustSigned, because bytecode in .cvd files already has these checks -# Note that by default only signed bytecode is loaded, currently you can only -# load unsigned bytecode in --enable-debug mode. -# -# Default: TrustSigned -#BytecodeSecurity TrustSigned - -# Set bytecode timeout in miliseconds. -# -# Default: 5000 -# BytecodeTimeout 1000 - -## -## Statistics gathering and submitting -## - -# Enable statistical reporting. -# Default: no -#StatsEnabled yes - -# Disable submission of individual PE sections for files flagged as malware. -# Default: no -#StatsPEDisabled yes - -# HostID in the form of an UUID to use when submitting statistical information. -# Default: auto -#StatsHostID auto - -# Time in seconds to wait for the stats server to come back with a response -# Default: 10 -#StatsTimeout 10 From c0e8b06076a9bc5c17522a54e434805769972050 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 27 Jan 2022 11:43:39 +0100 Subject: [PATCH 10/22] Revert "Moving OSSEC pkill to use process_manager role instead. (#258)" (#544) This reverts commit 73c7bd0adb1105436e484fe794182c915b2d25dd. --- roles/ossec/handlers/main.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/roles/ossec/handlers/main.yml b/roles/ossec/handlers/main.yml index e32a1dd97..3aaa40de4 100644 --- a/roles/ossec/handlers/main.yml +++ b/roles/ossec/handlers/main.yml @@ -7,11 +7,8 @@ state: restarted - name: stop ossec-authd - ansible.builtin.include_role: - name: process_manager - vars: - process_manager: - process_name: ossec-authd + ignore_errors: true + command: "pkill ossec-authd" - name: start ossec-authd - command: "/var/ossec/bin/ossec-authd -k {{ ssl_facts[ossec.ssl.domain].key }} -x {{ ssl_facts[ossec.ssl.domain].certificate }}" + command: "/var/ossec/bin/ossec-authd -k {{ ssl_facts[ossec.ssl.domain].key }} -x {{ ssl_facts[ossec.ssl.domain].certificate }}" \ No newline at end of file From b8a1f6091a65218207426acf9a939382951f540b Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 28 Jan 2022 17:25:38 +0100 Subject: [PATCH 11/22] Backing out of Packer logging. --- roles/aws/aws_ami/tasks/create.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/aws/aws_ami/tasks/create.yml b/roles/aws/aws_ami/tasks/create.yml index 3f08beb73..88cdadbcb 100644 --- a/roles/aws/aws_ami/tasks/create.yml +++ b/roles/aws/aws_ami/tasks/create.yml @@ -14,5 +14,3 @@ ansible.builtin.command: cmd: "packer build {{ _ce_provision_build_tmp_dir }}/packer.json" chdir: "{{ _ce_provision_base_dir }}" - environment: - PACKER_LOG: "{{ packer_log }}" \ No newline at end of file From 9b41cf720a42aa5f7c459da09417c00b936040d5 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 3 Feb 2022 12:03:48 +0100 Subject: [PATCH 12/22] Moving key servers to a variable so we can set them. (#555) * Moving key servers to a variable so we can set them. * Allowing us to disable sending keys completely. * Oops, doubled up on existing functionality. * Fixing var name. --- roles/gpg_key/defaults/main.yml | 3 +++ roles/gpg_key/tasks/gpg.yml | 6 +----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/roles/gpg_key/defaults/main.yml b/roles/gpg_key/defaults/main.yml index fc67f7f4c..dccbcf101 100644 --- a/roles/gpg_key/defaults/main.yml +++ b/roles/gpg_key/defaults/main.yml @@ -1,4 +1,7 @@ --- +gpg_key_servers: + - hkps://pgp.mit.edu + - hkps://keys.openpgp.org gpg_key: - username: example # Must exist already on the server. publish: false # Whether to publish to HKS public servers. diff --git a/roles/gpg_key/tasks/gpg.yml b/roles/gpg_key/tasks/gpg.yml index fdfc4e345..9983f5a5b 100644 --- a/roles/gpg_key/tasks/gpg.yml +++ b/roles/gpg_key/tasks/gpg.yml @@ -46,10 +46,6 @@ when: - gpg_user.publish - not user_ansible_gpg_key_exists.stdout - with_items: - - hkp://hkps.pool.sks-keyservers.net - - hkps://pgp.mit.edu - - hkps://keys.openpgp.org - - hkp://keys.gnupg.net + with_items: "{{ gpg_key_servers }}" loop_control: loop_var: server From 7804c19a66b6203459fd288fdbaf21fa223f74f4 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 7 Feb 2022 13:49:15 +0100 Subject: [PATCH 13/22] Adding a reboot option to the patching role. (#557) --- roles/ce_patcher/defaults/main.yml | 3 ++- roles/ce_patcher/tasks/main.yml | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/roles/ce_patcher/defaults/main.yml b/roles/ce_patcher/defaults/main.yml index 5dd020e11..631deee83 100644 --- a/roles/ce_patcher/defaults/main.yml +++ b/roles/ce_patcher/defaults/main.yml @@ -1,3 +1,4 @@ --- ce_patcher: - username: controller \ No newline at end of file + username: controller + reboot: false \ No newline at end of file diff --git a/roles/ce_patcher/tasks/main.yml b/roles/ce_patcher/tasks/main.yml index 1e401b6db..a4d6eecfc 100644 --- a/roles/ce_patcher/tasks/main.yml +++ b/roles/ce_patcher/tasks/main.yml @@ -11,3 +11,10 @@ autoremove: true become: true remote_user: "{{ ce_patcher.username }}" + +- name: Reboot the server. + ansible.builtin.reboot: + post_reboot_delay: 30 + become: true + remote_user: "{{ ce_patcher.username }}" + when: ce_patcher.reboot From 71d1ea762ef2fb984e61c2be2a85bfe3594f2973 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Mon, 7 Feb 2022 13:04:34 +0000 Subject: [PATCH 14/22] Add minimal support for Aurora RDS instances (#567) * Attempt to create an RDS read replica. * Use new task to create Aurora RDS instances. * Try and fix linting issues. * Don't pass max_storage variable for Aurora instances. * Remove more storage related vars from Aurora RDS instance creation task. * Add profile and region to read replica creation. * Try creating the Aurora read replica another way. * Add some debug info. * Work around the silly registering of variables in Ansible. * Rename an RDS CloudWatch task for Aurora DBs and remove RDS debug info. * Add some Aurora info to aws_rds README file. * Use reader instead of replica for Aurora readers. * Remove db_cluster_identifier variable from non-Aurora RDS task. --- roles/aws/aws_rds/README.md | 9 ++++ roles/aws/aws_rds/defaults/main.yml | 1 + roles/aws/aws_rds/tasks/main.yml | 67 +++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/roles/aws/aws_rds/README.md b/roles/aws/aws_rds/README.md index a1ad86e2d..b448714aa 100644 --- a/roles/aws/aws_rds/README.md +++ b/roles/aws/aws_rds/README.md @@ -1,5 +1,14 @@ # AWS RDS Creates an RDS instance and associated ressources. + +If the `engine` variable is set to **aurora-mysql**, you'll need to manually create the Aurora cluster first. Typically, a controller will already exist, so something like this can be run from the controller: + +``` +AWS_PROFILE=example aws rds create-db-cluster --db-cluster-identifier example-aurora-cluster --engine aurora-mysql --engine-version 5.7.mysql_aurora.2.10.2 --db-subnet-group-name example-aurora --vpc-security-group-ids sg-abcdefghijklmnop --storage-encrypted --master-username "auroradev" --master-user-password "aurora12345" +``` + +You'll need to have created the subnet group first as well as the security groups. + diff --git a/roles/aws/aws_rds/defaults/main.yml b/roles/aws/aws_rds/defaults/main.yml index 0b15cc247..f5bb2a831 100644 --- a/roles/aws/aws_rds/defaults/main.yml +++ b/roles/aws/aws_rds/defaults/main.yml @@ -11,6 +11,7 @@ aws_rds: state: present description: example engine: mariadb + aurora_reader: false # If true, an Aurora reader instance will be created. # engine_version: '5.7.2' # Omit to use latest. # See parameter group docs: https://docs.ansible.com/ansible/latest/collections/community/aws/rds_param_group_module.html # db_parameter_group_name: "example" # Omit to use default diff --git a/roles/aws/aws_rds/tasks/main.yml b/roles/aws/aws_rds/tasks/main.yml index 4b398d1e3..ac7bd67d5 100644 --- a/roles/aws/aws_rds/tasks/main.yml +++ b/roles/aws/aws_rds/tasks/main.yml @@ -28,6 +28,45 @@ tags: "{{ aws_rds.tags | combine({ 'Name': aws_rds.name }) }}" when: aws_rds.db_parameters is defined +- name: Create Aurora RDS instance. + community.aws.rds_instance: + db_instance_identifier: "{{ aws_rds.name }}" + db_instance_class: "{{ aws_rds.db_instance_class }}" + db_cluster_identifier: "{{ aws_rds.db_cluster_identifier }}" + state: "{{ aws_rds.state }}" + engine: "{{ aws_rds.engine }}" + copy_tags_to_snapshot: true + publicly_accessible: "{{ aws_rds.publicly_accessible }}" + profile: "{{ aws_rds.aws_profile }}" + tags: "{{ aws_rds.tags | combine({ 'Name': aws_rds.name }) }}" + allow_major_version_upgrade: false + apply_immediately: true + wait: true + region: "{{ aws_rds.region }}" + engine_version: "{{ aws_rds.engine_version | default(omit) }}" + register: _rds_instance_info_aurora + when: "'aurora' in aws_rds.engine" + +- name: Create Aurora RDS reader instance. + community.aws.rds_instance: + db_instance_identifier: "{{ aws_rds.name }}-reader" + db_cluster_identifier: "{{ aws_rds.db_cluster_identifier }}" + db_instance_class: "{{ aws_rds.db_instance_class }}" + state: "{{ aws_rds.state }}" + engine: "{{ aws_rds.engine }}" + copy_tags_to_snapshot: true + publicly_accessible: "{{ aws_rds.publicly_accessible }}" + profile: "{{ aws_rds.aws_profile }}" + tags: "{{ aws_rds.tags | combine({ 'Name': aws_rds.name + '-reader' }) }}" + allow_major_version_upgrade: false + apply_immediately: true + wait: true + region: "{{ aws_rds.region }}" + engine_version: "{{ aws_rds.engine_version | default(omit) }}" + when: + - "'aurora' in aws_rds.engine" + - aws_rds.aurora_reader + - name: Create RDS instance community.aws.rds_instance: db_instance_identifier: "{{ aws_rds.name }}" @@ -60,6 +99,7 @@ engine_version: "{{ aws_rds.engine_version | default(omit) }}" #preferred_maintenance_window: undefined # not required. The weekly time range (in UTC) of at least 30 minutes, during which system maintenance can occur. The option must be in the format "ddd:hh24:mi-ddd:hh24:mi" where ddd is one of Mon, Tue, Wed, Thu, Fri, Sat, Sun. register: _rds_instance_info + when: "'aurora' not in aws_rds.engine" - name: Create SNS topic. ansible.builtin.include_role: @@ -116,6 +156,33 @@ with_items: "{{ aws_rds.rds_cloudwatch_alarms }}" when: - aws_rds.rds_cloudwatch_alarms + - _rds_instance_info.db_instance_identifier is defined + +- name: Create Aurora RDS CloudWatch alarms. + ansible.builtin.include_role: + name: aws/aws_ec2_metric_alarm + vars: + aws_ec2_metric_alarm: + aws_profile: "{{ aws_ec2_autoscale_cluster.aws_profile }}" + region: "{{ aws_ec2_autoscale_cluster.region }}" + name: "{{ _rds_instance_info_aurora.db_instance_identifier }}_{{ item.name }}" + description: "{{ item.description }}" + metric: "{{ item.metric }}" + namespace: "{{ item.namespace }}" + statistic: "{{ item.statistic }}" + comparison: "{{ item.comparison }}" + threshold: "{{ item.threshold }}" + unit: "{{ item.unit }}" + period: "{{ item.period }}" + evaluation_periods: "{{ item.evaluation_periods }}" + alarm_actions: "{{ _alarm_actions | default([]) }}" + ok_actions: "{{ _alarm_actions | default([]) }}" + dimensions: + "DBInstanceIdentifier": "{{ _rds_instance_info_aurora.db_instance_identifier }}" + with_items: "{{ aws_rds.rds_cloudwatch_alarms }}" + when: + - aws_rds.rds_cloudwatch_alarms + - _rds_instance_info_aurora.db_instance_identifier is defined - name: (Re) set facts for instance. ansible.builtin.set_fact: From 81ee940c333b56091ba24a2806b6acd730389fce Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 8 Feb 2022 11:50:13 +0100 Subject: [PATCH 15/22] Gpg servers fix pr 1.x (#571) * Moving key servers to a variable so we can set them. * Allowing us to disable sending keys completely. * Oops, doubled up on existing functionality. * Fixing var name. * Using a pipe to grep with 'command' cannot work, refactoring. * Making CI use the meta deploy role to test gitlab. * We mustn't assume AWS servers for deploy and controller. --- ce-dev/ansible/plays/gitlab/gitlab.yml | 4 +--- roles/_meta/controller/meta/main.yml | 1 - roles/_meta/deploy/meta/main.yml | 1 - roles/gpg_key/tasks/gpg.yml | 6 +++--- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ce-dev/ansible/plays/gitlab/gitlab.yml b/ce-dev/ansible/plays/gitlab/gitlab.yml index 921608205..4c4afe965 100644 --- a/ce-dev/ansible/plays/gitlab/gitlab.yml +++ b/ce-dev/ansible/plays/gitlab/gitlab.yml @@ -16,8 +16,6 @@ - ansible.builtin.import_role: name: _init - ansible.builtin.import_role: - name: gitlab - - ansible.builtin.import_role: - name: haproxy + name: _meta/deploy - ansible.builtin.import_role: name: _exit diff --git a/roles/_meta/controller/meta/main.yml b/roles/_meta/controller/meta/main.yml index 91b3d8b32..ac2d9c58c 100644 --- a/roles/_meta/controller/meta/main.yml +++ b/roles/_meta/controller/meta/main.yml @@ -2,7 +2,6 @@ dependencies: - role: user_provision - role: _meta/common_base - - role: _meta/aws_client_instance - role: aws/aws_credentials - role: ssh_server - role: ce_provision diff --git a/roles/_meta/deploy/meta/main.yml b/roles/_meta/deploy/meta/main.yml index ab039886e..6f7f4cf66 100644 --- a/roles/_meta/deploy/meta/main.yml +++ b/roles/_meta/deploy/meta/main.yml @@ -2,7 +2,6 @@ dependencies: - role: user_provision - role: _meta/common_base - - role: _meta/aws_client_instance - role: ce_deploy - role: aws/aws_credentials - role: gitlab diff --git a/roles/gpg_key/tasks/gpg.yml b/roles/gpg_key/tasks/gpg.yml index 9983f5a5b..f779e870d 100644 --- a/roles/gpg_key/tasks/gpg.yml +++ b/roles/gpg_key/tasks/gpg.yml @@ -30,17 +30,17 @@ - name: Gather key ID. ansible.builtin.command: - cmd: "gpg --list-secret-keys {{ gpg_user.email }} | grep '^ '" + cmd: "gpg --list-secret-keys {{ gpg_user.email }}" become: true become_user: "{{ gpg_user.username }}" - register: user_ansible_gpg_key_id + register: user_ansible_gpg_key when: - gpg_user.publish - not user_ansible_gpg_key_exists.stdout - name: Publish GPG key to openpgp.org. ansible.builtin.command: - cmd: "gpg --keyserver {{ server }} --send-keys {{ user_ansible_gpg_key_id.stdout }}" + cmd: "gpg --keyserver {{ server }} --send-keys {{ user_ansible_gpg_key.stdout_lines[1] }}" become: true become_user: "{{ gpg_user.username }}" when: From c268b547147da3035354acc98ef90b518a7ddb38 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 8 Feb 2022 12:39:59 +0100 Subject: [PATCH 16/22] Support termination protection in EC2. (#573) * Support termination protection in EC2. * Fixing CI vars. * Fixing CI vars. --- ce-dev/ansible/vars/gitlab/ce_deploy.yml | 6 ++++++ ce-dev/ansible/vars/gitlab/haproxy.yml | 7 ------- roles/aws/aws_ec2_with_eip/defaults/main.yml | 1 + roles/aws/aws_ec2_with_eip/tasks/main.yml | 1 + 4 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 ce-dev/ansible/vars/gitlab/ce_deploy.yml delete mode 100644 ce-dev/ansible/vars/gitlab/haproxy.yml diff --git a/ce-dev/ansible/vars/gitlab/ce_deploy.yml b/ce-dev/ansible/vars/gitlab/ce_deploy.yml new file mode 100644 index 000000000..675c45cc6 --- /dev/null +++ b/ce-dev/ansible/vars/gitlab/ce_deploy.yml @@ -0,0 +1,6 @@ +ce_deploy: + own_repository: https://github.com/codeenigma/ce-deploy.git + own_repository_branch: 1.x + username: deploy + utility_username: deploy + local_dir: /home/deploy/ce-deploy \ No newline at end of file diff --git a/ce-dev/ansible/vars/gitlab/haproxy.yml b/ce-dev/ansible/vars/gitlab/haproxy.yml deleted file mode 100644 index 8a7186e32..000000000 --- a/ce-dev/ansible/vars/gitlab/haproxy.yml +++ /dev/null @@ -1,7 +0,0 @@ -haproxy: - template: "haproxy-ssl-passthru" - backends: - - name: gitlab - ip: 127.0.0.1 - port: 8881 - domain: "provision-privileged.local" diff --git a/roles/aws/aws_ec2_with_eip/defaults/main.yml b/roles/aws/aws_ec2_with_eip/defaults/main.yml index 9b1d0a565..02a9c1dcd 100644 --- a/roles/aws/aws_ec2_with_eip/defaults/main.yml +++ b/roles/aws/aws_ec2_with_eip/defaults/main.yml @@ -10,6 +10,7 @@ aws_ec2_with_eip: # An IAM Role name to associate with the instance. iam_role_name: "example" state: started + termination_protection: false # set to true to disable termination and avoid accidents instance_name: "{{ _domain_name }}" root_volume_size: 80 ebs_optimized: true diff --git a/roles/aws/aws_ec2_with_eip/tasks/main.yml b/roles/aws/aws_ec2_with_eip/tasks/main.yml index fd2030f27..338e04376 100644 --- a/roles/aws/aws_ec2_with_eip/tasks/main.yml +++ b/roles/aws/aws_ec2_with_eip/tasks/main.yml @@ -29,6 +29,7 @@ image_id: "{{ aws_ec2_with_eip_image_latest.image_id }}" state: "{{ aws_ec2_with_eip.state }}" wait: true + termination_protection: "{{ aws_ec2_with_eip.termination_protection }}" vpc_subnet_id: "{{ aws_ec2_with_eip.vpc_subnet_id }}" name: "{{ aws_ec2_with_eip.instance_name }}" tags: "{{ aws_ec2_with_eip.tags }}" From 43eaefa3d445e071244b02f5a9420a0f659b0dcd Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 8 Feb 2022 12:19:07 +0000 Subject: [PATCH 17/22] Fix managed SSL key perms and the variable used for the private key. (#575) --- roles/ssl/tasks/manual.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/ssl/tasks/manual.yml b/roles/ssl/tasks/manual.yml index 9ad497b86..db553d5df 100644 --- a/roles/ssl/tasks/manual.yml +++ b/roles/ssl/tasks/manual.yml @@ -47,11 +47,11 @@ - name: Copy private key file to destination ansible.builtin.copy: - content: "{{ ssl.private_key }}" + content: "{{ ssl.key }}" dest: "{{ _ssl_current_key_path }}" owner: root group: root - mode: 0644 + mode: 0400 when: not ssl_key.stat.exists - name: Add key/cert pairs to know paths information. From f61e540d6d3dda10f3c6e3d928124deb732be80a Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 10 Feb 2022 12:48:35 +0100 Subject: [PATCH 18/22] Ec2 subnet lookup pr 1.x (#583) * First pass at EC2 subnet detection. * Touching subnet file to ensure it exists. * Trying a different approach, file module didn't work. * Switching back to file module. * We need to create the directory for new servers too. * Bad variable name. --- roles/aws/aws_ec2_with_eip/defaults/main.yml | 4 +- roles/aws/aws_ec2_with_eip/tasks/main.yml | 62 ++++++++++++++++++- .../templates/ec2-subnet-id.j2 | 1 + 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 roles/aws/aws_ec2_with_eip/templates/ec2-subnet-id.j2 diff --git a/roles/aws/aws_ec2_with_eip/defaults/main.yml b/roles/aws/aws_ec2_with_eip/defaults/main.yml index 02a9c1dcd..72c90c231 100644 --- a/roles/aws/aws_ec2_with_eip/defaults/main.yml +++ b/roles/aws/aws_ec2_with_eip/defaults/main.yml @@ -6,7 +6,9 @@ aws_ec2_with_eip: key_name: "{{ ce_provision.username }}@{{ ansible_hostname }}" # This needs to match your "provision" user SSH key. ami_name: "{{ _domain_name }}" # The name of an AMI image to use. Image must exists in the same region. ami_owner: self # Default to self-created image. - vpc_subnet_id: subnet-xxx + # vpc_subnet_id: subnet-xxx # One of vpc_subnet_id or vpc_name + vpc_subnet_profile is mandatory. + vpc_name: "{{ _infra_name }}" + vpc_subnet_profile: core # if you are looking up subnets we need a Profile tag to search against # An IAM Role name to associate with the instance. iam_role_name: "example" state: started diff --git a/roles/aws/aws_ec2_with_eip/tasks/main.yml b/roles/aws/aws_ec2_with_eip/tasks/main.yml index 338e04376..8e2741d40 100644 --- a/roles/aws/aws_ec2_with_eip/tasks/main.yml +++ b/roles/aws/aws_ec2_with_eip/tasks/main.yml @@ -19,6 +19,66 @@ name: "{{ aws_ec2_with_eip.iam_role_name }}" register: _aws_ec2_with_eip_iam_role_info +# Subnet ID is stored in ce-provision's data directory +- name: Ensure server data directory exists. + ansible.builtin.file: + path: "{{ _ce_provision_data_dir + '/' + _aws_resource_name }}" + state: directory + +- name: Ensure subnet file exists. + ansible.builtin.file: + path: "{{ _ce_provision_data_dir + '/' + _aws_resource_name + '/ec2-subnet-id' }}" + state: touch + +- name: Try to load subnet from file. + ansible.builtin.set_fact: + _aws_ec2_with_eip_vpc_subnet_id: "{{ lookup('file', _ce_provision_data_dir + '/' + _aws_resource_name + '/ec2-subnet-id') }}" + +# If we have no stored subnet ID and vpc_name is defined we'll look up available subnets +- name: Gather VPC information. + amazon.aws.ec2_vpc_net_info: + profile: "{{ aws_ec2_with_eip.aws_profile }}" + region: "{{ aws_ec2_with_eip.region }}" + filters: + "tag:Name": "{{ aws_ec2_with_eip.vpc_name }}" + when: ( aws_ec2_with_eip.vpc_name is defined and aws_ec2_with_eip.vpc_name | length > 0 and _aws_ec2_with_eip_vpc_subnet_id | length == 0 ) + register: _aws_ec2_with_eip_vpc + +- name: Set the VPC id from name. + ansible.builtin.set_fact: + _aws_ec2_with_eip_vpc_id: "{{ _aws_ec2_with_eip_vpc.vpcs[0].vpc_id }}" + when: ( aws_ec2_with_eip.vpc_name is defined and aws_ec2_with_eip.vpc_name | length > 0 and _aws_ec2_with_eip_vpc_subnet_id | length == 0 ) + +- name: Gather public subnet information. + amazon.aws.ec2_vpc_subnet_info: + profile: "{{ aws_ec2_with_eip.aws_profile }}" + region: "{{ aws_ec2_with_eip.region }}" + filters: + vpc-id: "{{ _aws_ec2_with_eip_vpc_id }}" + tag:Env: "{{ _env_type }}" + tag:Profile: "{{ aws_ec2_with_eip.vpc_subnet_profile }}" + when: ( aws_ec2_with_eip.vpc_name is defined and aws_ec2_with_eip.vpc_name | length > 0 and _aws_ec2_with_eip_vpc_subnet_id | length == 0 ) + register: _aws_ec2_with_eip_subnets + +- name: Pick a subnet at random. + ansible.builtin.set_fact: + _aws_ec2_with_eip_vpc_subnet_id: "{{ item.subnet_id }}" + with_random_choice: "{{ _aws_ec2_with_eip_subnets.subnets }}" + when: ( aws_ec2_with_eip.vpc_name is defined and aws_ec2_with_eip.vpc_name | length > 0 and _aws_ec2_with_eip_vpc_subnet_id | length == 0 ) + +# If we have no stored subnet ID and one has been set in variables this will override whatever might be looked up. +- name: Load defined subnet from variables. + ansible.builtin.set_fact: + _aws_ec2_with_eip_vpc_subnet_id: "{{ aws_ec2_with_eip.vpc_subnet_id }}" + when: ( aws_ec2_with_eip.vpc_subnet_id is defined and aws_ec2_with_eip.vpc_subnet_id | length > 0 and _aws_ec2_with_eip_vpc_subnet_id | length == 0 ) + +# Save the subnet ID to file. +- name: Save the subnet ID in the data directory. + ansible.builtin.template: + src: ec2-subnet-id.j2 + dest: "{{ _ce_provision_data_dir + '/' + _aws_resource_name + '/ec2-subnet-id' }}" + force: true + - name: Create EC2 instance. community.aws.ec2_instance: # @TODO: soon to move to amazon.aws collection profile: "{{ aws_ec2_with_eip.aws_profile }}" @@ -30,7 +90,7 @@ state: "{{ aws_ec2_with_eip.state }}" wait: true termination_protection: "{{ aws_ec2_with_eip.termination_protection }}" - vpc_subnet_id: "{{ aws_ec2_with_eip.vpc_subnet_id }}" + vpc_subnet_id: "{{ _aws_ec2_with_eip_vpc_subnet_id }}" name: "{{ aws_ec2_with_eip.instance_name }}" tags: "{{ aws_ec2_with_eip.tags }}" security_groups: "{{ aws_ec2_with_eip.security_groups }}" diff --git a/roles/aws/aws_ec2_with_eip/templates/ec2-subnet-id.j2 b/roles/aws/aws_ec2_with_eip/templates/ec2-subnet-id.j2 new file mode 100644 index 000000000..60d0bc4df --- /dev/null +++ b/roles/aws/aws_ec2_with_eip/templates/ec2-subnet-id.j2 @@ -0,0 +1 @@ +{{ _aws_ec2_with_eip_vpc_subnet_id }} \ No newline at end of file From c75366ec74f946c5a3a47a4453eb5b734b669354 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 10 Feb 2022 13:15:25 +0100 Subject: [PATCH 19/22] Ec2 subnet lookup pr 1.x (#589) * First pass at EC2 subnet detection. * Touching subnet file to ensure it exists. * Trying a different approach, file module didn't work. * Switching back to file module. * We need to create the directory for new servers too. * Bad variable name. * Changing subnet lookup order to check for defined subnet first. --- roles/aws/aws_ec2_with_eip/tasks/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/aws/aws_ec2_with_eip/tasks/main.yml b/roles/aws/aws_ec2_with_eip/tasks/main.yml index 8e2741d40..71b19f773 100644 --- a/roles/aws/aws_ec2_with_eip/tasks/main.yml +++ b/roles/aws/aws_ec2_with_eip/tasks/main.yml @@ -34,6 +34,12 @@ ansible.builtin.set_fact: _aws_ec2_with_eip_vpc_subnet_id: "{{ lookup('file', _ce_provision_data_dir + '/' + _aws_resource_name + '/ec2-subnet-id') }}" +# If we have no stored subnet ID and one has been set in variables this will override whatever might be looked up. +- name: Load defined subnet from variables. + ansible.builtin.set_fact: + _aws_ec2_with_eip_vpc_subnet_id: "{{ aws_ec2_with_eip.vpc_subnet_id }}" + when: ( aws_ec2_with_eip.vpc_subnet_id is defined and aws_ec2_with_eip.vpc_subnet_id | length > 0 and _aws_ec2_with_eip_vpc_subnet_id | length == 0 ) + # If we have no stored subnet ID and vpc_name is defined we'll look up available subnets - name: Gather VPC information. amazon.aws.ec2_vpc_net_info: @@ -66,12 +72,6 @@ with_random_choice: "{{ _aws_ec2_with_eip_subnets.subnets }}" when: ( aws_ec2_with_eip.vpc_name is defined and aws_ec2_with_eip.vpc_name | length > 0 and _aws_ec2_with_eip_vpc_subnet_id | length == 0 ) -# If we have no stored subnet ID and one has been set in variables this will override whatever might be looked up. -- name: Load defined subnet from variables. - ansible.builtin.set_fact: - _aws_ec2_with_eip_vpc_subnet_id: "{{ aws_ec2_with_eip.vpc_subnet_id }}" - when: ( aws_ec2_with_eip.vpc_subnet_id is defined and aws_ec2_with_eip.vpc_subnet_id | length > 0 and _aws_ec2_with_eip_vpc_subnet_id | length == 0 ) - # Save the subnet ID to file. - name: Save the subnet ID in the data directory. ansible.builtin.template: From 4ea0d3c483780bbe4d419bd19fdd82ba46c37591 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 10 Feb 2022 15:16:22 +0100 Subject: [PATCH 20/22] Fixing gitlab-runner overriders so upgrades do not break the runner. (#586) * Fixing gitlab-runner overriders so upgrades do not break the runner. * Fixing override file template. * Hopefully fixing CI. * Making sure the service directory exists. * We cannot use the deploy meta role in CI because of LDAP. * Changing dir perms and adding a force. --- .gitignore | 3 ++- ce-dev/ansible/plays/gitlab/gitlab.yml | 20 ++++++++++++++++++- ce-dev/ansible/vars/gitlab/gpg_key.yml | 8 ++++++++ roles/gitlab_runner/tasks/main.yml | 16 ++++++++++++--- .../templates/gitlab-runner.service.j2 | 18 ----------------- .../gitlab_runner/templates/override.conf.j2 | 3 +++ 6 files changed, 45 insertions(+), 23 deletions(-) create mode 100644 ce-dev/ansible/vars/gitlab/gpg_key.yml delete mode 100644 roles/gitlab_runner/templates/gitlab-runner.service.j2 create mode 100644 roles/gitlab_runner/templates/override.conf.j2 diff --git a/.gitignore b/.gitignore index 767ce8ead..fbee4fe08 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ /files /ansible.cfg *geerlingguy* -/ce-dev/ansible/vars/*/_aws_credentials.yml \ No newline at end of file +/ce-dev/ansible/vars/*/_aws_credentials.yml +/ce-dev/ansible/vars/_common/*_exporter.yml diff --git a/ce-dev/ansible/plays/gitlab/gitlab.yml b/ce-dev/ansible/plays/gitlab/gitlab.yml index 4c4afe965..262a16f6c 100644 --- a/ce-dev/ansible/plays/gitlab/gitlab.yml +++ b/ce-dev/ansible/plays/gitlab/gitlab.yml @@ -16,6 +16,24 @@ - ansible.builtin.import_role: name: _init - ansible.builtin.import_role: - name: _meta/deploy + name: user_provision + - ansible.builtin.import_role: + name: _meta/common_base + - ansible.builtin.import_role: + name: ce_deploy + - ansible.builtin.import_role: + name: aws/aws_credentials + - ansible.builtin.import_role: + name: gitlab + - ansible.builtin.import_role: + name: gitlab_runner + - ansible.builtin.import_role: + name: ssh_server + - ansible.builtin.import_role: + name: sops + - ansible.builtin.import_role: + name: gpg_key + - ansible.builtin.import_role: + name: firewall_config - ansible.builtin.import_role: name: _exit diff --git a/ce-dev/ansible/vars/gitlab/gpg_key.yml b/ce-dev/ansible/vars/gitlab/gpg_key.yml new file mode 100644 index 000000000..aa13e6283 --- /dev/null +++ b/ce-dev/ansible/vars/gitlab/gpg_key.yml @@ -0,0 +1,8 @@ +--- +gpg_key: + - username: ce-dev # Must exist already on the server. + publish: false # Whether to publish to HKS public servers. + key_type: "RSA" + key_length: 4096 + email: sysadm@codeenigma.com + expire: 0 diff --git a/roles/gitlab_runner/tasks/main.yml b/roles/gitlab_runner/tasks/main.yml index 8e1df408b..236e4da5d 100644 --- a/roles/gitlab_runner/tasks/main.yml +++ b/roles/gitlab_runner/tasks/main.yml @@ -77,10 +77,20 @@ group: "{{ gitlab_runner.username }}" become: true -- name: Place the gitlab-runner service def file. +- name: Ensure the Gitlab runner service override directory exists. + ansible.builtin.file: + path: /etc/systemd/system/gitlab-runner.service.d + state: directory + mode: '0755' + owner: root + group: root + force: true + become: true + +- name: Place the gitlab-runner unit override file. # see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2422#note_838191146 ansible.builtin.template: - src: gitlab-runner.service.j2 - dest: /etc/systemd/system/gitlab-runner.service + src: override.conf.j2 + dest: /etc/systemd/system/gitlab-runner.service.d/override.conf mode: '0644' owner: root group: root diff --git a/roles/gitlab_runner/templates/gitlab-runner.service.j2 b/roles/gitlab_runner/templates/gitlab-runner.service.j2 deleted file mode 100644 index b596f0d50..000000000 --- a/roles/gitlab_runner/templates/gitlab-runner.service.j2 +++ /dev/null @@ -1,18 +0,0 @@ -[Unit] -Description=GitLab Runner -ConditionFileIsExecutable=/usr/bin/gitlab-runner - -After=syslog.target network.target - -[Service] -StartLimitInterval=5 -StartLimitBurst=10 -ExecStart=/usr/bin/gitlab-runner "run" "--working-directory" "{{ gitlab_runner.runner_workingdir }}" "--config" "{{ gitlab_runner.runner_config }}" "--service" "gitlab-runner" "--user" "{{ gitlab_runner.username }}" - -Restart=always - -RestartSec=120 -EnvironmentFile=-/etc/sysconfig/gitlab-runner - -[Install] -WantedBy=multi-user.target diff --git a/roles/gitlab_runner/templates/override.conf.j2 b/roles/gitlab_runner/templates/override.conf.j2 new file mode 100644 index 000000000..c28213f43 --- /dev/null +++ b/roles/gitlab_runner/templates/override.conf.j2 @@ -0,0 +1,3 @@ +[Service] +ExecStart= +ExecStart=/usr/bin/gitlab-runner "run" "--working-directory" "{{ gitlab_runner.runner_workingdir }}" "--config" "{{ gitlab_runner.runner_config }}" "--service" "gitlab-runner" "--user" "{{ gitlab_runner.username }}" From 9b10e44ddc0e73e5e7c16cc4e7a0af4c9e19c59a Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 10 Feb 2022 17:07:57 +0100 Subject: [PATCH 21/22] Gitlab runner service override pr 1.x (#591) * Fixing gitlab-runner overriders so upgrades do not break the runner. * Fixing override file template. * Hopefully fixing CI. * Making sure the service directory exists. * We cannot use the deploy meta role in CI because of LDAP. * Changing dir perms and adding a force. * Debugging gitlab-runner directory creation issues in CI. * Fixing linting error. * Removing verbosity again but leaving 'stat' command in. --- roles/gitlab_runner/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/gitlab_runner/tasks/main.yml b/roles/gitlab_runner/tasks/main.yml index 236e4da5d..2c0467ece 100644 --- a/roles/gitlab_runner/tasks/main.yml +++ b/roles/gitlab_runner/tasks/main.yml @@ -87,6 +87,11 @@ force: true become: true +- name: Check directory exists. # forces any filesystem caching to clear + ansible.builtin.stat: + path: /etc/systemd/system/gitlab-runner.service.d + register: _service_override_dir + - name: Place the gitlab-runner unit override file. # see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2422#note_838191146 ansible.builtin.template: src: override.conf.j2 From 8203244c39a06f61ad8c2eb83f35366491ee8cdd Mon Sep 17 00:00:00 2001 From: gregharvey Date: Mon, 21 Feb 2022 09:38:45 +0100 Subject: [PATCH 22/22] Making the _exit role reset variables to defaults prior to next run. --- roles/_exit/defaults/main.yml | 4 ++++ roles/_exit/tasks/main.yml | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 roles/_exit/defaults/main.yml diff --git a/roles/_exit/defaults/main.yml b/roles/_exit/defaults/main.yml new file mode 100644 index 000000000..c05de3d02 --- /dev/null +++ b/roles/_exit/defaults/main.yml @@ -0,0 +1,4 @@ +_exit: + roles_path: + - "{{ _ce_provision_base_dir }}/roles" # ce-provision roles + - "$HOME/.ansible/roles" # Galaxy roles \ No newline at end of file diff --git a/roles/_exit/tasks/main.yml b/roles/_exit/tasks/main.yml index 61907d3b7..2f4082809 100644 --- a/roles/_exit/tasks/main.yml +++ b/roles/_exit/tasks/main.yml @@ -11,3 +11,20 @@ content: "{{ current_play_dir_md5 }}" dest: "{{ _ce_provision_data_dir }}/{{ current_play_md5_file }}" delegate_to: localhost + +- name: Find all the role 'defaults' files in the specified paths. + ansible.builtin.find: + paths: "{{ _exit.roles_path }}" + recurse: true + file_type: directory + patterns: 'defaults' + register: _defaults_dirs + +- name: Reload variable defaults prior to next run. + ansible.builtin.include_vars: + dir: "{{ item.path }}" + extensions: + - "yml" + with_items: "{{ _defaults_dirs.files }}" + loop_control: + label: "{{ item.path }}"