From 195ff54953910c50f2d62a83d8f42b5eebce975a Mon Sep 17 00:00:00 2001 From: EmlynK Date: Wed, 5 Jan 2022 15:30:21 +0000 Subject: [PATCH 01/17] Need to check if is_local is defined in webserver meta dependencies. (#522) --- roles/_meta/webserver/meta/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/_meta/webserver/meta/main.yml b/roles/_meta/webserver/meta/main.yml index 838c86baa..17d8d92ce 100644 --- a/roles/_meta/webserver/meta/main.yml +++ b/roles/_meta/webserver/meta/main.yml @@ -2,7 +2,7 @@ dependencies: - role: _meta/common_base - role: firewall_config - - { role: ssh_server, when: not is_local } + - { role: ssh_server, when: ( is_local is not defined or not is_local ) } - role: user_provision - role: user_deploy - role: mysql_client From a21e8d1796b0c20a4ddc230de756b342796c081c Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 5 Jan 2022 16:58:54 +0100 Subject: [PATCH 02/17] Ce dev refactor pr 1.x (#518) * Making it easier to test with provision-target and ce-dev. * Moving the provision forcing var back to plays so _init has it. * Adding defaults vars and test script extra options. * Adding a web server test to CI. * examples string needs to be in quotes. * Making sure is_local and _ce_provision_force_play are available to the _init role. * Adding SSH keys to the provision user. * Adding a --force to the test script. * Explicitly adding vars to role. * Fixing _init behaviour and adding SSH key for web role. * Setting default PHP version to 7.4. * Looking up the generated ce-dev SSH key instead of hard-coding one. * We cannot run the ssh_server role locally, so excluding for tests of webserver role. * Trying to remove user_root.yml in case it's breaking CI. * Adding a verbose mode to the test script. * Exposing the command in the test script. * Trying hard-coded keys again. * Changing location of data dir for test containers. * Putting vars back and restricting CI to the 'web' example. --- .github/workflows/ce-provision-test.yml | 2 +- ce-dev/ansible/test.sh | 10 ++++++++++ ce-dev/ansible/vars/_common/user_deploy.yml | 3 ++- ce-dev/ansible/vars/_common/user_provision.yml | 3 ++- ce-dev/ansible/vars/_common/user_root.yml | 3 --- 5 files changed, 15 insertions(+), 6 deletions(-) delete mode 100644 ce-dev/ansible/vars/_common/user_root.yml diff --git a/.github/workflows/ce-provision-test.yml b/.github/workflows/ce-provision-test.yml index c9631ec71..0dfbf714e 100644 --- a/.github/workflows/ce-provision-test.yml +++ b/.github/workflows/ce-provision-test.yml @@ -44,7 +44,7 @@ jobs: if: ${{ github.event.pull_request.head.ref != 'documentation' }} run: | git clone --branch 1.x https://github.com/codeenigma/ce-dev-ce-provision-config.git config - /bin/bash ce-dev/ansible/test.sh --examples "web gitlab" --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} shell: bash # Builds the docs diff --git a/ce-dev/ansible/test.sh b/ce-dev/ansible/test.sh index 13918e4d6..61124de0a 100755 --- a/ce-dev/ansible/test.sh +++ b/ce-dev/ansible/test.sh @@ -12,6 +12,7 @@ usage(){ echo '--config-branch: Branch to use for the main stack config repository' echo '--no-rebuild: Do not tear down an existing ce-dev stack' echo '--no-provision: Do not run ce-provision against the ce-dev stack' + echo '--verbose: Run ce-provision and Ansible in verbose mode' } # Set defaults @@ -20,6 +21,7 @@ OWN_BRANCH="1.x" CONFIG_BRANCH="1.x" NO_REBUILD=false NO_PROVISION=false +VERBOSE=false # Parse options arguments. parse_options(){ @@ -43,6 +45,9 @@ parse_options(){ "--no-provision") NO_PROVISION=true ;; + "--verbose") + VERBOSE=true + ;; *) usage exit 1 @@ -91,6 +96,11 @@ EOT PROVISION_CMD="/bin/sh /home/ce-dev/ce-provision/scripts/provision.sh" echo "# Executing $1 project" PROVISION_CMD="$PROVISION_CMD --repo dummy --branch dummy --workspace /home/ce-dev/ce-provision/ce-dev/ansible --playbook plays/$1/$1.yml --own-branch $2 --config-branch $3 --force" + if [ $VERBOSE = true ]; then + echo "# In verbose mode" + PROVISION_CMD="$PROVISION_CMD --verbose" + fi + echo "# Running command: $PROVISION_CMD" # shellcheck disable=SC2086 sudo docker exec -t --workdir /home/ce-dev/ce-provision --user ce-dev provision-controller $PROVISION_CMD echo "### $1 project completed ###" diff --git a/ce-dev/ansible/vars/_common/user_deploy.yml b/ce-dev/ansible/vars/_common/user_deploy.yml index 58bbbcee5..bb18c1fde 100644 --- a/ce-dev/ansible/vars/_common/user_deploy.yml +++ b/ce-dev/ansible/vars/_common/user_deploy.yml @@ -5,4 +5,5 @@ user_deploy: sudoer: false groups: [] ssh_keys: - - "{{ lookup('file', '{{ _ce_provision_data_dir }}/provision-controller/home/ce-dev/.ssh/id_rsa.pub') }}" + - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCbzq9srh/W23MBCx/GmlTO3aJckDjZnCcczAAGtx3rAzGFEtvFgDaZbbHK+ya2SSazzQbfSXmwWAYABtFu6qRmZtDCBKlHhR7rI9MysiACmSX6X6fVIDbCZoVXIZIVGR0wdv52+kMArMktO9Y456jlman5whd3b7VD5LOt1xKQRLVQ7KjsxgQ/hpolSo6ZmsJuJe2xF9NMSfpQetmprTEFoCbXbVOl6XDUVqYsNF/e55A2jXBng2UNluztWQDnaP2sDODPk5uw3Fy6Znk0auYe9CmImUZX5x+BLulJIJWmsDa2+Ls3MCpF9VKPqV/UInETJ5P3ge1ieC8a1dboPL4/T9XERPXuGxeZtISNHwFZ3mGSjrbqFpMpJHZ7G41oGqBOq9dsw4eY3G0YlvMtv8Nqu2MtOQz/nGFSniES6e7Q4aHtBOoCbpJt7tDcpIDy9YfEhgQHxNtdAmt0n7GS71O0HS+15y89EzJ46139vZeuvSUkdgUxhQ5ZWhp1u3/6XyaCKt+SHKH5CbbrXQrfP/g98wm4W6kVAAjtkEXZ6rwsc71WdRLxZr17ArOpG+YVTJ67+iMEgsaEw9bi2DNDVeYl51NmNc4d873iH7d/0gsRq4ECzZwoNB0X9N6n0F5cVjQvx2802FzwHUMA2V8gS64UBuoC2OIv/r3YSkwuplzggw== controller@127.0.0.1" + #- "{{ lookup('file', '{{ _ce_provision_data_dir }}/provision-controller/home/ce-dev/.ssh/id_rsa.pub') }}" # works locally diff --git a/ce-dev/ansible/vars/_common/user_provision.yml b/ce-dev/ansible/vars/_common/user_provision.yml index a1b811429..fb9c97b97 100644 --- a/ce-dev/ansible/vars/_common/user_provision.yml +++ b/ce-dev/ansible/vars/_common/user_provision.yml @@ -5,4 +5,5 @@ user_provision: sudoer: true groups: [] ssh_keys: - - "{{ lookup('file', '{{ _ce_provision_data_dir }}/provision-controller/home/ce-dev/.ssh/id_rsa.pub') }}" + - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCbzq9srh/W23MBCx/GmlTO3aJckDjZnCcczAAGtx3rAzGFEtvFgDaZbbHK+ya2SSazzQbfSXmwWAYABtFu6qRmZtDCBKlHhR7rI9MysiACmSX6X6fVIDbCZoVXIZIVGR0wdv52+kMArMktO9Y456jlman5whd3b7VD5LOt1xKQRLVQ7KjsxgQ/hpolSo6ZmsJuJe2xF9NMSfpQetmprTEFoCbXbVOl6XDUVqYsNF/e55A2jXBng2UNluztWQDnaP2sDODPk5uw3Fy6Znk0auYe9CmImUZX5x+BLulJIJWmsDa2+Ls3MCpF9VKPqV/UInETJ5P3ge1ieC8a1dboPL4/T9XERPXuGxeZtISNHwFZ3mGSjrbqFpMpJHZ7G41oGqBOq9dsw4eY3G0YlvMtv8Nqu2MtOQz/nGFSniES6e7Q4aHtBOoCbpJt7tDcpIDy9YfEhgQHxNtdAmt0n7GS71O0HS+15y89EzJ46139vZeuvSUkdgUxhQ5ZWhp1u3/6XyaCKt+SHKH5CbbrXQrfP/g98wm4W6kVAAjtkEXZ6rwsc71WdRLxZr17ArOpG+YVTJ67+iMEgsaEw9bi2DNDVeYl51NmNc4d873iH7d/0gsRq4ECzZwoNB0X9N6n0F5cVjQvx2802FzwHUMA2V8gS64UBuoC2OIv/r3YSkwuplzggw== controller@127.0.0.1" + #- "{{ lookup('file', '{{ _ce_provision_data_dir }}/provision-controller/home/ce-dev/.ssh/id_rsa.pub') }}" # works locally diff --git a/ce-dev/ansible/vars/_common/user_root.yml b/ce-dev/ansible/vars/_common/user_root.yml deleted file mode 100644 index 3dda2aed2..000000000 --- a/ce-dev/ansible/vars/_common/user_root.yml +++ /dev/null @@ -1,3 +0,0 @@ -user_root: - authorized_keys: - - "{{ lookup('file', '{{ _ce_provision_data_dir }}/provision-controller/home/ce-dev/.ssh/id_rsa.pub') }}" From cb681f8ae83535cc14a78ab75221e277b4546ed0 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 10 Jan 2022 14:45:56 +0100 Subject: [PATCH 03/17] Adding backup handling to ldap_server. (#525) * Adding backup handling to ldap_server. * Improving SSL docs and handling perms for openldap and letsencrypt. * Cron user must be specified with file. * Running as root, do not need a 'sudo' in this cron. --- roles/ldap_server/defaults/main.yml | 12 +++++++- roles/ldap_server/tasks/main.yml | 38 +++++++++++++++++++++++++ roles/ldap_server/templates/slap-bak.j2 | 11 +++++++ 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 roles/ldap_server/templates/slap-bak.j2 diff --git a/roles/ldap_server/defaults/main.yml b/roles/ldap_server/defaults/main.yml index 13cbfda10..f6a04b345 100644 --- a/roles/ldap_server/defaults/main.yml +++ b/roles/ldap_server/defaults/main.yml @@ -14,9 +14,19 @@ ldap_server: # Execution order can be important, so ensure your files are named in alphabetical order. path: "{{ _ce_provision_base_dir }}/config/files/ldap_server/config" purge: false + backup: false # set to true to create local backups of LDAP + backup_path: /opt/slap-bak + backup_minute: "45" + backup_hour: "23" + # TLS settings in LDAP are not separately handled, you need to manually set it up or use the config import feature. + # If you use 'manual' SSL handling you need to provide a separate CA certificate. + # If you use 'letsencrypt' SSL handling then the LDAP TLS settings in your imported config should be as follows: + # olcTLSCACertificateFile: /etc/letsencrypt/live/{{ _domain_name }}/chain.pem + # olcTLSCertificateFile: /etc/letsencrypt/live/{{ _domain_name }}/cert.pem + # olcTLSCertificateKeyFile: /etc/letsencrypt/live/{{ _domain_name }}/privkey.pem ssl: # @see the 'ssl' role - does nothing by default. domain: "{{ _domain_name }}" - handling: "unmanaged" # LDAP needs to use 'manual' handling for SSL because a separate CA certificate is required. + handling: "unmanaged" key: "" cert: "" ca_cert: "" diff --git a/roles/ldap_server/tasks/main.yml b/roles/ldap_server/tasks/main.yml index 07938b012..a9e48b532 100644 --- a/roles/ldap_server/tasks/main.yml +++ b/roles/ldap_server/tasks/main.yml @@ -129,6 +129,25 @@ vars: ssl: "{{ ldap_server.ssl }}" +# LetsEncrypt handling - see https://serverfault.com/a/864655 +- name: Create user for LetsEncrypt. + ansible.builtin.user: + name: letsencrypt + group: letsencrypt + system: true + create_home: false + password: "*" + groups: openldap + when: ldap_server.ssl.handling == 'letsencrypt' + +- name: Allow OpenLDAP to view the LetsEncrypt directory. + ansible.builtin.file: + path: /etc/letsencrypt + state: directory + owner: openldap + group: letsencrypt + when: ldap_server.ssl.handling == 'letsencrypt' + - name: Start slapd service. ansible.builtin.service: name: slapd @@ -140,3 +159,22 @@ when: - ldap_server.replication.host - ldap_server.config.import + +- name: Create the slapd backup script. + ansible.builtin.template: + src: slap-bak.j2 + dest: /usr/local/bin/slap-bak + owner: root + group: root + mode: 0755 + when: ldap_server.config.backup + +- name: Create cron job for slapd backups. + ansible.builtin.cron: + name: "ldap backup" + minute: "{{ ldap_server.config.backup_minute }}" + hour: "{{ ldap_server.config.backup_hour }}" + job: "/usr/local/bin/slap-bak && find /opt/slap-bak/ -type f -mtime +60 -print0 | xargs -0 -r rm" + cron_file: slapd-backup + user: root + when: ldap_server.config.backup diff --git a/roles/ldap_server/templates/slap-bak.j2 b/roles/ldap_server/templates/slap-bak.j2 new file mode 100644 index 000000000..6ab8b38f2 --- /dev/null +++ b/roles/ldap_server/templates/slap-bak.j2 @@ -0,0 +1,11 @@ +#!/bin/sh +BACKUPDIR="{{ ldap_server.config.backup_path }}" +mkdir -p "$BACKUPDIR" + +CONFIG_LDAPBK="ldap-config-$( date +%y%m%d-%H%M ).ldif" +slapcat -n 0 > "$BACKUPDIR/$CONFIG_LDAPBK" +gzip -9 "$BACKUPDIR/$CONFIG_LDAPBK" + +DATA_LDAPBK="ldap-data-$( date +%y%m%d-%H%M ).ldif" +slapcat -n 1 > "$BACKUPDIR/$DATA_LDAPBK" +gzip -9 "$BACKUPDIR/$DATA_LDAPBK" From f134be0219aace7f6d5fd0b2ac09aee8f17168d7 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 17 Jan 2022 13:12:09 +0100 Subject: [PATCH 04/17] Allowing 'gitLab' to disable Prometheus. (#530) * Allowing 'gitLab' to disable Prometheus. * Booleans to use in jinja2 as strings must be cast as strings. --- ce-dev/ansible/vars/provision/ce_provision.yml | 2 +- ce-dev/ansible/vars/provision/galaxy-requirements.yml | 8 ++++++++ roles/ce_provision/meta/requirements.yml | 1 - roles/gitlab/defaults/main.yml | 5 ++++- roles/gitlab/templates/gitlab.rb.j2 | 6 +++--- 5 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 ce-dev/ansible/vars/provision/galaxy-requirements.yml diff --git a/ce-dev/ansible/vars/provision/ce_provision.yml b/ce-dev/ansible/vars/provision/ce_provision.yml index a93890652..6818f88bb 100644 --- a/ce-dev/ansible/vars/provision/ce_provision.yml +++ b/ce-dev/ansible/vars/provision/ce_provision.yml @@ -8,7 +8,7 @@ ce_provision: config_repository_branch: "1.x" config_repository_skip_checkout: true local_dir: "/home/ce-dev/ce-provision" - galaxy_custom_requirements_file: "" + galaxy_custom_requirements_file: "/home/ce-dev/ce-provision/ce-dev/ansible/vars/provision/galaxy-requirements.yml" _ce_provision_base_dir: /home/ce-dev/ce-provision _ce_provision_build_dir: /home/ce-dev/ce-provision/ce-dev/ansible/local _ce_provision_build_tmp_dir: /tmp diff --git a/ce-dev/ansible/vars/provision/galaxy-requirements.yml b/ce-dev/ansible/vars/provision/galaxy-requirements.yml new file mode 100644 index 000000000..eeab8eed2 --- /dev/null +++ b/ce-dev/ansible/vars/provision/galaxy-requirements.yml @@ -0,0 +1,8 @@ +--- +roles: + - name: geerlingguy.solr + - name: geerlingguy.java + - name: cloudalchemy.prometheus + - name: cloudalchemy.node_exporter + - name: cloudalchemy.alertmanager + - name: cloudalchemy.process_exporter diff --git a/roles/ce_provision/meta/requirements.yml b/roles/ce_provision/meta/requirements.yml index 0d87ad9c7..38b0d3b6a 100644 --- a/roles/ce_provision/meta/requirements.yml +++ b/roles/ce_provision/meta/requirements.yml @@ -5,7 +5,6 @@ collections: - name: amazon.aws roles: - - name: cloudalchemy.process_exporter - name: geerlingguy.varnish - name: geerlingguy.firewall - name: geerlingguy.composer diff --git a/roles/gitlab/defaults/main.yml b/roles/gitlab/defaults/main.yml index 1fa32e471..dd004aeb5 100644 --- a/roles/gitlab/defaults/main.yml +++ b/roles/gitlab/defaults/main.yml @@ -14,11 +14,14 @@ gitlab: unicorn_worker_processes: 2 puma_worker_processes: 2 initial_root_password: "" - ldap: false + ldap: false # enable/disable LDAP integration ldap_endpoint: "{{ ldap_client.endpoints[0] }}" ldap_lookup_base: "{{ ldap_client.lookup_base }}" ldap_binddn: "{{ ldap_client.binddn }}" ldap_bindpw: "{{ ldap_client.bindpw }}" + prometheus: "true" # enable/disable built-in Prometheus + node_exporter: "true" # enable/disable built-in Prometheus Node Exporter + alertmanager: "true" # enable/disable built-in Prometheus Alertmanager nginx: listen_port: 8881 listen_https: nil diff --git a/roles/gitlab/templates/gitlab.rb.j2 b/roles/gitlab/templates/gitlab.rb.j2 index 29588f0c1..b5ffaa8bd 100644 --- a/roles/gitlab/templates/gitlab.rb.j2 +++ b/roles/gitlab/templates/gitlab.rb.j2 @@ -1345,7 +1345,7 @@ nginx['listen_https'] = {{ gitlab.nginx.listen_https }} ##! Docs: https://docs.gitlab.com/ce/administration/monitoring/prometheus/ ################################################################################ -# prometheus['enable'] = true +prometheus['enable'] = {{ gitlab.prometheus }} # prometheus['monitor_kubernetes'] = true # prometheus['username'] = 'gitlab-prometheus' # prometheus['group'] = 'gitlab-prometheus' @@ -1427,7 +1427,7 @@ nginx['listen_https'] = {{ gitlab.nginx.listen_https }} ##! Docs: https://docs.gitlab.com/ce/administration/monitoring/prometheus/alertmanager.html ################################################################################ -# alertmanager['enable'] = true +alertmanager['enable'] = {{ gitlab.alertmanager }} # alertmanager['home'] = '/var/opt/gitlab/alertmanager' # alertmanager['log_directory'] = '/var/log/gitlab/alertmanager' # alertmanager['admin_email'] = 'admin@example.com' @@ -1449,7 +1449,7 @@ nginx['listen_https'] = {{ gitlab.nginx.listen_https }} ##! Docs: https://docs.gitlab.com/ce/administration/monitoring/prometheus/node_exporter.html ################################################################################ -# node_exporter['enable'] = true +node_exporter['enable'] = {{ gitlab.node_exporter }} # node_exporter['home'] = '/var/opt/gitlab/node-exporter' # node_exporter['log_directory'] = '/var/log/gitlab/node-exporter' # node_exporter['flags'] = { From 450d4090d3002053a3be1398b6e574f0fb14dc7b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:32:13 +0100 Subject: [PATCH 05/17] GitHub Actions - Rebuilt documentation. (#526) Co-authored-by: Code Enigma CI From 965d7b2c1169351638530f4c22b18d23bfc03fe8 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 17 Jan 2022 14:14:51 +0100 Subject: [PATCH 06/17] Prometheus pr 1.x (#533) * Allowing 'gitLab' to disable Prometheus. * Booleans to use in jinja2 as strings must be cast as strings. * Tidying up CI and adding a GitLab test. * Fixing CI job description. --- ...n-test.yml => ce-provision-build-docs.yml} | 29 ++----------- .../workflows/ce-provision-test-gitlab.yml | 42 +++++++++++++++++++ .github/workflows/ce-provision-test-web.yml | 42 +++++++++++++++++++ 3 files changed, 87 insertions(+), 26 deletions(-) rename .github/workflows/{ce-provision-test.yml => ce-provision-build-docs.yml} (59%) create mode 100644 .github/workflows/ce-provision-test-gitlab.yml create mode 100644 .github/workflows/ce-provision-test-web.yml diff --git a/.github/workflows/ce-provision-test.yml b/.github/workflows/ce-provision-build-docs.yml similarity index 59% rename from .github/workflows/ce-provision-test.yml rename to .github/workflows/ce-provision-build-docs.yml index 0dfbf714e..bbd1c649f 100644 --- a/.github/workflows/ce-provision-test.yml +++ b/.github/workflows/ce-provision-build-docs.yml @@ -1,4 +1,4 @@ -name: Run tests and build docs +name: Build docs # Run this workflow every time a new commit pushed to your repository on: pull_request @@ -6,9 +6,9 @@ on: pull_request jobs: # Set the job key. The key is displayed as the job name # when a job name is not provided - run-tests: + build-docs: # Name the Job - name: Run tests against Ansible code base + name: Build the documentation # Set the type of machine to run on runs-on: ubuntu-20.04 @@ -18,35 +18,12 @@ jobs: if: ${{ github.event.pull_request.head.ref != 'documentation' }} uses: actions/checkout@v2 - # Installs the ce-dev stack - - name: Install ce-dev - if: ${{ github.event.pull_request.head.ref != 'documentation' }} - run: | - cd /tmp - wget https://golang.org/dl/go1.15.8.linux-amd64.tar.gz - sudo tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz - export PATH=$PATH:/usr/local/go/bin - git clone https://github.com/FiloSottile/mkcert && cd mkcert - go build -ldflags "-X main.Version=$(git describe --tags)" - sudo mv ./mkcert /usr/local/bin && cd ../ - sudo chmod +x /usr/local/bin/mkcert - rm -Rf mkcert - curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux - # Configures global Git variables for committing - name: Configure Git run: | git config --global user.email "sysadm@codeenigma.com" git config --global user.name "Code Enigma CI" - # Uses the ce-dev stack to run a test provision - - name: Run a test provision - if: ${{ github.event.pull_request.head.ref != 'documentation' }} - run: | - git clone --branch 1.x https://github.com/codeenigma/ce-dev-ce-provision-config.git config - /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} - shell: bash - # Builds the docs - name: Build documentation if: ${{ github.event.pull_request.head.ref != 'documentation' }} diff --git a/.github/workflows/ce-provision-test-gitlab.yml b/.github/workflows/ce-provision-test-gitlab.yml new file mode 100644 index 000000000..1357e9cc2 --- /dev/null +++ b/.github/workflows/ce-provision-test-gitlab.yml @@ -0,0 +1,42 @@ +name: Run GitLab server test build + +# Run this workflow every time a new commit pushed to your repository +on: pull_request + +jobs: + # Set the job key. The key is displayed as the job name + # when a job name is not provided + test-gitlab: + # Name the Job + name: Run tests against Ansible code base + # Set the type of machine to run on + runs-on: ubuntu-20.04 + + steps: + # Checks out a copy of your repository on the ubuntu-latest machine + - name: Checkout code + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + uses: actions/checkout@v2 + + # Installs the ce-dev stack + - name: Install ce-dev + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + run: | + cd /tmp + wget https://golang.org/dl/go1.15.8.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz + export PATH=$PATH:/usr/local/go/bin + git clone https://github.com/FiloSottile/mkcert && cd mkcert + go build -ldflags "-X main.Version=$(git describe --tags)" + sudo mv ./mkcert /usr/local/bin && cd ../ + sudo chmod +x /usr/local/bin/mkcert + rm -Rf mkcert + curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux + + # Uses the ce-dev stack to run a test provision + - name: Run a test provision + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + run: | + git clone --branch 1.x https://github.com/codeenigma/ce-dev-ce-provision-config.git config + /bin/bash ce-dev/ansible/test.sh --examples gitlab --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + shell: bash diff --git a/.github/workflows/ce-provision-test-web.yml b/.github/workflows/ce-provision-test-web.yml new file mode 100644 index 000000000..55a6942f4 --- /dev/null +++ b/.github/workflows/ce-provision-test-web.yml @@ -0,0 +1,42 @@ +name: Run web server test build + +# Run this workflow every time a new commit pushed to your repository +on: pull_request + +jobs: + # Set the job key. The key is displayed as the job name + # when a job name is not provided + test-web: + # Name the Job + name: Run tests against Ansible code base + # Set the type of machine to run on + runs-on: ubuntu-20.04 + + steps: + # Checks out a copy of your repository on the ubuntu-latest machine + - name: Checkout code + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + uses: actions/checkout@v2 + + # Installs the ce-dev stack + - name: Install ce-dev + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + run: | + cd /tmp + wget https://golang.org/dl/go1.15.8.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf go1.15.8.linux-amd64.tar.gz + export PATH=$PATH:/usr/local/go/bin + git clone https://github.com/FiloSottile/mkcert && cd mkcert + go build -ldflags "-X main.Version=$(git describe --tags)" + sudo mv ./mkcert /usr/local/bin && cd ../ + sudo chmod +x /usr/local/bin/mkcert + rm -Rf mkcert + curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux + + # Uses the ce-dev stack to run a test provision + - name: Run a test provision + if: ${{ github.event.pull_request.head.ref != 'documentation' }} + run: | + git clone --branch 1.x https://github.com/codeenigma/ce-dev-ce-provision-config.git config + /bin/bash ce-dev/ansible/test.sh --examples web --own-branch ${{ github.event.pull_request.base.ref }} --config-branch ${{ github.event.pull_request.base.ref }} + shell: bash From cbb386d21fed5cad5748effbf7728b8ff9889a8b Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 18 Jan 2022 13:47:45 +0000 Subject: [PATCH 07/17] Add private files support for Drupal in Nginx. (#535) --- roles/nginx/templates/drupal_common.j2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/roles/nginx/templates/drupal_common.j2 b/roles/nginx/templates/drupal_common.j2 index 19db7b1e9..2c6a5064a 100644 --- a/roles/nginx/templates/drupal_common.j2 +++ b/roles/nginx/templates/drupal_common.j2 @@ -89,6 +89,13 @@ location ~ ^/sites/.*/files/.* { try_files $uri @rewrite; } +# Allow private files support +location ~* ^/system/files/.*\.(png|jpg|jpeg|gif|ico|svg)$ { # If the image does not exist, it must be a private file. + try_files $uri @rewrite; + expires 7d; + log_not_found off; +} + # Core and contrib assets can be pretty much anywhere. location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|eot|woff2|ttf|otf)$ { try_files $uri @rewrite =404; From 8904ba8b515957954c90c6450ade2965c9221dec Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 18 Jan 2022 16:10:33 +0100 Subject: [PATCH 08/17] Prometheus pr 1.x (#539) * Allowing 'gitLab' to disable Prometheus. * Booleans to use in jinja2 as strings must be cast as strings. * Tidying up CI and adding a GitLab test. * Fixing CI job description. * Adding a firewall config preset to open port 80 for LetsEncrypt. --- ce-dev/.gitignore | 4 +++- roles/firewall_config/defaults/main.yml | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ce-dev/.gitignore b/ce-dev/.gitignore index 990b2ccfc..3b8ced0db 100644 --- a/ce-dev/.gitignore +++ b/ce-dev/.gitignore @@ -3,4 +3,6 @@ docker-compose.yml ansible/local/* !ansible/local/.gitkeep ansible/vars/provision-target -ansible/plays/provision-target/*.yml \ No newline at end of file +ansible/plays/provision-target/*.yml +ansible/vars/provision-privileged +ansible/plays/provision-privileged/*.yml \ No newline at end of file diff --git a/roles/firewall_config/defaults/main.yml b/roles/firewall_config/defaults/main.yml index 6625b9181..29e143853 100644 --- a/roles/firewall_config/defaults/main.yml +++ b/roles/firewall_config/defaults/main.yml @@ -36,6 +36,9 @@ firewall_config: firewall_allowed_tcp_ports: - "989" - "990" + letsencrypt: + firewall_allowed_tcp_ports: + - "80" ossec: firewall_allowed_udp_ports: - "1514" From df86ca38db6418829029cde4bfacce461c2e210a Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 25 Jan 2022 13:33:56 +0100 Subject: [PATCH 09/17] Removing our unused ClamAV roles and adding a Galaxy role to common base. (#541) --- roles/_meta/common_base/meta/main.yml | 1 + roles/ce_provision/meta/requirements.yml | 1 + roles/clamav_clamscan/README.md | 22 - roles/clamav_clamscan/defaults/main.yml | 7 - roles/clamav_clamscan/tasks/main.yml | 25 - .../templates/clamscan-cron.j2 | 2 - .../templates/clamscan-report.j2 | 79 --- roles/clamav_daemon/README.md | 20 - roles/clamav_daemon/defaults/main.yml | 6 - roles/clamav_daemon/handlers/main.yml | 7 - roles/clamav_daemon/tasks/main.yml | 20 - roles/clamav_daemon/templates/clamd.conf.j2 | 605 ------------------ 12 files changed, 2 insertions(+), 793 deletions(-) delete mode 100644 roles/clamav_clamscan/README.md delete mode 100644 roles/clamav_clamscan/defaults/main.yml delete mode 100644 roles/clamav_clamscan/tasks/main.yml delete mode 100644 roles/clamav_clamscan/templates/clamscan-cron.j2 delete mode 100644 roles/clamav_clamscan/templates/clamscan-report.j2 delete mode 100644 roles/clamav_daemon/README.md delete mode 100644 roles/clamav_daemon/defaults/main.yml delete mode 100644 roles/clamav_daemon/handlers/main.yml delete mode 100644 roles/clamav_daemon/tasks/main.yml delete mode 100644 roles/clamav_daemon/templates/clamd.conf.j2 diff --git a/roles/_meta/common_base/meta/main.yml b/roles/_meta/common_base/meta/main.yml index 86f97952c..f23097474 100644 --- a/roles/_meta/common_base/meta/main.yml +++ b/roles/_meta/common_base/meta/main.yml @@ -7,3 +7,4 @@ dependencies: - { role: hosts, when: ( is_local is not defined or not is_local ) } - role: rkhunter - role: postfix + - role: geerlingguy.clamav diff --git a/roles/ce_provision/meta/requirements.yml b/roles/ce_provision/meta/requirements.yml index 38b0d3b6a..1a062f40e 100644 --- a/roles/ce_provision/meta/requirements.yml +++ b/roles/ce_provision/meta/requirements.yml @@ -8,4 +8,5 @@ roles: - name: geerlingguy.varnish - name: geerlingguy.firewall - name: geerlingguy.composer + - name: geerlingguy.clamav - name: robertdebock.openvpn diff --git a/roles/clamav_clamscan/README.md b/roles/clamav_clamscan/README.md deleted file mode 100644 index adbac2052..000000000 --- a/roles/clamav_clamscan/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# ClamAV Clamscan - - - -## Configuration -This role will install the ClamAV base package which will allow us to run clamscan on demand and email scan reports to an email address. -NOTE: This approach will not install clamd. Check the clamav_daemon role if this is what you are looking for. - - -## Default variables -```yaml ---- -# defaults file for clamav - -clamav_clamscan: - email: "admins@example.com" - schedule: "0 0 * * *" - path: "/var/www" - -``` - - diff --git a/roles/clamav_clamscan/defaults/main.yml b/roles/clamav_clamscan/defaults/main.yml deleted file mode 100644 index 4531b0fa4..000000000 --- a/roles/clamav_clamscan/defaults/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# defaults file for clamav - -clamav_clamscan: - email: "admins@example.com" - schedule: "0 0 * * *" - path: "/var/www" diff --git a/roles/clamav_clamscan/tasks/main.yml b/roles/clamav_clamscan/tasks/main.yml deleted file mode 100644 index 07be70701..000000000 --- a/roles/clamav_clamscan/tasks/main.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# tasks file for clamav - -- name: Ensure ClamAV packages are installed - ansible.builtin.apt: - pkg: ["clamav"] - state: present - update_cache: true - cache_valid_time: 240 - -- name: Copy clamscan report config - ansible.builtin.template: - src: "clamscan-report.j2" - dest: "/usr/local/bin/clamscan-report" - owner: root - group: root - mode: 0700 - -- name: Copy clamscan cron config - ansible.builtin.template: - src: "clamscan-cron.j2" - dest: "/etc/cron.d/clamscan_cron" - owner: root - group: root - mode: 0644 diff --git a/roles/clamav_clamscan/templates/clamscan-cron.j2 b/roles/clamav_clamscan/templates/clamscan-cron.j2 deleted file mode 100644 index b5d68185e..000000000 --- a/roles/clamav_clamscan/templates/clamscan-cron.j2 +++ /dev/null @@ -1,2 +0,0 @@ -MAILTO="" -{{ clamav_clamscan.schedule }} root /usr/local/bin/clamscan-report -d {{ clamav_clamscan.path }} diff --git a/roles/clamav_clamscan/templates/clamscan-report.j2 b/roles/clamav_clamscan/templates/clamscan-report.j2 deleted file mode 100644 index 25d808e16..000000000 --- a/roles/clamav_clamscan/templates/clamscan-report.j2 +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -# Variables -TODAY=$(date +%y%m%d) -EMAIL="{{ clamav_clamscan.email }}" -SERVER=`hostname -f` - -# A simple color function to report errors in red -color() { - printf '\033[%sm%s\033[m\n' "$@" - # usage color "31;5" "error message" - # 0 default - # 5 blink, 1 strong, 4 underlined - # fg: 31 red, 32 green, 33 yellow, 34 blue, 35 purple, 36 cyan, 37 white - # bg: 40 black, 41 red, 44 blue, 45 purple -} - -usage() -{ -cat << EOF -usage: $0 ARGUMENT OPTIONS - -This script scans a directory with clamscan and e-mails -a report if it finds any infected files. - -ARGUMENTS: - -d Directory to scan - -h This help message. -EOF -} - -# Parse the args -while getopts ":d:h" OPTION -do - case $OPTION in - h) - usage - exit - ;; - d) - DIRECTORY=$OPTARG - ;; - ?) - usage - exit - ;; - esac -done - -which clamscan > /dev/null -if [ $? -eq 1 ]; then - echo "clamscan doesn't seem to be installed!" - exit 1 -fi - -# Check for appropriate arguments and options - -# No task provided -if [[ -z $DIRECTORY ]]; then - color '31;1' "You didn't provide a directory to scan! Pass one as an argument with -d" - usage - exit 1 -fi - -if [[ ! -d $DIRECTORY ]]; then - echo "That directory $DIRECTORY doesn't exist!" - exit -fi - -# Scan the directory and log to a file -clamscan -r -i --cross-fs=no --log=/var/log/clamscan-report-${TODAY}.log --quiet $DIRECTORY - -# Check the log to see if there were any infected files -grep -q "Infected files: 0" /var/log/clamscan-report-${TODAY}.log - -# If we found infected files, send an e-mail -if [ $? -eq 1 ]; then - cat /var/log/clamscan-report-${TODAY}.log | mail -s "ClamScan report for ${SERVER}" $EMAIL -fi diff --git a/roles/clamav_daemon/README.md b/roles/clamav_daemon/README.md deleted file mode 100644 index 00f9582d1..000000000 --- a/roles/clamav_daemon/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# ClamAV Daemon - - - -## Configuration -This role will install the ClamAV daemon. If you want to install clamscan and generate reports check the clamav_clamscan role. - - -## Default variables -```yaml ---- -# defaults file for clamav - -clamav_daemon: - host: "127.0.0.1" - port: "3310" - -``` - - diff --git a/roles/clamav_daemon/defaults/main.yml b/roles/clamav_daemon/defaults/main.yml deleted file mode 100644 index 113aa4363..000000000 --- a/roles/clamav_daemon/defaults/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -# defaults file for clamav - -clamav_daemon: - host: "127.0.0.1" - port: "3310" diff --git a/roles/clamav_daemon/handlers/main.yml b/roles/clamav_daemon/handlers/main.yml deleted file mode 100644 index 75c6d105f..000000000 --- a/roles/clamav_daemon/handlers/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# handlers file for clamav - -- name: restart clamav - service: - name: "clamav-daemon" - state: restarted diff --git a/roles/clamav_daemon/tasks/main.yml b/roles/clamav_daemon/tasks/main.yml deleted file mode 100644 index d23e49050..000000000 --- a/roles/clamav_daemon/tasks/main.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- name: Ensure ClamAV packages are installed - ansible.builtin.apt: - pkg: ["clamav-daemon"] - state: present - update_cache: true - cache_valid_time: 240 - -- name: Copy main ClamAV config - ansible.builtin.template: - src: "clamd.conf.j2" - dest: "/etc/clamav/clamd.conf" - owner: root - group: root - mode: "0644" - notify: - - restart clamav - -- pause: - minutes: 1 diff --git a/roles/clamav_daemon/templates/clamd.conf.j2 b/roles/clamav_daemon/templates/clamd.conf.j2 deleted file mode 100644 index 55ccfc991..000000000 --- a/roles/clamav_daemon/templates/clamd.conf.j2 +++ /dev/null @@ -1,605 +0,0 @@ -## -## Example config file for the Clam AV daemon -## Please read the clamd.conf(5) manual before editing this file. -## - - -# Comment or remove the line below. -# Example - -# Uncomment this option to enable logging. -# LogFile must be writable for the user running daemon. -# A full path is required. -# Default: disabled -#LogFile /tmp/clamd.log - -# By default the log file is locked for writing - the lock protects against -# running clamd multiple times (if want to run another clamd, please -# copy the configuration file, change the LogFile variable, and run -# the daemon with --config-file option). -# This option disables log file locking. -# Default: no -#LogFileUnlock yes - -# Maximum size of the log file. -# Value of 0 disables the limit. -# You may use 'M' or 'm' for megabytes (1M = 1m = 1048576 bytes) -# and 'K' or 'k' for kilobytes (1K = 1k = 1024 bytes). To specify the size -# in bytes just don't use modifiers. If LogFileMaxSize is enabled, log -# rotation (the LogRotate option) will always be enabled. -# Default: 1M -#LogFileMaxSize 2M - -# Log time with each message. -# Default: no -LogTime yes - -# Also log clean files. Useful in debugging but drastically increases the -# log size. -# Default: no -#LogClean yes - -# Use system logger (can work together with LogFile). -# Default: no -LogSyslog yes - -# Specify the type of syslog messages - please refer to 'man syslog' -# for facility names. -# Default: LOG_LOCAL6 -#LogFacility LOG_MAIL - -# Enable verbose logging. -# Default: no -#LogVerbose yes - -# Enable log rotation. Always enabled when LogFileMaxSize is enabled. -# Default: no -#LogRotate yes - -# Log additional information about the infected file, such as its -# size and hash, together with the virus name. -#ExtendedDetectionInfo yes - -# This option allows you to save a process identifier of the listening -# daemon (main thread). -# Default: disabled -# PidFile /var/run/clamd.pid - -# Optional path to the global temporary directory. -# Default: system specific (usually /tmp or /var/tmp). -#TemporaryDirectory /var/tmp - -# Path to the database directory. -# Default: hardcoded (depends on installation options) -#DatabaseDirectory /var/lib/clamav - -# Only load the official signatures published by the ClamAV project. -# Default: no -#OfficialDatabaseOnly no - -# The daemon can work in local mode, network mode or both. -# Due to security reasons we recommend the local mode. - -# Path to a local socket file the daemon will listen on. -# Default: disabled (must be specified by a user) -#LocalSocket /tmp/clamd.socket - -# Sets the group ownership on the unix socket. -# Default: disabled (the primary group of the user running clamd) -#LocalSocketGroup virusgroup - -# Sets the permissions on the unix socket to the specified mode. -# Default: disabled (socket is world accessible) -#LocalSocketMode 660 - -# Remove stale socket after unclean shutdown. -# Default: true -#FixStaleSocket yes - -# TCP port address. -# Default: no -TCPSocket {{ clamav_daemon.port }} - -# TCP address. -# By default we bind to INADDR_ANY, probably not wise. -# Enable the following to provide some degree of protection -# from the outside world. This option can be specified multiple -# times if you want to listen on multiple IPs. IPv6 is now supported. -# Default: no - -TCPAddr {{ clamav_daemon.host }} - -# Maximum length the queue of pending connections may grow to. -# Default: 200 -#MaxConnectionQueueLength 30 - -# Clamd uses FTP-like protocol to receive data from remote clients. -# If you are using clamav-milter to balance load between remote clamd daemons -# on firewall servers you may need to tune the options below. - -# Close the connection when the data size limit is exceeded. -# The value should match your MTA's limit for a maximum attachment size. -# Default: 25M -#StreamMaxLength 10M - -# Limit port range. -# Default: 1024 -#StreamMinPort 30000 -# Default: 2048 -#StreamMaxPort 32000 - -# Maximum number of threads running at the same time. -# Default: 10 -#MaxThreads 20 - -# Waiting for data from a client socket will timeout after this time (seconds). -# Default: 120 -#ReadTimeout 300 - -# This option specifies the time (in seconds) after which clamd should -# timeout if a client doesn't provide any initial command after connecting. -# Default: 5 -#CommandReadTimeout 5 - -# This option specifies how long to wait (in miliseconds) if the send buffer is full. -# Keep this value low to prevent clamd hanging -# -# Default: 500 -#SendBufTimeout 200 - -# Maximum number of queued items (including those being processed by MaxThreads threads) -# It is recommended to have this value at least twice MaxThreads if possible. -# WARNING: you shouldn't increase this too much to avoid running out of file descriptors, -# the following condition should hold: -# MaxThreads*MaxRecursion + (MaxQueue - MaxThreads) + 6< RLIMIT_NOFILE (usual max is 1024) -# -# Default: 100 -#MaxQueue 200 - -# Waiting for a new job will timeout after this time (seconds). -# Default: 30 -#IdleTimeout 60 - -# Don't scan files and directories matching regex -# This directive can be used multiple times -# Default: scan all -#ExcludePath ^/proc/ -#ExcludePath ^/sys/ - -# Maximum depth directories are scanned at. -# Default: 15 -#MaxDirectoryRecursion 20 - -# Follow directory symlinks. -# Default: no -#FollowDirectorySymlinks yes - -# Follow regular file symlinks. -# Default: no -#FollowFileSymlinks yes - -# Scan files and directories on other filesystems. -# Default: true -#CrossFilesystems yes - -# Perform a database check. -# Default: 600 (10 min) -#SelfCheck 600 - -# Execute a command when virus is found. In the command string %v will -# be replaced with the virus name. -# Default: no -#VirusEvent /usr/local/bin/send_sms 123456789 "VIRUS ALERT: %v" - -# Run as another user (clamd must be started by root for this option to work) -# Default: don't drop privileges -User clamav - -# Initialize supplementary group access (clamd must be started by root). -# Default: no -#AllowSupplementaryGroups no - -# Stop daemon when libclamav reports out of memory condition. -#ExitOnOOM yes - -# Don't fork into background. -# Default: no -#Foreground yes - -# Enable debug messages in libclamav. -# Default: no -#Debug yes - -# Do not remove temporary files (for debug purposes). -# Default: no -#LeaveTemporaryFiles yes - -# Permit use of the ALLMATCHSCAN command. If set to no, clamd will reject -# any ALLMATCHSCAN command as invalid. -# Default: true -#AllowAllMatchScan no - -# Detect Possibly Unwanted Applications. -# Default: no -#DetectPUA yes - -# Exclude a specific PUA category. This directive can be used multiple times. -# See https://github.com/vrtadmin/clamav-faq/blob/master/faq/faq-pua.md for -# the complete list of PUA categories. -# Default: Load all categories (if DetectPUA is activated) -#ExcludePUA NetTool -#ExcludePUA PWTool - -# Only include a specific PUA category. This directive can be used multiple -# times. -# Default: Load all categories (if DetectPUA is activated) -#IncludePUA Spy -#IncludePUA Scanner -#IncludePUA RAT - -# In some cases (eg. complex malware, exploits in graphic files, and others), -# ClamAV uses special algorithms to provide accurate detection. This option -# controls the algorithmic detection. -# Default: true -#AlgorithmicDetection yes - -# This option causes memory or nested map scans to dump the content to disk. -# If you turn on this option, more data is written to disk and is available -# when the LeaveTemporaryFiles option is enabled. -#ForceToDisk yes - -# This option allows you to disable the caching feature of the engine. By -# default, the engine will store an MD5 in a cache of any files that are -# not flagged as virus or that hit limits checks. Disabling the cache will -# have a negative performance impact on large scans. -# Default: no -#DisableCache yes - -## -## Executable files -## - -# PE stands for Portable Executable - it's an executable file format used -# in all 32 and 64-bit versions of Windows operating systems. This option allows -# ClamAV to perform a deeper analysis of executable files and it's also -# required for decompression of popular executable packers such as UPX, FSG, -# and Petite. If you turn off this option, the original files will still be -# scanned, but without additional processing. -# Default: true -#ScanPE yes - -# Certain PE files contain an authenticode signature. By default, we check -# the signature chain in the PE file against a database of trusted and -# revoked certificates if the file being scanned is marked as a virus. -# If any certificate in the chain validates against any trusted root, but -# does not match any revoked certificate, the file is marked as whitelisted. -# If the file does match a revoked certificate, the file is marked as virus. -# The following setting completely turns off authenticode verification. -# Default: no -#DisableCertCheck yes - -# Executable and Linking Format is a standard format for UN*X executables. -# This option allows you to control the scanning of ELF files. -# If you turn off this option, the original files will still be scanned, but -# without additional processing. -# Default: true -#ScanELF yes - -# With this option clamav will try to detect broken executables (both PE and -# ELF) and mark them as Broken.Executable. -# Default: no -#DetectBrokenExecutables yes - - -## -## Documents -## - -# This option enables scanning of OLE2 files, such as Microsoft Office -# documents and .msi files. -# If you turn off this option, the original files will still be scanned, but -# without additional processing. -# Default: true -#ScanOLE2 yes - -# With this option enabled OLE2 files with VBA macros, which were not -# detected by signatures will be marked as "Heuristics.OLE2.ContainsMacros". -# Default: no -#OLE2BlockMacros no - -# This option enables scanning within PDF files. -# If you turn off this option, the original files will still be scanned, but -# without decoding and additional processing. -# Default: true -#ScanPDF yes - -# This option enables scanning within SWF files. -# If you turn off this option, the original files will still be scanned, but -# without decoding and additional processing. -# Default: true -#ScanSWF yes - - -## -## Mail files -## - -# Enable internal e-mail scanner. -# If you turn off this option, the original files will still be scanned, but -# without parsing individual messages/attachments. -# Default: true -#ScanMail yes - -# Scan RFC1341 messages split over many emails. -# You will need to periodically clean up $TemporaryDirectory/clamav-partial directory. -# WARNING: This option may open your system to a DoS attack. -# Never use it on loaded servers. -# Default: no -#ScanPartialMessages yes - -# With this option enabled ClamAV will try to detect phishing attempts by using -# signatures. -# Default: true -#PhishingSignatures yes - -# Scan URLs found in mails for phishing attempts using heuristics. -# Default: true -#PhishingScanURLs yes - -# Always block SSL mismatches in URLs, even if the URL isn't in the database. -# This can lead to false positives. -# -# Default: no -#PhishingAlwaysBlockSSLMismatch no - -# Always block cloaked URLs, even if URL isn't in database. -# This can lead to false positives. -# -# Default: no -#PhishingAlwaysBlockCloak no - -# Detect partition intersections in raw disk images using heuristics. -# Default: no -#PartitionIntersection no - -# Allow heuristic match to take precedence. -# When enabled, if a heuristic scan (such as phishingScan) detects -# a possible virus/phish it will stop scan immediately. Recommended, saves CPU -# scan-time. -# When disabled, virus/phish detected by heuristic scans will be reported only at -# the end of a scan. If an archive contains both a heuristically detected -# virus/phish, and a real malware, the real malware will be reported -# -# Keep this disabled if you intend to handle "*.Heuristics.*" viruses -# differently from "real" malware. -# If a non-heuristically-detected virus (signature-based) is found first, -# the scan is interrupted immediately, regardless of this config option. -# -# Default: no -#HeuristicScanPrecedence yes - - -## -## Data Loss Prevention (DLP) -## - -# Enable the DLP module -# Default: No -#StructuredDataDetection yes - -# This option sets the lowest number of Credit Card numbers found in a file -# to generate a detect. -# Default: 3 -#StructuredMinCreditCardCount 5 - -# This option sets the lowest number of Social Security Numbers found -# in a file to generate a detect. -# Default: 3 -#StructuredMinSSNCount 5 - -# With this option enabled the DLP module will search for valid -# SSNs formatted as xxx-yy-zzzz -# Default: true -#StructuredSSNFormatNormal yes - -# With this option enabled the DLP module will search for valid -# SSNs formatted as xxxyyzzzz -# Default: no -#StructuredSSNFormatStripped yes - - -## -## HTML -## - -# Perform HTML normalisation and decryption of MS Script Encoder code. -# Default: true -# If you turn off this option, the original files will still be scanned, but -# without additional processing. -#ScanHTML yes - - -## -## Archives -## - -# ClamAV can scan within archives and compressed files. -# If you turn off this option, the original files will still be scanned, but -# without unpacking and additional processing. -# Default: true -#ScanArchive yes - -# Mark encrypted archives as viruses (Encrypted.Zip, Encrypted.RAR). -# Default: no -#ArchiveBlockEncrypted no - - -## -## Limits -## - -# The options below protect your system against Denial of Service attacks -# using archive bombs. - -# This option sets the maximum amount of data to be scanned for each input file. -# Archives and other containers are recursively extracted and scanned up to this -# value. -# Value of 0 disables the limit -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 100M -#MaxScanSize 150M - -# Files larger than this limit won't be scanned. Affects the input file itself -# as well as files contained inside it (when the input file is an archive, a -# document or some other kind of container). -# Value of 0 disables the limit. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 25M -#MaxFileSize 30M - -# Nested archives are scanned recursively, e.g. if a Zip archive contains a RAR -# file, all files within it will also be scanned. This options specifies how -# deeply the process should be continued. -# Note: setting this limit too high may result in severe damage to the system. -# Default: 16 -#MaxRecursion 10 - -# Number of files to be scanned within an archive, a document, or any other -# container file. -# Value of 0 disables the limit. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 10000 -#MaxFiles 15000 - -# Maximum size of a file to check for embedded PE. Files larger than this value -# will skip the additional analysis step. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 10M -#MaxEmbeddedPE 10M - -# Maximum size of a HTML file to normalize. HTML files larger than this value -# will not be normalized or scanned. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 10M -#MaxHTMLNormalize 10M - -# Maximum size of a normalized HTML file to scan. HTML files larger than this -# value after normalization will not be scanned. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 2M -#MaxHTMLNoTags 2M - -# Maximum size of a script file to normalize. Script content larger than this -# value will not be normalized or scanned. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 5M -#MaxScriptNormalize 5M - -# Maximum size of a ZIP file to reanalyze type recognition. ZIP files larger -# than this value will skip the step to potentially reanalyze as PE. -# Note: disabling this limit or setting it too high may result in severe damage -# to the system. -# Default: 1M -#MaxZipTypeRcg 1M - -# This option sets the maximum number of partitions of a raw disk image to be scanned. -# Raw disk images with more partitions than this value will have up to the value number -# partitions scanned. Negative values are not allowed. -# Note: setting this limit too high may result in severe damage or impact performance. -# Default: 50 -#MaxPartitions 128 - -# This option sets the maximum number of icons within a PE to be scanned. -# PE files with more icons than this value will have up to the value number icons scanned. -# Negative values are not allowed. -# WARNING: setting this limit too high may result in severe damage or impact performance. -# Default: 100 -#MaxIconsPE 200 - -## -## On-access Scan Settings -## - -# Enable on-access scanning. Currently, this is supported via fanotify. -# Clamuko/Dazuko support has been deprecated. -# Default: no -#ScanOnAccess yes - -# Don't scan files larger than OnAccessMaxFileSize -# Value of 0 disables the limit. -# Default: 5M -#OnAccessMaxFileSize 10M - -# Set the include paths (all files inside them will be scanned). You can have -# multiple OnAccessIncludePath directives but each directory must be added -# in a separate line. (On-access scan only) -# Default: disabled -#OnAccessIncludePath /home -#OnAccessIncludePath /students - -# Set the exclude paths. All subdirectories are also excluded. -# (On-access scan only) -# Default: disabled -#OnAccessExcludePath /home/bofh - -# With this option you can whitelist specific UIDs. Processes with these UIDs -# will be able to access all files. -# This option can be used multiple times (one per line). -# Default: disabled -#OnAccessExcludeUID 0 - - -## -## Bytecode -## - -# With this option enabled ClamAV will load bytecode from the database. -# It is highly recommended you keep this option on, otherwise you'll miss detections for many new viruses. -# Default: true -#Bytecode yes - -# Set bytecode security level. -# Possible values: -# None - no security at all, meant for debugging. DO NOT USE THIS ON PRODUCTION SYSTEMS -# This value is only available if clamav was built with --enable-debug! -# TrustSigned - trust bytecode loaded from signed .c[lv]d files, -# insert runtime safety checks for bytecode loaded from other sources -# Paranoid - don't trust any bytecode, insert runtime checks for all -# Recommended: TrustSigned, because bytecode in .cvd files already has these checks -# Note that by default only signed bytecode is loaded, currently you can only -# load unsigned bytecode in --enable-debug mode. -# -# Default: TrustSigned -#BytecodeSecurity TrustSigned - -# Set bytecode timeout in miliseconds. -# -# Default: 5000 -# BytecodeTimeout 1000 - -## -## Statistics gathering and submitting -## - -# Enable statistical reporting. -# Default: no -#StatsEnabled yes - -# Disable submission of individual PE sections for files flagged as malware. -# Default: no -#StatsPEDisabled yes - -# HostID in the form of an UUID to use when submitting statistical information. -# Default: auto -#StatsHostID auto - -# Time in seconds to wait for the stats server to come back with a response -# Default: 10 -#StatsTimeout 10 From c0e8b06076a9bc5c17522a54e434805769972050 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 27 Jan 2022 11:43:39 +0100 Subject: [PATCH 10/17] Revert "Moving OSSEC pkill to use process_manager role instead. (#258)" (#544) This reverts commit 73c7bd0adb1105436e484fe794182c915b2d25dd. --- roles/ossec/handlers/main.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/roles/ossec/handlers/main.yml b/roles/ossec/handlers/main.yml index e32a1dd97..3aaa40de4 100644 --- a/roles/ossec/handlers/main.yml +++ b/roles/ossec/handlers/main.yml @@ -7,11 +7,8 @@ state: restarted - name: stop ossec-authd - ansible.builtin.include_role: - name: process_manager - vars: - process_manager: - process_name: ossec-authd + ignore_errors: true + command: "pkill ossec-authd" - name: start ossec-authd - command: "/var/ossec/bin/ossec-authd -k {{ ssl_facts[ossec.ssl.domain].key }} -x {{ ssl_facts[ossec.ssl.domain].certificate }}" + command: "/var/ossec/bin/ossec-authd -k {{ ssl_facts[ossec.ssl.domain].key }} -x {{ ssl_facts[ossec.ssl.domain].certificate }}" \ No newline at end of file From 9b41cf720a42aa5f7c459da09417c00b936040d5 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 3 Feb 2022 12:03:48 +0100 Subject: [PATCH 11/17] Moving key servers to a variable so we can set them. (#555) * Moving key servers to a variable so we can set them. * Allowing us to disable sending keys completely. * Oops, doubled up on existing functionality. * Fixing var name. --- roles/gpg_key/defaults/main.yml | 3 +++ roles/gpg_key/tasks/gpg.yml | 6 +----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/roles/gpg_key/defaults/main.yml b/roles/gpg_key/defaults/main.yml index fc67f7f4c..dccbcf101 100644 --- a/roles/gpg_key/defaults/main.yml +++ b/roles/gpg_key/defaults/main.yml @@ -1,4 +1,7 @@ --- +gpg_key_servers: + - hkps://pgp.mit.edu + - hkps://keys.openpgp.org gpg_key: - username: example # Must exist already on the server. publish: false # Whether to publish to HKS public servers. diff --git a/roles/gpg_key/tasks/gpg.yml b/roles/gpg_key/tasks/gpg.yml index fdfc4e345..9983f5a5b 100644 --- a/roles/gpg_key/tasks/gpg.yml +++ b/roles/gpg_key/tasks/gpg.yml @@ -46,10 +46,6 @@ when: - gpg_user.publish - not user_ansible_gpg_key_exists.stdout - with_items: - - hkp://hkps.pool.sks-keyservers.net - - hkps://pgp.mit.edu - - hkps://keys.openpgp.org - - hkp://keys.gnupg.net + with_items: "{{ gpg_key_servers }}" loop_control: loop_var: server From 89ef9ef144d6707da237101b2e3c28928226b079 Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Fri, 4 Feb 2022 11:05:31 +0000 Subject: [PATCH 12/17] Attempt to create an RDS read replica. --- roles/aws/aws_rds/defaults/main.yml | 1 + roles/aws/aws_rds/tasks/main.yml | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/roles/aws/aws_rds/defaults/main.yml b/roles/aws/aws_rds/defaults/main.yml index 0b15cc247..ff7e8f4fd 100644 --- a/roles/aws/aws_rds/defaults/main.yml +++ b/roles/aws/aws_rds/defaults/main.yml @@ -11,6 +11,7 @@ aws_rds: state: present description: example engine: mariadb + replica: false # If true, a read replica will be created. # engine_version: '5.7.2' # Omit to use latest. # See parameter group docs: https://docs.ansible.com/ansible/latest/collections/community/aws/rds_param_group_module.html # db_parameter_group_name: "example" # Omit to use default diff --git a/roles/aws/aws_rds/tasks/main.yml b/roles/aws/aws_rds/tasks/main.yml index 4b398d1e3..8acfa201e 100644 --- a/roles/aws/aws_rds/tasks/main.yml +++ b/roles/aws/aws_rds/tasks/main.yml @@ -32,6 +32,7 @@ community.aws.rds_instance: db_instance_identifier: "{{ aws_rds.name }}" db_instance_class: "{{ aws_rds.db_instance_class }}" + db_cluster_identifier: "{{ aws_rds.db_cluster_identifier | default(omit) }}" #storage_type: standard # not required. choices: standard;gp2;io1. The storage type to be associated with the DB instance. I(storage_type) does not apply to Aurora DB instances. master_username: "{{ aws_rds.master_username }}" master_user_password: "{{ aws_rds.master_user_password }}" @@ -61,6 +62,16 @@ #preferred_maintenance_window: undefined # not required. The weekly time range (in UTC) of at least 30 minutes, during which system maintenance can occur. The option must be in the format "ddd:hh24:mi-ddd:hh24:mi" where ddd is one of Mon, Tue, Wed, Thu, Fri, Sat, Sun. register: _rds_instance_info +- name: Create RDS read replica instance. + community.aws.rds_instance: + db_instance_identifier: "{{ aws_rds.name }}-replica" + db_cluster_identifier: "{{ aws_rds.db_cluster_identifier | default(omit) }}" + db_instance_class: "{{ aws_rds.db_instance_class }}" + read_replica: true + creation_source: "instance" + source_db_instance_identifier: "{{ aws_rds.name }}" + when: aws_rds.replica + - name: Create SNS topic. ansible.builtin.include_role: name: aws/aws_sns From 2e7d5b8c142c587e2e388536f473f9803cef61ae Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Fri, 4 Feb 2022 13:18:16 +0000 Subject: [PATCH 13/17] Use new task to create Aurora RDS instances. --- roles/aws/aws_rds/tasks/main.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/roles/aws/aws_rds/tasks/main.yml b/roles/aws/aws_rds/tasks/main.yml index 8acfa201e..ba3bb693a 100644 --- a/roles/aws/aws_rds/tasks/main.yml +++ b/roles/aws/aws_rds/tasks/main.yml @@ -28,6 +28,28 @@ tags: "{{ aws_rds.tags | combine({ 'Name': aws_rds.name }) }}" when: aws_rds.db_parameters is defined +- name: Create Aurora RDS instance. + community.aws.rds_instance: + db_instance_identifier: "{{ aws_rds.name }}" + db_instance_class: "{{ aws_rds.db_instance_class }}" + db_cluster_identifier: "{{ aws_rds.db_cluster_identifier }}" + state: "{{ aws_rds.state }}" + engine: "{{ aws_rds.engine }}" + copy_tags_to_snapshot: true + publicly_accessible: "{{ aws_rds.publicly_accessible }}" + allocated_storage: "{{ aws_rds.allocated_storage }}" + max_allocated_storage: "{{ aws_rds.max_allocated_storage }}" + profile: "{{ aws_rds.aws_profile }}" + tags: "{{ aws_rds.tags | combine({ 'Name': aws_rds.name }) }}" + allow_major_version_upgrade: false + storage_encrypted: "{{ aws_rds.storage_encrypted }}" + apply_immediately: true + wait: true + region: "{{ aws_rds.region }}" + engine_version: "{{ aws_rds.engine_version | default(omit) }}" + register: _rds_instance_info + when: 'aurora' in aws_rds.engine + - name: Create RDS instance community.aws.rds_instance: db_instance_identifier: "{{ aws_rds.name }}" @@ -61,6 +83,7 @@ engine_version: "{{ aws_rds.engine_version | default(omit) }}" #preferred_maintenance_window: undefined # not required. The weekly time range (in UTC) of at least 30 minutes, during which system maintenance can occur. The option must be in the format "ddd:hh24:mi-ddd:hh24:mi" where ddd is one of Mon, Tue, Wed, Thu, Fri, Sat, Sun. register: _rds_instance_info + when: 'aurora' not in aws_rds.engine - name: Create RDS read replica instance. community.aws.rds_instance: From d9b4c030f89c58929ed6af89702289f5c8ee7a38 Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Fri, 4 Feb 2022 13:19:28 +0000 Subject: [PATCH 14/17] Try and fix linting issues. --- roles/aws/aws_rds/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/aws/aws_rds/tasks/main.yml b/roles/aws/aws_rds/tasks/main.yml index ba3bb693a..3d708438e 100644 --- a/roles/aws/aws_rds/tasks/main.yml +++ b/roles/aws/aws_rds/tasks/main.yml @@ -48,7 +48,7 @@ region: "{{ aws_rds.region }}" engine_version: "{{ aws_rds.engine_version | default(omit) }}" register: _rds_instance_info - when: 'aurora' in aws_rds.engine + when: "'aurora' in aws_rds.engine" - name: Create RDS instance community.aws.rds_instance: @@ -83,7 +83,7 @@ engine_version: "{{ aws_rds.engine_version | default(omit) }}" #preferred_maintenance_window: undefined # not required. The weekly time range (in UTC) of at least 30 minutes, during which system maintenance can occur. The option must be in the format "ddd:hh24:mi-ddd:hh24:mi" where ddd is one of Mon, Tue, Wed, Thu, Fri, Sat, Sun. register: _rds_instance_info - when: 'aurora' not in aws_rds.engine + when: "'aurora' not in aws_rds.engine" - name: Create RDS read replica instance. community.aws.rds_instance: From 796eebc4355c6ebb4f09571e5658f8a14273a10f Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Fri, 4 Feb 2022 13:37:17 +0000 Subject: [PATCH 15/17] Don't pass max_storage variable for Aurora instances. --- roles/aws/aws_rds/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/aws/aws_rds/tasks/main.yml b/roles/aws/aws_rds/tasks/main.yml index 3d708438e..85d4878ba 100644 --- a/roles/aws/aws_rds/tasks/main.yml +++ b/roles/aws/aws_rds/tasks/main.yml @@ -38,7 +38,6 @@ copy_tags_to_snapshot: true publicly_accessible: "{{ aws_rds.publicly_accessible }}" allocated_storage: "{{ aws_rds.allocated_storage }}" - max_allocated_storage: "{{ aws_rds.max_allocated_storage }}" profile: "{{ aws_rds.aws_profile }}" tags: "{{ aws_rds.tags | combine({ 'Name': aws_rds.name }) }}" allow_major_version_upgrade: false From d748ba1c65bd2581e2e452366fa5b34014626de8 Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Fri, 4 Feb 2022 13:55:16 +0000 Subject: [PATCH 16/17] Remove more storage related vars from Aurora RDS instance creation task. --- roles/aws/aws_rds/tasks/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/aws/aws_rds/tasks/main.yml b/roles/aws/aws_rds/tasks/main.yml index 85d4878ba..33379e826 100644 --- a/roles/aws/aws_rds/tasks/main.yml +++ b/roles/aws/aws_rds/tasks/main.yml @@ -37,11 +37,9 @@ engine: "{{ aws_rds.engine }}" copy_tags_to_snapshot: true publicly_accessible: "{{ aws_rds.publicly_accessible }}" - allocated_storage: "{{ aws_rds.allocated_storage }}" profile: "{{ aws_rds.aws_profile }}" tags: "{{ aws_rds.tags | combine({ 'Name': aws_rds.name }) }}" allow_major_version_upgrade: false - storage_encrypted: "{{ aws_rds.storage_encrypted }}" apply_immediately: true wait: true region: "{{ aws_rds.region }}" From ef7b5b32c6ff578add4550df99cb74c567a5a54d Mon Sep 17 00:00:00 2001 From: Emlyn Kinzett Date: Fri, 4 Feb 2022 14:14:23 +0000 Subject: [PATCH 17/17] Add profile and region to read replica creation. --- roles/aws/aws_rds/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/aws/aws_rds/tasks/main.yml b/roles/aws/aws_rds/tasks/main.yml index 33379e826..c1a66411d 100644 --- a/roles/aws/aws_rds/tasks/main.yml +++ b/roles/aws/aws_rds/tasks/main.yml @@ -90,6 +90,8 @@ read_replica: true creation_source: "instance" source_db_instance_identifier: "{{ aws_rds.name }}" + profile: "{{ aws_rds.aws_profile }}" + region: "{{ aws_rds.region }}" when: aws_rds.replica - name: Create SNS topic.