From 894dd42318a3ef043fe3b3d22119ed6fae4ffef3 Mon Sep 17 00:00:00 2001 From: Alicia Cozine Date: Fri, 2 Feb 2024 12:31:43 -0600 Subject: [PATCH 01/11] templated nginx config work so far Co-authored-by: Christina Chortaria Co-authored-by: Francis Kayiwa Co-authored-by: Jane Sandberg Co-authored-by: Ryan Laddusaw Co-authored-by: Vickie Karasic --- roles/nginxplus/defaults/main.yml | 13 ++ .../files/conf/http/abid_staging.conf | 57 ------ .../files/conf/http/allsearch-api_prod.conf | 54 ------ .../conf/http/allsearch-api_staging.conf | 60 ------ roles/nginxplus/tasks/conf/upload-config.yml | 172 +++++++++++++---- .../nginxplus/templates/http/library.conf.j2 | 108 +++++++++++ roles/nginxplus/templates/nginx.conf.j2 | 1 + roles/nginxplus/vars/main.yml | 178 +++++++++++++++++- 8 files changed, 434 insertions(+), 209 deletions(-) delete mode 100644 roles/nginxplus/files/conf/http/abid_staging.conf delete mode 100644 roles/nginxplus/files/conf/http/allsearch-api_prod.conf delete mode 100644 roles/nginxplus/files/conf/http/allsearch-api_staging.conf create mode 100644 roles/nginxplus/templates/http/library.conf.j2 diff --git a/roles/nginxplus/defaults/main.yml b/roles/nginxplus/defaults/main.yml index 63a05591e8..e4d43fe7c3 100644 --- a/roles/nginxplus/defaults/main.yml +++ b/roles/nginxplus/defaults/main.yml @@ -105,6 +105,13 @@ nginx_main_upload_enable: false # Upload HTTP NGINX configuration files. # Set to false for testing nginx_http_upload_enable: false +nginx_http_upload_src: conf/http/*.conf +nginx_http_upload_dest: /etc/nginx/conf.d/ +nginx_http_dynamic_config_dest: /etc/nginx/conf.d/dynamic +# Upload Stream NGINX configuration files. +nginx_stream_upload_enable: false +nginx_stream_upload_src: conf/stream/*.conf +nginx_stream_upload_dest: /etc/nginx/conf.d/ # Upload HTML files. # Set to false for testing nginx_html_upload_enable: false @@ -431,3 +438,9 @@ nginx_stream_template: port: 8080 weight: 1 health_check: max_fails=1 fail_timeout=10s + +# manage the hash table that stores server names for fast lookup +# increase the size first, when that stops working, increase the number of hash buckets in the table +nginx_server_names_hash_bucket_size: 64 +nginx_server_names_hash_max_size: 1024 +client_max_body_size: 0 diff --git a/roles/nginxplus/files/conf/http/abid_staging.conf b/roles/nginxplus/files/conf/http/abid_staging.conf deleted file mode 100644 index f792f9e8da..0000000000 --- a/roles/nginxplus/files/conf/http/abid_staging.conf +++ /dev/null @@ -1,57 +0,0 @@ -# Ansible managed -proxy_cache_path /data/nginx/abid-staging/NGINX_cache/ keys_zone=abid-stagingcache:10m; - -upstream abid-staging { - zone abid-staging 64k; - server abid-staging2.princeton.edu resolve; - sticky learn - create=$upstream_cookie_abidstagingcookie - lookup=$cookie_abidstagingcookie - zone=abidclient_sessions:1m; -} - -server { - listen 80; - server_name abid-staging.princeton.edu; - - location / { - return 301 https://$server_name$request_uri; - } -} - -server { - listen 443 ssl; - http2 on; - server_name abid-staging.princeton.edu; - - ssl_certificate /etc/letsencrypt/live/abid-staging/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/abid-staging/privkey.pem; - ssl_session_cache shared:SSL:1m; - ssl_prefer_server_ciphers on; - - location / { - # app_protect_enable on; - # app_protect_security_log_enable on; - proxy_pass http://abid-staging; - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_cache abid-stagingcache; - proxy_connect_timeout 2h; - proxy_send_timeout 2h; - proxy_read_timeout 2h; - # handle errors using errors.conf - proxy_intercept_errors on; - # configure health checks - health_check interval=10 fails=3 passes=2; - # allow princeton network - include /etc/nginx/conf.d/templates/restrict.conf; - # block all IPs outside the princeton network - deny all; - } - - # both staging-maintenance.conf and errors.conf define error_page - # only use one! - # include /etc/nginx/conf.d/templates/staging-maintenance.conf; - include /etc/nginx/conf.d/templates/errors.conf; - -} diff --git a/roles/nginxplus/files/conf/http/allsearch-api_prod.conf b/roles/nginxplus/files/conf/http/allsearch-api_prod.conf deleted file mode 100644 index 20aeac8d28..0000000000 --- a/roles/nginxplus/files/conf/http/allsearch-api_prod.conf +++ /dev/null @@ -1,54 +0,0 @@ -# Ansible managed -proxy_cache_path /data/nginx/allsearch-api/NGINX_cache/ keys_zone=allsearch-apicache:10m; - -map $limit $external_traffic { - 0 ""; - 1 $binary_remote_addr; -} - -# zone: 10mb can hold 160K IP addresses in memory -limit_req_zone $external_traffic zone=allsearch-api-prod-ratelimit:10m rate=10r/s; - -upstream allsearch-api { - zone allsearch-api 128k; - least_conn; - server allsearch-api-prod1.princeton.edu resolve; - server allsearch-api-prod2.princeton.edu resolve; - sticky learn - create=$upstream_cookie_allsearch-apicookie - lookup=$cookie_allsearch-apicookie - zone=allsearch-apiclient_sessions:1m; -} - -server { - listen 80; - server_name allsearch-api.princeton.edu; - - location / { - return 301 https://$server_name$request_uri; - } -} - -server { - listen 443 ssl; - http2 on; - server_name allsearch-api.princeton.edu; - - client_max_body_size 8m; - - ssl_certificate /etc/letsencrypt/live/allsearch-api/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/allsearch-api/privkey.pem; - ssl_session_cache shared:SSL:1m; - ssl_prefer_server_ciphers on; - - location / { - proxy_pass http://allsearch-api; - # handle errors using errors.conf - proxy_intercept_errors on; - proxy_set_header Host $host; - limit_req zone=allsearch-api-prod-ratelimit burst=20 nodelay; - proxy_cache allsearch-apicache; - } - - include /etc/nginx/conf.d/templates/errors.conf; -} diff --git a/roles/nginxplus/files/conf/http/allsearch-api_staging.conf b/roles/nginxplus/files/conf/http/allsearch-api_staging.conf deleted file mode 100644 index 1ea6e8c5c8..0000000000 --- a/roles/nginxplus/files/conf/http/allsearch-api_staging.conf +++ /dev/null @@ -1,60 +0,0 @@ -# Ansible managed -proxy_cache_path /data/nginx/allsearch-api-staging/NGINX_cache/ keys_zone=allsearch-api-stagingcache:10m; - -map $limit $external_traffic { - 0 ""; - 1 $binary_remote_addr; -} - -# zone: 10mb can hold 160K IP addresses in memory -limit_req_zone $external_traffic zone=allsearch-api-staging-ratelimit:10m rate=10r/s; - -upstream allsearch-api-staging { - zone allsearch-api-staging 64k; - least_conn; - server allsearch-api-staging1.princeton.edu resolve; - sticky learn - create=$upstream_cookie_allsearch-apistagingcookie - lookup=$cookie_allsearch-apistagingcookie - zone=allsearch-apistagingclient_sessions:1m; -} - -server { - listen 80; - server_name allsearch-api-staging.princeton.edu; - - location / { - return 301 https://$server_name$request_uri; - } -} - -server { - listen 443 ssl; - http2 on; - server_name allsearch-api-staging.princeton.edu; - - client_max_body_size 8m; - - ssl_certificate /etc/letsencrypt/live/allsearch-api-staging/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/allsearch-api-staging/privkey.pem; - ssl_session_cache shared:SSL:1m; - ssl_prefer_server_ciphers on; - - location / { - app_protect_enable on; - app_protect_security_log_enable on; - proxy_pass http://allsearch-api-staging; - proxy_cache allsearch-api-stagingcache; - # handle errors using errors.conf - proxy_intercept_errors on; - proxy_set_header Host $host; - limit_req zone=allsearch-api-staging-ratelimit burst=20 nodelay; - # allow princeton network - include /etc/nginx/conf.d/templates/restrict.conf; - # block all - deny all; - } - - include /etc/nginx/conf.d/templates/errors.conf; - -} diff --git a/roles/nginxplus/tasks/conf/upload-config.yml b/roles/nginxplus/tasks/conf/upload-config.yml index 18103e3850..68abbb0a93 100644 --- a/roles/nginxplus/tasks/conf/upload-config.yml +++ b/roles/nginxplus/tasks/conf/upload-config.yml @@ -1,6 +1,6 @@ --- -- name: "Setup: Upload NGINX Main Configuration File" - copy: +- name: Setup | Upload NGINX Main Configuration File + ansible.builtin.copy: src: "{{ nginx_main_upload_src }}" dest: "{{ nginx_main_upload_dest }}" mode: 0755 @@ -8,22 +8,22 @@ notify: Reload NGINX when: nginx_main_upload_enable -- name: "Setup: Ensure NGINX HTTP Directory Exists" - file: +- name: Setup | Ensure NGINX HTTP Directory Exists + ansible.builtin.file: path: "{{ nginx_http_upload_dest }}" state: directory mode: 0755 when: nginx_http_upload_enable -- name: "Setup: Ensure NGINX template Directory Exists" - file: +- name: Setup | Ensure NGINX partial config "template" Directory Exists + ansible.builtin.file: path: "{{ nginx_template_upload_dest }}" state: directory mode: 0755 when: nginx_template_upload_enable -- name: "Setup: Upload NGINX HTTP Configuration Files" - copy: +- name: Setup | Upload NGINX HTTP Static Site Config Files + ansible.builtin.copy: src: "{{ item }}" dest: "{{ nginx_http_upload_dest }}" mode: 0755 @@ -31,10 +31,64 @@ with_fileglob: "{{ nginx_http_upload_src }}" notify: Reload NGINX when: nginx_http_upload_enable - tags: update_conf + register: uploaded_static_files + tags: + - update_conf + - test_template + +# identify and remove obsolete static files +- name: Setup | list files in static config dir + ansible.builtin.find: + paths: "{{ nginx_http_upload_dest }}/" + recurse: false + file_type: file + register: all_static_files + tags: test_template + +- name: Setup | set facts for removing obsolete static config files + set_fact: + static_files_on_server: "{{ all_static_files.files|map(attribute='path') }}" + static_files_we_want: "{{ uploaded_static_files.results|map(attribute='dest') }}" + tags: test_template + +- name: Debug - view static files on server + ansible.builtin.debug: + var: static_files_on_server + tags: test_template + +- name: Debug - view static files we want + ansible.builtin.debug: + var: static_files_we_want + tags: test_template + +- name: Find obsolete static files + # ITEMS IN static_files_on_server NOT IN static_files_we_want + set_fact: obsolete_static_files="{{ static_files_on_server | difference(static_files_we_want) }}" + tags: test_template + +- name: Debug - view obsolete static files + ansible.builtin.debug: + var: obsolete_static_files + tags: test_template -- name: "Setup: Upload NGINX template Configuration Files" - copy: +# when we're confident this works: +# - name: Delete obsolete static files +# ansible.builtin.file: +# path: "{{ item }}" +# state: absent +# loop: "{{ obsolete_static_files }}" +# tags: test_template + +- name: Setup | Ensure cache directories exist + ansible.builtin.file: + path: /tmp/nginx/{{ item.short_name }}/NGINX_cache + state: directory + mode: 0755 + notify: Reload NGINX + loop: "{{ sites }}" + +- name: Setup | Upload NGINX partial config "template" files + ansible.builtin.copy: src: "{{ item }}" dest: "{{ nginx_template_upload_dest }}" mode: 0755 @@ -44,35 +98,79 @@ when: nginx_template_upload_enable tags: update_conf -- name: "Setup: Ensure NGINX Stream Directory Exists" - file: - path: "{{ nginx_stream_upload_dest }}" +- name: Setup | Ensure NGINX HTTP Dynamic Site Config Directory Exists + ansible.builtin.file: + path: "{{ nginx_http_dynamic_config_dest }}" state: directory mode: 0755 - when: nginx_stream_upload_enable - tags: update_conf + when: nginx_template_upload_enable + tags: test_template -- name: "Setup: Upload NGINX Stream Configuration Files" - copy: - src: "{{ item }}" - dest: "{{ nginx_stream_upload_dest }}" +- name: Setup | Upload NGINX HTTP Dynamic Site Config Files + ansible.builtin.template: + src: http/library.conf.j2 + dest: "{{ nginx_http_dynamic_config_dest }}/{{ item.name }}.conf" + # dest: /tmp/{{ item.name }}.conf + # dest: /etc/nginx/conf.d/{{ item.name }}.conf mode: 0755 - backup: true - with_fileglob: "{{ nginx_stream_upload_src }}" - notify: Reload NGINX - when: nginx_stream_upload_enable - tags: update_conf + when: nginx_http_upload_enable + loop: "{{ sites }}" + tags: + - update_conf + - test_template + register: sites_loop + +- name: Setup | list files in dynamic config dir + ansible.builtin.find: + paths: "{{ nginx_http_dynamic_config_dest }}/" + recurse: false + file_type: file + register: all_files + tags: test_template + +- name: Setup | set facts for removing obsolete dynamic config files + set_fact: + files_on_server: "{{ all_files.files|map(attribute='path') }}" + sites_we_want: "{{ sites_loop.results|map(attribute='dest') }}" + tags: test_template + +- name: Debug - view files on server + ansible.builtin.debug: + var: files_on_server + tags: test_template + +- name: Debug - view sites we want + ansible.builtin.debug: + var: sites_we_want + tags: test_template + +- name: Find obsolete dynamic config files + # ITEMS in files on server NOT in sites we want + set_fact: obsolete_files="{{ files_on_server | difference(sites_we_want) }}" + tags: test_template + +- name: Debug - view obsolete dynamic config files + ansible.builtin.debug: + var: obsolete_files + tags: test_template + +- name: Delete obsolete dynamic config files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: "{{ obsolete_files }}" + tags: test_template -- name: "Setup: Ensure NGINX HTML Directory Exists" - file: +- name: Setup | Ensure NGINX HTML Directory Exists + ansible.builtin.file: path: "{{ nginx_html_upload_dest }}" state: directory mode: 0755 when: nginx_html_upload_enable tags: update_conf -- name: "Setup: Upload NGINX HTML Files" - copy: +- name: Setup | Upload NGINX HTML Files + ansible.builtin.copy: src: "{{ item }}" dest: "{{ nginx_html_upload_dest }}" mode: 0755 @@ -82,23 +180,23 @@ when: nginx_html_upload_enable tags: update_conf -- name: "Setup: Ensure SSL Certificate Directory Exists" - file: +- name: Setup | Ensure SSL Certificate Directory Exists + ansible.builtin.file: path: "{{ nginx_ssl_crt_upload_dest }}" state: directory mode: 0755 when: nginx_ssl_upload_enable -- name: "Setup: Ensure SSL Key Directory Exists" - file: +- name: Setup | Ensure SSL Key Directory Exists + ansible.builtin.file: path: "{{ nginx_ssl_key_upload_dest }}" state: directory mode: 0755 when: nginx_ssl_upload_enable tags: SSL -- name: "Setup: Upload NGINX SSL Certificates" - copy: +- name: Setup | Upload NGINX SSL Certificates + ansible.builtin.copy: src: "{{ item }}" dest: "{{ nginx_ssl_crt_upload_dest }}" mode: 0640 @@ -108,8 +206,8 @@ when: nginx_ssl_upload_enable tags: SSL -- name: "(Setup: All NGINX) Upload NGINX SSL Keys" - copy: +- name: Setup | All NGINX | Upload NGINX SSL Keys + ansible.builtin.copy: src: "{{ item }}" dest: "{{ nginx_ssl_key_upload_dest }}" mode: 0640 diff --git a/roles/nginxplus/templates/http/library.conf.j2 b/roles/nginxplus/templates/http/library.conf.j2 new file mode 100644 index 0000000000..4a11a5a553 --- /dev/null +++ b/roles/nginxplus/templates/http/library.conf.j2 @@ -0,0 +1,108 @@ +# Ansible managed +# local changes will be overwritten with each playbook run +# +proxy_cache_path /data/nginx/{{ item.short_name }}/NGINX_cache/ keys_zone={{ item.short_name }}cache:10m; + +{% if item.rate_limit is defined %} + include /etc/nginx/conf.d/templates/rate-limit-allow-list.conf; + + map $limit $external_traffic { + 0 ""; + 1 $binary_remote_addr; + } + + # zone: 10mb can hold 160K IP addresses in memory + limit_req_zone $external_traffic zone={{ item.name }}-ratelimit:{{ item.rate_limit_mb }}m rate={{ item.rate_limit_per_sec }}r/s; +{% endif %} + +upstream {{ item.short_name }} { + {% if item.lbtechnique is defined %} + {{ item.lbtechnique }}; + {% endif %} + zone {{ item.short_name }} {{ item.zone_size | default("64k") }}; + {% for name in item.servers %} + server {{ name }}.princeton.edu resolve; + {% endfor %} + sticky learn + create=$upstream_cookie_{{ item.short_name }}cookie + lookup=$cookie_{{ item.short_name }}cookie + zone={{ item.short_name }}client_sessions:1m; +} + +# redirect HTTP traffic to HTTPS +server { + listen 80; + server_name {{ item.short_name }}.princeton.edu; + + location / { + return 301 https://$server_name$request_uri; + } +} + +server { + listen 443 ssl; + http2 on; + server_name {{ item.short_name }}.princeton.edu; + + ssl_certificate /etc/letsencrypt/live/{{ item.name }}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/{{ item.name }}/privkey.pem; + ssl_session_cache shared:SSL:1m; + ssl_prefer_server_ciphers on; + client_max_body_size {{ client_max_body_size }}; + + {% if item.visibility == "public" %} + location {{ item.location }} { + {% if item.app_protect == "enabled" %} + app_protect_enable on; + app_protect_security_log_enable on; + {% endif %} + proxy_pass http://{{ item.short_name }}; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_cache {{ item.short_name }}cache; + proxy_connect_timeout {{ item.proxy_connect_timeout | default("60s") }}; + proxy_send_timeout {{ item.proxy_send_timeout | default("60s") }}; + proxy_read_timeout {{ item.proxy_read_timeout | default("60s") }}; + # handle errors using errors.conf + proxy_intercept_errors on; + # TODO don't do a health check if there's only one server + {% if item.health_check_URI is defined %} + health_check interval=10 fails=3 passes=2 uri={{ item.health_check_URI }}; + {% else %} + health_check interval=10 fails=3 passes=2; + {% endif %} + {% if item.rate_limit is defined %} + limit_req zone={{ item.name }}-ratelimit burst={{ item.rate_limit_burst_rate }} nodelay; + {% endif %} + } + {% else %} + location {{ item.location }} { + {% if item.app_protect == "enabled" %} + app_protect_enable on; + app_protect_security_log_enable on; + {% endif %} + proxy_pass http://{{ item.short_name }}; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_cache {{ item.short_name }}cache; + proxy_connect_timeout {{ item.proxy_connect_timeout | default("60s") }}; + proxy_send_timeout {{ item.proxy_send_timeout | default("60s") }}; + proxy_read_timeout {{ item.proxy_read_timeout | default("60s") }}; + proxy_intercept_errors on; + health_check interval=10 fails=3 passes=2; + # allow princeton network + include /etc/nginx/conf.d/templates/restrict.conf; + {% if item.added_restrictions is defined %} + {% for name in item.added_restrictions %} + include /etc/nginx/conf.d/templates/{{ name }}; + {% endfor %} + {% endif %} + # block non-princeton traffic + deny all; + } + {% endif %} + + # template config for error handling + include /etc/nginx/conf.d/templates/{{ error_handling_template | default("errors.conf") }}; + +} diff --git a/roles/nginxplus/templates/nginx.conf.j2 b/roles/nginxplus/templates/nginx.conf.j2 index 9afeac8407..fca2d8e34d 100644 --- a/roles/nginxplus/templates/nginx.conf.j2 +++ b/roles/nginxplus/templates/nginx.conf.j2 @@ -90,6 +90,7 @@ http { autoindex on; {% endif %} include /etc/nginx/conf.d/*.conf; + include /etc/nginx/conf.d/dynamic/*.conf; } {% endif %} diff --git a/roles/nginxplus/vars/main.yml b/roles/nginxplus/vars/main.yml index 7d1d7df3c8..d6c67b02a7 100644 --- a/roles/nginxplus/vars/main.yml +++ b/roles/nginxplus/vars/main.yml @@ -1,2 +1,178 @@ --- -# vars file for roles/nginxplus +# the 'sites' dictionary is used by the library.conf.j2 template +sites: + # - name: example-prod + ## 'servers' is a list of VMs that serve this site + # servers: + # - example-prod1 + # - example-prod2 + ## short_name is used for cookies for sticky learning + # short_name: example + ## zone_size is the amount of memory for the cache zone + ## default is 64 + # zone_size: 128 + ## location defines the URL for the nginx location directive + # location: "/" + ## the template defaults to restricted configuration (VPN-only) + ## set 'visibility: public' to create a site that is visible to the world + # visibility: public + ## default load balancing technique is round-robin + ## see https://www.nginx.com/blog/choosing-nginx-plus-load-balancing-techniques/ for options + # lbtechnique: least_time + ## most sites have app_protect disabled + ## set app_protect: enabled to enable it + # app_protect: enabled + ## client_max_body_size defines the upper limit on request sizes + ## default is 0 (don't check), if you change this, you must include the unit of measurement + # client_max_body_size: 8m + ## proxy timeouts default to 60 seconds + ## to increase or decrease these values, set: + # proxy_connect_timeout: 2h + # proxy_send_timeout: 2h + # proxy_read_timeout: 2h + ## error handling templates live in /etc/nginx/conf.d/templates/ + ## the default template for handling errors is errors.conf + # error_handling_template: cdh-errors.conf + ## the load balancers do health checks at the root of the site + ## set health_check_URI to define a different health URI + # health_check_URI: /solr + # - name: abid-prod + # servers: + # - abid-prod1 + # - abid-prod2 + # short_name: abid + # location: "/" + # visibility: public + # app_protect: disabled + - name: abid-staging + servers: + - abid-staging1 + short_name: abid-staging + location: "/" + visibility: private + app_protect: enabled + - name: allsearch-api-prod + servers: + - allsearch-api-prod1 + - allsearch-api-prod2 + short_name: allsearch-api + location: "/" + visibility: public + lbtechnique: least_conn + client_max_body_size: 512k + app_protect: disabled + rate_limit: enabled + rate_limit_mb: 10 + rate_limit_per_sec: 10 + rate_limit_burst_rate: 20 + # error_handling_template: changeme + - name: allsearch-api-staging + servers: + - allsearch-api-staging1 + short_name: allsearch-api-staging + location: "/" + visibility: private + lbtechnique: least_conn + client_max_body_size: 512k + app_protect: enabled + # error_handling_template: changeme + + # - name: annotations-prod + # servers: + # - annotations-prod1 + # - annotations-prod2 + # short_name: annotations + # location: "/" + # visibility: public + # lbtechnique: least_conn + # app_protect: disabled + # - name: annotations-staging + # servers: + # - annotations-staging1 + # short_name: annotations-staging + # location: "/" + # visibility: private + # lbtechnique: least_conn + # app_protect: disabled + # - name: ansible-tower + # servers: + # - ansible-tower1 + # short_name: ansible-tower + # location: "/" + # visibility: public + # app_protect: disabled + # - name: approvals-prod + # servers: + # - lib-approvals-prod1 + # - lib-approvals-prod2 + # short_name: approvals + # location: "/" + # visibility: public + # app_protect: disabled + # - name: approvals-staging + # servers: + # - lib-approvals-staging1 + # - lib-approvals-staging2 + # short_name: approvals-staging + # location: "/" + # visibility: private + # app_protect: disabled + # - name: bibdata-prod + # servers: + # - bibdata-alma1 + # - bibdata-alma2 + # short_name: bibdata + # location: "/" + # visibility: public + # app_protect: disabled + # proxy_connect_timeout: 2h + # proxy_send_timeout: 2h + # proxy_read_timeout: 2h + # - name: bibdata-qa + # servers: + # # - bibdata-qa1 + # - bibdata-qa2 + # short_name: bibdata-qa + # location: "/" + # visibility: private + # added_restrictions: htc_restrict.conf + # app_protect: disabled + # proxy_connect_timeout: 2h + # proxy_send_timeout: 2h + # proxy_read_timeout: 2h + - name: bibdata-staging + servers: + # - bibdata-alma-staging1 + - bibdata-alma-staging2 + short_name: bibdata-staging + location: "/" + visibility: private + added_restrictions: + - "htc_restrict.conf" + app_protect: enabled + proxy_connect_timeout: 2h + proxy_send_timeout: 2h + proxy_read_timeout: 2h + # - name: byzantine-tsp-prod + # servers: + # - byzantine-tsp-prod + # short_name: byzantine-tsp + # location: "/" + # visibility: public + # app_protect: disabled + # - name: byzantine-tsp-staging + # servers: + # - byzantine-tsp-staging1 + # short_name: byzantine-tsp-staging + # location: "/" + # visibility: public + # app_protect: disabled + # - name: catalog-qa + # servers: + # # - catalog-qa1 + # - catalog-qa2 + # short_name: catalogqa + # location: "/" + # visibility: public + # app_protect: disabled + # health_check_URI: "/catalog/1234567" From aec0a95e97df53b3bf17c88cbd49c80d34d6c708 Mon Sep 17 00:00:00 2001 From: Alicia Cozine Date: Fri, 2 Feb 2024 13:41:46 -0600 Subject: [PATCH 02/11] do not make backups of static config files --- roles/nginxplus/tasks/conf/upload-config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/nginxplus/tasks/conf/upload-config.yml b/roles/nginxplus/tasks/conf/upload-config.yml index 68abbb0a93..ce64675288 100644 --- a/roles/nginxplus/tasks/conf/upload-config.yml +++ b/roles/nginxplus/tasks/conf/upload-config.yml @@ -27,7 +27,6 @@ src: "{{ item }}" dest: "{{ nginx_http_upload_dest }}" mode: 0755 - backup: true with_fileglob: "{{ nginx_http_upload_src }}" notify: Reload NGINX when: nginx_http_upload_enable From 48f34ab641095c02b0c63a7cf010b4fa44291867 Mon Sep 17 00:00:00 2001 From: Alicia Cozine Date: Fri, 2 Feb 2024 15:33:51 -0600 Subject: [PATCH 03/11] make a backup of all config --- roles/nginxplus/tasks/conf/upload-config.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/roles/nginxplus/tasks/conf/upload-config.yml b/roles/nginxplus/tasks/conf/upload-config.yml index ce64675288..19a5e0d87c 100644 --- a/roles/nginxplus/tasks/conf/upload-config.yml +++ b/roles/nginxplus/tasks/conf/upload-config.yml @@ -22,6 +22,14 @@ mode: 0755 when: nginx_template_upload_enable +- name: Setup | Make a backup of nginx config files dir + ansible.builtin.copy: + src: "{{ nginx_http_upload_dest }}" + dest: /tmp/nginx_backup + mode: 0755 + remote_src: true + tags: test_template + - name: Setup | Upload NGINX HTTP Static Site Config Files ansible.builtin.copy: src: "{{ item }}" @@ -215,3 +223,4 @@ with_fileglob: "{{ nginx_ssl_key_upload_src }}" when: nginx_ssl_upload_enable tags: SSL + From 85ef21542ba8b0e8288d3573b4fb0f8579983f92 Mon Sep 17 00:00:00 2001 From: Alicia Cozine Date: Tue, 6 Feb 2024 11:50:12 -0600 Subject: [PATCH 04/11] make the config validation run when testing templatized configs Co-authored-by: Carolyn Cole Co-authored-by: Christina Chortaria Co-authored-by: Francis Kayiwa Co-authored-by: Ryan Laddusaw --- roles/nginxplus/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/nginxplus/tasks/main.yml b/roles/nginxplus/tasks/main.yml index 27f4c7582f..cc732111b0 100644 --- a/roles/nginxplus/tasks/main.yml +++ b/roles/nginxplus/tasks/main.yml @@ -35,6 +35,10 @@ or nginx_stream_template_enable or nginx_rest_api_enable + - import_tasks: validate.yml + when: nginx_type == "plus" + tags: test_template + - import_tasks: conf/setup-status.yml when: nginx_status_enable From 11c23471351d63cc17eea6c06b7d9448b16e187a Mon Sep 17 00:00:00 2001 From: Alicia Cozine Date: Thu, 8 Feb 2024 17:38:52 -0600 Subject: [PATCH 05/11] handle 'nginx -t' failures with custom message Co-authored-by: Jane Sandberg --- roles/nginxplus/tasks/validate.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/roles/nginxplus/tasks/validate.yml b/roles/nginxplus/tasks/validate.yml index 6123626b69..e258db3489 100644 --- a/roles/nginxplus/tasks/validate.yml +++ b/roles/nginxplus/tasks/validate.yml @@ -1,6 +1,19 @@ --- - name: nginxplus | make sure nginx configs are valid + # ignoring errors so we can provide a custom message if validation fails command: /usr/sbin/nginx -t changed_when: false + ignore_errors: true become: true tags: always + register: validate_config_msg + +- name: display full failure message + ansible.builtin.debug: + var: validate_config_msg + when: validate_config_msg.rc != 0 or 'test failed' in validate_config_msg.stderr + +- name: fail with message if nginx -t errors out + ansible.builtin.fail: + msg: "Nginx config update failed. Time to get a human involved, please restore from /tmp/nginx_backup." + when: validate_config_msg.rc != 0 or 'test failed' in validate_config_msg.stderr From ac3eca0a84d399d6164349bdfe7f6a91e4317194 Mon Sep 17 00:00:00 2001 From: Alicia Cozine Date: Fri, 9 Feb 2024 10:24:42 -0600 Subject: [PATCH 06/11] make backup dir a variable --- roles/nginxplus/defaults/main.yml | 3 +++ roles/nginxplus/tasks/conf/upload-config.yml | 3 +-- roles/nginxplus/tasks/validate.yml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/roles/nginxplus/defaults/main.yml b/roles/nginxplus/defaults/main.yml index e4d43fe7c3..5d738e933d 100644 --- a/roles/nginxplus/defaults/main.yml +++ b/roles/nginxplus/defaults/main.yml @@ -444,3 +444,6 @@ nginx_stream_template: nginx_server_names_hash_bucket_size: 64 nginx_server_names_hash_max_size: 1024 client_max_body_size: 0 + +# define the directory where we copy config before deploying changes, for rollback +config_backup_dir: /tmp/nginx_backup diff --git a/roles/nginxplus/tasks/conf/upload-config.yml b/roles/nginxplus/tasks/conf/upload-config.yml index 19a5e0d87c..785da3dc1a 100644 --- a/roles/nginxplus/tasks/conf/upload-config.yml +++ b/roles/nginxplus/tasks/conf/upload-config.yml @@ -25,7 +25,7 @@ - name: Setup | Make a backup of nginx config files dir ansible.builtin.copy: src: "{{ nginx_http_upload_dest }}" - dest: /tmp/nginx_backup + dest: "{{ config_backup_dir }}" mode: 0755 remote_src: true tags: test_template @@ -223,4 +223,3 @@ with_fileglob: "{{ nginx_ssl_key_upload_src }}" when: nginx_ssl_upload_enable tags: SSL - diff --git a/roles/nginxplus/tasks/validate.yml b/roles/nginxplus/tasks/validate.yml index e258db3489..a7ad9c3136 100644 --- a/roles/nginxplus/tasks/validate.yml +++ b/roles/nginxplus/tasks/validate.yml @@ -15,5 +15,5 @@ - name: fail with message if nginx -t errors out ansible.builtin.fail: - msg: "Nginx config update failed. Time to get a human involved, please restore from /tmp/nginx_backup." + msg: "Nginx config update failed. Time to get a human involved, please restore from {{ config_backup_dir }}." when: validate_config_msg.rc != 0 or 'test failed' in validate_config_msg.stderr From 30067b4189cdd45c507ff71d779bd91c5c297cb8 Mon Sep 17 00:00:00 2001 From: Alicia Cozine Date: Fri, 9 Feb 2024 10:38:28 -0600 Subject: [PATCH 07/11] creates static subdir for nginx config files, loads static and dynamic dirs in main config --- roles/nginxplus/defaults/main.yml | 2 +- roles/nginxplus/templates/nginx.conf.j2 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/nginxplus/defaults/main.yml b/roles/nginxplus/defaults/main.yml index 5d738e933d..19e8a46dd2 100644 --- a/roles/nginxplus/defaults/main.yml +++ b/roles/nginxplus/defaults/main.yml @@ -106,7 +106,7 @@ nginx_main_upload_enable: false # Set to false for testing nginx_http_upload_enable: false nginx_http_upload_src: conf/http/*.conf -nginx_http_upload_dest: /etc/nginx/conf.d/ +nginx_http_upload_dest: /etc/nginx/conf.d/static nginx_http_dynamic_config_dest: /etc/nginx/conf.d/dynamic # Upload Stream NGINX configuration files. nginx_stream_upload_enable: false diff --git a/roles/nginxplus/templates/nginx.conf.j2 b/roles/nginxplus/templates/nginx.conf.j2 index fca2d8e34d..021858d34d 100644 --- a/roles/nginxplus/templates/nginx.conf.j2 +++ b/roles/nginxplus/templates/nginx.conf.j2 @@ -61,7 +61,7 @@ http { ~^[23] 0; default 1; } - + access_log /var/log/nginx/access.log combined if=$loggable; @@ -89,7 +89,7 @@ http { {% if nginx_main_template.http_global_autoindex | default(false) %} autoindex on; {% endif %} - include /etc/nginx/conf.d/*.conf; + include /etc/nginx/conf.d/static/*.conf; include /etc/nginx/conf.d/dynamic/*.conf; } {% endif %} From 13b418886af87c462bcfa97e77b60030c5e22dc3 Mon Sep 17 00:00:00 2001 From: Alicia Cozine Date: Fri, 9 Feb 2024 10:50:57 -0600 Subject: [PATCH 08/11] turn on removing obsolete static files, make var names clearer --- roles/nginxplus/tasks/conf/upload-config.yml | 34 +++++++++----------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/roles/nginxplus/tasks/conf/upload-config.yml b/roles/nginxplus/tasks/conf/upload-config.yml index 785da3dc1a..10fc8a9cb7 100644 --- a/roles/nginxplus/tasks/conf/upload-config.yml +++ b/roles/nginxplus/tasks/conf/upload-config.yml @@ -78,13 +78,13 @@ var: obsolete_static_files tags: test_template -# when we're confident this works: -# - name: Delete obsolete static files -# ansible.builtin.file: -# path: "{{ item }}" -# state: absent -# loop: "{{ obsolete_static_files }}" -# tags: test_template +- name: Delete obsolete static files + # will remove static files as we migrate sites to dynamic config + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: "{{ obsolete_static_files }}" + tags: test_template - name: Setup | Ensure cache directories exist ansible.builtin.file: @@ -117,55 +117,53 @@ ansible.builtin.template: src: http/library.conf.j2 dest: "{{ nginx_http_dynamic_config_dest }}/{{ item.name }}.conf" - # dest: /tmp/{{ item.name }}.conf - # dest: /etc/nginx/conf.d/{{ item.name }}.conf mode: 0755 when: nginx_http_upload_enable loop: "{{ sites }}" tags: - update_conf - test_template - register: sites_loop + register: uploaded_dynamic_files - name: Setup | list files in dynamic config dir ansible.builtin.find: paths: "{{ nginx_http_dynamic_config_dest }}/" recurse: false file_type: file - register: all_files + register: all_dynamic_files tags: test_template - name: Setup | set facts for removing obsolete dynamic config files set_fact: - files_on_server: "{{ all_files.files|map(attribute='path') }}" - sites_we_want: "{{ sites_loop.results|map(attribute='dest') }}" + dynamic_files_on_server: "{{ all_dynamic_files.files|map(attribute='path') }}" + dynamic_files_we_want: "{{ uploaded_dynamic_files.results|map(attribute='dest') }}" tags: test_template - name: Debug - view files on server ansible.builtin.debug: - var: files_on_server + var: dynamic_files_on_server tags: test_template - name: Debug - view sites we want ansible.builtin.debug: - var: sites_we_want + var: dynamic_files_we_want tags: test_template - name: Find obsolete dynamic config files # ITEMS in files on server NOT in sites we want - set_fact: obsolete_files="{{ files_on_server | difference(sites_we_want) }}" + set_fact: obsolete_dynamic_files="{{ dynamic_files_on_server | difference(dynamic_files_we_want) }}" tags: test_template - name: Debug - view obsolete dynamic config files ansible.builtin.debug: - var: obsolete_files + var: obsolete_dynamic_files tags: test_template - name: Delete obsolete dynamic config files ansible.builtin.file: path: "{{ item }}" state: absent - loop: "{{ obsolete_files }}" + loop: "{{ obsolete_dynamic_files }}" tags: test_template - name: Setup | Ensure NGINX HTML Directory Exists From cf28802a6a8dae3fe51a233f98f7ce8a8528e2a1 Mon Sep 17 00:00:00 2001 From: Alicia Cozine Date: Fri, 9 Feb 2024 12:00:12 -0600 Subject: [PATCH 09/11] removes task that deleted all nginx config --- roles/nginxplus/tasks/conf/cleanup-config.yml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 roles/nginxplus/tasks/conf/cleanup-config.yml diff --git a/roles/nginxplus/tasks/conf/cleanup-config.yml b/roles/nginxplus/tasks/conf/cleanup-config.yml deleted file mode 100644 index 5222942689..0000000000 --- a/roles/nginxplus/tasks/conf/cleanup-config.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: "Setup: Remove NGINX configuration files" - file: - path: "{{ item }}" - state: absent - with_items: - - "{{ nginx_cleanup_config_path }}" From 07e825f9ddfc6f8809c38a8690e8f2a2b7d65a8e Mon Sep 17 00:00:00 2001 From: Alicia Cozine Date: Fri, 9 Feb 2024 14:10:04 -0600 Subject: [PATCH 10/11] remove tag for testing template work --- roles/nginxplus/tasks/conf/upload-config.yml | 59 +++++++++++++------- roles/nginxplus/tasks/main.yml | 1 - 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/roles/nginxplus/tasks/conf/upload-config.yml b/roles/nginxplus/tasks/conf/upload-config.yml index 10fc8a9cb7..3c90971468 100644 --- a/roles/nginxplus/tasks/conf/upload-config.yml +++ b/roles/nginxplus/tasks/conf/upload-config.yml @@ -28,7 +28,8 @@ dest: "{{ config_backup_dir }}" mode: 0755 remote_src: true - tags: test_template + tags: + - update_conf - name: Setup | Upload NGINX HTTP Static Site Config Files ansible.builtin.copy: @@ -41,7 +42,6 @@ register: uploaded_static_files tags: - update_conf - - test_template # identify and remove obsolete static files - name: Setup | list files in static config dir @@ -50,33 +50,39 @@ recurse: false file_type: file register: all_static_files - tags: test_template + tags: + - update_conf - name: Setup | set facts for removing obsolete static config files set_fact: static_files_on_server: "{{ all_static_files.files|map(attribute='path') }}" static_files_we_want: "{{ uploaded_static_files.results|map(attribute='dest') }}" - tags: test_template + tags: + - update_conf - name: Debug - view static files on server ansible.builtin.debug: var: static_files_on_server - tags: test_template + tags: + - update_conf - name: Debug - view static files we want ansible.builtin.debug: var: static_files_we_want - tags: test_template + tags: + - update_conf - name: Find obsolete static files # ITEMS IN static_files_on_server NOT IN static_files_we_want set_fact: obsolete_static_files="{{ static_files_on_server | difference(static_files_we_want) }}" - tags: test_template + tags: + - update_conf - name: Debug - view obsolete static files ansible.builtin.debug: var: obsolete_static_files - tags: test_template + tags: + - update_conf - name: Delete obsolete static files # will remove static files as we migrate sites to dynamic config @@ -84,7 +90,8 @@ path: "{{ item }}" state: absent loop: "{{ obsolete_static_files }}" - tags: test_template + tags: + - update_conf - name: Setup | Ensure cache directories exist ansible.builtin.file: @@ -103,7 +110,8 @@ with_fileglob: "{{ nginx_template_upload_src }}" notify: Reload NGINX when: nginx_template_upload_enable - tags: update_conf + tags: + - update_conf - name: Setup | Ensure NGINX HTTP Dynamic Site Config Directory Exists ansible.builtin.file: @@ -111,7 +119,8 @@ state: directory mode: 0755 when: nginx_template_upload_enable - tags: test_template + tags: + - update_conf - name: Setup | Upload NGINX HTTP Dynamic Site Config Files ansible.builtin.template: @@ -122,7 +131,6 @@ loop: "{{ sites }}" tags: - update_conf - - test_template register: uploaded_dynamic_files - name: Setup | list files in dynamic config dir @@ -131,40 +139,47 @@ recurse: false file_type: file register: all_dynamic_files - tags: test_template + tags: + - update_conf - name: Setup | set facts for removing obsolete dynamic config files set_fact: dynamic_files_on_server: "{{ all_dynamic_files.files|map(attribute='path') }}" dynamic_files_we_want: "{{ uploaded_dynamic_files.results|map(attribute='dest') }}" - tags: test_template + tags: + - update_conf - name: Debug - view files on server ansible.builtin.debug: var: dynamic_files_on_server - tags: test_template + tags: + - update_conf - name: Debug - view sites we want ansible.builtin.debug: var: dynamic_files_we_want - tags: test_template + tags: + - update_conf - name: Find obsolete dynamic config files # ITEMS in files on server NOT in sites we want set_fact: obsolete_dynamic_files="{{ dynamic_files_on_server | difference(dynamic_files_we_want) }}" - tags: test_template + tags: + - update_conf - name: Debug - view obsolete dynamic config files ansible.builtin.debug: var: obsolete_dynamic_files - tags: test_template + tags: + - update_conf - name: Delete obsolete dynamic config files ansible.builtin.file: path: "{{ item }}" state: absent loop: "{{ obsolete_dynamic_files }}" - tags: test_template + tags: + - update_conf - name: Setup | Ensure NGINX HTML Directory Exists ansible.builtin.file: @@ -172,7 +187,8 @@ state: directory mode: 0755 when: nginx_html_upload_enable - tags: update_conf + tags: + - update_conf - name: Setup | Upload NGINX HTML Files ansible.builtin.copy: @@ -183,7 +199,8 @@ with_fileglob: "{{ nginx_html_upload_src }}" notify: Reload NGINX when: nginx_html_upload_enable - tags: update_conf + tags: + - update_conf - name: Setup | Ensure SSL Certificate Directory Exists ansible.builtin.file: diff --git a/roles/nginxplus/tasks/main.yml b/roles/nginxplus/tasks/main.yml index cc732111b0..d7cd9ee7c0 100644 --- a/roles/nginxplus/tasks/main.yml +++ b/roles/nginxplus/tasks/main.yml @@ -37,7 +37,6 @@ - import_tasks: validate.yml when: nginx_type == "plus" - tags: test_template - import_tasks: conf/setup-status.yml when: nginx_status_enable From 7020a5ac5de0c3d3703e03f87864e385aca6baf5 Mon Sep 17 00:00:00 2001 From: Alicia Cozine Date: Tue, 5 Mar 2024 15:14:43 -0600 Subject: [PATCH 11/11] removes import task for deleted tasks file, we do config removal differently now Co-authored-by: Carolyn Cole Co-authored-by: Jane Sandberg Co-authored-by: regineheberlein --- roles/nginxplus/tasks/main.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/roles/nginxplus/tasks/main.yml b/roles/nginxplus/tasks/main.yml index d7cd9ee7c0..f77bb01d84 100644 --- a/roles/nginxplus/tasks/main.yml +++ b/roles/nginxplus/tasks/main.yml @@ -8,32 +8,28 @@ - name: Install NGINX block: - - import_tasks: opensource/install-oss.yml when: nginx_type == "opensource" - import_tasks: plus/install-plus.yml when: nginx_type == "plus" - - import_tasks: conf/cleanup-config.yml - when: nginx_cleanup_config - - import_tasks: modules/install-modules.yml when: true in nginx_modules.values() - import_tasks: conf/upload-config.yml when: nginx_main_upload_enable - or nginx_http_upload_enable - or nginx_template_upload_enable - or nginx_stream_upload_enable - or nginx_html_upload_enable - or nginx_ssl_upload_enable + or nginx_http_upload_enable + or nginx_template_upload_enable + or nginx_stream_upload_enable + or nginx_html_upload_enable + or nginx_ssl_upload_enable - import_tasks: conf/template-config.yml when: nginx_main_template_enable - or nginx_http_template_enable - or nginx_stream_template_enable - or nginx_rest_api_enable + or nginx_http_template_enable + or nginx_stream_template_enable + or nginx_rest_api_enable - import_tasks: validate.yml when: nginx_type == "plus"