diff --git a/docs/logstash-pipelines.md b/docs/logstash-pipelines.md index 6b256761..496d1ce2 100644 --- a/docs/logstash-pipelines.md +++ b/docs/logstash-pipelines.md @@ -1,5 +1,13 @@ # Pipelines # +## Keeping an overview ## + +It can be quite difficult to stay on top of your pipeline configuration because they tend to become very complex. + +This collection will leave some comments about how pipelines are interconnected within the `/etc/logstash/pipelines.yml` configuration file. + +If you set `logstash_mermaid` to `true` (which is the default), then you will also get a new file in `/etc/logstash/pipelines.mermaid`. You can paste it into a Mermaid editor in your documentation tool or in an [online Mermaid editor](https://mermaid.live/). The same content will be available on your control node in a temporary file. You can even add arbitrary code to reflect manually managed pipelines by using `logstash_mermaid_extra`. + ## Git managed ## If you have pipeline code managed in (and available via) Git repositories, you can use this role to check them out and integrate them into `pipelines.yml`. diff --git a/docs/role-logstash.md b/docs/role-logstash.md index e0a8d142..1b4e4b78 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -89,6 +89,11 @@ Aside from `logstash.yml` we can manage Logstashs pipelines. * *logstash_legacy_monitoring*: Enables legacy monitoring - ignored when `elasticstack_full_stack` is not set. (default: `true`) * *logstash_redis_password*: If set this will use this password when connecting our simple inputs and outputs to Redis. (default: not set) +* *logstash_mermaid*: Print overview over Logstash pipelines in Mermaid syntax. (default: `true`) +* *logstash_mermaid_logstash*: Place Mermaid syntax into `/etc/logstash/pipelines.mermaid` on Logstash hosts. (default: `true`) +* *logstash_mermaid_local*: Place Mermaid syntax into temporary file on control node. (default: `false`) +* *logstash_mermaid_extra*: You can add extra Mermaid syntax to the output by adding it to this variable. YAML-multiline is supported. (default: none) + The following variables configure Log4j for Logstash. All default to `true` as this is the default after the installation. * *logstash_logging_console*: Log to console - syslog when run via systemd diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 6c152923..88eead19 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -88,6 +88,10 @@ logstash_pipeline_identifier: true logstash_pipeline_identifier_field_name: "[netways][pipeline]" logstash_pipeline_identifier_defaults: false +logstash_mermaid: true +logstash_mermaid_local: false +logstash_mermaid_logstash: true + # Only for internal use logstash_freshstart: diff --git a/roles/logstash/tasks/logstash-mermaid.yml b/roles/logstash/tasks/logstash-mermaid.yml new file mode 100644 index 00000000..5d904237 --- /dev/null +++ b/roles/logstash/tasks/logstash-mermaid.yml @@ -0,0 +1,36 @@ +--- + +- name: Print Logstash pipelines in Mermaid syntax on Logstash hosts + ansible.builtin.template: + src: pipelines.mermaid.j2 + dest: /etc/logstash/pipelines.mermaid + owner: root + group: root + mode: 0655 + when: + - logstash_mermaid_logstash | bool + +- name: Provide Logstash pipelines in Mermaid syntax on control node + when: + - logstash_mermaid_local | bool + block: + + - name: Create temporary directory on control node + ansible.builtin.tempfile: + prefix: logstash_mermaid + register: mermaid_path + delegate_to: localhost + + - name: Print Logstash pipelines in Mermaid syntax on control node + ansible.builtin.template: + src: pipelines.mermaid.j2 + dest: "{{ mermaid_path.path }}" + owner: root + group: root + mode: 0655 + delegate_to: localhost + + - name: Send user to Mermaid file + ansible.builtin.debug: + msg: + - "You can find an overview of your pipeline configuration in Mermaid syntax at {{ mermaid_path.path }}" diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 84dc81b6..52d87054 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -251,6 +251,13 @@ - configuration - logstash_configuration +- name: Print Logstash pipelines in Mermaid syntax + ansible.builtin.import_tasks: logstash-mermaid.yml + when: + - logstash_mermaid | bool + tags: + - mermaid + - name: Install Logstash plugins community.general.logstash_plugin: state: present diff --git a/roles/logstash/templates/pipelines.mermaid.j2 b/roles/logstash/templates/pipelines.mermaid.j2 new file mode 100644 index 00000000..adc30895 --- /dev/null +++ b/roles/logstash/templates/pipelines.mermaid.j2 @@ -0,0 +1,31 @@ +# Managed via Ansible role +# https://github.com/netways/ansible-role-logstash + +# Use the following code with your favorite Mermaid editor +# Or paste into: https://mermaid.live/ +# To get a graphical overview of your Logstash pipelines + +flowchart TD +{% if logstash_beats_input | bool %} +p_ansible-input[ansible-input] --> k_input{input} +{% endif %} +{% if logstash_elasticsearch_output | bool %} +k_forwarder{forwarder} --> p_ansible-forwarder[ansible-forwarder] +{% endif %} +{% if logstash_pipelines is defined %} +{% for item in logstash_pipelines %} +{% if item.input is defined %} +{% for input in item.input %} +k_{{ input.key }}{{ '{' }}{{ input.key }}{{ '}' }} --> p_{{ item.name }}{{ '[' }}{{item.name}}{{ ']' }} +{% endfor %} +{% endif %} +{% if item.output is defined %} +{% for output in item.output %} +p_{{ item.name }}{{ '[' }}{{ item.name }}{{ ']' }} --> {% if output.condition is defined %}{{ '|' }}if {{ output.condition | replace("][", ".") | replace("[","") | replace("]","") | replace('"', '')}}{{ '|' }}{% endif %}k_{{ output.key }}{{ '{' }}{{ output.key }}} +{% endfor %} +{% endif %} +{% endfor %} +{% endif %} +{% if logstash_mermaid_extra is defined %} +{{ logstash_mermaid_extra }} +{% endif %}