[syslog] Add remote syslog configuration (cherry-pick to 202305) (#15897

) cherry-pick: #14513 depends: sonic-net/sonic-utilities#2939 * Add an ability to configure remote syslog servers * Add an initial configuration for remote syslog * Extend YANG module and add unit tests #### Why I did it Adding the following functionality to rsyslog feature: * Configure remote syslog servers: protocol, filter, severity level * Update global syslog configuration: severity level, message format #### How I did it added parameters to syslog server and global configuration. #### How to verify it create syslog server using CLI/adding to Redis-DB verify server is added to file /etc/rsyslog.conf and server is functional. #### Description for the changelog extend rsyslog capabilities, added server and global configuration parameters. #### Link to config_db schema for YANG module changes [sonic-syslog.yang](https://github.com/sonic-net/sonic-buildimage/blob/master/src/sonic-yang-models/yang-models/sonic-syslog.yang)
sonic-net · Aug 14, 2023 · 4602d30 · 4602d30
1 parent ac34681
commit 4602d30
Show file tree

Hide file tree

Showing 8 changed files with 438 additions and 83 deletions.
diff --git a/files/image_config/rsyslog/rsyslog-config.sh b/files/image_config/rsyslog/rsyslog-config.sh
@@ -17,7 +17,10 @@ if [[ ($NUM_ASIC -gt 1) ]]; then
 else
  udp_server_ip=$(ip -j -4 addr list lo scope host | jq -r -M '.[0].addr_info[0].local')
 fi
+hostname=$(hostname)
 
-sonic-cfggen -d -t /usr/share/sonic/templates/rsyslog.conf.j2 -a "{\"udp_server_ip\": \"$udp_server_ip\"}" >/etc/rsyslog.conf
+sonic-cfggen -d -t /usr/share/sonic/templates/rsyslog.conf.j2 \
+ -a "{\"udp_server_ip\": \"$udp_server_ip\", \"hostname\": \"$hostname\"}" \
+ > /etc/rsyslog.conf
 
 systemctl restart rsyslog
diff --git a/files/image_config/rsyslog/rsyslog.conf.j2 b/files/image_config/rsyslog/rsyslog.conf.j2
@@ -15,21 +15,14 @@
 
 $ModLoad imuxsock # provides support for local system logging
 
-{% if SYSLOG_CONFIG is defined %}
-{% if 'GLOBAL' in SYSLOG_CONFIG %}
-{% if 'rate_limit_interval' in SYSLOG_CONFIG['GLOBAL']%}
-{% set rate_limit_interval = SYSLOG_CONFIG['GLOBAL']['rate_limit_interval'] %}
-{% endif %}
-{% if 'rate_limit_burst' in SYSLOG_CONFIG['GLOBAL']%}
-{% set rate_limit_burst = SYSLOG_CONFIG['GLOBAL']['rate_limit_burst'] %}
-{% endif %}
-{% endif %}
-{% endif %}
+{% set gconf = (SYSLOG_CONFIG | d({})).get('GLOBAL', {}) -%}
+{% set rate_limit_interval = gconf.get('rate_limit_interval') %}
+{% set rate_limit_burst = gconf.get('rate_limit_burst') %}
 
-{% if rate_limit_interval is defined %}
+{% if rate_limit_interval is not none %}
 $SystemLogRateLimitInterval {{ rate_limit_interval }}
 {% endif %}
-{% if rate_limit_burst is defined %}
+{% if rate_limit_burst is not none %}
 $SystemLogRateLimitBurst {{ rate_limit_burst }}
 {% endif %}
 
@@ -49,6 +42,8 @@ $UDPServerRun 514
 ###########################
 #### GLOBAL DIRECTIVES ####
 ###########################
+{% set format = gconf.get('format', 'standard') -%}
+{% set fw_name = gconf.get('welf_firewall_name', hostname) -%}
 #
 # Use traditional timestamp format.
 # To enable high precision timestamps, comment out the following line.
@@ -59,6 +54,10 @@ $UDPServerRun 514
 $template SONiCFileFormat,"%timegenerated%.%timegenerated:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
 $ActionFileDefaultTemplate SONiCFileFormat
 
+template(name="WelfRemoteFormat" type="string" string="%TIMESTAMP% id=firewall time=\"%timereported\
+:::date-year%-%timereported:::date-month%-%timereported:::date-day% %timereported:::date-hour%:%timereported:::date-minute%:%timereported\
+:::date-second%\" fw=\"{{ fw_name }}\" pri=%syslogpriority% msg=\"%syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\"\n")
+
 #
 # Set the default permissions for all log files.
 #
@@ -91,25 +90,36 @@ $RepeatedMsgReduction on
 # Remote syslog logging
 #
 
-# The omfwd plug-in provides the core functionality of traditional message forwarding via UDP and plain TCP.
-# It is a built-in module that does not need to be loaded.
+# The omfwd plug-in provides the core functionality of traditional message
+# forwarding via UDP and plain TCP. It is a built-in module that does not need
+# to be loaded.
 
-{% if SYSLOG_SERVER is defined %}
-{% for server, data in SYSLOG_SERVER.items() %}
-{% set params_list = [] %}
-{% if 'source' in data %}
-{% set dummy = params_list.append('address=' + '"' + data.source|string + '"') %}
-{% endif %}
-{% if 'port' in data %}
-{% set dummy = params_list.append('port=' + '"' + data.port|string + '"') %}
-{% endif %}
-{% if 'vrf' in data and data['vrf'] != "default" %}
-{% set dummy = params_list.append('device=' + '"' + data.vrf|string + '"') %}
-{% endif %}
-{% if params_list %}
-*.* action(type="omfwd" target="{{ server }}" protocol="udp" {{ params_list|join(' ') }} template="SONiCFileFormat")
-{% else %}
-*.* action(type="omfwd" target="{{ server }}" protocol="udp" template="SONiCFileFormat")
+{% set servers = SYSLOG_SERVER | d({}) -%}
+{% for server in servers %}
+{% set conf = servers[server] | d({}) -%}
+
+{% set source = conf.get('source') -%}
+{% set port = conf.get('port', 514) -%}
+{% set proto = conf.get('protocol', 'udp') -%}
+{% set vrf = conf.get('vrf', 'default') -%}
+{% set severity = conf.get('severity', gconf.get('severity', 'notice')) -%}
+{% set filter = conf.get('filter') -%}
+{% set regex = conf.get('filter_regex') -%}
+
+{% set fmodifier = '!' if filter == 'exclude' else '' %}
+{% set device = 'eth0' if vrf == 'default' else vrf -%}
+{% set template = 'WelfRemoteFormat' if format == 'welf' else 'SONiCFileFormat' -%}
+
+{# Server extra options -#}
+{% set options = '' -%}
+
+{% if source -%}
+ {% set options = options ~ ' Address="' ~ source ~ '"'-%}
+{% endif -%}
+
+{% if filter %}
+:msg, {{ fmodifier }}ereregex, "{{ regex }}"
 {% endif %}
+*.{{ severity }}
+action(type="omfwd" Target="{{ server }}" Port="{{ port }}" Protocol="{{ proto }}" Device="{{ device }}" Template="{{ template }}"{{ options }})
 {% endfor %}
-{% endif %}
diff --git a/src/sonic-utilities b/src/sonic-utilities
diff --git a/src/sonic-yang-models/doc/Configuration.md b/src/sonic-yang-models/doc/Configuration.md
@@ -50,7 +50,7 @@ Table of Contents
  * [MUX_LINKMGR](#mux_linkmgr)
  * [NEIGH](#neigh)
  * [NTP Global Configuration](#ntp-global-configuration)
- * [NTP and SYSLOG servers](#ntp-and-syslog-servers)
+ * [NTP Servers](#ntp-servers)
  * [Peer Switch](#peer-switch)
  * [Policer](#policer)
  * [Port](#port)
@@ -59,7 +59,8 @@ Table of Contents
  * [Scheduler](#scheduler)
  * [Port QoS Map](#port-qos-map)
  * [Queue](#queue)
- * [Syslog Rate Limit](#syslog-rate-limit)
+ * [Syslog Global Configuration](#syslog-global-configuration)
+ * [Syslog Servers](#syslog-servers)
  * [Sflow](#sflow)
  * [Restapi](#restapi)
  * [System Port](#system-port)
@@ -1495,7 +1496,7 @@ for that address.
 }
 ```
 
-### NTP and SYSLOG servers
+### NTP servers
 
 These information are configured in individual tables. Domain name or IP
 address of the server is used as object key. Currently there are no
@@ -1518,35 +1519,6 @@ attributes in those objects.
 }
 ```
 
-***Syslog server***
-```
-{
- "SYSLOG_SERVER": {
- "10.0.0.5": {},
- "10.0.0.6": {},
- "10.11.150.5": {}
- },
-
- "SYSLOG_SERVER" : {
- "2.2.2.2": {
- "source": "1.1.1.1",
- "port": "514",
- "vrf": "default"
- },
- "4.4.4.4": {
- "source": "3.3.3.3",
- "port": "514",
- "vrf": "mgmt"
- },
- "2222::2222": {
- "source": "1111::1111",
- "port": "514",
- "vrf": "Vrf-Data"
- }
- }
-}
-```
-
 ### Peer Switch
 
 Below is an exmaple of the peer switch table configuration.
@@ -1832,7 +1804,33 @@ key - name
 | collector_port | Destination L4 port of the Sflow collector | | 6343 | |
 | collector_vrf | Specify the Collector VRF. In this revision, it is either default VRF or Management VRF.| | | |
 
-### Syslog Rate Limit
+### Syslog Global Configuration
+
+These configuration options are used to configure rsyslog utility and the way
+the system generates logs.
+
+***Configuration sample***
+```
+{
+ "SYSLOG_CONFIG": {
+ "GLOBAL": {
+ "rate_limit_interval": "5",
+ "rate_limit_burst": "100",
+ "format": "welf",
+ "welf_firewall_name": "bla",
+ "severity": "info"
+ }
+ }
+}
+```
+
+* `rate_limit_interval` - determines the amount of time that is being measured for rate limiting: `unsigned integer`
+* `rate_limit_burst` - defines the amount of messages, that have to occur in the time limit: `unsigned integer`
+* `format` - syslog log format: `{standard, welf}`
+* `welf_firewall_name` - WELF format firewall name: `string`
+* `severity` - global log severity: `{emerg, alert, crit, error, warning, notice, info, debug}`
+
+***Syslog Rate Limit***
 
 Host side configuration:
 
@@ -1864,6 +1862,50 @@ Container side configuration:
 }
 ```
 
+### Syslog servers
+
+These information are configured in individual tables. Domain name or IP
+address of the server is used as object key. Each server can be configurable.
+
+***Configuration sample***
+```
+{
+ "SYSLOG_SERVER": {
+ "10.0.0.5": {},
+ "10.0.0.6": {},
+ "10.11.150.5": {}
+ },
+
+ "SYSLOG_SERVER" : {
+ "4.4.4.4": {
+ "source": "3.3.3.3",
+ "port": "514",
+ "vrf": "mgmt"
+ },
+ "2222::2222": {
+ "source": "1111::1111",
+ "port": "514",
+ "vrf": "Vrf-Data"
+ },
+ "somehostname": {
+ "filter": "include",
+ "filter_regex": "ololo",
+ "port": "514",
+ "protocol": "tcp",
+ "severity": "notice",
+ "vrf": "default"
+ }
+ }
+}
+```
+
+* `filter` - determines if syslog will include or exclude messages specified by regex: `{include, exclude}`
+* `filter_regex` - filter messages by this regex: `string`
+* `port` - network port to use to connect to remote server: `integer: 1..65535`
+* `protocol` - network protocol to use to connect to remote server: `{tcp, udp}`
+* `severity` - per-server log severity, overrifes global one: `{emerg, alert, crit, error, warning, notice, info, debug}`
+
+
 ### System Port
 Every port on the system requires a global representation, known as a System Port,
 and is listed in this table.

diff --git a/src/sonic-yang-models/tests/files/sample_config_db.json b/src/sonic-yang-models/tests/files/sample_config_db.json
@@ -499,12 +499,23 @@
  "source": "1111::1111",
  "port": "514",
  "vrf": "Vrf_blue"
+ },
+ "somehostname": {
+ "filter": "include",
+ "filter_regex": "ololo",
+ "port": "514",
+ "protocol": "tcp",
+ "severity": "notice",
+ "vrf": "default"
  }
  },
  "SYSLOG_CONFIG" : {
  "GLOBAL": {
  "rate_limit_interval": "5",
- "rate_limit_burst": "100"
+ "rate_limit_burst": "100",
+ "format": "welf",
+ "welf_firewall_name": "bla",
+ "severity": "info"
  }
  },
  "SYSLOG_CONFIG_FEATURE" : {

diff --git a/src/sonic-yang-models/tests/yang_model_tests/tests/syslog.json b/src/sonic-yang-models/tests/yang_model_tests/tests/syslog.json
@@ -29,10 +29,6 @@
  "desc": "Load syslog server table with empty address as syslog server.",
  "eStrKey": "InvalidValue"
  },
- "SYSLOG_SERVER_INVALID_IPADDR_TEST": {
- "desc": "Load syslog server table with invalid ipv4 address as syslog server.",
- "eStrKey": "InvalidValue"
- },
  "SYSLOG_SERVER_INVALID_IPV6_ADDR_TEST": {
  "desc": "Load syslog server table with invalid ipv6 address as syslog server.",
  "eStrKey": "InvalidValue"
@@ -62,5 +58,60 @@
  "SYSLOG_CONFIG_FEATURE_INVALID_BURST": {
  "desc": "Configure invalid rate_limit_burst in SYSLOG_CONFIG_FEATURE.",
  "eStrKey": "InvalidValue"
+ },
+ "SYSLOG_SERVER_HOSTNAME": {
+ "desc": "Load syslog server table with hostname"
+ },
+ "SYSLOG_SERVER_HOSTNAME_INVALID": {
+ "desc": "Load syslog server table with invalid hostname",
+ "eStrKey": "InvalidValue"
+ },
+ "SYSLOG_SERVER_FILTER_TYPE": {
+ "desc": "Valid filter type for syslog server"
+ },
+ "SYSLOG_SERVER_FILTER_TYPE_INVALID": {
+ "desc": "Invalid filter type for syslog server",
+ "eStrKey": "InvalidValue"
+ },
+ "SYSLOG_SERVER_FILTER_REGEX": {
+ "desc": "Valid filter regex"
+ },
+ "SYSLOG_SERVER_PROTOCOL": {
+ "desc": "Valid syslog server protocol"
+ },
+ "SYSLOG_SERVER_PROTOCOL_INVALID": {
+ "desc": "Invalid syslog server protocol",
+ "eStrKey": "InvalidValue"
+ },
+ "SYSLOG_SERVER_SEVERITY": {
+ "desc": "Syslog server valid severity"
+ },
+ "SYSLOG_SERVER_SEVERITY_INVALID": {
+ "desc": "Syslog server invalid severity",
+ "eStrKey": "InvalidValue"
+ },
+ "SYSLOG_CONFIG_GLOBAL_VALID": {
+ "desc": "Global syslog configuration"
+ },
+ "SYSLOG_CONFIG_FORMAT": {
+ "desc": "Syslog format type"
+ },
+ "SYSLOG_CONFIG_FORMAT_INVALID": {
+ "desc": "Invalid syslog format",
+ "eStrKey": "InvalidValue"
+ },
+ "SYSLOG_CONFIG_FORMAT_WELF_FW_NAME": {
+ "desc": "Syslog format WELF firewall name"
+ },
+ "SYSLOG_CONFIG_FORMAT_WELF_FW_NAME_INVALID": {
+ "desc": "Syslog format WELF invalid firewall name",
+ "eStrKey": "Must"
+ },
+ "SYSLOG_CONFIG_SEVERITY": {
+ "desc": "Global syslog severity"
+ },
+ "SYSLOG_CONFIG_SEVERITY_INVALID": {
+ "desc": "Global invalid syslog severity",
+ "eStrKey": "InvalidValue"
  }
 }