Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add remote syslog configuration #14513

Merged
merged 8 commits into from
Jul 10, 2023
+438 −83
Merged

Add remote syslog configuration #14513

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2e43092
Add remote syslog configuration
fastiuk Dec 20, 2022
e2ebb72
Doc: update configuration reference
fastiuk May 23, 2023
0d7bb0d
Update sonic-utilities submodule
fastiuk May 31, 2023
df9d120
YANG: Add defaults for new syslog GLOBAL parameters
fastiuk Jul 4, 2023
5dd69f8
Merge branch 'master' into rsyslog-merge
fastiuk Jul 5, 2023
b46bd00
Merge branch 'master' into rsyslog-merge
fastiuk Jul 6, 2023
cd0f659
RSyslog: Move defaults to Jinja2 template
fastiuk Jul 10, 2023
4091c1c
Merge remote-tracking branch 'upstream/master' into rsyslog-merge
fastiuk Jul 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion files/image_config/rsyslog/rsyslog-config.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ if [[ ($NUM_ASIC -gt 1) ]]; then
else
udp_server_ip=$(ip -j -4 addr list lo scope host | jq -r -M '.[0].addr_info[0].local')
fi
hostname=$(hostname)
Copy link
Collaborator

@qiluo-msft qiluo-msft May 13, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hostname

There is DEVICE_METADATA/hostname in ConfigDB. Could you check which one is better? #Closed

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it is better to take from linux than DEVICE_METADATA/hostname.
example: DB can be empty if not configured with CLI. hostname can also be dynamically assigned using DHCP.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If DEVICE_METADATA/hostname has value, it should be preferred.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The hostname can be dynamically assigned by DHCP, in that case, DB value may contain the wrong hostname.
Plus DEVICE_METADATA is a configuration and not a STATE, once data is placed to DEVICE_METADATA it will update the system hostname according to https://github.com/sonic-net/sonic-host-services/blob/bc08806b64002c506b8401eae5d9e1c760651e49/scripts/hostcfgd#L1568
So in my opinion it is better to take it from the hostname utility as the most appropriate one.


sonic-cfggen -d -t /usr/share/sonic/templates/rsyslog.conf.j2 -a "{\"udp_server_ip\": \"$udp_server_ip\"}" >/etc/rsyslog.conf
sonic-cfggen -d -t /usr/share/sonic/templates/rsyslog.conf.j2 \
-a "{\"udp_server_ip\": \"$udp_server_ip\", \"hostname\": \"$hostname\"}" \
> /etc/rsyslog.conf

systemctl restart rsyslog
72 changes: 41 additions & 31 deletions files/image_config/rsyslog/rsyslog.conf.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,21 +15,14 @@

$ModLoad imuxsock # provides support for local system logging

{% if SYSLOG_CONFIG is defined %}
{% if 'GLOBAL' in SYSLOG_CONFIG %}
{% if 'rate_limit_interval' in SYSLOG_CONFIG['GLOBAL']%}
{% set rate_limit_interval = SYSLOG_CONFIG['GLOBAL']['rate_limit_interval'] %}
{% endif %}
{% if 'rate_limit_burst' in SYSLOG_CONFIG['GLOBAL']%}
{% set rate_limit_burst = SYSLOG_CONFIG['GLOBAL']['rate_limit_burst'] %}
{% endif %}
{% endif %}
{% endif %}
{% set gconf = (SYSLOG_CONFIG | d({})).get('GLOBAL', {}) -%}
{% set rate_limit_interval = gconf.get('rate_limit_interval') %}
{% set rate_limit_burst = gconf.get('rate_limit_burst') %}

{% if rate_limit_interval is defined %}
{% if rate_limit_interval is not none %}
$SystemLogRateLimitInterval {{ rate_limit_interval }}
{% endif %}
{% if rate_limit_burst is defined %}
{% if rate_limit_burst is not none %}
$SystemLogRateLimitBurst {{ rate_limit_burst }}
{% endif %}

Expand All @@ -49,6 +42,8 @@ $UDPServerRun 514
###########################
#### GLOBAL DIRECTIVES ####
###########################
{% set format = gconf.get('format', 'standard') -%}
{% set fw_name = gconf.get('welf_firewall_name', hostname) -%}
#
# Use traditional timestamp format.
# To enable high precision timestamps, comment out the following line.
Expand All @@ -59,6 +54,10 @@ $UDPServerRun 514
$template SONiCFileFormat,"%timegenerated%.%timegenerated:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
$ActionFileDefaultTemplate SONiCFileFormat

template(name="WelfRemoteFormat" type="string" string="%TIMESTAMP% id=firewall time=\"%timereported\
:::date-year%-%timereported:::date-month%-%timereported:::date-day% %timereported:::date-hour%:%timereported:::date-minute%:%timereported\
:::date-second%\" fw=\"{{ fw_name }}\" pri=%syslogpriority% msg=\"%syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\"\n")

#
# Set the default permissions for all log files.
#
Expand Down Expand Up @@ -91,25 +90,36 @@ $RepeatedMsgReduction on
# Remote syslog logging
#

# The omfwd plug-in provides the core functionality of traditional message forwarding via UDP and plain TCP.
# It is a built-in module that does not need to be loaded.
# The omfwd plug-in provides the core functionality of traditional message
# forwarding via UDP and plain TCP. It is a built-in module that does not need
# to be loaded.

{% if SYSLOG_SERVER is defined %}
{% for server, data in SYSLOG_SERVER.items() %}
{% set params_list = [] %}
{% if 'source' in data %}
{% set dummy = params_list.append('address=' + '"' + data.source|string + '"') %}
{% endif %}
{% if 'port' in data %}
{% set dummy = params_list.append('port=' + '"' + data.port|string + '"') %}
{% endif %}
{% if 'vrf' in data and data['vrf'] != "default" %}
{% set dummy = params_list.append('device=' + '"' + data.vrf|string + '"') %}
{% endif %}
{% if params_list %}
*.* action(type="omfwd" target="{{ server }}" protocol="udp" {{ params_list|join(' ') }} template="SONiCFileFormat")
{% else %}
*.* action(type="omfwd" target="{{ server }}" protocol="udp" template="SONiCFileFormat")
{% set servers = SYSLOG_SERVER | d({}) -%}
{% for server in servers %}
{% set conf = servers[server] | d({}) -%}

{% set source = conf.get('source') -%}
{% set port = conf.get('port', 514) -%}
{% set proto = conf.get('protocol', 'udp') -%}
{% set vrf = conf.get('vrf', 'default') -%}
{% set severity = conf.get('severity', gconf.get('severity', 'notice')) -%}
{% set filter = conf.get('filter') -%}
{% set regex = conf.get('filter_regex') -%}

{% set fmodifier = '!' if filter == 'exclude' else '' %}
{% set device = 'eth0' if vrf == 'default' else vrf -%}
{% set template = 'WelfRemoteFormat' if format == 'welf' else 'SONiCFileFormat' -%}

{# Server extra options -#}
{% set options = '' -%}

{% if source -%}
{% set options = options ~ ' Address="' ~ source ~ '"'-%}
{% endif -%}

{% if filter %}
:msg, {{ fmodifier }}ereregex, "{{ regex }}"
{% endif %}
*.{{ severity }}
action(type="omfwd" Target="{{ server }}" Port="{{ port }}" Protocol="{{ proto }}" Device="{{ device }}" Template="{{ template }}"{{ options }})
{% endfor %}
{% endif %}
2 changes: 1 addition & 1 deletion src/sonic-utilities
Open in desktop
Submodule sonic-utilities updated 14 files
+12 −6 scripts/db_migrator.py
+2 −2 scripts/portstat
+3 −3 scripts/sfpshow
+3 −3 sfputil/main.py
+1 −1 show/main.py
+6 −4 tests/db_migrator_input/appl_db/routes_migrate_expected.json
+2 −2 tests/db_migrator_input/appl_db/routes_migrate_input.json
+2 −2 tests/db_migrator_test.py
+105 −0 tests/mock_tables/state_db.json
+0 −1 tests/portstat_test.py
+75 −0 tests/sfp_test.py
+132 −24 tests/sfputil_test.py
+25 −1 tests/show_test.py
+77 −0 tests/syslog_input/rsyslog.conf
108 changes: 75 additions & 33 deletions src/sonic-yang-models/doc/Configuration.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ Table of Contents
* [MUX_LINKMGR](#mux_linkmgr)
* [NEIGH](#neigh)
* [NTP Global Configuration](#ntp-global-configuration)
* [NTP and SYSLOG servers](#ntp-and-syslog-servers)
* [NTP Servers](#ntp-servers)
* [Peer Switch](#peer-switch)
* [Policer](#policer)
* [Port](#port)
Expand All @@ -60,7 +60,8 @@ Table of Contents
* [Scheduler](#scheduler)
* [Port QoS Map](#port-qos-map)
* [Queue](#queue)
* [Syslog Rate Limit](#syslog-rate-limit)
* [Syslog Global Configuration](#syslog-global-configuration)
* [Syslog Servers](#syslog-servers)
* [Sflow](#sflow)
* [Restapi](#restapi)
* [System Port](#system-port)
Expand Down Expand Up @@ -1511,7 +1512,7 @@ for that address.
}
```

### NTP and SYSLOG servers
### NTP servers

These information are configured in individual tables. Domain name or IP
address of the server is used as object key. Currently there are no
Expand All @@ -1534,35 +1535,6 @@ attributes in those objects.
}
```

***Syslog server***
```
{
"SYSLOG_SERVER": {
"10.0.0.5": {},
"10.0.0.6": {},
"10.11.150.5": {}
},

"SYSLOG_SERVER" : {
"2.2.2.2": {
"source": "1.1.1.1",
"port": "514",
"vrf": "default"
},
"4.4.4.4": {
"source": "3.3.3.3",
"port": "514",
"vrf": "mgmt"
},
"2222::2222": {
"source": "1111::1111",
"port": "514",
"vrf": "Vrf-Data"
}
}
}
```

### Peer Switch

Below is an exmaple of the peer switch table configuration.
Expand Down Expand Up @@ -1848,7 +1820,33 @@ key - name
| collector_port | Destination L4 port of the Sflow collector | | 6343 | |
| collector_vrf | Specify the Collector VRF. In this revision, it is either default VRF or Management VRF.| | | |

### Syslog Rate Limit
### Syslog Global Configuration

These configuration options are used to configure rsyslog utility and the way
the system generates logs.

***Configuration sample***
```
{
"SYSLOG_CONFIG": {
"GLOBAL": {
"rate_limit_interval": "5",
"rate_limit_burst": "100",
"format": "welf",
"welf_firewall_name": "bla",
"severity": "info"
}
}
}
```

* `rate_limit_interval` - determines the amount of time that is being measured for rate limiting: `unsigned integer`
* `rate_limit_burst` - defines the amount of messages, that have to occur in the time limit: `unsigned integer`
* `format` - syslog log format: `{standard, welf}`
* `welf_firewall_name` - WELF format firewall name: `string`
* `severity` - global log severity: `{emerg, alert, crit, error, warning, notice, info, debug}`

***Syslog Rate Limit***

Host side configuration:

Expand Down Expand Up @@ -1880,6 +1878,50 @@ Container side configuration:
}
```

### Syslog servers

These information are configured in individual tables. Domain name or IP
address of the server is used as object key. Each server can be configurable.

***Configuration sample***
```
{
"SYSLOG_SERVER": {
"10.0.0.5": {},
"10.0.0.6": {},
"10.11.150.5": {}
},

"SYSLOG_SERVER" : {
"4.4.4.4": {
"source": "3.3.3.3",
"port": "514",
"vrf": "mgmt"
},
"2222::2222": {
"source": "1111::1111",
"port": "514",
"vrf": "Vrf-Data"
},
"somehostname": {
"filter": "include",
"filter_regex": "ololo",
"port": "514",
"protocol": "tcp",
"severity": "notice",
"vrf": "default"
}
}
}
```

* `filter` - determines if syslog will include or exclude messages specified by regex: `{include, exclude}`
* `filter_regex` - filter messages by this regex: `string`
* `port` - network port to use to connect to remote server: `integer: 1..65535`
* `protocol` - network protocol to use to connect to remote server: `{tcp, udp}`
* `severity` - per-server log severity, overrifes global one: `{emerg, alert, crit, error, warning, notice, info, debug}`


### System Port
Every port on the system requires a global representation, known as a System Port,
and is listed in this table.
Expand Down
13 changes: 12 additions & 1 deletion src/sonic-yang-models/tests/files/sample_config_db.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -499,12 +499,23 @@
"source": "1111::1111",
"port": "514",
"vrf": "Vrf_blue"
},
"somehostname": {
"filter": "include",
"filter_regex": "ololo",
"port": "514",
"protocol": "tcp",
"severity": "notice",
"vrf": "default"
}
},
"SYSLOG_CONFIG" : {
"GLOBAL": {
"rate_limit_interval": "5",
"rate_limit_burst": "100"
"rate_limit_burst": "100",
"format": "welf",
"welf_firewall_name": "bla",
"severity": "info"
}
},
"SYSLOG_CONFIG_FEATURE" : {
Expand Down
59 changes: 55 additions & 4 deletions src/sonic-yang-models/tests/yang_model_tests/tests/syslog.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,6 @@
"desc": "Load syslog server table with empty address as syslog server.",
"eStrKey": "InvalidValue"
},
"SYSLOG_SERVER_INVALID_IPADDR_TEST": {
"desc": "Load syslog server table with invalid ipv4 address as syslog server.",
"eStrKey": "InvalidValue"
},
"SYSLOG_SERVER_INVALID_IPV6_ADDR_TEST": {
"desc": "Load syslog server table with invalid ipv6 address as syslog server.",
"eStrKey": "InvalidValue"
Expand Down Expand Up @@ -62,5 +58,60 @@
"SYSLOG_CONFIG_FEATURE_INVALID_BURST": {
"desc": "Configure invalid rate_limit_burst in SYSLOG_CONFIG_FEATURE.",
"eStrKey": "InvalidValue"
},
"SYSLOG_SERVER_HOSTNAME": {
"desc": "Load syslog server table with hostname"
},
"SYSLOG_SERVER_HOSTNAME_INVALID": {
"desc": "Load syslog server table with invalid hostname",
"eStrKey": "InvalidValue"
},
"SYSLOG_SERVER_FILTER_TYPE": {
"desc": "Valid filter type for syslog server"
},
"SYSLOG_SERVER_FILTER_TYPE_INVALID": {
"desc": "Invalid filter type for syslog server",
"eStrKey": "InvalidValue"
},
"SYSLOG_SERVER_FILTER_REGEX": {
"desc": "Valid filter regex"
},
"SYSLOG_SERVER_PROTOCOL": {
"desc": "Valid syslog server protocol"
},
"SYSLOG_SERVER_PROTOCOL_INVALID": {
"desc": "Invalid syslog server protocol",
"eStrKey": "InvalidValue"
},
"SYSLOG_SERVER_SEVERITY": {
"desc": "Syslog server valid severity"
},
"SYSLOG_SERVER_SEVERITY_INVALID": {
"desc": "Syslog server invalid severity",
"eStrKey": "InvalidValue"
},
"SYSLOG_CONFIG_GLOBAL_VALID": {
"desc": "Global syslog configuration"
},
"SYSLOG_CONFIG_FORMAT": {
"desc": "Syslog format type"
},
"SYSLOG_CONFIG_FORMAT_INVALID": {
"desc": "Invalid syslog format",
"eStrKey": "InvalidValue"
},
"SYSLOG_CONFIG_FORMAT_WELF_FW_NAME": {
"desc": "Syslog format WELF firewall name"
},
"SYSLOG_CONFIG_FORMAT_WELF_FW_NAME_INVALID": {
"desc": "Syslog format WELF invalid firewall name",
"eStrKey": "Must"
},
"SYSLOG_CONFIG_SEVERITY": {
"desc": "Global syslog severity"
},
"SYSLOG_CONFIG_SEVERITY_INVALID": {
"desc": "Global invalid syslog severity",
"eStrKey": "InvalidValue"
}
}
Loading