Skip to content

Commit 0517bfa

Browse files
Junchao-MellanoxJunchao-Mellanox
authored
Add test case for syslog rate limit feature (#6627)
Add test case for syslog rate limit feature test_syslog_rate_limit_container 1. Randomly choose on container 2. Change the syslog rate limit of current container 3. Use a generated script to print log from current container which is fast enough to hit the limit 4, Check syslog that some logs are dropped test_syslog_rate_limit_host 1. Change the syslog rate limit of host 2. Use a generated script to print log from host which is fast enough to hit the limit 3. Check syslog that some logs are dropped
1 parent db5144a commit 0517bfa

File tree

2 files changed

+253
-0
lines changed

2 files changed

+253
-0
lines changed

‎tests/syslog/log_generator.py

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import time
2+
from sonic_py_common.logger import Logger
3+
4+
logger = Logger(log_identifier="rate-limit-test")
5+
logger.set_min_log_priority_info()
6+
7+
begin = 1
8+
end = 101
9+
10+
while begin <= end:
11+
logger.log_info('This is a test log: {}'.format(begin))
12+
begin += 1
13+
14+
# Let log flush to file
15+
time.sleep(2)

‎tests/syslog/test_syslog_rate_limit.py

+238
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
import logging
2+
import os
3+
import pytest
4+
import random
5+
6+
from tests.common.config_reload import config_reload
7+
from tests.common.utilities import skip_release
8+
from tests.common.helpers.assertions import pytest_assert
9+
from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer
10+
from tests.common.helpers.sonic_db import SonicDbCli
11+
12+
logger = logging.getLogger(__name__)
13+
14+
RATE_LIMIT_BURST = 100
15+
RATE_LIMIT_INTERVAL = 10
16+
# Generate 101 packets in tests/syslog/log_generator.py, so that 1 log message will be dropped by rsyslogd
17+
LOG_MESSAGE_GENERATE_COUNT = 101
18+
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
19+
LOCAL_LOG_GENERATOR_FILE = os.path.join(BASE_DIR, 'log_generator.py')
20+
REMOTE_LOG_GENERATOR_FILE = os.path.join('/tmp', 'log_generator.py')
21+
DOCKER_LOG_GENERATOR_FILE = '/log_generator.py'
22+
# rsyslogd prints this log when rate-limiting reached
23+
LOG_EXPECT_SYSLOG_RATE_LIMIT_REACHED = '.*begin to drop messages due to rate-limiting.*'
24+
# Log pattern for tests/syslog/log_generator.py
25+
LOG_EXPECT_LAST_MESSAGE = '.*{}rate-limit-test: This is a test log:.*'
26+
27+
pytestmark = [
28+
pytest.mark.topology("any")
29+
]
30+
31+
32+
@pytest.fixture(autouse=True, scope="module")
33+
def check_image_version(rand_selected_dut):
34+
"""Skips this test if the SONiC image installed on DUT is older than 202205
35+
36+
Args:
37+
duthost: Hostname of DUT.
38+
39+
Returns:
40+
None.
41+
"""
42+
skip_release(rand_selected_dut, ["201811", "201911", "202012", "202106", "202205"])
43+
44+
45+
@pytest.fixture(autouse=True, scope="module")
46+
def restore_rate_limit(rand_selected_dut):
47+
"""Fixture to automatically restore syslog rate limit configuration
48+
49+
Args:
50+
rand_selected_dut (object): DUT host object
51+
"""
52+
container_data = rand_selected_dut.show_and_parse('show syslog rate-limit-container')
53+
host_data = rand_selected_dut.show_and_parse('show syslog rate-limit-host')
54+
55+
yield
56+
57+
for item in container_data:
58+
rand_selected_dut.command('config syslog rate-limit-container {} -b {} -i {}'.format(
59+
item['service'], item['burst'], item['interval']))
60+
61+
rand_selected_dut.command('config syslog rate-limit-host -b {} -i {}'.format(
62+
host_data[0]['burst'], host_data[0]['interval']))
63+
rand_selected_dut.command('config save -y')
64+
65+
66+
def test_syslog_rate_limit(rand_selected_dut):
67+
"""Test case for syslog rate limit
68+
69+
Args:
70+
rand_selected_dut (object): DUT host object
71+
"""
72+
# Copy tests/syslog/log_generator.py to DUT
73+
rand_selected_dut.copy(src=LOCAL_LOG_GENERATOR_FILE, dest=REMOTE_LOG_GENERATOR_FILE)
74+
75+
verify_container_rate_limit(rand_selected_dut)
76+
verify_host_rate_limit(rand_selected_dut)
77+
78+
# Save configuration and reload, verify the configuration can be loaded
79+
logger.info('Persist syslog rate limit configuration to DB and do config reload')
80+
rand_selected_dut.command('config save -y')
81+
config_reload(rand_selected_dut)
82+
83+
# database does not support syslog rate limit configuration persist
84+
verify_container_rate_limit(rand_selected_dut, ignore_containers=['database'])
85+
verify_host_rate_limit(rand_selected_dut)
86+
87+
88+
def verify_container_rate_limit(rand_selected_dut, ignore_containers=[]):
89+
"""Config syslog rate limit for each container and verify it works. Basic flow:
90+
1. For each container
91+
2. Filter disabled container
92+
3. Filter un-supported container
93+
4. Config syslog rate limit
94+
5. Verify syslog rate limit with "show syslog rate-limit-container" command
95+
6. Generate syslog via a script and verify the rate limit configuration works
96+
7. Disable syslog rate limit by setting interval and burst to 0
97+
8. Verify syslog rate limit with "show syslog rate-limit-container" command
98+
9. Generate syslog via a script and verify there is no rate limit anymore
99+
100+
Args:
101+
rand_selected_dut (object): DUT host object
102+
ignore_containers (list): container list that will be ignored for this test
103+
"""
104+
config_db = SonicDbCli(rand_selected_dut, 'CONFIG_DB')
105+
feature_data = rand_selected_dut.show_and_parse('show feature status')
106+
random.shuffle(feature_data)
107+
for item in feature_data:
108+
service_name = item['feature']
109+
if service_name in ignore_containers:
110+
continue
111+
112+
logger.info('Start syslog rate limit test for container {}'.format(service_name))
113+
if item['state'] in ['disabled', 'always_disabled']:
114+
logger.info('Container {} is {}'.format(service_name, item['state']))
115+
continue
116+
117+
support_syslog_rate_limit = config_db.hget_key_value('FEATURE|{}'.format(service_name),
118+
'support_syslog_rate_limit')
119+
if support_syslog_rate_limit.lower() != 'true':
120+
logger.info('Container {} does not support syslog rate limit configuration'.format(service_name))
121+
verify_config_rate_limit_fail(rand_selected_dut, service_name)
122+
continue
123+
124+
rand_selected_dut.command('config syslog rate-limit-container {} -b {} -i {}'.format(
125+
service_name, RATE_LIMIT_BURST, RATE_LIMIT_INTERVAL))
126+
rate_limit_data = rand_selected_dut.show_and_parse('show syslog rate-limit-container {}'.format(service_name))
127+
pytest_assert(rate_limit_data[0]['interval'] == str(RATE_LIMIT_INTERVAL),
128+
'Expect rate limit interval {}, actual {}'.format(RATE_LIMIT_INTERVAL,
129+
rate_limit_data[0]['interval']))
130+
pytest_assert(rate_limit_data[0]['burst'] == str(RATE_LIMIT_BURST),
131+
'Expect rate limit burst {}, actual {}'.format(RATE_LIMIT_BURST, rate_limit_data[0]['burst']))
132+
133+
rand_selected_dut.command(
134+
'docker cp {} {}:{}'.format(REMOTE_LOG_GENERATOR_FILE, service_name, DOCKER_LOG_GENERATOR_FILE))
135+
verify_rate_limit_with_log_generator(rand_selected_dut,
136+
service_name,
137+
'syslog_rate_limit_{}-interval_{}_burst_{}'.format(service_name,
138+
RATE_LIMIT_INTERVAL,
139+
RATE_LIMIT_BURST),
140+
[LOG_EXPECT_SYSLOG_RATE_LIMIT_REACHED,
141+
LOG_EXPECT_LAST_MESSAGE.format(service_name + '#')],
142+
RATE_LIMIT_BURST + 1)
143+
144+
rand_selected_dut.command('config syslog rate-limit-container {} -b {} -i {}'.format(service_name, 0, 0))
145+
rate_limit_data = rand_selected_dut.show_and_parse('show syslog rate-limit-container {}'.format(service_name))
146+
pytest_assert(rate_limit_data[0]['interval'] == '0',
147+
'Expect rate limit interval {}, actual {}'.format(0, rate_limit_data[0]['interval']))
148+
pytest_assert(rate_limit_data[0]['burst'] == '0',
149+
'Expect rate limit burst {}, actual {}'.format(0, rate_limit_data[0]['burst']))
150+
151+
verify_rate_limit_with_log_generator(rand_selected_dut,
152+
service_name,
153+
'syslog_rate_limit_{}-interval_{}_burst_{}'.format(service_name, 0, 0),
154+
[LOG_EXPECT_LAST_MESSAGE.format(service_name + '#')],
155+
LOG_MESSAGE_GENERATE_COUNT)
156+
break # we only randomly test 1 container to reduce test time
157+
158+
159+
def verify_host_rate_limit(rand_selected_dut):
160+
"""Config syslog rate limit for host and verify it works. Basic flow:
161+
1. Config syslog rate limit
162+
2. Verify syslog rate limit with "show syslog rate-limit-host" command
163+
3. Generate syslog via a script and verify the rate limit configuration works
164+
4. Disable syslog rate limit by setting interval and burst to 0
165+
5. Verify syslog rate limit with "show syslog rate-limit-host" command
166+
6. Generate syslog via a script and verify there is no rate limit anymore
167+
168+
Args:
169+
rand_selected_dut (object): DUT host object
170+
"""
171+
logger.info('Start syslog rate limit test for host')
172+
rand_selected_dut.command('config syslog rate-limit-host -b {} -i {}'.format(RATE_LIMIT_BURST, RATE_LIMIT_INTERVAL))
173+
rate_limit_data = rand_selected_dut.show_and_parse('show syslog rate-limit-host')
174+
pytest_assert(rate_limit_data[0]['interval'] == str(RATE_LIMIT_INTERVAL),
175+
'Expect rate limit interval {}, actual {}'.format(RATE_LIMIT_INTERVAL,
176+
rate_limit_data[0]['interval']))
177+
pytest_assert(rate_limit_data[0]['burst'] == str(RATE_LIMIT_BURST),
178+
'Expect rate limit burst {}, actual {}'.format(RATE_LIMIT_BURST, rate_limit_data[0]['burst']))
179+
180+
verify_rate_limit_with_log_generator(rand_selected_dut,
181+
'host',
182+
'syslog_rate_limit_host_interval_{}_burst_{}'.format(RATE_LIMIT_INTERVAL,
183+
RATE_LIMIT_BURST),
184+
[LOG_EXPECT_SYSLOG_RATE_LIMIT_REACHED, LOG_EXPECT_LAST_MESSAGE.format('')],
185+
RATE_LIMIT_BURST + 1,
186+
is_host=True)
187+
188+
rand_selected_dut.command('config syslog rate-limit-host -b {} -i {}'.format(0, 0))
189+
rate_limit_data = rand_selected_dut.show_and_parse('show syslog rate-limit-host')
190+
pytest_assert(rate_limit_data[0]['interval'] == '0',
191+
'Expect rate limit interval {}, actual {}'.format(0, rate_limit_data[0]['interval']))
192+
pytest_assert(rate_limit_data[0]['burst'] == '0',
193+
'Expect rate limit burst {}, actual {}'.format(0, rate_limit_data[0]['burst']))
194+
195+
verify_rate_limit_with_log_generator(rand_selected_dut,
196+
'host',
197+
'syslog_rate_limit_host_interval_{}_burst_{}'.format(0, 0),
198+
[LOG_EXPECT_LAST_MESSAGE.format('')],
199+
LOG_MESSAGE_GENERATE_COUNT,
200+
is_host=True)
201+
202+
203+
def verify_config_rate_limit_fail(duthost, service_name):
204+
"""For service that does not support rate limit configuration, verify that config rate limit fails
205+
206+
Args:
207+
duthost (object): DUT object
208+
service_name (str): Service name
209+
"""
210+
cmd = 'config syslog rate-limit-container {} -b {} -i {}'.format(
211+
service_name, RATE_LIMIT_BURST, RATE_LIMIT_INTERVAL)
212+
output = duthost.command(cmd, module_ignore_errors=True)['stderr']
213+
pytest_assert('Error' in output, 'Error: config syslog rate limit for {}: {}'.format(service_name, output))
214+
215+
216+
def verify_rate_limit_with_log_generator(duthost, service_name, log_marker, expect_log_regex, expect_log_matches,
217+
is_host=False):
218+
"""Generator syslog with a script and verify that syslog rate limit reached
219+
220+
Args:
221+
duthost (object): DUT host object
222+
service_name (str): Service name
223+
log_marker (str): Log start marker
224+
expect_log_regex (list): A list of expected log message regular expression
225+
expect_log_matches (int): Number of log lines matches the expect_log_regex
226+
is_host (bool, optional): Verify on host side or container side. Defaults to False.
227+
"""
228+
loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=log_marker)
229+
loganalyzer.expect_regex = expect_log_regex
230+
loganalyzer.expected_matches_target = expect_log_matches
231+
232+
if is_host:
233+
run_generator_cmd = "python3 {}".format(REMOTE_LOG_GENERATOR_FILE)
234+
else:
235+
run_generator_cmd = "docker exec -i {} bash -c 'python3 {}'".format(service_name, DOCKER_LOG_GENERATOR_FILE)
236+
237+
with loganalyzer:
238+
duthost.command(run_generator_cmd)

0 commit comments

Comments
 (0)