[dockers][supervisor] Increase event buffer size for dependent-startup (

#5247) When stopping the swss, pmon or bgp containers, log messages like the following can be seen: ``` Aug 23 22:50:43.789760 sonic-dut INFO swss#supervisord 2020-08-23 22:50:10,061 ERRO pool dependent-startup event buffer overflowed, discarding event 34 Aug 23 22:50:43.789760 sonic-dut INFO swss#supervisord 2020-08-23 22:50:10,063 ERRO pool dependent-startup event buffer overflowed, discarding event 35 Aug 23 22:50:43.789760 sonic-dut INFO swss#supervisord 2020-08-23 22:50:10,064 ERRO pool dependent-startup event buffer overflowed, discarding event 36 Aug 23 22:50:43.789760 sonic-dut INFO swss#supervisord 2020-08-23 22:50:10,066 ERRO pool dependent-startup event buffer overflowed, discarding event 37 ``` This is due to the number of programs in the container managed by supervisor, all generating events at the same time. The default event queue buffer size in supervisor is 10. This patch increases that value in all containers in order to eliminate these errors. As more programs are added to the containers, we may need to further adjust these values. I increased all buffer sizes to 25 except for containers with more programs or templated supervisor.conf files which allow for a variable number of programs. In these cases I increased the buffer size to 50. One final exception is the swss container, where the buffer fills up to ~50, so I increased this buffer to 100. Resolves #5241
sonic-net · Sep 28, 2020 · b70c6f7 · b70c6f7
1 parent 615086e
commit b70c6f7
Show file tree

Hide file tree

Showing 22 changed files with 22 additions and 0 deletions.
diff --git a/dockers/docker-dhcp-relay/docker-dhcp-relay.supervisord.conf.j2 b/dockers/docker-dhcp-relay/docker-dhcp-relay.supervisord.conf.j2
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=50
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name dhcp_relay

diff --git a/dockers/docker-lldp-sv2/supervisord.conf.j2 b/dockers/docker-lldp-sv2/supervisord.conf.j2
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name lldp

diff --git a/dockers/docker-nat/supervisord.conf b/dockers/docker-nat/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name nat

diff --git a/dockers/docker-orchagent/supervisord.conf b/dockers/docker-orchagent/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=100
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name swss

diff --git a/dockers/docker-platform-monitor/docker-pmon.supervisord.conf.j2 b/dockers/docker-platform-monitor/docker-pmon.supervisord.conf.j2
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=100
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name pmon

diff --git a/dockers/docker-router-advertiser/docker-router-advertiser.supervisord.conf.j2 b/dockers/docker-router-advertiser/docker-router-advertiser.supervisord.conf.j2
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [eventlistener:supervisor-proc-exit-script]
 command=/usr/bin/supervisor-proc-exit-listener --container-name radv

diff --git a/dockers/docker-sflow/supervisord.conf b/dockers/docker-sflow/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name sflow

diff --git a/dockers/docker-snmp-sv2/supervisord.conf b/dockers/docker-snmp-sv2/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=50
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name snmp

diff --git a/dockers/docker-sonic-mgmt-framework/supervisord.conf b/dockers/docker-sonic-mgmt-framework/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [program:rsyslogd]
 command=/usr/sbin/rsyslogd -n -iNONE

diff --git a/dockers/docker-sonic-restapi/supervisord.conf b/dockers/docker-sonic-restapi/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name restapi

diff --git a/dockers/docker-sonic-telemetry/supervisord.conf b/dockers/docker-sonic-telemetry/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=50
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name telemetry

diff --git a/dockers/docker-teamd/supervisord.conf b/dockers/docker-teamd/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=50
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name teamd

diff --git a/platform/barefoot/docker-syncd-bfn/supervisord.conf b/platform/barefoot/docker-syncd-bfn/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

diff --git a/platform/broadcom/docker-syncd-brcm/supervisord.conf b/platform/broadcom/docker-syncd-brcm/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

diff --git a/platform/innovium/docker-syncd-invm/supervisord.conf b/platform/innovium/docker-syncd-invm/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [program:rsyslogd]
 command=/usr/sbin/rsyslogd -n -iNONE

diff --git a/platform/marvell-arm64/docker-syncd-mrvl/supervisord.conf b/platform/marvell-arm64/docker-syncd-mrvl/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

diff --git a/platform/marvell-armhf/docker-syncd-mrvl/supervisord.conf b/platform/marvell-armhf/docker-syncd-mrvl/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

diff --git a/platform/marvell/docker-syncd-mrvl/supervisord.conf b/platform/marvell/docker-syncd-mrvl/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

diff --git a/platform/mellanox/docker-syncd-mlnx/supervisord.conf b/platform/mellanox/docker-syncd-mlnx/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

diff --git a/platform/nephos/docker-syncd-nephos/supervisord.conf b/platform/nephos/docker-syncd-nephos/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

diff --git a/platform/vs/docker-syncd-vs/supervisord.conf b/platform/vs/docker-syncd-vs/supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=25
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name syncd

diff --git a/src/sonic-config-engine/tests/sample_output/docker-dhcp-relay.supervisord.conf b/src/sonic-config-engine/tests/sample_output/docker-dhcp-relay.supervisord.conf
@@ -10,6 +10,7 @@ autorestart=unexpected
 startretries=0
 exitcodes=0,3
 events=PROCESS_STATE
+buffer_size=50
 
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name dhcp_relay