Streaming structured events implementation (sonic-net#11848)

With this PR in, you flap BGP and use events_tool to see the published events.
With telemetry PR sonic-net#111 in and corresponding submodule update done in buildimage, one could run gnmi_cli to capture BGP flap events.

dockers/docker-eventd/Dockerfile.j2

@@ -0,0 +1,36 @@
+{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+FROM docker-config-engine-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+ARG docker_container_name
+ARG image_version
+RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf
+
+# Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Pass the image_version to container
+ENV IMAGE_VERSION=$image_version
+
+# Update apt's cache of available packages
+RUN apt-get update
+
+{% if docker_eventd_debs.strip() -%}
+# Copy built Debian packages
+{{ copy_files("debs/", docker_eventd_debs.split(' '), "/debs/") }}
+
+# Install built Debian packages and implicitly install their dependencies
+{{ install_debian_packages(docker_eventd_debs.split(' ')) }}
+{%- endif %}
+
+# Clean up
+RUN apt-get clean -y && \
+ apt-get autoclean -y && \
+ apt-get autoremove -y && \
+ rm -rf /debs
+
+COPY ["start.sh", "/usr/bin/"]
+COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor"]
+
+ENTRYPOINT ["/usr/local/bin/supervisord"]

dockers/docker-eventd/critical_processes

@@ -0,0 +1 @@
+program:eventd

dockers/docker-eventd/start.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+if [ "${RUNTIME_OWNER}" == "" ]; then
+ RUNTIME_OWNER="kube"
+fi
+

dockers/docker-eventd/supervisord.conf

@@ -0,0 +1,52 @@
+[supervisord]
+logfile_maxbytes=1MB
+logfile_backups=2
+nodaemon=true
+
+[eventlistener:dependent-startup]
+command=python3 -m supervisord_dependent_startup
+autostart=true
+autorestart=unexpected
+startretries=0
+exitcodes=0,3
+events=PROCESS_STATE
+buffer_size=1024
+
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener --container-name eventd
+events=PROCESS_STATE_EXITED,PROCESS_STATE_RUNNING
+autostart=true
+autorestart=unexpected
+buffer_size=1024
+
+[program:rsyslogd]
+command=/usr/sbin/rsyslogd -n -iNONE
+priority=1
+autostart=false
+autorestart=unexpected
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+
+[program:start]
+command=/usr/bin/start.sh
+priority=2
+autostart=false
+autorestart=false
+startsecs=0
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=rsyslogd:running
+
+
+[program:eventd]
+command=/usr/sbin/eventd
+priority=3
+autostart=false
+autorestart=false
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=start:exited
+

dockers/docker-fpm-frr/Dockerfile.j2

@@ -55,9 +55,15 @@ COPY ["TSC", "/usr/bin/TSC"]
 COPY ["TS", "/usr/bin/TS"]
 COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
 COPY ["zsocket.sh", "/usr/bin/"]
+COPY ["*.json", "/etc/rsyslog.d/"]
+COPY ["files/rsyslog_plugin.conf.j2", "/etc/rsyslog.d/"]
 RUN chmod a+x /usr/bin/TSA && \
  chmod a+x /usr/bin/TSB && \
  chmod a+x /usr/bin/TSC && \
  chmod a+x /usr/bin/zsocket.sh
 
+RUN j2 -f json /etc/rsyslog.d/rsyslog_plugin.conf.j2 /etc/rsyslog.d/events_info.json > /etc/rsyslog.d/bgp_events.conf
+RUN rm -f /etc/rsyslog.d/rsyslog_plugin.conf.j2*
+RUN rm -f /etc/rsyslog.d/events_info.json*
+
 ENTRYPOINT ["/usr/bin/docker_init.sh"]

dockers/docker-fpm-frr/bgp_regex.json

@@ -0,0 +1,8 @@
+[
+ {
+ "tag": "bgp-state",
+ "regex": "Peer .default\\|([0-9a-f:.]*[0-9a-f]*). admin state is set to .(up|down).",
+ "params": [ "ip", "status" ]
+ }
+]
+

dockers/docker-fpm-frr/events_info.json

@@ -0,0 +1,10 @@
+{
+ "yang_module": "sonic-events-bgp",
+ "proclist": [
+ {
+ "name": "bgp",
+ "parse_json": "bgp_regex.json"
+ }
+ ]
+}
+

files/build_templates/docker_image_ctl.j2

@@ -515,6 +515,7 @@ start() {
 {%- endif -%}
 {%- if docker_container_name == "bgp" %}
  -v /etc/sonic/frr/$DEV:/etc/frr:rw \
+ -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro \
 {%- endif %}
 {%- if docker_container_name == "database" %}
  $DB_OPT \

files/build_templates/eventd.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=EVENTD container
+Requires=updategraph.service
+After=updategraph.service
+BindsTo=sonic.target
+After=sonic.target
+StartLimitIntervalSec=1200
+StartLimitBurst=3
+
+[Service]
+ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
+ExecStart=/usr/bin/{{docker_container_name}}.sh wait
+ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+RestartSec=30
+
+[Install]
+WantedBy=sonic.target

files/build_templates/init_cfg.json.j2

@@ -39,6 +39,7 @@
  ("pmon", "enabled", false, "enabled"),
  ("radv", "enabled", false, "enabled"),
  ("snmp", "enabled", true, "enabled"),
+ ("eventd", "enabled", true, "enabled"),
  ("swss", "enabled", false, "enabled"),
  ("syncd", "enabled", false, "enabled"),
  ("teamd", "enabled", false, "enabled")] %}
@@ -69,7 +70,7 @@
  "check_up_status" : "false",
 {%- endif %}
 {%- if include_kubernetes == "y" %}
-{%- if feature in ["lldp", "pmon", "radv", "snmp", "telemetry"] %}
+{%- if feature in ["lldp", "pmon", "radv", "eventd", "snmp", "telemetry"] %}
  "set_owner": "kube", {% else %}
  "set_owner": "local", {% endif %} {% endif %}
  "high_mem_alert": "disabled"

files/build_templates/rsyslog_plugin.conf.j2

@@ -0,0 +1,19 @@
+## rsyslog-plugin for streaming telemetry via gnmi
+
+
+
+template(name="prog_msg" type="list") {
+ property(name="msg")
+ constant(value="\n")
+}
+
+$ModLoad omprog
+
+{% for proc in proclist %}
+if re_match($programname, "{{ proc.name }}") then {
+ action(type="omprog"
+ binary="/usr/share/sonic/scripts/rsyslog_plugin -r /etc/rsyslog.d/{{ proc.parse_json }} -m {{ yang_module }}"
+ output="/var/log/rsyslog_plugin.log"
+ template="prog_msg")
+}
+{% endfor %}

files/build_templates/sonic_debian_extension.j2

@@ -799,6 +799,10 @@ sudo bash -c "echo { > $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_name
 {% endfor %}
 sudo bash -c "echo } >> $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_names.json"
 
+# copy rsyslog plugin binary for use by all dockers that use plugin to publish events.
+sudo mkdir -p ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}
+sudo cp ${files_path}/rsyslog_plugin ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/
+
 {% for script in installer_start_scripts.split(' ') -%}
 if [ -f $TARGET_MACHINE"_{{script}}" ]; then
  sudo cp $TARGET_MACHINE"_{{script}}" $FILESYSTEM_ROOT/usr/bin/{{script}}

rules/docker-config-engine-bullseye.mk

@@ -8,13 +8,15 @@ $(DOCKER_CONFIG_ENGINE_BULLSEYE)_DEPENDS += $(LIBSWSSCOMMON) \
  $(LIBYANG_CPP) \
  $(LIBYANG_PY3) \
  $(PYTHON3_SWSSCOMMON) \
- $(SONIC_DB_CLI)
+ $(SONIC_DB_CLI) \
+ $(SONIC_EVENTD)
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) \
  $(SONIC_YANG_MGMT_PY3) \
  $(SONIC_YANG_MODELS_PY3)
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3)
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_LOAD_DOCKERS += $(DOCKER_BASE_BULLSEYE)
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $(SWSS_VARS_TEMPLATE)
+$(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $(RSYSLOG_PLUGIN_CONF_J2)
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT)
 
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_DBG_DEPENDS = $($(DOCKER_BASE_BULLSEYE)_DBG_DEPENDS) \

rules/docker-config-engine-buster.mk

@@ -15,6 +15,7 @@ $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) \
 $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(SWSS_VARS_TEMPLATE)
+$(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(RSYSLOG_PLUGIN_CONF_J2)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT)
 
 $(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS = $($(DOCKER_BASE_BUSTER)_DBG_DEPENDS) \

rules/docker-eventd.dep

@@ -0,0 +1,11 @@
+
+DPATH := $($(DOCKER_EVENTD)_PATH)
+DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/docker-eventd.mk rules/docker-eventd.dep 
+DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES += $(shell git ls-files $(DPATH))
+
+$(DOCKER_EVENTD)_CACHE_MODE := GIT_CONTENT_SHA 
+$(DOCKER_EVENTD)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
+$(DOCKER_EVENTD)_DEP_FILES := $(DEP_FILES)
+
+$(eval $(call add_dbg_docker,$(DOCKER_EVENTD),$(DOCKER_EVENTD_DBG)))

rules/docker-eventd.mk

@@ -0,0 +1,47 @@
+# docker image for eventd
+
+DOCKER_EVENTD_STEM = docker-eventd
+DOCKER_EVENTD = $(DOCKER_EVENTD_STEM).gz
+DOCKER_EVENTD_DBG = $(DOCKER_EVENTD_STEM)-$(DBG_IMAGE_MARK).gz
+
+$(DOCKER_EVENTD)_DEPENDS += $(SONIC_EVENTD)
+
+$(DOCKER_EVENTD)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_BULLSEYE)_DBG_DEPENDS)
+$(DOCKER_EVENTD)_DBG_DEPENDS += $(SONIC_EVENTD_DBG) $(LIBSWSSCOMMON_DBG)
+
+$(DOCKER_EVENTD)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BULLSEYE)_DBG_IMAGE_PACKAGES)
+
+$(DOCKER_EVENTD)_LOAD_DOCKERS = $(DOCKER_CONFIG_ENGINE_BULLSEYE)
+
+$(DOCKER_EVENTD)_PATH = $(DOCKERS_PATH)/$(DOCKER_EVENTD_STEM)
+
+$(DOCKER_EVENTD)_INSTALL_PYTHON_WHEELS = $(SONIC_UTILITIES_PY3)
+$(DOCKER_EVENTD)_INSTALL_DEBS = $(PYTHON3_SWSSCOMMON)
+
+$(DOCKER_EVENTD)_VERSION = 1.0.0
+$(DOCKER_EVENTD)_PACKAGE_NAME = eventd
+
+$(DOCKER_DHCP)_SERVICE_REQUIRES = updategraph
+$(DOCKER_DHCP)_SERVICE_AFTER = database
+
+SONIC_DOCKER_IMAGES += $(DOCKER_EVENTD)
+SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_EVENTD)
+
+SONIC_DOCKER_DBG_IMAGES += $(DOCKER_EVENTD_DBG)
+SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_EVENTD_DBG)
+
+$(DOCKER_EVENTD)_CONTAINER_NAME = eventd
+$(DOCKER_EVENTD)_RUN_OPT += --privileged -t
+$(DOCKER_EVENTD)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
+
+SONIC_BULLSEYE_DOCKERS += $(DOCKER_EVENTD)
+SONIC_BULLSEYE_DBG_DOCKERS += $(DOCKER_EVENTD_DBG)
+
+$(DOCKER_EVENTD)_FILESPATH = $($(SONIC_EVENTD)_SRC_PATH)/rsyslog_plugin
+
+$(DOCKER_EVENTD)_PLUGIN = rsyslog_plugin
+$($(DOCKER_EVENTD)_PLUGIN)_PATH = $($(DOCKER_EVENTD)_FILESPATH)
+
+SONIC_COPY_FILES += $($(DOCKER_EVENTD)_PLUGIN)
+$(DOCKER_EVENTD)_SHARED_FILES = $($(DOCKER_EVENTD)_PLUGIN)
+

rules/eventd.dep

@@ -0,0 +1,10 @@
+
+SPATH := $($(SONIC_EVENTD)_SRC_PATH)
+DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/eventd.mk rules/eventd.dep 
+DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
+
+$(SONIC_EVENTD)_CACHE_MODE := GIT_CONTENT_SHA 
+$(SONIC_EVENTD)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_EVENTD)_DEP_FILES := $(DEP_FILES)
+

rules/eventd.mk

@@ -0,0 +1,19 @@
+# eventd package
+
+SONIC_EVENTD_VERSION = 1.0.0-0
+SONIC_EVENTD_PKG_NAME = eventd
+
+SONIC_EVENTD = sonic-$(SONIC_EVENTD_PKG_NAME)_$(SONIC_EVENTD_VERSION)_$(CONFIGURED_ARCH).deb
+$(SONIC_EVENTD)_SRC_PATH = $(SRC_PATH)/sonic-eventd
+$(SONIC_EVENTD)_DEPENDS += $(LIBSWSSCOMMON) $(LIBSWSSCOMMON_DEV)
+
+SONIC_DPKG_DEBS += $(SONIC_EVENTD)
+
+SONIC_EVENTD_DBG = sonic-$(SONIC_EVENTD_PKG_NAME)-dbgsym_$(SONIC_EVENTD_VERSION)_$(CONFIGURED_ARCH).deb
+$(eval $(call add_derived_package,$(SONIC_EVENTD),$(SONIC_EVENTD_DBG)))
+
+# The .c, .cpp, .h & .hpp files under src/{$DBG_SRC_ARCHIVE list}
+# are archived into debug one image to facilitate debugging.
+#
+DBG_SRC_ARCHIVE += sonic-eventd
+

rules/scripts.mk

@@ -32,6 +32,9 @@ $(SWSS_VARS_TEMPLATE)_PATH = files/build_templates
 COPP_CONFIG_TEMPLATE = copp_cfg.j2
 $(COPP_CONFIG_TEMPLATE)_PATH = files/image_config/copp
 
+RSYSLOG_PLUGIN_CONF_J2 = rsyslog_plugin.conf.j2
+$(RSYSLOG_PLUGIN_CONF_J2)_PATH = files/build_templates
+
 SONIC_COPY_FILES += $(CONFIGDB_LOAD_SCRIPT) \
  $(ARP_UPDATE_SCRIPT) \
  $(ARP_UPDATE_VARS_TEMPLATE) \
@@ -42,4 +45,5 @@ SONIC_COPY_FILES += $(CONFIGDB_LOAD_SCRIPT) \
  $(SYSCTL_NET_CONFIG) \
  $(UPDATE_CHASSISDB_CONFIG_SCRIPT) \
  $(SWSS_VARS_TEMPLATE) \
+ $(RSYSLOG_PLUGIN_CONF_J2) \
  $(COPP_CONFIG_TEMPLATE)

rules/telemetry.mk

@@ -2,6 +2,7 @@
 
 SONIC_TELEMETRY = sonic-gnmi_0.1_$(CONFIGURED_ARCH).deb
 $(SONIC_TELEMETRY)_SRC_PATH = $(SRC_PATH)/sonic-gnmi
-$(SONIC_TELEMETRY)_DEPENDS = $(SONIC_MGMT_COMMON) $(SONIC_MGMT_COMMON_CODEGEN)
-$(SONIC_TELEMETRY)_RDEPENDS = 
+$(SONIC_TELEMETRY)_DEPENDS = $(SONIC_MGMT_COMMON) $(SONIC_MGMT_COMMON_CODEGEN) \
+ $(LIBSWSSCOMMON_DEV) $(LIBSWSSCOMMON) 
+$(SONIC_TELEMETRY)_RDEPENDS = $(LIBSWSSCOMMON) $(LIBSWSSCOMMON_DEV)
 SONIC_DPKG_DEBS += $(SONIC_TELEMETRY)

slave.mk

@@ -1292,6 +1292,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
  $(if $($(docker:-dbg.gz=.gz)_MACHINE),\
  mv $($(docker:-dbg.gz=.gz)_CONTAINER_NAME).sh $($(docker:-dbg.gz=.gz)_MACHINE)_$($(docker:-dbg.gz=.gz)_CONTAINER_NAME).sh
  )
+ $(foreach file, $($(docker)_SHARED_FILES), \
+ { cp $($(file)_PATH)/$(file) $(FILES_PATH)/ $(LOG) || exit 1 ; } ; )
  )
 
  # Exported variables are used by sonic_debian_extension.sh