Add events to host and create rsyslog_plugin deb pkg (#12059)

Why I did it

Create rsyslog plugin deb for other containers/host to install
Add events for bgp and host events

dockers/docker-eventd/supervisord.conf

@@ -41,7 +41,7 @@ dependent_startup_wait_for=rsyslogd:running
 
 
 [program:eventd]
-command=/usr/sbin/eventd
+command=/usr/bin/eventd
 priority=3
 autostart=false
 autorestart=false

dockers/docker-fpm-frr/Dockerfile.j2

@@ -56,14 +56,14 @@ COPY ["TS", "/usr/bin/TS"]
 COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
 COPY ["zsocket.sh", "/usr/bin/"]
 COPY ["*.json", "/etc/rsyslog.d/"]
-# COPY ["files/rsyslog_plugin.conf.j2", "/etc/rsyslog.d/"]
+COPY ["files/rsyslog_plugin.conf.j2", "/etc/rsyslog.d/"]
 RUN chmod a+x /usr/bin/TSA && \
     chmod a+x /usr/bin/TSB && \
     chmod a+x /usr/bin/TSC && \
     chmod a+x /usr/bin/zsocket.sh
 
-# RUN j2 -f json /etc/rsyslog.d/rsyslog_plugin.conf.j2 /etc/rsyslog.d/events_info.json  > /etc/rsyslog.d/bgp_events.conf
-# RUN rm -f /etc/rsyslog.d/rsyslog_plugin.conf.j2*
-RUN rm -f /etc/rsyslog.d/events_info.json*
+RUN j2 -f json /etc/rsyslog.d/rsyslog_plugin.conf.j2 /etc/rsyslog.d/events_info.json  > /etc/rsyslog.d/bgp_events.conf
+RUN rm -f /etc/rsyslog.d/rsyslog_plugin.conf.j2
+RUN rm -f /etc/rsyslog.d/events_info.json
 
 ENTRYPOINT ["/usr/bin/docker_init.sh"]

files/build_templates/events_info.json

@@ -0,0 +1,17 @@
+{
+    "yang_module": "sonic-events-host",
+    "proclist": [
+        {
+            "name": "monit",
+	    "parse_json": "monit_regex.json"
+        },
+        {
+            "name": "sshd",
+            "parse_json": "sshd_regex.json"
+        },
+        {
+            "name": "systemd",
+            "parse_json": "systemd_regex.json"
+        }
+    ]
+}

files/build_templates/init_cfg.json.j2

@@ -39,7 +39,7 @@
                    ("pmon", "enabled", false, "enabled"),
                    ("radv", "enabled", false, "enabled"),
                    ("snmp", "enabled", true, "enabled"),
-                   ("eventd", "enabled", true, "enabled"),
+                   ("eventd", "enabled", false, "enabled"),
                    ("swss", "enabled", false, "enabled"),
                    ("syncd", "enabled", false, "enabled"),
                    ("teamd", "enabled", false, "enabled")] %}

files/build_templates/monit_regex.json

@@ -0,0 +1,17 @@
+[
+    {
+        "tag": "disk-usage",
+        "regex": ".([a-zA-Z0-9-_]*). space usage (\\d+\\.\\d+)% matches resource limit .space usage.(\\d+\\.\\d+)%.",
+        "params": [ "fs", "usage", "limit" ]
+    },
+    {
+        "tag": "memory-usage",
+        "regex": "mem usage of (\\d+\\.\\d+)% matches resource limit .mem usage>(\\d+\\.\\d+)%.",
+        "params": [ "usage", "limit" ]
+    },
+    {
+        "tag": "cpu-usage",
+        "regex": "cpu user usage of (\\d+\\.\\d+)% matches resource limit .cpu user usage>(\\d+\\.\\d+)%.",
+        "params": [ "usage", "limit" ]
+    }
+]

files/build_templates/rsyslog_plugin.conf.j2

@@ -12,7 +12,7 @@ $ModLoad omprog
 {% for proc in proclist %}
 if re_match($programname, "{{ proc.name }}") then {
     action(type="omprog"
-        binary="/usr/share/sonic/scripts/rsyslog_plugin -r /etc/rsyslog.d/{{ proc.parse_json }} -m {{ yang_module }}"
+        binary="/usr/bin/rsyslog_plugin -r /etc/rsyslog.d/{{ proc.parse_json }} -m {{ yang_module }}"
         output="/var/log/rsyslog_plugin.log"
         template="prog_msg")
 }

files/build_templates/sonic_debian_extension.j2

@@ -319,6 +319,16 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i {{deb}} || sudo LANG=C DEBIAN_FRONTEND=noni
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-db-cli_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 
+# Install sonic-rsyslog-plugin
+sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-rsyslog-plugin_*.deb || \
+    sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
+
+# Generate host conf for rsyslog_plugin
+j2 -f json $BUILD_TEMPLATES/rsyslog_plugin.conf.j2 $BUILD_TEMPLATES/events_info.json | sudo tee $FILESYSTEM_ROOT_ETC/rsyslog.d/host_events.conf
+sudo cp $BUILD_TEMPLATES/monit_regex.json $FILESYSTEM_ROOT_ETC/rsyslog.d/
+sudo cp $BUILD_TEMPLATES/sshd_regex.json $FILESYSTEM_ROOT_ETC/rsyslog.d/
+sudo cp $BUILD_TEMPLATES/systemd_regex.json $FILESYSTEM_ROOT_ETC/rsyslog.d/
+
 # Install custom-built monit package and SONiC configuration files
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/monit_*.deb || \
     sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
@@ -799,10 +809,6 @@ sudo bash -c "echo { > $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_name
 {% endfor %}
 sudo bash -c "echo } >> $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_names.json"
 
-# copy rsyslog plugin binary for use by all dockers that use plugin to publish events.
-# sudo mkdir -p ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}
-# sudo cp ${files_path}/rsyslog_plugin ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/
-
 {% for script in installer_start_scripts.split(' ') -%}
 if [ -f $TARGET_MACHINE"_{{script}}" ]; then
     sudo cp $TARGET_MACHINE"_{{script}}" $FILESYSTEM_ROOT/usr/bin/{{script}}

files/build_templates/sshd_regex.json

@@ -0,0 +1,7 @@
+[
+    {
+        "tag": "event-sshd",
+        "regex": "auth fail: Password Incorrect. user:.([a-zA-Z0-9-_]*)",
+        "params": [ "username" ]
+    }
+]

files/build_templates/systemd_regex.json

@@ -0,0 +1,7 @@
+[
+    {
+        "tag": "event-stopped-ctr",
+        "regex": "Stopped ([a-zA-Z-_\\s]*) container",
+        "params": [ "ctr-name" ]
+    }
+]

files/image_config/monit/container_checker

@@ -22,6 +22,9 @@ import sys
 from sonic_py_common import multi_asic, device_info
 from swsscommon import swsscommon
 
+EVENTS_PUBLISHER_SOURCE = "sonic-events-host"
+EVENTS_PUBLISHER_TAG = "event-down-ctr"
+
 def get_expected_running_containers():
     """
     @summary: This function will get the expected running & always-enabled containers by following the rule:
@@ -150,6 +153,17 @@ def get_current_running_containers(always_running_containers):
     return current_running_containers
 
 
+def publish_events(lst):
+    events_handle = swsscommon.events_init_publisher(EVENTS_PUBLISHER_SOURCE)
+    params = swsscommon.FieldValueMap()
+
+    for ctr in lst:
+        params["name"] = ctr;
+        swsscommon.event_publish(events_handle, EVENTS_PUBLISHER_TAG, params)
+
+    swsscommon.events_deinit_publisher(events_handle)
+
+
 def main():
     """
     @summary: This function will compare the difference between the current running containers
@@ -162,6 +176,7 @@ def main():
     expected_running_containers |= always_running_containers
     not_running_containers = expected_running_containers.difference(current_running_containers)
     if not_running_containers:
+        publish_events(not_running_containers)
         print("Expected containers not running: " + ", ".join(not_running_containers))
         sys.exit(3)
 

rules/docker-eventd.mk

@@ -36,13 +36,3 @@ $(DOCKER_EVENTD)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
 
 SONIC_BULLSEYE_DOCKERS += $(DOCKER_EVENTD)
 SONIC_BULLSEYE_DBG_DOCKERS += $(DOCKER_EVENTD_DBG)
-
-$(DOCKER_EVENTD)_FILESPATH = $($(SONIC_EVENTD)_SRC_PATH)/rsyslog_plugin
-
-$(DOCKER_EVENTD)_PLUGIN = rsyslog_plugin
-$($(DOCKER_EVENTD)_PLUGIN)_PATH = $($(DOCKER_EVENTD)_FILESPATH)
-
-SONIC_COPY_FILES += $($(DOCKER_EVENTD)_PLUGIN)
-# Some builds fails to find this file. Remove until we root cause it.
-# $(DOCKER_EVENTD)_SHARED_FILES = $($(DOCKER_EVENTD)_PLUGIN)
-

rules/docker-fpm-frr.mk

@@ -7,10 +7,10 @@ DOCKER_FPM_FRR_DBG = $(DOCKER_FPM_FRR_STEM)-$(DBG_IMAGE_MARK).gz
 $(DOCKER_FPM_FRR)_PATH = $(DOCKERS_PATH)/$(DOCKER_FPM_FRR_STEM)
 $(DOCKER_FPM_FRR)_PYTHON_WHEELS += $(SONIC_BGPCFGD) $(SONIC_FRR_MGMT_FRAMEWORK)
 
-$(DOCKER_FPM_FRR)_DEPENDS += $(FRR) $(FRR_SNMP) $(SWSS) $(LIBYANG2)
+$(DOCKER_FPM_FRR)_DEPENDS += $(FRR) $(FRR_SNMP) $(SWSS) $(LIBYANG2) $(SONIC_RSYSLOG_PLUGIN)
 $(DOCKER_FPM_FRR)_DBG_DEPENDS = $($(DOCKER_SWSS_LAYER_BULLSEYE)_DBG_DEPENDS)
 $(DOCKER_FPM_FRR)_DBG_DEPENDS += $(SWSS_DBG) $(LIBSWSSCOMMON_DBG) \
-                                $(FRR_DBG) $(FRR_SNMP_DBG) $(LIBYANG2_DBG)
+                                $(FRR_DBG) $(FRR_SNMP_DBG) $(LIBYANG2_DBG) $(SONIC_RSYSLOG_PLUGIN)
 
 $(DOCKER_FPM_FRR)_DBG_IMAGE_PACKAGES = $($(DOCKER_SWSS_LAYER_BULLSEYE)_DBG_IMAGE_PACKAGES)
 

rules/eventd.mk

@@ -11,9 +11,9 @@ SONIC_DPKG_DEBS += $(SONIC_EVENTD)
 
 SONIC_EVENTD_DBG = sonic-$(SONIC_EVENTD_PKG_NAME)-dbgsym_$(SONIC_EVENTD_VERSION)_$(CONFIGURED_ARCH).deb
 $(eval $(call add_derived_package,$(SONIC_EVENTD),$(SONIC_EVENTD_DBG)))
-
+SONIC_RSYSLOG_PLUGIN = sonic-rsyslog-plugin_$(SONIC_EVENTD_VERSION)_$(CONFIGURED_ARCH).deb
+$(eval $(call add_derived_package,$(SONIC_EVENTD),$(SONIC_RSYSLOG_PLUGIN)))
 # The .c, .cpp, .h & .hpp files under src/{$DBG_SRC_ARCHIVE list}
 # are archived into debug one image to facilitate debugging.
 #
 DBG_SRC_ARCHIVE += sonic-eventd
-

slave.mk

@@ -1129,6 +1129,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
                 $(PYTHON_SWSSCOMMON) \
                 $(PYTHON3_SWSSCOMMON) \
                 $(SONIC_DB_CLI) \
+                $(SONIC_RSYSLOG_PLUGIN) \
                 $(SONIC_UTILITIES_DATA) \
                 $(SONIC_HOST_SERVICES_DATA) \
                 $(BASH) \
@@ -1292,8 +1293,6 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 		$(if $($(docker:-dbg.gz=.gz)_MACHINE),\
 			mv $($(docker:-dbg.gz=.gz)_CONTAINER_NAME).sh $($(docker:-dbg.gz=.gz)_MACHINE)_$($(docker:-dbg.gz=.gz)_CONTAINER_NAME).sh
 		)
-		$(foreach file, $($(docker)_SHARED_FILES), \
-			{ cp $($(file)_PATH)/$(file) $(FILES_PATH)/ $(LOG) || exit 1 ; } ; )
 	)
 
 	# Exported variables are used by sonic_debian_extension.sh

src/sonic-eventd/Makefile

@@ -5,6 +5,9 @@ EVENTD_TOOL := tools/events_tool
 EVENTD_PUBLISH_TOOL := tools/events_publish_tool.py
 RSYSLOG-PLUGIN_TARGET := rsyslog_plugin/rsyslog_plugin
 RSYSLOG-PLUGIN_TEST := rsyslog_plugin_tests/tests
+EVENTD_MONIT := tools/events_monit_test.py
+EVENTD_MONIT_CONF := tools/monit_events
+
 CP := cp
 MKDIR := mkdir
 CC := g++
@@ -68,15 +71,18 @@ rsyslog-plugin-tests: $(RSYSLOG-PLUGIN-TEST_OBJS)
 	@echo ' '
 
 install:
-	$(MKDIR) -p $(DESTDIR)/usr/sbin
-	$(CP) $(EVENTD_TARGET) $(DESTDIR)/usr/sbin
-	$(CP) $(EVENTD_TOOL) $(DESTDIR)/usr/sbin
-	$(CP) $(EVENTD_PUBLISH_TOOL) $(DESTDIR)/usr/sbin
+	$(MKDIR) -p $(DESTDIR)/usr/bin
+	$(MKDIR) -p $(DESTDIR)/etc/monit/conf.d
+	$(CP) $(EVENTD_TARGET) $(DESTDIR)/usr/bin
+	$(CP) $(EVENTD_TOOL) $(DESTDIR)/usr/bin
+	$(CP) $(EVENTD_PUBLISH_TOOL) $(DESTDIR)/usr/bin
+	$(CP) $(RSYSLOG-PLUGIN_TARGET) $(DESTDIR)/usr/bin
+	$(CP) $(EVENTD_MONIT) $(DESTDIR)/usr/bin
+	$(CP) $(EVENTD_MONIT_CONF) $(DESTDIR)/etc/monit/conf.d
 
 deinstall:
-	$(RM) $(DESTDIR)/usr/sbin/$(EVENTD_TARGET)
-	$(RM) $(DESTDIR)/usr/sbin/$(RSYSLOG-PLUGIN_TARGET)
-	$(RM) -rf $(DESTDIR)/usr/sbin
+	$(RM) -rf $(DESTDIR)/usr
+	$(RM) -rf $(DESTDIR)/etc
 
 clean:
 	-@echo ' '

src/sonic-eventd/debian/control

@@ -2,7 +2,7 @@ Source: sonic-eventd
 Section: devel
 Priority: optional
 Maintainer: Renuka Manavalan <remanava@microsoft.com>
-Build-Depends: debhelper (>= 12.0.0), libevent-dev, libboost-thread-dev, libboost-system-dev, libswsscommon-dev
+Build-Depends: debhelper (>= 12.0.0), libevent-dev, libboost-thread-dev, libboost-system-dev, libswsscommon-dev, liblua5.1-0
 Standards-Version: 3.9.3
 Homepage: https://github.com/Azure/sonic-buildimage
 XS-Go-Import-Path: github.com/Azure/sonic-buildimage
@@ -12,3 +12,9 @@ Architecture: any
 Built-Using: ${misc:Built-Using}
 Depends: ${shlibs:Depends}
 Description: SONiC event service
+
+Package: sonic-rsyslog-plugin
+Architecture: any
+Built-Using: ${misc:Built-Using}
+Depends: ${shlibs:Depends}
+Description: SONiC rsyslog plugin service

src/sonic-eventd/debian/sonic-eventd.install

@@ -0,0 +1,3 @@
+usr/bin/eventd
+usr/bin/events_tool
+usr/bin/events_publish_tool.py

src/sonic-eventd/debian/sonic-rsyslog-plugin.install

@@ -0,0 +1,3 @@
+usr/bin/rsyslog_plugin
+usr/bin/events_monit_test.py
+etc/monit/conf.d/monit_events