[containercfgd] Add containercfgd and syslog rate limit configuration…

… support (sonic-net#12489)

* [containercfgd] Add containercfgd and syslog rate limit configuration support

* Fix build issue

* Fix checker issue

* Fix review comment

* Fix review comment

* Update containercfgd.py

dockers/docker-base-bullseye/Dockerfile.j2

@@ -81,7 +81,7 @@ RUN pip3 install supervisor==4.2.1
 # Add support for supervisord to handle startup dependencies
 RUN pip3 install supervisord-dependent-startup==1.4.0
 
-RUN mkdir -p /etc/supervisor /var/log/supervisor
+RUN mkdir -p /var/log/supervisor /etc/supervisor/conf.d
 
 RUN apt-get -y purge   \
     exim4              \
@@ -104,10 +104,10 @@ RUN apt-get clean -y                     && \
     apt-get autoremove -y                && \
     rm -rf /var/lib/apt/lists/* /tmp/* ~/.cache
 
-COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
 COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
 COPY ["root/.vimrc", "/root/.vimrc"]
 
 RUN ln /usr/bin/vim.tiny /usr/bin/vim
 
 COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]
+COPY ["etc/supervisor/containercfgd.conf", "/etc/supervisor/conf.d/"]

dockers/docker-base-bullseye/etc/supervisor/containercfgd.conf

@@ -0,0 +1,9 @@
+[program:containercfgd]
+command=python3 /usr/local/bin/containercfgd
+priority=99
+autostart=false
+autorestart=unexpected
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=rsyslogd:running

dockers/docker-base-buster/Dockerfile.j2

@@ -94,7 +94,7 @@ RUN pip3 install supervisor==4.2.1
 # Add support for supervisord to handle startup dependencies
 RUN pip3 install supervisord-dependent-startup==1.4.0
 
-RUN mkdir -p /etc/supervisor /var/log/supervisor
+RUN mkdir -p /var/log/supervisor /etc/supervisor/conf.d
 
 RUN apt-get -y purge   \
     exim4              \
@@ -117,10 +117,10 @@ RUN apt-get clean -y                     && \
     apt-get autoremove -y                && \
     rm -rf /var/lib/apt/lists/* /tmp/* ~/.cache/
 
-COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
 COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
 COPY ["root/.vimrc", "/root/.vimrc"]
 
 RUN ln /usr/bin/vim.tiny /usr/bin/vim
 
 COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]
+COPY ["etc/supervisor/containercfgd.conf", "/etc/supervisor/conf.d/"]

dockers/docker-base-buster/etc/supervisor/containercfgd.conf

@@ -0,0 +1,9 @@
+[program:containercfgd]
+command=python3 /usr/local/bin/containercfgd
+priority=99
+autostart=false
+autorestart=unexpected
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=rsyslogd:running

dockers/docker-base-stretch/Dockerfile.j2

@@ -90,7 +90,7 @@ RUN pip install supervisor>=3.4.0
 # Add support for supervisord to handle startup dependencies
 RUN pip install supervisord-dependent-startup==1.4.0
 
-RUN mkdir -p /etc/supervisor /var/log/supervisor
+RUN mkdir -p /var/log/supervisor /etc/supervisor/conf.d
 
 RUN apt-get -y purge   \
     exim4              \
@@ -113,10 +113,10 @@ RUN apt-get clean -y                     && \
     apt-get autoremove -y                && \
     rm -rf /var/lib/apt/lists/* /tmp/*
 
-COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
 COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
 COPY ["root/.vimrc", "/root/.vimrc"]
 
 RUN ln /usr/bin/vim.tiny /usr/bin/vim
 
 COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]
+COPY ["etc/supervisor/containercfgd.conf", "/etc/supervisor/conf.d/"]

dockers/docker-base-stretch/etc/supervisor/containercfgd.conf

@@ -0,0 +1,9 @@
+[program:containercfgd]
+command=python /usr/local/bin/containercfgd
+priority=99
+autostart=false
+autorestart=unexpected
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=rsyslogd:running

dockers/docker-base/Dockerfile.j2

@@ -49,7 +49,6 @@ RUN apt-get -y install \
     rsyslog            \
     less
 
-COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
 COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
 COPY ["root/.vimrc", "/root/.vimrc"]
 
@@ -64,10 +63,11 @@ RUN pip install wheel
 # Install supervisor
 RUN pip install supervisor>=3.4.0
 
-RUN mkdir -p /etc/supervisor
+RUN mkdir -p /etc/supervisor/conf.d
 RUN mkdir -p /var/log/supervisor
 
 COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]
+COPY ["etc/supervisor/containercfgd.conf", "/etc/supervisor/conf.d/"]
 
 RUN apt-get -y purge   \
     exim4              \

dockers/docker-base/etc/supervisor/containercfgd.conf

@@ -0,0 +1,9 @@
+[program:containercfgd]
+command=python /usr/local/bin/containercfgd
+priority=99
+autostart=false
+autorestart=unexpected
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=rsyslogd:running

dockers/docker-database/supervisord.conf.j2

@@ -3,6 +3,15 @@ logfile_maxbytes=1MB
 logfile_backups=2
 nodaemon=true
 
+[eventlistener:dependent-startup]
+command=python3 -m supervisord_dependent_startup
+autostart=true
+autorestart=unexpected
+startretries=0
+exitcodes=0,3
+events=PROCESS_STATE
+buffer_size=1024
+
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener --container-name database
 events=PROCESS_STATE_EXITED,PROCESS_STATE_RUNNING
@@ -13,32 +22,37 @@ buffer_size=1024
 [program:rsyslogd]
 command=/usr/sbin/rsyslogd -n -iNONE
 priority=1
-autostart=true
+autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+dependent_startup=true
 
 {% if INSTANCES %}
 {%     for redis_inst, redis_items in INSTANCES.items() %}
-[program: {{ redis_inst }}]
+[program:{{ redis_inst }}]
 {% if redis_items['hostname'] != '127.0.0.1' and redis_inst != 'redis_chassis' %}
 {%- set LOOPBACK_IP = '127.0.0.1' -%}
 {%- else -%}
 {%- set LOOPBACK_IP = '' -%}
 {%- endif -%}
 command=/bin/bash -c "{ [[ -s /var/lib/{{ redis_inst }}/dump.rdb ]] || rm -f /var/lib/{{ redis_inst }}/dump.rdb; } && mkdir -p /var/lib/{{ redis_inst }} && exec /usr/bin/redis-server /etc/redis/redis.conf --bind {{ LOOPBACK_IP }} {{ redis_items['hostname'] }} --port {{ redis_items['port'] }} --unixsocket {{ redis_items['unix_socket_path'] }} --pidfile /var/run/redis/{{ redis_inst }}.pid --dir /var/lib/{{ redis_inst }}"
 priority=2
-autostart=true
+autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=rsyslogd:running
 {%     endfor %}
 {% endif %}
 
 [program:flushdb]
 command=/bin/bash -c "sleep 300 && /usr/local/bin/flush_unused_database"
 priority=3
-autostart=true
+autostart=false
 autorestart=false
 stdout_logfile=syslog
 stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=rsyslogd:running

files/build_templates/docker_image_ctl.j2

@@ -34,16 +34,23 @@ function updateSyslogConf()
     # Also update the container name
     if [[ ($NUM_ASIC -gt 1) ]]; then
         TARGET_IP=$(docker network inspect bridge --format={{ "'{{(index .IPAM.Config 0).Gateway}}'" }})
-        CONTAINER_NAME="$DOCKERNAME"
-        TMP_FILE="/tmp/rsyslog.$CONTAINER_NAME.conf"
-        {%- if docker_container_name == "database" %}
-        python -c "import jinja2, os; paths=['/usr/share/sonic/templates']; loader = jinja2.FileSystemLoader(paths); env = jinja2.Environment(loader=loader, trim_blocks=True); template_file='/usr/share/sonic/templates/rsyslog-container.conf.j2'; template = env.get_template(os.path.basename(template_file)); data=template.render({\"target_ip\":\"$TARGET_IP\",\"container_name\":\"$CONTAINER_NAME\"}); print(data)" > $TMP_FILE
-        {%- else %}
-        sonic-cfggen -t /usr/share/sonic/templates/rsyslog-container.conf.j2 -a "{\"target_ip\": \"$TARGET_IP\", \"container_name\": \"$CONTAINER_NAME\" }"  > $TMP_FILE
-        {%- endif %}
-        docker cp $TMP_FILE ${DOCKERNAME}:/etc/rsyslog.conf
-        rm -rf $TMP_FILE
+    else
+        if [ "$CONTAINER_EXISTS" = "yes" ]; then
+            # database configuration has been synced to /etc/rsyslog.conf
+            # no need generate it to save boot time
+            return
+        fi
+        TARGET_IP="127.0.0.1"
     fi
+    CONTAINER_NAME="$DOCKERNAME"
+    TMP_FILE="/tmp/rsyslog.$CONTAINER_NAME.conf"
+    {%- if docker_container_name == "database" %}
+    python -c "import jinja2, os; paths=['/usr/share/sonic/templates']; loader = jinja2.FileSystemLoader(paths); env = jinja2.Environment(loader=loader, trim_blocks=True); template_file='/usr/share/sonic/templates/rsyslog-container.conf.j2'; template = env.get_template(os.path.basename(template_file)); data=template.render({\"target_ip\":\"$TARGET_IP\",\"container_name\":\"$CONTAINER_NAME\"}); print(data)" > $TMP_FILE
+    {%- else %}
+    sonic-cfggen -d -t /usr/share/sonic/templates/rsyslog-container.conf.j2 -a "{\"target_ip\": \"$TARGET_IP\", \"container_name\": \"$CONTAINER_NAME\" }"  > $TMP_FILE
+    {%- endif %}
+    docker cp $TMP_FILE ${DOCKERNAME}:/etc/rsyslog.conf
+    rm -rf $TMP_FILE
 }
 function ebtables_config()
 {
@@ -342,6 +349,7 @@ start() {
         DOCKERMOUNT=`getMountPoint "$DOCKERCHECK"`
         {%- endif %}
         if [ x"$DOCKERMOUNT" == x"$MOUNTPATH" ]; then
+            CONTAINER_EXISTS="yes"
             preStartAction
             {%- if docker_container_name == "database" %}
             echo "Starting existing ${DOCKERNAME} container"
@@ -536,6 +544,7 @@ start() {
 {%- endif %}
         $REDIS_MNT \
         -v /usr/share/sonic/device/$PLATFORM:/usr/share/sonic/platform:ro \
+        -v /usr/share/sonic/templates/rsyslog-container.conf.j2:/usr/share/sonic/templates/rsyslog-container.conf.j2:ro \
 {%- if sonic_asic_platform != "mellanox" %}
 {%- if mount_default_tmpfs|default("n") == "y" %}
         --tmpfs /tmp \
@@ -547,6 +556,7 @@ start() {
         --env "NAMESPACE_ID"="$DEV" \
         --env "NAMESPACE_PREFIX"="$NAMESPACE_PREFIX" \
         --env "NAMESPACE_COUNT"=$NUM_ASIC \
+        --env "CONTAINER_NAME"=$DOCKERNAME \
         --name=$DOCKERNAME \
 {%- if docker_container_name == "gbsyncd" %}
         -v /var/run/docker-syncd$DEV:/var/run/sswsyncd \
@@ -617,6 +627,7 @@ fi
 {%- endif %}
 NAMESPACE_PREFIX="asic"
 DOCKERNAME=$DOCKERNAME$DEV
+CONTAINER_EXISTS="no"
 if [ "$DEV" ]; then
     NET_NS="$NAMESPACE_PREFIX$DEV" #name of the network namespace
 
@@ -639,4 +650,4 @@ case "$1" in
         echo "Usage: $0 {start namespace(optional)|wait namespace(optional)|stop namespace(optional)}"
         exit 1
         ;;
-esac
+esac

files/image_config/rsyslog/rsyslog-container.conf.j2

@@ -14,8 +14,24 @@ $ModLoad imuxsock # provides support for local system logging
 #
 # Set a rate limit on messages from the container
 #
-$SystemLogRateLimitInterval 300
-$SystemLogRateLimitBurst 20000
+
+{% if SYSLOG_CONFIG_FEATURE is defined %}
+{% if container_name in SYSLOG_CONFIG_FEATURE %}
+{% if 'rate_limit_interval' in SYSLOG_CONFIG_FEATURE[container_name]%}
+{% set rate_limit_interval = SYSLOG_CONFIG_FEATURE[container_name]['rate_limit_interval'] %}
+{% endif %}
+{% if 'rate_limit_burst' in SYSLOG_CONFIG_FEATURE[container_name]%}
+{% set rate_limit_burst = SYSLOG_CONFIG_FEATURE[container_name]['rate_limit_burst'] %}
+{% endif %}
+{% endif %}
+{% endif %}
+
+{% if rate_limit_interval is defined %}
+$SystemLogRateLimitInterval {{ rate_limit_interval }}
+{% endif %}
+{% if rate_limit_burst is defined %}
+$SystemLogRateLimitBurst {{ rate_limit_burst }}
+{% endif %}
 
 #$ModLoad imklog  # provides kernel logging support
 #$ModLoad immark  # provides --MARK-- message capability
@@ -73,4 +89,4 @@ $RepeatedMsgReduction on
 
 ###############
 #### RULES ####
-###############
+###############

rules/docker-config-engine-bullseye.mk

@@ -11,8 +11,9 @@ $(DOCKER_CONFIG_ENGINE_BULLSEYE)_DEPENDS += $(LIBSWSSCOMMON) \
                                           $(SONIC_DB_CLI) \
                                           $(SONIC_EVENTD)
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) \
-                                                $(SONIC_YANG_MGMT_PY3) \
-                                                $(SONIC_YANG_MODELS_PY3)
+                                                  $(SONIC_YANG_MGMT_PY3) \
+                                                  $(SONIC_YANG_MODELS_PY3) \
+                                                  $(SONIC_CONTAINERCFGD)
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3)
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_LOAD_DOCKERS += $(DOCKER_BASE_BULLSEYE)
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $(SWSS_VARS_TEMPLATE)

rules/docker-config-engine-buster.mk

@@ -11,7 +11,8 @@ $(DOCKER_CONFIG_ENGINE_BUSTER)_DEPENDS += $(LIBSWSSCOMMON) \
                                           $(SONIC_DB_CLI)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) \
                                                 $(SONIC_YANG_MGMT_PY3) \
-                                                $(SONIC_YANG_MODELS_PY3)
+                                                $(SONIC_YANG_MODELS_PY3) \
+                                                $(SONIC_CONTAINERCFGD)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(SWSS_VARS_TEMPLATE)

rules/sonic-containercfgd.dep

@@ -0,0 +1,10 @@
+SPATH       := $($(SONIC_CONTAINERCFGD)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/sonic-containercfgd.mk rules/sonic-containercfgd.dep
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
+
+$(SONIC_CONTAINERCFGD)_CACHE_MODE  := GIT_CONTENT_SHA
+$(SONIC_CONTAINERCFGD)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_CONTAINERCFGD)_DEP_FILES   := $(DEP_FILES)
+$(SONIC_CONTAINERCFGD)_SMDEP_FILES := $(SMDEP_FILES)
+$(SONIC_CONTAINERCFGD)_SMDEP_PATHS := $(SPATH)

rules/sonic-containercfgd.mk

@@ -0,0 +1,8 @@
+# sonic-bgpcfgd package
+
+SONIC_CONTAINERCFGD = sonic_containercfgd-1.0-py3-none-any.whl
+$(SONIC_CONTAINERCFGD)_SRC_PATH = $(SRC_PATH)/sonic-containercfgd
+$(SONIC_CONTAINERCFGD)_DEPENDS += $(SONIC_PY_COMMON_PY3)
+$(SONIC_CONTAINERCFGD)_DEBS_DEPENDS = $(LIBSWSSCOMMON) $(PYTHON3_SWSSCOMMON)
+$(SONIC_CONTAINERCFGD)_PYTHON_VERSION = 3
+SONIC_PYTHON_WHEELS += $(SONIC_CONTAINERCFGD)

src/sonic-containercfgd/.gitignore

@@ -0,0 +1,13 @@
+# Compiled Python files
+*.pyc
+
+# Generated by packaging
+*.egg-info/
+.eggs/
+build/
+dist/
+
+# Unit test coverage
+.coverage
+coverage.xml
+htmlcov/