[Services] Restart Teamd service upon unexpected critical process exi…

…t. (#3703) Signed-off-by: Yong Zhao <yozhao@microsoft.com>
sonic-net · Nov 5, 2019 · 4c31ef3 · 4c31ef3
1 parent 4fa3a1e
commit 4c31ef3
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 1 deletion.
diff --git a/dockers/docker-teamd/Dockerfile.j2 b/dockers/docker-teamd/Dockerfile.j2
@@ -36,5 +36,7 @@ RUN apt-get clean -y      && \
 
 COPY ["start.sh", "/usr/bin/"]
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor"]
 
 ENTRYPOINT ["/usr/bin/supervisord"]
diff --git a/dockers/docker-teamd/critical_processes b/dockers/docker-teamd/critical_processes
@@ -0,0 +1,2 @@
+teammgrd
+teamsyncd
diff --git a/dockers/docker-teamd/supervisord.conf b/dockers/docker-teamd/supervisord.conf
@@ -3,6 +3,12 @@ logfile_maxbytes=1MB
 logfile_backups=2
 nodaemon=true
 
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener
+events=PROCESS_STATE_EXITED
+autostart=true
+autorestart=unexpected
+
 [program:start.sh]
 command=/usr/bin/start.sh
 priority=1
@@ -15,7 +21,7 @@ stderr_logfile=syslog
 command=/usr/sbin/rsyslogd -n
 priority=2
 autostart=false
-autorestart=false
+autorestart=unexpected
 stdout_logfile=syslog
 stderr_logfile=syslog
 

diff --git a/files/build_templates/teamd.service.j2 b/files/build_templates/teamd.service.j2
@@ -3,12 +3,16 @@ Description=TEAMD container
 Requires=updategraph.service
 After=updategraph.service swss.service
 Before=ntp-config.service
+StartLimitIntervalSec=1200
+StartLimitBurst=3
 
 [Service]
 User={{ sonicadmin_user }}
 ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh wait
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+Restart=always
+RestartSec=30
 
 [Install]
 WantedBy=multi-user.target
diff --git a/rules/docker-teamd.mk b/rules/docker-teamd.mk
@@ -29,3 +29,4 @@ $(DOCKER_TEAMD)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
 $(DOCKER_TEAMD)_RUN_OPT += -v /host/warmboot:/var/warmboot
 
 $(DOCKER_TEAMD)_BASE_IMAGE_FILES += teamdctl:/usr/bin/teamdctl
+$(DOCKER_TEAMD)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -29,3 +29,4 @@ $(DOCKER_TEAMD)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
		$(DOCKER_TEAMD)_RUN_OPT += -v /host/warmboot:/var/warmboot

		$(DOCKER_TEAMD)_BASE_IMAGE_FILES += teamdctl:/usr/bin/teamdctl
		$(DOCKER_TEAMD)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)