From 95b7badfcd238958dbbdc99c84950bde1d2bf3b5 Mon Sep 17 00:00:00 2001 From: JiaminZhu Date: Thu, 23 Sep 2021 10:15:00 -0700 Subject: [PATCH 1/2] launch flag (#112) Co-authored-by: jiamzhu --- utils/pod/pod.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/pod/pod.go b/utils/pod/pod.go index 7280d16..5e7152f 100644 --- a/utils/pod/pod.go +++ b/utils/pod/pod.go @@ -323,8 +323,8 @@ func LaunchPod(files []string) (types.PodStatus, error) { go dockerLogToPodLogFile(files, true) - err = cmd.Run() LaunchCmdAttempted = true + err = cmd.Run() log.Println("Updated the state of LaunchCmdAttempted to true.") if err != nil { log.Printf("POD_LAUNCH_FAIL -- Error running launch task command : %v", err) From d1eb0cde295a7545feca9204fba6c8b972e41b2b Mon Sep 17 00:00:00 2001 From: JiaminZhu Date: Mon, 11 Oct 2021 18:08:08 -0700 Subject: [PATCH 2/2] update health check order (#114) * check exit code first * skip check health if exit Co-authored-by: jiamzhu --- dce/monitor/plugin/default/monitor.go | 16 ++++++++-------- utils/pod/pod.go | 19 +++++++++++++------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/dce/monitor/plugin/default/monitor.go b/dce/monitor/plugin/default/monitor.go index c9fe298..7616504 100644 --- a/dce/monitor/plugin/default/monitor.go +++ b/dce/monitor/plugin/default/monitor.go @@ -52,20 +52,20 @@ func (m *monitor) Start(ctx context.Context) (types.PodStatus, error) { return types.POD_FAILED, nil } - if healthy == types.UNHEALTHY { - err = pod.PrintInspectDetail(pod.MonitorContainerList[i].ContainerId) - if err != nil { - log.Warnf("failed to get container detail: %s ", err) - } - return types.POD_FAILED, nil - } - if exitCode == 0 && !running { logger.Infof("Removed finished(exit with 0) container %s from monitor list", pod.MonitorContainerList[i]) pod.MonitorContainerList = append(pod.MonitorContainerList[:i], pod.MonitorContainerList[i+1:]...) i-- + continue + } + if healthy == types.UNHEALTHY { + err = pod.PrintInspectDetail(pod.MonitorContainerList[i].ContainerId) + if err != nil { + log.Warnf("failed to get container detail: %s ", err) + } + return types.POD_FAILED, nil } } diff --git a/utils/pod/pod.go b/utils/pod/pod.go index 5e7152f..574620e 100644 --- a/utils/pod/pod.go +++ b/utils/pod/pod.go @@ -1177,13 +1177,8 @@ healthCheck: EndStep(StepMetrics, fmt.Sprintf("HealthCheck-%s", containers[i].ServiceName), types.GetInstanceStatusTag(containers[i], healthy, running, exitCode), err) - if err != nil || healthy == types.UNHEALTHY { + if err != nil { log.Println("POD_INIT_HEALTH_CHECK_FAILURE -- Send Failed") - - if err == nil { - err = errors.New("POD_INIT_HEALTH_CHECK_FAILURE") - } - err = PrintInspectDetail(containers[i].ContainerId) if err != nil { log.Warnf("Error during docker inspect: %v ", err) @@ -1195,6 +1190,7 @@ healthCheck: if healthy == types.HEALTHY { healthCount++ + continue } if exitCode == 0 && !running { @@ -1202,6 +1198,17 @@ healthCheck: containers = append(containers[:i], containers[i+1:]...) i-- healthCount-- + continue + } + + if healthy == types.UNHEALTHY { + log.Println("POD_INIT_HEALTH_CHECK_FAILURE -- Send Failed") + err = PrintInspectDetail(containers[i].ContainerId) + if err != nil { + log.Warnf("Error during docker inspect: %v ", err) + } + out <- types.POD_FAILED.String() + return } // Break health check IF only system proxy is running