Merge pull request #30 from paypal/develop

cut release 0.2.0

config/config.go

@@ -45,13 +45,15 @@ const (
 	INFRA_CONTAINER                      = "infracontainer"
 	PULL_RETRY                           = "pullretry"
 	MAX_RETRY                            = "maxretry"
+	RETRY_INTERVAL                       = "retryinterval"
 	NETWORKS                             = "networks"
 	PRE_EXIST                            = "pre_existing"
 	NETWORK_NAME                         = "name"
 	NETWORK_DRIVER                       = "driver"
 	CLEANPOD                             = "cleanpod"
 	CLEAN_CONTAINER_VOLUME_ON_MESOS_KILL = "cleanvolumeandcontaineronmesoskill"
 	CLEAN_IMAGE_ON_MESOS_KILL            = "cleanimageonmesoskill"
+	DOCKER_COMPOSE_VERBOSE               = "dockercomposeverbose"
 )
 
 // Read from default configuration file and set config as key/values
@@ -69,12 +71,12 @@ func ConfigInit(pluginConfig string) {
 
 	file, err := os.Open(pluginConfig)
 	if err != nil {
-		log.Errorf("Fail to open file, err: %s\n", err.Error())
+		log.Fatalf("Fail to open file, err: %s\n", err.Error())
 	}
 
 	err = viper.MergeConfig(file)
 	if err != nil {
-		log.Errorf("Fail to merge config, err: %s\n", err.Error())
+		log.Fatalf("Fail to merge config, err: %s\n", err.Error())
 	}
 }
 
@@ -98,7 +100,7 @@ func getConfigFromFile(cfgFile string) error {
 
 	err := viper.ReadInConfig()
 	if err != nil {
-		log.Fatalf("No config file loaded, err: %s\n", err.Error())
+		log.Errorf("No config file loaded, err: %s\n", err.Error())
 		return err
 	}
 	return nil
@@ -143,13 +145,36 @@ func GetPullRetryCount() int {
 	return _retry
 }
 
-func GetTimeout() time.Duration {
+func GetLaunchTimeout() time.Duration {
 	timeout := GetConfigSection(LAUNCH_TASK)[TIMEOUT]
 	if timeout == "" {
 		log.Warningln("pod timeout doesn't set in config...timeout will be set as 500s")
 		return time.Duration(500000)
 	}
 	t, err := strconv.Atoi(timeout)
+	if err != nil {
+		log.Errorf("Error converting timeout from string to int : %s...timeout will be set as 500s\n", err.Error())
+		return time.Duration(500000)
+	}
+	return time.Duration(t)
+}
+
+func GetStopTimeout() string {
+	timeout := GetConfigSection(CLEANPOD)[TIMEOUT]
+	if timeout == "" {
+		log.Warningln("pod timeout doesn't set in config...timeout will be set as 10s")
+		return "10"
+	}
+	return timeout
+}
+
+func GetRetryInterval() time.Duration {
+	interval := GetConfigSection(LAUNCH_TASK)[RETRY_INTERVAL]
+	if interval == "" {
+		log.Warningln("retry interval doesn't set in config...timeout will be set as 10s")
+		return time.Duration(10000)
+	}
+	t, err := strconv.Atoi(interval)
 	if err != nil {
 		log.Fatalf("Error converting timeout from string to int : %s\n", err.Error())
 	}
@@ -164,7 +189,8 @@ func GetMaxRetry() int {
 	}
 	i, err := strconv.Atoi(retry)
 	if err != nil {
-		log.Fatalf("Error converting retry from string to int : %s\n", err.Error())
+		log.Errorf("Error converting retry from string to int : %s...setting max retry to zero\n", err.Error())
+		return 0
 	}
 	return i
 }
@@ -195,3 +221,7 @@ func GetNetwork() (types.Network, bool) {
 	}
 	return network, true
 }
+
+func EnableVerbose() bool {
+	return GetConfig().GetBool(DOCKER_COMPOSE_VERBOSE)
+}

config/config.yaml

@@ -2,10 +2,19 @@ launchtask:
    podmonitorinterval: 10000
    pullretry: 3
    maxretry: 3
+   retryinterval: 10000
    timeout: 500000
 plugins:
    pluginorder: general
 cleanpod:
+   timeout: 20
    unhealthy: true
    cleanvolumeandcontaineronmesoskill: true
-   cleanimageonmesoskill: true
+   cleanimageonmesoskill: true
+dockerdump:
+   enable: true
+   dumppath: /home/ubuntu
+   dockerpidfile: /var/run/docker.pid
+   containerpidfile: /run/docker/libcontainerd/docker-containerd.pid
+   dockerlogpath: /var/log/upstart/docker.log
+dockercomposeverbose: true

dce/main.go

@@ -108,7 +108,7 @@ func (exec *dockerComposeExecutor) LaunchTask(driver exec.ExecutorDriver, taskIn
 	var ctx context.Context
 	var cancel context.CancelFunc
 	ctx = context.Background()
-	ctx, cancel = context.WithTimeout(ctx, config.GetTimeout()*time.Millisecond)
+	ctx, cancel = context.WithTimeout(ctx, config.GetLaunchTimeout()*time.Millisecond)
 	go pod.WaitOnPod(&ctx)
 
 	// Get order of plugins from config or mesos labels
@@ -173,13 +173,6 @@ func (exec *dockerComposeExecutor) LaunchTask(driver exec.ExecutorDriver, taskIn
 			cancel()
 			pod.SendPodStatus(types.POD_FAILED)
 		}
-		if res == types.POD_RUNNING {
-			cancel()
-			if pod.GetPodStatus() != types.POD_RUNNING {
-				pod.SendPodStatus(types.POD_RUNNING)
-				go monitor.MonitorPoller()
-			}
-		}
 
 		// Temp status keeps the pod status returned by PostLaunchTask
 		var tempStatus string
@@ -198,6 +191,14 @@ func (exec *dockerComposeExecutor) LaunchTask(driver exec.ExecutorDriver, taskIn
 			}
 		}
 
+		if res == types.POD_RUNNING {
+			cancel()
+			if pod.GetPodStatus() != types.POD_RUNNING {
+				pod.SendPodStatus(types.POD_RUNNING)
+				go monitor.MonitorPoller()
+			}
+		}
+
 	default:
 		logger.Printf("default: Unknown status -- %s from pullAndLaunchPod ", replyPodStatus)
 
@@ -294,7 +295,7 @@ func pullAndLaunchPod() string {
 }
 
 func initHealthCheck(podServices map[string]bool) (string, error) {
-	res, err := wait.WaitUntil(config.GetTimeout()*time.Millisecond, wait.ConditionCHFunc(func(healthCheckReply chan string) {
+	res, err := wait.WaitUntil(config.GetLaunchTimeout()*time.Millisecond, wait.ConditionCHFunc(func(healthCheckReply chan string) {
 		pod.HealthCheck(pod.ComposeFiles, podServices, healthCheckReply)
 	}))
 

dce/monitor/monitor.go

@@ -32,10 +32,57 @@ import (
 // Watching pod status and notifying executor if any container in the pod goes wrong
 func podMonitor() string {
 	var err error
+
+	for i := 0; i < len(pod.PodContainers); i++ {
+		var healthy string
+		var exitCode int
+
+		if hc, ok := pod.HealthCheckListId[pod.PodContainers[i]]; ok && hc {
+			healthy, exitCode, err = pod.CheckContainer(pod.PodContainers[i], true)
+			//log.Printf("container %s has health check, health status: %s, exitCode: %d, err : %v", containers[i], healthy, exitCode, err)
+		} else {
+			healthy, exitCode, err = pod.CheckContainer(pod.PodContainers[i], false)
+			//log.Printf("container %s no health check, exitCode: %d, err : %v", containers[i], healthy, exitCode, err)
+		}
+
+		if err != nil {
+			log.Println(fmt.Sprintf("Error inspecting container with id : %s, %v", pod.PodContainers[i], err.Error()))
+			log.Println("Pod Monitor : Send Failed")
+			return types.POD_FAILED
+		}
+
+		if exitCode != 0 && exitCode != -1 {
+			log.Println("Pod Monitor : Stopped and send Failed")
+			return types.POD_FAILED
+		}
+
+		if healthy == types.UNHEALTHY {
+			if config.GetConfigSection(config.CLEANPOD) == nil ||
+				config.GetConfigSection(config.CLEANPOD)[types.UNHEALTHY] == "true" {
+				log.Println("Pod Monitor : Stopped and send Failed")
+				return types.POD_FAILED
+			}
+			log.Warnf("Container %s became unhealthy, but pod won't be killed due to cleanpod config", pod.PodContainers[i])
+		}
+
+		if exitCode == 0 {
+			log.Printf("Removed finished(exit with 0) container %s from monitor list", pod.PodContainers[i])
+			pod.PodContainers = append(pod.PodContainers[:i], pod.PodContainers[i+1:]...)
+			i--
+
+		}
+	}
+
+	if len(pod.PodContainers) == 0 {
+		log.Println("Pod Monitor : Finished")
+		return types.POD_FINISHED
+	}
+
+	/*var healthCount int
 	containers := make([]string, len(pod.PodContainers))
 	copy(containers, pod.PodContainers)
 
-	for i := 0; i < len(containers); i++ {
+		for i := 0; i < len(containers); i++ {
 		var healthy string
 		var exitCode int
 
@@ -70,9 +117,9 @@ func podMonitor() string {
 		if exitCode == 0 || exitCode == -1 {
 			containers = append(containers[:i], containers[i+1:]...)
 			i--
-		}
+		}*/
 
-	}
+	//}
 	return ""
 }
 
@@ -82,10 +129,11 @@ func MonitorPoller() {
 
 	gap, err := strconv.Atoi(config.GetConfigSection(config.LAUNCH_TASK)[config.POD_MONITOR_INTERVAL])
 	if err != nil {
-		log.Fatalf("Error converting podmonitorinterval from string to int : %s|\n", err.Error())
+		log.Errorf("Error converting podmonitorinterval from string to int : %s", err.Error())
+		gap = 10000
 	}
 
-	res, _ := wait.PollForever(time.Duration(gap)*time.Millisecond, nil, wait.ConditionFunc(func() (string, error) {
+	res, err := wait.PollForever(time.Duration(gap)*time.Millisecond, nil, wait.ConditionFunc(func() (string, error) {
 		return podMonitor(), nil
 	}))
 
@@ -97,6 +145,11 @@ func MonitorPoller() {
 		return
 	}
 
+	if err != nil {
+		pod.SendPodStatus(types.POD_FAILED)
+		return
+	}
+
 	switch res {
 	case types.POD_FAILED:
 		pod.SendPodStatus(types.POD_FAILED)

examples/vagrant/config/daemon.json

@@ -0,0 +1,3 @@
+{
+  "live-restore": true
+}

examples/vagrant/config/docker-compose-executor.json

@@ -2,7 +2,7 @@
   {
     "executor": {
       "command": {
-        "value": "./executor",
+        "value": "./executor -log_dir=/var/log -logtostderr=true -v=1",
         "shell": "true",
         "uris": [
           {
@@ -45,4 +45,4 @@
     },
     "task_prefix": "compose-"
   }
-]
+]

examples/vagrant/provision-dev-cluster.sh

@@ -91,6 +91,8 @@ function prepare_extra {
   mkdir -p /etc/mesos-slave
   mkdir -p /etc/mesos-master
   mkdir -p /output/usr/local/lib/mesos
+  mkdir -p /etc/docker || true
+  cp /vagrant/examples/vagrant/config/daemon.json /etc/docker/
   cp /vagrant/examples/vagrant/mesos_config/etc_mesos-slave/* /etc/mesos-slave
   cp /vagrant/examples/vagrant/mesos_config/etc_mesos-master/* /etc/mesos-master
   cp /vagrant/examples/vagrant/mesos-module/${MESOS_MODULE}/* /output/usr/local/lib/mesos/

plugin/general/impl.go

@@ -109,6 +109,10 @@ func (ge *generalExt) PreLaunchTask(ctx *context.Context, composeFiles *[]string
 		delete(filesMap, editedFiles[indexInfra])
 		*ctx = context.WithValue(*ctx, types.SERVICE_DETAIL, filesMap)
 		editedFiles = append(editedFiles[:indexInfra], editedFiles[indexInfra+1:]...)
+		err = utils.DeleteFile(types.INFRA_CONTAINER_YML)
+		if err != nil {
+			log.Errorf("Error deleting infra yml file %v", err)
+		}
 	}
 
 	logger.Println("====================context out====================")

types/types.go

@@ -17,52 +17,52 @@ package types
 import (
 	exec_cmd "os/exec"
 	"sync"
-	"time"
 
 	"github.com/mesos/mesos-go/mesosproto"
 )
 
 const (
-	HEALTHY                               = "healthy"
-	UNHEALTHY                             = "unhealthy"
-	POD_STAGING                           = "POD_STAGING"
-	POD_STARTING                          = "POD_STARTING"
-	POD_RUNNING                           = "POD_RUNNING"
-	POD_FAILED                            = "POD_FAILED"
-	POD_KILLED                            = "POD_KILLED"
-	POD_FINISHED                          = "POD_FINISHED"
-	POD_PULL_FAILED                       = "PULL_IMAGE_FAILED"
-	HEALTHCHECK                           = "healthcheck"
-	CONTAINER_NAME                        = "container_name"
-	NETWORK_MODE                          = "network_mode"
-	LINKS                                 = "links"
-	PORTS                                 = "ports"
-	LABELS                                = "labels"
-	RESTART                               = "restart"
-	SERVICES                              = "services"
-	IMAGE                                 = "image"
-	VERSION                               = "version"
-	NETWORKS                              = "networks"
-	TASK_ID                               = "taskId"
-	EXECUTOR_ID                           = "executorId"
-	CGROUP_PARENT                         = "cgroup_parent"
-	PLUGIN_ORDER                          = "pluginorder"
-	INFRA_CONTAINER_YML                   = "docker-infra-container.yml"
-	INFRA_CONTAINER_NAME                  = "infra"
-	HOST_MODE                             = "host"
-	NONE_NETWORK_MODE                     = "none"
-	NAME                                  = "name"
-	NETWORK_DRIVER                        = "driver"
-	NETWORK_DEFAULT_DRIVER                = "bridge"
-	NETWORK_DEFAULT_NAME                  = "default"
-	NETWORK_EXTERNAL                      = "external"
-	INFRA_CONTAINER_GEN_YML               = "docker-infra-container.yml-generated.yml"
-	DEFAULT_FOLDER                        = "poddata"
-	NO_FOLDER                             = "dontcreatefolder"
-	RM_INFRA_CONTAINER                    = "rm_infra_container"
-	COMPOSE_HTTP_TIMEOUT                  = "COMPOSE_HTTP_TIMEOUT"
-	SERVICE_DETAIL                        = "serviceDetail"
-	MAX_DURATION            time.Duration = 1<<63 - 1
+	HEALTHY                 = "healthy"
+	UNHEALTHY               = "unhealthy"
+	STARTING                = "starting"
+	POD_STAGING             = "POD_STAGING"
+	POD_STARTING            = "POD_STARTING"
+	POD_RUNNING             = "POD_RUNNING"
+	POD_FAILED              = "POD_FAILED"
+	POD_KILLED              = "POD_KILLED"
+	POD_FINISHED            = "POD_FINISHED"
+	POD_PULL_FAILED         = "PULL_IMAGE_FAILED"
+	HEALTHCHECK             = "healthcheck"
+	CONTAINER_NAME          = "container_name"
+	NETWORK_MODE            = "network_mode"
+	LINKS                   = "links"
+	PORTS                   = "ports"
+	LABELS                  = "labels"
+	RESTART                 = "restart"
+	SERVICES                = "services"
+	IMAGE                   = "image"
+	VERSION                 = "version"
+	NETWORKS                = "networks"
+	TASK_ID                 = "taskId"
+	EXECUTOR_ID             = "executorId"
+	CGROUP_PARENT           = "cgroup_parent"
+	PLUGIN_ORDER            = "pluginorder"
+	INFRA_CONTAINER_YML     = "docker-infra-container.yml"
+	INFRA_CONTAINER_NAME    = "infra"
+	HOST_MODE               = "host"
+	NONE_NETWORK_MODE       = "none"
+	NAME                    = "name"
+	NETWORK_DRIVER          = "driver"
+	NETWORK_DEFAULT_DRIVER  = "bridge"
+	NETWORK_DEFAULT_NAME    = "default"
+	NETWORK_EXTERNAL        = "external"
+	INFRA_CONTAINER_GEN_YML = "docker-infra-container.yml-generated.yml"
+	DEFAULT_FOLDER          = "poddata"
+	NO_FOLDER               = "dontcreatefolder"
+	RM_INFRA_CONTAINER      = "rm_infra_container"
+	COMPOSE_HTTP_TIMEOUT    = "COMPOSE_HTTP_TIMEOUT"
+	SERVICE_DETAIL          = "serviceDetail"
+	FOREVER                 = 1<<63 - 1
 )
 
 type PodStatus struct {