diff --git a/stacks/clouformation.go b/stacks/clouformation.go index 5b8d5bd..b6ac231 100644 --- a/stacks/clouformation.go +++ b/stacks/clouformation.go @@ -2,10 +2,11 @@ package stacks import ( "fmt" - "github.com/daidokoro/qaz/utils" "sync" "time" + "github.com/daidokoro/qaz/utils" + "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" ) @@ -59,6 +60,10 @@ func DeployHandler(runstacks map[string]string, stacks map[string]*Stack) { // status - pending, failed, completed var status = make(map[string]string) + // kick off tail mechanism + tail = make(chan *TailServiceInput) + go TailService(tail) + for _, stk := range stacks { if _, ok := runstacks[stk.Name]; !ok && len(runstacks) > 0 { @@ -161,6 +166,10 @@ func DeployHandler(runstacks map[string]string, stacks map[string]*Stack) { // TerminateHandler - Handles terminating stacks in the correct order func TerminateHandler(runstacks map[string]string, stacks map[string]*Stack) { + // kick off tail mechanism + tail = make(chan *TailServiceInput) + go TailService(tail) + for _, stk := range stacks { if _, ok := runstacks[stk.Name]; !ok && len(runstacks) > 0 { Log.Debug(fmt.Sprintf("%s: not in run.stacks, skipping", stk.Name)) diff --git a/stacks/deploy.go b/stacks/deploy.go index 951e6cb..95753b4 100644 --- a/stacks/deploy.go +++ b/stacks/deploy.go @@ -65,7 +65,8 @@ func (s *Stack) Deploy() error { // If bucket - upload to s3 if s.Bucket != "" { - url, err := resolveBucket(s) + var url string + url, err = resolveBucket(s) if err != nil { return err } @@ -75,18 +76,30 @@ func (s *Stack) Deploy() error { } Log.Debug(fmt.Sprintln("Calling [CreateStack] with parameters:", createParams)) - if _, err := svc.CreateStack(createParams); err != nil { + if _, err = svc.CreateStack(createParams); err != nil { return errors.New(fmt.Sprintln("Deploying failed: ", err.Error())) } - go s.tail("CREATE", done) - if err := Wait(s.StackStatus); err != nil { + // go s.tail("CREATE", done) + var tailinput = TailServiceInput{ + printed: make(map[string]interface{}), + stk: *s, + command: "CREATE", + } + + go tailWait(done, &tailinput) + + err = svc.WaitUntilStackCreateComplete(&cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + }) + + if err != nil { return err } done <- true - Log.Info(fmt.Sprintf("deployment successful: [%s]", s.Stackname)) + Log.Info(fmt.Sprintf("deployment completed: [%s]", s.Stackname)) return nil } diff --git a/stacks/services.go b/stacks/services.go new file mode 100644 index 0000000..b11d8fa --- /dev/null +++ b/stacks/services.go @@ -0,0 +1,88 @@ +package stacks + +import ( + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +var tail chan *TailServiceInput + +// TailServiceInput used for tailing cloudfomation outputs +type TailServiceInput struct { + stk Stack + command string + printed map[string]interface{} +} + +// TailService - handles all tailing events +func TailService(tail <-chan *TailServiceInput) { + Log.Debug("Tail.Service started") + for { + select { + case input := <-tail: + svc := cloudformation.New( + input.stk.Session, + &aws.Config{Credentials: input.stk.creds()}, + ) + + params := &cloudformation.DescribeStackEventsInput{ + StackName: aws.String(input.stk.Stackname), + } + + // If channel is not populated, run verbose cf print + Log.Debug(fmt.Sprintf("Calling [DescribeStackEvents] with parameters: %s", params)) + stackevents, err := svc.DescribeStackEvents(params) + if err != nil { + Log.Debug(fmt.Sprintln("Error when tailing events: ", err.Error())) + continue + } + + Log.Debug(fmt.Sprintln("Response:", stackevents)) + + event := stackevents.StackEvents[0] + + statusReason := "" + var lg = Log.Info + if strings.Contains(*event.ResourceStatus, "FAILED") { + statusReason = *event.ResourceStatusReason + lg = Log.Error + } + + line := strings.Join([]string{ + *event.StackName, + Log.ColorMap(*event.ResourceStatus), + *event.ResourceType, + *event.LogicalResourceId, + statusReason, + }, " - ") + + if _, ok := input.printed[line]; !ok { + evt := strings.Split(*event.ResourceStatus, "_")[0] + if evt == input.command || input.command == "" || strings.Contains(strings.ToLower(evt), "rollback") { + lg(strings.Trim(line, "- ")) + } + + input.printed[line] = nil + } + + default: + // TODO + } + } +} + +// populates tail channel and returns when done +func tailWait(done <-chan bool, tailinput *TailServiceInput) { + for ch := time.Tick(time.Millisecond * 1300); ; <-ch { + select { + case <-done: + return + default: + tail <- tailinput + } + } +} diff --git a/stacks/terminate.go b/stacks/terminate.go index 6335b97..7ae5e19 100644 --- a/stacks/terminate.go +++ b/stacks/terminate.go @@ -3,7 +3,6 @@ package stacks import ( "errors" "fmt" - "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudformation" @@ -23,37 +22,28 @@ func (s *Stack) terminate() error { StackName: aws.String(s.Stackname), } - Log.Debug(fmt.Sprintln("Calling [DeleteStack] with parameters:", params)) - _, err := svc.DeleteStack(params) + // create wait handler for tail + var tailinput = TailServiceInput{ + printed: make(map[string]interface{}), + stk: *s, + command: "DELETE", + } - go s.tail("DELETE", done) + go tailWait(done, &tailinput) - if err != nil { + Log.Debug(fmt.Sprintln("Calling [DeleteStack] with parameters:", params)) + if _, err := svc.DeleteStack(params); err != nil { done <- true return errors.New(fmt.Sprintln("Deleting failed: ", err)) } - // describeStacksInput := &cloudformation.DescribeStacksInput{ - // StackName: aws.String(s.Stackname), - // } - // - // Log(fmt.Sprintln("Calling [WaitUntilStackDeleteComplete] with parameters:", describeStacksInput), level.debug) - // - // if err := svc.WaitUntilStackDeleteComplete(describeStacksInput); err != nil { - // return err - // } - - // NOTE: The [WaitUntilStackDeleteComplete] api call suddenly stopped playing nice. - // Implemented this crude loop as a patch fix for now - for { - if !s.StackExists() { - done <- true - break - } - - time.Sleep(time.Second * 1) + if err := svc.WaitUntilStackDeleteComplete(&cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + }); err != nil { + return err } + done <- true Log.Info(fmt.Sprintf("Deletion successful: [%s]", s.Stackname)) return nil