From 58bd19431cbe527e1e53448b3b0746cad7b34401 Mon Sep 17 00:00:00 2001 From: daidokoro Date: Sun, 23 Jul 2017 01:19:43 +0100 Subject: [PATCH 1/3] added tail service mechanism to reduce api calls --- stacks/services.go | 88 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 stacks/services.go diff --git a/stacks/services.go b/stacks/services.go new file mode 100644 index 0000000..b11d8fa --- /dev/null +++ b/stacks/services.go @@ -0,0 +1,88 @@ +package stacks + +import ( + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +var tail chan *TailServiceInput + +// TailServiceInput used for tailing cloudfomation outputs +type TailServiceInput struct { + stk Stack + command string + printed map[string]interface{} +} + +// TailService - handles all tailing events +func TailService(tail <-chan *TailServiceInput) { + Log.Debug("Tail.Service started") + for { + select { + case input := <-tail: + svc := cloudformation.New( + input.stk.Session, + &aws.Config{Credentials: input.stk.creds()}, + ) + + params := &cloudformation.DescribeStackEventsInput{ + StackName: aws.String(input.stk.Stackname), + } + + // If channel is not populated, run verbose cf print + Log.Debug(fmt.Sprintf("Calling [DescribeStackEvents] with parameters: %s", params)) + stackevents, err := svc.DescribeStackEvents(params) + if err != nil { + Log.Debug(fmt.Sprintln("Error when tailing events: ", err.Error())) + continue + } + + Log.Debug(fmt.Sprintln("Response:", stackevents)) + + event := stackevents.StackEvents[0] + + statusReason := "" + var lg = Log.Info + if strings.Contains(*event.ResourceStatus, "FAILED") { + statusReason = *event.ResourceStatusReason + lg = Log.Error + } + + line := strings.Join([]string{ + *event.StackName, + Log.ColorMap(*event.ResourceStatus), + *event.ResourceType, + *event.LogicalResourceId, + statusReason, + }, " - ") + + if _, ok := input.printed[line]; !ok { + evt := strings.Split(*event.ResourceStatus, "_")[0] + if evt == input.command || input.command == "" || strings.Contains(strings.ToLower(evt), "rollback") { + lg(strings.Trim(line, "- ")) + } + + input.printed[line] = nil + } + + default: + // TODO + } + } +} + +// populates tail channel and returns when done +func tailWait(done <-chan bool, tailinput *TailServiceInput) { + for ch := time.Tick(time.Millisecond * 1300); ; <-ch { + select { + case <-done: + return + default: + tail <- tailinput + } + } +} From c37e6aa5ac20587d74cb3b09a7bed78ad96bd06c Mon Sep 17 00:00:00 2001 From: daidokoro Date: Sun, 23 Jul 2017 01:23:51 +0100 Subject: [PATCH 2/3] added support for service-model to deploy/terminate commands --- stacks/deploy.go | 23 ++++++++++++++++++----- stacks/terminate.go | 38 ++++++++++++++------------------------ 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/stacks/deploy.go b/stacks/deploy.go index 951e6cb..95753b4 100644 --- a/stacks/deploy.go +++ b/stacks/deploy.go @@ -65,7 +65,8 @@ func (s *Stack) Deploy() error { // If bucket - upload to s3 if s.Bucket != "" { - url, err := resolveBucket(s) + var url string + url, err = resolveBucket(s) if err != nil { return err } @@ -75,18 +76,30 @@ func (s *Stack) Deploy() error { } Log.Debug(fmt.Sprintln("Calling [CreateStack] with parameters:", createParams)) - if _, err := svc.CreateStack(createParams); err != nil { + if _, err = svc.CreateStack(createParams); err != nil { return errors.New(fmt.Sprintln("Deploying failed: ", err.Error())) } - go s.tail("CREATE", done) - if err := Wait(s.StackStatus); err != nil { + // go s.tail("CREATE", done) + var tailinput = TailServiceInput{ + printed: make(map[string]interface{}), + stk: *s, + command: "CREATE", + } + + go tailWait(done, &tailinput) + + err = svc.WaitUntilStackCreateComplete(&cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + }) + + if err != nil { return err } done <- true - Log.Info(fmt.Sprintf("deployment successful: [%s]", s.Stackname)) + Log.Info(fmt.Sprintf("deployment completed: [%s]", s.Stackname)) return nil } diff --git a/stacks/terminate.go b/stacks/terminate.go index 6335b97..7ae5e19 100644 --- a/stacks/terminate.go +++ b/stacks/terminate.go @@ -3,7 +3,6 @@ package stacks import ( "errors" "fmt" - "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudformation" @@ -23,37 +22,28 @@ func (s *Stack) terminate() error { StackName: aws.String(s.Stackname), } - Log.Debug(fmt.Sprintln("Calling [DeleteStack] with parameters:", params)) - _, err := svc.DeleteStack(params) + // create wait handler for tail + var tailinput = TailServiceInput{ + printed: make(map[string]interface{}), + stk: *s, + command: "DELETE", + } - go s.tail("DELETE", done) + go tailWait(done, &tailinput) - if err != nil { + Log.Debug(fmt.Sprintln("Calling [DeleteStack] with parameters:", params)) + if _, err := svc.DeleteStack(params); err != nil { done <- true return errors.New(fmt.Sprintln("Deleting failed: ", err)) } - // describeStacksInput := &cloudformation.DescribeStacksInput{ - // StackName: aws.String(s.Stackname), - // } - // - // Log(fmt.Sprintln("Calling [WaitUntilStackDeleteComplete] with parameters:", describeStacksInput), level.debug) - // - // if err := svc.WaitUntilStackDeleteComplete(describeStacksInput); err != nil { - // return err - // } - - // NOTE: The [WaitUntilStackDeleteComplete] api call suddenly stopped playing nice. - // Implemented this crude loop as a patch fix for now - for { - if !s.StackExists() { - done <- true - break - } - - time.Sleep(time.Second * 1) + if err := svc.WaitUntilStackDeleteComplete(&cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + }); err != nil { + return err } + done <- true Log.Info(fmt.Sprintf("Deletion successful: [%s]", s.Stackname)) return nil From f87ef9e57c204cdac726a199ef88f6f7a90b3e01 Mon Sep 17 00:00:00 2001 From: daidokoro Date: Sun, 23 Jul 2017 01:24:19 +0100 Subject: [PATCH 3/3] added support for service tail model to deploy/terminate handlers --- stacks/clouformation.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/stacks/clouformation.go b/stacks/clouformation.go index 5b8d5bd..b6ac231 100644 --- a/stacks/clouformation.go +++ b/stacks/clouformation.go @@ -2,10 +2,11 @@ package stacks import ( "fmt" - "github.com/daidokoro/qaz/utils" "sync" "time" + "github.com/daidokoro/qaz/utils" + "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudformation" ) @@ -59,6 +60,10 @@ func DeployHandler(runstacks map[string]string, stacks map[string]*Stack) { // status - pending, failed, completed var status = make(map[string]string) + // kick off tail mechanism + tail = make(chan *TailServiceInput) + go TailService(tail) + for _, stk := range stacks { if _, ok := runstacks[stk.Name]; !ok && len(runstacks) > 0 { @@ -161,6 +166,10 @@ func DeployHandler(runstacks map[string]string, stacks map[string]*Stack) { // TerminateHandler - Handles terminating stacks in the correct order func TerminateHandler(runstacks map[string]string, stacks map[string]*Stack) { + // kick off tail mechanism + tail = make(chan *TailServiceInput) + go TailService(tail) + for _, stk := range stacks { if _, ok := runstacks[stk.Name]; !ok && len(runstacks) > 0 { Log.Debug(fmt.Sprintf("%s: not in run.stacks, skipping", stk.Name))