Feature #54 (#65)

* WIP: Adding code to generate new main.tf and use local backend. * WIP: Using local but logging still a mess. * WIP: Adding new TF to use binary instead of go module * WIP: Mostly done, just need automated testing now and some logging tweaks. * WIP: need automated tests; very nearly there now. * WIP: moved tf state file out of .cache * WIP: first unit tests and changed Fatal calls to returning error * Passing unit tests now. * Resolved issue for prompting * Fixed a bad bug that reversed file exist logic. * Fixed logging per PR suggestions and fixed nasty bug withe file exist logic reversed after code change. * WIP: Added config stuct for backend * Added changes per PR review * Resolved merge conflicts * WIP: adding abilily to supply init and apply args via CLI * WIP: put in post deploy hook and unit test to rewrite config file after sucessful completion. * Added test and fixed logic for writing back out DCE file. * WIP: added ability to pass arguments to underlying TF and to persist them in the config file. * Cleaned up unused interface * Removed unused interface * Updated changelog * Fixed order of loading vars per comments. * Added --save-options flag for saving tf options to config file, if specified * Updated save options, regenerated documentation, fixed broken test. * Added better prompter and tests for prompt * Added prompting at earlier point in process and tests to assert. * Fixed typo in prompt * Made note of potential breaking change for STDERR output * Made note of potential breaking change for STDERR output
Optum · Jan 27, 2020 · 3790782 · 3790782
1 parent a6c1740
commit 3790782
Show file tree

Hide file tree

Showing 33 changed files with 778 additions and 108 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,10 @@
+## vNext
+- Added `--tf-init-options` and `--tf-apply-options` for greater control over underlying provisioner
+- Added `--save-options` flag to persist the `--tf-init-options` and `--tf-apply-options` to the
+  configuration file when specified. Default behavior is false.
+- Persist API host name and path after running `system deploy`
+- **Potential breaking change**: explicitly configured logging output to go to STDOUT; command output should go to STDOUT
+
 ## v0.3.1
 - Moved quick start to readthedocs
 - Download deployment assets from the public url rather than using Github's GraphQL API

diff --git a/Makefile b/Makefile
@@ -22,7 +22,7 @@ test:
 	go test -count=1 -v ./...
 
 cover:
-	go test -coverprofile=coverage.out -coverpkg="./pkg/...,./internal/...,./cmd/..."  ./tests/...
+	go test -coverprofile=coverage.out -coverpkg="./pkg/...,./internal/...,./cmd/...,./configs/..."  ./tests/...
 
 test_functional:
 	go test -count=1 -v ./tests/functional/

diff --git a/cmd/root.go b/cmd/root.go
@@ -21,8 +21,6 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/mitchellh/go-homedir"
-
 	"github.com/Optum/dce-cli/configs"
 	"github.com/Optum/dce-cli/internal/constants"
 	observ "github.com/Optum/dce-cli/internal/observation"
@@ -43,19 +41,14 @@ var log observ.Logger
 var Log observ.Logger
 
 func init() {
-	homeDir, err := homedir.Dir()
-	if err != nil {
-		log.Fatalf("error: %v", err)
-	}
-
 	// Global Flags
 	// ---------------
 	// --config flag, to specify path to dce.yml config
-	// default to ~/.dce.yml
+	// default to ~/.dce/config.yaml
 	RootCmd.PersistentFlags().StringVar(
 		&cfgFile, "config",
-		filepath.Join(homeDir, ".dce", constants.DefaultConfigFileName),
-		"config file",
+		"",
+		"config file (default is \"$HOME/.dce/config.yaml\")",
 	)
 }
 
@@ -122,6 +115,14 @@ func onInit(cmd *cobra.Command, args []string) error {
 	log = Observation.Logger
 	Log = log
 
+	if len(cfgFile) == 0 {
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			log.Fatalf("error: %v", err)
+		}
+		cfgFile = filepath.Join(homeDir, ".dce", constants.DefaultConfigFileName)
+	}
+
 	fsUtil := &utl.FileSystemUtil{Config: Config, ConfigFile: cfgFile}
 
 	// Initialize config
@@ -132,7 +133,7 @@ func onInit(cmd *cobra.Command, args []string) error {
 			return errors.New("Config file not found. Please type 'dce init' to generate one.")
 		}
 	} else {
-		// Load config from dce.yaml
+		// Load config from the configuration file
 		err := fsUtil.ReadInConfig()
 		if err != nil {
 			return fmt.Errorf("Failed to parse dce.yml: %s", err)
@@ -151,6 +152,7 @@ func onInit(cmd *cobra.Command, args []string) error {
 // initialize anything related to logging, metrics, or tracing
 func initObservation() {
 	logrusInstance := logrus.New()
+	logrusInstance.SetOutput(os.Stderr)
 
 	//TODO: Make configurable
 	var logLevel logrus.Level

diff --git a/cmd/system.go b/cmd/system.go
@@ -15,15 +15,24 @@ var (
 	deployConfig    cfg.DeployConfig
 )
 
+const (
+	DCETFInitOptionsEnvVar  string = "DCE_TF_INIT_OPTIONS"
+	DCETFApplyOptionsEnvVar string = "DCE_TF_APPLY_OPTIONS"
+	Empty                   string = ""
+)
+
 func init() {
 	deployOverrides = svc.DeployOverrides{}
 	deployConfig = cfg.DeployConfig{}
 	systemDeployCmd.Flags().StringVarP(&deployConfig.DeployLocalPath, "local", "l", "", "Path to a local DCE repo to deploy.")
-	systemDeployCmd.Flags().BoolVarP(&deployConfig.UseCached, "use-cached", "c", true, "Overwrite local backend state.")
+	systemDeployCmd.Flags().BoolVarP(&deployConfig.UseCached, "use-cached", "c", true, "Uses locally-cached files, if available.")
 	systemDeployCmd.Flags().BoolVarP(&deployConfig.BatchMode, "batch-mode", "b", false, "Skip prompting for resource creation.")
+	systemDeployCmd.Flags().StringVar(&deployConfig.TFInitOptions, "tf-init-options", "", "Options to pass to the underlying \"tf init\" command.")
+	systemDeployCmd.Flags().StringVar(&deployConfig.TFApplyOptions, "tf-apply-options", "", "Options to pass to the underlying \"tf apply\" command.")
+	systemDeployCmd.Flags().BoolVar(&deployConfig.SaveTFOptions, "save-options", false, "If specified, saves the values provided by \"--tf-init-options\" and \"--tf-apply-options\" in the config file.")
 	systemDeployCmd.Flags().StringVarP(&deployOverrides.Namespace, "namespace", "n", "", "Set a custom terraform namespace (Optional)")
 	systemDeployCmd.Flags().StringVarP(&deployOverrides.AWSRegion, "region", "r", "", "The aws region that DCE will be deployed to (Default: us-east-1)")
-	systemDeployCmd.Flags().StringArrayVarP(&deployOverrides.GlobalTags, "tag", "t", []string{}, "Tags to be placed on all DCE resources. E.g. `dce system deploy --tag key1:value1 --tag key2:value2`")
+	systemDeployCmd.Flags().StringArrayVarP(&deployOverrides.GlobalTags, "tag", "t", []string{}, "Tags to be placed on all DCE resources. E.g. \"dce system deploy --tag key1:value1 --tag key2:value2\"")
 	systemDeployCmd.Flags().StringVar(&deployOverrides.BudgetNotificationFromEmail, "budget-notification-from-email", "", "Email address from which budget notifications will be sent")
 	systemDeployCmd.Flags().StringArrayVar(&deployOverrides.BudgetNotificationBCCEmails, "budget-notification-bcc-emails", []string{}, "Email address from which budget notifications will be sent")
 	systemDeployCmd.Flags().StringVar(&deployOverrides.BudgetNotificationTemplateHTML, "budget-notification-template-html", "", "HTML template for budget notification emails")
@@ -43,10 +52,36 @@ var systemCmd = &cobra.Command{
 var systemDeployCmd = &cobra.Command{
 	Use:   "deploy",
 	Short: "Deploy DCE to a new master account",
-	Run: func(cmd *cobra.Command, args []string) {
+	RunE: func(cmd *cobra.Command, args []string) error {
+		// Before running the command, resolve the configuration by using
+		// the default load order.
+		deployConfig.TFInitOptions = *cfg.Coalesce(&deployConfig.TFInitOptions, Config.Terraform.TFInitOptions, stringp(DCETFInitOptionsEnvVar), stringp(Empty))
+		deployConfig.TFApplyOptions = *cfg.Coalesce(&deployConfig.TFApplyOptions, Config.Terraform.TFApplyOptions, stringp(DCETFApplyOptionsEnvVar), stringp(Empty))
+
 		ctx := context.WithValue(context.Background(), constants.DeployConfig, &deployConfig)
 		if err := Service.Deploy(ctx, &deployOverrides); err != nil {
-			log.Fatalln(err)
+			cmd.SilenceUsage = true
+			return err
 		}
+		return nil
 	},
+	// If the command was successful, read the config file back in
+	// and if it's different than the running config, then save it.
+	// We don't want to do this unless the command was succesful,
+	// though, because of cases like bad tf opts we don't want to
+	// create an usuable state.
+	PostRunE: func(cmd *cobra.Command, args []string) error {
+		deployConfig.TFInitOptions = *cfg.Coalesce(&deployConfig.TFInitOptions, Config.Terraform.TFInitOptions, stringp(DCETFInitOptionsEnvVar), stringp(Empty))
+		deployConfig.TFApplyOptions = *cfg.Coalesce(&deployConfig.TFApplyOptions, Config.Terraform.TFApplyOptions, stringp(DCETFApplyOptionsEnvVar), stringp(Empty))
+
+		ctx := context.WithValue(context.Background(), constants.DeployConfig, &deployConfig)
+		if err := Service.PostDeploy(ctx); err != nil {
+			return err
+		}
+		return nil
+	},
+}
+
+func stringp(s string) *string {
+	return &s
 }
diff --git a/configs/config.go b/configs/config.go
@@ -1,5 +1,7 @@
 package configs
 
+import "os"
+
 // Root contains config
 type Root struct {
 	API       API
@@ -15,15 +17,56 @@ type API struct {
 	Token *string `yaml:"token,omitempty"`
 }
 
+// Terraform contains configuration for the underlying terraform
+// command used to provision the DCE infrastructure.
 type Terraform struct {
-	Bin    *string
-	Source *string
+	Bin            *string
+	Source         *string
+	TFInitOptions  *string `yaml:"initOptions,omitempty"`
+	TFApplyOptions *string `yaml:"applyOptions,omitempty"`
 }
 
+// DeployConfig holds configuration values for the `system deploy`
+// command
 type DeployConfig struct {
-	UseCached       bool
+	// UseCached, if true, tells DCE to use files already in the
+	// `~/.dce/.cache` folder
+	UseCached bool
+	// DeployLocalPath, if set, specifies a path from which to pull
+	// local resources
 	DeployLocalPath string
-	BatchMode       bool
+	// BatchMode, if enabled, forces DCE to run non-interactively
+	// and supplies Terraform with -auto-approve and input=false
+	BatchMode bool
+	// TFInitOptions are options passed along to `terraform init`
+	TFInitOptions string
+	// TFApplyOptions are options passed along to `terraform apply`
+	TFApplyOptions string
+	// SaveTFOptions, if yes, will save the provided terraform options to the config file.
+	SaveTFOptions bool
 }
 
 var Regions = []string{"us-east-1", "us-east-2", "us-west-1", "us-west-2"}
+
+// Coalesce returns the first non-empty vluae, but takes into account a loading order,
+// which is CLI args > environment variables > config file > some default
+func Coalesce(arg *string, config *string, envvar *string, def *string) *string {
+
+	if arg != nil && len(*arg) > 0 {
+		return arg
+	}
+
+	if envvar != nil {
+		envval, ok := os.LookupEnv(*envvar)
+
+		if ok && len(envval) > 0 {
+			return &envval
+		}
+	}
+
+	if config != nil && len(*config) > 0 {
+		return config
+	}
+
+	return def
+}
diff --git a/docs/dce.md b/docs/dce.md
@@ -14,7 +14,7 @@ Disposable Cloud Environment (DCE)
 ### Options
 
 ```
-      --config string   config file (default "$HOME/.dce.yaml")
+      --config string   config file (default is "$HOME/.dce/config.yaml")
   -h, --help            help for dce
 ```
 
@@ -27,4 +27,4 @@ Disposable Cloud Environment (DCE)
 * [dce system](dce_system.md)	 - Deploy and configure the DCE system
 * [dce usage](dce_usage.md)	 - View lease budget information
 
-###### Auto generated by spf13/cobra on 4-Dec-2019
+###### Auto generated by spf13/cobra on 21-Jan-2020
diff --git a/docs/dce_accounts.md b/docs/dce_accounts.md
@@ -15,7 +15,7 @@ Manage dce accounts
 ### Options inherited from parent commands
 
 ```
-      --config string   config file (default "$HOME/.dce.yaml")
+      --config string   config file (default is "$HOME/.dce/config.yaml")
 ```
 
 ### SEE ALSO
@@ -26,4 +26,4 @@ Manage dce accounts
 * [dce accounts list](dce_accounts_list.md)	 - list accounts
 * [dce accounts remove](dce_accounts_remove.md)	 - Remove an account from the accounts pool.
 
-###### Auto generated by spf13/cobra on 4-Dec-2019
+###### Auto generated by spf13/cobra on 21-Jan-2020
diff --git a/docs/dce_accounts_add.md b/docs/dce_accounts_add.md
@@ -21,11 +21,11 @@ dce accounts add [flags]
 ### Options inherited from parent commands
 
 ```
-      --config string   config file (default "$HOME/.dce.yaml")
+      --config string   config file (default is "$HOME/.dce/config.yaml")
 ```
 
 ### SEE ALSO
 
 * [dce accounts](dce_accounts.md)	 - Manage dce accounts
 
-###### Auto generated by spf13/cobra on 4-Dec-2019
+###### Auto generated by spf13/cobra on 21-Jan-2020
diff --git a/docs/dce_accounts_describe.md b/docs/dce_accounts_describe.md
@@ -19,11 +19,11 @@ dce accounts describe [Accound ID] [flags]
 ### Options inherited from parent commands
 
 ```
-      --config string   config file (default "$HOME/.dce.yaml")
+      --config string   config file (default is "$HOME/.dce/config.yaml")
 ```
 
 ### SEE ALSO
 
 * [dce accounts](dce_accounts.md)	 - Manage dce accounts
 
-###### Auto generated by spf13/cobra on 4-Dec-2019
+###### Auto generated by spf13/cobra on 21-Jan-2020
diff --git a/docs/dce_accounts_list.md b/docs/dce_accounts_list.md
@@ -19,11 +19,11 @@ dce accounts list [flags]
 ### Options inherited from parent commands
 
 ```
-      --config string   config file (default "$HOME/.dce.yaml")
+      --config string   config file (default is "$HOME/.dce/config.yaml")
 ```
 
 ### SEE ALSO
 
 * [dce accounts](dce_accounts.md)	 - Manage dce accounts
 
-###### Auto generated by spf13/cobra on 4-Dec-2019
+###### Auto generated by spf13/cobra on 21-Jan-2020
diff --git a/docs/dce_accounts_remove.md b/docs/dce_accounts_remove.md
@@ -19,11 +19,11 @@ dce accounts remove [Account ID] [flags]
 ### Options inherited from parent commands
 
 ```
-      --config string   config file (default "$HOME/.dce.yaml")
+      --config string   config file (default is "$HOME/.dce/config.yaml")
 ```
 
 ### SEE ALSO
 
 * [dce accounts](dce_accounts.md)	 - Manage dce accounts
 
-###### Auto generated by spf13/cobra on 4-Dec-2019
+###### Auto generated by spf13/cobra on 21-Jan-2020
diff --git a/docs/dce_auth.md b/docs/dce_auth.md
@@ -19,11 +19,11 @@ dce auth [flags]
 ### Options inherited from parent commands
 
 ```
-      --config string   config file (default "$HOME/.dce.yaml")
+      --config string   config file (default is "$HOME/.dce/config.yaml")
 ```
 
 ### SEE ALSO
 
 * [dce](dce.md)	 - Disposable Cloud Environment (DCE)
 
-###### Auto generated by spf13/cobra on 4-Dec-2019
+###### Auto generated by spf13/cobra on 21-Jan-2020
diff --git a/docs/dce_init.md b/docs/dce_init.md
@@ -19,11 +19,11 @@ dce init [flags]
 ### Options inherited from parent commands
 
 ```
-      --config string   config file (default "$HOME/.dce.yaml")
+      --config string   config file (default is "$HOME/.dce/config.yaml")
 ```
 
 ### SEE ALSO
 
 * [dce](dce.md)	 - Disposable Cloud Environment (DCE)
 
-###### Auto generated by spf13/cobra on 4-Dec-2019
+###### Auto generated by spf13/cobra on 21-Jan-2020
diff --git a/docs/dce_leases.md b/docs/dce_leases.md
@@ -15,7 +15,7 @@ Manage dce leases
 ### Options inherited from parent commands
 
 ```
-      --config string   config file (default "$HOME/.dce.yaml")
+      --config string   config file (default is "$HOME/.dce/config.yaml")
 ```
 
 ### SEE ALSO
@@ -27,4 +27,4 @@ Manage dce leases
 * [dce leases list](dce_leases_list.md)	 - List leases using various query filters.
 * [dce leases login](dce_leases_login.md)	 - Login to a leased DCE account. (Sets AWS CLI credentials if used with no flags)
 
-###### Auto generated by spf13/cobra on 4-Dec-2019
+###### Auto generated by spf13/cobra on 21-Jan-2020
diff --git a/docs/dce_leases_create.md b/docs/dce_leases_create.md
@@ -16,18 +16,19 @@ dce leases create [flags]
   -b, --budget-amount float      The leased accounts budget amount
   -c, --budget-currency string   The leased accounts budget currency (default "USD")
   -e, --email stringArray        The email address that budget notifications will be sent to
+  -E, --expires-on string        The leased accounts expiry date as a long (UNIX epoch) or string (eg., '7d', '8h' (default "7d")
   -h, --help                     help for create
   -p, --principal-id string      Principle ID for the user of the leased account
 ```
 
 ### Options inherited from parent commands
 
 ```
-      --config string   config file (default "$HOME/.dce.yaml")
+      --config string   config file (default is "$HOME/.dce/config.yaml")
 ```
 
 ### SEE ALSO
 
 * [dce leases](dce_leases.md)	 - Manage dce leases
 
-###### Auto generated by spf13/cobra on 4-Dec-2019
+###### Auto generated by spf13/cobra on 21-Jan-2020
diff --git a/docs/dce_leases_describe.md b/docs/dce_leases_describe.md
@@ -19,11 +19,11 @@ dce leases describe [Lease ID] [flags]
 ### Options inherited from parent commands
 
 ```
-      --config string   config file (default "$HOME/.dce.yaml")
+      --config string   config file (default is "$HOME/.dce/config.yaml")
 ```
 
 ### SEE ALSO
 
 * [dce leases](dce_leases.md)	 - Manage dce leases
 
-###### Auto generated by spf13/cobra on 4-Dec-2019
+###### Auto generated by spf13/cobra on 21-Jan-2020
diff --git a/docs/dce_leases_end.md b/docs/dce_leases_end.md
@@ -21,11 +21,11 @@ dce leases end [flags]
 ### Options inherited from parent commands
 
 ```
-      --config string   config file (default "$HOME/.dce.yaml")
+      --config string   config file (default is "$HOME/.dce/config.yaml")
 ```
 
 ### SEE ALSO
 
 * [dce leases](dce_leases.md)	 - Manage dce leases
 
-###### Auto generated by spf13/cobra on 4-Dec-2019
+###### Auto generated by spf13/cobra on 21-Jan-2020