Terraform 0.12 support

README.md

@@ -288,7 +288,7 @@ When Terragrunt finds the `terraform` block with a `source` parameter in `live/s
 
 1. Download the configurations specified via the `source` parameter into the `--terragrunt-download-dir` folder (by
    default `.terragrunt-cache` in the working directory, which we recommend adding to `.gitignore`). This downloading
-   is done by using the [terraform init command](https://www.terraform.io/docs/commands/init.html), so the `source`
+   is done by using the same [go-getter library](https://github.com/hashicorp/go-getter) Terraform uses, so the `source`
    parameter supports the exact same syntax as the [module source](https://www.terraform.io/docs/modules/sources.html)
    parameter, including local file paths, Git URLs, and Git URLs with `ref` parameters (useful for checking out a
    specific tag, commit, or branch of Git repo). Terragrunt will download all the code in the repo (i.e. the part
@@ -1741,13 +1741,12 @@ Hooks support the following arguments:
 * `execute` (required): the shell command to execute. 
 * `run_on_error` (optional): if set to true, this hook will run even if a previous hook hit an error, or in the case of
   "after" hooks, if the Terraform command hit an error. Default is false.
-* `init_from_module` vs `init`: This is not an argument, but a special name for a hook that runs on `terraform init`. 
-  Terragrunt uses `terraform init` in two different ways: one is to download
-  [remote configurations](#keep-your-terraform-code-dry) using `terraform init -from-module`; the other is to as part 
-  of [Auto-Init](#auto-init), which includes configuring the backend, retrieving provider plugins, and remote modules 
-  specified within the root module. If you wish to execute a hook when Terragrunt is using `terraform init` to download
-  remote configurations, name the hook `init_from_module`. If you wish to execute a hook when Terragrunt is using
-  `terraform init` for Auto-Init, name the hook `init`.   
+* `init_from_module` and `init`: This is not an argument, but a special name you can use for hooks that run during
+  initialization. There are two stages of initialization: one is to download 
+  [remote configurations](#keep-your-terraform-code-dry) using `go-getter`; the other is [Auto-Init](#auto-init), which 
+  configures the backend and downloads provider plugins and modules. If you wish to execute a hook when Terragrunt is 
+  using `go-getter` to download remote configurations, name the hook `init_from_module`. If you wish to execute a hook 
+  when Terragrunt is using `terraform init` for Auto-Init, name the hook `init`.   
 
 
 

cli/cli_app.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"os"
 	"regexp"
 	"strings"
 	"time"
@@ -401,11 +400,26 @@ func runTerragruntWithConfig(terragruntOptions *options.TerragruntOptions, terra
 		return err
 	}
 
+	return runActionWithHooks("terraform", terragruntOptions, terragruntConfig, func() error {
+		return runTerraformWithRetry(terragruntOptions)
+	})
+}
+
+// Run the given action function surrounded by hooks. That is, run the before hooks first, then, if there were no
+// errors, run the action, and finally, run the after hooks. Return any errors hit from the hooks or action.
+func runActionWithHooks(description string, terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig, action func() error) error {
 	beforeHookErrors := processHooks(terragruntConfig.Terraform.GetBeforeHooks(), terragruntOptions)
-	terraformError := runTerraformCommandIfNoErrors(beforeHookErrors, terragruntOptions)
-	postHookErrors := processHooks(terragruntConfig.Terraform.GetAfterHooks(), terragruntOptions, beforeHookErrors, terraformError)
 
-	return errors.NewMultiError(beforeHookErrors, terraformError, postHookErrors)
+	var actionErrors error
+	if beforeHookErrors == nil {
+		actionErrors = action()
+	} else {
+		terragruntOptions.Logger.Printf("Errors encountered running before_hooks. Not running '%s'.", description)
+	}
+
+	postHookErrors := processHooks(terragruntConfig.Terraform.GetAfterHooks(), terragruntOptions, beforeHookErrors, actionErrors)
+
+	return errors.NewMultiError(beforeHookErrors, actionErrors, postHookErrors)
 }
 
 // The Terragrunt configuration can contain a set of inputs to pass to Terraform as environment variables. This method
@@ -430,35 +444,6 @@ func setTerragruntInputsAsEnvVars(terragruntOptions *options.TerragruntOptions,
 	return nil
 }
 
-var moduleNotFoundErr = regexp.MustCompile(`Error loading modules: module .+?: not found, may need to run 'terraform init'`)
-
-func runTerraformCommandIfNoErrors(possibleErrors error, terragruntOptions *options.TerragruntOptions) error {
-	if possibleErrors != nil {
-		terragruntOptions.Logger.Println("Errors encountered running before_hooks. Not running terraform.")
-		return nil
-	}
-
-	// Workaround for https://github.com/hashicorp/terraform/issues/18460. Calling 'terraform init -get=false '
-	// sometimes results in Terraform trying to download/validate modules anyway, so we need to ignore that error.
-	if terragruntOptions.TerraformCommand == CMD_INIT_FROM_MODULE {
-		// Redirect all log output to stderr to make sure we don't pollute stdout with this extra call to 'init'
-		terragruntOptionsCopy := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath)
-		terragruntOptionsCopy.Writer = terragruntOptionsCopy.ErrWriter
-
-		out, err := shell.RunTerraformCommandWithOutput(terragruntOptionsCopy, terragruntOptionsCopy.TerraformCliArgs...)
-
-		// If we got an error and the error output included this error message, ignore the error and keep going
-		if err != nil && (len(moduleNotFoundErr.FindStringSubmatch(out.Stderr)) > 0 || strings.Contains(out.Stderr, "Missing required providers.")) {
-			terragruntOptions.Logger.Println("Ignoring error from call to init, as this is a known Terraform bug: https://github.com/hashicorp/terraform/issues/18460")
-			return nil
-		}
-
-		return err
-	}
-
-	return runTerraformWithRetry(terragruntOptions)
-}
-
 func runTerraformWithRetry(terragruntOptions *options.TerragruntOptions) error {
 	// Retry the command configurable time with sleep in between
 	for i := 0; i < terragruntOptions.MaxRetryAttempts; i++ {
@@ -477,20 +462,9 @@ func runTerraformWithRetry(terragruntOptions *options.TerragruntOptions) error {
 	return errors.WithStackTrace(MaxRetriesExceeded{terragruntOptions})
 }
 
-// Prepare for running 'terraform init' by
-// preventing users from passing source download arguments,
-// initializing remote state storage, and
-// adding backend configuration arguments to the TerraformCliArgs
+// Prepare for running 'terraform init' by initializing remote state storage and adding backend configuration arguments
+// to the TerraformCliArgs
 func prepareInitCommand(terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig, allowSourceDownload bool) error {
-
-	// Do not allow the user to specify the source or DIR arguments
-	// on the command line or as part of extra_arguments
-	//
-	// However, allow download_source.go to specify the source and DIR arguments.
-	if err := verifySourceDownloadArguments(allowSourceDownload, terragruntOptions); err != nil {
-		return err
-	}
-
 	if terragruntConfig.RemoteState != nil {
 		// Initialize the remote state if necessary  (e.g. create S3 bucket and DynamoDB table)
 		remoteStateNeedsInit, err := remoteStateNeedsInit(terragruntConfig.RemoteState, terragruntOptions)
@@ -629,69 +603,12 @@ func prepareInitOptions(terragruntOptions *options.TerragruntOptions, terraformS
 	initOptions.WorkingDir = terragruntOptions.WorkingDir
 	initOptions.TerraformCommand = CMD_INIT
 
-	// Don't pollute stdout with the stdout from Aoto Init
+	// Don't pollute stdout with the stdout from Auto Init
 	initOptions.Writer = initOptions.ErrWriter
 
-	// Only add the arguments to download source if terraformSource was specified
-	if terraformSource != nil {
-		initOptions.WorkingDir = terraformSource.WorkingDir
-		if util.FileExists(terraformSource.WorkingDir) {
-			if err := os.MkdirAll(terraformSource.WorkingDir, 0700); err != nil {
-				return nil, errors.WithStackTrace(err)
-			}
-		}
-
-		// terraform init -from-module will only work if the destination folder is empty, so we have to
-		if util.FileExists(terraformSource.DownloadDir) {
-			terragruntOptions.Logger.Printf("Download dir %s already exists, so deleting it before downloading into it.", terraformSource.DownloadDir)
-			if err := os.RemoveAll(terraformSource.DownloadDir); err != nil {
-				return nil, errors.WithStackTrace(err)
-			}
-			if err := os.MkdirAll(terraformSource.DownloadDir, 0700); err != nil {
-				return nil, errors.WithStackTrace(err)
-			}
-		}
-
-		// We will run init separately to download modules, plugins, backend state, etc, so don't run it at this point
-		initOptions.AppendTerraformCliArgs("-get=false")
-		initOptions.AppendTerraformCliArgs("-get-plugins=false")
-		initOptions.AppendTerraformCliArgs("-backend=false")
-
-		// Set the TerraformCommand attribute to match hooks on `init-from-module`
-		initOptions.TerraformCommand = CMD_INIT_FROM_MODULE
-
-		// Use the -from-module parameter to tell Terraform to download the module for us
-		initOptions.AppendTerraformCliArgs("-from-module="+terraformSource.CanonicalSourceURL.String(), "-no-color")
-
-		initOptions.AppendTerraformCliArgs(terraformSource.DownloadDir)
-	}
 	return initOptions, nil
 }
 
-// Returns an error if allowSourceDownload is false, and terragruntOptions.TerraformCliArgs contains source download related arguments
-func verifySourceDownloadArguments(allowSourceDownload bool, terragruntOptions *options.TerragruntOptions) error {
-	if allowSourceDownload || len(terragruntOptions.TerraformCliArgs) <= 1 {
-		return nil
-	}
-	for _, arg := range terragruntOptions.TerraformCliArgs[1:] {
-		// Enforce that the user did not specify -from-module
-		if strings.Contains(arg, "-from-module") {
-			return errors.WithStackTrace(ArgumentNotAllowed{
-				Argument: arg,
-				Message:  "Option not allowed: %s.  Terragrunt will handle setting -from-module automatically.",
-			})
-		}
-		// The user is not allowed to pass non-option arguments (such as DIR)
-		if !strings.HasPrefix(arg, "-") {
-			return errors.WithStackTrace(ArgumentNotAllowed{
-				Argument: arg,
-				Message:  "Argument not allowed: %s.  Terragrunt will handle setting the module source and DIR arguments automatically.",
-			})
-		}
-	}
-	return nil
-}
-
 // Returns true if the command the user wants to execute is supposed to affect multiple Terraform modules, such as the
 // apply-all or destroy-all command.
 func isMultiModuleCommand(command string) bool {

cli/download_source.go

@@ -89,8 +89,17 @@ func downloadTerraformSourceIfNecessary(terraformSource *TerraformSource, terrag
 		return err
 	}
 
-	if err := terraformInit(terraformSource, terragruntOptions, terragruntConfig); err != nil {
-		return err
+	// When downloading source, we need to process any hooks waiting on `init-from-module`. Therefore, we clone the
+	// options struct, set the command to the value the hooks are expecting, and run the download action surrounded by
+	// before and after hooks (if any).
+	terragruntOptionsForDownload := terragruntOptions.Clone(terragruntOptions.TerragruntConfigPath)
+	terragruntOptionsForDownload.TerraformCommand = CMD_INIT_FROM_MODULE
+	downloadErr := runActionWithHooks("download source", terragruntOptionsForDownload, terragruntConfig, func() error {
+		return downloadSource(terraformSource, terragruntOptions, terragruntConfig)
+	})
+
+	if downloadErr != nil {
+		return downloadErr
 	}
 
 	if err := writeVersionFile(terraformSource); err != nil {
@@ -356,9 +365,38 @@ func getTerraformSourceUrl(terragruntOptions *options.TerragruntOptions, terragr
 	}
 }
 
-// Download the code from the Canonical Source URL into the Download Folder using the terraform init command
-func terraformInit(terraformSource *TerraformSource, terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) error {
-	terragruntOptions.Logger.Printf("Downloading Terraform configurations from %s into %s using terraform init", terraformSource.CanonicalSourceURL, terraformSource.DownloadDir)
+// Download the code from the Canonical Source URL into the Download Folder using the go-getter library
+func downloadSource(terraformSource *TerraformSource, terragruntOptions *options.TerragruntOptions, terragruntConfig *config.TerragruntConfig) error {
+	terragruntOptions.Logger.Printf("Downloading Terraform configurations from %s into %s", terraformSource.CanonicalSourceURL, terraformSource.DownloadDir)
+
+	// go-getter will not download into folders that already exist, so initially, we download into a brand new temp
+	// folder
+	if err := os.MkdirAll(terragruntOptions.DownloadDir, 0700); err != nil {
+		return errors.WithStackTrace(err)
+	}
+	tmpDownloadDir, err := ioutil.TempDir(terragruntOptions.DownloadDir, "terragrunt-download-temp")
+	if err != nil {
+		return errors.WithStackTrace(err)
+	}
+	if err := os.RemoveAll(tmpDownloadDir); err != nil {
+		return errors.WithStackTrace(err)
+	}
+	if err := getter.GetAny(tmpDownloadDir, terraformSource.CanonicalSourceURL.String()); err != nil {
+		return errors.WithStackTrace(err)
+	}
+
+	// Now copy all the contents of the tmp folder into the original download dir
+	if err := os.MkdirAll(terraformSource.DownloadDir, 0700); err != nil {
+		return errors.WithStackTrace(err)
+	}
+	if err := util.CopyFolderContents(tmpDownloadDir, terraformSource.DownloadDir); err != nil {
+		return err
+	}
+
+	// Clean up the tmp folder
+	if err := os.RemoveAll(tmpDownloadDir); err != nil {
+		return errors.WithStackTrace(err)
+	}
 
-	return runTerraformInit(terragruntOptions, terragruntConfig, terraformSource)
+	return nil
 }

cli/download_source_test.go

@@ -232,7 +232,7 @@ func testDownloadTerraformSourceIfNecessary(t *testing.T, canonicalUrl string, d
 	assert.Nil(t, err, "For terraform source %v: %v", terraformSource, err)
 
 	err = downloadTerraformSourceIfNecessary(terraformSource, terragruntOptions, terragruntConfig)
-	assert.Nil(t, err, "For terraform source %v: %v", terraformSource, err)
+	require.NoError(t, err, "For terraform source %v: %v", terraformSource, err)
 
 	expectedFilePath := util.JoinPath(downloadDir, "main.tf")
 	if assert.True(t, util.FileExists(expectedFilePath), "For terraform source %v", terraformSource) {

config/config_test.go

@@ -17,8 +17,8 @@ func TestParseTerragruntConfigRemoteStateMinimalConfig(t *testing.T) {
 
 	config := `
 remote_state {
-	backend = "s3"
-	config = {}
+  backend = "s3"
+  config  = {}
 }
 `
 

configstack/module.go

@@ -417,8 +417,13 @@ func resolveExternalDependenciesForModule(module *TerraformModule, moduleMap map
 }
 
 // Confirm with the user whether they want Terragrunt to assume the given dependency of the given module is already
-// applied. If the user selects "no", then Terragrunt will apply that module as well.
+// applied. If the user selects "yes", then Terragrunt will apply that module as well.
 func confirmShouldApplyExternalDependency(module *TerraformModule, dependency *TerraformModule, terragruntOptions *options.TerragruntOptions) (bool, error) {
+	if terragruntOptions.NonInteractive {
+		terragruntOptions.Logger.Printf("The --non-interactive flag is set. To avoid accidentally affecting external dependencies with an xxx-all command, will not run this command against module %s, which is a dependency of module %s.", dependency.Path, module.Path)
+		return false, nil
+	}
+
 	prompt := fmt.Sprintf("Module %s depends on module %s, which is an external dependency outside of the current working directory. Should Terragrunt run this external dependency? Warning, if you say 'yes', Terragrunt will make changes in %s as well!", module.Path, dependency.Path, dependency.Path)
 	return shell.PromptUserForYesNo(prompt, terragruntOptions)
 }

configstack/module_test.go

@@ -382,7 +382,7 @@ func TestResolveTerraformModulesMultipleModulesWithExternalDependencies(t *testi
 		Dependencies:         []*TerraformModule{},
 		Config:               config.TerragruntConfig{},
 		TerragruntOptions:    mockOptions.Clone(canonical(t, "../test/fixture-modules/module-f/"+config.DefaultTerragruntConfigPath)),
-		AssumeAlreadyApplied: false,
+		AssumeAlreadyApplied: true,
 	}
 
 	moduleG := &TerraformModule{
@@ -411,7 +411,7 @@ func TestResolveTerraformModulesMultipleModulesWithNestedExternalDependencies(t
 		Dependencies:         []*TerraformModule{},
 		Config:               config.TerragruntConfig{},
 		TerragruntOptions:    mockOptions.Clone(canonical(t, "../test/fixture-modules/module-h/"+config.DefaultTerragruntConfigPath)),
-		AssumeAlreadyApplied: false,
+		AssumeAlreadyApplied: true,
 	}
 
 	moduleI := &TerraformModule{
@@ -421,7 +421,7 @@ func TestResolveTerraformModulesMultipleModulesWithNestedExternalDependencies(t
 			Dependencies: &config.ModuleDependencies{Paths: []string{"../module-h"}},
 		},
 		TerragruntOptions:    mockOptions.Clone(canonical(t, "../test/fixture-modules/module-i/"+config.DefaultTerragruntConfigPath)),
-		AssumeAlreadyApplied: false,
+		AssumeAlreadyApplied: true,
 	}
 
 	moduleJ := &TerraformModule{

remote/remote_state.go

@@ -94,7 +94,7 @@ func (remoteState *RemoteState) differsFrom(existingBackend *TerraformBackend, t
 		return true
 	}
 
-	if !reflect.DeepEqual(existingBackend.Config, remoteState.Config) {
+	if !terraformStateConfigEqual(existingBackend.Config, remoteState.Config) {
 		terragruntOptions.Logger.Printf("Backend config has changed from %s to %s", existingBackend.Config, remoteState.Config)
 		return true
 	}
@@ -103,6 +103,28 @@ func (remoteState *RemoteState) differsFrom(existingBackend *TerraformBackend, t
 	return false
 }
 
+// Return true if the existing config from a .tfstate file is equal to the new config from the user's backend
+// configuration. Under the hood, this method does a reflect.DeepEqual check, but with one twist: we strip out any
+// null values in the existing config. This is because Terraform >= 0.12 stores ALL possible keys for a given backend
+// in the .tfstate file, even if the user hasn't configured that key, in which case the value will be null, and cause
+// reflect.DeepEqual to fail.
+func terraformStateConfigEqual(existingConfig map[string]interface{}, newConfig map[string]interface{}) bool {
+	if existingConfig == nil {
+		return newConfig == nil
+	}
+
+	existingConfigNonNil := map[string]interface{}{}
+	for existingKey, existingValue := range existingConfig {
+		_, newValueIsSet := newConfig[existingKey]
+		if existingValue == nil && !newValueIsSet {
+			continue
+		}
+		existingConfigNonNil[existingKey] = existingValue
+	}
+
+	return reflect.DeepEqual(existingConfigNonNil, newConfig)
+}
+
 // Convert the RemoteState config into the format used by the terraform init command
 func (remoteState RemoteState) ToTerraformInitArgs() []string {
 

remote/remote_state_s3.go

@@ -176,7 +176,7 @@ func configValuesEqual(config map[string]interface{}, existingBackend *Terraform
 		delete(config, key)
 	}
 
-	if !reflect.DeepEqual(existingBackend.Config, config) {
+	if !terraformStateConfigEqual(existingBackend.Config, config) {
 		terragruntOptions.Logger.Printf("Backend config has changed from %s to %s", existingBackend.Config, config)
 		return false
 	}

remote/remote_state_s3_test.go

@@ -99,6 +99,12 @@ func TestConfigValuesEqual(t *testing.T) {
 			&TerraformBackend{Type: "s3", Config: map[string]interface{}{"something": "false"}},
 			false,
 		},
+		{
+			"equal-null-ignored",
+			map[string]interface{}{"something": "foo"},
+			&TerraformBackend{Type: "s3", Config: map[string]interface{}{"something": "foo", "ignored-because-null": nil}},
+			true,
+		},
 	}
 
 	for _, testCase := range testCases {