From faa027acbd3c2d9d9973333625a1d79e0592afcb Mon Sep 17 00:00:00 2001 From: Adam Verigin <116602331+adam-verigin@users.noreply.github.com> Date: Thu, 22 Dec 2022 11:37:35 -0800 Subject: [PATCH] feat: Add flag to disable downloading tf for airgapped environments (#2843) * Move terraform version listing into Terraform client * Add flag to disable Terraform downloads * Fallback to exact version matching if unable to list TF versions * Add new config option to documentation * Fix and update tests * Fix check-lint errors * Change option name * Remove obsolete commented code * Migrate version detection logic into terraform client * Improve consistency in new function signatures & log statements * Test requests before calling terraform-switch, to prevent unnecessary crashes * Fix broken tests * Silence gosec error * Close response body --- cmd/server.go | 6 + runatlantis.io/docs/server-configuration.md | 11 + .../events/events_controller_e2e_test.go | 3 +- .../terraform/mocks/mock_terraform_client.go | 92 ++++++++ server/core/terraform/terraform_client.go | 126 ++++++++++- .../core/terraform/terraform_client_test.go | 196 +++++++++++++++++- server/events/project_command_builder.go | 10 + .../project_command_builder_internal_test.go | 11 +- server/events/project_command_builder_test.go | 136 ++++++------ .../events/project_command_context_builder.go | 71 +------ .../project_command_context_builder_test.go | 10 +- server/server.go | 2 + server/user_config.go | 1 + 13 files changed, 521 insertions(+), 154 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 506b537608..ad07254e49 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -117,6 +117,7 @@ const ( SSLCertFileFlag = "ssl-cert-file" SSLKeyFileFlag = "ssl-key-file" RestrictFileList = "restrict-file-list" + TFDownloadFlag = "tf-download" TFDownloadURLFlag = "tf-download-url" VarFileAllowlistFlag = "var-file-allowlist" VCSStatusName = "vcs-status-name" @@ -151,6 +152,7 @@ const ( DefaultRedisTLSEnabled = false DefaultRedisInsecureSkipVerify = false DefaultTFDownloadURL = "https://releases.hashicorp.com" + DefaultTFDownload = true DefaultTFEHostname = "app.terraform.io" DefaultVCSStatusName = "atlantis" DefaultWebBasicAuth = false @@ -500,6 +502,10 @@ var boolFlags = map[string]boolFlag{ description: "Skips cloning the PR repo if there are no projects were changed in the PR.", defaultValue: false, }, + TFDownloadFlag: { + description: "Allow Atlantis to list & download Terraform versions. Setting this to false can be helpful in air-gapped environments.", + defaultValue: DefaultTFDownload, + }, TFELocalExecutionModeFlag: { description: "Enable if you're using local execution mode (instead of TFE/C's remote execution mode).", defaultValue: false, diff --git a/runatlantis.io/docs/server-configuration.md b/runatlantis.io/docs/server-configuration.md index 2af2e8b188..8e2a1a97e5 100644 --- a/runatlantis.io/docs/server-configuration.md +++ b/runatlantis.io/docs/server-configuration.md @@ -917,6 +917,15 @@ and set `--autoplan-modules` to `false`. ``` Namespace for emitting stats/metrics. See [stats](stats.html) section. +### `--tf--download` + ```bash + atlantis server --tf-download=false + # or + ATLANTIS_TF_DOWNLOAD=false + ``` +Defaults to `true`. Allow Atlantis to list and download additional versions of Terraform. +Setting this to `false` can be useful in an air-gapped environment where a download mirror is not available. + ### `--tf-download-url` ```bash atlantis server --tf-download-url="https://releases.company.com" @@ -926,6 +935,8 @@ and set `--autoplan-modules` to `false`. An alternative URL to download Terraform versions if they are missing. Useful in an airgapped environment where releases.hashicorp.com is not available. Directory structure of the custom endpoint should match that of releases.hashicorp.com. + + This has no impact if `--tf-download` is set to `false`. ### `--tfe-hostname` ```bash diff --git a/server/controllers/events/events_controller_e2e_test.go b/server/controllers/events/events_controller_e2e_test.go index 7e56c2b9f2..717286af88 100644 --- a/server/controllers/events/events_controller_e2e_test.go +++ b/server/controllers/events/events_controller_e2e_test.go @@ -909,7 +909,7 @@ func setupE2E(t *testing.T, repoDir, repoConfigFile string) (events_controllers. GitlabUser: "gitlab-user", ExecutableName: "atlantis", } - terraformClient, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "", "default-tf-version", "https://releases.hashicorp.com", &NoopTFDownloader{}, false, projectCmdOutputHandler) + terraformClient, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "", "default-tf-version", "https://releases.hashicorp.com", &NoopTFDownloader{}, true, false, projectCmdOutputHandler) Ok(t, err) boltdb, err := db.New(dataDir) Ok(t, err) @@ -1005,6 +1005,7 @@ func setupE2E(t *testing.T, repoDir, repoConfigFile string) (events_controllers. false, statsScope, logger, + terraformClient, ) showStepRunner, err := runtime.NewShowStepRunner(terraformClient, defaultTFVersion) diff --git a/server/core/terraform/mocks/mock_terraform_client.go b/server/core/terraform/mocks/mock_terraform_client.go index 746f39a0c7..2f0488f42d 100644 --- a/server/core/terraform/mocks/mock_terraform_client.go +++ b/server/core/terraform/mocks/mock_terraform_client.go @@ -61,6 +61,40 @@ func (mock *MockClient) EnsureVersion(log logging.SimpleLogging, v *go_version.V return ret0 } +func (mock *MockClient) ListAvailableVersions(log logging.SimpleLogging) ([]string, error) { + if mock == nil { + panic("mock must not be nil. Use myMock := NewMockClient().") + } + params := []pegomock.Param{log} + result := pegomock.GetGenericMockFrom(mock).Invoke("ListAvailableVersions", params, []reflect.Type{reflect.TypeOf((*[]string)(nil)).Elem(), reflect.TypeOf((*error)(nil)).Elem()}) + var ret0 []string + var ret1 error + if len(result) != 0 { + if result[0] != nil { + ret0 = result[0].([]string) + } + if result[1] != nil { + ret1 = result[1].(error) + } + } + return ret0, ret1 +} + +func (mock *MockClient) DetectVersion(log logging.SimpleLogging, projectDirectory string) *go_version.Version { + if mock == nil { + panic("mock must not be nil. Use myMock := NewMockClient().") + } + params := []pegomock.Param{log, projectDirectory} + result := pegomock.GetGenericMockFrom(mock).Invoke("DetectVersion", params, []reflect.Type{reflect.TypeOf((**go_version.Version)(nil)).Elem()}) + var ret0 *go_version.Version + if len(result) != 0 { + if result[0] != nil { + ret0 = result[0].(*go_version.Version) + } + } + return ret0 +} + func (mock *MockClient) VerifyWasCalledOnce() *VerifierMockClient { return &VerifierMockClient{ mock: mock, @@ -175,3 +209,61 @@ func (c *MockClient_EnsureVersion_OngoingVerification) GetAllCapturedArguments() } return } + +func (verifier *VerifierMockClient) ListAvailableVersions(log logging.SimpleLogging) *MockClient_ListAvailableVersions_OngoingVerification { + params := []pegomock.Param{log} + methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "ListAvailableVersions", params, verifier.timeout) + return &MockClient_ListAvailableVersions_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} +} + +type MockClient_ListAvailableVersions_OngoingVerification struct { + mock *MockClient + methodInvocations []pegomock.MethodInvocation +} + +func (c *MockClient_ListAvailableVersions_OngoingVerification) GetCapturedArguments() logging.SimpleLogging { + log := c.GetAllCapturedArguments() + return log[len(log)-1] +} + +func (c *MockClient_ListAvailableVersions_OngoingVerification) GetAllCapturedArguments() (_param0 []logging.SimpleLogging) { + params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) + if len(params) > 0 { + _param0 = make([]logging.SimpleLogging, len(c.methodInvocations)) + for u, param := range params[0] { + _param0[u] = param.(logging.SimpleLogging) + } + } + return +} + +func (verifier *VerifierMockClient) DetectVersion(log logging.SimpleLogging, projectDirectory string) *MockClient_DetectVersion_OngoingVerification { + params := []pegomock.Param{log, projectDirectory} + methodInvocations := pegomock.GetGenericMockFrom(verifier.mock).Verify(verifier.inOrderContext, verifier.invocationCountMatcher, "DetectVersion", params, verifier.timeout) + return &MockClient_DetectVersion_OngoingVerification{mock: verifier.mock, methodInvocations: methodInvocations} +} + +type MockClient_DetectVersion_OngoingVerification struct { + mock *MockClient + methodInvocations []pegomock.MethodInvocation +} + +func (c *MockClient_DetectVersion_OngoingVerification) GetCapturedArguments() (logging.SimpleLogging, string) { + log, projectDirectory := c.GetAllCapturedArguments() + return log[len(log)-1], projectDirectory[len(projectDirectory)-1] +} + +func (c *MockClient_DetectVersion_OngoingVerification) GetAllCapturedArguments() (_param0 []logging.SimpleLogging, _param1 []string) { + params := pegomock.GetGenericMockFrom(c.mock).GetInvocationParams(c.methodInvocations) + if len(params) > 0 { + _param0 = make([]logging.SimpleLogging, len(c.methodInvocations)) + for u, param := range params[0] { + _param0[u] = param.(logging.SimpleLogging) + } + _param1 = make([]string, len(c.methodInvocations)) + for u, param := range params[1] { + _param1[u] = param.(string) + } + } + return +} diff --git a/server/core/terraform/terraform_client.go b/server/core/terraform/terraform_client.go index 3260b2d6d4..199fc245b3 100644 --- a/server/core/terraform/terraform_client.go +++ b/server/core/terraform/terraform_client.go @@ -18,18 +18,23 @@ package terraform import ( "fmt" + "net/http" "os" "os/exec" "path/filepath" "regexp" "runtime" + "sort" "strings" "sync" + "github.com/Masterminds/semver" "github.com/hashicorp/go-getter" "github.com/hashicorp/go-version" + "github.com/hashicorp/terraform-config-inspect/tfconfig" "github.com/mitchellh/go-homedir" "github.com/pkg/errors" + "github.com/warrensbox/terraform-switcher/lib" "github.com/runatlantis/atlantis/server/core/runtime/models" "github.com/runatlantis/atlantis/server/events/command" @@ -53,6 +58,12 @@ type Client interface { // EnsureVersion makes sure that terraform version `v` is available to use EnsureVersion(log logging.SimpleLogging, v *version.Version) error + + // ListAvailableVersions returns all available version of Terraform, if available; otherwise this will return an empty list. + ListAvailableVersions(log logging.SimpleLogging) ([]string, error) + + // DetectVersion Extracts required_version from Terraform configuration in the specified project directory. Returns nil if unable to determine the version. + DetectVersion(log logging.SimpleLogging, projectDirectory string) *version.Version } type DefaultClient struct { @@ -69,6 +80,7 @@ type DefaultClient struct { // downloader downloads terraform versions. downloader Downloader downloadBaseURL string + downloadAllowed bool // versions maps from the string representation of a tf version (ex. 0.11.10) // to the absolute path of that binary on disk (if it exists). // Use versionsLock to control access. @@ -111,6 +123,7 @@ func NewClientWithDefaultVersion( defaultVersionFlagName string, tfDownloadURL string, tfDownloader Downloader, + tfDownloadAllowed bool, usePluginCache bool, fetchAsync bool, projectCmdOutputHandler jobs.ProjectCommandOutputHandler, @@ -147,7 +160,7 @@ func NewClientWithDefaultVersion( // Since ensureVersion might end up downloading terraform, // we call it asynchronously so as to not delay server startup. versionsLock.Lock() - _, err := ensureVersion(log, tfDownloader, versions, defaultVersion, binDir, tfDownloadURL) + _, err := ensureVersion(log, tfDownloader, versions, defaultVersion, binDir, tfDownloadURL, tfDownloadAllowed) versionsLock.Unlock() if err != nil { log.Err("could not download terraform %s: %s", defaultVersion.String(), err) @@ -177,6 +190,7 @@ func NewClientWithDefaultVersion( binDir: binDir, downloader: tfDownloader, downloadBaseURL: tfDownloadURL, + downloadAllowed: tfDownloadAllowed, versionsLock: &versionsLock, versions: versions, usePluginCache: usePluginCache, @@ -195,6 +209,7 @@ func NewTestClient( defaultVersionFlagName string, tfDownloadURL string, tfDownloader Downloader, + tfDownloadAllowed bool, usePluginCache bool, projectCmdOutputHandler jobs.ProjectCommandOutputHandler, ) (*DefaultClient, error) { @@ -208,6 +223,7 @@ func NewTestClient( defaultVersionFlagName, tfDownloadURL, tfDownloader, + tfDownloadAllowed, usePluginCache, false, projectCmdOutputHandler, @@ -232,6 +248,7 @@ func NewClient( defaultVersionFlagName string, tfDownloadURL string, tfDownloader Downloader, + tfDownloadAllowed bool, usePluginCache bool, projectCmdOutputHandler jobs.ProjectCommandOutputHandler, ) (*DefaultClient, error) { @@ -245,6 +262,7 @@ func NewClient( defaultVersionFlagName, tfDownloadURL, tfDownloader, + tfDownloadAllowed, usePluginCache, true, projectCmdOutputHandler, @@ -262,6 +280,96 @@ func (c *DefaultClient) TerraformBinDir() string { return c.binDir } +// ListAvailableVersions returns all available version of Terraform. If downloads are not allowed, this will return an empty list. +func (c *DefaultClient) ListAvailableVersions(log logging.SimpleLogging) ([]string, error) { + url := fmt.Sprintf("%s/terraform", c.downloadBaseURL) + + if !c.downloadAllowed { + log.Debug("Terraform downloads disabled. Won't list Terraform versions available at %s", url) + return []string{}, nil + } + + log.Debug("Listing Terraform versions available at: %s", url) + + // terraform-switcher calls os.Exit(1) if it fails to successfully GET the configured URL. + // So, before calling it, test if we can connect. Then we can return an error instead if the request fails. + resp, err := http.Get(url) // #nosec G107 -- terraform-switch makes this same call below. Also, we don't process the response payload. + if err != nil { + return nil, fmt.Errorf("Unable to list Terraform versions: %s", err) + } + defer resp.Body.Close() // nolint: errcheck + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Unable to list Terraform versions: response code %d from %s", resp.StatusCode, url) + } + + versions, err := lib.GetTFList(url, true) + return versions, err +} + +// DetectVersion Extracts required_version from Terraform configuration in the specified project directory. Returns nil if unable to determine the version. +// This will also try to intelligently evaluate non-exact matches by listing the available versions of Terraform and picking the best match. +func (c *DefaultClient) DetectVersion(log logging.SimpleLogging, projectDirectory string) *version.Version { + module, diags := tfconfig.LoadModule(projectDirectory) + if diags.HasErrors() { + log.Err("Trying to detect required version: %s", diags.Error()) + } + + if len(module.RequiredCore) != 1 { + log.Info("Cannot determine which version to use from terraform configuration, detected %d possibilities.", len(module.RequiredCore)) + return nil + } + requiredVersionSetting := module.RequiredCore[0] + log.Debug("Found required_version setting of %q", requiredVersionSetting) + + tfVersions, err := c.ListAvailableVersions(log) + if err != nil { + log.Err("Unable to list Terraform versions, may fall back to default: %s", err) + } + + if len(tfVersions) == 0 { + // Fall back to an exact required version string + // We allow `= x.y.z`, `=x.y.z` or `x.y.z` where `x`, `y` and `z` are integers. + re := regexp.MustCompile(`^=?\s*([0-9.]+)\s*$`) + matched := re.FindStringSubmatch(requiredVersionSetting) + if len(matched) == 0 { + log.Debug("Did not specify exact version in terraform configuration, found %q", requiredVersionSetting) + return nil + } + tfVersions = []string{matched[1]} + } + + constraint, _ := semver.NewConstraint(requiredVersionSetting) + versions := make([]*semver.Version, len(tfVersions)) + + for i, tfvals := range tfVersions { + newVersion, err := semver.NewVersion(tfvals) + if err == nil { + versions[i] = newVersion + } + } + + if len(versions) == 0 { + log.Debug("Did not specify exact valid version in terraform configuration, found %q", requiredVersionSetting) + return nil + } + + sort.Sort(sort.Reverse(semver.Collection(versions))) + + for _, element := range versions { + if constraint.Check(element) { // Validate a version against a constraint + tfversionStr := element.String() + if lib.ValidVersionFormat(tfversionStr) { //check if version format is correct + tfversion, _ := version.NewVersion(tfversionStr) + log.Info("Detected module requires version: %s", tfversionStr) + return tfversion + } + } + } + log.Debug("Could not match any valid terraform version with %q", requiredVersionSetting) + return nil +} + // See Client.EnsureVersion. func (c *DefaultClient) EnsureVersion(log logging.SimpleLogging, v *version.Version) error { if v == nil { @@ -270,7 +378,7 @@ func (c *DefaultClient) EnsureVersion(log logging.SimpleLogging, v *version.Vers var err error c.versionsLock.Lock() - _, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir, c.downloadBaseURL) + _, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir, c.downloadBaseURL, c.downloadAllowed) c.versionsLock.Unlock() if err != nil { return err @@ -314,7 +422,7 @@ func (c *DefaultClient) RunCommandWithVersion(ctx command.ProjectContext, path s ctx.Log.Err(err.Error()) return ansi.Strip(string(out)), err } - ctx.Log.Info("successfully ran %q in %q", tfCmd, path) + ctx.Log.Info("Successfully ran %q in %q", tfCmd, path) return ansi.Strip(string(out)), nil } @@ -347,7 +455,7 @@ func (c *DefaultClient) prepCmd(log logging.SimpleLogging, v *version.Version, w } else { var err error c.versionsLock.Lock() - binPath, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir, c.downloadBaseURL) + binPath, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir, c.downloadBaseURL, c.downloadAllowed) c.versionsLock.Unlock() if err != nil { return "", nil, err @@ -418,7 +526,7 @@ func MustConstraint(v string) version.Constraints { // ensureVersion returns the path to a terraform binary of version v. // It will download this version if we don't have it. -func ensureVersion(log logging.SimpleLogging, dl Downloader, versions map[string]string, v *version.Version, binDir string, downloadURL string) (string, error) { +func ensureVersion(log logging.SimpleLogging, dl Downloader, versions map[string]string, v *version.Version, binDir string, downloadURL string, downloadsAllowed bool) (string, error) { if binPath, ok := versions[v.String()]; ok { return binPath, nil } @@ -439,7 +547,11 @@ func ensureVersion(log logging.SimpleLogging, dl Downloader, versions map[string versions[v.String()] = dest return dest, nil } - log.Info("could not find terraform version %s in PATH or %s, downloading from %s", v.String(), binDir, downloadURL) + if !downloadsAllowed { + return "", fmt.Errorf("Could not find terraform version %s in PATH or %s, and downloads are disabled", v.String(), binDir) + } + + log.Info("Could not find terraform version %s in PATH or %s, downloading from %s", v.String(), binDir, downloadURL) urlPrefix := fmt.Sprintf("%s/terraform/%s/terraform_%s", downloadURL, v.String(), v.String()) binURL := fmt.Sprintf("%s_%s_%s.zip", urlPrefix, runtime.GOOS, runtime.GOARCH) checksumURL := fmt.Sprintf("%s_SHA256SUMS", urlPrefix) @@ -448,7 +560,7 @@ func ensureVersion(log logging.SimpleLogging, dl Downloader, versions map[string return "", errors.Wrapf(err, "downloading terraform version %s at %q", v.String(), fullSrcURL) } - log.Info("downloaded terraform %s to %s", v.String(), dest) + log.Info("Downloaded terraform %s to %s", v.String(), dest) versions[v.String()] = dest return dest, nil } diff --git a/server/core/terraform/terraform_client_test.go b/server/core/terraform/terraform_client_test.go index 2ce495f37e..b4833b9b43 100644 --- a/server/core/terraform/terraform_client_test.go +++ b/server/core/terraform/terraform_client_test.go @@ -18,6 +18,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "testing" "time" @@ -76,7 +77,7 @@ is 0.11.13. You can update by downloading from developer.hashicorp.com/terraform Ok(t, err) defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))() - c, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true, projectCmdOutputHandler) + c, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true, true, projectCmdOutputHandler) Ok(t, err) Ok(t, err) @@ -110,7 +111,7 @@ is 0.11.13. You can update by downloading from developer.hashicorp.com/terraform Ok(t, err) defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))() - c, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true, projectCmdOutputHandler) + c, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true, true, projectCmdOutputHandler) Ok(t, err) Ok(t, err) @@ -131,7 +132,7 @@ func TestNewClient_NoTF(t *testing.T) { // Set PATH to only include our empty directory. defer tempSetEnv(t, "PATH", tmp)() - _, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true, projectCmdOutputHandler) + _, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true, true, projectCmdOutputHandler) ErrEquals(t, "terraform not found in $PATH. Set --default-tf-version or download terraform from https://developer.hashicorp.com/terraform/downloads", err) } @@ -154,7 +155,7 @@ func TestNewClient_DefaultTFFlagInPath(t *testing.T) { Ok(t, err) defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))() - c, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true, projectCmdOutputHandler) + c, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, false, true, projectCmdOutputHandler) Ok(t, err) Ok(t, err) @@ -182,7 +183,7 @@ func TestNewClient_DefaultTFFlagInBinDir(t *testing.T) { Ok(t, err) defer tempSetEnv(t, "PATH", fmt.Sprintf("%s:%s", tmp, os.Getenv("PATH")))() - c, err := terraform.NewClient(logging.NewNoopLogger(t), binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true, projectCmdOutputHandler) + c, err := terraform.NewClient(logging.NewNoopLogger(t), binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true, true, projectCmdOutputHandler) Ok(t, err) Ok(t, err) @@ -214,7 +215,7 @@ func TestNewClient_DefaultTFFlagDownload(t *testing.T) { err := os.WriteFile(params[0].(string), []byte("#!/bin/sh\necho '\nTerraform v0.11.10\n'"), 0700) // #nosec G306 return []pegomock.ReturnValue{err} }) - c, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, "https://my-mirror.releases.mycompany.com", mockDownloader, true, projectCmdOutputHandler) + c, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, "https://my-mirror.releases.mycompany.com", mockDownloader, true, true, projectCmdOutputHandler) Ok(t, err) Ok(t, err) @@ -240,7 +241,7 @@ func TestNewClient_BadVersion(t *testing.T) { logger := logging.NewNoopLogger(t) _, binDir, cacheDir := mkSubDirs(t) projectCmdOutputHandler := jobmocks.NewMockProjectCommandOutputHandler() - _, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "malformed", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true, projectCmdOutputHandler) + _, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "malformed", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, nil, true, true, projectCmdOutputHandler) ErrEquals(t, "Malformed version: malformed", err) } @@ -269,7 +270,7 @@ func TestRunCommandWithVersion_DLsTF(t *testing.T) { return []pegomock.ReturnValue{err} }) - c, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, mockDownloader, true, projectCmdOutputHandler) + c, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, mockDownloader, true, true, projectCmdOutputHandler) Ok(t, err) Equals(t, "0.11.10", c.DefaultVersion().String()) @@ -282,7 +283,7 @@ func TestRunCommandWithVersion_DLsTF(t *testing.T) { Equals(t, "\nTerraform v99.99.99\n\n", output) } -// Test the EnsureVersion downloads terraform. +// Test that EnsureVersion downloads terraform. func TestEnsureVersion_downloaded(t *testing.T) { logger := logging.NewNoopLogger(t) RegisterMockTestingT(t) @@ -290,8 +291,8 @@ func TestEnsureVersion_downloaded(t *testing.T) { projectCmdOutputHandler := jobmocks.NewMockProjectCommandOutputHandler() mockDownloader := mocks.NewMockDownloader() - - c, err := terraform.NewTestClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, mockDownloader, true, projectCmdOutputHandler) + downloadsAllowed := true + c, err := terraform.NewTestClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, mockDownloader, downloadsAllowed, true, projectCmdOutputHandler) Ok(t, err) Equals(t, "0.11.10", c.DefaultVersion().String()) @@ -312,6 +313,30 @@ func TestEnsureVersion_downloaded(t *testing.T) { mockDownloader.VerifyWasCalledEventually(Once(), 2*time.Second).GetFile(filepath.Join(tmp, "bin", "terraform99.99.99"), expURL) } +// Test that EnsureVersion throws an error when downloads are disabled +func TestEnsureVersion_downloaded_downloadingDisabled(t *testing.T) { + logger := logging.NewNoopLogger(t) + RegisterMockTestingT(t) + _, binDir, cacheDir := mkSubDirs(t) + projectCmdOutputHandler := jobmocks.NewMockProjectCommandOutputHandler() + + mockDownloader := mocks.NewMockDownloader() + + downloadsAllowed := false + c, err := terraform.NewTestClient(logger, binDir, cacheDir, "", "", "0.11.10", cmd.DefaultTFVersionFlag, cmd.DefaultTFDownloadURL, mockDownloader, downloadsAllowed, true, projectCmdOutputHandler) + Ok(t, err) + + Equals(t, "0.11.10", c.DefaultVersion().String()) + + v, err := version.NewVersion("99.99.99") + Ok(t, err) + + err = c.EnsureVersion(logger, v) + ErrContains(t, "Could not find terraform version", err) + ErrContains(t, "downloads are disabled", err) + mockDownloader.VerifyWasCalled(Never()) +} + // tempSetEnv sets env var key to value. It returns a function that when called // will reset the env var to its original value. func tempSetEnv(t *testing.T, key string, value string) func() { @@ -333,3 +358,152 @@ func mkSubDirs(t *testing.T) (string, string, string) { return tmp, binDir, cachedir } + +// If TF downloads are disabled, test that terraform version is used when specified in terraform configuration only if an exact version +func TestDefaultProjectCommandBuilder_TerraformVersion(t *testing.T) { + // For the following tests: + // If terraform configuration is used, result should be `0.12.8`. + // If project configuration is used, result should be `0.12.6`. + // If an inexact version is used, the result should be `nil` + // If default is to be used, result should be `nil`. + + baseVersionConfig := ` +terraform { + required_version = "%s0.12.8" +} +` + + exactSymbols := []string{"", "="} + // Depending on when the tests are run, the > and >= matching versions will have to be increased. + // It's probably not worth testing the terraform-switcher version here so we only test <, <=, and ~>. + // One way to test this in the future is to mock tfswitcher.GetTFList() to return the highest + // version of 1.3.5. + // nonExactSymbols := []string{">", ">=", "<", "<=", "~>"} + nonExactSymbols := []string{"<", "<=", "~>"} + nonExactVersions := map[string]map[string]string{ + // ">": { + // "project1": "1.3.5", + // }, + // ">=": { + // "project1": "1.3.5", + // }, + "<": { + "project1": "0.12.7", + }, + "<=": { + "project1": "0.12.8", + }, + "~>": { + "project1": "0.12.31", + }, + } + + type testCase struct { + DirStructure map[string]interface{} + Exp map[string]string + IsExact bool + } + + testCases := make(map[string]testCase) + + for _, exactSymbol := range exactSymbols { + testCases[fmt.Sprintf("exact version using \"%s\"", exactSymbol)] = testCase{ + DirStructure: map[string]interface{}{ + "project1": map[string]interface{}{ + "main.tf": fmt.Sprintf(baseVersionConfig, exactSymbol), + }, + }, + Exp: map[string]string{ + "project1": "0.12.8", + }, + IsExact: true, + } + } + + for _, nonExactSymbol := range nonExactSymbols { + testCases[fmt.Sprintf("non-exact version using \"%s\"", nonExactSymbol)] = testCase{ + DirStructure: map[string]interface{}{ + "project1": map[string]interface{}{ + "main.tf": fmt.Sprintf(baseVersionConfig, nonExactSymbol), + }, + }, + Exp: nonExactVersions[nonExactSymbol], + IsExact: false, + } + } + + testCases["no version specified"] = testCase{ + DirStructure: map[string]interface{}{ + "project1": map[string]interface{}{ + "main.tf": nil, + }, + }, + Exp: map[string]string{ + "project1": "", + }, + IsExact: true, + } + + testCases["projects with different terraform versions"] = testCase{ + DirStructure: map[string]interface{}{ + "project1": map[string]interface{}{ + "main.tf": fmt.Sprintf(baseVersionConfig, exactSymbols[0]), + }, + "project2": map[string]interface{}{ + "main.tf": strings.Replace(fmt.Sprintf(baseVersionConfig, exactSymbols[0]), "0.12.8", "0.12.9", -1), + }, + }, + Exp: map[string]string{ + "project1": "0.12.8", + "project2": "0.12.9", + }, + IsExact: true, + } + + runDetectVersionTestCase := func(t *testing.T, name string, testCase testCase, downloadsAllowed bool) bool { + return t.Run(name, func(t *testing.T) { + RegisterMockTestingT(t) + + logger := logging.NewNoopLogger(t) + RegisterMockTestingT(t) + _, binDir, cacheDir := mkSubDirs(t) + projectCmdOutputHandler := jobmocks.NewMockProjectCommandOutputHandler() + + mockDownloader := mocks.NewMockDownloader() + c, err := terraform.NewTestClient(logger, + binDir, + cacheDir, + "", + "", + "", + cmd.DefaultTFVersionFlag, + cmd.DefaultTFDownloadURL, + mockDownloader, + downloadsAllowed, + true, + projectCmdOutputHandler) + Ok(t, err) + + tmpDir := DirStructure(t, testCase.DirStructure) + + for project, expectedVersion := range testCase.Exp { + detectedVersion := c.DetectVersion(logger, filepath.Join(tmpDir, project)) + + expectNil := expectedVersion == "" || (!testCase.IsExact && !downloadsAllowed) + if expectNil { + Assert(t, detectedVersion == nil, "TerraformVersion is supposed to be nil.") + } else { + Assert(t, detectedVersion != nil, "TerraformVersion is nil.") + Ok(t, err) + Equals(t, expectedVersion, detectedVersion.String()) + } + } + + }) + } + + for name, testCase := range testCases { + runDetectVersionTestCase(t, name+": Downloads Allowed", testCase, true) + runDetectVersionTestCase(t, name+": Downloads Disabled", testCase, false) + } +} diff --git a/server/events/project_command_builder.go b/server/events/project_command_builder.go index 6d58a79b96..64a6aa6a3b 100644 --- a/server/events/project_command_builder.go +++ b/server/events/project_command_builder.go @@ -10,6 +10,7 @@ import ( "github.com/uber-go/tally" "github.com/runatlantis/atlantis/server/core/config/valid" + "github.com/runatlantis/atlantis/server/core/terraform" "github.com/runatlantis/atlantis/server/logging" "github.com/runatlantis/atlantis/server/metrics" @@ -54,6 +55,7 @@ func NewInstrumentedProjectCommandBuilder( RestrictFileList bool, scope tally.Scope, logger logging.SimpleLogging, + terraformClient terraform.Client, ) *InstrumentedProjectCommandBuilder { scope = scope.SubScope("builder") @@ -79,6 +81,7 @@ func NewInstrumentedProjectCommandBuilder( RestrictFileList, scope, logger, + terraformClient, ), Logger: logger, scope: scope, @@ -102,6 +105,7 @@ func NewProjectCommandBuilder( RestrictFileList bool, scope tally.Scope, logger logging.SimpleLogging, + terraformClient terraform.Client, ) *DefaultProjectCommandBuilder { return &DefaultProjectCommandBuilder{ ParserValidator: parserValidator, @@ -121,6 +125,7 @@ func NewProjectCommandBuilder( commentBuilder, scope, ), + TerraformExecutor: terraformClient, } } @@ -181,6 +186,7 @@ type DefaultProjectCommandBuilder struct { AutoplanFileList string EnableDiffMarkdownFormat bool RestrictFileList bool + TerraformExecutor terraform.Client } // See ProjectCommandBuilder.BuildAutoplanCommands. @@ -330,6 +336,7 @@ func (p *DefaultProjectCommandBuilder) buildPlanAllCommands(ctx *command.Context repoCfg.ParallelApply, repoCfg.ParallelPlan, verbose, + p.TerraformExecutor, )...) } } else { @@ -367,6 +374,7 @@ func (p *DefaultProjectCommandBuilder) buildPlanAllCommands(ctx *command.Context DefaultParallelApplyEnabled, DefaultParallelPlanEnabled, verbose, + p.TerraformExecutor, )...) } } @@ -701,6 +709,7 @@ func (p *DefaultProjectCommandBuilder) buildProjectCommandCtx(ctx *command.Conte parallelApply, parallelPlan, verbose, + p.TerraformExecutor, )...) } } else { @@ -716,6 +725,7 @@ func (p *DefaultProjectCommandBuilder) buildProjectCommandCtx(ctx *command.Conte parallelApply, parallelPlan, verbose, + p.TerraformExecutor, )...) } diff --git a/server/events/project_command_builder_internal_test.go b/server/events/project_command_builder_internal_test.go index 62239521d9..d2e1487c82 100644 --- a/server/events/project_command_builder_internal_test.go +++ b/server/events/project_command_builder_internal_test.go @@ -7,9 +7,9 @@ import ( version "github.com/hashicorp/go-version" . "github.com/petergtz/pegomock" - "github.com/runatlantis/atlantis/server/core/config" "github.com/runatlantis/atlantis/server/core/config/valid" + "github.com/runatlantis/atlantis/server/core/terraform/mocks" "github.com/runatlantis/atlantis/server/events/command" "github.com/runatlantis/atlantis/server/events/matchers" "github.com/runatlantis/atlantis/server/events/models" @@ -621,6 +621,8 @@ projects: Ok(t, os.WriteFile(filepath.Join(tmp, "atlantis.yaml"), []byte(c.repoCfg), 0600)) } + terraformClient := mocks.NewMockClient() + builder := NewProjectCommandBuilder( false, parser, @@ -638,6 +640,7 @@ projects: false, statsScope, logger, + terraformClient, ) // We run a test for each type of command. @@ -825,6 +828,8 @@ projects: logger := logging.NewNoopLogger(t) statsScope, _, _ := metrics.NewLoggingScope(logging.NewNoopLogger(t), "atlantis") + terraformClient := mocks.NewMockClient() + builder := NewProjectCommandBuilder( false, parser, @@ -842,6 +847,7 @@ projects: false, statsScope, logger, + terraformClient, ) // We run a test for each type of command, again specific projects @@ -1058,6 +1064,8 @@ workflows: } statsScope, _, _ := metrics.NewLoggingScope(logging.NewNoopLogger(t), "atlantis") + terraformClient := mocks.NewMockClient() + builder := NewProjectCommandBuilder( true, parser, @@ -1075,6 +1083,7 @@ workflows: false, statsScope, logger, + terraformClient, ) cmd := command.PolicyCheck diff --git a/server/events/project_command_builder_test.go b/server/events/project_command_builder_test.go index 9c3f8b8349..7122eebb1f 100644 --- a/server/events/project_command_builder_test.go +++ b/server/events/project_command_builder_test.go @@ -1,13 +1,14 @@ package events_test import ( - "fmt" "os" "path/filepath" "strings" "testing" + "github.com/hashicorp/go-version" . "github.com/petergtz/pegomock" + terraform_mocks "github.com/runatlantis/atlantis/server/core/terraform/mocks" "github.com/runatlantis/atlantis/server/core/config" "github.com/runatlantis/atlantis/server/core/config/valid" @@ -123,6 +124,9 @@ projects: logger := logging.NewNoopLogger(t) scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") + terraformClient := terraform_mocks.NewMockClient() + When(terraformClient.ListAvailableVersions(matchers.AnyLoggingSimpleLogging())).ThenReturn([]string{}, nil) + for _, c := range cases { t.Run(c.Description, func(t *testing.T) { RegisterMockTestingT(t) @@ -163,6 +167,7 @@ projects: false, scope, logger, + terraformClient, ) ctxs, err := builder.BuildAutoplanCommands(&command.Context{ @@ -415,6 +420,9 @@ projects: UnDivergedReq: false, } + terraformClient := terraform_mocks.NewMockClient() + When(terraformClient.ListAvailableVersions(matchers.AnyLoggingSimpleLogging())).ThenReturn([]string{}, nil) + builder := events.NewProjectCommandBuilder( false, &config.ParserValidator{}, @@ -432,6 +440,7 @@ projects: false, scope, logger, + terraformClient, ) var actCtxs []command.ProjectContext @@ -588,6 +597,9 @@ projects: UnDivergedReq: false, } + terraformClient := terraform_mocks.NewMockClient() + When(terraformClient.ListAvailableVersions(matchers.AnyLoggingSimpleLogging())).ThenReturn([]string{}, nil) + builder := events.NewProjectCommandBuilder( false, &config.ParserValidator{}, @@ -605,6 +617,7 @@ projects: true, scope, logger, + terraformClient, ) var actCtxs []command.ProjectContext @@ -758,6 +771,9 @@ projects: UnDivergedReq: false, } + terraformClient := terraform_mocks.NewMockClient() + When(terraformClient.ListAvailableVersions(matchers.AnyLoggingSimpleLogging())).ThenReturn([]string{}, nil) + builder := events.NewProjectCommandBuilder( false, &config.ParserValidator{}, @@ -775,6 +791,7 @@ projects: false, scope, logger, + terraformClient, ) ctxs, err := builder.BuildPlanCommands( @@ -850,6 +867,9 @@ func TestDefaultProjectCommandBuilder_BuildMultiApply(t *testing.T) { } scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") + terraformClient := terraform_mocks.NewMockClient() + When(terraformClient.ListAvailableVersions(matchers.AnyLoggingSimpleLogging())).ThenReturn([]string{}, nil) + builder := events.NewProjectCommandBuilder( false, &config.ParserValidator{}, @@ -867,6 +887,7 @@ func TestDefaultProjectCommandBuilder_BuildMultiApply(t *testing.T) { false, scope, logger, + terraformClient, ) ctxs, err := builder.BuildApplyCommands( @@ -935,6 +956,8 @@ projects: } logger := logging.NewNoopLogger(t) scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") + terraformClient := terraform_mocks.NewMockClient() + When(terraformClient.ListAvailableVersions(matchers.AnyLoggingSimpleLogging())).ThenReturn([]string{}, nil) builder := events.NewProjectCommandBuilder( false, @@ -953,6 +976,7 @@ projects: false, scope, logger, + terraformClient, ) ctx := &command.Context{ @@ -1016,6 +1040,9 @@ func TestDefaultProjectCommandBuilder_EscapeArgs(t *testing.T) { UnDivergedReq: false, } + terraformClient := terraform_mocks.NewMockClient() + When(terraformClient.ListAvailableVersions(matchers.AnyLoggingSimpleLogging())).ThenReturn([]string{}, nil) + builder := events.NewProjectCommandBuilder( false, &config.ParserValidator{}, @@ -1033,6 +1060,7 @@ func TestDefaultProjectCommandBuilder_EscapeArgs(t *testing.T) { false, scope, logger, + terraformClient, ) var actCtxs []command.ProjectContext @@ -1064,7 +1092,7 @@ func TestDefaultProjectCommandBuilder_TerraformVersion(t *testing.T) { baseVersionConfig := ` terraform { - required_version = "%s0.12.8" + required_version = "0.12.8" } ` @@ -1075,77 +1103,26 @@ projects: terraform_version: v0.12.6 ` - exactSymbols := []string{"", "="} - // Depending on when the tests are run, the > and >= matching versions will have to be increased. - // It's probably not worth testing the terraform-switcher version here so we only test <, <=, and ~>. - // One way to test this in the future is to mock tfswitcher.GetTFList() to return the highest - // version of 1.3.5. - // nonExactSymbols := []string{">", ">=", "<", "<=", "~>"} - nonExactSymbols := []string{"<", "<=", "~>"} - nonExactVersions := map[string]map[string][]int{ - // ">": { - // "project1": {1, 3, 5}, - // }, - // ">=": { - // "project1": {1, 3, 5}, - // }, - "<": { - "project1": {0, 12, 7}, - }, - "<=": { - "project1": {0, 12, 8}, - }, - "~>": { - "project1": {0, 12, 31}, - }, - } - type testCase struct { DirStructure map[string]interface{} AtlantisYAML string ModifiedFiles []string - Exp map[string][]int + Exp map[string]string } testCases := make(map[string]testCase) - for _, exactSymbol := range exactSymbols { - testCases[fmt.Sprintf("exact version in terraform config using \"%s\"", exactSymbol)] = testCase{ - DirStructure: map[string]interface{}{ - "project1": map[string]interface{}{ - "main.tf": fmt.Sprintf(baseVersionConfig, exactSymbol), - }, - }, - ModifiedFiles: []string{"project1/main.tf"}, - Exp: map[string][]int{ - "project1": {0, 12, 8}, - }, - } - } - - for _, nonExactSymbol := range nonExactSymbols { - testCases[fmt.Sprintf("non-exact version in terraform config using \"%s\"", nonExactSymbol)] = testCase{ - DirStructure: map[string]interface{}{ - "project1": map[string]interface{}{ - "main.tf": fmt.Sprintf(baseVersionConfig, nonExactSymbol), - }, - }, - ModifiedFiles: []string{"project1/main.tf"}, - Exp: nonExactVersions[nonExactSymbol], - } - } - // atlantis.yaml should take precedence over terraform config testCases["with project config and terraform config"] = testCase{ DirStructure: map[string]interface{}{ "project1": map[string]interface{}{ - "main.tf": fmt.Sprintf(baseVersionConfig, exactSymbols[0]), + "main.tf": baseVersionConfig, }, valid.DefaultAtlantisFile: atlantisYamlContent, }, ModifiedFiles: []string{"project1/main.tf", "project2/main.tf"}, - Exp: map[string][]int{ - "project1": {0, 12, 6}, + Exp: map[string]string{ + "project1": "0.12.6", }, } @@ -1157,8 +1134,8 @@ projects: valid.DefaultAtlantisFile: atlantisYamlContent, }, ModifiedFiles: []string{"project1/main.tf"}, - Exp: map[string][]int{ - "project1": {0, 12, 6}, + Exp: map[string]string{ + "project1": "0.12.6", }, } @@ -1169,24 +1146,24 @@ projects: }, }, ModifiedFiles: []string{"project1/main.tf", "project2/main.tf"}, - Exp: map[string][]int{ - "project1": nil, + Exp: map[string]string{ + "project1": "", }, } testCases["project with different terraform config"] = testCase{ DirStructure: map[string]interface{}{ "project1": map[string]interface{}{ - "main.tf": fmt.Sprintf(baseVersionConfig, exactSymbols[0]), + "main.tf": baseVersionConfig, }, "project2": map[string]interface{}{ - "main.tf": strings.Replace(fmt.Sprintf(baseVersionConfig, exactSymbols[0]), "0.12.8", "0.12.9", -1), + "main.tf": strings.Replace(baseVersionConfig, "0.12.8", "0.12.9", -1), }, }, ModifiedFiles: []string{"project1/main.tf", "project2/main.tf"}, - Exp: map[string][]int{ - "project1": {0, 12, 8}, - "project2": {0, 12, 9}, + Exp: map[string]string{ + "project1": "0.12.8", + "project2": "0.12.9", }, } @@ -1221,6 +1198,17 @@ projects: UnDivergedReq: false, } + terraformClient := terraform_mocks.NewMockClient() + When(terraformClient.DetectVersion(matchers.AnyLoggingSimpleLogging(), AnyString())).Then(func(params []Param) ReturnValues { + projectName := filepath.Base(params[1].(string)) + testVersion := testCase.Exp[projectName] + if testVersion != "" { + v, _ := version.NewVersion(testVersion) + return []ReturnValue{v} + } + return nil + }) + builder := events.NewProjectCommandBuilder( false, &config.ParserValidator{}, @@ -1238,6 +1226,7 @@ projects: false, scope, logger, + terraformClient, ) actCtxs, err := builder.BuildPlanCommands( @@ -1255,9 +1244,9 @@ projects: Ok(t, err) Equals(t, len(testCase.Exp), len(actCtxs)) for _, actCtx := range actCtxs { - if testCase.Exp[actCtx.RepoRelDir] != nil { - Assert(t, actCtx.TerraformVersion != nil, "TerraformVersion is nil.") - Equals(t, testCase.Exp[actCtx.RepoRelDir], actCtx.TerraformVersion.Segments()) + if testCase.Exp[actCtx.RepoRelDir] != "" { + Assert(t, actCtx.TerraformVersion != nil, "TerraformVersion is nil, not %s for %s", testCase.Exp[actCtx.RepoRelDir], actCtx.RepoRelDir) + Equals(t, testCase.Exp[actCtx.RepoRelDir], actCtx.TerraformVersion.String()) } else { Assert(t, actCtx.TerraformVersion == nil, "TerraformVersion is supposed to be nil.") } @@ -1310,6 +1299,8 @@ parallel_plan: true`, UnDivergedReq: false, } scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") + terraformClient := terraform_mocks.NewMockClient() + When(terraformClient.ListAvailableVersions(matchers.AnyLoggingSimpleLogging())).ThenReturn([]string{}, nil) builder := events.NewProjectCommandBuilder( false, @@ -1328,6 +1319,7 @@ parallel_plan: true`, false, scope, logger, + terraformClient, ) var actCtxs []command.ProjectContext @@ -1370,6 +1362,8 @@ func TestDefaultProjectCommandBuilder_WithPolicyCheckEnabled_BuildAutoplanComman } globalCfg := valid.NewGlobalCfgFromArgs(globalCfgArgs) + terraformClient := terraform_mocks.NewMockClient() + When(terraformClient.ListAvailableVersions(matchers.AnyLoggingSimpleLogging())).ThenReturn([]string{}, nil) builder := events.NewProjectCommandBuilder( true, @@ -1388,6 +1382,7 @@ func TestDefaultProjectCommandBuilder_WithPolicyCheckEnabled_BuildAutoplanComman false, scope, logger, + terraformClient, ) ctxs, err := builder.BuildAutoplanCommands(&command.Context{ @@ -1453,6 +1448,8 @@ func TestDefaultProjectCommandBuilder_BuildVersionCommand(t *testing.T) { ApprovedReq: false, UnDivergedReq: false, } + terraformClient := terraform_mocks.NewMockClient() + When(terraformClient.ListAvailableVersions(matchers.AnyLoggingSimpleLogging())).ThenReturn([]string{}, nil) builder := events.NewProjectCommandBuilder( false, @@ -1471,6 +1468,7 @@ func TestDefaultProjectCommandBuilder_BuildVersionCommand(t *testing.T) { false, scope, logger, + terraformClient, ) ctxs, err := builder.BuildVersionCommands( diff --git a/server/events/project_command_context_builder.go b/server/events/project_command_context_builder.go index d18d2b294f..8f896181c3 100644 --- a/server/events/project_command_context_builder.go +++ b/server/events/project_command_context_builder.go @@ -2,22 +2,15 @@ package events import ( "path/filepath" - "sort" - "github.com/Masterminds/semver" "github.com/google/uuid" - "github.com/hashicorp/go-version" - - "github.com/hashicorp/terraform-config-inspect/tfconfig" "github.com/runatlantis/atlantis/server/core/config/valid" + "github.com/runatlantis/atlantis/server/core/terraform" "github.com/runatlantis/atlantis/server/events/command" "github.com/runatlantis/atlantis/server/events/models" "github.com/uber-go/tally" - lib "github.com/warrensbox/terraform-switcher/lib" ) -var mirrorURL = "https://releases.hashicorp.com/terraform" - func NewProjectCommandContextBuilder(policyCheckEnabled bool, commentBuilder CommentBuilder, scope tally.Scope) ProjectCommandContextBuilder { projectCommandContextBuilder := &DefaultProjectCommandContextBuilder{ CommentBuilder: commentBuilder, @@ -44,7 +37,7 @@ type ProjectCommandContextBuilder interface { prjCfg valid.MergedProjectCfg, commentFlags []string, repoDir string, - automerge, parallelApply, parallelPlan, verbose bool, + automerge, parallelApply, parallelPlan, verbose bool, terraformClient terraform.Client, ) []command.ProjectContext } @@ -63,12 +56,12 @@ func (cb *CommandScopedStatsProjectCommandContextBuilder) BuildProjectContext( prjCfg valid.MergedProjectCfg, commentFlags []string, repoDir string, - automerge, parallelApply, parallelPlan, verbose bool, + automerge, parallelApply, parallelPlan, verbose bool, terraformClient terraform.Client, ) (projectCmds []command.ProjectContext) { cb.ProjectCounter.Inc(1) cmds := cb.ProjectCommandContextBuilder.BuildProjectContext( - ctx, cmdName, prjCfg, commentFlags, repoDir, automerge, parallelApply, parallelPlan, verbose, + ctx, cmdName, prjCfg, commentFlags, repoDir, automerge, parallelApply, parallelPlan, verbose, terraformClient, ) projectCmds = []command.ProjectContext{} @@ -95,7 +88,7 @@ func (cb *DefaultProjectCommandContextBuilder) BuildProjectContext( prjCfg valid.MergedProjectCfg, commentFlags []string, repoDir string, - automerge, parallelApply, parallelPlan, verbose bool, + automerge, parallelApply, parallelPlan, verbose bool, terraformClient terraform.Client, ) (projectCmds []command.ProjectContext) { ctx.Log.Debug("Building project command context for %s", cmdName) @@ -115,7 +108,7 @@ func (cb *DefaultProjectCommandContextBuilder) BuildProjectContext( // If TerraformVersion not defined in config file look for a // terraform.require_version block. if prjCfg.TerraformVersion == nil { - prjCfg.TerraformVersion = getTfVersion(ctx, filepath.Join(repoDir, prjCfg.RepoRelDir)) + prjCfg.TerraformVersion = terraformClient.DetectVersion(ctx.Log, filepath.Join(repoDir, prjCfg.RepoRelDir)) } projectCmdContext := newProjectCommandContext( @@ -152,14 +145,14 @@ func (cb *PolicyCheckProjectCommandContextBuilder) BuildProjectContext( prjCfg valid.MergedProjectCfg, commentFlags []string, repoDir string, - automerge, parallelApply, parallelPlan, verbose bool, + automerge, parallelApply, parallelPlan, verbose bool, terraformClient terraform.Client, ) (projectCmds []command.ProjectContext) { ctx.Log.Debug("PolicyChecks are enabled") // If TerraformVersion not defined in config file look for a // terraform.require_version block. if prjCfg.TerraformVersion == nil { - prjCfg.TerraformVersion = getTfVersion(ctx, filepath.Join(repoDir, prjCfg.RepoRelDir)) + prjCfg.TerraformVersion = terraformClient.DetectVersion(ctx.Log, filepath.Join(repoDir, prjCfg.RepoRelDir)) } projectCmds = cb.ProjectCommandContextBuilder.BuildProjectContext( @@ -172,6 +165,7 @@ func (cb *PolicyCheckProjectCommandContextBuilder) BuildProjectContext( parallelApply, parallelPlan, verbose, + terraformClient, ) if cmdName == command.Plan { @@ -282,50 +276,3 @@ func escapeArgs(args []string) []string { } return escaped } - -// Extracts required_version from Terraform configuration. -// Returns nil if unable to determine version from configuration. -func getTfVersion(ctx *command.Context, absProjDir string) *version.Version { - module, diags := tfconfig.LoadModule(absProjDir) - if diags.HasErrors() { - ctx.Log.Err("trying to detect required version: %s", diags.Error()) - } - - if len(module.RequiredCore) != 1 { - ctx.Log.Info("cannot determine which version to use from terraform configuration, detected %d possibilities.", len(module.RequiredCore)) - return nil - } - requiredVersionSetting := module.RequiredCore[0] - ctx.Log.Debug("found required_version setting of %q", requiredVersionSetting) - - tflist, _ := lib.GetTFList(mirrorURL, true) - constrains, _ := semver.NewConstraint(requiredVersionSetting) - versions := make([]*semver.Version, len(tflist)) - - for i, tfvals := range tflist { - version, err := semver.NewVersion(tfvals) //NewVersion parses a given version and returns an instance of Version or an error if unable to parse the version. - if err == nil { - versions[i] = version - } - } - - if len(versions) == 0 { - ctx.Log.Debug("did not specify exact valid version in terraform configuration, found %q", requiredVersionSetting) - return nil - } - - sort.Sort(sort.Reverse(semver.Collection(versions))) - - for _, element := range versions { - if constrains.Check(element) { // Validate a version against a constraint - tfversionStr := element.String() - if lib.ValidVersionFormat(tfversionStr) { //check if version format is correct - tfversion, _ := version.NewVersion(tfversionStr) - ctx.Log.Info("detected module requires version: %s", tfversionStr) - return tfversion - } - } - } - ctx.Log.Debug("could not match any valid terraform version with %q", requiredVersionSetting) - return nil -} diff --git a/server/events/project_command_context_builder_test.go b/server/events/project_command_context_builder_test.go index a9039e24ba..aa90dd90ee 100644 --- a/server/events/project_command_context_builder_test.go +++ b/server/events/project_command_context_builder_test.go @@ -5,6 +5,7 @@ import ( . "github.com/petergtz/pegomock" "github.com/runatlantis/atlantis/server/core/config/valid" + terraform_mocks "github.com/runatlantis/atlantis/server/core/terraform/mocks" "github.com/runatlantis/atlantis/server/events" "github.com/runatlantis/atlantis/server/events/command" "github.com/runatlantis/atlantis/server/events/mocks" @@ -46,6 +47,9 @@ func TestProjectCommandContextBuilder_PullStatus(t *testing.T) { expectedApplyCmt := "Apply Comment" expectedPlanCmt := "Plan Comment" + terraformClient := terraform_mocks.NewMockClient() + When(terraformClient.ListAvailableVersions(commandCtx.Log)) + t.Run("with project name defined", func(t *testing.T) { When(mockCommentBuilder.BuildPlanComment(projRepoRelDir, projWorkspace, projName, []string{})).ThenReturn(expectedPlanCmt) When(mockCommentBuilder.BuildApplyComment(projRepoRelDir, projWorkspace, projName, false)).ThenReturn(expectedApplyCmt) @@ -58,7 +62,7 @@ func TestProjectCommandContextBuilder_PullStatus(t *testing.T) { }, } - result := subject.BuildProjectContext(commandCtx, command.Plan, projCfg, []string{}, "some/dir", false, false, false, false) + result := subject.BuildProjectContext(commandCtx, command.Plan, projCfg, []string{}, "some/dir", false, false, false, false, terraformClient) assert.Equal(t, models.ErroredPolicyCheckStatus, result[0].ProjectPlanStatus) }) @@ -77,7 +81,7 @@ func TestProjectCommandContextBuilder_PullStatus(t *testing.T) { }, } - result := subject.BuildProjectContext(commandCtx, command.Plan, projCfg, []string{}, "some/dir", false, false, false, false) + result := subject.BuildProjectContext(commandCtx, command.Plan, projCfg, []string{}, "some/dir", false, false, false, false, terraformClient) assert.Equal(t, models.ErroredPolicyCheckStatus, result[0].ProjectPlanStatus) }) @@ -97,7 +101,7 @@ func TestProjectCommandContextBuilder_PullStatus(t *testing.T) { }, } - result := subject.BuildProjectContext(commandCtx, command.Plan, projCfg, []string{}, "some/dir", false, true, false, false) + result := subject.BuildProjectContext(commandCtx, command.Plan, projCfg, []string{}, "some/dir", false, true, false, false, terraformClient) assert.True(t, result[0].ParallelApplyEnabled) assert.False(t, result[0].ParallelPlanEnabled) diff --git a/server/server.go b/server/server.go index 2e84a37ea6..c37328157a 100644 --- a/server/server.go +++ b/server/server.go @@ -393,6 +393,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { config.DefaultTFVersionFlag, userConfig.TFDownloadURL, &terraform.DefaultDownloader{}, + userConfig.TFDownload, true, projectCmdOutputHandler) // The flag.Lookup call is to detect if we're running in a unit test. If we @@ -557,6 +558,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { userConfig.RestrictFileList, statsScope, logger, + terraformClient, ) showStepRunner, err := runtime.NewShowStepRunner(terraformClient, defaultTfVersion) diff --git a/server/user_config.go b/server/user_config.go index 771e89faf8..4203ca6139 100644 --- a/server/user_config.go +++ b/server/user_config.go @@ -99,6 +99,7 @@ type UserConfig struct { SSLCertFile string `mapstructure:"ssl-cert-file"` SSLKeyFile string `mapstructure:"ssl-key-file"` RestrictFileList bool `mapstructure:"restrict-file-list"` + TFDownload bool `mapstructure:"tf-download"` TFDownloadURL string `mapstructure:"tf-download-url"` TFEHostname string `mapstructure:"tfe-hostname"` TFELocalExecutionMode bool `mapstructure:"tfe-local-execution-mode"`