feat: Add flag to disable downloading tf for airgapped environments

cmd/server.go

@@ -117,6 +117,7 @@ const (
 	SSLCertFileFlag            = "ssl-cert-file"
 	SSLKeyFileFlag             = "ssl-key-file"
 	RestrictFileList           = "restrict-file-list"
+	TFDownloadFlag             = "tf-download"
 	TFDownloadURLFlag          = "tf-download-url"
 	VarFileAllowlistFlag       = "var-file-allowlist"
 	VCSStatusName              = "vcs-status-name"
@@ -151,6 +152,7 @@ const (
 	DefaultRedisTLSEnabled              = false
 	DefaultRedisInsecureSkipVerify      = false
 	DefaultTFDownloadURL                = "https://releases.hashicorp.com"
+	DefaultTFDownload                   = true
 	DefaultTFEHostname                  = "app.terraform.io"
 	DefaultVCSStatusName                = "atlantis"
 	DefaultWebBasicAuth                 = false
@@ -500,6 +502,10 @@ var boolFlags = map[string]boolFlag{
 		description:  "Skips cloning the PR repo if there are no projects were changed in the PR.",
 		defaultValue: false,
 	},
+	TFDownloadFlag: {
+		description:  "Allow Atlantis to list & download Terraform versions. Setting this to false can be helpful in air-gapped environments.",
+		defaultValue: DefaultTFDownload,
+	},
 	TFELocalExecutionModeFlag: {
 		description:  "Enable if you're using local execution mode (instead of TFE/C's remote execution mode).",
 		defaultValue: false,

runatlantis.io/docs/server-configuration.md

@@ -917,6 +917,15 @@ and set `--autoplan-modules` to `false`.
   ```
   Namespace for emitting stats/metrics. See [stats](stats.html) section.
 
+### `--tf--download`
+  ```bash
+  atlantis server --tf-download=false
+  # or
+  ATLANTIS_TF_DOWNLOAD=false
+  ```
+Defaults to `true`. Allow Atlantis to list and download additional versions of Terraform.
+Setting this to `false` can be useful in an air-gapped environment where a download mirror is not available.
+
 ### `--tf-download-url`
   ```bash
   atlantis server --tf-download-url="https://releases.company.com"
@@ -926,6 +935,8 @@ and set `--autoplan-modules` to `false`.
   An alternative URL to download Terraform versions if they are missing. Useful in an airgapped
   environment where releases.hashicorp.com is not available. Directory structure of the custom
   endpoint should match that of releases.hashicorp.com.
+
+  This has no impact if `--tf-download` is set to `false`.
 
 ### `--tfe-hostname`
   ```bash

server/controllers/events/events_controller_e2e_test.go

@@ -914,7 +914,7 @@ func setupE2E(t *testing.T, repoDir, repoConfigFile string) (events_controllers.
 		GitlabUser:     "gitlab-user",
 		ExecutableName: "atlantis",
 	}
-	terraformClient, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "", "default-tf-version", "https://releases.hashicorp.com", &NoopTFDownloader{}, false, projectCmdOutputHandler)
+	terraformClient, err := terraform.NewClient(logger, binDir, cacheDir, "", "", "", "default-tf-version", "https://releases.hashicorp.com", &NoopTFDownloader{}, true, false, projectCmdOutputHandler)
 	Ok(t, err)
 	boltdb, err := db.New(dataDir)
 	Ok(t, err)
@@ -1010,6 +1010,7 @@ func setupE2E(t *testing.T, repoDir, repoConfigFile string) (events_controllers.
 		false,
 		statsScope,
 		logger,
+		terraformClient,
 	)
 
 	showStepRunner, err := runtime.NewShowStepRunner(terraformClient, defaultTFVersion)

server/core/terraform/terraform_client.go

@@ -23,13 +23,17 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"sort"
 	"strings"
 	"sync"
 
+	"github.com/Masterminds/semver"
 	"github.com/hashicorp/go-getter"
 	"github.com/hashicorp/go-version"
+	"github.com/hashicorp/terraform-config-inspect/tfconfig"
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
+	"github.com/warrensbox/terraform-switcher/lib"
 
 	"github.com/runatlantis/atlantis/server/core/runtime/models"
 	"github.com/runatlantis/atlantis/server/events/command"
@@ -53,6 +57,12 @@ type Client interface {
 
 	// EnsureVersion makes sure that terraform version `v` is available to use
 	EnsureVersion(log logging.SimpleLogging, v *version.Version) error
+
+	// ListAvailableVersions returns all available version of Terraform, if available; otherwise this will return an empty list.
+	ListAvailableVersions(log logging.SimpleLogging) ([]string, error)
+
+	// DetectVersion Extracts required_version from Terraform configuration in the specified project directory. Returns nil if unable to determine the version.
+	DetectVersion(projectDirectory string, log logging.SimpleLogging) *version.Version
 }
 
 type DefaultClient struct {
@@ -67,8 +77,9 @@ type DefaultClient struct {
 	// with another binary, ex. echo.
 	overrideTF string
 	// downloader downloads terraform versions.
-	downloader      Downloader
-	downloadBaseURL string
+	downloader       Downloader
+	downloadBaseURL  string
+	downloadsAllowed bool
 	// versions maps from the string representation of a tf version (ex. 0.11.10)
 	// to the absolute path of that binary on disk (if it exists).
 	// Use versionsLock to control access.
@@ -111,6 +122,7 @@ func NewClientWithDefaultVersion(
 	defaultVersionFlagName string,
 	tfDownloadURL string,
 	tfDownloader Downloader,
+	allowDownloads bool,
 	usePluginCache bool,
 	fetchAsync bool,
 	projectCmdOutputHandler jobs.ProjectCommandOutputHandler,
@@ -147,7 +159,7 @@ func NewClientWithDefaultVersion(
 			// Since ensureVersion might end up downloading terraform,
 			// we call it asynchronously so as to not delay server startup.
 			versionsLock.Lock()
-			_, err := ensureVersion(log, tfDownloader, versions, defaultVersion, binDir, tfDownloadURL)
+			_, err := ensureVersion(log, tfDownloader, versions, defaultVersion, binDir, tfDownloadURL, allowDownloads)
 			versionsLock.Unlock()
 			if err != nil {
 				log.Err("could not download terraform %s: %s", defaultVersion.String(), err)
@@ -177,6 +189,7 @@ func NewClientWithDefaultVersion(
 		binDir:                  binDir,
 		downloader:              tfDownloader,
 		downloadBaseURL:         tfDownloadURL,
+		downloadsAllowed:        allowDownloads,
 		versionsLock:            &versionsLock,
 		versions:                versions,
 		usePluginCache:          usePluginCache,
@@ -195,6 +208,7 @@ func NewTestClient(
 	defaultVersionFlagName string,
 	tfDownloadURL string,
 	tfDownloader Downloader,
+	downloadsAllowed bool,
 	usePluginCache bool,
 	projectCmdOutputHandler jobs.ProjectCommandOutputHandler,
 ) (*DefaultClient, error) {
@@ -208,6 +222,7 @@ func NewTestClient(
 		defaultVersionFlagName,
 		tfDownloadURL,
 		tfDownloader,
+		downloadsAllowed,
 		usePluginCache,
 		false,
 		projectCmdOutputHandler,
@@ -232,6 +247,7 @@ func NewClient(
 	defaultVersionFlagName string,
 	tfDownloadURL string,
 	tfDownloader Downloader,
+	allowDownloads bool,
 	usePluginCache bool,
 	projectCmdOutputHandler jobs.ProjectCommandOutputHandler,
 ) (*DefaultClient, error) {
@@ -245,6 +261,7 @@ func NewClient(
 		defaultVersionFlagName,
 		tfDownloadURL,
 		tfDownloader,
+		allowDownloads,
 		usePluginCache,
 		true,
 		projectCmdOutputHandler,
@@ -262,6 +279,83 @@ func (c *DefaultClient) TerraformBinDir() string {
 	return c.binDir
 }
 
+// ListAvailableVersions returns all available version of Terraform. If downloads are not allowed, this will return an empty list.
+func (c *DefaultClient) ListAvailableVersions(log logging.SimpleLogging) ([]string, error) {
+	url := fmt.Sprintf("%s/terraform", c.downloadBaseURL)
+
+	if !c.downloadsAllowed {
+		log.Debug("Terraform downloads disabled. Won't list Terraform versions available at %s", url)
+		return []string{}, nil
+	}
+
+	log.Debug("Listing Terraform versions available at: %s", url)
+	versions, err := lib.GetTFList(url, true)
+	return versions, err
+}
+
+// DetectVersion Extracts required_version from Terraform configuration in the specified project directory. Returns nil if unable to determine the version.
+// This will also try to intelligently evaluate non-exact matches by listing the available versions of Terraform and picking the best match.
+func (c *DefaultClient) DetectVersion(projectDirectory string, log logging.SimpleLogging) *version.Version {
+	module, diags := tfconfig.LoadModule(projectDirectory)
+	if diags.HasErrors() {
+		log.Err("trying to detect required version: %s", diags.Error())
+	}
+
+	if len(module.RequiredCore) != 1 {
+		log.Info("cannot determine which version to use from terraform configuration, detected %d possibilities.", len(module.RequiredCore))
+		return nil
+	}
+	requiredVersionSetting := module.RequiredCore[0]
+	log.Debug("found required_version setting of %q", requiredVersionSetting)
+
+	tfVersions, err := c.ListAvailableVersions(log)
+	if err != nil {
+		log.Err("Unable to list Terraform versions, may fall back to default: %s", err)
+	}
+
+	if len(tfVersions) == 0 {
+		// Fall back to an exact required version string
+		// We allow `= x.y.z`, `=x.y.z` or `x.y.z` where `x`, `y` and `z` are integers.
+		re := regexp.MustCompile(`^=?\s*([0-9.]+)\s*$`)
+		matched := re.FindStringSubmatch(requiredVersionSetting)
+		if len(matched) == 0 {
+			log.Debug("Did not specify exact version in terraform configuration, found %q", requiredVersionSetting)
+			return nil
+		}
+		tfVersions = []string{matched[1]}
+	}
+
+	constraint, _ := semver.NewConstraint(requiredVersionSetting)
+	versions := make([]*semver.Version, len(tfVersions))
+
+	for i, tfvals := range tfVersions {
+		newVersion, err := semver.NewVersion(tfvals) //NewVersion parses a given version and returns an instance of Version or an error if unable to parse the version.
+		if err == nil {
+			versions[i] = newVersion
+		}
+	}
+
+	if len(versions) == 0 {
+		log.Debug("did not specify exact valid version in terraform configuration, found %q", requiredVersionSetting)
+		return nil
+	}
+
+	sort.Sort(sort.Reverse(semver.Collection(versions)))
+
+	for _, element := range versions {
+		if constraint.Check(element) { // Validate a version against a constraint
+			tfversionStr := element.String()
+			if lib.ValidVersionFormat(tfversionStr) { //check if version format is correct
+				tfversion, _ := version.NewVersion(tfversionStr)
+				log.Info("detected module requires version: %s", tfversionStr)
+				return tfversion
+			}
+		}
+	}
+	log.Debug("could not match any valid terraform version with %q", requiredVersionSetting)
+	return nil
+}
+
 // See Client.EnsureVersion.
 func (c *DefaultClient) EnsureVersion(log logging.SimpleLogging, v *version.Version) error {
 	if v == nil {
@@ -270,7 +364,7 @@ func (c *DefaultClient) EnsureVersion(log logging.SimpleLogging, v *version.Vers
 
 	var err error
 	c.versionsLock.Lock()
-	_, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir, c.downloadBaseURL)
+	_, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir, c.downloadBaseURL, c.downloadsAllowed)
 	c.versionsLock.Unlock()
 	if err != nil {
 		return err
@@ -347,7 +441,7 @@ func (c *DefaultClient) prepCmd(log logging.SimpleLogging, v *version.Version, w
 	} else {
 		var err error
 		c.versionsLock.Lock()
-		binPath, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir, c.downloadBaseURL)
+		binPath, err = ensureVersion(log, c.downloader, c.versions, v, c.binDir, c.downloadBaseURL, c.downloadsAllowed)
 		c.versionsLock.Unlock()
 		if err != nil {
 			return "", nil, err
@@ -418,7 +512,7 @@ func MustConstraint(v string) version.Constraints {
 
 // ensureVersion returns the path to a terraform binary of version v.
 // It will download this version if we don't have it.
-func ensureVersion(log logging.SimpleLogging, dl Downloader, versions map[string]string, v *version.Version, binDir string, downloadURL string) (string, error) {
+func ensureVersion(log logging.SimpleLogging, dl Downloader, versions map[string]string, v *version.Version, binDir string, downloadURL string, downloadsAllowed bool) (string, error) {
 	if binPath, ok := versions[v.String()]; ok {
 		return binPath, nil
 	}
@@ -439,6 +533,10 @@ func ensureVersion(log logging.SimpleLogging, dl Downloader, versions map[string
 		versions[v.String()] = dest
 		return dest, nil
 	}
+	if !downloadsAllowed {
+		return "", fmt.Errorf("could not find terraform version %s in PATH or %s, and downloads are disabled", v.String(), binDir)
+	}
+
 	log.Info("could not find terraform version %s in PATH or %s, downloading from %s", v.String(), binDir, downloadURL)
 	urlPrefix := fmt.Sprintf("%s/terraform/%s/terraform_%s", downloadURL, v.String(), v.String())
 	binURL := fmt.Sprintf("%s_%s_%s.zip", urlPrefix, runtime.GOOS, runtime.GOARCH)