Pick core schema only based on required_version constraint (#1027)

* Pick schema based on constraints + account for too new/old version

* tests: Wait for all jobs from didOpen

* handlers: add tests for edge cases (old & new TF version)

* deps: bump tfschema to latest main rev

go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/hashicorp/terraform-exec v0.17.2
 	github.com/hashicorp/terraform-json v0.14.0
 	github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c
-	github.com/hashicorp/terraform-schema v0.0.0-20220805102629-49cc6ae5bfa3
+	github.com/hashicorp/terraform-schema v0.0.0-20220809100530-1ef9558a761c
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5
 	github.com/mitchellh/cli v1.1.4

go.sum

@@ -338,8 +338,8 @@ github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e
 github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM=
 github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c h1:D8aRO6+mTqHfLsK/BC3j5OAoogv1WLRWzY1AaTo3rBg=
 github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c/go.mod h1:Wn3Na71knbXc1G8Lh+yu/dQWWJeFQEpDeJMtWMtlmNI=
-github.com/hashicorp/terraform-schema v0.0.0-20220805102629-49cc6ae5bfa3 h1:1MdlLPlGxrTKSrkIf8ehb8b9TQuiMwvZSZ0CPlGQRQk=
-github.com/hashicorp/terraform-schema v0.0.0-20220805102629-49cc6ae5bfa3/go.mod h1:YfAL2BROQ9jq7ei+c8HqGyx0C1lc3xenQ/2FTr4AtKI=
+github.com/hashicorp/terraform-schema v0.0.0-20220809100530-1ef9558a761c h1:NlNpLN/GHk2ycoeGcfGbVFt0nMIC+c0WLvYPu8s8faU=
+github.com/hashicorp/terraform-schema v0.0.0-20220809100530-1ef9558a761c/go.mod h1:vZFv3NuAOpCxkP8j4d08ofVXVeCEjIbeKy/fNxYFbhg=
 github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 h1:HKLsbzeOsfXmKNpr3GiT18XAblV0BjCbzL8KQAMZGa0=
 github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734/go.mod h1:kNDNcF7sN4DocDLBkQYz73HGKwN1ANB1blq4lIYLYvg=
 github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=

internal/decoder/module_schema.go

@@ -9,30 +9,14 @@ import (
 )
 
 func schemaForModule(mod *state.Module, schemaReader state.SchemaReader, modReader state.ModuleCallReader) (*schema.BodySchema, error) {
-	var coreSchema *schema.BodySchema
-	coreRequirements := make(version.Constraints, 0)
-	if mod.TerraformVersion != nil {
-		var err error
-		coreSchema, err = tfschema.CoreModuleSchemaForVersion(mod.TerraformVersion)
-		if err != nil {
-			return nil, err
-		}
-		coreRequirements, err = version.NewConstraint(mod.TerraformVersion.String())
-		if err != nil {
-			return nil, err
-		}
-	} else {
-		coreSchema = tfschema.UniversalCoreModuleSchema()
-	}
-
-	sm := tfschema.NewSchemaMerger(coreSchema)
+	sm := tfschema.NewSchemaMerger(coreSchema(mod))
 	sm.SetSchemaReader(schemaReader)
 	sm.SetTerraformVersion(mod.TerraformVersion)
 	sm.SetModuleReader(modReader)
 
 	meta := &tfmodule.Meta{
 		Path:                 mod.Path,
-		CoreRequirements:     coreRequirements,
+		CoreRequirements:     mod.Meta.CoreRequirements,
 		ProviderRequirements: mod.Meta.ProviderRequirements,
 		ProviderReferences:   mod.Meta.ProviderReferences,
 		Variables:            mod.Meta.Variables,
@@ -42,3 +26,36 @@ func schemaForModule(mod *state.Module, schemaReader state.SchemaReader, modRead
 
 	return sm.SchemaForModule(meta)
 }
+
+func coreSchema(mod *state.Module) *schema.BodySchema {
+	if mod.TerraformVersion != nil {
+		s, err := tfschema.CoreModuleSchemaForVersion(mod.TerraformVersion)
+		if err == nil {
+			return s
+		}
+		if mod.TerraformVersion.LessThan(tfschema.OldestAvailableVersion) {
+			return mustCoreSchemaForVersion(tfschema.OldestAvailableVersion)
+		}
+
+		return mustCoreSchemaForVersion(tfschema.LatestAvailableVersion)
+	}
+
+	s, err := tfschema.CoreModuleSchemaForConstraint(mod.Meta.CoreRequirements)
+	if err == nil {
+		return s
+	}
+	if mod.Meta.CoreRequirements.Check(tfschema.OldestAvailableVersion) {
+		return mustCoreSchemaForVersion(tfschema.OldestAvailableVersion)
+	}
+
+	return mustCoreSchemaForVersion(tfschema.LatestAvailableVersion)
+}
+
+func mustCoreSchemaForVersion(v *version.Version) *schema.BodySchema {
+	s, err := tfschema.CoreModuleSchemaForVersion(v)
+	if err != nil {
+		// this should never happen
+		panic(err)
+	}
+	return s
+}

internal/indexer/document_open.go

@@ -0,0 +1,84 @@
+package indexer
+
+import (
+	"context"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/terraform-ls/internal/document"
+	"github.com/hashicorp/terraform-ls/internal/job"
+	"github.com/hashicorp/terraform-ls/internal/terraform/exec"
+	"github.com/hashicorp/terraform-ls/internal/terraform/module"
+	op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation"
+)
+
+func (idx *Indexer) DocumentOpened(modHandle document.DirHandle) (job.IDs, error) {
+	mod, err := idx.modStore.ModuleByPath(modHandle.Path())
+	if err != nil {
+		return nil, err
+	}
+
+	ids := make(job.IDs, 0)
+	var errs *multierror.Error
+
+	if mod.TerraformVersionState == op.OpStateUnknown {
+		_, err := idx.jobStore.EnqueueJob(job.Job{
+			Dir: modHandle,
+			Func: func(ctx context.Context) error {
+				ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
+				return module.GetTerraformVersion(ctx, idx.modStore, modHandle.Path())
+			},
+			Type: op.OpTypeGetTerraformVersion.String(),
+		})
+		if err != nil {
+			errs = multierror.Append(errs, err)
+		}
+		// Given that getting version may take time and we only use it to
+		// enhance the UX, we ignore the outcome (job ID) here
+		// to avoid delays when documents of new modules are open.
+	}
+
+	parseId, err := idx.jobStore.EnqueueJob(job.Job{
+		Dir: modHandle,
+		Func: func(ctx context.Context) error {
+			return module.ParseModuleConfiguration(idx.fs, idx.modStore, modHandle.Path())
+		},
+		Type: op.OpTypeParseModuleConfiguration.String(),
+	})
+	if err != nil {
+		return ids, err
+	}
+	ids = append(ids, parseId)
+
+	modIds, err := idx.decodeModule(modHandle, job.IDs{parseId})
+	if err != nil {
+		return ids, err
+	}
+	ids = append(ids, modIds...)
+
+	parseVarsId, err := idx.jobStore.EnqueueJob(job.Job{
+		Dir: modHandle,
+		Func: func(ctx context.Context) error {
+			return module.ParseVariables(idx.fs, idx.modStore, modHandle.Path())
+		},
+		Type: op.OpTypeParseVariables.String(),
+	})
+	if err != nil {
+		return ids, err
+	}
+	ids = append(ids, parseVarsId)
+
+	varsRefsId, err := idx.jobStore.EnqueueJob(job.Job{
+		Dir: modHandle,
+		Func: func(ctx context.Context) error {
+			return module.DecodeVarsReferences(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
+		},
+		Type:      op.OpTypeDecodeVarsReferences.String(),
+		DependsOn: job.IDs{parseVarsId},
+	})
+	if err != nil {
+		return ids, err
+	}
+	ids = append(ids, varsRefsId)
+
+	return ids, errs.ErrorOrNil()
+}

internal/indexer/walker.go

@@ -83,21 +83,6 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
 		}
 	}
 
-	tfVersionId, err := idx.jobStore.EnqueueJob(job.Job{
-		Dir: modHandle,
-		Func: func(ctx context.Context) error {
-			ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
-			return module.GetTerraformVersion(ctx, idx.modStore, modHandle.Path())
-		},
-		Type: op.OpTypeGetTerraformVersion.String(),
-	})
-	if err != nil {
-		errs = multierror.Append(errs, err)
-	} else {
-		ids = append(ids, tfVersionId)
-		refCollectionDeps = append(refCollectionDeps, tfVersionId)
-	}
-
 	dataDir := datadir.WalkDataDirOfModule(idx.fs, modHandle.Path())
 	idx.logger.Printf("parsed datadir: %#v", dataDir)
 

internal/indexer/watcher.go

@@ -83,23 +83,5 @@ func (idx *Indexer) PluginLockChanged(ctx context.Context, modHandle document.Di
 	}
 	ids = append(ids, id)
 
-	id, err = idx.jobStore.EnqueueJob(job.Job{
-		Dir: modHandle,
-		Func: func(ctx context.Context) error {
-			ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
-			eo, ok := exec.ExecutorOptsFromContext(ctx)
-			if ok {
-				ctx = exec.WithExecutorOpts(ctx, eo)
-			}
-
-			return module.GetTerraformVersion(ctx, idx.modStore, modHandle.Path())
-		},
-		Type: op.OpTypeGetTerraformVersion.String(),
-	})
-	if err != nil {
-		return ids, err
-	}
-	ids = append(ids, id)
-
 	return ids, nil
 }

internal/langserver/handlers/code_action_test.go

@@ -104,6 +104,8 @@ func TestLangServer_codeAction_basic(t *testing.T) {
 			"uri": "%s/main.tf"
 		}
 	}`, tmpDir.URI)})
+	waitForAllJobs(t, ss)
+
 	ls.CallAndExpectResponse(t, &langserver.CallRequest{
 		Method: "textDocument/codeAction",
 		ReqParams: fmt.Sprintf(`{
@@ -351,6 +353,7 @@ func TestLangServer_codeAction_no_code_action_requested(t *testing.T) {
 					"uri": "%s/main.tf"
 				}
 			}`, tmpDir.URI)})
+			waitForAllJobs(t, ss)
 
 			ls.CallAndExpectResponse(t, tt.request, tt.want)
 		})

internal/langserver/handlers/code_lens_test.go

@@ -41,7 +41,16 @@ func TestCodeLens_withoutOptIn(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	ls := langserver.NewLangServerMock(t, NewMockSession(nil))
+	ss, err := state.NewStateStore()
+	if err != nil {
+		t.Fatal(err)
+	}
+	wc := walker.NewWalkerCollector()
+
+	ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{
+		StateStore:      ss,
+		WalkerCollector: wc,
+	}))
 	stop := ls.Start(t)
 	defer stop()
 
@@ -52,6 +61,8 @@ func TestCodeLens_withoutOptIn(t *testing.T) {
 		"rootUri": %q,
 		"processId": 12345
 	}`, tmpDir.URI)})
+	waitForWalkerPath(t, ss, wc, tmpDir)
+
 	ls.Notify(t, &langserver.CallRequest{
 		Method:    "initialized",
 		ReqParams: "{}",
@@ -66,6 +77,8 @@ func TestCodeLens_withoutOptIn(t *testing.T) {
 			"uri": "%s/main.tf"
 		}
 	}`, tmpDir.URI)})
+	waitForAllJobs(t, ss)
+
 	ls.CallAndExpectResponse(t, &langserver.CallRequest{
 		Method: "textDocument/codeLens",
 		ReqParams: fmt.Sprintf(`{
@@ -170,6 +183,8 @@ output "test" {
 	value = var.test
 }
 `, tmpDir.URI)})
+	waitForAllJobs(t, ss)
+
 	ls.CallAndExpectResponse(t, &langserver.CallRequest{
 		Method: "textDocument/codeLens",
 		ReqParams: fmt.Sprintf(`{
@@ -282,6 +297,8 @@ variable "instances" {
   type = number
 }
 `, submodUri.URI)})
+	waitForAllJobs(t, ss)
+
 	ls.CallAndExpectResponse(t, &langserver.CallRequest{
 		Method: "textDocument/codeLens",
 		ReqParams: fmt.Sprintf(`{