diff --git a/.github/gobenchdata-checks.yml b/.github/gobenchdata-checks.yml index 43cb3864..5a4a2b00 100644 --- a/.github/gobenchdata-checks.yml +++ b/.github/gobenchdata-checks.yml @@ -4,21 +4,21 @@ checks: benchmarks: [BenchmarkInitializeFolder_basic/local-single-module-no-provider] diff: current.NsPerOp / 1000000 # ms thresholds: - min: 25 - max: 55 + min: 3 + max: 20 - package: ./internal/langserver/handlers name: local-single-submodule-no-provider benchmarks: [BenchmarkInitializeFolder_basic/local-single-submodule-no-provider] diff: current.NsPerOp / 1000000 # ms thresholds: - min: 140 - max: 310 + min: 100 + max: 250 - package: ./internal/langserver/handlers name: local-single-module-random benchmarks: [BenchmarkInitializeFolder_basic/local-single-module-random] diff: current.NsPerOp / 1000000 # ms thresholds: - min: 140 + min: 100 max: 300 - package: ./internal/langserver/handlers name: local-single-module-aws @@ -47,35 +47,35 @@ checks: diff: current.NsPerOp / 1000000 # ms thresholds: min: 1400 - max: 2200 + max: 5000 - package: ./internal/langserver/handlers name: google-project benchmarks: [BenchmarkInitializeFolder_basic/google-project] diff: current.NsPerOp / 1000000 # ms thresholds: min: 1570 - max: 2450 + max: 9000 - package: ./internal/langserver/handlers name: google-network benchmarks: [BenchmarkInitializeFolder_basic/google-network] diff: current.NsPerOp / 1000000 # ms thresholds: min: 1430 - max: 2700 + max: 15000 - package: ./internal/langserver/handlers name: google-gke benchmarks: [BenchmarkInitializeFolder_basic/google-gke] diff: current.NsPerOp / 1000000 # ms thresholds: min: 1500 - max: 5100 + max: 20000 - package: ./internal/langserver/handlers name: k8s-metrics-server benchmarks: [BenchmarkInitializeFolder_basic/k8s-metrics-server] diff: current.NsPerOp / 1000000 # ms thresholds: min: 1000 - max: 3200 + max: 4000 - package: ./internal/langserver/handlers name: k8s-dashboard benchmarks: [BenchmarkInitializeFolder_basic/k8s-dashboard] diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 4191ddf4..e16c5adb 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -41,7 +41,7 @@ jobs: -bench=InitializeFolder_basic \ -run=^# \ -benchtime=60s \ - -timeout=30m | tee ${{ runner.temp }}/benchmarks.txt + -timeout=60m | tee ${{ runner.temp }}/benchmarks.txt - name: Evaluate benchmarks id: bench-eval diff --git a/internal/indexer/document_change.go b/internal/indexer/document_change.go index b0840fb0..03552e80 100644 --- a/internal/indexer/document_change.go +++ b/internal/indexer/document_change.go @@ -15,16 +15,17 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro parseId, err := idx.jobStore.EnqueueJob(job.Job{ Dir: modHandle, Func: func(ctx context.Context) error { - return module.ParseModuleConfiguration(idx.fs, idx.modStore, modHandle.Path()) + return module.ParseModuleConfiguration(ctx, idx.fs, idx.modStore, modHandle.Path()) }, - Type: op.OpTypeParseModuleConfiguration.String(), + Type: op.OpTypeParseModuleConfiguration.String(), + IgnoreState: true, }) if err != nil { return ids, err } ids = append(ids, parseId) - modIds, err := idx.decodeModule(modHandle, job.IDs{parseId}) + modIds, err := idx.decodeModule(modHandle, job.IDs{parseId}, true) if err != nil { return ids, err } @@ -33,9 +34,10 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro parseVarsId, err := idx.jobStore.EnqueueJob(job.Job{ Dir: modHandle, Func: func(ctx context.Context) error { - return module.ParseVariables(idx.fs, idx.modStore, modHandle.Path()) + return module.ParseVariables(ctx, idx.fs, idx.modStore, modHandle.Path()) }, - Type: op.OpTypeParseVariables.String(), + Type: op.OpTypeParseVariables.String(), + IgnoreState: true, }) if err != nil { return ids, err @@ -47,8 +49,9 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro Func: func(ctx context.Context) error { return module.DecodeVarsReferences(ctx, idx.modStore, idx.schemaStore, modHandle.Path()) }, - Type: op.OpTypeDecodeVarsReferences.String(), - DependsOn: job.IDs{parseVarsId}, + Type: op.OpTypeDecodeVarsReferences.String(), + DependsOn: job.IDs{parseVarsId}, + IgnoreState: true, }) if err != nil { return ids, err @@ -58,16 +61,17 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro return ids, nil } -func (idx *Indexer) decodeModule(modHandle document.DirHandle, dependsOn job.IDs) (job.IDs, error) { +func (idx *Indexer) decodeModule(modHandle document.DirHandle, dependsOn job.IDs, ignoreState bool) (job.IDs, error) { ids := make(job.IDs, 0) metaId, err := idx.jobStore.EnqueueJob(job.Job{ Dir: modHandle, Func: func(ctx context.Context) error { - return module.LoadModuleMetadata(idx.modStore, modHandle.Path()) + return module.LoadModuleMetadata(ctx, idx.modStore, modHandle.Path()) }, - Type: op.OpTypeLoadModuleMetadata.String(), - DependsOn: dependsOn, + Type: op.OpTypeLoadModuleMetadata.String(), + DependsOn: dependsOn, + IgnoreState: ignoreState, }) if err != nil { return ids, err @@ -79,8 +83,9 @@ func (idx *Indexer) decodeModule(modHandle document.DirHandle, dependsOn job.IDs Func: func(ctx context.Context) error { return module.DecodeReferenceTargets(ctx, idx.modStore, idx.schemaStore, modHandle.Path()) }, - Type: op.OpTypeDecodeReferenceTargets.String(), - DependsOn: job.IDs{metaId}, + Type: op.OpTypeDecodeReferenceTargets.String(), + DependsOn: job.IDs{metaId}, + IgnoreState: ignoreState, }) if err != nil { return ids, err @@ -92,8 +97,9 @@ func (idx *Indexer) decodeModule(modHandle document.DirHandle, dependsOn job.IDs Func: func(ctx context.Context) error { return module.DecodeReferenceOrigins(ctx, idx.modStore, idx.schemaStore, modHandle.Path()) }, - Type: op.OpTypeDecodeReferenceOrigins.String(), - DependsOn: job.IDs{metaId}, + Type: op.OpTypeDecodeReferenceOrigins.String(), + DependsOn: job.IDs{metaId}, + IgnoreState: ignoreState, }) if err != nil { return ids, err diff --git a/internal/indexer/document_open.go b/internal/indexer/document_open.go index f6fc80c2..bcd3902c 100644 --- a/internal/indexer/document_open.go +++ b/internal/indexer/document_open.go @@ -40,16 +40,17 @@ func (idx *Indexer) DocumentOpened(modHandle document.DirHandle) (job.IDs, error parseId, err := idx.jobStore.EnqueueJob(job.Job{ Dir: modHandle, Func: func(ctx context.Context) error { - return module.ParseModuleConfiguration(idx.fs, idx.modStore, modHandle.Path()) + return module.ParseModuleConfiguration(ctx, idx.fs, idx.modStore, modHandle.Path()) }, - Type: op.OpTypeParseModuleConfiguration.String(), + Type: op.OpTypeParseModuleConfiguration.String(), + IgnoreState: true, }) if err != nil { return ids, err } ids = append(ids, parseId) - modIds, err := idx.decodeModule(modHandle, job.IDs{parseId}) + modIds, err := idx.decodeModule(modHandle, job.IDs{parseId}, true) if err != nil { return ids, err } @@ -58,9 +59,10 @@ func (idx *Indexer) DocumentOpened(modHandle document.DirHandle) (job.IDs, error parseVarsId, err := idx.jobStore.EnqueueJob(job.Job{ Dir: modHandle, Func: func(ctx context.Context) error { - return module.ParseVariables(idx.fs, idx.modStore, modHandle.Path()) + return module.ParseVariables(ctx, idx.fs, idx.modStore, modHandle.Path()) }, - Type: op.OpTypeParseVariables.String(), + Type: op.OpTypeParseVariables.String(), + IgnoreState: true, }) if err != nil { return ids, err diff --git a/internal/indexer/module_calls.go b/internal/indexer/module_calls.go index 6537b210..960def83 100644 --- a/internal/indexer/module_calls.go +++ b/internal/indexer/module_calls.go @@ -11,7 +11,7 @@ import ( op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation" ) -func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (job.IDs, error) { +func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle, ignoreState bool) (job.IDs, error) { jobIds := make(job.IDs, 0) moduleCalls, err := idx.modStore.ModuleCalls(modHandle.Path()) @@ -43,9 +43,10 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo parseId, err := idx.jobStore.EnqueueJob(job.Job{ Dir: mcHandle, Func: func(ctx context.Context) error { - return module.ParseModuleConfiguration(idx.fs, idx.modStore, mcPath) + return module.ParseModuleConfiguration(ctx, idx.fs, idx.modStore, mcPath) }, - Type: op.OpTypeParseModuleConfiguration.String(), + Type: op.OpTypeParseModuleConfiguration.String(), + IgnoreState: ignoreState, }) if err != nil { multierror.Append(errs, err) @@ -60,9 +61,10 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo Dir: mcHandle, Type: op.OpTypeLoadModuleMetadata.String(), Func: func(ctx context.Context) error { - return module.LoadModuleMetadata(idx.modStore, mcPath) + return module.LoadModuleMetadata(ctx, idx.modStore, mcPath) }, - DependsOn: job.IDs{parseId}, + DependsOn: job.IDs{parseId}, + IgnoreState: ignoreState, }) if err != nil { multierror.Append(errs, err) @@ -73,7 +75,7 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo } if parseId != "" { - ids, err := idx.collectReferences(mcHandle, refCollectionDeps) + ids, err := idx.collectReferences(mcHandle, refCollectionDeps, ignoreState) if err != nil { multierror.Append(errs, err) } else { @@ -84,9 +86,10 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo varsParseId, err := idx.jobStore.EnqueueJob(job.Job{ Dir: mcHandle, Func: func(ctx context.Context) error { - return module.ParseVariables(idx.fs, idx.modStore, mcPath) + return module.ParseVariables(ctx, idx.fs, idx.modStore, mcPath) }, - Type: op.OpTypeParseVariables.String(), + Type: op.OpTypeParseVariables.String(), + IgnoreState: ignoreState, }) if err != nil { multierror.Append(errs, err) @@ -100,8 +103,9 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo Func: func(ctx context.Context) error { return module.DecodeVarsReferences(ctx, idx.modStore, idx.schemaStore, mcPath) }, - Type: op.OpTypeDecodeVarsReferences.String(), - DependsOn: job.IDs{varsParseId}, + Type: op.OpTypeDecodeVarsReferences.String(), + DependsOn: job.IDs{varsParseId}, + IgnoreState: ignoreState, }) if err != nil { multierror.Append(errs, err) @@ -114,7 +118,7 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo return jobIds, errs.ErrorOrNil() } -func (idx *Indexer) collectReferences(modHandle document.DirHandle, dependsOn job.IDs) (job.IDs, error) { +func (idx *Indexer) collectReferences(modHandle document.DirHandle, dependsOn job.IDs, ignoreState bool) (job.IDs, error) { ids := make(job.IDs, 0) var errs *multierror.Error @@ -124,8 +128,9 @@ func (idx *Indexer) collectReferences(modHandle document.DirHandle, dependsOn jo Func: func(ctx context.Context) error { return module.DecodeReferenceTargets(ctx, idx.modStore, idx.schemaStore, modHandle.Path()) }, - Type: op.OpTypeDecodeReferenceTargets.String(), - DependsOn: dependsOn, + Type: op.OpTypeDecodeReferenceTargets.String(), + DependsOn: dependsOn, + IgnoreState: ignoreState, }) if err != nil { errs = multierror.Append(errs, err) @@ -138,8 +143,9 @@ func (idx *Indexer) collectReferences(modHandle document.DirHandle, dependsOn jo Func: func(ctx context.Context) error { return module.DecodeReferenceOrigins(ctx, idx.modStore, idx.schemaStore, modHandle.Path()) }, - Type: op.OpTypeDecodeReferenceOrigins.String(), - DependsOn: dependsOn, + Type: op.OpTypeDecodeReferenceOrigins.String(), + DependsOn: dependsOn, + IgnoreState: ignoreState, }) if err != nil { errs = multierror.Append(errs, err) diff --git a/internal/indexer/walker.go b/internal/indexer/walker.go index b803a37e..94d633a4 100644 --- a/internal/indexer/walker.go +++ b/internal/indexer/walker.go @@ -22,7 +22,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand parseId, err := idx.jobStore.EnqueueJob(job.Job{ Dir: modHandle, Func: func(ctx context.Context) error { - return module.ParseModuleConfiguration(idx.fs, idx.modStore, modHandle.Path()) + return module.ParseModuleConfiguration(ctx, idx.fs, idx.modStore, modHandle.Path()) }, Type: op.OpTypeParseModuleConfiguration.String(), }) @@ -40,7 +40,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand Dir: modHandle, Type: op.OpTypeLoadModuleMetadata.String(), Func: func(ctx context.Context) error { - return module.LoadModuleMetadata(idx.modStore, modHandle.Path()) + return module.LoadModuleMetadata(ctx, idx.modStore, modHandle.Path()) }, DependsOn: job.IDs{parseId}, }) @@ -56,7 +56,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand parseVarsId, err := idx.jobStore.EnqueueJob(job.Job{ Dir: modHandle, Func: func(ctx context.Context) error { - return module.ParseVariables(idx.fs, idx.modStore, modHandle.Path()) + return module.ParseVariables(ctx, idx.fs, idx.modStore, modHandle.Path()) }, Type: op.OpTypeParseVariables.String(), }) @@ -93,11 +93,11 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand modManifestId, err = idx.jobStore.EnqueueJob(job.Job{ Dir: modHandle, Func: func(ctx context.Context) error { - return module.ParseModuleManifest(idx.fs, idx.modStore, modHandle.Path()) + return module.ParseModuleManifest(ctx, idx.fs, idx.modStore, modHandle.Path()) }, Type: op.OpTypeParseModuleManifest.String(), Defer: func(ctx context.Context, jobErr error) (job.IDs, error) { - return idx.decodeInstalledModuleCalls(modHandle) + return idx.decodeInstalledModuleCalls(modHandle, false) }, }) if err != nil { @@ -111,47 +111,31 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand } if dataDir.PluginLockFilePath != "" { - pSchemaId, err := idx.jobStore.EnqueueJob(job.Job{ + dependsOn := make(job.IDs, 0) + pSchemaVerId, err := idx.jobStore.EnqueueJob(job.Job{ Dir: modHandle, Func: func(ctx context.Context) error { - return module.ParseProviderVersions(idx.fs, idx.modStore, modHandle.Path()) + return module.ParseProviderVersions(ctx, idx.fs, idx.modStore, modHandle.Path()) }, Type: op.OpTypeParseProviderVersions.String(), DependsOn: providerVersionDeps, - Defer: func(ctx context.Context, jobErr error) (job.IDs, error) { - ids := make(job.IDs, 0) - - pReqs, err := idx.modStore.ProviderRequirementsForModule(modHandle.Path()) - if err != nil { - return ids, err - } - - exist, err := idx.schemaStore.AllSchemasExist(pReqs) - if err != nil { - return ids, err - } - if exist { - idx.logger.Printf("Avoiding obtaining schemas as they all exist: %#v", pReqs) - // avoid obtaining schemas if we already have it - return ids, nil - } - idx.logger.Printf("Obtaining schemas for: %#v", pReqs) - - id, err := idx.jobStore.EnqueueJob(job.Job{ - Dir: modHandle, - Func: func(ctx context.Context) error { - ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory) - return module.ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path()) - }, - Type: op.OpTypeObtainSchema.String(), - }) - if err != nil { - return ids, err - } - ids = append(ids, id) + }) + if err != nil { + errs = multierror.Append(errs, err) + } else { + ids = append(ids, pSchemaVerId) + dependsOn = append(dependsOn, pSchemaVerId) + refCollectionDeps = append(refCollectionDeps, pSchemaVerId) + } - return ids, nil + pSchemaId, err := idx.jobStore.EnqueueJob(job.Job{ + Dir: modHandle, + Func: func(ctx context.Context) error { + ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory) + return module.ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path()) }, + Type: op.OpTypeObtainSchema.String(), + DependsOn: dependsOn, }) if err != nil { errs = multierror.Append(errs, err) @@ -162,7 +146,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand } if parseId != "" { - rIds, err := idx.collectReferences(modHandle, refCollectionDeps) + rIds, err := idx.collectReferences(modHandle, refCollectionDeps, false) if err != nil { errs = multierror.Append(errs, err) } else { diff --git a/internal/indexer/watcher.go b/internal/indexer/watcher.go index 6fa667af..9142c257 100644 --- a/internal/indexer/watcher.go +++ b/internal/indexer/watcher.go @@ -3,6 +3,7 @@ package indexer import ( "context" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/terraform-ls/internal/document" "github.com/hashicorp/terraform-ls/internal/job" "github.com/hashicorp/terraform-ls/internal/terraform/exec" @@ -16,11 +17,12 @@ func (idx *Indexer) ModuleManifestChanged(ctx context.Context, modHandle documen modManifestId, err := idx.jobStore.EnqueueJob(job.Job{ Dir: modHandle, Func: func(ctx context.Context) error { - return module.ParseModuleManifest(idx.fs, idx.modStore, modHandle.Path()) + return module.ParseModuleManifest(ctx, idx.fs, idx.modStore, modHandle.Path()) }, - Type: op.OpTypeParseModuleManifest.String(), + Type: op.OpTypeParseModuleManifest.String(), + IgnoreState: true, Defer: func(ctx context.Context, jobErr error) (job.IDs, error) { - return idx.decodeInstalledModuleCalls(modHandle) + return idx.decodeInstalledModuleCalls(modHandle, true) }, }) if err != nil { @@ -33,55 +35,39 @@ func (idx *Indexer) ModuleManifestChanged(ctx context.Context, modHandle documen func (idx *Indexer) PluginLockChanged(ctx context.Context, modHandle document.DirHandle) (job.IDs, error) { ids := make(job.IDs, 0) + dependsOn := make(job.IDs, 0) + var errs *multierror.Error - id, err := idx.jobStore.EnqueueJob(job.Job{ + pSchemaVerId, err := idx.jobStore.EnqueueJob(job.Job{ Dir: modHandle, Func: func(ctx context.Context) error { - return module.ParseProviderVersions(idx.fs, idx.modStore, modHandle.Path()) + return module.ParseProviderVersions(ctx, idx.fs, idx.modStore, modHandle.Path()) }, - Defer: func(ctx context.Context, jobErr error) (job.IDs, error) { - ids := make(job.IDs, 0) - - mod, err := idx.modStore.ModuleByPath(modHandle.Path()) - if err != nil { - return ids, err - } - - exist, err := idx.schemaStore.AllSchemasExist(mod.Meta.ProviderRequirements) - if err != nil { - return ids, err - } - if exist { - // avoid obtaining schemas if we already have it - return ids, nil - } - - id, err := idx.jobStore.EnqueueJob(job.Job{ - Dir: modHandle, - Func: func(ctx context.Context) error { - ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory) - eo, ok := exec.ExecutorOptsFromContext(ctx) - if ok { - ctx = exec.WithExecutorOpts(ctx, eo) - } - - return module.ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path()) - }, - Type: op.OpTypeObtainSchema.String(), - }) - if err != nil { - return ids, err - } - ids = append(ids, id) + IgnoreState: true, + Type: op.OpTypeParseProviderVersions.String(), + }) + if err != nil { + errs = multierror.Append(errs, err) + } else { + ids = append(ids, pSchemaVerId) + dependsOn = append(dependsOn, pSchemaVerId) + } - return ids, nil + pSchemaId, err := idx.jobStore.EnqueueJob(job.Job{ + Dir: modHandle, + Func: func(ctx context.Context) error { + ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory) + return module.ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path()) }, - Type: op.OpTypeParseProviderVersions.String(), + IgnoreState: true, + Type: op.OpTypeObtainSchema.String(), + DependsOn: dependsOn, }) if err != nil { - return ids, err + errs = multierror.Append(errs, err) + } else { + ids = append(ids, pSchemaId) } - ids = append(ids, id) - return ids, nil + return ids, errs.ErrorOrNil() } diff --git a/internal/job/ignore_state.go b/internal/job/ignore_state.go new file mode 100644 index 00000000..3bc946a9 --- /dev/null +++ b/internal/job/ignore_state.go @@ -0,0 +1,30 @@ +package job + +import ( + "context" + "fmt" + + "github.com/hashicorp/terraform-ls/internal/document" +) + +type ignoreState struct{} + +func IgnoreState(ctx context.Context) bool { + v, ok := ctx.Value(ignoreState{}).(bool) + if !ok { + return false + } + return v +} + +func WithIgnoreState(ctx context.Context, ignore bool) context.Context { + return context.WithValue(ctx, ignoreState{}, ignore) +} + +type StateNotChangedErr struct { + Dir document.DirHandle +} + +func (e StateNotChangedErr) Error() string { + return fmt.Sprintf("%s: state not changed", e.Dir.URI) +} diff --git a/internal/job/job.go b/internal/job/job.go index 6ad1cc8e..a504e752 100644 --- a/internal/job/job.go +++ b/internal/job/job.go @@ -32,6 +32,11 @@ type Job struct { // This will be taken into account when scheduling, so that only // jobs with no dependencies are dispatched at any time. DependsOn IDs + + // IgnoreState indicates to the job (as defined by Func) + // whether to ignore existing state, i.e. whether to invalidate cache. + // It is up to [Func] to read this flag from ctx and reflect it. + IgnoreState bool } // DeferFunc represents a deferred function scheduling more jobs @@ -41,12 +46,13 @@ type DeferFunc func(ctx context.Context, jobErr error) (IDs, error) func (job Job) Copy() Job { return Job{ - Func: job.Func, - Dir: job.Dir, - Type: job.Type, - Priority: job.Priority, - Defer: job.Defer, - DependsOn: job.DependsOn.Copy(), + Func: job.Func, + Dir: job.Dir, + Type: job.Type, + Priority: job.Priority, + Defer: job.Defer, + IgnoreState: job.IgnoreState, + DependsOn: job.DependsOn.Copy(), } } diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 902f6b4f..6d0dac97 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -65,6 +65,8 @@ func (s *Scheduler) eval(ctx context.Context) { return } + ctx = job.WithIgnoreState(ctx, nextJob.IgnoreState) + jobErr := nextJob.Func(ctx) deferredJobIds := make(job.IDs, 0) diff --git a/internal/scheduler/scheduler_test.go b/internal/scheduler/scheduler_test.go index 7fe82141..f11e8c91 100644 --- a/internal/scheduler/scheduler_test.go +++ b/internal/scheduler/scheduler_test.go @@ -18,6 +18,67 @@ import ( "github.com/hashicorp/terraform-ls/internal/state" ) +func TestScheduler_withIgnoreExistingState(t *testing.T) { + ss, err := state.NewStateStore() + if err != nil { + t.Fatal(err) + } + + tmpDir := t.TempDir() + + ctx := context.Background() + + s := NewScheduler(ss.JobStore, 1, job.LowPriority) + s.SetLogger(testLogger()) + s.Start(ctx) + t.Cleanup(func() { + s.Stop() + }) + + var stateIgnored int64 = 0 + firstJobId, err := ss.JobStore.EnqueueJob(job.Job{ + Func: func(ctx context.Context) error { + if job.IgnoreState(ctx) { + atomic.AddInt64(&stateIgnored, 1) + } + return nil + }, + Dir: document.DirHandleFromPath(tmpDir), + Type: "test-type", + IgnoreState: true, + }) + if err != nil { + t.Fatal(err) + } + + var stateNotIgnored int64 = 0 + secondJobId, err := ss.JobStore.EnqueueJob(job.Job{ + Func: func(ctx context.Context) error { + if !job.IgnoreState(ctx) { + atomic.AddInt64(&stateNotIgnored, 1) + } + return nil + }, + Dir: document.DirHandleFromPath(tmpDir), + Type: "test-type", + }) + if err != nil { + t.Fatal(err) + } + + err = ss.JobStore.WaitForJobs(ctx, firstJobId, secondJobId) + if err != nil { + t.Fatal(err) + } + + if stateIgnored != 1 { + t.Fatalf("expected state to be ignored once, given: %d", stateIgnored) + } + if stateNotIgnored != 1 { + t.Fatalf("expected state not to be ignored once, given: %d", stateNotIgnored) + } +} + func TestScheduler_closedOnly(t *testing.T) { ss, err := state.NewStateStore() if err != nil { diff --git a/internal/state/jobs.go b/internal/state/jobs.go index 425a5ae8..26a7ae8e 100644 --- a/internal/state/jobs.go +++ b/internal/state/jobs.go @@ -58,22 +58,6 @@ const ( ) func (js *JobStore) EnqueueJob(newJob job.Job) (job.ID, error) { - jobID, queued, err := js.jobExists(newJob, StateQueued) - if err != nil { - return "", err - } - if queued { - return jobID, nil - } - - jobID, running, err := js.jobExists(newJob, StateRunning) - if err != nil { - return "", err - } - if running { - return jobID, nil - } - txn := js.db.Txn(true) defer txn.Abort() @@ -98,13 +82,13 @@ func (js *JobStore) EnqueueJob(newJob job.Job) (job.ID, error) { State: StateQueued, } - err = txn.Insert(js.tableName, sJob) + err := txn.Insert(js.tableName, sJob) if err != nil { return "", fmt.Errorf("failed to insert new job: %w", err) } - js.logger.Printf("JOBS: Enqueueing new job %q: %q for %q (IsDirOpen: %t)", - sJob.ID, sJob.Type, sJob.Dir, sJob.IsDirOpen) + js.logger.Printf("JOBS: Enqueueing new job %q: %q for %q (IsDirOpen: %t, IgnoreState: %t)", + sJob.ID, sJob.Type, sJob.Dir, sJob.IsDirOpen, sJob.IgnoreState) txn.Commit() diff --git a/internal/terraform/module/module_ops.go b/internal/terraform/module/module_ops.go index 8b672eab..06704063 100644 --- a/internal/terraform/module/module_ops.go +++ b/internal/terraform/module/module_ops.go @@ -10,6 +10,8 @@ import ( "github.com/hashicorp/hcl-lang/decoder" "github.com/hashicorp/hcl-lang/lang" idecoder "github.com/hashicorp/terraform-ls/internal/decoder" + "github.com/hashicorp/terraform-ls/internal/document" + "github.com/hashicorp/terraform-ls/internal/job" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" "github.com/hashicorp/terraform-ls/internal/registry" "github.com/hashicorp/terraform-ls/internal/state" @@ -58,6 +60,11 @@ func GetTerraformVersion(ctx context.Context, modStore *state.ModuleStore, modPa return err } + // Avoid getting version if getting is already in progress or already known + if mod.TerraformVersionState != op.OpStateUnknown && !job.IgnoreState(ctx) { + return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)} + } + err = modStore.SetTerraformVersionState(modPath, op.OpStateLoading) if err != nil { return err @@ -108,7 +115,26 @@ func ObtainSchema(ctx context.Context, modStore *state.ModuleStore, schemaStore return err } - tfExec, err := TerraformExecutorForModule(ctx, mod.Path) + // Avoid obtaining schema if it is already in progress or already known + if mod.ProviderSchemaState != op.OpStateUnknown && !job.IgnoreState(ctx) { + return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)} + } + + pReqs, err := modStore.ProviderRequirementsForModule(modPath) + if err != nil { + return err + } + + exist, err := schemaStore.AllSchemasExist(pReqs) + if err != nil { + return err + } + if exist { + // avoid obtaining schemas if we already have it + return nil + } + + tfExec, err := TerraformExecutorForModule(ctx, modPath) if err != nil { sErr := modStore.FinishProviderSchemaLoading(modPath, err) if sErr != nil { @@ -148,8 +174,22 @@ func ObtainSchema(ctx context.Context, modStore *state.ModuleStore, schemaStore return nil } -func ParseModuleConfiguration(fs ReadOnlyFS, modStore *state.ModuleStore, modPath string) error { - err := modStore.SetModuleParsingState(modPath, op.OpStateLoading) +func ParseModuleConfiguration(ctx context.Context, fs ReadOnlyFS, modStore *state.ModuleStore, modPath string) error { + mod, err := modStore.ModuleByPath(modPath) + if err != nil { + return err + } + + // TODO: Avoid parsing if the content matches existing AST + + // TODO: Only parse the file that's being changed/opened, unless this is 1st-time parsing + + // Avoid parsing if it is already in progress or already known + if mod.ModuleParsingState != op.OpStateUnknown && !job.IgnoreState(ctx) { + return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)} + } + + err = modStore.SetModuleParsingState(modPath, op.OpStateLoading) if err != nil { return err } @@ -169,8 +209,22 @@ func ParseModuleConfiguration(fs ReadOnlyFS, modStore *state.ModuleStore, modPat return err } -func ParseVariables(fs ReadOnlyFS, modStore *state.ModuleStore, modPath string) error { - err := modStore.SetVarsParsingState(modPath, op.OpStateLoading) +func ParseVariables(ctx context.Context, fs ReadOnlyFS, modStore *state.ModuleStore, modPath string) error { + mod, err := modStore.ModuleByPath(modPath) + if err != nil { + return err + } + + // TODO: Avoid parsing if the content matches existing AST + + // TODO: Only parse the file that's being changed/opened, unless this is 1st-time parsing + + // Avoid parsing if it is already in progress or already known + if mod.VarsParsingState != op.OpStateUnknown && !job.IgnoreState(ctx) { + return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)} + } + + err = modStore.SetVarsParsingState(modPath, op.OpStateLoading) if err != nil { return err } @@ -190,8 +244,18 @@ func ParseVariables(fs ReadOnlyFS, modStore *state.ModuleStore, modPath string) return err } -func ParseModuleManifest(fs ReadOnlyFS, modStore *state.ModuleStore, modPath string) error { - err := modStore.SetModManifestState(modPath, op.OpStateLoading) +func ParseModuleManifest(ctx context.Context, fs ReadOnlyFS, modStore *state.ModuleStore, modPath string) error { + mod, err := modStore.ModuleByPath(modPath) + if err != nil { + return err + } + + // Avoid parsing if it is already in progress or already known + if mod.ModManifestState != op.OpStateUnknown && !job.IgnoreState(ctx) { + return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)} + } + + err = modStore.SetModManifestState(modPath, op.OpStateLoading) if err != nil { return err } @@ -224,8 +288,18 @@ func ParseModuleManifest(fs ReadOnlyFS, modStore *state.ModuleStore, modPath str return err } -func ParseProviderVersions(fs ReadOnlyFS, modStore *state.ModuleStore, modPath string) error { - err := modStore.SetInstalledProvidersState(modPath, op.OpStateLoading) +func ParseProviderVersions(ctx context.Context, fs ReadOnlyFS, modStore *state.ModuleStore, modPath string) error { + mod, err := modStore.ModuleByPath(modPath) + if err != nil { + return err + } + + // Avoid parsing if it is already in progress or already known + if mod.InstalledProvidersState != op.OpStateUnknown && !job.IgnoreState(ctx) { + return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)} + } + + err = modStore.SetInstalledProvidersState(modPath, op.OpStateLoading) if err != nil { return err } @@ -240,13 +314,20 @@ func ParseProviderVersions(fs ReadOnlyFS, modStore *state.ModuleStore, modPath s return err } -func LoadModuleMetadata(modStore *state.ModuleStore, modPath string) error { - err := modStore.SetMetaState(modPath, op.OpStateLoading) +func LoadModuleMetadata(ctx context.Context, modStore *state.ModuleStore, modPath string) error { + mod, err := modStore.ModuleByPath(modPath) if err != nil { return err } - mod, err := modStore.ModuleByPath(modPath) + // TODO: Avoid parsing if upstream (parsing) job reported no changes + + // Avoid parsing if it is already in progress or already known + if mod.MetaState != op.OpStateUnknown && !job.IgnoreState(ctx) { + return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)} + } + + err = modStore.SetMetaState(modPath, op.OpStateLoading) if err != nil { return err } @@ -279,7 +360,19 @@ func LoadModuleMetadata(modStore *state.ModuleStore, modPath string) error { } func DecodeReferenceTargets(ctx context.Context, modStore *state.ModuleStore, schemaReader state.SchemaReader, modPath string) error { - err := modStore.SetReferenceTargetsState(modPath, op.OpStateLoading) + mod, err := modStore.ModuleByPath(modPath) + if err != nil { + return err + } + + // TODO: Avoid collection if upstream jobs reported no changes + + // Avoid collection if it is already in progress or already done + if mod.RefTargetsState != op.OpStateUnknown && !job.IgnoreState(ctx) { + return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)} + } + + err = modStore.SetReferenceTargetsState(modPath, op.OpStateLoading) if err != nil { return err } @@ -310,7 +403,19 @@ func DecodeReferenceTargets(ctx context.Context, modStore *state.ModuleStore, sc } func DecodeReferenceOrigins(ctx context.Context, modStore *state.ModuleStore, schemaReader state.SchemaReader, modPath string) error { - err := modStore.SetReferenceOriginsState(modPath, op.OpStateLoading) + mod, err := modStore.ModuleByPath(modPath) + if err != nil { + return err + } + + // TODO: Avoid collection if upstream jobs reported no changes + + // Avoid collection if it is already in progress or already done + if mod.RefOriginsState != op.OpStateUnknown && !job.IgnoreState(ctx) { + return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)} + } + + err = modStore.SetReferenceOriginsState(modPath, op.OpStateLoading) if err != nil { return err } @@ -340,7 +445,19 @@ func DecodeReferenceOrigins(ctx context.Context, modStore *state.ModuleStore, sc } func DecodeVarsReferences(ctx context.Context, modStore *state.ModuleStore, schemaReader state.SchemaReader, modPath string) error { - err := modStore.SetVarsReferenceOriginsState(modPath, op.OpStateLoading) + mod, err := modStore.ModuleByPath(modPath) + if err != nil { + return err + } + + // TODO: Avoid collection if upstream (parsing) job reported no changes + + // Avoid collection if it is already in progress or already done + if mod.VarsRefOriginsState != op.OpStateUnknown && !job.IgnoreState(ctx) { + return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)} + } + + err = modStore.SetVarsReferenceOriginsState(modPath, op.OpStateLoading) if err != nil { return err } @@ -375,6 +492,8 @@ func GetModuleDataFromRegistry(ctx context.Context, regClient registry.Client, m return err } + // TODO: Avoid collection if upstream jobs (parsing, meta) reported no changes + var errs *multierror.Error for _, declaredModule := range calls.Declared { diff --git a/internal/terraform/module/module_ops_test.go b/internal/terraform/module/module_ops_test.go index 7bac65f8..b87c8fa1 100644 --- a/internal/terraform/module/module_ops_test.go +++ b/internal/terraform/module/module_ops_test.go @@ -43,12 +43,12 @@ func TestGetModuleDataFromRegistry_singleModule(t *testing.T) { } fs := filesystem.NewFilesystem(ss.DocumentStore) - err = ParseModuleConfiguration(fs, ss.Modules, modPath) + err = ParseModuleConfiguration(ctx, fs, ss.Modules, modPath) if err != nil { t.Fatal(err) } - err = LoadModuleMetadata(ss.Modules, modPath) + err = LoadModuleMetadata(ctx, ss.Modules, modPath) if err != nil { t.Fatal(err) } @@ -115,12 +115,12 @@ func TestGetModuleDataFromRegistry_moduleNotFound(t *testing.T) { } fs := filesystem.NewFilesystem(ss.DocumentStore) - err = ParseModuleConfiguration(fs, ss.Modules, modPath) + err = ParseModuleConfiguration(ctx, fs, ss.Modules, modPath) if err != nil { t.Fatal(err) } - err = LoadModuleMetadata(ss.Modules, modPath) + err = LoadModuleMetadata(ctx, ss.Modules, modPath) if err != nil { t.Fatal(err) } @@ -194,12 +194,12 @@ func TestGetModuleDataFromRegistry_apiTimeout(t *testing.T) { } fs := filesystem.NewFilesystem(ss.DocumentStore) - err = ParseModuleConfiguration(fs, ss.Modules, modPath) + err = ParseModuleConfiguration(ctx, fs, ss.Modules, modPath) if err != nil { t.Fatal(err) } - err = LoadModuleMetadata(ss.Modules, modPath) + err = LoadModuleMetadata(ctx, ss.Modules, modPath) if err != nil { t.Fatal(err) } @@ -368,7 +368,8 @@ func TestParseProviderVersions(t *testing.T) { t.Fatal(err) } - err = ParseProviderVersions(fs, ss.Modules, modPath) + ctx := context.Background() + err = ParseProviderVersions(ctx, fs, ss.Modules, modPath) if err != nil { t.Fatal(err) }