Skip to content

Commit

Permalink
indexer: use IgnoreState
Browse files Browse the repository at this point in the history
  • Loading branch information
@radeksimko
radeksimko committed Aug 9, 2022
1 parent 1ac6cae commit 15f0482
Show file tree
Hide file tree
Showing 7 changed files with 85 additions and 39 deletions.
30 changes: 18 additions & 12 deletions internal/indexer/document_change.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,15 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro
Func: func(ctx context.Context) error {
return module.ParseModuleConfiguration(ctx, idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseModuleConfiguration.String(),
Type: op.OpTypeParseModuleConfiguration.String(),
IgnoreState: true,
})
if err != nil {
return ids, err
}
ids = append(ids, parseId)

modIds, err := idx.decodeModule(modHandle, job.IDs{parseId})
modIds, err := idx.decodeModule(modHandle, job.IDs{parseId}, true)
if err != nil {
return ids, err
}
Expand All @@ -35,7 +36,8 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro
Func: func(ctx context.Context) error {
return module.ParseVariables(ctx, idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseVariables.String(),
Type: op.OpTypeParseVariables.String(),
IgnoreState: true,
})
if err != nil {
return ids, err
Expand All @@ -47,8 +49,9 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro
Func: func(ctx context.Context) error {
return module.DecodeVarsReferences(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeDecodeVarsReferences.String(),
DependsOn: job.IDs{parseVarsId},
Type: op.OpTypeDecodeVarsReferences.String(),
DependsOn: job.IDs{parseVarsId},
IgnoreState: true,
})
if err != nil {
return ids, err
Expand All @@ -58,16 +61,17 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro
return ids, nil
}

func (idx *Indexer) decodeModule(modHandle document.DirHandle, dependsOn job.IDs) (job.IDs, error) {
func (idx *Indexer) decodeModule(modHandle document.DirHandle, dependsOn job.IDs, ignoreState bool) (job.IDs, error) {
ids := make(job.IDs, 0)

metaId, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.LoadModuleMetadata(ctx, idx.modStore, modHandle.Path())
},
Type: op.OpTypeLoadModuleMetadata.String(),
DependsOn: dependsOn,
Type: op.OpTypeLoadModuleMetadata.String(),
DependsOn: dependsOn,
IgnoreState: ignoreState,
})
if err != nil {
return ids, err
Expand All @@ -79,8 +83,9 @@ func (idx *Indexer) decodeModule(modHandle document.DirHandle, dependsOn job.IDs
Func: func(ctx context.Context) error {
return module.DecodeReferenceTargets(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeDecodeReferenceTargets.String(),
DependsOn: job.IDs{metaId},
Type: op.OpTypeDecodeReferenceTargets.String(),
DependsOn: job.IDs{metaId},
IgnoreState: ignoreState,
})
if err != nil {
return ids, err
Expand All @@ -92,8 +97,9 @@ func (idx *Indexer) decodeModule(modHandle document.DirHandle, dependsOn job.IDs
Func: func(ctx context.Context) error {
return module.DecodeReferenceOrigins(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeDecodeReferenceOrigins.String(),
DependsOn: job.IDs{metaId},
Type: op.OpTypeDecodeReferenceOrigins.String(),
DependsOn: job.IDs{metaId},
IgnoreState: ignoreState,
})
if err != nil {
return ids, err
Expand Down
8 changes: 5 additions & 3 deletions internal/indexer/document_open.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,15 @@ func (idx *Indexer) DocumentOpened(modHandle document.DirHandle) (job.IDs, error
Func: func(ctx context.Context) error {
return module.ParseModuleConfiguration(ctx, idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseModuleConfiguration.String(),
Type: op.OpTypeParseModuleConfiguration.String(),
IgnoreState: true,
})
if err != nil {
return ids, err
}
ids = append(ids, parseId)

modIds, err := idx.decodeModule(modHandle, job.IDs{parseId})
modIds, err := idx.decodeModule(modHandle, job.IDs{parseId}, true)
if err != nil {
return ids, err
}
Expand All @@ -60,7 +61,8 @@ func (idx *Indexer) DocumentOpened(modHandle document.DirHandle) (job.IDs, error
Func: func(ctx context.Context) error {
return module.ParseVariables(ctx, idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseVariables.String(),
Type: op.OpTypeParseVariables.String(),
IgnoreState: true,
})
if err != nil {
return ids, err
Expand Down
30 changes: 18 additions & 12 deletions internal/indexer/module_calls.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation"
)

func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (job.IDs, error) {
func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle, ignoreState bool) (job.IDs, error) {
jobIds := make(job.IDs, 0)

moduleCalls, err := idx.modStore.ModuleCalls(modHandle.Path())
Expand Down Expand Up @@ -45,7 +45,8 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo
Func: func(ctx context.Context) error {
return module.ParseModuleConfiguration(ctx, idx.fs, idx.modStore, mcPath)
},
Type: op.OpTypeParseModuleConfiguration.String(),
Type: op.OpTypeParseModuleConfiguration.String(),
IgnoreState: ignoreState,
})
if err != nil {
multierror.Append(errs, err)
Expand All @@ -62,7 +63,8 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo
Func: func(ctx context.Context) error {
return module.LoadModuleMetadata(ctx, idx.modStore, mcPath)
},
DependsOn: job.IDs{parseId},
DependsOn: job.IDs{parseId},
IgnoreState: ignoreState,
})
if err != nil {
multierror.Append(errs, err)
Expand All @@ -73,7 +75,7 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo
}

if parseId != "" {
ids, err := idx.collectReferences(mcHandle, refCollectionDeps)
ids, err := idx.collectReferences(mcHandle, refCollectionDeps, ignoreState)
if err != nil {
multierror.Append(errs, err)
} else {
Expand All @@ -86,7 +88,8 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo
Func: func(ctx context.Context) error {
return module.ParseVariables(ctx, idx.fs, idx.modStore, mcPath)
},
Type: op.OpTypeParseVariables.String(),
Type: op.OpTypeParseVariables.String(),
IgnoreState: ignoreState,
})
if err != nil {
multierror.Append(errs, err)
Expand All @@ -100,8 +103,9 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo
Func: func(ctx context.Context) error {
return module.DecodeVarsReferences(ctx, idx.modStore, idx.schemaStore, mcPath)
},
Type: op.OpTypeDecodeVarsReferences.String(),
DependsOn: job.IDs{varsParseId},
Type: op.OpTypeDecodeVarsReferences.String(),
DependsOn: job.IDs{varsParseId},
IgnoreState: ignoreState,
})
if err != nil {
multierror.Append(errs, err)
Expand All @@ -114,7 +118,7 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo
return jobIds, errs.ErrorOrNil()
}

func (idx *Indexer) collectReferences(modHandle document.DirHandle, dependsOn job.IDs) (job.IDs, error) {
func (idx *Indexer) collectReferences(modHandle document.DirHandle, dependsOn job.IDs, ignoreState bool) (job.IDs, error) {
ids := make(job.IDs, 0)

var errs *multierror.Error
Expand All @@ -124,8 +128,9 @@ func (idx *Indexer) collectReferences(modHandle document.DirHandle, dependsOn jo
Func: func(ctx context.Context) error {
return module.DecodeReferenceTargets(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeDecodeReferenceTargets.String(),
DependsOn: dependsOn,
Type: op.OpTypeDecodeReferenceTargets.String(),
DependsOn: dependsOn,
IgnoreState: ignoreState,
})
if err != nil {
errs = multierror.Append(errs, err)
Expand All @@ -138,8 +143,9 @@ func (idx *Indexer) collectReferences(modHandle document.DirHandle, dependsOn jo
Func: func(ctx context.Context) error {
return module.DecodeReferenceOrigins(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeDecodeReferenceOrigins.String(),
DependsOn: dependsOn,
Type: op.OpTypeDecodeReferenceOrigins.String(),
DependsOn: dependsOn,
IgnoreState: ignoreState,
})
if err != nil {
errs = multierror.Append(errs, err)
Expand Down
6 changes: 3 additions & 3 deletions internal/indexer/walker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
},
Type: op.OpTypeParseModuleManifest.String(),
Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
return idx.decodeInstalledModuleCalls(modHandle)
return idx.decodeInstalledModuleCalls(modHandle, false)
},
})
if err != nil {
Expand All @@ -114,7 +114,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
pSchemaVerId, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.ParseProviderVersions(idx.fs, idx.modStore, modHandle.Path())
return module.ParseProviderVersions(ctx, idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseProviderVersions.String(),
DependsOn: providerVersionDeps,
Expand Down Expand Up @@ -144,7 +144,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
}

if parseId != "" {
rIds, err := idx.collectReferences(modHandle, refCollectionDeps)
rIds, err := idx.collectReferences(modHandle, refCollectionDeps, false)
if err != nil {
errs = multierror.Append(errs, err)
} else {
Expand Down
15 changes: 9 additions & 6 deletions internal/indexer/watcher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@ func (idx *Indexer) ModuleManifestChanged(ctx context.Context, modHandle documen
Func: func(ctx context.Context) error {
return module.ParseModuleManifest(ctx, idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseModuleManifest.String(),
Type: op.OpTypeParseModuleManifest.String(),
IgnoreState: true,
Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
return idx.decodeInstalledModuleCalls(modHandle)
return idx.decodeInstalledModuleCalls(modHandle, true)
},
})
if err != nil {
Expand All @@ -39,9 +40,10 @@ func (idx *Indexer) PluginLockChanged(ctx context.Context, modHandle document.Di
pSchemaVerId, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.ParseProviderVersions(idx.fs, idx.modStore, modHandle.Path())
return module.ParseProviderVersions(ctx, idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseProviderVersions.String(),
IgnoreState: true,
Type: op.OpTypeParseProviderVersions.String(),
})
if err != nil {
errs = multierror.Append(errs, err)
Expand All @@ -55,8 +57,9 @@ func (idx *Indexer) PluginLockChanged(ctx context.Context, modHandle document.Di
ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
return module.ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeObtainSchema.String(),
DependsOn: job.IDs{pSchemaVerId},
IgnoreState: true,
Type: op.OpTypeObtainSchema.String(),
DependsOn: job.IDs{pSchemaVerId},
})
if err != nil {
errs = multierror.Append(errs, err)
Expand Down
32 changes: 30 additions & 2 deletions internal/terraform/module/module_ops.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,10 @@ func ParseModuleConfiguration(ctx context.Context, fs ReadOnlyFS, modStore *stat
return err
}

// TODO: Avoid parsing if the content matches existing AST

// TODO: Only parse the file that's being changed/opened, unless this is 1st-time parsing

// Avoid parsing if it is already in progress or already known
if mod.ModuleParsingState != op.OpStateUnknown && !job.IgnoreState(ctx) {
return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)}
Expand Down Expand Up @@ -211,6 +215,10 @@ func ParseVariables(ctx context.Context, fs ReadOnlyFS, modStore *state.ModuleSt
return err
}

// TODO: Avoid parsing if the content matches existing AST

// TODO: Only parse the file that's being changed/opened, unless this is 1st-time parsing

// Avoid parsing if it is already in progress or already known
if mod.VarsParsingState != op.OpStateUnknown && !job.IgnoreState(ctx) {
return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)}
Expand Down Expand Up @@ -280,8 +288,18 @@ func ParseModuleManifest(ctx context.Context, fs ReadOnlyFS, modStore *state.Mod
return err
}

func ParseProviderVersions(fs ReadOnlyFS, modStore *state.ModuleStore, modPath string) error {
err := modStore.SetInstalledProvidersState(modPath, op.OpStateLoading)
func ParseProviderVersions(ctx context.Context, fs ReadOnlyFS, modStore *state.ModuleStore, modPath string) error {
mod, err := modStore.ModuleByPath(modPath)
if err != nil {
return err
}

// Avoid parsing if it is already in progress or already known
if mod.InstalledProvidersState != op.OpStateUnknown && !job.IgnoreState(ctx) {
return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)}
}

err = modStore.SetInstalledProvidersState(modPath, op.OpStateLoading)
if err != nil {
return err
}
Expand All @@ -302,6 +320,8 @@ func LoadModuleMetadata(ctx context.Context, modStore *state.ModuleStore, modPat
return err
}

// TODO: Avoid parsing if upstream (parsing) job reported no changes

// Avoid parsing if it is already in progress or already known
if mod.MetaState != op.OpStateUnknown && !job.IgnoreState(ctx) {
return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)}
Expand Down Expand Up @@ -345,6 +365,8 @@ func DecodeReferenceTargets(ctx context.Context, modStore *state.ModuleStore, sc
return err
}

// TODO: Avoid collection if upstream jobs reported no changes

// Avoid collection if it is already in progress or already done
if mod.RefTargetsState != op.OpStateUnknown && !job.IgnoreState(ctx) {
return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)}
Expand Down Expand Up @@ -386,6 +408,8 @@ func DecodeReferenceOrigins(ctx context.Context, modStore *state.ModuleStore, sc
return err
}

// TODO: Avoid collection if upstream jobs reported no changes

// Avoid collection if it is already in progress or already done
if mod.RefOriginsState != op.OpStateUnknown && !job.IgnoreState(ctx) {
return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)}
Expand Down Expand Up @@ -426,6 +450,8 @@ func DecodeVarsReferences(ctx context.Context, modStore *state.ModuleStore, sche
return err
}

// TODO: Avoid collection if upstream (parsing) job reported no changes

// Avoid collection if it is already in progress or already done
if mod.VarsRefOriginsState != op.OpStateUnknown && !job.IgnoreState(ctx) {
return job.StateNotChangedErr{Dir: document.DirHandleFromPath(modPath)}
Expand Down Expand Up @@ -466,6 +492,8 @@ func GetModuleDataFromRegistry(ctx context.Context, regClient registry.Client, m
return err
}

// TODO: Avoid collection if upstream jobs (parsing, meta) reported no changes

var errs *multierror.Error

for _, declaredModule := range calls.Declared {
Expand Down
3 changes: 2 additions & 1 deletion internal/terraform/module/module_ops_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,8 @@ func TestParseProviderVersions(t *testing.T) {
t.Fatal(err)
}

err = ParseProviderVersions(fs, ss.Modules, modPath)
ctx := context.Background()
err = ParseProviderVersions(ctx, fs, ss.Modules, modPath)
if err != nil {
t.Fatal(err)
}
Expand Down

0 comments on commit 15f0482

Please sign in to comment.