Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Only schedule jobs relevant to the files when indexing a directory #1630

Closed
Tracked by #1614
dbanck opened this issue Feb 12, 2024 · 2 comments
Closed
Tracked by #1614

Only schedule jobs relevant to the files when indexing a directory #1630

dbanck opened this issue Feb 12, 2024 · 2 comments
Assignees
Labels

Comments

@dbanck
Copy link
Member

dbanck commented Feb 12, 2024


Whenever we encounter a directory that contains any file relevant to terraform-ls, we schedule all kind of jobs for that directory. But for a directory that only contains module files, we only need to schedule jobs related to that language and can avoid scheduling any variable definition jobs.

func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHandle) (job.IDs, error) {
ids := make(job.IDs, 0)
var errs *multierror.Error
refCollectionDeps := make(job.IDs, 0)
providerVersionDeps := make(job.IDs, 0)
parseId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.ParseModuleConfiguration(ctx, idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseModuleConfiguration.String(),
})
if err != nil {
errs = multierror.Append(errs, err)
} else {
ids = append(ids, parseId)
refCollectionDeps = append(refCollectionDeps, parseId)
providerVersionDeps = append(providerVersionDeps, parseId)
}
var metaId job.ID
if parseId != "" {
metaId, err = idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Type: op.OpTypeLoadModuleMetadata.String(),
Func: func(ctx context.Context) error {
return module.LoadModuleMetadata(ctx, idx.modStore, modHandle.Path())
},
DependsOn: job.IDs{parseId},
})
if err != nil {
return ids, err
} else {
ids = append(ids, metaId)
refCollectionDeps = append(refCollectionDeps, metaId)
providerVersionDeps = append(providerVersionDeps, metaId)
}
}
parseVarsId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.ParseVariables(ctx, idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseVariables.String(),
})
if err != nil {
errs = multierror.Append(errs, err)
} else {
ids = append(ids, parseVarsId)
}
if parseVarsId != "" {
varsRefsId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.DecodeVarsReferences(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeDecodeVarsReferences.String(),
DependsOn: job.IDs{parseVarsId},
})
if err != nil {
return ids, err
} else {
ids = append(ids, varsRefsId)
refCollectionDeps = append(refCollectionDeps, varsRefsId)
}
}
dataDir := datadir.WalkDataDirOfModule(idx.fs, modHandle.Path())
idx.logger.Printf("parsed datadir: %#v", dataDir)
var modManifestId job.ID
if dataDir.ModuleManifestPath != "" {
// References are collected *after* manifest parsing
// so that we reflect any references to submodules.
modManifestId, err = idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.ParseModuleManifest(ctx, idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseModuleManifest.String(),
Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
return idx.decodeInstalledModuleCalls(ctx, modHandle, false)
},
})
if err != nil {
errs = multierror.Append(errs, err)
} else {
ids = append(ids, modManifestId)
refCollectionDeps = append(refCollectionDeps, modManifestId)
// provider requirements may be within the (installed) modules
providerVersionDeps = append(providerVersionDeps, modManifestId)
}
}
if dataDir.PluginLockFilePath != "" {
dependsOn := make(job.IDs, 0)
pSchemaVerId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.ParseProviderVersions(ctx, idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseProviderVersions.String(),
DependsOn: providerVersionDeps,
})
if err != nil {
errs = multierror.Append(errs, err)
} else {
ids = append(ids, pSchemaVerId)
dependsOn = append(dependsOn, pSchemaVerId)
refCollectionDeps = append(refCollectionDeps, pSchemaVerId)
}
pSchemaId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
return module.ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeObtainSchema.String(),
DependsOn: dependsOn,
})
if err != nil {
errs = multierror.Append(errs, err)
} else {
ids = append(ids, pSchemaId)
refCollectionDeps = append(refCollectionDeps, pSchemaId)
}
}
eSchemaId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.PreloadEmbeddedSchema(ctx, idx.logger, schemas.FS, idx.modStore, idx.schemaStore, modHandle.Path())
},
// This could theoretically also depend on ObtainSchema to avoid
// attempt to preload the same schema twice but we avoid that dependency
// as obtaining schema via CLI often takes a long time (multiple
// seconds) and this would then defeat the main benefit
// of preloaded schemas which can be loaded in miliseconds.
DependsOn: providerVersionDeps,
Type: op.OpTypePreloadEmbeddedSchema.String(),
})
if err != nil {
return ids, err
}
ids = append(ids, eSchemaId)
if parseId != "" {
rIds, err := idx.collectReferences(ctx, modHandle, refCollectionDeps, false)
if err != nil {
errs = multierror.Append(errs, err)
} else {
ids = append(ids, rIds...)
}
}
return ids, errs.ErrorOrNil()
}

@dbanck
Copy link
Member Author

dbanck commented Jun 11, 2024

Closed via #1667

@dbanck dbanck closed this as completed Jun 11, 2024
Copy link

I'm going to lock this issue because it has been closed for 30 days ⏳. This helps our maintainers find and focus on the active issues.
If you have found a problem that seems similar to this, please open a new issue and complete the issue template so we can capture all the details necessary to investigate further.

@github-actions github-actions bot locked as resolved and limited conversation to collaborators Jul 12, 2024
Sign up for free to subscribe to this conversation on GitHub. Already have an account? Sign in.
Labels
Projects
None yet
Development

No branches or pull requests

1 participant