From 168d44e2153fb8aef01edc2a475ef73c0926b319 Mon Sep 17 00:00:00 2001 From: James Pogran Date: Tue, 6 Aug 2024 11:24:04 -0400 Subject: [PATCH] (TF-18664) Add DecodeReferenceOrigins and DecodeReferenceTargets jobs (#1786) * Add DecodeReferenceOrigins and DecodeReferenceTargets jobs This commit adds two new jobs, DecodeReferenceOrigins and DecodeReferenceTargets, and their supporting plumbing to the stacks feature. These jobs are responsible for collecting reference origins and targets, respectively. Reference origins and targets are used to determine where a reference is defined and where it is used. This information is useful for features like go-to-definition and go-to-references. --- .../ENHANCEMENTS-20240805-140526.yaml | 6 + .../features/stacks/decoder/path_reader.go | 26 ++++ internal/features/stacks/events.go | 56 +++++-- internal/features/stacks/jobs/references.go | 143 ++++++++++++++++++ .../features/stacks/state/stack_record.go | 25 ++- internal/features/stacks/state/stack_store.go | 87 +++++++++++ 6 files changed, 328 insertions(+), 15 deletions(-) create mode 100644 .changes/unreleased/ENHANCEMENTS-20240805-140526.yaml create mode 100644 internal/features/stacks/jobs/references.go diff --git a/.changes/unreleased/ENHANCEMENTS-20240805-140526.yaml b/.changes/unreleased/ENHANCEMENTS-20240805-140526.yaml new file mode 100644 index 000000000..2d2b05eba --- /dev/null +++ b/.changes/unreleased/ENHANCEMENTS-20240805-140526.yaml @@ -0,0 +1,6 @@ +kind: ENHANCEMENTS +body: Add DecodeReferenceOrigins and DecodeReferenceTargets jobs +time: 2024-08-05T14:05:26.030294-04:00 +custom: + Issue: "1786" + Repository: terraform-ls diff --git a/internal/features/stacks/decoder/path_reader.go b/internal/features/stacks/decoder/path_reader.go index 04c5db295..fe4d10f93 100644 --- a/internal/features/stacks/decoder/path_reader.go +++ b/internal/features/stacks/decoder/path_reader.go @@ -106,6 +106,19 @@ func stackPathContext(record *state.StackRecord, stateReader CombinedReader) (*d } // TODO: Add reference origins and targets if needed + for _, origin := range record.RefOrigins { + if ast.IsStackFilename(origin.OriginRange().Filename) { + pathCtx.ReferenceOrigins = append(pathCtx.ReferenceOrigins, origin) + } + } + + for _, target := range record.RefTargets { + if target.RangePtr != nil && ast.IsStackFilename(target.RangePtr.Filename) { + pathCtx.ReferenceTargets = append(pathCtx.ReferenceTargets, target) + } else if target.RangePtr == nil { + pathCtx.ReferenceTargets = append(pathCtx.ReferenceTargets, target) + } + } for name, f := range record.ParsedFiles { if _, ok := name.(ast.StackFilename); ok { @@ -153,6 +166,19 @@ func deployPathContext(record *state.StackRecord) (*decoder.PathContext, error) } // TODO: Add reference origins and targets if needed + for _, origin := range record.RefOrigins { + if ast.IsDeployFilename(origin.OriginRange().Filename) { + pathCtx.ReferenceOrigins = append(pathCtx.ReferenceOrigins, origin) + } + } + + for _, target := range record.RefTargets { + if target.RangePtr != nil && ast.IsDeployFilename(target.RangePtr.Filename) { + pathCtx.ReferenceTargets = append(pathCtx.ReferenceTargets, target) + } else if target.RangePtr == nil { + pathCtx.ReferenceTargets = append(pathCtx.ReferenceTargets, target) + } + } for name, f := range record.ParsedFiles { if _, ok := name.(ast.DeployFilename); ok { diff --git a/internal/features/stacks/events.go b/internal/features/stacks/events.go index 0e34bc395..f7a614cdd 100644 --- a/internal/features/stacks/events.go +++ b/internal/features/stacks/events.go @@ -192,8 +192,13 @@ func (f *StacksFeature) decodeStack(ctx context.Context, dir document.DirHandle, } ids = append(ids, parseId) - // this needs to be here because the setting context - // is not available in the validate job + // Changes to a setting currently requires a LS restart, so the LS + // setting context cannot change during the execution of a job. That's + // why we can extract it here and use it in Defer. + // See https://github.com/hashicorp/terraform-ls/issues/1008 + // We can safely ignore the error here. If we can't get the options from + // the context, validationOptions.EnableEnhancedValidation will be false + // by default. So we don't run the validation jobs. validationOptions, _ := lsctx.ValidationOptions(ctx) metaId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{ @@ -211,10 +216,13 @@ func (f *StacksFeature) decodeStack(ctx context.Context, dir document.DirHandle, f.logger.Printf("loading module metadata returned error: %s", jobErr) } - spawnedIds, err := loadStackComponentSources(ctx, f.store, f.bus, path) - deferIds = append(deferIds, spawnedIds...) + componentIds, err := loadStackComponentSources(ctx, f.store, f.bus, path) + deferIds = append(deferIds, componentIds...) if err != nil { f.logger.Printf("loading stack component sources returned error: %s", err) + // We log the error but still continue scheduling other jobs + // which are still valuable for the rest of the configuration + // even if they may not have the data for module calls. } // while we now have the job ids in here, depending on the metaId job is not enough @@ -238,20 +246,47 @@ func (f *StacksFeature) decodeStack(ctx context.Context, dir document.DirHandle, } deferIds = append(deferIds, eSchemaId) + refTargetsId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{ + Dir: dir, + Func: func(ctx context.Context) error { + return jobs.DecodeReferenceTargets(ctx, f.store, f.moduleFeature, path) + }, + Type: operation.OpTypeDecodeReferenceTargets.String(), + DependsOn: append(componentIds, eSchemaId), + IgnoreState: ignoreState, + }) + if err != nil { + return deferIds, err + } + deferIds = append(deferIds, refTargetsId) + + refOriginsId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{ + Dir: dir, + Func: func(ctx context.Context) error { + return jobs.DecodeReferenceOrigins(ctx, f.store, f.moduleFeature, path) + }, + Type: operation.OpTypeDecodeReferenceOrigins.String(), + DependsOn: append(componentIds, eSchemaId), + IgnoreState: ignoreState, + }) + if err != nil { + return deferIds, err + } + deferIds = append(deferIds, refOriginsId) + if validationOptions.EnableEnhancedValidation { - validationId, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{ + _, err := f.stateStore.JobStore.EnqueueJob(ctx, job.Job{ Dir: dir, Func: func(ctx context.Context) error { return jobs.SchemaStackValidation(ctx, f.store, f.moduleFeature, dir.Path()) }, Type: operation.OpTypeSchemaStackValidation.String(), - DependsOn: deferIds, + DependsOn: job.IDs{refOriginsId, refTargetsId}, IgnoreState: ignoreState, }) if err != nil { - return deferIds, err + return ids, err } - deferIds = append(deferIds, validationId) } return deferIds, nil @@ -262,11 +297,6 @@ func (f *StacksFeature) decodeStack(ctx context.Context, dir document.DirHandle, } ids = append(ids, metaId) - // TODO: Implement the following functions where appropriate to stacks - // Future: decodeDeclaredModuleCalls(ctx, dir, ignoreState) - // Future: DecodeReferenceTargets(ctx, f.Store, f.rootFeature, path) - // Future: DecodeReferenceOrigins(ctx, f.Store, f.rootFeature, path) - return ids, nil } diff --git a/internal/features/stacks/jobs/references.go b/internal/features/stacks/jobs/references.go new file mode 100644 index 000000000..a6a8c4bf9 --- /dev/null +++ b/internal/features/stacks/jobs/references.go @@ -0,0 +1,143 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package jobs + +import ( + "context" + + "github.com/hashicorp/hcl-lang/decoder" + "github.com/hashicorp/hcl-lang/lang" + "github.com/hashicorp/hcl-lang/reference" + idecoder "github.com/hashicorp/terraform-ls/internal/decoder" + "github.com/hashicorp/terraform-ls/internal/document" + sdecoder "github.com/hashicorp/terraform-ls/internal/features/stacks/decoder" + "github.com/hashicorp/terraform-ls/internal/features/stacks/state" + "github.com/hashicorp/terraform-ls/internal/job" + ilsp "github.com/hashicorp/terraform-ls/internal/lsp" + "github.com/hashicorp/terraform-ls/internal/terraform/module/operation" +) + +// DecodeReferenceTargets collects reference targets, +// using previously parsed AST (via [ParseStackConfiguration]), +// core schema of appropriate version (as obtained via [GetTerraformVersion]) +// and provider schemas ([PreloadEmbeddedSchema] or [ObtainSchema]). +// +// For example it tells us that variable block between certain LOC +// can be referred to as var.foobar. This is useful e.g. during completion, +// go-to-definition or go-to-references. +func DecodeReferenceTargets(ctx context.Context, stackStore *state.StackStore, moduleReader sdecoder.ModuleReader, stackPath string) error { + mod, err := stackStore.StackRecordByPath(stackPath) + if err != nil { + return err + } + + // TODO: Avoid collection if upstream jobs reported no changes + + // Avoid collection if it is already in progress or already done + if mod.RefTargetsState != operation.OpStateUnknown && !job.IgnoreState(ctx) { + return job.StateNotChangedErr{Dir: document.DirHandleFromPath(stackPath)} + } + + err = stackStore.SetReferenceTargetsState(stackPath, operation.OpStateLoading) + if err != nil { + return err + } + + d := decoder.NewDecoder(&sdecoder.PathReader{ + StateReader: stackStore, + ModuleReader: moduleReader, + }) + d.SetContext(idecoder.DecoderContext(ctx)) + + stackDecoder, err := d.Path(lang.Path{ + Path: stackPath, + LanguageID: ilsp.Stacks.String(), + }) + if err != nil { + return err + } + stackTargets, rErr := stackDecoder.CollectReferenceTargets() + + deployDecoder, err := d.Path(lang.Path{ + Path: stackPath, + LanguageID: ilsp.Deploy.String(), + }) + if err != nil { + return err + } + deployTargets, rErr := deployDecoder.CollectReferenceTargets() + + targets := make(reference.Targets, 0) + targets = append(targets, stackTargets...) + targets = append(targets, deployTargets...) + + sErr := stackStore.UpdateReferenceTargets(stackPath, targets, rErr) + if sErr != nil { + return sErr + } + + return rErr +} + +// DecodeReferenceOrigins collects reference origins, +// using previously parsed AST (via [ParseStackConfiguration]), +// core schema of appropriate version (as obtained via [GetTerraformVersion]) +// and provider schemas ([PreloadEmbeddedSchema] or [ObtainSchema]). +// +// For example it tells us that there is a reference address var.foobar +// at a particular LOC. This can be later matched with targets +// (as obtained via [DecodeReferenceTargets]) during hover or go-to-definition. +func DecodeReferenceOrigins(ctx context.Context, stackStore *state.StackStore, moduleReader sdecoder.ModuleReader, stackPath string) error { + mod, err := stackStore.StackRecordByPath(stackPath) + if err != nil { + return err + } + + // TODO: Avoid collection if upstream jobs reported no changes + + // Avoid collection if it is already in progress or already done + if mod.RefOriginsState != operation.OpStateUnknown && !job.IgnoreState(ctx) { + return job.StateNotChangedErr{Dir: document.DirHandleFromPath(stackPath)} + } + + err = stackStore.SetReferenceOriginsState(stackPath, operation.OpStateLoading) + if err != nil { + return err + } + + d := decoder.NewDecoder(&sdecoder.PathReader{ + StateReader: stackStore, + ModuleReader: moduleReader, + }) + d.SetContext(idecoder.DecoderContext(ctx)) + + stackDecoder, err := d.Path(lang.Path{ + Path: stackPath, + LanguageID: ilsp.Stacks.String(), + }) + if err != nil { + return err + } + stackOrigins, rErr := stackDecoder.CollectReferenceOrigins() + + deployDecoder, err := d.Path(lang.Path{ + Path: stackPath, + LanguageID: ilsp.Deploy.String(), + }) + if err != nil { + return err + } + deployOrigins, rErr := deployDecoder.CollectReferenceOrigins() + + origins := make(reference.Origins, 0) + origins = append(origins, stackOrigins...) + origins = append(origins, deployOrigins...) + + sErr := stackStore.UpdateReferenceOrigins(stackPath, origins, rErr) + if sErr != nil { + return sErr + } + + return rErr +} diff --git a/internal/features/stacks/state/stack_record.go b/internal/features/stacks/state/stack_record.go index 9d4f58937..a6c80d587 100644 --- a/internal/features/stacks/state/stack_record.go +++ b/internal/features/stacks/state/stack_record.go @@ -5,6 +5,7 @@ package state import ( "github.com/hashicorp/go-version" + "github.com/hashicorp/hcl-lang/reference" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/terraform-ls/internal/features/stacks/ast" globalAst "github.com/hashicorp/terraform-ls/internal/terraform/ast" @@ -34,6 +35,14 @@ type StackRecord struct { RequiredTerraformVersion *version.Version RequiredTerraformVersionErr error RequiredTerraformVersionState operation.OpState + + RefTargets reference.Targets + RefTargetsErr error + RefTargetsState operation.OpState + + RefOrigins reference.Origins + RefOriginsErr error + RefOriginsState operation.OpState } func (m *StackRecord) Path() string { @@ -59,6 +68,14 @@ func (m *StackRecord) Copy() *StackRecord { RequiredTerraformVersion: m.RequiredTerraformVersion, RequiredTerraformVersionErr: m.RequiredTerraformVersionErr, RequiredTerraformVersionState: m.RequiredTerraformVersionState, + + RefTargets: m.RefTargets.Copy(), + RefTargetsErr: m.RefTargetsErr, + RefTargetsState: m.RefTargetsState, + + RefOrigins: m.RefOrigins.Copy(), + RefOriginsErr: m.RefOriginsErr, + RefOriginsState: m.RefOriginsState, } if m.ParsedFiles != nil { @@ -85,9 +102,13 @@ func (m *StackRecord) Copy() *StackRecord { return newRecord } -func newStack(modPath string) *StackRecord { +func newStack(stackPath string) *StackRecord { return &StackRecord{ - path: modPath, + path: stackPath, + PreloadEmbeddedSchemaState: operation.OpStateUnknown, + RefOriginsState: operation.OpStateUnknown, + RefTargetsState: operation.OpStateUnknown, + MetaState: operation.OpStateUnknown, DiagnosticsState: globalAst.DiagnosticSourceState{ globalAst.HCLParsingSource: operation.OpStateUnknown, globalAst.SchemaValidationSource: operation.OpStateUnknown, diff --git a/internal/features/stacks/state/stack_store.go b/internal/features/stacks/state/stack_store.go index 4afc266bc..d16e69e43 100644 --- a/internal/features/stacks/state/stack_store.go +++ b/internal/features/stacks/state/stack_store.go @@ -8,6 +8,7 @@ import ( "github.com/hashicorp/go-memdb" "github.com/hashicorp/go-version" + "github.com/hashicorp/hcl-lang/reference" "github.com/hashicorp/terraform-ls/internal/document" "github.com/hashicorp/terraform-ls/internal/features/stacks/ast" globalState "github.com/hashicorp/terraform-ls/internal/state" @@ -336,6 +337,92 @@ func (s *StackStore) SetPreloadEmbeddedSchemaState(path string, state operation. return nil } +func (s *StackStore) SetReferenceTargetsState(path string, state operation.OpState) error { + txn := s.db.Txn(true) + defer txn.Abort() + + record, err := stackCopyByPath(txn, path) + if err != nil { + return err + } + + record.RefTargetsState = state + err = txn.Insert(s.tableName, record) + if err != nil { + return err + } + + txn.Commit() + return nil +} + +func (s *StackStore) UpdateReferenceTargets(path string, refs reference.Targets, rErr error) error { + txn := s.db.Txn(true) + txn.Defer(func() { + s.SetReferenceTargetsState(path, operation.OpStateLoaded) + }) + defer txn.Abort() + + record, err := stackCopyByPath(txn, path) + if err != nil { + return err + } + + record.RefTargets = refs + record.RefTargetsErr = rErr + + err = txn.Insert(s.tableName, record) + if err != nil { + return err + } + + txn.Commit() + return nil +} + +func (s *StackStore) SetReferenceOriginsState(path string, state operation.OpState) error { + txn := s.db.Txn(true) + defer txn.Abort() + + stack, err := stackCopyByPath(txn, path) + if err != nil { + return err + } + + stack.RefOriginsState = state + err = txn.Insert(s.tableName, stack) + if err != nil { + return err + } + + txn.Commit() + return nil +} + +func (s *StackStore) UpdateReferenceOrigins(path string, origins reference.Origins, roErr error) error { + txn := s.db.Txn(true) + txn.Defer(func() { + s.SetReferenceOriginsState(path, operation.OpStateLoaded) + }) + defer txn.Abort() + + stack, err := stackCopyByPath(txn, path) + if err != nil { + return err + } + + stack.RefOrigins = origins + stack.RefOriginsErr = roErr + + err = txn.Insert(s.tableName, stack) + if err != nil { + return err + } + + txn.Commit() + return nil +} + func (s *StackStore) add(txn *memdb.Txn, stackPath string) error { // TODO: Introduce Exists method to Txn? obj, err := txn.First(s.tableName, "id", stackPath)