walker: Index uninitialized modules

go.mod

@@ -1,6 +1,6 @@
 module github.com/hashicorp/terraform-ls
 
-go 1.13
+go 1.16
 
 require (
  github.com/agext/levenshtein v1.2.2 // indirect
@@ -19,7 +19,7 @@ require (
  github.com/hashicorp/terraform-exec v0.17.2
  github.com/hashicorp/terraform-json v0.14.0
  github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c
- github.com/hashicorp/terraform-schema v0.0.0-20220708122804-408b2aefd4d6
+ github.com/hashicorp/terraform-schema v0.0.0-20220712135911-08c17f04e2bc
  github.com/kylelemons/godebug v1.1.0 // indirect
  github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5
  github.com/mitchellh/cli v1.1.4

go.sum

@@ -86,7 +86,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
-github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/antonmedv/expr v1.9.0 h1:j4HI3NHEdgDnN9p6oI6Ndr0G5QryMY0FNxT4ONrFDGU=
@@ -102,7 +101,6 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -163,7 +161,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
-github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
@@ -175,14 +172,12 @@ github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo
 github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gizak/termui/v3 v3.1.0/go.mod h1:bXQEBkJpzxUAKf0+xq9MSWAvWZlE7c+aidmyFlkYTrY=
-github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
 github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
 github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
 github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34=
 github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
-github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8=
 github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0=
 github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4=
 github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc=
@@ -344,8 +339,8 @@ github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e
 github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM=
 github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c h1:D8aRO6+mTqHfLsK/BC3j5OAoogv1WLRWzY1AaTo3rBg=
 github.com/hashicorp/terraform-registry-address v0.0.0-20220623143253-7d51757b572c/go.mod h1:Wn3Na71knbXc1G8Lh+yu/dQWWJeFQEpDeJMtWMtlmNI=
-github.com/hashicorp/terraform-schema v0.0.0-20220708122804-408b2aefd4d6 h1:nyYG4twPkXiAUnLSgF5HfhKUGNp3rErv2sYOS2bDsZY=
-github.com/hashicorp/terraform-schema v0.0.0-20220708122804-408b2aefd4d6/go.mod h1:GL+zHwXHrTc/MfKnQP8K2Z4hmaTck85ndiIbvZueMng=
+github.com/hashicorp/terraform-schema v0.0.0-20220712135911-08c17f04e2bc h1:08Pp12m9Rv4cQgk2GOlkoG6njaiAyCR+rKDpdrgvlVs=
+github.com/hashicorp/terraform-schema v0.0.0-20220712135911-08c17f04e2bc/go.mod h1:GL+zHwXHrTc/MfKnQP8K2Z4hmaTck85ndiIbvZueMng=
 github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 h1:HKLsbzeOsfXmKNpr3GiT18XAblV0BjCbzL8KQAMZGa0=
 github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734/go.mod h1:kNDNcF7sN4DocDLBkQYz73HGKwN1ANB1blq4lIYLYvg=
 github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
@@ -405,7 +400,6 @@ github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1
 github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
 github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
-github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
 github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
@@ -470,7 +464,6 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6
 github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE=
 github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
-github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI=
 github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI=

internal/cmd/inspect_module_command.go

@@ -103,7 +103,7 @@ func (c *InspectModuleCommand) inspect(rootPath string) error {
  exec.NewExecutor, registry.NewClient())
 
  pa := state.NewPathAwaiter(ss.WalkerPaths, false)
- w := walker.NewWalker(pa, ss.Modules, indexer.WalkedModule)
+ w := walker.NewWalker(fs, pa, ss.Modules, indexer.WalkedModule)
  w.Collector = walker.NewWalkerCollector()
  w.SetLogger(c.logger)
  err = ss.WalkerPaths.WaitForDirs(ctx, []document.DirHandle{dir})
@@ -199,7 +199,7 @@ func formatModuleRecords(mds []datadir.ModuleRecord) []string {
  continue
  }
  if m.IsExternal() {
- out = append(out, "EXTERNAL(%s)", m.SourceAddr)
+ out = append(out, "EXTERNAL(%s)", m.SourceAddr.String())
  continue
  }
  out = append(out, fmt.Sprintf("%s (%s)", m.Dir, m.SourceAddr))

internal/indexer/module_calls.go

@@ -49,29 +49,24 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) job
  Type: op.OpTypeParseModuleConfiguration.String(),
  Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
  ids := make(job.IDs, 0)
- var errs *multierror.Error
 
  id, err := idx.jobStore.EnqueueJob(job.Job{
  Dir: mcHandle,
  Type: op.OpTypeLoadModuleMetadata.String(),
  Func: func(ctx context.Context) error {
  return module.LoadModuleMetadata(idx.modStore, mcPath)
  },
+ Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
+ return idx.collectReferences(ctx, mcHandle)
+ },
  })
  if err != nil {
- errs = multierror.Append(errs, err)
+ return ids, err
  } else {
  ids = append(ids, id)
  }
 
- rIds, err := idx.collectReferences(ctx, mcHandle)
- if err != nil {
- errs = multierror.Append(errs, err)
- } else {
- ids = append(ids, rIds...)
- }
-
- return ids, errs.ErrorOrNil()
+ return ids, nil
  },
  })
  if err != nil {

internal/indexer/walker.go

@@ -16,17 +16,40 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
  ids := make(job.IDs, 0)
  var errs *multierror.Error
 
+ // blockingJobIds tracks job IDs which need to finish
+ // prior to collecting references
+ blockingJobIds := make(job.IDs, 0)
+
  id, err := idx.jobStore.EnqueueJob(job.Job{
  Dir: modHandle,
  Func: func(ctx context.Context) error {
  return module.ParseModuleConfiguration(idx.fs, idx.modStore, modHandle.Path())
  },
  Type: op.OpTypeParseModuleConfiguration.String(),
+ Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
+ ids := make(job.IDs, 0)
+
+ id, err := idx.jobStore.EnqueueJob(job.Job{
+ Dir: modHandle,
+ Type: op.OpTypeLoadModuleMetadata.String(),
+ Func: func(ctx context.Context) error {
+ return module.LoadModuleMetadata(idx.modStore, modHandle.Path())
+ },
+ })
+ if err != nil {
+ return ids, err
+ } else {
+ ids = append(ids, id)
+ }
+
+ return ids, nil
+ },
  })
  if err != nil {
  errs = multierror.Append(errs, err)
  } else {
  ids = append(ids, id)
+ blockingJobIds = append(blockingJobIds, id)
  }
 
  id, err = idx.jobStore.EnqueueJob(job.Job{
@@ -70,6 +93,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
  errs = multierror.Append(errs, err)
  } else {
  ids = append(ids, id)
+ blockingJobIds = append(blockingJobIds, id)
  }
 
  dataDir := datadir.WalkDataDirOfModule(idx.fs, modHandle.Path())
@@ -88,6 +112,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
  errs = multierror.Append(errs, err)
  } else {
  ids = append(ids, id)
+ blockingJobIds = append(blockingJobIds, id)
  }
  }
 
@@ -106,13 +131,17 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
  errs = multierror.Append(errs, err)
  } else {
  ids = append(ids, id)
+ blockingJobIds = append(blockingJobIds, id)
  }
+ }
 
- // Here we wait for all module calls to be processed to
- // reflect any metadata required to collect reference origins.
- // This assumes scheduler is running to consume the jobs
- // by the time we reach this point.
- idx.jobStore.WaitForJobs(ctx, id)
+ // Here we wait for all dependent jobs to be processed to
+ // reflect any data required to collect reference origins.
+ // This assumes scheduler is running to consume the jobs
+ // by the time we reach this point.
+ err = idx.jobStore.WaitForJobs(ctx, blockingJobIds...)
+ if err != nil {
+ return ids, err
  }
 
  rIds, err := idx.collectReferences(ctx, modHandle)

internal/langserver/handlers/complete_test.go

@@ -41,7 +41,7 @@ func TestModuleCompletion_withoutInitialization(t *testing.T) {
  }`, TempDir(t).URI)}, session.SessionNotInitialized.Err())
 }
 
-func TestModuleCompletion_withValidData(t *testing.T) {
+func TestModuleCompletion_withValidData_basic(t *testing.T) {
  tmpDir := TempDir(t)
  InitPluginCache(t, tmpDir.Path())
 

internal/langserver/handlers/did_change_watched_files_test.go

@@ -735,22 +735,22 @@ func TestLangServer_DidChangeWatchedFiles_pluginChange(t *testing.T) {
  },
  {
  Method: "GetExecPath",
- Repeatability: 2,
+ Repeatability: 1,
  ReturnArguments: []interface{}{
  "",
  },
  },
  {
  Method: "ProviderSchemas",
- Repeatability: 2,
+ Repeatability: 1,
  Arguments: []interface{}{
  mock.AnythingOfType(""),
  },
  ReturnArguments: []interface{}{
  &tfjson.ProviderSchemas{
  FormatVersion: "0.1",
  Schemas: map[string]*tfjson.ProviderSchema{
- "test": {
+ "foo": {
  ConfigSchema: &tfjson.Schema{},
  },
  },
@@ -780,12 +780,12 @@ func TestLangServer_DidChangeWatchedFiles_pluginChange(t *testing.T) {
  ReqParams: "{}",
  })
 
- addr := tfaddr.MustParseProviderSource("-/test")
+ addr := tfaddr.MustParseProviderSource("-/foo")
  vc := version.MustConstraints(version.NewConstraint(">= 1.0"))
 
  _, err = ss.ProviderSchemas.ProviderSchema(testHandle.Path(), addr, vc)
  if err == nil {
- t.Fatal("expected -/test schema to be missing")
+ t.Fatal("expected -/foo schema to be missing")
  }
 
  // Install Terraform
@@ -876,7 +876,7 @@ func TestLangServer_DidChangeWatchedFiles_moduleInstalled(t *testing.T) {
  ReqParams: "{}",
  })
 
- submodulePath := filepath.Join(testDir, "application")
+ submodulePath := filepath.Join(testDir, ".terraform", "modules", "azure-hcp-consul")
  _, err = ss.Modules.ModuleByPath(submodulePath)
  if err == nil || !state.IsModuleNotFound(err) {
  t.Fatalf("expected submodule not to be found: %s", err)
@@ -922,7 +922,7 @@ func TestLangServer_DidChangeWatchedFiles_moduleInstalled(t *testing.T) {
  t.Fatal(err)
  }
 
- if len(mod.Meta.Variables) != 3 {
- t.Fatalf("expected exactly 3 variables, %d given", len(mod.Meta.Variables))
+ if len(mod.Meta.Variables) != 8 {
+ t.Fatalf("expected exactly 8 variables, %d given", len(mod.Meta.Variables))
  }
 }

internal/langserver/handlers/handlers_test.go

@@ -285,4 +285,14 @@ func InitPluginCache(t *testing.T, dir string) {
  if err != nil {
  t.Fatal(err)
  }
+
+ // create an empty file such that it's recognized as an indexable workspace
+ f, err = os.Create(filepath.Join(dir, "empty.tf"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = f.Close()
+ if err != nil {
+ t.Fatal(err)
+ }
 }

internal/langserver/handlers/service.go

@@ -493,12 +493,12 @@ func (svc *service) configureSessionDependencies(ctx context.Context, cfgOpts *s
  }
 
  closedPa := state.NewPathAwaiter(svc.stateStore.WalkerPaths, false)
- svc.closedDirWalker = walker.NewWalker(closedPa, svc.modStore, svc.indexer.WalkedModule)
+ svc.closedDirWalker = walker.NewWalker(svc.fs, closedPa, svc.modStore, svc.indexer.WalkedModule)
  svc.closedDirWalker.Collector = svc.walkerCollector
  svc.closedDirWalker.SetLogger(svc.logger)
 
  opendPa := state.NewPathAwaiter(svc.stateStore.WalkerPaths, true)
- svc.openDirWalker = walker.NewWalker(opendPa, svc.modStore, svc.indexer.WalkedModule)
+ svc.openDirWalker = walker.NewWalker(svc.fs, opendPa, svc.modStore, svc.indexer.WalkedModule)
  svc.closedDirWalker.Collector = svc.walkerCollector
  svc.openDirWalker.SetLogger(svc.logger)
 

internal/langserver/handlers/testdata/uninitialized-single-submodule/main.tf

@@ -1,6 +1,3 @@
-module "gorilla-app" {
- source = "./application"
- environment_name = "prod"
- app_prefix = "protect-gorillas"
- instances = 5
+module "azure-hcp-consul" {
+ source = "github.com/hashicorp/terraform-azurerm-hcp-consul?ref=v0.2.4"
 }

internal/state/module.go

@@ -1,6 +1,7 @@
 package state
 
 import (
+ "fmt"
  "path/filepath"
 
  "github.com/hashicorp/go-memdb"
@@ -73,7 +74,7 @@ func (mm ModuleMetadata) Copy() ModuleMetadata {
  if mm.ModuleCalls != nil {
  newMm.ModuleCalls = make(map[string]tfmod.DeclaredModuleCall, len(mm.ModuleCalls))
  for name, moduleCall := range mm.ModuleCalls {
- newMm.ModuleCalls[name] = moduleCall
+ newMm.ModuleCalls[name] = moduleCall.Copy()
  }
  }
 
@@ -362,6 +363,7 @@ func (s *ModuleStore) ModuleCalls(modPath string) (tfmod.ModuleCalls, error) {
  LocalName: mc.LocalName,
  SourceAddr: mc.SourceAddr,
  Version: mc.Version,
+ InputNames: mc.InputNames,
  }
  }
 
@@ -373,6 +375,9 @@ func (s *ModuleStore) LocalModuleMeta(modPath string) (*tfmod.Meta, error) {
  if err != nil {
  return nil, err
  }
+ if mod.MetaState != op.OpStateLoaded {
+ return nil, fmt.Errorf("%s: module data not available", modPath)
+ }
  return &tfmod.Meta{
  Path: mod.Path,
  ProviderReferences: mod.Meta.ProviderReferences,