Queue decoding of submodules asynchronously (#591)

internal/state/module.go

@@ -255,6 +255,9 @@ func (s *ModuleStore) ModuleCalls(modPath string) ([]tfmod.ModuleCall, error) {
  for _, mod := range modList {
  if mod.ModManifest != nil {
  for _, record := range mod.ModManifest.Records {
+ if record.IsRoot() {
+ continue
+ }
  result = append(result, tfmod.ModuleCall{
  SourceAddr: record.SourceAddr,
  Path: filepath.Join(modPath, record.Dir),

internal/terraform/module/module_loader.go

@@ -147,7 +147,6 @@ func (ml *moduleLoader) nonPrioCapacity() int64 {
 func (ml *moduleLoader) executeModuleOp(ctx context.Context, modOp ModuleOperation) {
  ml.logger.Printf("executing %q for %s", modOp.Type, modOp.ModulePath)
  // TODO: Report progress in % for each op based on queue length
- defer ml.logger.Printf("finished %q for %s", modOp.Type, modOp.ModulePath)
  defer modOp.markAsDone()
 
  var opErr error
@@ -198,9 +197,10 @@ func (ml *moduleLoader) executeModuleOp(ctx context.Context, modOp ModuleOperati
  modOp.ModulePath, modOp.Type)
  return
  }
+ ml.logger.Printf("finished %q for %s", modOp.Type, modOp.ModulePath)
 
  if modOp.Defer != nil {
- modOp.Defer(opErr)
+ go modOp.Defer(opErr)
  }
 }
 

internal/terraform/module/module_manager.go

@@ -170,6 +170,10 @@ func (mm *moduleManager) CallersOfModule(modPath string) ([]Module, error) {
  return modules, nil
 }
 
+func (mm *moduleManager) ModuleCalls(modPath string) ([]tfmodule.ModuleCall, error) {
+ return mm.moduleStore.ModuleCalls(modPath)
+}
+
 // SchemaSourcesForModule is DEPRECATED and should NOT be used anymore
 // it is just maintained for backwards compatibility in the "rootmodules"
 // custom LSP command which itself will be DEPRECATED as external parties

internal/terraform/module/module_ops.go

@@ -296,18 +296,6 @@ func ParseModuleManifest(fs filesystem.Filesystem, modStore *state.ModuleStore,
  }
  return err
  }
- if mm != nil {
- for _, modRecord := range mm.Records {
- modModulePath := filepath.Join(modPath, modRecord.Dir)
- _, err := modStore.ModuleByPath(modModulePath)
- if err != nil {
- modStore.Add(modModulePath)
- ParseModuleConfiguration(fs, modStore, modModulePath)
- ParseVariables(fs, modStore, modModulePath)
- LoadModuleMetadata(modStore, modModulePath)
- }
- }
- }
 
  sErr := modStore.UpdateModManifest(modPath, mm, err)
 

internal/terraform/module/types.go

@@ -8,6 +8,7 @@ import (
  "github.com/hashicorp/terraform-ls/internal/filesystem"
  "github.com/hashicorp/terraform-ls/internal/state"
  op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation"
+ tfmodule "github.com/hashicorp/terraform-schema/module"
 )
 
 type File interface {
@@ -25,6 +26,7 @@ type ModuleFinder interface {
  SchemaForVariables(path string) (*schema.BodySchema, error)
  SchemaSourcesForModule(path string) ([]SchemaSource, error)
  ListModules() ([]Module, error)
+ ModuleCalls(modPath string) ([]tfmodule.ModuleCall, error)
  CallersOfModule(modPath string) ([]Module, error)
 }
 

internal/terraform/module/walker.go

@@ -246,7 +246,8 @@ func (w *Walker) walk(ctx context.Context, rootPath string) error {
 
  dataDir := datadir.WalkDataDirOfModule(w.fs, dir)
  if dataDir.ModuleManifestPath != "" {
- err = w.modMgr.EnqueueModuleOp(dir, op.OpTypeParseModuleManifest, nil)
+ err = w.modMgr.EnqueueModuleOp(dir, op.OpTypeParseModuleManifest,
+ decodeCalledModulesFunc(w.modMgr, w.watcher, dir))
  if err != nil {
  return err
  }

internal/terraform/module/watcher.go

@@ -149,7 +149,8 @@ func (w *watcher) processEvent(event fsnotify.Event) {
  if event.Op&fsnotify.Write == fsnotify.Write {
  for _, mod := range w.modules {
  if containsPath(mod.Watchable.ModuleManifests, eventPath) {
- w.modMgr.EnqueueModuleOp(mod.Path, op.OpTypeParseModuleManifest, nil)
+ w.modMgr.EnqueueModuleOp(mod.Path, op.OpTypeParseModuleManifest,
+ decodeCalledModulesFunc(w.modMgr, w, mod.Path))
  return
  }
  if containsPath(mod.Watchable.PluginLockFiles, eventPath) {
@@ -175,7 +176,8 @@ func (w *watcher) processEvent(event fsnotify.Event) {
  return nil
  }
  if containsPath(mod.Watchable.ModuleManifests, path) {
- return w.modMgr.EnqueueModuleOp(mod.Path, op.OpTypeParseModuleManifest, nil)
+ return w.modMgr.EnqueueModuleOp(mod.Path, op.OpTypeParseModuleManifest,
+ decodeCalledModulesFunc(w.modMgr, w, mod.Path))
  }
  if containsPath(mod.Watchable.PluginLockFiles, path) {
  w.modMgr.EnqueueModuleOp(mod.Path, op.OpTypeObtainSchema, nil)
@@ -189,7 +191,8 @@ func (w *watcher) processEvent(event fsnotify.Event) {
  }
 
  if containsPath(mod.Watchable.ModuleManifests, eventPath) {
- w.modMgr.EnqueueModuleOp(mod.Path, op.OpTypeParseModuleManifest, nil)
+ w.modMgr.EnqueueModuleOp(mod.Path, op.OpTypeParseModuleManifest,
+ decodeCalledModulesFunc(w.modMgr, w, mod.Path))
  return
  }
 
@@ -224,6 +227,38 @@ func (w *watcher) processEvent(event fsnotify.Event) {
  }
 }
 
+func decodeCalledModulesFunc(modMgr ModuleManager, w Watcher, modPath string) DeferFunc {
+ return func(opErr error) {
+ if opErr != nil {
+ return
+ }
+
+ moduleCalls, err := modMgr.ModuleCalls(modPath)
+ if err != nil {
+ return
+ }
+
+ for _, mc := range moduleCalls {
+ fi, err := os.Stat(mc.Path)
+ if err != nil || !fi.IsDir() {
+ continue
+ }
+ modMgr.AddModule(mc.Path)
+
+ modMgr.EnqueueModuleOpWait(mc.Path, op.OpTypeParseModuleConfiguration)
+
+ modMgr.EnqueueModuleOp(mc.Path, op.OpTypeParseVariables, nil)
+ modMgr.EnqueueModuleOp(mc.Path, op.OpTypeLoadModuleMetadata, nil)
+ modMgr.EnqueueModuleOp(mc.Path, op.OpTypeDecodeReferenceTargets, nil)
+ modMgr.EnqueueModuleOp(mc.Path, op.OpTypeDecodeReferenceOrigins, nil)
+
+ if w != nil {
+ w.AddModule(mc.Path)
+ }
+ }
+ }
+}
+
 func containsPath(paths []string, path string) bool {
  for _, p := range paths {
  if pathcmp.PathEquals(p, path) {