Walk hierarchy to add root modules (#176)

* Walk hierarchy to add root modules

Because detection of the relevant root module for a file/dir
is a non trivial task, we also introduce candidate finder
which can report all potential candidates for root modules
in the context of a directory being initialized/opened.

* Add testdata

* rootmodule: Store data in slice for consistent results

This turns lookups from O(1) to O(n), but the n is in most cases so low
that it likely doesn't matter anyway.

* Shorten warning for multiple candidates

* Update internal/terraform/schema/schema_storage.go

Co-authored-by: Paul Tyng <ptyng@hashicorp.com>

* rootmodule: embed interfaces

Co-authored-by: Paul Tyng <ptyng@hashicorp.com>

internal/context/context.go

@@ -29,6 +29,8 @@ var (
  ctxRootModuleMngr = &contextKey{"root module manager"}
  ctxParserFinder = &contextKey{"parser finder"}
  ctxTfExecFinder = &contextKey{"terraform exec finder"}
+ ctxRootModuleCaFi = &contextKey{"root module candidate finder"}
+ ctxRootDir = &contextKey{"root directory"}
 )
 
 func missingContextErr(ctxKey *contextKey) *MissingContextErr {
@@ -149,3 +151,37 @@ func TerraformExecPath(ctx context.Context) (string, bool) {
  path, ok := ctx.Value(ctxTfExecPath).(string)
  return path, ok
 }
+
+func WithRootModuleCandidateFinder(rmcf rootmodule.RootModuleCandidateFinder, ctx context.Context) context.Context {
+ return context.WithValue(ctx, ctxRootModuleCaFi, rmcf)
+}
+
+func RootModuleCandidateFinder(ctx context.Context) (rootmodule.RootModuleCandidateFinder, error) {
+ cf, ok := ctx.Value(ctxRootModuleCaFi).(rootmodule.RootModuleCandidateFinder)
+ if !ok {
+ return nil, missingContextErr(ctxRootModuleCaFi)
+ }
+ return cf, nil
+}
+
+func WithRootDirectory(dir *string, ctx context.Context) context.Context {
+ return context.WithValue(ctx, ctxRootDir, dir)
+}
+
+func SetRootDirectory(ctx context.Context, dir string) error {
+ rootDir, ok := ctx.Value(ctxRootDir).(*string)
+ if !ok {
+ return missingContextErr(ctxRootDir)
+ }
+
+ *rootDir = dir
+ return nil
+}
+
+func RootDirectory(ctx context.Context) (string, bool) {
+ rootDir, ok := ctx.Value(ctxRootDir).(*string)
+ if !ok {
+ return "", false
+ }
+ return *rootDir, true
+}

internal/terraform/rootmodule/root_module.go

@@ -18,6 +18,7 @@ import (
 
 type rootModule struct {
  ctx context.Context
+ path string
  logger *log.Logger
  pluginLockFile File
  moduleManifestFile File
@@ -39,9 +40,10 @@ type rootModule struct {
  moduleMu *sync.RWMutex
 }
 
-func newRootModule(ctx context.Context) *rootModule {
+func newRootModule(ctx context.Context, dir string) *rootModule {
  return &rootModule{
  ctx: ctx,
+ path: dir,
  logger: defaultLogger,
  pluginMu: &sync.RWMutex{},
  moduleMu: &sync.RWMutex{},
@@ -51,7 +53,7 @@ func newRootModule(ctx context.Context) *rootModule {
 var defaultLogger = log.New(ioutil.Discard, "", 0)
 
 func NewRootModule(ctx context.Context, dir string) (RootModule, error) {
- rm := newRootModule(ctx)
+ rm := newRootModule(ctx, dir)
 
  d := &discovery.Discovery{}
  rm.tfDiscoFunc = d.LookPath
@@ -63,15 +65,16 @@ func NewRootModule(ctx context.Context, dir string) (RootModule, error) {
  return ss
  }
 
- return rm, rm.init(ctx, dir)
+ return rm, rm.init(ctx)
 }
 
 func (rm *rootModule) SetLogger(logger *log.Logger) {
  rm.logger = logger
 }
 
-func (rm *rootModule) init(ctx context.Context, dir string) error {
- tf, err := rm.initTfExecutor(dir)
+func (rm *rootModule) init(ctx context.Context) error {
+ rm.logger.Printf("initing new root module: %s", rm.path)
+ tf, err := rm.initTfExecutor(rm.path)
  if err != nil {
  return err
  }
@@ -105,11 +108,11 @@ func (rm *rootModule) init(ctx context.Context, dir string) error {
  rm.tfExec = tf
  rm.tfVersion = version
 
- err = rm.initPluginCache(dir)
+ err = rm.initPluginCache(rm.path)
  if err != nil {
  return fmt.Errorf("plugin initialization failed: %w", err)
  }
- err = rm.initModuleCache(dir)
+ err = rm.initModuleCache(rm.path)
  if err != nil {
  return err
  }
@@ -216,6 +219,10 @@ func (rm *rootModule) initModuleCache(dir string) error {
  return rm.UpdateModuleManifest(lf)
 }
 
+func (rm *rootModule) Path() string {
+ return rm.path
+}
+
 func (rm *rootModule) UpdateModuleManifest(lockFile File) error {
  rm.moduleMu.Lock()
  rm.logger.Printf("updating module manifest based on %s ...", lockFile.Path())

internal/terraform/rootmodule/root_module_manager.go

@@ -16,7 +16,7 @@ import (
 )
 
 type rootModuleManager struct {
- rms map[string]*rootModule
+ rms []*rootModule
  tfExecPath string
  tfExecTimeout time.Duration
  tfExecLogPath string
@@ -31,15 +31,15 @@ func NewRootModuleManager(ctx context.Context) RootModuleManager {
 
 func newRootModuleManager(ctx context.Context) *rootModuleManager {
  rmm := &rootModuleManager{
- rms: make(map[string]*rootModule, 0),
+ rms: make([]*rootModule, 0),
  logger: defaultLogger,
  }
  rmm.newRootModule = rmm.defaultRootModuleFactory
  return rmm
 }
 
 func (rmm *rootModuleManager) defaultRootModuleFactory(ctx context.Context, dir string) (*rootModule, error) {
- rm := newRootModule(ctx)
+ rm := newRootModule(ctx, dir)
 
  rm.SetLogger(rmm.logger)
 
@@ -52,7 +52,7 @@ func (rmm *rootModuleManager) defaultRootModuleFactory(ctx context.Context, dir
  rm.tfExecTimeout = rmm.tfExecTimeout
  rm.tfExecLogPath = rmm.tfExecLogPath
 
- return rm, rm.init(ctx, dir)
+ return rm, rm.init(ctx)
 }
 
 func (rmm *rootModuleManager) SetTerraformExecPath(path string) {
@@ -76,8 +76,7 @@ func (rmm *rootModuleManager) AddRootModule(dir string) error {
 
  // TODO: Follow symlinks (requires proper test data)
 
- _, exists := rmm.rms[dir]
- if exists {
+ if rmm.exists(dir) {
  return fmt.Errorf("root module %s was already added", dir)
  }
 
@@ -86,33 +85,66 @@ func (rmm *rootModuleManager) AddRootModule(dir string) error {
  return err
  }
 
- rmm.rms[dir] = rm
+ rmm.rms = append(rmm.rms, rm)
 
  return nil
 }
 
-func (rmm *rootModuleManager) RootModuleByPath(path string) (RootModule, error) {
+func (rmm *rootModuleManager) exists(dir string) bool {
+ for _, rm := range rmm.rms {
+ if rm.Path() == dir {
+ return true
+ }
+ }
+ return false
+}
+
+func (rmm *rootModuleManager) rootModuleByPath(dir string) *rootModule {
+ for _, rm := range rmm.rms {
+ if rm.Path() == dir {
+ return rm
+ }
+ }
+ return nil
+}
+
+func (rmm *rootModuleManager) RootModuleCandidatesByPath(path string) []string {
  path = filepath.Clean(path)
 
  // TODO: Follow symlinks (requires proper test data)
 
- if rm, ok := rmm.rms[path]; ok {
+ if rmm.exists(path) {
  rmm.logger.Printf("direct root module lookup succeeded: %s", path)
- return rm, nil
+ return []string{path}
  }
 
  dir := rootModuleDirFromFilePath(path)
- if rm, ok := rmm.rms[dir]; ok {
+ if rmm.exists(dir) {
  rmm.logger.Printf("dir-based root module lookup succeeded: %s", dir)
- return rm, nil
+ return []string{dir}
  }
 
+ candidates := make([]string, 0)
  for _, rm := range rmm.rms {
- rmm.logger.Printf("looking up %s in module references", dir)
+ rmm.logger.Printf("looking up %s in module references of %s", dir, rm.Path())
  if rm.ReferencesModulePath(dir) {
  rmm.logger.Printf("module-ref-based root module lookup succeeded: %s", dir)
- return rm, nil
+ candidates = append(candidates, rm.Path())
+ }
+ }
+
+ return candidates
+}
+
+func (rmm *rootModuleManager) RootModuleByPath(path string) (RootModule, error) {
+ candidates := rmm.RootModuleCandidatesByPath(path)
+ if len(candidates) > 0 {
+ firstMatch := candidates[0]
+ if !rmm.exists(firstMatch) {
+ return nil, fmt.Errorf("Discovered root module %s not available,"+
+ " this is most likely a bug, please report it", firstMatch)
  }
+ return rmm.rootModuleByPath(firstMatch), nil
  }
 
  return nil, &RootModuleNotFoundErr{path}

internal/terraform/rootmodule/root_module_manager_mock.go

@@ -22,29 +22,38 @@ type RootModuleMockFactory struct {
 }
 
 func (rmf *RootModuleMockFactory) New(ctx context.Context, dir string) (*rootModule, error) {
- rm, ok := rmf.rmm[dir]
+ rmm, ok := rmf.rmm[dir]
  if !ok {
  return nil, fmt.Errorf("unexpected root module requested: %s (%d available: %#v)", dir, len(rmf.rmm), rmf.rmm)
  }
 
- w := newRootModule(ctx)
- w.SetLogger(rmf.logger)
+ mock := NewRootModuleMock(ctx, rmm, dir)
+ mock.SetLogger(rmf.logger)
+ return mock, mock.init(ctx)
+}
+
+func NewRootModuleMock(ctx context.Context, rmm *RootModuleMock, dir string) *rootModule {
+ rm := newRootModule(ctx, dir)
 
  md := &discovery.MockDiscovery{Path: "tf-mock"}
- w.tfDiscoFunc = md.LookPath
+ rm.tfDiscoFunc = md.LookPath
 
  // For now, until we have better testing strategy to mimic real lock files
- w.ignorePluginCache = true
+ rm.ignorePluginCache = true
 
- w.tfNewExecutor = exec.MockExecutor(rm.TerraformExecQueue)
+ rm.tfNewExecutor = exec.MockExecutor(rmm.TerraformExecQueue)
 
- if rm.ProviderSchemas == nil {
- w.newSchemaStorage = schema.MockStorage(rm.ProviderSchemas)
+ if rmm.ProviderSchemas == nil {
+ rm.newSchemaStorage = func() *schema.Storage {
+ ss := schema.NewStorage()
+ ss.SetSynchronous()
+ return ss
+ }
  } else {
- w.newSchemaStorage = schema.NewStorage
+ rm.newSchemaStorage = schema.MockStorage(rmm.ProviderSchemas)
  }
 
- return w, w.init(ctx, dir)
+ return rm
 }
 
 func NewRootModuleManagerMock(m map[string]*RootModuleMock) RootModuleManagerFactory {