fix: check module existence & add in the same txn

This is to prevent a race condition

internal/state/module.go

@@ -230,6 +230,16 @@ func (s *ModuleStore) Add(modPath string) error {
  txn := s.db.Txn(true)
  defer txn.Abort()
 
+ err := s.add(txn, modPath)
+ if err != nil {
+ return err
+ }
+ txn.Commit()
+
+ return nil
+}
+
+func (s *ModuleStore) add(txn *memdb.Txn, modPath string) error {
  // TODO: Introduce Exists method to Txn?
  obj, err := txn.First(s.tableName, "id", modPath)
  if err != nil {
@@ -252,7 +262,6 @@ func (s *ModuleStore) Add(modPath string) error {
  return err
  }
 
- txn.Commit()
  return nil
 }
 
@@ -318,19 +327,25 @@ func (s *ModuleStore) ModuleByPath(path string) (*Module, error) {
  return mod, nil
 }
 
-func (s *ModuleStore) Exists(path string) (bool, error) {
- txn := s.db.Txn(false)
+func (s *ModuleStore) AddIfNotExists(path string) error {
+ txn := s.db.Txn(true)
+ defer txn.Abort()
 
  _, err := moduleByPath(txn, path)
  if err != nil {
  if IsModuleNotFound(err) {
- return false, nil
+ err := s.add(txn, path)
+ if err != nil {
+ return err
+ }
+ txn.Commit()
+ return nil
  }
 
- return false, err
+ return err
  }
 
- return true, nil
+ return nil
 }
 
 func (s *ModuleStore) ModuleCalls(modPath string) (tfmod.ModuleCalls, error) {

internal/walker/walker.go

@@ -56,8 +56,7 @@ type PathStore interface {
 }
 
 type ModuleStore interface {
- Exists(dir string) (bool, error)
- Add(dir string) error
+ AddIfNotExists(dir string) error
 }
 
 func NewWalker(fs fs.ReadDirFS, pathStore PathStore, modStore ModuleStore, walkFunc WalkFunc) *Walker {
@@ -190,20 +189,14 @@ func (w *Walker) walk(ctx context.Context, dir document.DirHandle) error {
  dirIndexed = true
  w.logger.Printf("found module %s", dir)
 
- exists, err := w.modStore.Exists(dir.Path())
+ err := w.modStore.AddIfNotExists(dir.Path())
  if err != nil {
  return err
  }
- if !exists {
- err := w.modStore.Add(dir.Path())
- if err != nil {
- return err
- }
- }
 
  ids, err := w.walkFunc(ctx, dir)
  if err != nil {
- w.collectError(err)
+ w.collectError(fmt.Errorf("walkFunc: %w", err))
  }
  w.collectJobIds(ids)
  continue