From eb7e47654e310a44d275a9c2c06a8c7cc4bf8f13 Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Thu, 13 May 2021 12:44:24 +0100 Subject: [PATCH] Support multiple folders natively --- internal/cmd/inspect_module_command.go | 3 +- .../handlers/did_change_workspace_folders.go | 86 +++++++++++++++++ internal/langserver/handlers/handlers_test.go | 5 +- internal/langserver/handlers/initialize.go | 51 ++++++++-- internal/langserver/handlers/service.go | 11 +++ internal/state/module.go | 13 +++ internal/terraform/module/module_loader.go | 4 + internal/terraform/module/module_manager.go | 5 + .../terraform/module/module_manager_test.go | 9 +- internal/terraform/module/module_ops_queue.go | 14 +++ internal/terraform/module/types.go | 2 + internal/terraform/module/walker.go | 94 ++++++++++++++++--- internal/terraform/module/walker_queue.go | 85 +++++++++++++++++ internal/terraform/module/watcher.go | 25 +++++ internal/terraform/module/watcher_mock.go | 4 + 15 files changed, 384 insertions(+), 27 deletions(-) create mode 100644 internal/langserver/handlers/did_change_workspace_folders.go create mode 100644 internal/terraform/module/walker_queue.go diff --git a/internal/cmd/inspect_module_command.go b/internal/cmd/inspect_module_command.go index 161bca0f3..0200543c6 100644 --- a/internal/cmd/inspect_module_command.go +++ b/internal/cmd/inspect_module_command.go @@ -101,7 +101,8 @@ func (c *InspectModuleCommand) inspect(rootPath string) error { c.logger, syscall.SIGINT, syscall.SIGTERM) defer cancel() - err = walker.StartWalking(ctx, rootPath) + walker.EnqueuePath(rootPath) + err = walker.StartWalking(ctx) if err != nil { return err } diff --git a/internal/langserver/handlers/did_change_workspace_folders.go b/internal/langserver/handlers/did_change_workspace_folders.go new file mode 100644 index 000000000..4683e9ed3 --- /dev/null +++ b/internal/langserver/handlers/did_change_workspace_folders.go @@ -0,0 +1,86 @@ +package handlers + +import ( + "context" + "fmt" + + "github.com/creachadair/jrpc2" + lsctx "github.com/hashicorp/terraform-ls/internal/context" + lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/uri" +) + +func (lh *logHandler) DidChangeWorkspaceFolders(ctx context.Context, params lsp.DidChangeWorkspaceFoldersParams) error { + watcher, err := lsctx.Watcher(ctx) + if err != nil { + return err + } + + walker, err := lsctx.ModuleWalker(ctx) + if err != nil { + return err + } + + mm, err := lsctx.ModuleManager(ctx) + if err != nil { + return err + } + + for _, removed := range params.Event.Removed { + modPath, err := pathFromDocumentURI(removed.URI) + if err != nil { + jrpc2.ServerFromContext(ctx).Notify(ctx, "window/showMessage", &lsp.ShowMessageParams{ + Type: lsp.Warning, + Message: fmt.Sprintf("Ignoring removed workspace folder %s: %s."+ + " This is most likely bug, please report it.", removed.URI, err), + }) + continue + } + walker.RemovePathFromQueue(modPath) + + err = watcher.RemoveModule(modPath) + if err != nil { + lh.logger.Printf("failed to remove module from watcher: %s", err) + continue + } + + callers, err := mm.CallersOfModule(modPath) + if err != nil { + lh.logger.Printf("failed to remove module from watcher: %s", err) + continue + } + if len(callers) == 0 { + mm.RemoveModule(modPath) + } + } + + for _, added := range params.Event.Added { + modPath, err := pathFromDocumentURI(added.URI) + if err != nil { + jrpc2.ServerFromContext(ctx).Notify(ctx, "window/showMessage", &lsp.ShowMessageParams{ + Type: lsp.Warning, + Message: fmt.Sprintf("Ignoring new workspace folder %s: %s."+ + " This is most likely bug, please report it.", added.URI, err), + }) + continue + } + err = watcher.AddModule(modPath) + if err != nil { + lh.logger.Printf("failed to add module to watcher: %s", err) + continue + } + + walker.EnqueuePath(modPath) + } + + return nil +} + +func pathFromDocumentURI(docUri string) (string, error) { + rawPath, err := uri.PathFromURI(docUri) + if err != nil { + return "", err + } + + return cleanupPath(rawPath) +} diff --git a/internal/langserver/handlers/handlers_test.go b/internal/langserver/handlers/handlers_test.go index 36d1ef954..ed7ce2f7d 100644 --- a/internal/langserver/handlers/handlers_test.go +++ b/internal/langserver/handlers/handlers_test.go @@ -55,7 +55,10 @@ func initializeResponse(t *testing.T, commandPrefix string) string { "full": false }, "workspace": { - "workspaceFolders": {} + "workspaceFolders": { + "supported": true, + "changeNotifications": "workspace/didChangeWorkspaceFolders" + } } }, "serverInfo": { diff --git a/internal/langserver/handlers/initialize.go b/internal/langserver/handlers/initialize.go index 359d84001..16df0fdaa 100644 --- a/internal/langserver/handlers/initialize.go +++ b/internal/langserver/handlers/initialize.go @@ -28,6 +28,12 @@ func (lh *logHandler) Initialize(ctx context.Context, params lsp.InitializeParam DocumentFormattingProvider: true, DocumentSymbolProvider: true, WorkspaceSymbolProvider: true, + Workspace: lsp.WorkspaceGn{ + WorkspaceFolders: lsp.WorkspaceFoldersGn{ + Supported: true, + ChangeNotifications: "workspace/didChangeWorkspaceFolders", + }, + }, }, } @@ -114,6 +120,36 @@ func (lh *logHandler) Initialize(ctx context.Context, params lsp.InitializeParam } cfgOpts := out.Options + if !clientCaps.Workspace.WorkspaceFolders && len(params.WorkspaceFolders) > 0 { + jrpc2.ServerFromContext(ctx).Notify(ctx, "window/showMessage", &lsp.ShowMessageParams{ + Type: lsp.Warning, + Message: "Client sent workspace folders despite not declaring support. " + + "Please report this as a bug.", + }) + } + + walker, err := lsctx.ModuleWalker(ctx) + if err != nil { + return serverCaps, err + } + walker.SetLogger(lh.logger) + + if len(params.WorkspaceFolders) > 0 { + for _, folderPath := range params.WorkspaceFolders { + modPath, err := pathFromDocumentURI(folderPath.URI) + if err != nil { + jrpc2.ServerFromContext(ctx).Notify(ctx, "window/showMessage", &lsp.ShowMessageParams{ + Type: lsp.Warning, + Message: fmt.Sprintf("Ignoring workspace folder %s: %s."+ + " This is most likely bug, please report it.", folderPath.URI, err), + }) + continue + } + + walker.EnqueuePath(modPath) + } + } + // Static user-provided paths take precedence over dynamic discovery if len(cfgOpts.ModulePaths) > 0 { lh.logger.Printf("Attempting to add %d static module paths", len(cfgOpts.ModulePaths)) @@ -146,17 +182,12 @@ func (lh *logHandler) Initialize(ctx context.Context, params lsp.InitializeParam excludeModulePaths = append(excludeModulePaths, modPath) } - walker, err := lsctx.ModuleWalker(ctx) - if err != nil { - return serverCaps, err - } - - walker.SetLogger(lh.logger) walker.SetExcludeModulePaths(excludeModulePaths) + walker.EnqueuePath(fh.Dir()) + // Walker runs asynchronously so we're intentionally *not* // passing the request context here - bCtx := context.Background() - err = walker.StartWalking(bCtx, fh.Dir()) + err = walker.StartWalking(context.Background()) return serverCaps, err } @@ -171,6 +202,10 @@ func resolvePath(rootDir, rawPath string) (string, error) { path = filepath.Join(rootDir, rawPath) } + return cleanupPath(path) +} + +func cleanupPath(path string) (string, error) { absPath, err := filepath.EvalSymlinks(path) return toLowerVolumePath(absPath), err } diff --git a/internal/langserver/handlers/service.go b/internal/langserver/handlers/service.go index b6cd0f4b8..677aadd4b 100644 --- a/internal/langserver/handlers/service.go +++ b/internal/langserver/handlers/service.go @@ -293,6 +293,17 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { return handle(ctx, req, lh.TextDocumentDidSave) }, + "workspace/didChangeWorkspaceFolders": func(ctx context.Context, req *jrpc2.Request) (interface{}, error) { + err := session.CheckInitializationIsConfirmed() + if err != nil { + return nil, err + } + + ctx = lsctx.WithModuleWalker(ctx, svc.walker) + ctx = lsctx.WithWatcher(ctx, svc.watcher) + + return handle(ctx, req, lh.DidChangeWorkspaceFolders) + }, "workspace/executeCommand": func(ctx context.Context, req *jrpc2.Request) (interface{}, error) { err := session.CheckInitializationIsConfirmed() if err != nil { diff --git a/internal/state/module.go b/internal/state/module.go index 765d9e3da..f141d8ac3 100644 --- a/internal/state/module.go +++ b/internal/state/module.go @@ -193,6 +193,19 @@ func (s *ModuleStore) Add(modPath string) error { return nil } +func (s *ModuleStore) Remove(modPath string) error { + txn := s.db.Txn(true) + defer txn.Abort() + + _, err := txn.DeleteAll(s.tableName, "id", modPath) + if err != nil { + return err + } + + txn.Commit() + return nil +} + func (s *ModuleStore) CallersOfModule(modPath string) ([]*Module, error) { txn := s.db.Txn(false) it, err := txn.Get(s.tableName, "id") diff --git a/internal/terraform/module/module_loader.go b/internal/terraform/module/module_loader.go index dc44078ab..ddb67a2dc 100644 --- a/internal/terraform/module/module_loader.go +++ b/internal/terraform/module/module_loader.go @@ -257,3 +257,7 @@ func (ml *moduleLoader) EnqueueModuleOp(modOp ModuleOperation) error { return nil } + +func (ml *moduleLoader) DequeueModule(modPath string) { + ml.queue.DequeueAllModuleOps(modPath) +} diff --git a/internal/terraform/module/module_manager.go b/internal/terraform/module/module_manager.go index 470996d8d..5da5d6d00 100644 --- a/internal/terraform/module/module_manager.go +++ b/internal/terraform/module/module_manager.go @@ -80,6 +80,11 @@ func (mm *moduleManager) AddModule(modPath string) (Module, error) { return mod, err } +func (mm *moduleManager) RemoveModule(modPath string) error { + mm.loader.DequeueModule(modPath) + return mm.moduleStore.Remove(modPath) +} + func (mm *moduleManager) EnqueueModuleOpWait(modPath string, opType op.OpType) error { modOp := NewModuleOperation(modPath, opType) mm.loader.EnqueueModuleOp(modOp) diff --git a/internal/terraform/module/module_manager_test.go b/internal/terraform/module/module_manager_test.go index 93ff17ff3..0946600bb 100644 --- a/internal/terraform/module/module_manager_test.go +++ b/internal/terraform/module/module_manager_test.go @@ -350,7 +350,8 @@ func TestModuleManager_ModuleCandidatesByPath(t *testing.T) { w := SyncWalker(fs, mm) w.SetLogger(testLogger()) - err = w.StartWalking(ctx, tc.walkerRoot) + w.EnqueuePath(tc.walkerRoot) + err = w.StartWalking(ctx) if err != nil { t.Fatal(err) } @@ -437,18 +438,18 @@ func TestSchemaForVariables(t *testing.T) { } mod.Meta.Variables = map[string]tfmodule.Variable{ - "name": tfmodule.Variable{ + "name": { Description: "name of the module", Type: cty.String, }, } expectedSchema := &schema.BodySchema{Attributes: map[string]*schema.AttributeSchema{ - "name": &schema.AttributeSchema{ + "name": { Description: lang.MarkupContent{ Value: "name of the module", Kind: lang.PlainTextKind, }, - Expr: schema.ExprConstraints{schema.LiteralTypeExpr{cty.String}}, + Expr: schema.ExprConstraints{schema.LiteralTypeExpr{Type: cty.String}}, }, }} diff --git a/internal/terraform/module/module_ops_queue.go b/internal/terraform/module/module_ops_queue.go index 59abf536a..7f418b1fa 100644 --- a/internal/terraform/module/module_ops_queue.go +++ b/internal/terraform/module/module_ops_queue.go @@ -44,6 +44,20 @@ func (q *moduleOpsQueue) PopOp() (ModuleOperation, bool) { return modOp, true } +func (q *moduleOpsQueue) DequeueAllModuleOps(modPath string) { + q.mu.Lock() + defer q.mu.Unlock() + if q.q.Len() == 0 { + return + } + + for i, p := range q.q.ops { + if p.ModulePath == modPath { + q.q.ops = append(q.q.ops[:i], q.q.ops[:i+1]...) + } + } +} + func (q *moduleOpsQueue) Len() int { q.mu.Lock() defer q.mu.Unlock() diff --git a/internal/terraform/module/types.go b/internal/terraform/module/types.go index 23a22999f..20e675382 100644 --- a/internal/terraform/module/types.go +++ b/internal/terraform/module/types.go @@ -35,6 +35,7 @@ type ModuleManager interface { SetLogger(logger *log.Logger) AddModule(modPath string) (Module, error) + RemoveModule(modPath string) error EnqueueModuleOp(modPath string, opType op.OpType) error EnqueueModuleOpWait(modPath string, opType op.OpType) error CancelLoading() @@ -54,5 +55,6 @@ type Watcher interface { Stop() error SetLogger(*log.Logger) AddModule(string) error + RemoveModule(string) error IsModuleWatched(string) bool } diff --git a/internal/terraform/module/walker.go b/internal/terraform/module/walker.go index 8866d6eab..2e30c66eb 100644 --- a/internal/terraform/module/walker.go +++ b/internal/terraform/module/walker.go @@ -1,6 +1,7 @@ package module import ( + "container/heap" "context" "fmt" "io/ioutil" @@ -9,6 +10,7 @@ import ( "path/filepath" "sync" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/terraform-ls/internal/filesystem" "github.com/hashicorp/terraform-ls/internal/terraform/datadir" op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation" @@ -27,6 +29,8 @@ var ( } ) +type pathToWatch struct{} + type Walker struct { fs filesystem.Filesystem modMgr ModuleManager @@ -34,6 +38,10 @@ type Walker struct { logger *log.Logger sync bool + queue *walkerQueue + queueMu *sync.Mutex + pushChan chan struct{} + walking bool walkingMu *sync.RWMutex cancelFunc context.CancelFunc @@ -48,6 +56,9 @@ func NewWalker(fs filesystem.Filesystem, modMgr ModuleManager) *Walker { modMgr: modMgr, logger: discardLogger, walkingMu: &sync.RWMutex{}, + queue: newWalkerQueue(fs), + queueMu: &sync.Mutex{}, + pushChan: make(chan struct{}, 1), doneCh: make(chan struct{}, 0), } } @@ -84,7 +95,21 @@ func (w *Walker) setWalking(isWalking bool) { w.walking = isWalking } -func (w *Walker) StartWalking(ctx context.Context, path string) error { +func (w *Walker) EnqueuePath(path string) { + w.queueMu.Lock() + defer w.queueMu.Unlock() + heap.Push(w.queue, path) + + w.pushChan <- struct{}{} +} + +func (w *Walker) RemovePathFromQueue(path string) { + w.queueMu.Lock() + defer w.queueMu.Unlock() + w.queue.RemoveFromQueue(path) +} + +func (w *Walker) StartWalking(ctx context.Context) error { if w.IsWalking() { return fmt.Errorf("walker is already running") } @@ -95,19 +120,64 @@ func (w *Walker) StartWalking(ctx context.Context, path string) error { w.setWalking(true) if w.sync { - w.logger.Printf("synchronously walking through %s", path) - return w.walk(ctx, path) + var errs *multierror.Error + for { + w.queueMu.Lock() + if w.queue.Len() == 0 { + w.queueMu.Unlock() + w.Stop() + return errs.ErrorOrNil() + } + nextPath := heap.Pop(w.queue) + w.queueMu.Unlock() + + path := nextPath.(string) + w.logger.Printf("synchronously walking through %s", path) + err := w.walk(ctx, path) + if err != nil { + multierror.Append(errs, err) + } + } } - go func(w *Walker, path string) { - w.logger.Printf("asynchronously walking through %s", path) - err := w.walk(ctx, path) - if err != nil { - w.logger.Printf("async walking through %s failed: %s", path, err) - return + var nextPathToWalk = make(chan string) + + go func(w *Walker) { + for { + w.queueMu.Lock() + if w.queue.Len() == 0 { + w.queueMu.Unlock() + select { + case <-w.pushChan: + // block to avoid infinite loop + continue + case <-w.doneCh: + return + } + } + nextPath := heap.Pop(w.queue) + w.queueMu.Unlock() + path := nextPath.(string) + nextPathToWalk <- path } - w.logger.Printf("async walking through %s finished", path) - }(w, path) + }(w) + + go func(w *Walker, pathsChan chan string) { + for { + select { + case <-w.doneCh: + return + case path := <-pathsChan: + w.logger.Printf("asynchronously walking through %s", path) + err := w.walk(ctx, path) + if err != nil { + w.logger.Printf("async walking through %s failed: %s", path, err) + return + } + w.logger.Printf("async walking through %s finished", path) + } + } + }(w, nextPathToWalk) return nil } @@ -120,8 +190,6 @@ func (w *Walker) IsWalking() bool { } func (w *Walker) walk(ctx context.Context, rootPath string) error { - defer w.Stop() - // We ignore the passed FS and instead read straight from OS FS // because that would require reimplementing filepath.Walk and // the data directory should never be on the virtual filesystem anyway diff --git a/internal/terraform/module/walker_queue.go b/internal/terraform/module/walker_queue.go new file mode 100644 index 000000000..18bcb0077 --- /dev/null +++ b/internal/terraform/module/walker_queue.go @@ -0,0 +1,85 @@ +package module + +import ( + "container/heap" + + "github.com/hashicorp/terraform-ls/internal/filesystem" +) + +type walkerQueue struct { + paths []string + + fs filesystem.Filesystem +} + +var _ heap.Interface = &walkerQueue{} + +func newWalkerQueue(fs filesystem.Filesystem) *walkerQueue { + wq := &walkerQueue{ + paths: make([]string, 0), + fs: fs, + } + heap.Init(wq) + return wq +} + +func (q *walkerQueue) Push(x interface{}) { + path := x.(string) + + if q.pathIsEnqueued(path) { + // avoid duplicate entries + return + } + + q.paths = append(q.paths, path) +} + +func (q *walkerQueue) pathIsEnqueued(path string) bool { + for _, p := range q.paths { + if p == path { + return true + } + } + return false +} + +func (q *walkerQueue) RemoveFromQueue(path string) { + for i, p := range q.paths { + if p == path { + q.paths = append(q.paths[:i], q.paths[i+1:]...) + } + } +} + +func (q *walkerQueue) Swap(i, j int) { + q.paths[i], q.paths[j] = q.paths[j], q.paths[i] +} + +func (q *walkerQueue) Pop() interface{} { + old := q.paths + n := len(old) + item := old[n-1] + q.paths = old[0 : n-1] + return item +} + +func (q *walkerQueue) Len() int { + return len(q.paths) +} + +func (q *walkerQueue) Less(i, j int) bool { + return q.moduleOperationLess(q.paths[i], q.paths[j]) +} + +func (q *walkerQueue) moduleOperationLess(leftModPath, rightModPath string) bool { + leftOpen, rightOpen := 0, 0 + + if hasOpenFiles, _ := q.fs.HasOpenFiles(leftModPath); hasOpenFiles { + leftOpen = 1 + } + if hasOpenFiles, _ := q.fs.HasOpenFiles(rightModPath); hasOpenFiles { + rightOpen = 1 + } + + return leftOpen > rightOpen +} diff --git a/internal/terraform/module/watcher.go b/internal/terraform/module/watcher.go index d65eed183..401cb31b6 100644 --- a/internal/terraform/module/watcher.go +++ b/internal/terraform/module/watcher.go @@ -101,6 +101,31 @@ func (w *watcher) AddModule(modPath string) error { return nil } +func (w *watcher) RemoveModule(modPath string) error { + modPath = filepath.Clean(modPath) + + w.logger.Printf("removing module from watching: %s", modPath) + + for modI, mod := range w.modules { + if pathcmp.PathEquals(mod.Path, modPath) { + for _, wPath := range mod.Watched { + w.fw.Remove(wPath) + } + w.fw.Remove(mod.Path) + w.modules = append(w.modules[:modI], w.modules[modI+1:]...) + } + + for i, wp := range mod.Watched { + if pathcmp.PathEquals(wp, modPath) { + w.fw.Remove(wp) + mod.Watched = append(mod.Watched[:i], mod.Watched[i+1:]...) + } + } + } + + return nil +} + func (w *watcher) run(ctx context.Context) { for { select { diff --git a/internal/terraform/module/watcher_mock.go b/internal/terraform/module/watcher_mock.go index bc5b0d9d2..01f0398f9 100644 --- a/internal/terraform/module/watcher_mock.go +++ b/internal/terraform/module/watcher_mock.go @@ -27,6 +27,10 @@ func (w *mockWatcher) AddModule(string) error { return nil } +func (w *mockWatcher) RemoveModule(string) error { + return nil +} + func (w *mockWatcher) IsModuleWatched(string) bool { return false }