From ab017e9f6fd03397c68cf46c617d1a001bd15ec8 Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Mon, 13 Jul 2020 09:06:41 +0100 Subject: [PATCH] New command: inspect-module --- commands/inspect_module_command.go | 178 ++++++++++++++++++ docs/TROUBLESHOOTING.md | 29 +++ internal/terraform/rootmodule/root_module.go | 26 ++- .../rootmodule/root_module_manager.go | 12 +- .../rootmodule/root_module_manager_test.go | 4 +- internal/terraform/rootmodule/types.go | 5 +- internal/terraform/rootmodule/walker.go | 45 +++-- langserver/handlers/initialize.go | 7 +- langserver/handlers/service.go | 10 +- main.go | 5 + 10 files changed, 299 insertions(+), 22 deletions(-) create mode 100644 commands/inspect_module_command.go diff --git a/commands/inspect_module_command.go b/commands/inspect_module_command.go new file mode 100644 index 000000000..e89f66513 --- /dev/null +++ b/commands/inspect_module_command.go @@ -0,0 +1,178 @@ +package commands + +import ( + "context" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/hashicorp/go-multierror" + ictx "github.com/hashicorp/terraform-ls/internal/context" + "github.com/hashicorp/terraform-ls/internal/terraform/rootmodule" + "github.com/hashicorp/terraform-ls/logging" + "github.com/mitchellh/cli" +) + +type InspectModuleCommand struct { + Ui cli.Ui + Verbose bool + + logger *log.Logger +} + +func (c *InspectModuleCommand) flags() *flag.FlagSet { + fs := defaultFlagSet("debug") + fs.BoolVar(&c.Verbose, "verbose", false, "whether to enable verbose output") + fs.Usage = func() { c.Ui.Error(c.Help()) } + return fs +} + +func (c *InspectModuleCommand) Run(args []string) int { + f := c.flags() + if err := f.Parse(args); err != nil { + c.Ui.Error(fmt.Sprintf("Error parsing command-line flags: %s", err)) + return 1 + } + + if f.NArg() != 1 { + c.Ui.Output(fmt.Sprintf("expected exactly 1 argument (%d given): %q", + f.NArg(), c.flags().Args())) + return 1 + } + + path := f.Arg(0) + + var logDestination io.Writer + if c.Verbose { + logDestination = os.Stderr + } else { + logDestination = ioutil.Discard + } + + c.logger = logging.NewLogger(logDestination) + + err := c.inspect(path) + if err != nil { + c.Ui.Output(err.Error()) + return 1 + } + + return 0 +} + +func (c *InspectModuleCommand) inspect(rootPath string) error { + rootPath, err := filepath.Abs(rootPath) + if err != nil { + return err + } + + fi, err := os.Stat(rootPath) + if err != nil { + return err + } + + if !fi.IsDir() { + return fmt.Errorf("expected %s to be a directory", rootPath) + } + + rmm := rootmodule.NewRootModuleManager() + rmm.SetLogger(c.logger) + walker := rootmodule.NewWalker() + walker.SetLogger(c.logger) + + ctx, cancel := ictx.WithSignalCancel(context.Background(), + c.logger, syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + err = walker.StartWalking(ctx, rootPath, func(ctx context.Context, dir string) error { + rm, err := rmm.AddAndStartLoadingRootModule(ctx, dir) + if err != nil { + return err + } + <-rm.LoadingDone() + + return nil + }) + if err != nil { + return err + } + + <-walker.Done() + + modules := rmm.ListRootModules() + c.Ui.Output(fmt.Sprintf("%d root modules found in total at %s", len(modules), rootPath)) + for _, rm := range modules { + errs := &multierror.Error{} + + err := rm.LoadError() + if err != nil { + var ok bool + errs, ok = err.(*multierror.Error) + if !ok { + return err + } + } + errs.ErrorFormat = formatErrors + + modules := formatModuleRecords(rm.Modules()) + subModules := fmt.Sprintf("%d modules", len(modules)) + if len(modules) > 0 { + subModules += "\n" + for _, m := range modules { + subModules += fmt.Sprintf(" - %s", m) + } + } + + c.Ui.Output(fmt.Sprintf(` - %s + - %s + - %s`, rm.Path(), errs, subModules)) + } + c.Ui.Output("") + + return nil +} + +func formatErrors(errors []error) string { + if len(errors) == 0 { + return "0 errors" + } + + out := fmt.Sprintf("%d errors:\n", len(errors)) + for _, err := range errors { + out += fmt.Sprintf(" - %s\n", err) + } + return strings.TrimSpace(out) +} + +func formatModuleRecords(mds []rootmodule.ModuleRecord) []string { + out := make([]string, 0) + for _, m := range mds { + if m.IsRoot() { + continue + } + if m.IsExternal() { + out = append(out, "EXTERNAL(%s)", m.SourceAddr) + continue + } + out = append(out, fmt.Sprintf("%s (%s)", m.Dir, m.SourceAddr)) + } + return out +} + +func (c *InspectModuleCommand) Help() string { + helpText := ` +Usage: terraform-ls inspect-module [path] + +` + c.Synopsis() + "\n\n" + helpForFlags(c.flags()) + return strings.TrimSpace(helpText) +} + +func (c *InspectModuleCommand) Synopsis() string { + return "Lists available debug items" +} diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 3d946ef87..7e396f7be 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -88,3 +88,32 @@ $ terraform-ls serve \ ``` The target file will be truncated before being written into. + +## "No root module found for ... functionality may be limited" + +Most of the language server features depend on initialized root modules +(i.e. folder with `*.tf` files where you ran `terraform init` successfully). +and server's ability to discover them within the hierarchy and match them +with files being open in the editor. + +This functionality should cover many hierarchies, but it may not cover yours. +If it appears that root modules aren't being discovered or matched the way +they should be, it can be useful to use `inspect-module` to obtain +the discovery results and provide them to maintainers in a bug report. + +Point it to the same directory that you tried to open in your IDE/editor +and wait for the output - it may take some seconds or low minutes +depending on the complexity of your hierarchy and number of root modules in it. + +``` +$ terraform-ls inspect-module /path/to/dir +``` + +## "Unable to retrieve schemas for ..." + +The process of obtaining the schema currently requires access to the state, +which in turn means that if the code itself doesn't have enough context +to obtain the state and/or there isn't context available from config file(s) +in standard locations you may need to provide that extra context. + +See https://github.com/hashicorp/terraform-ls/issues/128 for more. diff --git a/internal/terraform/rootmodule/root_module.go b/internal/terraform/rootmodule/root_module.go index 7d5e16293..3d1d5af2a 100644 --- a/internal/terraform/rootmodule/root_module.go +++ b/internal/terraform/rootmodule/root_module.go @@ -25,6 +25,7 @@ type rootModule struct { // loading isLoading bool isLoadingMu *sync.RWMutex + loadingDone <-chan struct{} cancelLoading context.CancelFunc loadErr error loadErrMu *sync.RWMutex @@ -146,17 +147,32 @@ func (rm *rootModule) discoverModuleCache(dir string) error { return nil } +func (rm *rootModule) Modules() []ModuleRecord { + rm.moduleMu.Lock() + defer rm.moduleMu.Unlock() + if rm.moduleManifest == nil { + return []ModuleRecord{} + } + + return rm.moduleManifest.Records +} + func (rm *rootModule) SetLogger(logger *log.Logger) { rm.logger = logger } -func (rm *rootModule) StartLoading() { +func (rm *rootModule) StartLoading() error { + if !rm.IsLoadingDone() { + return fmt.Errorf("root module is already being loaded") + } ctx, cancelFunc := context.WithCancel(context.Background()) rm.cancelLoading = cancelFunc + rm.loadingDone = ctx.Done() go func(ctx context.Context) { rm.setLoadErr(rm.load(ctx)) }(ctx) + return nil } func (rm *rootModule) CancelLoading() { @@ -166,6 +182,10 @@ func (rm *rootModule) CancelLoading() { rm.setLoadingState(false) } +func (rm *rootModule) LoadingDone() <-chan struct{} { + return rm.loadingDone +} + func (rm *rootModule) load(ctx context.Context) error { var errs *multierror.Error defer rm.CancelLoading() @@ -327,7 +347,7 @@ func (rm *rootModule) UpdateModuleManifest(lockFile File) error { mm, err := ParseModuleManifestFromFile(lockFile.Path()) if err != nil { - return err + return fmt.Errorf("failed to update module manifest: %w", err) } rm.moduleManifest = mm @@ -376,6 +396,8 @@ func (rm *rootModule) setSchemaLoaded(isLoaded bool) { } func (rm *rootModule) ReferencesModulePath(path string) bool { + rm.moduleMu.Lock() + defer rm.moduleMu.Unlock() if rm.moduleManifest == nil { return false } diff --git a/internal/terraform/rootmodule/root_module_manager.go b/internal/terraform/rootmodule/root_module_manager.go index a4b73ecaf..4f1b47d10 100644 --- a/internal/terraform/rootmodule/root_module_manager.go +++ b/internal/terraform/rootmodule/root_module_manager.go @@ -103,7 +103,10 @@ func (rmm *rootModuleManager) AddAndStartLoadingRootModule(ctx context.Context, } rmm.logger.Printf("asynchronously loading root module %s", dir) - rm.StartLoading() + err = rm.StartLoading() + if err != nil { + return rm, err + } return rm, nil } @@ -145,6 +148,13 @@ func (rmm *rootModuleManager) RootModuleCandidatesByPath(path string) RootModule return candidates } +func (rmm *rootModuleManager) ListRootModules() RootModules { + modules := make([]RootModule, 0) + for _, rm := range rmm.rms { + modules = append(modules, rm) + } + return modules +} func (rmm *rootModuleManager) RootModuleByPath(path string) (RootModule, error) { candidates := rmm.RootModuleCandidatesByPath(path) if len(candidates) > 0 { diff --git a/internal/terraform/rootmodule/root_module_manager_test.go b/internal/terraform/rootmodule/root_module_manager_test.go index 5dabb7921..d1ab00f9d 100644 --- a/internal/terraform/rootmodule/root_module_manager_test.go +++ b/internal/terraform/rootmodule/root_module_manager_test.go @@ -435,7 +435,9 @@ func TestRootModuleManager_RootModuleCandidatesByPath(t *testing.T) { t.Run(fmt.Sprintf("%d-%s/%s", i, tc.name, base), func(t *testing.T) { rmm := testRootModuleManager(t) w := MockWalker() - err := w.StartWalking(tc.walkerRoot, func(ctx context.Context, rmPath string) error { + w.SetLogger(testLogger()) + ctx := context.Background() + err := w.StartWalking(ctx, tc.walkerRoot, func(ctx context.Context, rmPath string) error { _, err := rmm.AddAndStartLoadingRootModule(ctx, rmPath) return err }) diff --git a/internal/terraform/rootmodule/types.go b/internal/terraform/rootmodule/types.go index 1d62b9373..358745dbd 100644 --- a/internal/terraform/rootmodule/types.go +++ b/internal/terraform/rootmodule/types.go @@ -42,6 +42,7 @@ type RootModuleManager interface { SetTerraformExecTimeout(timeout time.Duration) AddAndStartLoadingRootModule(ctx context.Context, dir string) (RootModule, error) + ListRootModules() RootModules PathsToWatch() []string RootModuleByPath(path string) (RootModule, error) CancelLoading() @@ -60,8 +61,9 @@ func (rms RootModules) Paths() []string { type RootModule interface { Path() string LoadError() error - StartLoading() + StartLoading() error IsLoadingDone() bool + LoadingDone() <-chan struct{} IsKnownPluginLockFile(path string) bool IsKnownModuleManifestFile(path string) bool PathsToWatch() []string @@ -72,6 +74,7 @@ type RootModule interface { IsParserLoaded() bool TerraformFormatter() (exec.Formatter, error) IsTerraformLoaded() bool + Modules() []ModuleRecord } type RootModuleFactory func(context.Context, string) (*rootModule, error) diff --git a/internal/terraform/rootmodule/walker.go b/internal/terraform/rootmodule/walker.go index 2c25d9ea3..2487d7a59 100644 --- a/internal/terraform/rootmodule/walker.go +++ b/internal/terraform/rootmodule/walker.go @@ -7,6 +7,7 @@ import ( "log" "os" "path/filepath" + "sync" ) var ( @@ -23,18 +24,20 @@ var ( ) type Walker struct { - logger *log.Logger - sync bool - walking bool + logger *log.Logger + sync bool + walking bool + walkingMu *sync.RWMutex cancelFunc context.CancelFunc - doneCh chan struct{} + doneCh <-chan struct{} } func NewWalker() *Walker { return &Walker{ - logger: discardLogger, - doneCh: make(chan struct{}, 0), + logger: discardLogger, + walkingMu: &sync.RWMutex{}, + doneCh: make(chan struct{}, 0), } } @@ -49,20 +52,31 @@ func (w *Walker) Stop() { w.cancelFunc() } - if w.walking { - w.walking = false - w.doneCh <- struct{}{} + if w.IsWalking() { + w.logger.Println("stopping walker") + w.setWalking(false) } } -func (w *Walker) StartWalking(path string, wf WalkFunc) error { - if w.walking { +func (w *Walker) setWalking(isWalking bool) { + w.walkingMu.Lock() + defer w.walkingMu.Unlock() + w.walking = isWalking +} + +func (w *Walker) Done() <-chan struct{} { + return w.doneCh +} + +func (w *Walker) StartWalking(ctx context.Context, path string, wf WalkFunc) error { + if w.IsWalking() { return fmt.Errorf("walker is already running") } - ctx, cancelFunc := context.WithCancel(context.Background()) + ctx, cancelFunc := context.WithCancel(ctx) w.cancelFunc = cancelFunc - w.walking = true + w.doneCh = ctx.Done() + w.setWalking(true) if w.sync { w.logger.Printf("synchronously walking through %s", path) @@ -83,10 +97,14 @@ func (w *Walker) StartWalking(path string, wf WalkFunc) error { } func (w *Walker) IsWalking() bool { + w.walkingMu.RLock() + defer w.walkingMu.RUnlock() + return w.walking } func (w *Walker) walk(ctx context.Context, rootPath string, wf WalkFunc) error { + defer w.Stop() err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { select { case <-w.doneCh: @@ -123,7 +141,6 @@ func (w *Walker) walk(ctx context.Context, rootPath string, wf WalkFunc) error { return nil }) w.logger.Printf("walking of %s finished", rootPath) - w.walking = false return err } diff --git a/langserver/handlers/initialize.go b/langserver/handlers/initialize.go index b91c33260..c57e4c47c 100644 --- a/langserver/handlers/initialize.go +++ b/langserver/handlers/initialize.go @@ -103,7 +103,12 @@ func (lh *logHandler) Initialize(ctx context.Context, params lsp.InitializeParam } walker.SetLogger(lh.logger) - err = walker.StartWalking(fh.Dir(), func(ctx context.Context, dir string) error { + + // Walker runs asynchronously so we're intentionally *not* + // passing the request context here + ctx = context.Background() + + err = walker.StartWalking(ctx, fh.Dir(), func(ctx context.Context, dir string) error { lh.logger.Printf("Adding root module: %s", dir) rm, err := rmm.AddAndStartLoadingRootModule(ctx, dir) if err != nil { diff --git a/langserver/handlers/service.go b/langserver/handlers/service.go index 3f6da928a..3512ef5c6 100644 --- a/langserver/handlers/service.go +++ b/langserver/handlers/service.go @@ -99,7 +99,10 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { } if w.IsKnownPluginLockFile(file.Path()) { svc.logger.Printf("detected plugin cache change, updating schema ...") - return w.UpdateSchemaCache(ctx, file) + err := w.UpdateSchemaCache(ctx, file) + if err != nil { + svc.logger.Printf(err.Error()) + } } return nil @@ -111,7 +114,10 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { } if rm.IsKnownModuleManifestFile(file.Path()) { svc.logger.Printf("detected module manifest change, updating ...") - return rm.UpdateModuleManifest(file) + err := rm.UpdateModuleManifest(file) + if err != nil { + svc.logger.Printf(err.Error()) + } } return nil diff --git a/main.go b/main.go index b38e67d6c..960e823aa 100644 --- a/main.go +++ b/main.go @@ -37,6 +37,11 @@ func main() { Version: VersionString(), }, nil }, + "inspect-module": func() (cli.Command, error) { + return &commands.InspectModuleCommand{ + Ui: ui, + }, nil + }, } exitStatus, err := c.Run()