Parse provider versions from lock file before obtaining schema

internal/indexer/walker.go

@@ -17,6 +17,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
  var errs *multierror.Error
 
  refCollectionDeps := make(job.IDs, 0)
+ providerVersionDeps := make(job.IDs, 0)
 
  parseId, err := idx.jobStore.EnqueueJob(job.Job{
  Dir: modHandle,
@@ -30,6 +31,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
  } else {
  ids = append(ids, parseId)
  refCollectionDeps = append(refCollectionDeps, parseId)
+ providerVersionDeps = append(providerVersionDeps, parseId)
  }
 
  var metaId job.ID
@@ -47,6 +49,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
  } else {
  ids = append(ids, metaId)
  refCollectionDeps = append(refCollectionDeps, metaId)
+ providerVersionDeps = append(providerVersionDeps, metaId)
  }
  }
 
@@ -98,41 +101,78 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
  dataDir := datadir.WalkDataDirOfModule(idx.fs, modHandle.Path())
  idx.logger.Printf("parsed datadir: %#v", dataDir)
 
- if dataDir.PluginLockFilePath != "" {
- pSchemaId, err := idx.jobStore.EnqueueJob(job.Job{
+ var modManifestId job.ID
+ if dataDir.ModuleManifestPath != "" {
+ // References are collected *after* manifest parsing
+ // so that we reflect any references to submodules.
+ modManifestId, err = idx.jobStore.EnqueueJob(job.Job{
  Dir: modHandle,
  Func: func(ctx context.Context) error {
- ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
- return module.ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
+ return module.ParseModuleManifest(idx.fs, idx.modStore, modHandle.Path())
+ },
+ Type: op.OpTypeParseModuleManifest.String(),
+ Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
+ return idx.decodeInstalledModuleCalls(modHandle)
  },
- Type: op.OpTypeObtainSchema.String(),
  })
  if err != nil {
  errs = multierror.Append(errs, err)
  } else {
- ids = append(ids, pSchemaId)
- refCollectionDeps = append(refCollectionDeps, pSchemaId)
+ ids = append(ids, modManifestId)
+ refCollectionDeps = append(refCollectionDeps, modManifestId)
+ // provider requirements may be within the (installed) modules
+ providerVersionDeps = append(providerVersionDeps, modManifestId)
  }
  }
 
- if dataDir.ModuleManifestPath != "" {
- // References are collected *after* manifest parsing
- // so that we reflect any references to submodules.
- modManifestId, err := idx.jobStore.EnqueueJob(job.Job{
+ if dataDir.PluginLockFilePath != "" {
+ pSchemaId, err := idx.jobStore.EnqueueJob(job.Job{
  Dir: modHandle,
  Func: func(ctx context.Context) error {
- return module.ParseModuleManifest(idx.fs, idx.modStore, modHandle.Path())
+ return module.ParseProviderVersions(idx.fs, idx.modStore, modHandle.Path())
  },
- Type: op.OpTypeParseModuleManifest.String(),
+ Type: op.OpTypeParseProviderVersions.String(),
+ DependsOn: providerVersionDeps,
  Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
- return idx.decodeInstalledModuleCalls(modHandle)
+ ids := make(job.IDs, 0)
+
+ pReqs, err := idx.modStore.ProviderRequirementsForModule(modHandle.Path())
+ if err != nil {
+ return ids, err
+ }
+
+ exist, err := idx.schemaStore.AllSchemasExist(pReqs)
+ if err != nil {
+ return ids, err
+ }
+ if exist {
+ idx.logger.Printf("Avoiding obtaining schemas as they all exist: %#v", pReqs)
+ // avoid obtaining schemas if we already have it
+ return ids, nil
+ }
+ idx.logger.Printf("Obtaining schemas for: %#v", pReqs)
+
+ id, err := idx.jobStore.EnqueueJob(job.Job{
+ Dir: modHandle,
+ Func: func(ctx context.Context) error {
+ ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
+ return module.ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
+ },
+ Type: op.OpTypeObtainSchema.String(),
+ })
+ if err != nil {
+ return ids, err
+ }
+ ids = append(ids, id)
+
+ return ids, nil
  },
  })
  if err != nil {
  errs = multierror.Append(errs, err)
  } else {
- ids = append(ids, modManifestId)
- refCollectionDeps = append(refCollectionDeps, modManifestId)
+ ids = append(ids, pSchemaId)
+ refCollectionDeps = append(refCollectionDeps, pSchemaId)
  }
  }
 

internal/indexer/watcher.go

@@ -37,15 +37,46 @@ func (idx *Indexer) PluginLockChanged(ctx context.Context, modHandle document.Di
  id, err := idx.jobStore.EnqueueJob(job.Job{
  Dir: modHandle,
  Func: func(ctx context.Context) error {
- ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
- eo, ok := exec.ExecutorOptsFromContext(ctx)
- if ok {
- ctx = exec.WithExecutorOpts(ctx, eo)
+ return module.ParseProviderVersions(idx.fs, idx.modStore, modHandle.Path())
+ },
+ Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
+ ids := make(job.IDs, 0)
+
+ mod, err := idx.modStore.ModuleByPath(modHandle.Path())
+ if err != nil {
+ return ids, err
+ }
+
+ exist, err := idx.schemaStore.AllSchemasExist(mod.Meta.ProviderRequirements)
+ if err != nil {
+ return ids, err
+ }
+ if exist {
+ // avoid obtaining schemas if we already have it
+ return ids, nil
+ }
+
+ id, err := idx.jobStore.EnqueueJob(job.Job{
+ Dir: modHandle,
+ Func: func(ctx context.Context) error {
+ ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
+ eo, ok := exec.ExecutorOptsFromContext(ctx)
+ if ok {
+ ctx = exec.WithExecutorOpts(ctx, eo)
+ }
+
+ return module.ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
+ },
+ Type: op.OpTypeObtainSchema.String(),
+ })
+ if err != nil {
+ return ids, err
  }
+ ids = append(ids, id)
 
- return module.ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
+ return ids, nil
  },
- Type: op.OpTypeObtainSchema.String(),
+ Type: op.OpTypeParseProviderVersions.String(),
  })
  if err != nil {
  return ids, err

internal/langserver/handlers/complete_test.go

@@ -4,6 +4,7 @@ import (
  "context"
  "encoding/json"
  "fmt"
+ "io/ioutil"
  "os"
  "path/filepath"
  "testing"
@@ -45,8 +46,13 @@ func TestModuleCompletion_withValidData_basic(t *testing.T) {
  tmpDir := TempDir(t)
  InitPluginCache(t, tmpDir.Path())
 
+ err := ioutil.WriteFile(filepath.Join(tmpDir.Path(), "main.tf"), []byte("provider \"test\" {\n\n}\n"), 0o755)
+ if err != nil {
+ t.Fatal(err)
+ }
+
  var testSchema tfjson.ProviderSchemas
- err := json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
+ err = json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
  if err != nil {
  t.Fatal(err)
  }
@@ -259,9 +265,13 @@ func TestModuleCompletion_withValidData_basic(t *testing.T) {
 func TestModuleCompletion_withValidDataAndSnippets(t *testing.T) {
  tmpDir := TempDir(t)
  InitPluginCache(t, tmpDir.Path())
+ err := ioutil.WriteFile(filepath.Join(tmpDir.Path(), "main.tf"), []byte("provider \"test\" {\n\n}\n"), 0o755)
+ if err != nil {
+ t.Fatal(err)
+ }
 
  var testSchema tfjson.ProviderSchemas
- err := json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
+ err = json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
  if err != nil {
  t.Fatal(err)
  }

internal/langserver/handlers/did_change_watched_files_test.go

@@ -701,7 +701,7 @@ func TestLangServer_DidChangeWatchedFiles_pluginChange(t *testing.T) {
  t.Fatal(err)
  }
 
- originalTestDir := filepath.Join(testData, "uninitialized-single-submodule")
+ originalTestDir := filepath.Join(testData, "single-fake-provider")
  testDir := t.TempDir()
  // Copy test configuration so the test can run in isolation
  err = copy.Copy(originalTestDir, testDir)
@@ -788,30 +788,6 @@ func TestLangServer_DidChangeWatchedFiles_pluginChange(t *testing.T) {
  t.Fatal("expected -/foo schema to be missing")
  }
 
- // Install Terraform
- tfVersion := version.Must(version.NewVersion("1.1.7"))
- i := install.NewInstaller()
- ctx := context.Background()
- execPath, err := i.Install(ctx, []src.Installable{
- &releases.ExactVersion{
- Product: product.Terraform,
- Version: tfVersion,
- },
- })
- if err != nil {
- t.Fatal(err)
- }
-
- // Install submodule
- tf, err := exec.NewExecutor(testHandle.Path(), execPath)
- if err != nil {
- t.Fatal(err)
- }
- err = tf.Init(ctx)
- if err != nil {
- t.Fatal(err)
- }
-
  ls.Call(t, &langserver.CallRequest{
  Method: "workspace/didChangeWatchedFiles",
  ReqParams: fmt.Sprintf(`{

internal/langserver/handlers/document_link_test.go

@@ -3,6 +3,8 @@ package handlers
 import (
  "encoding/json"
  "fmt"
+ "io/ioutil"
+ "path/filepath"
  "testing"
 
  "github.com/hashicorp/go-version"
@@ -17,9 +19,13 @@ import (
 func TestDocumentLink_withValidData(t *testing.T) {
  tmpDir := TempDir(t)
  InitPluginCache(t, tmpDir.Path())
+ err := ioutil.WriteFile(filepath.Join(tmpDir.Path(), "main.tf"), []byte("provider \"test\" {\n\n}\n"), 0o755)
+ if err != nil {
+ t.Fatal(err)
+ }
 
  var testSchema tfjson.ProviderSchemas
- err := json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
+ err = json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
  if err != nil {
  t.Fatal(err)
  }

internal/langserver/handlers/execute_command_module_providers_test.go

@@ -97,7 +97,7 @@ func TestLangServer_workspaceExecuteCommand_moduleProviders_basic(t *testing.T)
  newDefaultProvider("aws"): version.Must(version.NewVersion("1.2.3")),
  newDefaultProvider("google"): version.Must(version.NewVersion("2.5.5")),
  }
- err = s.Modules.UpdateInstalledProviders(modDir, pVersions)
+ err = s.Modules.UpdateInstalledProviders(modDir, pVersions, nil)
  if err != nil {
  t.Fatal(err)
  }

internal/langserver/handlers/go_to_ref_target_test.go

@@ -3,6 +3,7 @@ package handlers
 import (
  "encoding/json"
  "fmt"
+ "io/ioutil"
  "path/filepath"
  "testing"
 
@@ -118,8 +119,13 @@ func TestDefinition_withLinkToDefLessBlock(t *testing.T) {
  tmpDir := TempDir(t)
  InitPluginCache(t, tmpDir.Path())
 
+ err := ioutil.WriteFile(filepath.Join(tmpDir.Path(), "main.tf"), []byte("provider \"test\" {\n\n}\n"), 0o755)
+ if err != nil {
+ t.Fatal(err)
+ }
+
  var testSchema tfjson.ProviderSchemas
- err := json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
+ err = json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
  if err != nil {
  t.Fatal(err)
  }
@@ -266,8 +272,13 @@ func TestDefinition_withLinkToDefBlock(t *testing.T) {
  tmpDir := TempDir(t)
  InitPluginCache(t, tmpDir.Path())
 
+ err := ioutil.WriteFile(filepath.Join(tmpDir.Path(), "main.tf"), []byte("provider \"test\" {\n\n}\n"), 0o755)
+ if err != nil {
+ t.Fatal(err)
+ }
+
  var testSchema tfjson.ProviderSchemas
- err := json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
+ err = json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
  if err != nil {
  t.Fatal(err)
  }
@@ -598,8 +609,13 @@ func TestDeclaration_withLinkSupport(t *testing.T) {
  tmpDir := TempDir(t)
  InitPluginCache(t, tmpDir.Path())
 
+ err := ioutil.WriteFile(filepath.Join(tmpDir.Path(), "main.tf"), []byte("provider \"test\" {\n\n}\n"), 0o755)
+ if err != nil {
+ t.Fatal(err)
+ }
+
  var testSchema tfjson.ProviderSchemas
- err := json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
+ err = json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
  if err != nil {
  t.Fatal(err)
  }

internal/langserver/handlers/testdata/single-fake-provider/main.tf

@@ -0,0 +1,3 @@
+provider "foo" {
+
+}