cli: Opt-in local cache directory for plugins

command/e2etest/init_test.go

@@ -1,7 +1,11 @@
 package e2etest
 
 import (
+ "bytes"
+ "fmt"
+ "os"
  "path/filepath"
+ "runtime"
  "strings"
  "testing"
 
@@ -45,3 +49,55 @@ func TestInitProviders(t *testing.T) {
  }
 
 }
+
+func TestInitProviders_pluginCache(t *testing.T) {
+ t.Parallel()
+
+ // This test reaches out to releases.hashicorp.com to access plugin
+ // metadata, and download the null plugin, though the template plugin
+ // should come from local cache.
+ skipIfCannotAccessNetwork(t)
+
+ fixturePath := filepath.Join("test-fixtures", "plugin-cache")
+ tf := e2e.NewBinary(terraformBin, fixturePath)
+ defer tf.Close()
+
+ // Our fixture dir has a generic os_arch dir, which we need to customize
+ // to the actual OS/arch where this test is running in order to get the
+ // desired result.
+ fixtMachineDir := tf.Path("cache/os_arch")
+ wantMachineDir := tf.Path("cache", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH))
+ os.Rename(fixtMachineDir, wantMachineDir)
+
+ cmd := tf.Cmd("init")
+ cmd.Env = append(cmd.Env, "TF_PLUGIN_CACHE_DIR=./cache")
+ cmd.Stdin = nil
+ cmd.Stderr = &bytes.Buffer{}
+
+ err := cmd.Run()
+ if err != nil {
+ t.Errorf("unexpected error: %s", err)
+ }
+
+ stderr := cmd.Stderr.(*bytes.Buffer).String()
+ if stderr != "" {
+ t.Errorf("unexpected stderr output:\n%s", stderr)
+ }
+
+ path := fmt.Sprintf(".terraform/plugins/%s_%s/terraform-provider-template_v0.1.0_x4", runtime.GOOS, runtime.GOARCH)
+ content, err := tf.ReadFile(path)
+ if err != nil {
+ t.Fatalf("failed to read installed plugin from %s: %s", path, err)
+ }
+ if strings.TrimSpace(string(content)) != "this is not a real plugin" {
+ t.Errorf("template plugin was not installed from local cache")
+ }
+
+ if !tf.FileExists(fmt.Sprintf(".terraform/plugins/%s_%s/terraform-provider-null_v0.1.0_x4", runtime.GOOS, runtime.GOARCH)) {
+ t.Errorf("null plugin was not installed")
+ }
+
+ if !tf.FileExists(fmt.Sprintf("cache/%s_%s/terraform-provider-null_v0.1.0_x4", runtime.GOOS, runtime.GOARCH)) {
+ t.Errorf("null plugin is not in cache after install")
+ }
+}

command/e2etest/test-fixtures/plugin-cache/cache/os_arch/terraform-provider-template_v0.1.0_x4

@@ -0,0 +1 @@
+this is not a real plugin

command/e2etest/test-fixtures/plugin-cache/main.tf

@@ -0,0 +1,7 @@
+provider "template" {
+ version = "0.1.0"
+}
+
+provider "null" {
+ version = "0.1.0"
+}

command/init.go

@@ -71,10 +71,11 @@ func (c *InitCommand) Run(args []string) int {
  c.getPlugins = false
  }
 
- // set getProvider if we don't have a test version already
+ // set providerInstaller if we don't have a test version already
  if c.providerInstaller == nil {
  c.providerInstaller = &discovery.ProviderInstaller{
- Dir: c.pluginDir(),
+ Dir: c.pluginDir(),
+ Cache: c.pluginCache(),
  PluginProtocolVersion: plugin.Handshake.ProtocolVersion,
  SkipVerify: !flagVerifyPlugins,
  Ui: c.Ui,

command/meta.go

@@ -55,6 +55,10 @@ type Meta struct {
  // the specific commands being run.
  RunningInAutomation bool
 
+ // PluginCacheDir, if non-empty, enables caching of downloaded plugins
+ // into the given directory.
+ PluginCacheDir string
+
  //----------------------------------------------------------
  // Protected: commands can set these
  //----------------------------------------------------------

command/plugins.go

@@ -168,6 +168,17 @@ func (m *Meta) pluginDirs(includeAutoInstalled bool) []string {
  return dirs
 }
 
+func (m *Meta) pluginCache() discovery.PluginCache {
+ dir := m.PluginCacheDir
+ if dir == "" {
+ return nil // cache disabled
+ }
+
+ dir = filepath.Join(dir, pluginMachineName)
+
+ return discovery.NewLocalPluginCache(dir)
+}
+
 // providerPluginSet returns the set of valid providers that were discovered in
 // the defined search paths.
 func (m *Meta) providerPluginSet() discovery.PluginMetaSet {

commands.go

@@ -25,15 +25,7 @@ const (
  OutputPrefix = "o:"
 )
 
-func init() {
- Ui = &cli.PrefixedUi{
- AskPrefix: OutputPrefix,
- OutputPrefix: OutputPrefix,
- InfoPrefix: OutputPrefix,
- ErrorPrefix: ErrorPrefix,
- Ui: &cli.BasicUi{Writer: os.Stdout},
- }
-
+func initCommands(config *Config) {
  var inAutomation bool
  if v := os.Getenv(runningInAutomationEnvName); v != "" {
  inAutomation = true
@@ -46,6 +38,7 @@ func init() {
  Ui: Ui,
 
  RunningInAutomation: inAutomation,
+ PluginCacheDir: config.PluginCacheDir,
  }
 
  // The command list is included in the terraform -help

config.go

@@ -11,6 +11,8 @@ import (
  "github.com/hashicorp/terraform/command"
 )
 
+const pluginCacheDirEnvVar = "TF_PLUGIN_CACHE_DIR"
+
 // Config is the structure of the configuration for the Terraform CLI.
 //
 // This is not the configuration for Terraform itself. That is in the
@@ -21,6 +23,10 @@ type Config struct {
 
  DisableCheckpoint bool `hcl:"disable_checkpoint"`
  DisableCheckpointSignature bool `hcl:"disable_checkpoint_signature"`
+
+ // If set, enables local caching of plugins in this directory to
+ // avoid repeatedly re-downloading over the Internet.
+ PluginCacheDir string `hcl:"plugin_cache_dir"`
 }
 
 // BuiltinConfig is the built-in defaults for the configuration. These
@@ -75,9 +81,31 @@ func LoadConfig(path string) (*Config, error) {
  result.Provisioners[k] = os.ExpandEnv(v)
  }
 
+ if result.PluginCacheDir != "" {
+ result.PluginCacheDir = os.ExpandEnv(result.PluginCacheDir)
+ }
+
  return &result, nil
 }
 
+// EnvConfig returns a Config populated from environment variables.
+//
+// Any values specified in this config should override those set in the
+// configuration file.
+func EnvConfig() *Config {
+ config := &Config{}
+
+ if envPluginCacheDir := os.Getenv(pluginCacheDirEnvVar); envPluginCacheDir != "" {
+ // No Expandenv here, because expanding environment variables inside
+ // an environment variable would be strange and seems unnecessary.
+ // (User can expand variables into the value while setting it using
+ // standard shell features.)
+ config.PluginCacheDir = envPluginCacheDir
+ }
+
+ return config
+}
+
 // Merge merges two configurations and returns a third entirely
 // new configuration with the two merged.
 func (c1 *Config) Merge(c2 *Config) *Config {
@@ -105,5 +133,10 @@ func (c1 *Config) Merge(c2 *Config) *Config {
  result.DisableCheckpoint = c1.DisableCheckpoint || c2.DisableCheckpoint
  result.DisableCheckpointSignature = c1.DisableCheckpointSignature || c2.DisableCheckpointSignature
 
+ result.PluginCacheDir = c1.PluginCacheDir
+ if result.PluginCacheDir == "" {
+ result.PluginCacheDir = c2.PluginCacheDir
+ }
+
  return &result
 }

main.go

@@ -96,6 +96,16 @@ func realMain() int {
  return wrappedMain()
 }
 
+func init() {
+ Ui = &cli.PrefixedUi{
+ AskPrefix: OutputPrefix,
+ OutputPrefix: OutputPrefix,
+ InfoPrefix: OutputPrefix,
+ ErrorPrefix: ErrorPrefix,
+ Ui: &cli.BasicUi{Writer: os.Stdout},
+ }
+}
+
 func wrappedMain() int {
  // We always need to close the DebugInfo before we exit.
  defer terraform.CloseDebugInfo()
@@ -128,6 +138,18 @@ func wrappedMain() int {
  config = *config.Merge(usrcfg)
  }
 
+ if envConfig := EnvConfig(); envConfig != nil {
+ // envConfig takes precedence
+ config = *envConfig.Merge(&config)
+ }
+
+ log.Printf("[DEBUG] CLI Config is %#v", config)
+
+ // In tests, Commands may already be set to provide mock commands
+ if Commands == nil {
+ initCommands(&config)
+ }
+
  // Run checkpoint
  go runCheckpoint(&config)
 

main_test.go

@@ -18,9 +18,12 @@ func TestMain_cliArgsFromEnv(t *testing.T) {
  defer func() { os.Args = oldArgs }()
 
  // Setup test command and restore that
+ Commands = make(map[string]cli.CommandFactory)
+ defer func() {
+ Commands = nil
+ }()
  testCommandName := "unit-test-cli-args"
  testCommand := &testCommandCLI{}
- defer func() { delete(Commands, testCommandName) }()
  Commands[testCommandName] = func() (cli.Command, error) {
  return testCommand, nil
  }
@@ -150,6 +153,12 @@ func TestMain_cliArgsFromEnvAdvanced(t *testing.T) {
  oldArgs := os.Args
  defer func() { os.Args = oldArgs }()
 
+ // Setup test command and restore that
+ Commands = make(map[string]cli.CommandFactory)
+ defer func() {
+ Commands = nil
+ }()
+
  cases := []struct {
  Name string
  Command string
@@ -230,7 +239,7 @@ func TestMain_cliArgsFromEnvAdvanced(t *testing.T) {
  testCommand.Args = nil
  exit := wrappedMain()
  if (exit != 0) != tc.Err {
- t.Fatalf("bad: %d", exit)
+ t.Fatalf("unexpected exit status %d; want 0", exit)
  }
  if tc.Err {
  return

plugin/discovery/find.go

@@ -3,6 +3,7 @@ package discovery
 import (
  "io/ioutil"
  "log"
+ "os"
  "path/filepath"
  "strings"
 )
@@ -70,6 +71,12 @@ func findPluginPaths(kind string, dirs []string) []string {
  continue
  }
 
+ // Check that the file we found is usable
+ if !pathIsFile(absPath) {
+ log.Printf("[ERROR] ignoring non-file %s", absPath)
+ continue
+ }
+
  log.Printf("[DEBUG] found %s %q", kind, fullName)
  ret = append(ret, filepath.Clean(absPath))
  continue
@@ -82,6 +89,12 @@ func findPluginPaths(kind string, dirs []string) []string {
  continue
  }
 
+ // Check that the file we found is usable
+ if !pathIsFile(absPath) {
+ log.Printf("[ERROR] ignoring non-file %s", absPath)
+ continue
+ }
+
  log.Printf("[WARNING] found legacy %s %q", kind, fullName)
 
  ret = append(ret, filepath.Clean(absPath))
@@ -91,6 +104,17 @@ func findPluginPaths(kind string, dirs []string) []string {
  return ret
 }
 
+// Returns true if and only if the given path refers to a file or a symlink
+// to a file.
+func pathIsFile(path string) bool {
+ info, err := os.Stat(path)
+ if err != nil {
+ return false
+ }
+
+ return !info.IsDir()
+}
+
 // ResolvePluginPaths takes a list of paths to plugin executables (as returned
 // by e.g. FindPluginPaths) and produces a PluginMetaSet describing the
 // referenced plugins.