From 2c98e6a3472cfb836c5ef68c3d3f6ee6ddcd1d44 Mon Sep 17 00:00:00 2001 From: Daniel Schmidt Date: Wed, 7 Feb 2024 18:54:34 +0100 Subject: [PATCH 1/6] improve provider lock speed by using cache Closes #33837 --- internal/command/providers_lock.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/command/providers_lock.go b/internal/command/providers_lock.go index fa0235735da7..cb6b26f2666d 100644 --- a/internal/command/providers_lock.go +++ b/internal/command/providers_lock.go @@ -243,9 +243,17 @@ func (c *ProvidersLockCommand) Run(args []string) int { } ctx := evts.OnContext(ctx) + // We can not use c.providerGlobalCacheDir() as we install the provider into a temp dir dir := providercache.NewDirWithPlatform(tempDir, platform) installer := providercache.NewInstaller(dir, source) + // Use global plugin cache for extra speed if this architecture matches the systems (and therefore the caches) one + globalCacheDir := c.providerGlobalCacheDir() + if globalCacheDir != nil && platform == getproviders.CurrentPlatform { + installer.SetGlobalCacheDir(globalCacheDir) + installer.SetGlobalCacheDirMayBreakDependencyLockFile(c.PluginCacheMayBreakDependencyLockFile) + } + newLocks, err := installer.EnsureProviderVersions(ctx, oldLocks, reqs, providercache.InstallNewProvidersForce) if err != nil { diags = diags.Append(tfdiags.Sourceless( From f720a060ffa814b6ffbb5ead02f4a54499b7eb90 Mon Sep 17 00:00:00 2001 From: Daniel Schmidt Date: Thu, 8 Feb 2024 13:10:41 +0100 Subject: [PATCH 2/6] use plugin cache for provider lock across for all platforms The cache architecture already supports multi-platform, the only downside is that this might grow the cache. This seems expected. --- internal/command/providers_lock.go | 4 ++-- internal/providercache/dir.go | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/command/providers_lock.go b/internal/command/providers_lock.go index cb6b26f2666d..d0b82abbe25c 100644 --- a/internal/command/providers_lock.go +++ b/internal/command/providers_lock.go @@ -249,8 +249,8 @@ func (c *ProvidersLockCommand) Run(args []string) int { // Use global plugin cache for extra speed if this architecture matches the systems (and therefore the caches) one globalCacheDir := c.providerGlobalCacheDir() - if globalCacheDir != nil && platform == getproviders.CurrentPlatform { - installer.SetGlobalCacheDir(globalCacheDir) + if globalCacheDir != nil { + installer.SetGlobalCacheDir(globalCacheDir.WithPlatform(platform)) installer.SetGlobalCacheDirMayBreakDependencyLockFile(c.PluginCacheMayBreakDependencyLockFile) } diff --git a/internal/providercache/dir.go b/internal/providercache/dir.go index c11a14192e4d..eab987e4534b 100644 --- a/internal/providercache/dir.go +++ b/internal/providercache/dir.go @@ -75,6 +75,12 @@ func (d *Dir) BasePath() string { return filepath.Clean(d.baseDir) } +// WithPlatform creates a new dir with the provided platform based +// on this dir +func (d *Dir) WithPlatform(platform getproviders.Platform) *Dir { + return NewDirWithPlatform(d.baseDir, platform) +} + // AllAvailablePackages returns a description of all of the packages already // present in the directory. The cache entries are grouped by the provider // they relate to and then sorted by version precedence, with highest From be262b52d7900025ed904ad2bc23b0bd7dc55327 Mon Sep 17 00:00:00 2001 From: Daniel Schmidt Date: Thu, 8 Feb 2024 13:32:25 +0100 Subject: [PATCH 3/6] make using the plugin cache opt-in --- internal/command/providers_lock.go | 72 ++++++++++++++++-------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/internal/command/providers_lock.go b/internal/command/providers_lock.go index d0b82abbe25c..3cb9b1f3e865 100644 --- a/internal/command/providers_lock.go +++ b/internal/command/providers_lock.go @@ -43,9 +43,11 @@ func (c *ProvidersLockCommand) Run(args []string) int { var optPlatforms FlagStringSlice var fsMirrorDir string var netMirrorURL string + cmdFlags.Var(&optPlatforms, "platform", "target platform") cmdFlags.StringVar(&fsMirrorDir, "fs-mirror", "", "filesystem mirror directory") cmdFlags.StringVar(&netMirrorURL, "net-mirror", "", "network mirror base URL") + pluginCache := cmdFlags.Bool("enable-plugin-cache", false, "") cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } if err := cmdFlags.Parse(args); err != nil { c.Ui.Error(fmt.Sprintf("Error parsing command-line flags: %s\n", err.Error())) @@ -249,7 +251,7 @@ func (c *ProvidersLockCommand) Run(args []string) int { // Use global plugin cache for extra speed if this architecture matches the systems (and therefore the caches) one globalCacheDir := c.providerGlobalCacheDir() - if globalCacheDir != nil { + if *pluginCache && globalCacheDir != nil { installer.SetGlobalCacheDir(globalCacheDir.WithPlatform(platform)) installer.SetGlobalCacheDirMayBreakDependencyLockFile(c.PluginCacheMayBreakDependencyLockFile) } @@ -378,38 +380,42 @@ Usage: terraform [global options] providers lock [options] [providers...] Options: - -fs-mirror=dir Consult the given filesystem mirror directory instead - of the origin registry for each of the given providers. - - This would be necessary to generate lock file entries for - a provider that is available only via a mirror, and not - published in an upstream registry. In this case, the set - of valid checksums will be limited only to what Terraform - can learn from the data in the mirror directory. - - -net-mirror=url Consult the given network mirror (given as a base URL) - instead of the origin registry for each of the given - providers. - - This would be necessary to generate lock file entries for - a provider that is available only via a mirror, and not - published in an upstream registry. In this case, the set - of valid checksums will be limited only to what Terraform - can learn from the data in the mirror indices. - - -platform=os_arch Choose a target platform to request package checksums - for. - - By default Terraform will request package checksums - suitable only for the platform where you run this - command. Use this option multiple times to include - checksums for multiple target systems. - - Target names consist of an operating system and a CPU - architecture. For example, "linux_amd64" selects the - Linux operating system running on an AMD64 or x86_64 - CPU. Each provider is available only for a limited - set of target platforms. + -fs-mirror=dir Consult the given filesystem mirror directory instead + of the origin registry for each of the given providers. + + This would be necessary to generate lock file entries for + a provider that is available only via a mirror, and not + published in an upstream registry. In this case, the set + of valid checksums will be limited only to what Terraform + can learn from the data in the mirror directory. + + -net-mirror=url Consult the given network mirror (given as a base URL) + instead of the origin registry for each of the given + providers. + + This would be necessary to generate lock file entries for + a provider that is available only via a mirror, and not + published in an upstream registry. In this case, the set + of valid checksums will be limited only to what Terraform + can learn from the data in the mirror indices. + + -platform=os_arch Choose a target platform to request package checksums + for. + + By default Terraform will request package checksums + suitable only for the platform where you run this + command. Use this option multiple times to include + checksums for multiple target systems. + + Target names consist of an operating system and a CPU + architecture. For example, "linux_amd64" selects the + Linux operating system running on an AMD64 or x86_64 + CPU. Each provider is available only for a limited + set of target platforms. + + -enable-plugin-cache Enable the usage of the globally configured plugin cache. + This will speed up the locking process, but the providers + wont be loaded from am authoritative source. ` } From 472a07ad6eaf7ddffc543daa3596b86008db86fd Mon Sep 17 00:00:00 2001 From: Daniel Schmidt Date: Thu, 8 Feb 2024 17:21:58 +0100 Subject: [PATCH 4/6] fix typo Co-authored-by: Liam Cervante --- internal/command/providers_lock.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/command/providers_lock.go b/internal/command/providers_lock.go index 3cb9b1f3e865..b7b237359f1b 100644 --- a/internal/command/providers_lock.go +++ b/internal/command/providers_lock.go @@ -415,7 +415,7 @@ Options: -enable-plugin-cache Enable the usage of the globally configured plugin cache. This will speed up the locking process, but the providers - wont be loaded from am authoritative source. + wont be loaded from an authoritative source. ` } From 14feb738a75eb5ec50624263097f3e8c13d3ed18 Mon Sep 17 00:00:00 2001 From: Daniel Schmidt Date: Thu, 8 Feb 2024 17:22:40 +0100 Subject: [PATCH 5/6] update comment --- internal/command/providers_lock.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/command/providers_lock.go b/internal/command/providers_lock.go index b7b237359f1b..a611864fd579 100644 --- a/internal/command/providers_lock.go +++ b/internal/command/providers_lock.go @@ -249,7 +249,7 @@ func (c *ProvidersLockCommand) Run(args []string) int { dir := providercache.NewDirWithPlatform(tempDir, platform) installer := providercache.NewInstaller(dir, source) - // Use global plugin cache for extra speed if this architecture matches the systems (and therefore the caches) one + // Use global plugin cache for extra speed if present and flag is set globalCacheDir := c.providerGlobalCacheDir() if *pluginCache && globalCacheDir != nil { installer.SetGlobalCacheDir(globalCacheDir.WithPlatform(platform)) From 91ea23aee5ac5f5065b5866ee73d226c1624f84b Mon Sep 17 00:00:00 2001 From: Daniel Schmidt Date: Thu, 8 Feb 2024 17:32:59 +0100 Subject: [PATCH 6/6] remove outdated comment --- internal/command/providers_lock.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/command/providers_lock.go b/internal/command/providers_lock.go index a611864fd579..c78d30f8bee0 100644 --- a/internal/command/providers_lock.go +++ b/internal/command/providers_lock.go @@ -245,7 +245,6 @@ func (c *ProvidersLockCommand) Run(args []string) int { } ctx := evts.OnContext(ctx) - // We can not use c.providerGlobalCacheDir() as we install the provider into a temp dir dir := providercache.NewDirWithPlatform(tempDir, platform) installer := providercache.NewInstaller(dir, source)