workdir: Start of a new package for working directory state management

Thus far our various interactions with the bits of state we keep
associated with a working directory have all been implemented directly
inside the "command" package -- often in the huge command.Meta type -- and
not managed collectively via a single component.

There's too many little codepaths reading and writing from the working
directory and data directory to refactor it all in one step, but this is
an attempt at a first step towards a future where everything that reads
and writes from the current working directory would do so via an object
that encapsulates the implementation details and offers a high-level API
to read and write all of these session-persistent settings.

The design here continues our gradual path towards using a dependency
injection style where "package main" is solely responsible for directly
interacting with the OS command line, the OS environment, the OS working
directory, the stdio streams, and the CLI configuration, and then
communicating the resulting information to the rest of Terraform by wiring
together objects. It seems likely that eventually we'll have enough wiring
code in package main to justify a more explicit organization of that code,
but for this commit the new "workdir.Dir" object is just wired directly in
place of its predecessors, without any significant change of code
organization at that top layer.

This first commit focuses on the main files and directories we use to
find provider plugins, because a subsequent commit will lightly reorganize
the separation of concerns for plugin launching with a similar goal of
collecting all of the relevant logic together into one spot.

commands.go

@@ -77,12 +77,12 @@ func initCommands(
 		configDir = "" // No config dir available (e.g. looking up a home directory failed)
 	}
 
-	dataDir := os.Getenv("TF_DATA_DIR")
+	wd := WorkingDir(originalWorkingDir, os.Getenv("TF_DATA_DIR"))
 
 	meta := command.Meta{
-		OriginalWorkingDir: originalWorkingDir,
-		Streams:            streams,
-		View:               views.NewView(streams).SetRunningInAutomation(inAutomation),
+		WorkingDir: wd,
+		Streams:    streams,
+		View:       views.NewView(streams).SetRunningInAutomation(inAutomation),
 
 		Color:            true,
 		GlobalPluginDirs: globalPluginDirs(),
@@ -94,7 +94,6 @@ func initCommands(
 		RunningInAutomation: inAutomation,
 		CLIConfigDir:        configDir,
 		PluginCacheDir:      config.PluginCacheDir,
-		OverrideDataDir:     dataDir,
 
 		ShutdownCh: makeShutdownCh(),
 

internal/command/command_test.go

@@ -24,6 +24,7 @@ import (
 	backendInit "github.com/hashicorp/terraform/internal/backend/init"
 	backendLocal "github.com/hashicorp/terraform/internal/backend/local"
 	"github.com/hashicorp/terraform/internal/command/views"
+	"github.com/hashicorp/terraform/internal/command/workdir"
 	"github.com/hashicorp/terraform/internal/configs"
 	"github.com/hashicorp/terraform/internal/configs/configload"
 	"github.com/hashicorp/terraform/internal/configs/configschema"
@@ -108,6 +109,65 @@ func tempDir(t *testing.T) string {
 	return dir
 }
 
+// tempWorkingDir constructs a workdir.Dir object referring to a newly-created
+// temporary directory, and returns that object along with a cleanup function
+// to call once the calling test is complete.
+//
+// Although workdir.Dir is built to support arbitrary base directories, the
+// not-yet-migrated behaviors in command.Meta tend to expect the root module
+// directory to be the real process working directory, and so if you intend
+// to use the result inside a command.Meta object you must use a pattern
+// similar to the following when initializing your test:
+//
+//     wd, cleanup := tempWorkingDir(t)
+//     defer cleanup()
+//     defer testChdir(t, wd.RootModuleDir())()
+//
+// Note that testChdir modifies global state for the test process, and so a
+// test using this pattern must never call t.Parallel().
+func tempWorkingDir(t *testing.T) (*workdir.Dir, func() error) {
+	t.Helper()
+
+	dirPath, err := os.MkdirTemp("", "tf-command-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	done := func() error {
+		return os.RemoveAll(dirPath)
+	}
+	t.Logf("temporary directory %s", dirPath)
+
+	return workdir.NewDir(dirPath), done
+}
+
+// tempWorkingDirFixture is like tempWorkingDir but it also copies the content
+// from a fixture directory into the temporary directory before returning it.
+//
+// The same caveats about working directory apply as for testWorkingDir. See
+// the testWorkingDir commentary for an example of how to use this function
+// along with testChdir to meet the expectations of command.Meta legacy
+// functionality.
+func tempWorkingDirFixture(t *testing.T, fixtureName string) (*workdir.Dir, func() error) {
+	t.Helper()
+
+	dirPath, err := os.MkdirTemp("", "tf-command-test-"+fixtureName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	done := func() error {
+		return os.RemoveAll(dirPath)
+	}
+	t.Logf("temporary directory %s with fixture %q", dirPath, fixtureName)
+
+	fixturePath := testFixturePath(fixtureName)
+	testCopyDir(t, fixturePath, dirPath)
+	// NOTE: Unfortunately because testCopyDir immediately aborts the test
+	// on failure, a failure to copy will prevent us from cleaning up the
+	// temporary directory. Oh well. :(
+
+	return workdir.NewDir(dirPath), done
+}
+
 func testFixturePath(name string) string {
 	return filepath.Join(fixtureDir, name)
 }
@@ -853,8 +913,10 @@ func testLockState(sourceDir, path string) (func(), error) {
 }
 
 // testCopyDir recursively copies a directory tree, attempting to preserve
-// permissions. Source directory must exist, destination directory must *not*
-// exist. Symlinks are ignored and skipped.
+// permissions. Source directory must exist, destination directory may exist
+// but will be created if not; it should typically be a temporary directory,
+// and thus already created using os.MkdirTemp or similar.
+// Symlinks are ignored and skipped.
 func testCopyDir(t *testing.T, src, dst string) {
 	t.Helper()
 
@@ -873,9 +935,6 @@ func testCopyDir(t *testing.T, src, dst string) {
 	if err != nil && !os.IsNotExist(err) {
 		t.Fatal(err)
 	}
-	if err == nil {
-		t.Fatal("destination already exists")
-	}
 
 	err = os.MkdirAll(dst, si.Mode())
 	if err != nil {

internal/command/get_test.go

@@ -1,25 +1,23 @@
 package command
 
 import (
-	"os"
 	"strings"
 	"testing"
 
 	"github.com/mitchellh/cli"
 )
 
 func TestGet(t *testing.T) {
-	td := tempDir(t)
-	testCopyDir(t, testFixturePath("get"), td)
-	defer os.RemoveAll(td)
-	defer testChdir(t, td)()
+	wd, cleanup := tempWorkingDirFixture(t, "get")
+	defer cleanup()
+	defer testChdir(t, wd.RootModuleDir())()
 
-	ui := new(cli.MockUi)
+	ui := cli.NewMockUi()
 	c := &GetCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(testProvider()),
 			Ui:               ui,
-			dataDir:          tempDir(t),
+			WorkingDir:       wd,
 		},
 	}
 
@@ -35,12 +33,16 @@ func TestGet(t *testing.T) {
 }
 
 func TestGet_multipleArgs(t *testing.T) {
-	ui := new(cli.MockUi)
+	wd, cleanup := tempWorkingDir(t)
+	defer cleanup()
+	defer testChdir(t, wd.RootModuleDir())()
+
+	ui := cli.NewMockUi()
 	c := &GetCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(testProvider()),
 			Ui:               ui,
-			dataDir:          tempDir(t),
+			WorkingDir:       wd,
 		},
 	}
 
@@ -54,17 +56,16 @@ func TestGet_multipleArgs(t *testing.T) {
 }
 
 func TestGet_update(t *testing.T) {
-	td := tempDir(t)
-	testCopyDir(t, testFixturePath("get"), td)
-	defer os.RemoveAll(td)
-	defer testChdir(t, td)()
+	wd, cleanup := tempWorkingDirFixture(t, "get")
+	defer cleanup()
+	defer testChdir(t, wd.RootModuleDir())()
 
-	ui := new(cli.MockUi)
+	ui := cli.NewMockUi()
 	c := &GetCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(testProvider()),
 			Ui:               ui,
-			dataDir:          tempDir(t),
+			WorkingDir:       wd,
 		},
 	}
 

internal/command/meta.go

@@ -16,24 +16,25 @@ import (
 
 	plugin "github.com/hashicorp/go-plugin"
 	"github.com/hashicorp/terraform-svchost/disco"
+	"github.com/mitchellh/cli"
+	"github.com/mitchellh/colorstring"
+
 	"github.com/hashicorp/terraform/internal/addrs"
 	"github.com/hashicorp/terraform/internal/backend"
 	"github.com/hashicorp/terraform/internal/backend/local"
 	"github.com/hashicorp/terraform/internal/command/arguments"
 	"github.com/hashicorp/terraform/internal/command/format"
 	"github.com/hashicorp/terraform/internal/command/views"
 	"github.com/hashicorp/terraform/internal/command/webbrowser"
+	"github.com/hashicorp/terraform/internal/command/workdir"
 	"github.com/hashicorp/terraform/internal/configs/configload"
 	"github.com/hashicorp/terraform/internal/getproviders"
+	legacy "github.com/hashicorp/terraform/internal/legacy/terraform"
 	"github.com/hashicorp/terraform/internal/providers"
 	"github.com/hashicorp/terraform/internal/provisioners"
 	"github.com/hashicorp/terraform/internal/terminal"
 	"github.com/hashicorp/terraform/internal/terraform"
 	"github.com/hashicorp/terraform/internal/tfdiags"
-	"github.com/mitchellh/cli"
-	"github.com/mitchellh/colorstring"
-
-	legacy "github.com/hashicorp/terraform/internal/legacy/terraform"
 )
 
 // Meta are the meta-options that are available on all or most commands.
@@ -42,16 +43,19 @@ type Meta struct {
 	// command with a Meta field. These are expected to be set externally
 	// (not from within the command itself).
 
-	// OriginalWorkingDir, if set, is the actual working directory where
-	// Terraform was run from. This might not be the _actual_ current working
-	// directory, because users can add the -chdir=... option to the beginning
-	// of their command line to ask Terraform to switch.
+	// WorkingDir is an object representing the "working directory" where we're
+	// running commands. In the normal case this literally refers to the
+	// working directory of the Terraform process, though this can take on
+	// a more symbolic meaning when the user has overridden default behavior
+	// to specify a different working directory or to override the special
+	// data directory where we'll persist settings that must survive between
+	// consecutive commands.
 	//
-	// Most things should just use the current working directory in order to
-	// respect the user's override, but we retain this for exceptional
-	// situations where we need to refer back to the original working directory
-	// for some reason.
-	OriginalWorkingDir string
+	// We're currently gradually migrating the various bits of state that
+	// must persist between consecutive commands in a session to be encapsulated
+	// in here, but we're not there yet and so there are also some methods on
+	// Meta which directly read and modify paths inside the data directory.
+	WorkingDir *workdir.Dir
 
 	// Streams tracks the raw Stdout, Stderr, and Stdin handles along with
 	// some basic metadata about them, such as whether each is connected to
@@ -102,11 +106,6 @@ type Meta struct {
 	// provider version can be obtained.
 	ProviderSource getproviders.Source
 
-	// OverrideDataDir, if non-empty, overrides the return value of the
-	// DataDir method for situations where the local .terraform/ directory
-	// is not suitable, e.g. because of a read-only filesystem.
-	OverrideDataDir string
-
 	// BrowserLauncher is used by commands that need to open a URL in a
 	// web browser.
 	BrowserLauncher webbrowser.Launcher
@@ -135,10 +134,6 @@ type Meta struct {
 	// Protected: commands can set these
 	//----------------------------------------------------------
 
-	// Modify the data directory location. This should be accessed through the
-	// DataDir method.
-	dataDir string
-
 	// pluginPath is a user defined set of directories to look for plugins.
 	// This is set during init with the `-plugin-dir` flag, saved to a file in
 	// the data directory.
@@ -265,13 +260,25 @@ func (m *Meta) Colorize() *colorstring.Colorize {
 	}
 }
 
+// fixupMissingWorkingDir is a compensation for various existing tests which
+// directly construct incomplete "Meta" objects. Specifically, it deals with
+// a test that omits a WorkingDir value by constructing one just-in-time.
+//
+// We shouldn't ever rely on this in any real codepath, because it doesn't
+// take into account the various ways users can override our default
+// directory selection behaviors.
+func (m *Meta) fixupMissingWorkingDir() {
+	if m.WorkingDir == nil {
+		log.Printf("[WARN] This 'Meta' object is missing its WorkingDir, so we're creating a default one suitable only for tests")
+		m.WorkingDir = workdir.NewDir(".")
+	}
+}
+
 // DataDir returns the directory where local data will be stored.
 // Defaults to DefaultDataDir in the current working directory.
 func (m *Meta) DataDir() string {
-	if m.OverrideDataDir != "" {
-		return m.OverrideDataDir
-	}
-	return DefaultDataDir
+	m.fixupMissingWorkingDir()
+	return m.WorkingDir.DataDir()
 }
 
 const (
@@ -499,7 +506,7 @@ func (m *Meta) contextOpts() (*terraform.ContextOpts, error) {
 
 	opts.Meta = &terraform.ContextMeta{
 		Env:                workspace,
-		OriginalWorkingDir: m.OriginalWorkingDir,
+		OriginalWorkingDir: m.WorkingDir.OriginalWorkingDir(),
 	}
 
 	return &opts, nil

internal/command/meta_backend_test.go

@@ -1855,17 +1855,19 @@ func TestMetaBackend_configToExtra(t *testing.T) {
 
 // no config; return inmem backend stored in state
 func TestBackendFromState(t *testing.T) {
-	td := tempDir(t)
-	testCopyDir(t, testFixturePath("backend-from-state"), td)
-	defer os.RemoveAll(td)
-	defer testChdir(t, td)()
+	wd, cleanup := tempWorkingDirFixture(t, "backend-from-state")
+	defer cleanup()
+	defer testChdir(t, wd.RootModuleDir())()
 
 	// Setup the meta
 	m := testMetaBackend(t, nil)
+	m.WorkingDir = wd
 	// terraform caches a small "state" file that stores the backend config.
 	// This test must override m.dataDir so it loads the "terraform.tfstate" file in the
-	// test directory as the backend config cache
-	m.OverrideDataDir = td
+	// test directory as the backend config cache. This fixture is really a
+	// fixture for the data dir rather than the module dir, so we'll override
+	// them to match just for this test.
+	wd.OverrideDataDir(".")
 
 	stateBackend, diags := m.backendFromState()
 	if diags.HasErrors() {

internal/command/meta_config.go

@@ -27,27 +27,8 @@ import (
 // paths used to load configuration, because we want to prefer recording
 // relative paths in source code references within the configuration.
 func (m *Meta) normalizePath(path string) string {
-	var err error
-
-	// First we will make it absolute so that we have a consistent place
-	// to start.
-	path, err = filepath.Abs(path)
-	if err != nil {
-		// We'll just accept what we were given, then.
-		return path
-	}
-
-	cwd, err := os.Getwd()
-	if err != nil || !filepath.IsAbs(cwd) {
-		return path
-	}
-
-	ret, err := filepath.Rel(cwd, path)
-	if err != nil {
-		return path
-	}
-
-	return ret
+	m.fixupMissingWorkingDir()
+	return m.WorkingDir.NormalizePath(path)
 }
 
 // loadConfig reads a configuration from the given directory, which should

internal/command/meta_providers.go

@@ -6,7 +6,6 @@ import (
 	"log"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"strings"
 
 	"github.com/hashicorp/go-multierror"
@@ -109,7 +108,8 @@ func (m *Meta) providerCustomLocalDirectorySource(dirs []string) getproviders.So
 // Only one object returned from this method should be live at any time,
 // because objects inside contain caches that must be maintained properly.
 func (m *Meta) providerLocalCacheDir() *providercache.Dir {
-	dir := filepath.Join(m.DataDir(), "providers")
+	m.fixupMissingWorkingDir()
+	dir := m.WorkingDir.ProviderLocalCacheDir()
 	return providercache.NewDir(dir)
 }