feat: configure temporary directory for local execute (#967)

* Add tmp_mount to cli.yaml and to setup command.

* remove tmp mount from setup

* add temp dir to local execute command and

* fix flags

* e2e test for docker execute

* add test cases

* setup remote docker in test jobs

* use --mount instead of --volume for docker bind mounts

* fix rebase mistake

* remove remote docker from non-linux test jobs

* skip local execute tests on non-linux OS

* omit empty temp dir

* remove local execute e2e tests

* remove unused setup_remote_docker

---------

Co-authored-by: Cirano Eusebi <ceusebi@eventbrite.com>

.circleci/config.yml

@@ -312,38 +312,38 @@ jobs:
       - run:
           name: Setup Scanning
           command: |
-              git config --global url."https://$GITHUB_USER:$GITHUB_TOKEN@github.com/circleci/".insteadOf "https://github.com/circleci/"
+            git config --global url."https://$GITHUB_USER:$GITHUB_TOKEN@github.com/circleci/".insteadOf "https://github.com/circleci/"
       - when:
           condition:
-              or:
-                - equal: [ main, << pipeline.git.branch >> ]
+            or:
+              - equal: [main, << pipeline.git.branch >>]
           steps:
-              - run:
-                  name: Launching Snyk Orb Scanning
-                  command: echo "Running snyk/scan on main; uploading the results"
-              - run:
-                  name: Cleanup RemoteRepoURL
-                  command: echo 'export REMOTE_REPO_URL="${CIRCLE_REPOSITORY_URL%".git"}"' >> "$BASH_ENV"
-              - snyk/scan:
-                  organization: "circleci-public"
-                  fail-on-issues: true
-                  severity-threshold: high
-                  monitor-on-build: true
-                  additional-arguments: "--all-projects --remote-repo-url=${REMOTE_REPO_URL} -d"
+            - run:
+                name: Launching Snyk Orb Scanning
+                command: echo "Running snyk/scan on main; uploading the results"
+            - run:
+                name: Cleanup RemoteRepoURL
+                command: echo 'export REMOTE_REPO_URL="${CIRCLE_REPOSITORY_URL%".git"}"' >> "$BASH_ENV"
+            - snyk/scan:
+                organization: "circleci-public"
+                fail-on-issues: true
+                severity-threshold: high
+                monitor-on-build: true
+                additional-arguments: "--all-projects --remote-repo-url=${REMOTE_REPO_URL} -d"
       - unless:
           condition:
-              or:
-                - equal: [ main, << pipeline.git.branch >> ]
+            or:
+              - equal: [main, << pipeline.git.branch >>]
           steps:
-              - run:
-                  name: Launching Snyk Orb Scanning
-                  command: echo "Running snyk/scan on branch; not uploading the results"
-              - snyk/scan:
-                  organization: "circleci-public"
-                  fail-on-issues: true
-                  severity-threshold: high
-                  monitor-on-build: false
-                  additional-arguments: "--all-projects -d"
+            - run:
+                name: Launching Snyk Orb Scanning
+                command: echo "Running snyk/scan on branch; not uploading the results"
+            - snyk/scan:
+                organization: "circleci-public"
+                fail-on-issues: true
+                severity-threshold: high
+                monitor-on-build: false
+                additional-arguments: "--all-projects -d"
 
 workflows:
   ci:

cmd/build.go

@@ -33,6 +33,7 @@ func newLocalExecuteCommand(config *settings.Config) *cobra.Command {
 	buildCommand.Flags().String("build-agent-version", "", `The version of the build agent image you want to use. This can be configured by writing in $HOME/.circleci/build_agent_settings.json: '{"LatestSha256":"<version-of-build-agent>"}'`)
 	buildCommand.Flags().StringP("org-slug", "o", "", "organization slug (for example: github/example-org), used when a config depends on private orbs belonging to that org")
 	buildCommand.Flags().String("org-id", "", "organization id, used when a config depends on private orbs belonging to that org")
+	buildCommand.Flags().String("temp-dir", "", "path to local directory to store temporary config files")
 
 	return buildCommand
 }

cmd/setup.go

@@ -243,7 +243,7 @@ func setupNoPrompt(opts setupOptions) error {
 		return nil
 	}
 
-	// Throw an error if both flags are blank are blank!
+	// Throw an error if host and token flags are blank
 	if opts.host == "" && opts.token == "" {
 		return errors.New("No existing host or token saved.\nThe proper format is `circleci setup --host HOST --token TOKEN --no-prompt")
 	}

cmd/setup_test.go

@@ -268,7 +268,7 @@ token: asdf
 				)
 			})
 
-			It("write the configuration to a file", func() {
+			It("writes the configuration to a file", func() {
 				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
 				Expect(err).ShouldNot(HaveOccurred())
 				stdout := session.Wait().Out.Contents()

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect
 	github.com/google/uuid v1.3.0
 	github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-isatty v0.0.18 // indirect
 	github.com/mitchellh/mapstructure v1.4.1
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/onsi/ginkgo v1.16.4
@@ -80,7 +80,7 @@ require (
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
 	github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect

go.sum

@@ -281,13 +281,14 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
+github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
-github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
+github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=

local/local.go

@@ -24,6 +24,18 @@ const DefaultDockerSocketPath = "/var/run/docker.sock"
 func Execute(flags *pflag.FlagSet, cfg *settings.Config, args []string) error {
 	var err error
 	var configResponse *config.ConfigResponse
+
+	// Get temp dir from flags
+	tempDir, _ := flags.GetString("temp-dir")
+	if tempDir == "" {
+		// If not specified, get from config
+		tempDir = cfg.TempDir
+		if tempDir == "" {
+			// If not specified, use system default
+			tempDir = os.TempDir()
+		}
+	}
+
 	processedArgs, configPath := buildAgentArguments(flags)
 
 	compiler, err := config.NewWithConfig(cfg)
@@ -50,14 +62,14 @@ func Execute(flags *pflag.FlagSet, cfg *settings.Config, args []string) error {
 		return fmt.Errorf("config errors %v", configResponse.Errors)
 	}
 
-	processedConfigPath, err := writeStringToTempFile(configResponse.OutputYaml)
+	processedConfigPath, err := writeStringToTempFile(tempDir, configResponse.OutputYaml)
 
 	// The file at processedConfigPath must be left in place until after the call
 	// to `docker run` has completed. Typically, we would `defer` a call to remove
 	// the file. In this case, we execute `docker` using `syscall.Exec`, which
 	// replaces the current process, and no more go code will execute at that
 	// point, so we cannot delete the file easily. We choose to leave the file
-	// in-place in /tmp.
+	// in-place in the temporary directory.
 
 	if err != nil {
 		return err
@@ -83,7 +95,7 @@ func Execute(flags *pflag.FlagSet, cfg *settings.Config, args []string) error {
 
 	job := args[0]
 	dockerSocketPath, _ := flags.GetString("docker-socket-path")
-	arguments := generateDockerCommand(processedConfigPath, image, pwd, job, dockerSocketPath, processedArgs...)
+	arguments := generateDockerCommand(tempDir, processedConfigPath, image, pwd, job, dockerSocketPath, processedArgs...)
 
 	if cfg.Debug {
 		_, err = fmt.Fprintf(os.Stderr, "Starting docker with args: %s", arguments)
@@ -111,13 +123,13 @@ func AddFlagsForDocumentation(flags *pflag.FlagSet) {
 	flags.Int("node-total", 1, "total number of parallel nodes")
 	flags.Int("index", 0, "node index of parallelism")
 	flags.Bool("skip-checkout", true, "use local path as-is")
-	flags.StringArrayP("volume", "v", nil, "Volume bind-mounting")
-	flags.String("checkout-key", "~/.ssh/id_rsa", "Git Checkout key")
-	flags.String("revision", "", "Git Revision")
-	flags.String("branch", "", "Git branch")
-	flags.String("repo-url", "", "Git Url")
-	flags.StringArrayP("env", "e", nil, "Set environment variables, e.g. `-e VAR=VAL`")
-	flags.String("docker-socket-path", DefaultDockerSocketPath, "Path to the host's docker socket")
+	flags.StringArrayP("volume", "v", nil, "volume bind-mounting")
+	flags.String("checkout-key", "~/.ssh/id_rsa", "git checkout key")
+	flags.String("revision", "", "git revision")
+	flags.String("branch", "", "git branch")
+	flags.String("repo-url", "", "git URL")
+	flags.StringArrayP("env", "e", nil, "set environment variables, e.g. `-e VAR=VAL`")
+	flags.String("docker-socket-path", DefaultDockerSocketPath, "path to the host's docker socket")
 }
 
 // Given the full set of flags that were passed to this command, return the path
@@ -135,7 +147,13 @@ func buildAgentArguments(flags *pflag.FlagSet) ([]string, string) {
 
 	// build a list of all supplied flags, that we will pass on to build-agent
 	flags.Visit(func(flag *pflag.Flag) {
-		if flag.Name != "build-agent-version" && flag.Name != "org-slug" && flag.Name != "config" && flag.Name != "debug" && flag.Name != "org-id" && flag.Name != "docker-socket-path" {
+		if flag.Name != "build-agent-version" &&
+			flag.Name != "org-slug" &&
+			flag.Name != "config" &&
+			flag.Name != "temp-dir" &&
+			flag.Name != "debug" &&
+			flag.Name != "org-id" &&
+			flag.Name != "docker-socket-path" {
 			result = append(result, unparseFlag(flags, flag)...)
 		}
 	})
@@ -199,14 +217,11 @@ func ensureDockerIsAvailable() (string, error) {
 }
 
 // Write data to a temp file, and return the path to that file.
-func writeStringToTempFile(data string) (string, error) {
-	// It's important to specify `/tmp` here as the location of the temp file.
-	// On macOS, the regular temp directories is not shared with Docker by default.
-	// The error message is along the lines of:
-	// > The path /var/folders/q0/2g2lcf6j79df6vxqm0cg_0zm0000gn/T/287575618-config.yml
-	// > is not shared from OS X and is not known to Docker.
-	// Docker has `/tmp` shared by default.
-	f, err := os.CreateTemp("/tmp", "*_circleci_config.yml")
+func writeStringToTempFile(tempDir, data string) (string, error) {
+	if tempDir == "" {
+		tempDir = os.TempDir()
+	}
+	f, err := os.CreateTemp(tempDir, "*_circleci_config.yml")
 
 	if err != nil {
 		return "", errors.Wrap(err, "Error creating temporary config file")
@@ -219,13 +234,13 @@ func writeStringToTempFile(data string) (string, error) {
 	return f.Name(), nil
 }
 
-func generateDockerCommand(configPath, image, pwd string, job string, dockerSocketPath string, arguments ...string) []string {
-	const configPathInsideContainer = "/tmp/local_build_config.yml"
-	core := []string{"docker", "run", "--interactive", "--tty", "--rm",
-		"--volume", fmt.Sprintf("%s:/var/run/docker.sock", dockerSocketPath),
-		"--volume", fmt.Sprintf("%s:%s", configPath, configPathInsideContainer),
-		"--volume", fmt.Sprintf("%s:%s", pwd, pwd),
-		"--volume", fmt.Sprintf("%s:/root/.circleci", settings.SettingsPath()),
+func generateDockerCommand(tempDir, configPath, image, pwd string, job string, dockerSocketPath string, arguments ...string) []string {
+	configPathInsideContainer := fmt.Sprintf("%s/local_build_config.yml", tempDir)
+	core := []string{"docker", "run", "--rm",
+		"--mount", fmt.Sprintf("type=bind,src=%s,dst=/var/run/docker.sock", dockerSocketPath),
+		"--mount", fmt.Sprintf("type=bind,src=%s,dst=%s", configPath, configPathInsideContainer),
+		"--mount", fmt.Sprintf("type=bind,src=%s,dst=%s", pwd, pwd),
+		"--mount", fmt.Sprintf("type=bind,src=%s,dst=/root/.circleci", settings.SettingsPath()),
 		"--workdir", pwd,
 		image, "circleci", "build", "--config", configPathInsideContainer, "--job", job}
 	return append(core, arguments...)

local/local_test.go

@@ -17,26 +17,24 @@ var _ = Describe("build", func() {
 		It("can generate a command line", func() {
 			home, err := os.UserHomeDir()
 			Expect(err).NotTo(HaveOccurred())
-			Expect(generateDockerCommand("/config/path", "docker-image-name", "/current/directory", "build", "/var/run/docker.sock", "extra-1", "extra-2")).To(ConsistOf(
+			Expect(generateDockerCommand("/tempdir", "/config/path", "docker-image-name", "/current/directory", "build", "/var/run/docker.sock", "extra-1", "extra-2")).To(ConsistOf(
 				"docker",
 				"run",
-				"--interactive",
-				"--tty",
 				"--rm",
-				"--volume", "/var/run/docker.sock:/var/run/docker.sock",
-				"--volume", "/config/path:/tmp/local_build_config.yml",
-				"--volume", "/current/directory:/current/directory",
-				"--volume", home+"/.circleci:/root/.circleci",
+				"--mount", "type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock",
+				"--mount", "type=bind,src=/config/path,dst=/tempdir/local_build_config.yml",
+				"--mount", "type=bind,src=/current/directory,dst=/current/directory",
+				"--mount", "type=bind,src="+home+"/.circleci,dst=/root/.circleci",
 				"--workdir", "/current/directory",
 				"docker-image-name", "circleci", "build",
-				"--config", "/tmp/local_build_config.yml",
+				"--config", "/tempdir/local_build_config.yml",
 				"--job", "build",
 				"extra-1", "extra-2",
 			))
 		})
 
 		It("can write temp files", func() {
-			path, err := writeStringToTempFile("cynosure")
+			path, err := writeStringToTempFile("/tmp", "cynosure")
 			Expect(err).NotTo(HaveOccurred())
 			defer os.Remove(path)
 			Expect(os.ReadFile(path)).To(BeEquivalentTo("cynosure"))

settings/settings.go

@@ -49,6 +49,7 @@ type Config struct {
 	// The value of this field is the path where the telemetry will be written
 	MockTelemetry string            `yaml:"-"`
 	OrbPublishing OrbPublishingInfo `yaml:"orb_publishing"`
+	TempDir       string            `yaml:"temp_dir,omitempty"`
 }
 
 type OrbPublishingInfo struct {