diff --git a/README.md b/README.md index 34237c58..51642345 100644 --- a/README.md +++ b/README.md @@ -388,5 +388,6 @@ On MacOS or Windows systems, we recommend either using a VM or the provided `.de | `--coder-agent-subsystem` | `CODER_AGENT_SUBSYSTEM` | | Coder agent subsystems to report when forwarding logs. The envbuilder subsystem is always included. | | `--push-image` | `ENVBUILDER_PUSH_IMAGE` | | Push the built image to a remote registry. This option forces a reproducible build. | | `--get-cached-image` | `ENVBUILDER_GET_CACHED_IMAGE` | | Print the digest of the cached image, if available. Exits with an error if not found. | +| `--remote-repo-build-mode` | `ENVBUILDER_REMOTE_REPO_BUILD_MODE` | `false` | Use the remote repository as the source of truth when building the image. Enabling this option ignores user changes to local files and they will not be reflected in the image. This can be used to improving cache utilization when multiple users are building working on the same repository. | | `--verbose` | `ENVBUILDER_VERBOSE` | | Enable verbose logging. | diff --git a/constants/constants.go b/constants/constants.go index ccdfcb8c..042660dd 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -28,4 +28,8 @@ var ( // to skip building when a container is restarting. // e.g. docker stop -> docker start MagicFile = filepath.Join(MagicDir, "built") + + // MagicFile is the location of the build context when + // using remote build mode. + MagicRemoteRepoDir = filepath.Join(MagicDir, "repo") ) diff --git a/envbuilder.go b/envbuilder.go index dc2ead8e..c65ae62f 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "context" - "crypto/x509" "encoding/base64" "encoding/json" "errors" @@ -42,7 +41,6 @@ import ( _ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem" "github.com/docker/cli/cli/config/configfile" "github.com/fatih/color" - "github.com/go-git/go-git/v5/plumbing/transport" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/kballard/go-shellquote" @@ -92,12 +90,6 @@ func Run(ctx context.Context, opts options.Options) error { opts.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) - var caBundle []byte - caBundle, err := initCABundle(opts.SSLCertBase64) - if err != nil { - return err - } - cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.DockerConfigBase64) if err != nil { return err @@ -108,51 +100,23 @@ func Run(ctx context.Context, opts options.Options) error { } }() // best effort + buildTimeWorkspaceFolder := opts.WorkspaceFolder var fallbackErr error var cloned bool if opts.GitURL != "" { + cloneOpts, err := git.CloneOptionsFromOptions(opts) + if err != nil { + return fmt.Errorf("git clone options: %w", err) + } + endStage := startStage("📦 Cloning %s to %s...", newColor(color.FgCyan).Sprintf(opts.GitURL), - newColor(color.FgCyan).Sprintf(opts.WorkspaceFolder), + newColor(color.FgCyan).Sprintf(cloneOpts.Path), ) - reader, writer := io.Pipe() - defer reader.Close() - defer writer.Close() - go func() { - data := make([]byte, 4096) - for { - read, err := reader.Read(data) - if err != nil { - return - } - content := data[:read] - for _, line := range strings.Split(string(content), "\r") { - if line == "" { - continue - } - opts.Logger(log.LevelInfo, "#1: %s", strings.TrimSpace(line)) - } - } - }() - - cloneOpts := git.CloneRepoOptions{ - Path: opts.WorkspaceFolder, - Storage: opts.Filesystem, - Insecure: opts.Insecure, - Progress: writer, - SingleBranch: opts.GitCloneSingleBranch, - Depth: int(opts.GitCloneDepth), - CABundle: caBundle, - } - - cloneOpts.RepoAuth = git.SetupRepoAuth(&opts) - if opts.GitHTTPProxyURL != "" { - cloneOpts.ProxyOptions = transport.ProxyOptions{ - URL: opts.GitHTTPProxyURL, - } - } - cloneOpts.RepoURL = opts.GitURL + w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) }) + defer w.Close() + cloneOpts.Progress = w cloned, fallbackErr = git.CloneRepo(ctx, cloneOpts) if fallbackErr == nil { @@ -165,6 +129,34 @@ func Run(ctx context.Context, opts options.Options) error { opts.Logger(log.LevelError, "Failed to clone repository: %s", fallbackErr.Error()) opts.Logger(log.LevelError, "Falling back to the default image...") } + + // Always clone the repo in remote repo build mode into a location that + // we control that isn't affected by the users changes. + if opts.RemoteRepoBuildMode { + cloneOpts, err := git.CloneOptionsFromOptions(opts) + if err != nil { + return fmt.Errorf("git clone options: %w", err) + } + cloneOpts.Path = constants.MagicRemoteRepoDir + + endStage := startStage("📦 Remote repo build mode enabled, cloning %s to %s for build context...", + newColor(color.FgCyan).Sprintf(opts.GitURL), + newColor(color.FgCyan).Sprintf(cloneOpts.Path), + ) + + w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) }) + defer w.Close() + cloneOpts.Progress = w + + fallbackErr = git.ShallowCloneRepo(ctx, cloneOpts) + if fallbackErr == nil { + endStage("📦 Cloned repository!") + buildTimeWorkspaceFolder = cloneOpts.Path + } else { + opts.Logger(log.LevelError, "Failed to clone repository for remote repo mode: %s", err.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") + } + } } defaultBuildParams := func() (*devcontainer.Compiled, error) { @@ -205,7 +197,7 @@ func Run(ctx context.Context, opts options.Options) error { // devcontainer is a standard, so it's reasonable to be the default. var devcontainerDir string var err error - devcontainerPath, devcontainerDir, err = findDevcontainerJSON(opts) + devcontainerPath, devcontainerDir, err = findDevcontainerJSON(buildTimeWorkspaceFolder, opts) if err != nil { opts.Logger(log.LevelError, "Failed to locate devcontainer.json: %s", err.Error()) opts.Logger(log.LevelError, "Falling back to the default image...") @@ -244,13 +236,13 @@ func Run(ctx context.Context, opts options.Options) error { } } else { // If a Dockerfile was specified, we use that. - dockerfilePath := filepath.Join(opts.WorkspaceFolder, opts.DockerfilePath) + dockerfilePath := filepath.Join(buildTimeWorkspaceFolder, opts.DockerfilePath) // If the dockerfilePath is specified and deeper than the base of WorkspaceFolder AND the BuildContextPath is // not defined, show a warning dockerfileDir := filepath.Dir(dockerfilePath) - if dockerfileDir != filepath.Clean(opts.WorkspaceFolder) && opts.BuildContextPath == "" { - opts.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, opts.WorkspaceFolder) + if dockerfileDir != filepath.Clean(buildTimeWorkspaceFolder) && opts.BuildContextPath == "" { + opts.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, buildTimeWorkspaceFolder) opts.Logger(log.LevelWarn, "\t-> set BUILD_CONTEXT_PATH to %q to fix", dockerfileDir) } @@ -263,7 +255,7 @@ func Run(ctx context.Context, opts options.Options) error { buildParams = &devcontainer.Compiled{ DockerfilePath: dockerfilePath, DockerfileContent: string(content), - BuildContext: filepath.Join(opts.WorkspaceFolder, opts.BuildContextPath), + BuildContext: filepath.Join(buildTimeWorkspaceFolder, opts.BuildContextPath), } } } @@ -552,10 +544,10 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) if err != nil { return fmt.Errorf("unmarshal metadata: %w", err) } - opts.Logger(log.LevelInfo, "#3: 👀 Found devcontainer.json label metadata in image...") + opts.Logger(log.LevelInfo, "#%d: 👀 Found devcontainer.json label metadata in image...", stageNumber) for _, container := range devContainer { if container.RemoteUser != "" { - opts.Logger(log.LevelInfo, "#3: 🧑 Updating the user to %q!", container.RemoteUser) + opts.Logger(log.LevelInfo, "#%d: 🧑 Updating the user to %q!", stageNumber, container.RemoteUser) configFile.Config.User = container.RemoteUser } @@ -654,7 +646,7 @@ ENTRYPOINT [%q]`, exePath, exePath, exePath) username = buildParams.User } if username == "" { - opts.Logger(log.LevelWarn, "#3: no user specified, using root") + opts.Logger(log.LevelWarn, "#%d: no user specified, using root", stageNumber) } userInfo, err := getUser(username) @@ -857,11 +849,6 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) opts.Logger(log.LevelInfo, "%s - Build development environments from repositories in a container", newColor(color.Bold).Sprintf("envbuilder")) - caBundle, err := initCABundle(opts.SSLCertBase64) - if err != nil { - return nil, err - } - cleanupDockerConfigJSON, err := initDockerConfigJSON(opts.DockerConfigBase64) if err != nil { return nil, err @@ -872,51 +859,23 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) } }() // best effort + buildTimeWorkspaceFolder := opts.WorkspaceFolder var fallbackErr error var cloned bool if opts.GitURL != "" { + cloneOpts, err := git.CloneOptionsFromOptions(opts) + if err != nil { + return nil, fmt.Errorf("git clone options: %w", err) + } + endStage := startStage("📦 Cloning %s to %s...", newColor(color.FgCyan).Sprintf(opts.GitURL), - newColor(color.FgCyan).Sprintf(opts.WorkspaceFolder), + newColor(color.FgCyan).Sprintf(cloneOpts.Path), ) - reader, writer := io.Pipe() - defer reader.Close() - defer writer.Close() - go func() { - data := make([]byte, 4096) - for { - read, err := reader.Read(data) - if err != nil { - return - } - content := data[:read] - for _, line := range strings.Split(string(content), "\r") { - if line == "" { - continue - } - opts.Logger(log.LevelInfo, "#1: %s", strings.TrimSpace(line)) - } - } - }() - - cloneOpts := git.CloneRepoOptions{ - Path: opts.WorkspaceFolder, - Storage: opts.Filesystem, - Insecure: opts.Insecure, - Progress: writer, - SingleBranch: opts.GitCloneSingleBranch, - Depth: int(opts.GitCloneDepth), - CABundle: caBundle, - } - - cloneOpts.RepoAuth = git.SetupRepoAuth(&opts) - if opts.GitHTTPProxyURL != "" { - cloneOpts.ProxyOptions = transport.ProxyOptions{ - URL: opts.GitHTTPProxyURL, - } - } - cloneOpts.RepoURL = opts.GitURL + w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) }) + defer w.Close() + cloneOpts.Progress = w cloned, fallbackErr = git.CloneRepo(ctx, cloneOpts) if fallbackErr == nil { @@ -929,6 +888,34 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) opts.Logger(log.LevelError, "Failed to clone repository: %s", fallbackErr.Error()) opts.Logger(log.LevelError, "Falling back to the default image...") } + + // Always clone the repo in remote repo build mode into a location that + // we control that isn't affected by the users changes. + if opts.RemoteRepoBuildMode { + cloneOpts, err := git.CloneOptionsFromOptions(opts) + if err != nil { + return nil, fmt.Errorf("git clone options: %w", err) + } + cloneOpts.Path = constants.MagicRemoteRepoDir + + endStage := startStage("📦 Remote repo build mode enabled, cloning %s to %s for build context...", + newColor(color.FgCyan).Sprintf(opts.GitURL), + newColor(color.FgCyan).Sprintf(cloneOpts.Path), + ) + + w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNumber, line) }) + defer w.Close() + cloneOpts.Progress = w + + fallbackErr = git.ShallowCloneRepo(ctx, cloneOpts) + if fallbackErr == nil { + endStage("📦 Cloned repository!") + buildTimeWorkspaceFolder = cloneOpts.Path + } else { + opts.Logger(log.LevelError, "Failed to clone repository for remote repo mode: %s", err.Error()) + opts.Logger(log.LevelError, "Falling back to the default image...") + } + } } defaultBuildParams := func() (*devcontainer.Compiled, error) { @@ -967,7 +954,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) // devcontainer is a standard, so it's reasonable to be the default. var devcontainerDir string var err error - devcontainerPath, devcontainerDir, err = findDevcontainerJSON(opts) + devcontainerPath, devcontainerDir, err = findDevcontainerJSON(buildTimeWorkspaceFolder, opts) if err != nil { opts.Logger(log.LevelError, "Failed to locate devcontainer.json: %s", err.Error()) opts.Logger(log.LevelError, "Falling back to the default image...") @@ -1005,13 +992,13 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) } } else { // If a Dockerfile was specified, we use that. - dockerfilePath := filepath.Join(opts.WorkspaceFolder, opts.DockerfilePath) + dockerfilePath := filepath.Join(buildTimeWorkspaceFolder, opts.DockerfilePath) // If the dockerfilePath is specified and deeper than the base of WorkspaceFolder AND the BuildContextPath is // not defined, show a warning dockerfileDir := filepath.Dir(dockerfilePath) - if dockerfileDir != filepath.Clean(opts.WorkspaceFolder) && opts.BuildContextPath == "" { - opts.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, opts.WorkspaceFolder) + if dockerfileDir != filepath.Clean(buildTimeWorkspaceFolder) && opts.BuildContextPath == "" { + opts.Logger(log.LevelWarn, "given dockerfile %q is below %q and no custom build context has been defined", dockerfilePath, buildTimeWorkspaceFolder) opts.Logger(log.LevelWarn, "\t-> set BUILD_CONTEXT_PATH to %q to fix", dockerfileDir) } @@ -1024,7 +1011,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error) buildParams = &devcontainer.Compiled{ DockerfilePath: dockerfilePath, DockerfileContent: string(content), - BuildContext: filepath.Join(opts.WorkspaceFolder, opts.BuildContextPath), + BuildContext: filepath.Join(buildTimeWorkspaceFolder, opts.BuildContextPath), } } } @@ -1281,7 +1268,11 @@ func newColor(value ...color.Attribute) *color.Color { return c } -func findDevcontainerJSON(options options.Options) (string, string, error) { +func findDevcontainerJSON(workspaceFolder string, options options.Options) (string, string, error) { + if workspaceFolder == "" { + workspaceFolder = options.WorkspaceFolder + } + // 0. Check if custom devcontainer directory or path is provided. if options.DevcontainerDir != "" || options.DevcontainerJSONPath != "" { devcontainerDir := options.DevcontainerDir @@ -1291,7 +1282,7 @@ func findDevcontainerJSON(options options.Options) (string, string, error) { // If `devcontainerDir` is not an absolute path, assume it is relative to the workspace folder. if !filepath.IsAbs(devcontainerDir) { - devcontainerDir = filepath.Join(options.WorkspaceFolder, devcontainerDir) + devcontainerDir = filepath.Join(workspaceFolder, devcontainerDir) } // An absolute location always takes a precedence. @@ -1310,20 +1301,20 @@ func findDevcontainerJSON(options options.Options) (string, string, error) { return devcontainerPath, devcontainerDir, nil } - // 1. Check `options.WorkspaceFolder`/.devcontainer/devcontainer.json. - location := filepath.Join(options.WorkspaceFolder, ".devcontainer", "devcontainer.json") + // 1. Check `workspaceFolder`/.devcontainer/devcontainer.json. + location := filepath.Join(workspaceFolder, ".devcontainer", "devcontainer.json") if _, err := options.Filesystem.Stat(location); err == nil { return location, filepath.Dir(location), nil } - // 2. Check `options.WorkspaceFolder`/devcontainer.json. - location = filepath.Join(options.WorkspaceFolder, "devcontainer.json") + // 2. Check `workspaceFolder`/devcontainer.json. + location = filepath.Join(workspaceFolder, "devcontainer.json") if _, err := options.Filesystem.Stat(location); err == nil { return location, filepath.Dir(location), nil } - // 3. Check every folder: `options.WorkspaceFolder`/.devcontainer//devcontainer.json. - devcontainerDir := filepath.Join(options.WorkspaceFolder, ".devcontainer") + // 3. Check every folder: `workspaceFolder`/.devcontainer//devcontainer.json. + devcontainerDir := filepath.Join(workspaceFolder, ".devcontainer") fileInfos, err := options.Filesystem.ReadDir(devcontainerDir) if err != nil { @@ -1389,25 +1380,6 @@ func copyFile(src, dst string) error { return nil } -func initCABundle(sslCertBase64 string) ([]byte, error) { - if sslCertBase64 == "" { - return []byte{}, nil - } - certPool, err := x509.SystemCertPool() - if err != nil { - return nil, fmt.Errorf("get global system cert pool: %w", err) - } - data, err := base64.StdEncoding.DecodeString(sslCertBase64) - if err != nil { - return nil, fmt.Errorf("base64 decode ssl cert: %w", err) - } - ok := certPool.AppendCertsFromPEM(data) - if !ok { - return nil, fmt.Errorf("failed to append the ssl cert to the global pool: %s", data) - } - return data, nil -} - func initDockerConfigJSON(dockerConfigBase64 string) (func() error, error) { var cleanupOnce sync.Once noop := func() error { return nil } diff --git a/envbuilder_internal_test.go b/envbuilder_internal_test.go index 3af4b5e4..eb756071 100644 --- a/envbuilder_internal_test.go +++ b/envbuilder_internal_test.go @@ -13,149 +13,169 @@ import ( func TestFindDevcontainerJSON(t *testing.T) { t.Parallel() - t.Run("empty filesystem", func(t *testing.T) { - t.Parallel() - - // given - fs := memfs.New() - - // when - _, _, err := findDevcontainerJSON(options.Options{ - Filesystem: fs, - WorkspaceFolder: "/workspace", + defaultWorkspaceFolder := "/workspace" + + for _, tt := range []struct { + name string + workspaceFolder string + }{ + { + name: "Default", + workspaceFolder: defaultWorkspaceFolder, + }, + { + name: "RepoMode", + workspaceFolder: "/.envbuilder/repo", + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + t.Run("empty filesystem", func(t *testing.T) { + t.Parallel() + + // given + fs := memfs.New() + + // when + _, _, err := findDevcontainerJSON(tt.workspaceFolder, options.Options{ + Filesystem: fs, + WorkspaceFolder: "/workspace", + }) + + // then + require.Error(t, err) + }) + + t.Run("devcontainer.json is missing", func(t *testing.T) { + t.Parallel() + + // given + fs := memfs.New() + err := fs.MkdirAll(tt.workspaceFolder+"/.devcontainer", 0o600) + require.NoError(t, err) + + // when + _, _, err = findDevcontainerJSON(tt.workspaceFolder, options.Options{ + Filesystem: fs, + WorkspaceFolder: "/workspace", + }) + + // then + require.Error(t, err) + }) + + t.Run("default configuration", func(t *testing.T) { + t.Parallel() + + // given + fs := memfs.New() + err := fs.MkdirAll(tt.workspaceFolder+"/.devcontainer", 0o600) + require.NoError(t, err) + _, err = fs.Create(tt.workspaceFolder + "/.devcontainer/devcontainer.json") + require.NoError(t, err) + + // when + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(tt.workspaceFolder, options.Options{ + Filesystem: fs, + WorkspaceFolder: "/workspace", + }) + + // then + require.NoError(t, err) + assert.Equal(t, tt.workspaceFolder+"/.devcontainer/devcontainer.json", devcontainerPath) + assert.Equal(t, tt.workspaceFolder+"/.devcontainer", devcontainerDir) + }) + + t.Run("overridden .devcontainer directory", func(t *testing.T) { + t.Parallel() + + // given + fs := memfs.New() + err := fs.MkdirAll(tt.workspaceFolder+"/experimental-devcontainer", 0o600) + require.NoError(t, err) + _, err = fs.Create(tt.workspaceFolder + "/experimental-devcontainer/devcontainer.json") + require.NoError(t, err) + + // when + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(tt.workspaceFolder, options.Options{ + Filesystem: fs, + WorkspaceFolder: "/workspace", + DevcontainerDir: "experimental-devcontainer", + }) + + // then + require.NoError(t, err) + assert.Equal(t, tt.workspaceFolder+"/experimental-devcontainer/devcontainer.json", devcontainerPath) + assert.Equal(t, tt.workspaceFolder+"/experimental-devcontainer", devcontainerDir) + }) + + t.Run("overridden devcontainer.json path", func(t *testing.T) { + t.Parallel() + + // given + fs := memfs.New() + err := fs.MkdirAll(tt.workspaceFolder+"/.devcontainer", 0o600) + require.NoError(t, err) + _, err = fs.Create(tt.workspaceFolder + "/.devcontainer/experimental.json") + require.NoError(t, err) + + // when + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(tt.workspaceFolder, options.Options{ + Filesystem: fs, + WorkspaceFolder: "/workspace", + DevcontainerJSONPath: "experimental.json", + }) + + // then + require.NoError(t, err) + assert.Equal(t, tt.workspaceFolder+"/.devcontainer/experimental.json", devcontainerPath) + assert.Equal(t, tt.workspaceFolder+"/.devcontainer", devcontainerDir) + }) + + t.Run("devcontainer.json in workspace root", func(t *testing.T) { + t.Parallel() + + // given + fs := memfs.New() + err := fs.MkdirAll(tt.workspaceFolder+"", 0o600) + require.NoError(t, err) + _, err = fs.Create(tt.workspaceFolder + "/devcontainer.json") + require.NoError(t, err) + + // when + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(tt.workspaceFolder, options.Options{ + Filesystem: fs, + WorkspaceFolder: "/workspace", + }) + + // then + require.NoError(t, err) + assert.Equal(t, tt.workspaceFolder+"/devcontainer.json", devcontainerPath) + assert.Equal(t, tt.workspaceFolder+"", devcontainerDir) + }) + + t.Run("devcontainer.json in subfolder of .devcontainer", func(t *testing.T) { + t.Parallel() + + // given + fs := memfs.New() + err := fs.MkdirAll(tt.workspaceFolder+"/.devcontainer/sample", 0o600) + require.NoError(t, err) + _, err = fs.Create(tt.workspaceFolder + "/.devcontainer/sample/devcontainer.json") + require.NoError(t, err) + + // when + devcontainerPath, devcontainerDir, err := findDevcontainerJSON(tt.workspaceFolder, options.Options{ + Filesystem: fs, + WorkspaceFolder: "/workspace", + }) + + // then + require.NoError(t, err) + assert.Equal(t, tt.workspaceFolder+"/.devcontainer/sample/devcontainer.json", devcontainerPath) + assert.Equal(t, tt.workspaceFolder+"/.devcontainer/sample", devcontainerDir) + }) }) - - // then - require.Error(t, err) - }) - - t.Run("devcontainer.json is missing", func(t *testing.T) { - t.Parallel() - - // given - fs := memfs.New() - err := fs.MkdirAll("/workspace/.devcontainer", 0o600) - require.NoError(t, err) - - // when - _, _, err = findDevcontainerJSON(options.Options{ - Filesystem: fs, - WorkspaceFolder: "/workspace", - }) - - // then - require.Error(t, err) - }) - - t.Run("default configuration", func(t *testing.T) { - t.Parallel() - - // given - fs := memfs.New() - err := fs.MkdirAll("/workspace/.devcontainer", 0o600) - require.NoError(t, err) - _, err = fs.Create("/workspace/.devcontainer/devcontainer.json") - require.NoError(t, err) - - // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ - Filesystem: fs, - WorkspaceFolder: "/workspace", - }) - - // then - require.NoError(t, err) - assert.Equal(t, "/workspace/.devcontainer/devcontainer.json", devcontainerPath) - assert.Equal(t, "/workspace/.devcontainer", devcontainerDir) - }) - - t.Run("overridden .devcontainer directory", func(t *testing.T) { - t.Parallel() - - // given - fs := memfs.New() - err := fs.MkdirAll("/workspace/experimental-devcontainer", 0o600) - require.NoError(t, err) - _, err = fs.Create("/workspace/experimental-devcontainer/devcontainer.json") - require.NoError(t, err) - - // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ - Filesystem: fs, - WorkspaceFolder: "/workspace", - DevcontainerDir: "experimental-devcontainer", - }) - - // then - require.NoError(t, err) - assert.Equal(t, "/workspace/experimental-devcontainer/devcontainer.json", devcontainerPath) - assert.Equal(t, "/workspace/experimental-devcontainer", devcontainerDir) - }) - - t.Run("overridden devcontainer.json path", func(t *testing.T) { - t.Parallel() - - // given - fs := memfs.New() - err := fs.MkdirAll("/workspace/.devcontainer", 0o600) - require.NoError(t, err) - _, err = fs.Create("/workspace/.devcontainer/experimental.json") - require.NoError(t, err) - - // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ - Filesystem: fs, - WorkspaceFolder: "/workspace", - DevcontainerJSONPath: "experimental.json", - }) - - // then - require.NoError(t, err) - assert.Equal(t, "/workspace/.devcontainer/experimental.json", devcontainerPath) - assert.Equal(t, "/workspace/.devcontainer", devcontainerDir) - }) - - t.Run("devcontainer.json in workspace root", func(t *testing.T) { - t.Parallel() - - // given - fs := memfs.New() - err := fs.MkdirAll("/workspace", 0o600) - require.NoError(t, err) - _, err = fs.Create("/workspace/devcontainer.json") - require.NoError(t, err) - - // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ - Filesystem: fs, - WorkspaceFolder: "/workspace", - }) - - // then - require.NoError(t, err) - assert.Equal(t, "/workspace/devcontainer.json", devcontainerPath) - assert.Equal(t, "/workspace", devcontainerDir) - }) - - t.Run("devcontainer.json in subfolder of .devcontainer", func(t *testing.T) { - t.Parallel() - - // given - fs := memfs.New() - err := fs.MkdirAll("/workspace/.devcontainer/sample", 0o600) - require.NoError(t, err) - _, err = fs.Create("/workspace/.devcontainer/sample/devcontainer.json") - require.NoError(t, err) - - // when - devcontainerPath, devcontainerDir, err := findDevcontainerJSON(options.Options{ - Filesystem: fs, - WorkspaceFolder: "/workspace", - }) - - // then - require.NoError(t, err) - assert.Equal(t, "/workspace/.devcontainer/sample/devcontainer.json", devcontainerPath) - assert.Equal(t, "/workspace/.devcontainer/sample", devcontainerDir) - }) + } } diff --git a/git/git.go b/git/git.go index 199e350b..d6c1371c 100644 --- a/git/git.go +++ b/git/git.go @@ -126,6 +126,41 @@ func CloneRepo(ctx context.Context, opts CloneRepoOptions) (bool, error) { return true, nil } +// ShallowCloneRepo will clone the repository at the given URL into the given path +// with a depth of 1. If the destination folder exists and is not empty, the +// clone will not be performed. +// +// The bool returned states whether the repository was cloned or not. +func ShallowCloneRepo(ctx context.Context, opts CloneRepoOptions) error { + opts.Depth = 1 + opts.SingleBranch = true + + if opts.Path == "" { + return errors.New("path is required") + } + + // Avoid clobbering the destination. + if _, err := opts.Storage.Stat(opts.Path); err == nil { + files, err := opts.Storage.ReadDir(opts.Path) + if err != nil { + return fmt.Errorf("read dir %q: %w", opts.Path, err) + } + if len(files) > 0 { + return fmt.Errorf("directory %q is not empty", opts.Path) + } + } + + cloned, err := CloneRepo(ctx, opts) + if err != nil { + return err + } + if !cloned { + return errors.New("repository already exists") + } + + return nil +} + // ReadPrivateKey attempts to read an SSH private key from path // and returns an ssh.Signer. func ReadPrivateKey(path string) (gossh.Signer, error) { @@ -253,3 +288,68 @@ func SetupRepoAuth(options *options.Options) transport.AuthMethod { } return auth } + +func CloneOptionsFromOptions(options options.Options) (CloneRepoOptions, error) { + caBundle, err := options.CABundle() + if err != nil { + return CloneRepoOptions{}, err + } + + cloneOpts := CloneRepoOptions{ + Path: options.WorkspaceFolder, + Storage: options.Filesystem, + Insecure: options.Insecure, + SingleBranch: options.GitCloneSingleBranch, + Depth: int(options.GitCloneDepth), + CABundle: caBundle, + } + + cloneOpts.RepoAuth = SetupRepoAuth(&options) + if options.GitHTTPProxyURL != "" { + cloneOpts.ProxyOptions = transport.ProxyOptions{ + URL: options.GitHTTPProxyURL, + } + } + cloneOpts.RepoURL = options.GitURL + + return cloneOpts, nil +} + +type progressWriter struct { + io.WriteCloser + r io.ReadCloser +} + +func (w *progressWriter) Close() error { + err := w.r.Close() + err2 := w.WriteCloser.Close() + if err != nil { + return err + } + return err2 +} + +func ProgressWriter(write func(line string)) io.WriteCloser { + reader, writer := io.Pipe() + go func() { + data := make([]byte, 4096) + for { + read, err := reader.Read(data) + if err != nil { + return + } + content := data[:read] + for _, line := range strings.Split(string(content), "\r") { + if line == "" { + continue + } + write(strings.TrimSpace(line)) + } + } + }() + + return &progressWriter{ + WriteCloser: writer, + r: reader, + } +} diff --git a/git/git_test.go b/git/git_test.go index 08ab1f93..14656886 100644 --- a/git/git_test.go +++ b/git/git_test.go @@ -169,6 +169,73 @@ func TestCloneRepo(t *testing.T) { } } +func TestShallowCloneRepo(t *testing.T) { + t.Parallel() + + t.Run("NotEmpty", func(t *testing.T) { + t.Parallel() + srvFS := memfs.New() + _ = gittest.NewRepo(t, srvFS, + gittest.Commit(t, "README.md", "Hello, world!", "Many wow!"), + gittest.Commit(t, "foo", "bar!", "Such commit!"), + gittest.Commit(t, "baz", "qux", "V nice!"), + ) + authMW := mwtest.BasicAuthMW("test", "test") + srv := httptest.NewServer(authMW(gittest.NewServer(srvFS))) + + clientFS := memfs.New() + // Not empty. + err := clientFS.MkdirAll("/repo", 0o500) + require.NoError(t, err) + f, err := clientFS.Create("/repo/not-empty") + require.NoError(t, err) + require.NoError(t, f.Close()) + + err = git.ShallowCloneRepo(context.Background(), git.CloneRepoOptions{ + Path: "/repo", + RepoURL: srv.URL, + Storage: clientFS, + RepoAuth: &githttp.BasicAuth{ + Username: "test", + Password: "test", + }, + }) + require.Error(t, err) + }) + t.Run("OK", func(t *testing.T) { + // 2024/08/01 13:22:08 unsupported capability: shallow + // clone "http://127.0.0.1:41499": unexpected client error: unexpected requesting "http://127.0.0.1:41499/git-upload-pack" status code: 500 + t.Skip("The gittest server doesn't support shallow cloning, skip for now...") + + t.Parallel() + srvFS := memfs.New() + _ = gittest.NewRepo(t, srvFS, + gittest.Commit(t, "README.md", "Hello, world!", "Many wow!"), + gittest.Commit(t, "foo", "bar!", "Such commit!"), + gittest.Commit(t, "baz", "qux", "V nice!"), + ) + authMW := mwtest.BasicAuthMW("test", "test") + srv := httptest.NewServer(authMW(gittest.NewServer(srvFS))) + + clientFS := memfs.New() + + err := git.ShallowCloneRepo(context.Background(), git.CloneRepoOptions{ + Path: "/repo", + RepoURL: srv.URL, + Storage: clientFS, + RepoAuth: &githttp.BasicAuth{ + Username: "test", + Password: "test", + }, + }) + require.NoError(t, err) + for _, path := range []string{"README.md", "foo", "baz"} { + _, err := clientFS.Stat(filepath.Join("/repo", path)) + require.NoError(t, err) + } + }) +} + func TestCloneRepoSSH(t *testing.T) { t.Parallel() diff --git a/options/options.go b/options/options.go index 5771b506..3432cee5 100644 --- a/options/options.go +++ b/options/options.go @@ -1,6 +1,9 @@ package options import ( + "crypto/x509" + "encoding/base64" + "fmt" "os" "strings" @@ -146,6 +149,13 @@ type Options struct { // GetCachedImage is a flag to determine if the cached image is available, // and if it is, to return it. GetCachedImage bool + + // RemoteRepoBuildMode uses the remote repository as the source of truth + // when building the image. Enabling this option ignores user changes to + // local files and they will not be reflected in the image. This can be + // used to improving cache utilization when multiple users are building + // working on the same repository. + RemoteRepoBuildMode bool } const envPrefix = "ENVBUILDER_" @@ -417,6 +427,17 @@ func (o *Options) CLI() serpent.OptionSet { Description: "Print the digest of the cached image, if available. " + "Exits with an error if not found.", }, + { + Flag: "remote-repo-build-mode", + Env: WithEnvPrefix("REMOTE_REPO_BUILD_MODE"), + Value: serpent.BoolOf(&o.RemoteRepoBuildMode), + Default: "false", + Description: "Use the remote repository as the source of truth " + + "when building the image. Enabling this option ignores user changes " + + "to local files and they will not be reflected in the image. This can " + + "be used to improving cache utilization when multiple users are building " + + "working on the same repository.", + }, { Flag: "verbose", Env: WithEnvPrefix("VERBOSE"), @@ -482,6 +503,26 @@ func (o *Options) Markdown() string { return sb.String() } +func (o *Options) CABundle() ([]byte, error) { + if o.SSLCertBase64 == "" { + return nil, nil + } + + certPool, err := x509.SystemCertPool() + if err != nil { + return nil, fmt.Errorf("get global system cert pool: %w", err) + } + data, err := base64.StdEncoding.DecodeString(o.SSLCertBase64) + if err != nil { + return nil, fmt.Errorf("base64 decode ssl cert: %w", err) + } + ok := certPool.AppendCertsFromPEM(data) + if !ok { + return nil, fmt.Errorf("failed to append the ssl cert to the global pool: %s", data) + } + return data, nil +} + func skipDeprecatedOptions(options []serpent.Option) []serpent.Option { var activeOptions []serpent.Option diff --git a/options/testdata/options.golden b/options/testdata/options.golden index d59ccd21..0bfbd64a 100644 --- a/options/testdata/options.golden +++ b/options/testdata/options.golden @@ -138,6 +138,13 @@ OPTIONS: Push the built image to a remote registry. This option forces a reproducible build. + --remote-repo-build-mode bool, $ENVBUILDER_REMOTE_REPO_BUILD_MODE (default: false) + Use the remote repository as the source of truth when building the + image. Enabling this option ignores user changes to local files and + they will not be reflected in the image. This can be used to improving + cache utilization when multiple users are building working on the same + repository. + --setup-script string, $ENVBUILDER_SETUP_SCRIPT The script to run before the init script. It runs as the root user regardless of the user specified in the devcontainer.json file.