Add copy on write support for CNB builds (#1213)

commands/apps_dev.go

@@ -250,7 +250,7 @@ func RunAppsDevBuild(c *CmdConfig) error {
 		}
 	}
 
-	cli, err := c.Doit.GetContainerEngineClient()
+	cli, err := c.Doit.GetDockerEngineClient()
 	if err != nil {
 		return err
 	}
@@ -290,7 +290,8 @@ func RunAppsDevBuild(c *CmdConfig) error {
 	var res builder.ComponentBuilderResult
 	err = func() error {
 		defer cancel()
-		builder, err := c.componentBuilderFactory.NewComponentBuilder(cli, spec, builder.NewBuilderOpts{
+
+		builder, err := c.componentBuilderFactory.NewComponentBuilder(cli, conf.contextDir, spec, builder.NewBuilderOpts{
 			Component:            component,
 			Registry:             registryName,
 			EnvOverride:          envs,

commands/apps_dev_config.go

@@ -139,8 +139,9 @@ func RunAppsDevConfigUnset(c *CmdConfig) error {
 }
 
 type appDevConfig struct {
-	cmdConfig *CmdConfig
-	viper     *viper.Viper
+	contextDir string
+	cmdConfig  *CmdConfig
+	viper      *viper.Viper
 }
 
 var validAppDevKeys = map[string]bool{
@@ -197,16 +198,23 @@ func newAppDevConfig(cmdConfig *CmdConfig) (*appDevConfig, error) {
 		return nil, err
 	}
 
+	config.contextDir, err = os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+	gitRoot, err := findTopLevelGitDir(config.contextDir)
+	if err != nil && !errors.Is(err, errNoGitRepo) {
+		return nil, err
+	}
+	if gitRoot != "" {
+		config.contextDir = gitRoot
+	}
+
 	if devConfigFilePath == "" {
-		cwd, err := os.Getwd()
-		if err != nil {
-			return nil, err
-		}
-		configDir, err := findTopLevelGitDir(cwd)
-		if err != nil {
-			return nil, err
+		if gitRoot == "" {
+			return nil, errNoGitRepo
 		}
-		configDir = filepath.Join(configDir, ".do")
+		configDir := filepath.Join(gitRoot, ".do")
 		err = os.MkdirAll(configDir, os.ModePerm)
 		if err != nil {
 			return nil, err
@@ -278,6 +286,8 @@ func ensureStringInFile(file string, val string) error {
 	return nil
 }
 
+var errNoGitRepo = errors.New("no git repository found")
+
 // findTopLevelGitDir ...
 func findTopLevelGitDir(workingDir string) (string, error) {
 	dir, err := filepath.Abs(workingDir)

commands/apps_dev_test.go

@@ -34,8 +34,11 @@ func TestRunAppsDevBuild(t *testing.T) {
 			config.Doit.Set(config.NS, doctl.ArgRegistryName, registryName)
 			config.Doit.Set(config.NS, doctl.ArgInteractive, false)
 
+			conf, err := newAppDevConfig(config)
+			require.NoError(t, err)
+
 			tm.appBuilder.EXPECT().Build(gomock.Any()).Return(builder.ComponentBuilderResult{}, nil)
-			tm.appBuilderFactory.EXPECT().NewComponentBuilder(gomock.Any(), sampleSpec, gomock.Any()).Return(tm.appBuilder, nil)
+			tm.appBuilderFactory.EXPECT().NewComponentBuilder(gomock.Any(), conf.contextDir, sampleSpec, gomock.Any()).Return(tm.appBuilder, nil)
 
 			err = RunAppsDevBuild(config)
 			require.NoError(t, err)
@@ -44,7 +47,9 @@ func TestRunAppsDevBuild(t *testing.T) {
 
 	t.Run("with appID", func(t *testing.T) {
 		withTestClient(t, func(config *CmdConfig, tm *tcMocks) {
-			tm.appBuilderFactory.EXPECT().NewComponentBuilder(gomock.Any(), sampleSpec, gomock.Any()).Return(tm.appBuilder, nil)
+			conf, err := newAppDevConfig(config)
+			require.NoError(t, err)
+			tm.appBuilderFactory.EXPECT().NewComponentBuilder(gomock.Any(), conf.contextDir, sampleSpec, gomock.Any()).Return(tm.appBuilder, nil)
 			tm.appBuilder.EXPECT().Build(gomock.Any()).Return(builder.ComponentBuilderResult{}, nil)
 
 			tm.apps.EXPECT().Get(appID).Times(1).Return(&godo.App{
@@ -56,7 +61,7 @@ func TestRunAppsDevBuild(t *testing.T) {
 			config.Doit.Set(config.NS, doctl.ArgRegistryName, registryName)
 			config.Doit.Set(config.NS, doctl.ArgInteractive, false)
 
-			err := RunAppsDevBuild(config)
+			err = RunAppsDevBuild(config)
 			require.NoError(t, err)
 		})
 	})

commands/commands_test.go

@@ -147,91 +147,91 @@ func assertCommandNames(t *testing.T, cmd *Command, expected ...string) {
 type testFn func(c *CmdConfig, tm *tcMocks)
 
 type tcMocks struct {
-	account                  *domocks.MockAccountService
-	actions                  *domocks.MockActionsService
-	apps                     *domocks.MockAppsService
-	balance                  *domocks.MockBalanceService
-	billingHistory           *domocks.MockBillingHistoryService
-	databases                *domocks.MockDatabasesService
-	dropletActions           *domocks.MockDropletActionsService
-	droplets                 *domocks.MockDropletsService
-	keys                     *domocks.MockKeysService
-	sizes                    *domocks.MockSizesService
-	regions                  *domocks.MockRegionsService
-	images                   *domocks.MockImagesService
-	imageActions             *domocks.MockImageActionsService
-	invoices                 *domocks.MockInvoicesService
-	reservedIPs              *domocks.MockReservedIPsService
-	reservedIPActions        *domocks.MockReservedIPActionsService
-	domains                  *domocks.MockDomainsService
-	volumes                  *domocks.MockVolumesService
-	volumeActions            *domocks.MockVolumeActionsService
-	tags                     *domocks.MockTagsService
-	snapshots                *domocks.MockSnapshotsService
-	certificates             *domocks.MockCertificatesService
-	loadBalancers            *domocks.MockLoadBalancersService
-	firewalls                *domocks.MockFirewallsService
-	cdns                     *domocks.MockCDNsService
-	projects                 *domocks.MockProjectsService
-	kubernetes               *domocks.MockKubernetesService
-	registry                 *domocks.MockRegistryService
-	sshRunner                *domocks.MockRunner
-	vpcs                     *domocks.MockVPCsService
-	oneClick                 *domocks.MockOneClickService
-	listen                   *domocks.MockListenerService
-	monitoring               *domocks.MockMonitoringService
-	sandbox                  *domocks.MockSandboxService
-	appBuilderFactory        *builder.MockComponentBuilderFactory
-	appBuilder               *builder.MockComponentBuilder
-	appContainerEngineClient *builder.MockContainerEngineClient
+	account               *domocks.MockAccountService
+	actions               *domocks.MockActionsService
+	apps                  *domocks.MockAppsService
+	balance               *domocks.MockBalanceService
+	billingHistory        *domocks.MockBillingHistoryService
+	databases             *domocks.MockDatabasesService
+	dropletActions        *domocks.MockDropletActionsService
+	droplets              *domocks.MockDropletsService
+	keys                  *domocks.MockKeysService
+	sizes                 *domocks.MockSizesService
+	regions               *domocks.MockRegionsService
+	images                *domocks.MockImagesService
+	imageActions          *domocks.MockImageActionsService
+	invoices              *domocks.MockInvoicesService
+	reservedIPs           *domocks.MockReservedIPsService
+	reservedIPActions     *domocks.MockReservedIPActionsService
+	domains               *domocks.MockDomainsService
+	volumes               *domocks.MockVolumesService
+	volumeActions         *domocks.MockVolumeActionsService
+	tags                  *domocks.MockTagsService
+	snapshots             *domocks.MockSnapshotsService
+	certificates          *domocks.MockCertificatesService
+	loadBalancers         *domocks.MockLoadBalancersService
+	firewalls             *domocks.MockFirewallsService
+	cdns                  *domocks.MockCDNsService
+	projects              *domocks.MockProjectsService
+	kubernetes            *domocks.MockKubernetesService
+	registry              *domocks.MockRegistryService
+	sshRunner             *domocks.MockRunner
+	vpcs                  *domocks.MockVPCsService
+	oneClick              *domocks.MockOneClickService
+	listen                *domocks.MockListenerService
+	monitoring            *domocks.MockMonitoringService
+	sandbox               *domocks.MockSandboxService
+	appBuilderFactory     *builder.MockComponentBuilderFactory
+	appBuilder            *builder.MockComponentBuilder
+	appDockerEngineClient *builder.MockDockerEngineClient
 }
 
 func withTestClient(t *testing.T, tFn testFn) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
 	tm := &tcMocks{
-		account:                  domocks.NewMockAccountService(ctrl),
-		actions:                  domocks.NewMockActionsService(ctrl),
-		apps:                     domocks.NewMockAppsService(ctrl),
-		balance:                  domocks.NewMockBalanceService(ctrl),
-		billingHistory:           domocks.NewMockBillingHistoryService(ctrl),
-		keys:                     domocks.NewMockKeysService(ctrl),
-		sizes:                    domocks.NewMockSizesService(ctrl),
-		regions:                  domocks.NewMockRegionsService(ctrl),
-		images:                   domocks.NewMockImagesService(ctrl),
-		imageActions:             domocks.NewMockImageActionsService(ctrl),
-		invoices:                 domocks.NewMockInvoicesService(ctrl),
-		reservedIPs:              domocks.NewMockReservedIPsService(ctrl),
-		reservedIPActions:        domocks.NewMockReservedIPActionsService(ctrl),
-		droplets:                 domocks.NewMockDropletsService(ctrl),
-		dropletActions:           domocks.NewMockDropletActionsService(ctrl),
-		domains:                  domocks.NewMockDomainsService(ctrl),
-		tags:                     domocks.NewMockTagsService(ctrl),
-		volumes:                  domocks.NewMockVolumesService(ctrl),
-		volumeActions:            domocks.NewMockVolumeActionsService(ctrl),
-		snapshots:                domocks.NewMockSnapshotsService(ctrl),
-		certificates:             domocks.NewMockCertificatesService(ctrl),
-		loadBalancers:            domocks.NewMockLoadBalancersService(ctrl),
-		firewalls:                domocks.NewMockFirewallsService(ctrl),
-		cdns:                     domocks.NewMockCDNsService(ctrl),
-		projects:                 domocks.NewMockProjectsService(ctrl),
-		kubernetes:               domocks.NewMockKubernetesService(ctrl),
-		databases:                domocks.NewMockDatabasesService(ctrl),
-		registry:                 domocks.NewMockRegistryService(ctrl),
-		sshRunner:                domocks.NewMockRunner(ctrl),
-		vpcs:                     domocks.NewMockVPCsService(ctrl),
-		oneClick:                 domocks.NewMockOneClickService(ctrl),
-		listen:                   domocks.NewMockListenerService(ctrl),
-		monitoring:               domocks.NewMockMonitoringService(ctrl),
-		sandbox:                  domocks.NewMockSandboxService(ctrl),
-		appBuilderFactory:        builder.NewMockComponentBuilderFactory(ctrl),
-		appBuilder:               builder.NewMockComponentBuilder(ctrl),
-		appContainerEngineClient: builder.NewMockContainerEngineClient(ctrl),
+		account:               domocks.NewMockAccountService(ctrl),
+		actions:               domocks.NewMockActionsService(ctrl),
+		apps:                  domocks.NewMockAppsService(ctrl),
+		balance:               domocks.NewMockBalanceService(ctrl),
+		billingHistory:        domocks.NewMockBillingHistoryService(ctrl),
+		keys:                  domocks.NewMockKeysService(ctrl),
+		sizes:                 domocks.NewMockSizesService(ctrl),
+		regions:               domocks.NewMockRegionsService(ctrl),
+		images:                domocks.NewMockImagesService(ctrl),
+		imageActions:          domocks.NewMockImageActionsService(ctrl),
+		invoices:              domocks.NewMockInvoicesService(ctrl),
+		reservedIPs:           domocks.NewMockReservedIPsService(ctrl),
+		reservedIPActions:     domocks.NewMockReservedIPActionsService(ctrl),
+		droplets:              domocks.NewMockDropletsService(ctrl),
+		dropletActions:        domocks.NewMockDropletActionsService(ctrl),
+		domains:               domocks.NewMockDomainsService(ctrl),
+		tags:                  domocks.NewMockTagsService(ctrl),
+		volumes:               domocks.NewMockVolumesService(ctrl),
+		volumeActions:         domocks.NewMockVolumeActionsService(ctrl),
+		snapshots:             domocks.NewMockSnapshotsService(ctrl),
+		certificates:          domocks.NewMockCertificatesService(ctrl),
+		loadBalancers:         domocks.NewMockLoadBalancersService(ctrl),
+		firewalls:             domocks.NewMockFirewallsService(ctrl),
+		cdns:                  domocks.NewMockCDNsService(ctrl),
+		projects:              domocks.NewMockProjectsService(ctrl),
+		kubernetes:            domocks.NewMockKubernetesService(ctrl),
+		databases:             domocks.NewMockDatabasesService(ctrl),
+		registry:              domocks.NewMockRegistryService(ctrl),
+		sshRunner:             domocks.NewMockRunner(ctrl),
+		vpcs:                  domocks.NewMockVPCsService(ctrl),
+		oneClick:              domocks.NewMockOneClickService(ctrl),
+		listen:                domocks.NewMockListenerService(ctrl),
+		monitoring:            domocks.NewMockMonitoringService(ctrl),
+		sandbox:               domocks.NewMockSandboxService(ctrl),
+		appBuilderFactory:     builder.NewMockComponentBuilderFactory(ctrl),
+		appBuilder:            builder.NewMockComponentBuilder(ctrl),
+		appDockerEngineClient: builder.NewMockDockerEngineClient(ctrl),
 	}
 
 	testConfig := doctl.NewTestConfig()
-	testConfig.ContainerEngineClient = tm.appContainerEngineClient
+	testConfig.DockerEngineClient = tm.appDockerEngineClient
 
 	config := &CmdConfig{
 		NS:   "test",

doit.go

@@ -177,7 +177,7 @@ func (glv *GithubLatestVersioner) LatestVersion() (string, error) {
 // Config is an interface that represent doit's config.
 type Config interface {
 	GetGodoClient(trace bool, accessToken string) (*godo.Client, error)
-	GetContainerEngineClient() (builder.ContainerEngineClient, error)
+	GetDockerEngineClient() (builder.DockerEngineClient, error)
 	SSH(user, host, keyPath string, port int, opts ssh.Options) runner.Runner
 	Listen(url *url.URL, token string, schemaFunc listen.SchemaFunc, out io.Writer) listen.ListenerService
 	Set(ns, key string, val interface{})
@@ -235,8 +235,8 @@ func (c *LiveConfig) GetGodoClient(trace bool, accessToken string) (*godo.Client
 	return godo.New(oauthClient, args...)
 }
 
-// GetContainerEngineClient returns a container engine client.
-func (c *LiveConfig) GetContainerEngineClient() (builder.ContainerEngineClient, error) {
+// GetDockerEngineClient returns a container engine client.
+func (c *LiveConfig) GetDockerEngineClient() (builder.DockerEngineClient, error) {
 	cli, err := client.NewClientWithOpts(client.FromEnv)
 	if err != nil {
 		return nil, err
@@ -405,11 +405,11 @@ func isRequired(key string) bool {
 
 // TestConfig is an implementation of Config for testing.
 type TestConfig struct {
-	SSHFn                 func(user, host, keyPath string, port int, opts ssh.Options) runner.Runner
-	ListenFn              func(url *url.URL, token string, schemaFunc listen.SchemaFunc, out io.Writer) listen.ListenerService
-	v                     *viper.Viper
-	IsSetMap              map[string]bool
-	ContainerEngineClient builder.ContainerEngineClient
+	SSHFn              func(user, host, keyPath string, port int, opts ssh.Options) runner.Runner
+	ListenFn           func(url *url.URL, token string, schemaFunc listen.SchemaFunc, out io.Writer) listen.ListenerService
+	v                  *viper.Viper
+	IsSetMap           map[string]bool
+	DockerEngineClient builder.DockerEngineClient
 }
 
 var _ Config = &TestConfig{}
@@ -434,10 +434,10 @@ func (c *TestConfig) GetGodoClient(trace bool, accessToken string) (*godo.Client
 	return &godo.Client{}, nil
 }
 
-// GetContainerEngineClient mocks a GetContainerEngineClient call. The returned client will
+// GetDockerEngineClient mocks a GetDockerEngineClient call. The returned client will
 // be nil unless configured in the test config.
-func (c *TestConfig) GetContainerEngineClient() (builder.ContainerEngineClient, error) {
-	return c.ContainerEngineClient, nil
+func (c *TestConfig) GetDockerEngineClient() (builder.DockerEngineClient, error) {
+	return c.DockerEngineClient, nil
 }
 
 // SSH returns a mock SSH runner.

internal/apps/builder/builder.go

@@ -17,7 +17,7 @@ import (
 
 // ComponentBuilderFactory is the interface for creating a component builder.
 type ComponentBuilderFactory interface {
-	NewComponentBuilder(ContainerEngineClient, *godo.AppSpec, NewBuilderOpts) (ComponentBuilder, error)
+	NewComponentBuilder(DockerEngineClient, string, *godo.AppSpec, NewBuilderOpts) (ComponentBuilder, error)
 }
 
 // ComponentBuilder is the interface of building one or more components.
@@ -33,12 +33,14 @@ type ComponentBuilderResult struct {
 }
 
 type baseComponentBuilder struct {
-	cli                  ContainerEngineClient
+	cli                  DockerEngineClient
+	contextDir           string
 	spec                 *godo.AppSpec
 	component            godo.AppBuildableComponentSpec
 	registry             string
 	envOverrides         map[string]string
 	buildCommandOverride string
+	copyOnWriteSemantics bool
 
 	logWriter io.Writer
 }
@@ -118,7 +120,7 @@ type DefaultComponentBuilderFactory struct{}
 
 // NewComponentBuilder returns the correct builder type depending upon the provided
 // app and component.
-func (f *DefaultComponentBuilderFactory) NewComponentBuilder(cli ContainerEngineClient, spec *godo.AppSpec, opts NewBuilderOpts) (ComponentBuilder, error) {
+func (f *DefaultComponentBuilderFactory) NewComponentBuilder(cli DockerEngineClient, contextDir string, spec *godo.AppSpec, opts NewBuilderOpts) (ComponentBuilder, error) {
 	// TODO(ntate): handle DetectionBuilder and allow empty component
 	if opts.Component == "" {
 		return nil, errors.New("component is required")
@@ -132,15 +134,22 @@ func (f *DefaultComponentBuilderFactory) NewComponentBuilder(cli ContainerEngine
 		return nil, fmt.Errorf("component %s does not exist", opts.Component)
 	}
 
+	// NOTE(ntate); We don't provide this as a configureable argument today.
+	// We always assume we want copy-on-write. Caching occurs through re-use of the built OCI image.
+	// This may change in the future so we provide as an argument to the baseComponentBuilder.
+	copyOnWriteSemantics := true
+
 	if component.GetDockerfilePath() == "" {
 		return &CNBComponentBuilder{
 			baseComponentBuilder: baseComponentBuilder{
 				cli,
+				contextDir,
 				spec,
 				component,
 				opts.Registry,
 				opts.EnvOverride,
 				opts.BuildCommandOverride,
+				copyOnWriteSemantics,
 				opts.LogWriter,
 			},
 		}, nil
@@ -149,11 +158,13 @@ func (f *DefaultComponentBuilderFactory) NewComponentBuilder(cli ContainerEngine
 	return &DockerComponentBuilder{
 		baseComponentBuilder: baseComponentBuilder{
 			cli,
+			contextDir,
 			spec,
 			component,
 			opts.Registry,
 			opts.EnvOverride,
 			opts.BuildCommandOverride,
+			copyOnWriteSemantics,
 			opts.LogWriter,
 		},
 	}, nil