[v0.22] cherry-picks for v0.22.0-rc2

api/services/control/control.proto

@@ -36,6 +36,7 @@ message PruneRequest {
 
 message DiskUsageRequest {
 	repeated string filter = 1; 
+	int64 ageLimit = 2;
 }
 
 message DiskUsageResponse {

cache/manager.go

@@ -1452,6 +1452,7 @@ func (cm *cacheManager) DiskUsage(ctx context.Context, opt client.DiskUsageInfo)
 	if err := cm.markShared(m); err != nil {
 		return nil, err
 	}
+	cutOff := time.Now().Add(-opt.AgeLimit)
 
 	var du []*client.UsageInfo
 	for id, cr := range m {
@@ -1468,9 +1469,15 @@ func (cm *cacheManager) DiskUsage(ctx context.Context, opt client.DiskUsageInfo)
 			RecordType:  cr.recordType,
 			Shared:      cr.shared,
 		}
-		if filter.Match(adaptUsageInfo(c)) {
-			du = append(du, c)
+		if !filter.Match(adaptUsageInfo(c)) {
+			continue
+		}
+		if opt.AgeLimit > 0 {
+			if c.LastUsedAt != nil && c.LastUsedAt.After(cutOff) {
+				continue
+			}
 		}
+		du = append(du, c)
 	}
 
 	eg, ctx := errgroup.WithContext(ctx)

client/diskusage.go

@@ -31,7 +31,7 @@ func (c *Client) DiskUsage(ctx context.Context, opts ...DiskUsageOption) ([]*Usa
 		o.SetDiskUsageOption(info)
 	}
 
-	req := &controlapi.DiskUsageRequest{Filter: info.Filter}
+	req := &controlapi.DiskUsageRequest{Filter: info.Filter, AgeLimit: int64(info.AgeLimit)}
 	resp, err := c.ControlClient().DiskUsage(ctx, req)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to call diskusage")
@@ -72,7 +72,8 @@ type DiskUsageOption interface {
 }
 
 type DiskUsageInfo struct {
-	Filter []string
+	Filter   []string
+	AgeLimit time.Duration
 }
 
 type UsageRecordType string
@@ -85,3 +86,15 @@ const (
 	UsageRecordTypeCacheMount  UsageRecordType = "exec.cachemount"
 	UsageRecordTypeRegular     UsageRecordType = "regular"
 )
+
+type diskUsageOptionFunc func(*DiskUsageInfo)
+
+func (f diskUsageOptionFunc) SetDiskUsageOption(info *DiskUsageInfo) {
+	f(info)
+}
+
+func WithAgeLimit(age time.Duration) DiskUsageOption {
+	return diskUsageOptionFunc(func(info *DiskUsageInfo) {
+		info.AgeLimit = age
+	})
+}

client/llb/source.go

@@ -322,6 +322,12 @@ func Git(url, ref string, opts ...GitOption) State {
 		addCap(&gi.Constraints, pb.CapSourceGitMountSSHSock)
 	}
 
+	checksum := gi.Checksum
+	if checksum != "" {
+		attrs[pb.AttrGitChecksum] = checksum
+		addCap(&gi.Constraints, pb.CapSourceGitChecksum)
+	}
+
 	addCap(&gi.Constraints, pb.CapSourceGit)
 
 	source := NewSource("git://"+id, attrs, gi.Constraints)
@@ -345,6 +351,7 @@ type GitInfo struct {
 	addAuthCap       bool
 	KnownSSHHosts    string
 	MountSSHSock     string
+	Checksum         string
 }
 
 func KeepGitDir() GitOption {
@@ -373,6 +380,12 @@ func MountSSHSock(sshID string) GitOption {
 	})
 }
 
+func GitChecksum(v string) GitOption {
+	return gitOptionFunc(func(gi *GitInfo) {
+		gi.Checksum = v
+	})
+}
+
 // AuthOption can be used with either HTTP or Git sources.
 type AuthOption interface {
 	GitOption

control/control.go

@@ -167,7 +167,8 @@ func (c *Controller) DiskUsage(ctx context.Context, r *controlapi.DiskUsageReque
 	}
 	for _, w := range workers {
 		du, err := w.DiskUsage(ctx, client.DiskUsageInfo{
-			Filter: r.Filter,
+			Filter:   r.Filter,
+			AgeLimit: time.Duration(r.AgeLimit),
 		})
 		if err != nil {
 			return nil, err

frontend/dockerfile/dockerfile2llb/convert.go

@@ -937,27 +937,21 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
 	case *instructions.WorkdirCommand:
 		err = dispatchWorkdir(d, c, true, &opt)
 	case *instructions.AddCommand:
-		var checksum digest.Digest
-		if c.Checksum != "" {
-			checksum, err = digest.Parse(c.Checksum)
-		}
-		if err == nil {
-			err = dispatchCopy(d, copyConfig{
-				params:          c.SourcesAndDest,
-				excludePatterns: c.ExcludePatterns,
-				source:          opt.buildContext,
-				isAddCommand:    true,
-				cmdToPrint:      c,
-				chown:           c.Chown,
-				chmod:           c.Chmod,
-				link:            c.Link,
-				keepGitDir:      c.KeepGitDir,
-				checksum:        checksum,
-				location:        c.Location(),
-				ignoreMatcher:   opt.dockerIgnoreMatcher,
-				opt:             opt,
-			})
-		}
+		err = dispatchCopy(d, copyConfig{
+			params:          c.SourcesAndDest,
+			excludePatterns: c.ExcludePatterns,
+			source:          opt.buildContext,
+			isAddCommand:    true,
+			cmdToPrint:      c,
+			chown:           c.Chown,
+			chmod:           c.Chmod,
+			link:            c.Link,
+			keepGitDir:      c.KeepGitDir,
+			checksum:        c.Checksum,
+			location:        c.Location(),
+			ignoreMatcher:   opt.dockerIgnoreMatcher,
+			opt:             opt,
+		})
 		if err == nil {
 			for _, src := range c.SourcePaths {
 				if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
@@ -1470,8 +1464,8 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 		if len(cfg.params.SourcePaths) != 1 {
 			return errors.New("checksum can't be specified for multiple sources")
 		}
-		if !isHTTPSource(cfg.params.SourcePaths[0]) {
-			return errors.New("checksum can't be specified for non-HTTP(S) sources")
+		if !isHTTPSource(cfg.params.SourcePaths[0]) && !isGitSource(cfg.params.SourcePaths[0]) {
+			return errors.New("checksum requires HTTP(S) or Git sources")
 		}
 	}
 
@@ -1519,6 +1513,9 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 			if cfg.keepGitDir {
 				gitOptions = append(gitOptions, llb.KeepGitDir())
 			}
+			if cfg.checksum != "" {
+				gitOptions = append(gitOptions, llb.GitChecksum(cfg.checksum))
+			}
 			st := llb.Git(gitRef.Remote, commit, gitOptions...)
 			opts := append([]llb.CopyOption{&llb.CopyInfo{
 				Mode:           chopt,
@@ -1547,7 +1544,15 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 				}
 			}
 
-			st := llb.HTTP(src, llb.Filename(f), llb.WithCustomName(pgName), llb.Checksum(cfg.checksum), dfCmd(cfg.params))
+			var checksum digest.Digest
+			if cfg.checksum != "" {
+				checksum, err = digest.Parse(cfg.checksum)
+				if err != nil {
+					return err
+				}
+			}
+
+			st := llb.HTTP(src, llb.Filename(f), llb.WithCustomName(pgName), llb.Checksum(checksum), dfCmd(cfg.params))
 
 			opts := append([]llb.CopyOption{&llb.CopyInfo{
 				Mode:           chopt,
@@ -1674,7 +1679,7 @@ type copyConfig struct {
 	chmod           string
 	link            bool
 	keepGitDir      bool
-	checksum        digest.Digest
+	checksum        string
 	parents         bool
 	location        []parser.Range
 	ignoreMatcher   *patternmatcher.PatternMatcher
@@ -2265,11 +2270,15 @@ func isHTTPSource(src string) bool {
 	if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
 		return false
 	}
+	return !isGitSource(src)
+}
+
+func isGitSource(src string) bool {
 	// https://github.com/ORG/REPO.git is a git source, not an http source
 	if gitRef, gitErr := gitutil.ParseGitRef(src); gitRef != nil && gitErr == nil {
-		return false
+		return true
 	}
-	return true
+	return false
 }
 
 func isEnabledForStage(stage string, value string) bool {

frontend/dockerfile/dockerfile_addchecksum_test.go

@@ -162,6 +162,6 @@ ADD --checksum=%s foo /tmp/foo
 				dockerui.DefaultLocalNameContext:    dir,
 			},
 		}, nil)
-		require.Error(t, err, "checksum can't be specified for non-HTTP(S) sources")
+		require.Error(t, err, "checksum requires HTTP(S) or Git sources")
 	})
 }