Skip to content

Commit fe00540

Browse files
authoredDec 5, 2023
Merge pull request #1719 from giuseppe/partial-images-store-expected-diffid
layers: add new TOCDigest attribute
2 parents 15c3cb7 + 15ac716 commit fe00540

File tree

5 files changed

+106
-40
lines changed

5 files changed

+106
-40
lines changed
 

‎cmd/containers-storage/diff.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ func applyDiffUsingStagingDirectory(flags *mflag.FlagSet, action string, m stora
176176
return 1, err
177177
}
178178

179-
var options graphdriver.ApplyDiffOpts
179+
var options graphdriver.ApplyDiffWithDifferOpts
180180
out, err := m.ApplyDiffWithDiffer("", &options, differ)
181181
if err != nil {
182182
return 1, err

‎drivers/driver.go

+9-2
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,13 @@ type ApplyDiffOpts struct {
7373
ForceMask *os.FileMode
7474
}
7575

76+
// ApplyDiffWithDifferOpts contains optional arguments for ApplyDiffWithDiffer methods.
77+
type ApplyDiffWithDifferOpts struct {
78+
ApplyDiffOpts
79+
80+
Flags map[string]interface{}
81+
}
82+
7683
// InitFunc initializes the storage driver.
7784
type InitFunc func(homedir string, options Options) (Driver, error)
7885

@@ -223,9 +230,9 @@ type DriverWithDiffer interface {
223230
Driver
224231
// ApplyDiffWithDiffer applies the changes using the callback function.
225232
// If id is empty, then a staging directory is created. The staging directory is guaranteed to be usable with ApplyDiffFromStagingDirectory.
226-
ApplyDiffWithDiffer(id, parent string, options *ApplyDiffOpts, differ Differ) (output DriverWithDifferOutput, err error)
233+
ApplyDiffWithDiffer(id, parent string, options *ApplyDiffWithDifferOpts, differ Differ) (output DriverWithDifferOutput, err error)
227234
// ApplyDiffFromStagingDirectory applies the changes using the specified staging directory.
228-
ApplyDiffFromStagingDirectory(id, parent, stagingDirectory string, diffOutput *DriverWithDifferOutput, options *ApplyDiffOpts) error
235+
ApplyDiffFromStagingDirectory(id, parent, stagingDirectory string, diffOutput *DriverWithDifferOutput, options *ApplyDiffWithDifferOpts) error
229236
// CleanupStagingDirectory cleanups the staging directory. It can be used to cleanup the staging directory on errors
230237
CleanupStagingDirectory(stagingDirectory string) error
231238
// DifferTarget gets the location where files are stored for the layer.

‎drivers/overlay/overlay.go

+2-4
Original file line numberDiff line numberDiff line change
@@ -2049,7 +2049,7 @@ func (d *Driver) useComposeFs() bool {
20492049
}
20502050

20512051
// ApplyDiff applies the changes in the new layer using the specified function
2052-
func (d *Driver) ApplyDiffWithDiffer(id, parent string, options *graphdriver.ApplyDiffOpts, differ graphdriver.Differ) (output graphdriver.DriverWithDifferOutput, err error) {
2052+
func (d *Driver) ApplyDiffWithDiffer(id, parent string, options *graphdriver.ApplyDiffWithDifferOpts, differ graphdriver.Differ) (output graphdriver.DriverWithDifferOutput, err error) {
20532053
var idMappings *idtools.IDMappings
20542054
if options != nil {
20552055
idMappings = options.Mappings
@@ -2100,7 +2100,7 @@ func (d *Driver) ApplyDiffWithDiffer(id, parent string, options *graphdriver.App
21002100
}
21012101

21022102
// ApplyDiffFromStagingDirectory applies the changes using the specified staging directory.
2103-
func (d *Driver) ApplyDiffFromStagingDirectory(id, parent, stagingDirectory string, diffOutput *graphdriver.DriverWithDifferOutput, options *graphdriver.ApplyDiffOpts) error {
2103+
func (d *Driver) ApplyDiffFromStagingDirectory(id, parent, stagingDirectory string, diffOutput *graphdriver.DriverWithDifferOutput, options *graphdriver.ApplyDiffWithDifferOpts) error {
21042104
if filepath.Dir(stagingDirectory) != d.getStagingDir() {
21052105
return fmt.Errorf("%q is not a staging directory", stagingDirectory)
21062106
}
@@ -2125,8 +2125,6 @@ func (d *Driver) ApplyDiffFromStagingDirectory(id, parent, stagingDirectory stri
21252125
return err
21262126
}
21272127

2128-
diffOutput.UncompressedDigest = diffOutput.TOCDigest
2129-
21302128
return os.Rename(stagingDirectory, diffPath)
21312129
}
21322130

‎layers.go

+79-29
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,13 @@ type Layer struct {
126126
// as a DiffID.
127127
UncompressedDigest digest.Digest `json:"diff-digest,omitempty"`
128128

129+
// TOCDigest represents the digest of the Table of Contents (TOC) of the blob.
130+
// This digest is utilized when the UncompressedDigest is not
131+
// validated during the partial image pull process, but the
132+
// TOC itself is validated.
133+
// It serves as an alternative reference under these specific conditions.
134+
TOCDigest digest.Digest `json:"toc-digest,omitempty"`
135+
129136
// UncompressedSize is the length of the blob that was last passed to
130137
// ApplyDiff() or create(), after we decompressed it. If
131138
// UncompressedDigest is not set, this should be treated as if it were
@@ -228,6 +235,10 @@ type roLayerStore interface {
228235
// specified uncompressed digest value recorded for them.
229236
LayersByUncompressedDigest(d digest.Digest) ([]Layer, error)
230237

238+
// LayersByTOCDigest returns a slice of the layers with the
239+
// specified uncompressed digest value recorded for them.
240+
LayersByTOCDigest(d digest.Digest) ([]Layer, error)
241+
231242
// Layers returns a slice of the known layers.
232243
Layers() ([]Layer, error)
233244
}
@@ -296,13 +307,13 @@ type rwLayerStore interface {
296307

297308
// ApplyDiffWithDiffer applies the changes through the differ callback function.
298309
// If to is the empty string, then a staging directory is created by the driver.
299-
ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error)
310+
ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffWithDifferOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error)
300311

301312
// CleanupStagingDirectory cleanups the staging directory. It can be used to cleanup the staging directory on errors
302313
CleanupStagingDirectory(stagingDirectory string) error
303314

304315
// ApplyDiffFromStagingDirectory uses stagingDirectory to create the diff.
305-
ApplyDiffFromStagingDirectory(id, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error
316+
ApplyDiffFromStagingDirectory(id, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffWithDifferOpts) error
306317

307318
// DifferTarget gets the location where files are stored for the layer.
308319
DifferTarget(id string) (string, error)
@@ -337,6 +348,7 @@ type layerStore struct {
337348
bymount map[string]*Layer
338349
bycompressedsum map[digest.Digest][]string
339350
byuncompressedsum map[digest.Digest][]string
351+
bytocsum map[digest.Digest][]string
340352
layerspathsModified [numLayerLocationIndex]time.Time
341353

342354
// FIXME: This field is only set when constructing layerStore, but locking rules of the driver
@@ -366,6 +378,7 @@ func copyLayer(l *Layer) *Layer {
366378
CompressedSize: l.CompressedSize,
367379
UncompressedDigest: l.UncompressedDigest,
368380
UncompressedSize: l.UncompressedSize,
381+
TOCDigest: l.TOCDigest,
369382
CompressionType: l.CompressionType,
370383
ReadOnly: l.ReadOnly,
371384
volatileStore: l.volatileStore,
@@ -745,6 +758,7 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) {
745758
names := make(map[string]*Layer)
746759
compressedsums := make(map[digest.Digest][]string)
747760
uncompressedsums := make(map[digest.Digest][]string)
761+
tocsums := make(map[digest.Digest][]string)
748762
var errorToResolveBySaving error // == nil; if there are multiple errors, this is one of them.
749763
if r.lockfile.IsReadWrite() {
750764
selinux.ClearLabels()
@@ -765,6 +779,9 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) {
765779
if layer.UncompressedDigest != "" {
766780
uncompressedsums[layer.UncompressedDigest] = append(uncompressedsums[layer.UncompressedDigest], layer.ID)
767781
}
782+
if layer.TOCDigest != "" {
783+
tocsums[layer.TOCDigest] = append(tocsums[layer.TOCDigest], layer.ID)
784+
}
768785
if layer.MountLabel != "" {
769786
selinux.ReserveLabel(layer.MountLabel)
770787
}
@@ -792,6 +809,7 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) {
792809
r.byname = names
793810
r.bycompressedsum = compressedsums
794811
r.byuncompressedsum = uncompressedsums
812+
r.bytocsum = tocsums
795813

796814
// Load and merge information about which layers are mounted, and where.
797815
if r.lockfile.IsReadWrite() {
@@ -1112,7 +1130,7 @@ func (r *layerStore) Size(name string) (int64, error) {
11121130
// We use the presence of a non-empty digest as an indicator that the size value was intentionally set, and that
11131131
// a zero value is not just present because it was never set to anything else (which can happen if the layer was
11141132
// created by a version of this library that didn't keep track of digest and size information).
1115-
if layer.UncompressedDigest != "" {
1133+
if layer.TOCDigest != "" || layer.UncompressedDigest != "" {
11161134
return layer.UncompressedSize, nil
11171135
}
11181136
return -1, nil
@@ -1201,6 +1219,9 @@ func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []s
12011219
if layer.UncompressedDigest != "" {
12021220
r.byuncompressedsum[layer.UncompressedDigest] = append(r.byuncompressedsum[layer.UncompressedDigest], layer.ID)
12031221
}
1222+
if layer.TOCDigest != "" {
1223+
r.bytocsum[layer.TOCDigest] = append(r.bytocsum[layer.TOCDigest], layer.ID)
1224+
}
12041225
if err := r.saveFor(layer); err != nil {
12051226
if e := r.Delete(layer.ID); e != nil {
12061227
logrus.Errorf("While recovering from a failure to save layers, error deleting layer %#v: %v", id, e)
@@ -1251,6 +1272,7 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount
12511272
templateCompressedDigest digest.Digest
12521273
templateCompressedSize int64
12531274
templateUncompressedDigest digest.Digest
1275+
templateTOCDigest digest.Digest
12541276
templateUncompressedSize int64
12551277
templateCompressionType archive.Compression
12561278
templateUIDs, templateGIDs []uint32
@@ -1263,6 +1285,7 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount
12631285
}
12641286
templateMetadata = templateLayer.Metadata
12651287
templateIDMappings = idtools.NewIDMappingsFromMaps(templateLayer.UIDMap, templateLayer.GIDMap)
1288+
templateTOCDigest = templateLayer.TOCDigest
12661289
templateCompressedDigest, templateCompressedSize = templateLayer.CompressedDigest, templateLayer.CompressedSize
12671290
templateUncompressedDigest, templateUncompressedSize = templateLayer.UncompressedDigest, templateLayer.UncompressedSize
12681291
templateCompressionType = templateLayer.CompressionType
@@ -1291,6 +1314,7 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount
12911314
CompressedDigest: templateCompressedDigest,
12921315
CompressedSize: templateCompressedSize,
12931316
UncompressedDigest: templateUncompressedDigest,
1317+
TOCDigest: templateTOCDigest,
12941318
UncompressedSize: templateUncompressedSize,
12951319
CompressionType: templateCompressionType,
12961320
UIDs: templateUIDs,
@@ -1413,6 +1437,9 @@ func (r *layerStore) create(id string, parentLayer *Layer, names []string, mount
14131437
if layer.UncompressedDigest != "" {
14141438
r.byuncompressedsum[layer.UncompressedDigest] = append(r.byuncompressedsum[layer.UncompressedDigest], layer.ID)
14151439
}
1440+
if layer.TOCDigest != "" {
1441+
r.bytocsum[layer.TOCDigest] = append(r.bytocsum[layer.TOCDigest], layer.ID)
1442+
}
14161443
}
14171444

14181445
delete(layer.Flags, incompleteFlag)
@@ -2197,6 +2224,25 @@ func (r *layerStore) DiffSize(from, to string) (size int64, err error) {
21972224
return r.driver.DiffSize(to, r.layerMappings(toLayer), from, r.layerMappings(fromLayer), toLayer.MountLabel)
21982225
}
21992226

2227+
func updateDigestMap(m *map[digest.Digest][]string, oldvalue, newvalue digest.Digest, id string) {
2228+
var newList []string
2229+
if oldvalue != "" {
2230+
for _, value := range (*m)[oldvalue] {
2231+
if value != id {
2232+
newList = append(newList, value)
2233+
}
2234+
}
2235+
if len(newList) > 0 {
2236+
(*m)[oldvalue] = newList
2237+
} else {
2238+
delete(*m, oldvalue)
2239+
}
2240+
}
2241+
if newvalue != "" {
2242+
(*m)[newvalue] = append((*m)[newvalue], id)
2243+
}
2244+
}
2245+
22002246
// Requires startWriting.
22012247
func (r *layerStore) ApplyDiff(to string, diff io.Reader) (size int64, err error) {
22022248
return r.applyDiffWithOptions(to, nil, diff)
@@ -2313,24 +2359,6 @@ func (r *layerStore) applyDiffWithOptions(to string, layerOptions *LayerOptions,
23132359
uncompressedDigest = uncompressedDigester.Digest()
23142360
}
23152361

2316-
updateDigestMap := func(m *map[digest.Digest][]string, oldvalue, newvalue digest.Digest, id string) {
2317-
var newList []string
2318-
if oldvalue != "" {
2319-
for _, value := range (*m)[oldvalue] {
2320-
if value != id {
2321-
newList = append(newList, value)
2322-
}
2323-
}
2324-
if len(newList) > 0 {
2325-
(*m)[oldvalue] = newList
2326-
} else {
2327-
delete(*m, oldvalue)
2328-
}
2329-
}
2330-
if newvalue != "" {
2331-
(*m)[newvalue] = append((*m)[newvalue], id)
2332-
}
2333-
}
23342362
updateDigestMap(&r.bycompressedsum, layer.CompressedDigest, compressedDigest, layer.ID)
23352363
layer.CompressedDigest = compressedDigest
23362364
layer.CompressedSize = compressedCounter.Count
@@ -2372,7 +2400,7 @@ func (r *layerStore) DifferTarget(id string) (string, error) {
23722400
}
23732401

23742402
// Requires startWriting.
2375-
func (r *layerStore) ApplyDiffFromStagingDirectory(id, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error {
2403+
func (r *layerStore) ApplyDiffFromStagingDirectory(id, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffWithDifferOpts) error {
23762404
ddriver, ok := r.driver.(drivers.DriverWithDiffer)
23772405
if !ok {
23782406
return ErrNotSupported
@@ -2382,20 +2410,35 @@ func (r *layerStore) ApplyDiffFromStagingDirectory(id, stagingDirectory string,
23822410
return ErrLayerUnknown
23832411
}
23842412
if options == nil {
2385-
options = &drivers.ApplyDiffOpts{
2386-
Mappings: r.layerMappings(layer),
2387-
MountLabel: layer.MountLabel,
2413+
options = &drivers.ApplyDiffWithDifferOpts{
2414+
ApplyDiffOpts: drivers.ApplyDiffOpts{
2415+
Mappings: r.layerMappings(layer),
2416+
MountLabel: layer.MountLabel,
2417+
},
2418+
Flags: nil,
23882419
}
23892420
}
2421+
23902422
err := ddriver.ApplyDiffFromStagingDirectory(layer.ID, layer.Parent, stagingDirectory, diffOutput, options)
23912423
if err != nil {
23922424
return err
23932425
}
23942426
layer.UIDs = diffOutput.UIDs
23952427
layer.GIDs = diffOutput.GIDs
2428+
updateDigestMap(&r.byuncompressedsum, layer.UncompressedDigest, diffOutput.UncompressedDigest, layer.ID)
23962429
layer.UncompressedDigest = diffOutput.UncompressedDigest
2430+
updateDigestMap(&r.bytocsum, diffOutput.TOCDigest, diffOutput.TOCDigest, layer.ID)
2431+
layer.TOCDigest = diffOutput.TOCDigest
23972432
layer.UncompressedSize = diffOutput.Size
23982433
layer.Metadata = diffOutput.Metadata
2434+
if options != nil && options.Flags != nil {
2435+
if layer.Flags == nil {
2436+
layer.Flags = make(map[string]interface{})
2437+
}
2438+
for k, v := range options.Flags {
2439+
layer.Flags[k] = v
2440+
}
2441+
}
23992442
if len(diffOutput.TarSplit) != 0 {
24002443
tsdata := bytes.Buffer{}
24012444
compressor, err := pgzip.NewWriterLevel(&tsdata, pgzip.BestSpeed)
@@ -2432,7 +2475,7 @@ func (r *layerStore) ApplyDiffFromStagingDirectory(id, stagingDirectory string,
24322475
}
24332476

24342477
// Requires startWriting.
2435-
func (r *layerStore) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) {
2478+
func (r *layerStore) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffWithDifferOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) {
24362479
ddriver, ok := r.driver.(drivers.DriverWithDiffer)
24372480
if !ok {
24382481
return nil, ErrNotSupported
@@ -2448,9 +2491,11 @@ func (r *layerStore) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOp
24482491
return nil, ErrLayerUnknown
24492492
}
24502493
if options == nil {
2451-
options = &drivers.ApplyDiffOpts{
2452-
Mappings: r.layerMappings(layer),
2453-
MountLabel: layer.MountLabel,
2494+
options = &drivers.ApplyDiffWithDifferOpts{
2495+
ApplyDiffOpts: drivers.ApplyDiffOpts{
2496+
Mappings: r.layerMappings(layer),
2497+
MountLabel: layer.MountLabel,
2498+
},
24542499
}
24552500
}
24562501
output, err := ddriver.ApplyDiffWithDiffer(layer.ID, layer.Parent, options, differ)
@@ -2494,6 +2539,11 @@ func (r *layerStore) LayersByUncompressedDigest(d digest.Digest) ([]Layer, error
24942539
return r.layersByDigestMap(r.byuncompressedsum, d)
24952540
}
24962541

2542+
// Requires startReading or startWriting.
2543+
func (r *layerStore) LayersByTOCDigest(d digest.Digest) ([]Layer, error) {
2544+
return r.layersByDigestMap(r.bytocsum, d)
2545+
}
2546+
24972547
func closeAll(closes ...func() error) (rErr error) {
24982548
for _, f := range closes {
24992549
if err := f(); err != nil {

‎store.go

+15-4
Original file line numberDiff line numberDiff line change
@@ -315,10 +315,10 @@ type Store interface {
315315
// ApplyDiffer applies a diff to a layer.
316316
// It is the caller responsibility to clean the staging directory if it is not
317317
// successfully applied with ApplyDiffFromStagingDirectory.
318-
ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error)
318+
ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffWithDifferOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error)
319319

320320
// ApplyDiffFromStagingDirectory uses stagingDirectory to create the diff.
321-
ApplyDiffFromStagingDirectory(to, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error
321+
ApplyDiffFromStagingDirectory(to, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffWithDifferOpts) error
322322

323323
// CleanupStagingDirectory cleanups the staging directory. It can be used to cleanup the staging directory on errors
324324
CleanupStagingDirectory(stagingDirectory string) error
@@ -334,6 +334,10 @@ type Store interface {
334334
// specified uncompressed digest value recorded for them.
335335
LayersByUncompressedDigest(d digest.Digest) ([]Layer, error)
336336

337+
// LayersByTOCDigest returns a slice of the layers with the
338+
// specified TOC digest value recorded for them.
339+
LayersByTOCDigest(d digest.Digest) ([]Layer, error)
340+
337341
// LayerSize returns a cached approximation of the layer's size, or -1
338342
// if we don't have a value on hand.
339343
LayerSize(id string) (int64, error)
@@ -2927,7 +2931,7 @@ func (s *store) Diff(from, to string, options *DiffOptions) (io.ReadCloser, erro
29272931
return nil, ErrLayerUnknown
29282932
}
29292933

2930-
func (s *store) ApplyDiffFromStagingDirectory(to, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffOpts) error {
2934+
func (s *store) ApplyDiffFromStagingDirectory(to, stagingDirectory string, diffOutput *drivers.DriverWithDifferOutput, options *drivers.ApplyDiffWithDifferOpts) error {
29312935
_, err := writeToLayerStore(s, func(rlstore rwLayerStore) (struct{}, error) {
29322936
if !rlstore.Exists(to) {
29332937
return struct{}{}, ErrLayerUnknown
@@ -2944,7 +2948,7 @@ func (s *store) CleanupStagingDirectory(stagingDirectory string) error {
29442948
return err
29452949
}
29462950

2947-
func (s *store) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) {
2951+
func (s *store) ApplyDiffWithDiffer(to string, options *drivers.ApplyDiffWithDifferOpts, differ drivers.Differ) (*drivers.DriverWithDifferOutput, error) {
29482952
return writeToLayerStore(s, func(rlstore rwLayerStore) (*drivers.DriverWithDifferOutput, error) {
29492953
if to != "" && !rlstore.Exists(to) {
29502954
return nil, ErrLayerUnknown
@@ -3006,6 +3010,13 @@ func (s *store) LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) {
30063010
return s.layersByMappedDigest(func(r roLayerStore, d digest.Digest) ([]Layer, error) { return r.LayersByUncompressedDigest(d) }, d)
30073011
}
30083012

3013+
func (s *store) LayersByTOCDigest(d digest.Digest) ([]Layer, error) {
3014+
if err := d.Validate(); err != nil {
3015+
return nil, fmt.Errorf("looking for TOC matching digest %q: %w", d, err)
3016+
}
3017+
return s.layersByMappedDigest(func(r roLayerStore, d digest.Digest) ([]Layer, error) { return r.LayersByTOCDigest(d) }, d)
3018+
}
3019+
30093020
func (s *store) LayerSize(id string) (int64, error) {
30103021
if res, done, err := readAllLayerStores(s, func(store roLayerStore) (int64, bool, error) {
30113022
if store.Exists(id) {

0 commit comments

Comments
 (0)
Please sign in to comment.