Skip to content

Commit

Permalink
storage.storageImageDestination.Commit(): leverage image options
Browse files Browse the repository at this point in the history 
When committing an image, pass big data items and the image metadata
string in as part of the set of options we pass to the storage library
when creating the image.

Fall back to setting them individually, as we would have originally,
only if we end up updating an image that was already in local storage.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>
  • Loading branch information
@nalind
nalind committed Aug 1, 2023
1 parent 68798a2 commit 3c47e19
Showing 1 changed file with 99 additions and 72 deletions.
171 changes: 99 additions & 72 deletions storage/storage_dest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -760,7 +760,7 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
if len(layerBlobs) > 0 { // Can happen when using caches
prev := s.indexToStorageID[len(layerBlobs)-1]
if prev == nil {
return fmt.Errorf("Internal error: StorageImageDestination.Commit(): previous layer %d hasn't been committed (lastLayer == nil)", len(layerBlobs)-1)
return fmt.Errorf("Internal error: storageImageDestination.Commit(): previous layer %d hasn't been committed (lastLayer == nil)", len(layerBlobs)-1)
}
lastLayer = *prev
}
Expand All @@ -772,45 +772,8 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
logrus.Debugf("setting image creation date to %s", inspect.Created)
options.CreationDate = *inspect.Created
}
// Create the image record, pointing to the most-recently added layer.
intendedID := s.imageRef.id
if intendedID == "" {
intendedID = s.computeID(man)
}
oldNames := []string{}
img, err := s.imageRef.transport.store.CreateImage(intendedID, nil, lastLayer, "", options)
if err != nil {
if !errors.Is(err, storage.ErrDuplicateID) {
logrus.Debugf("error creating image: %q", err)
return fmt.Errorf("creating image %q: %w", intendedID, err)
}
img, err = s.imageRef.transport.store.Image(intendedID)
if err != nil {
return fmt.Errorf("reading image %q: %w", intendedID, err)
}
if img.TopLayer != lastLayer {
logrus.Debugf("error creating image: image with ID %q exists, but uses different layers", intendedID)
return fmt.Errorf("image with ID %q already exists, but uses a different top layer: %w", intendedID, storage.ErrDuplicateID)
}
logrus.Debugf("reusing image ID %q", img.ID)
oldNames = append(oldNames, img.Names...)
} else {
logrus.Debugf("created new image ID %q", img.ID)
}

// Clean up the unfinished image on any error.
// (Is this the right thing to do if the image has existed before?)
commitSucceeded := false
defer func() {
if !commitSucceeded {
logrus.Errorf("Updating image %q (old names %v) failed, deleting it", img.ID, oldNames)
if _, err := s.imageRef.transport.store.DeleteImage(img.ID, true); err != nil {
logrus.Errorf("Error deleting incomplete image %q: %v", img.ID, err)
}
}
}()

// Add the non-layer blobs as data items. Since we only share layers, they should all be in files, so
// Set up to save the non-layer blobs as data items. Since we only share layers, they should all be in files, so
// we just need to screen out the ones that are actually layers to get the list of non-layers.
dataBlobs := set.New[digest.Digest]()
for blob := range s.filenames {
Expand All @@ -824,64 +787,128 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
if err != nil {
return fmt.Errorf("copying non-layer blob %q to image: %w", blob, err)
}
if err := s.imageRef.transport.store.SetImageBigData(img.ID, blob.String(), v, manifest.Digest); err != nil {
logrus.Debugf("error saving big data %q for image %q: %v", blob.String(), img.ID, err)
return fmt.Errorf("saving big data %q for image %q: %w", blob.String(), img.ID, err)
dataDigest, err := manifest.Digest(v)
if err != nil {
dataDigest = digest.Canonical.FromBytes(v)
}
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: blob.String(),
Data: v,
Digest: dataDigest,
})
}
// Save the unparsedToplevel's manifest if it differs from the per-platform one, which is saved below.
// Set up to save the unparsedToplevel's manifest if it differs from
// the per-platform one, which is saved below.
if len(toplevelManifest) != 0 && !bytes.Equal(toplevelManifest, s.manifest) {
manifestDigest, err := manifest.Digest(toplevelManifest)
if err != nil {
return fmt.Errorf("digesting top-level manifest: %w", err)
}
key := manifestBigDataKey(manifestDigest)
if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, toplevelManifest, manifest.Digest); err != nil {
logrus.Debugf("error saving top-level manifest for image %q: %v", img.ID, err)
return fmt.Errorf("saving top-level manifest for image %q: %w", img.ID, err)
}
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: key,
Data: toplevelManifest,
Digest: manifestDigest,
})
}
// Save the image's manifest. Allow looking it up by digest by using the key convention defined by the Store.
// Set up to save the image's manifest. Allow looking it up by digest by using the key convention defined by the Store.
// Record the manifest twice: using a digest-specific key to allow references to that specific digest instance,
// and using storage.ImageDigestBigDataKey for future users that don’t specify any digest and for compatibility with older readers.
key := manifestBigDataKey(s.manifestDigest)
if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil {
logrus.Debugf("error saving manifest for image %q: %v", img.ID, err)
return fmt.Errorf("saving manifest for image %q: %w", img.ID, err)
manifestDigest, err := manifest.Digest(s.manifest)
if err != nil {
return fmt.Errorf("digesting image manifest: %w", err)
}
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: key,
Data: s.manifest,
Digest: manifestDigest,
})
key = storage.ImageDigestBigDataKey
if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil {
logrus.Debugf("error saving manifest for image %q: %v", img.ID, err)
return fmt.Errorf("saving manifest for image %q: %w", img.ID, err)
}
// Save the signatures, if we have any.
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: key,
Data: s.manifest,
Digest: manifestDigest,
})
// Set up to save the signatures, if we have any.
if len(s.signatures) > 0 {
if err := s.imageRef.transport.store.SetImageBigData(img.ID, "signatures", s.signatures, manifest.Digest); err != nil {
logrus.Debugf("error saving signatures for image %q: %v", img.ID, err)
return fmt.Errorf("saving signatures for image %q: %w", img.ID, err)
}
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: "signatures",
Data: s.signatures,
Digest: digest.Canonical.FromBytes(s.signatures),
})
}
for instanceDigest, signatures := range s.signatureses {
key := signatureBigDataKey(instanceDigest)
if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, signatures, manifest.Digest); err != nil {
logrus.Debugf("error saving signatures for image %q: %v", img.ID, err)
return fmt.Errorf("saving signatures for image %q: %w", img.ID, err)
}
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: key,
Data: signatures,
Digest: digest.Canonical.FromBytes(signatures),
})
}
// Save our metadata.

// Set up to save our metadata.
metadata, err := json.Marshal(s)
if err != nil {
logrus.Debugf("error encoding metadata for image %q: %v", img.ID, err)
return fmt.Errorf("encoding metadata for image %q: %w", img.ID, err)
return fmt.Errorf("encoding metadata for image: %w", err)
}
if len(metadata) != 0 {
if err = s.imageRef.transport.store.SetMetadata(img.ID, string(metadata)); err != nil {
logrus.Debugf("error saving metadata for image %q: %v", img.ID, err)
return fmt.Errorf("saving metadata for image %q: %w", img.ID, err)
options.Metadata = string(metadata)
}

// Create the image record, pointing to the most-recently added layer.
intendedID := s.imageRef.id
if intendedID == "" {
intendedID = s.computeID(man)
}
oldNames := []string{}
img, err := s.imageRef.transport.store.CreateImage(intendedID, nil, lastLayer, "", options)
if err != nil {
if !errors.Is(err, storage.ErrDuplicateID) {
logrus.Debugf("error creating image: %q", err)
return fmt.Errorf("creating image %q: %w", intendedID, err)
}
logrus.Debugf("saved image metadata %q", string(metadata))
img, err = s.imageRef.transport.store.Image(intendedID)
if err != nil {
return fmt.Errorf("reading image %q: %w", intendedID, err)
}
if img.TopLayer != lastLayer {
logrus.Debugf("error creating image: image with ID %q exists, but uses different layers", intendedID)
return fmt.Errorf("image with ID %q already exists, but uses a different top layer: %w", intendedID, storage.ErrDuplicateID)
}
logrus.Debugf("reusing image ID %q", img.ID)
oldNames = append(oldNames, img.Names...)
// set the data items and metadata on the already-present image
for _, data := range options.BigData {
if err := s.imageRef.transport.store.SetImageBigData(img.ID, data.Key, data.Data, manifest.Digest); err != nil {
logrus.Debugf("error saving big data %q for image %q: %v", data.Key, img.ID, err)
return fmt.Errorf("saving big data %q for image %q: %w", data.Key, img.ID, err)
}
}
if options.Metadata != "" {
if err := s.imageRef.transport.store.SetMetadata(img.ID, options.Metadata); err != nil {
logrus.Debugf("error saving metadata for image %q: %v", img.ID, err)
return fmt.Errorf("saving metadata for image %q: %w", img.ID, err)
}
logrus.Debugf("saved image metadata %q", options.Metadata)
}
} else {
logrus.Debugf("created new image ID %q with metadata %q", img.ID, options.Metadata)
}
// Adds the reference's name on the image. We don't need to worry about avoiding duplicate

// Clean up the unfinished image on any error.
// (Is this the right thing to do if the image has existed before?)
commitSucceeded := false
defer func() {
if !commitSucceeded {
logrus.Errorf("Updating image %q (old names %v) failed, deleting it", img.ID, oldNames)
if _, err := s.imageRef.transport.store.DeleteImage(img.ID, true); err != nil {
logrus.Errorf("Error deleting incomplete image %q: %v", img.ID, err)
}
}
}()

// Add the reference's name on the image. We don't need to worry about avoiding duplicate
// values because AddNames() will deduplicate the list that we pass to it.
if name := s.imageRef.DockerReference(); name != nil {
if err := s.imageRef.transport.store.AddNames(img.ID, []string{name.String()}); err != nil {
Expand Down

0 comments on commit 3c47e19

Please sign in to comment.