Merge pull request #3990 from jedevc/manifest-oci

Add OCI support to manifest subcommand

cli/command/manifest/push.go

@@ -12,6 +12,7 @@ import (
 	registryclient "github.com/docker/cli/cli/registry/client"
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/ocischema"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/registry"
@@ -217,18 +218,53 @@ func buildPutManifestRequest(imageManifest types.ImageManifest, targetRef refere
 		return mountRequest{}, err
 	}
 
-	// This indentation has to be added to ensure sha parity with the registry
-	v2ManifestBytes, err := json.MarshalIndent(imageManifest.SchemaV2Manifest, "", "   ")
-	if err != nil {
-		return mountRequest{}, err
-	}
-	// indent only the DeserializedManifest portion of this, in order to maintain parity with the registry
-	// and not alter the sha
-	var v2Manifest schema2.DeserializedManifest
-	if err = v2Manifest.UnmarshalJSON(v2ManifestBytes); err != nil {
-		return mountRequest{}, err
+	// Attempt to reconstruct indentation of the manifest to ensure sha parity
+	// with the registry - if we haven't preserved the raw content.
+	//
+	// This is necessary because our previous internal storage format did not
+	// preserve whitespace. If we don't have the newer format present, we can
+	// attempt the reconstruction like before, but explicitly error if the
+	// reconstruction failed!
+	switch {
+	case imageManifest.SchemaV2Manifest != nil:
+		dt := imageManifest.Raw
+		if len(dt) == 0 {
+			dt, err = json.MarshalIndent(imageManifest.SchemaV2Manifest, "", "   ")
+			if err != nil {
+				return mountRequest{}, err
+			}
+		}
+
+		dig := imageManifest.Descriptor.Digest
+		if dig2 := dig.Algorithm().FromBytes(dt); dig != dig2 {
+			return mountRequest{}, errors.Errorf("internal digest mismatch for %s: expected %s, got %s", imageManifest.Ref, dig, dig2)
+		}
+
+		var manifest schema2.DeserializedManifest
+		if err = manifest.UnmarshalJSON(dt); err != nil {
+			return mountRequest{}, err
+		}
+		imageManifest.SchemaV2Manifest = &manifest
+	case imageManifest.OCIManifest != nil:
+		dt := imageManifest.Raw
+		if len(dt) == 0 {
+			dt, err = json.MarshalIndent(imageManifest.OCIManifest, "", "  ")
+			if err != nil {
+				return mountRequest{}, err
+			}
+		}
+
+		dig := imageManifest.Descriptor.Digest
+		if dig2 := dig.Algorithm().FromBytes(dt); dig != dig2 {
+			return mountRequest{}, errors.Errorf("internal digest mismatch for %s: expected %s, got %s", imageManifest.Ref, dig, dig2)
+		}
+
+		var manifest ocischema.DeserializedManifest
+		if err = manifest.UnmarshalJSON(dt); err != nil {
+			return mountRequest{}, err
+		}
+		imageManifest.OCIManifest = &manifest
 	}
-	imageManifest.SchemaV2Manifest = &v2Manifest
 
 	return mountRequest{ref: mountRef, manifest: imageManifest}, err
 }

cli/command/manifest/testdata/inspect-annotate.golden

@@ -14,6 +14,7 @@
 			"variant": "v7"
 		}
 	},
+	"Raw": "ewogICAic2NoZW1hVmVyc2lvbiI6IDIsCiAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5kaXN0cmlidXRpb24ubWFuaWZlc3QudjIranNvbiIsCiAgICJjb25maWciOiB7CiAgICAgICJtZWRpYVR5cGUiOiAiYXBwbGljYXRpb24vdm5kLmRvY2tlci5jb250YWluZXIuaW1hZ2UudjEranNvbiIsCiAgICAgICJzaXplIjogMTUyMCwKICAgICAgImRpZ2VzdCI6ICJzaGEyNTY6NzMyOGY2ZjhiNDE4OTA1OTc1NzVjYmFhZGM4ODRlNzM4NmFlMGFjYzUzYjc0NzQwMWViY2U1Y2YwZDYyNDU2MCIKICAgfSwKICAgImxheWVycyI6IFsKICAgICAgewogICAgICAgICAibWVkaWFUeXBlIjogImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLAogICAgICAgICAic2l6ZSI6IDE5OTA0MDIsCiAgICAgICAgICJkaWdlc3QiOiAic2hhMjU2Ojg4Mjg2ZjQxNTMwZTkzZGZmZDRiOTY0ZTFkYjIyY2U0OTM5ZmZmYTRhNGM2NjVkYWI4NTkxZmJhYjAzZDQ5MjYiCiAgICAgIH0KICAgXQp9",
 	"SchemaV2Manifest": {
 		"schemaVersion": 2,
 		"mediaType": "application/vnd.docker.distribution.manifest.v2+json",

cli/command/manifest/util.go

@@ -76,6 +76,8 @@ func getManifest(ctx context.Context, dockerCli command.Cli, listRef, namedRef r
 		return dockerCli.RegistryClient(insecure).GetManifest(ctx, namedRef)
 	case err != nil:
 		return types.ImageManifest{}, err
+	case len(data.Raw) == 0:
+		return dockerCli.RegistryClient(insecure).GetManifest(ctx, namedRef)
 	default:
 		return data, nil
 	}

cli/manifest/types/types.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/ocischema"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/reference"
 	"github.com/opencontainers/go-digest"
@@ -16,10 +17,12 @@ import (
 type ImageManifest struct {
 	Ref        *SerializableNamed
 	Descriptor ocispec.Descriptor
+	Raw        []byte `json:",omitempty"`
 
 	// SchemaV2Manifest is used for inspection
-	// TODO: Deprecate this and store manifest blobs
 	SchemaV2Manifest *schema2.DeserializedManifest `json:",omitempty"`
+	// OCIManifest is used for inspection
+	OCIManifest *ocischema.DeserializedManifest `json:",omitempty"`
 }
 
 // OCIPlatform creates an OCI platform from a manifest list platform spec
@@ -53,8 +56,15 @@ func PlatformSpecFromOCI(p *ocispec.Platform) *manifestlist.PlatformSpec {
 // Blobs returns the digests for all the blobs referenced by this manifest
 func (i ImageManifest) Blobs() []digest.Digest {
 	digests := []digest.Digest{}
-	for _, descriptor := range i.SchemaV2Manifest.References() {
-		digests = append(digests, descriptor.Digest)
+	switch {
+	case i.SchemaV2Manifest != nil:
+		for _, descriptor := range i.SchemaV2Manifest.References() {
+			digests = append(digests, descriptor.Digest)
+		}
+	case i.OCIManifest != nil:
+		for _, descriptor := range i.OCIManifest.References() {
+			digests = append(digests, descriptor.Digest)
+		}
 	}
 	return digests
 }
@@ -65,6 +75,8 @@ func (i ImageManifest) Payload() (string, []byte, error) {
 	switch {
 	case i.SchemaV2Manifest != nil:
 		return i.SchemaV2Manifest.Payload()
+	case i.OCIManifest != nil:
+		return i.OCIManifest.Payload()
 	default:
 		return "", nil, errors.Errorf("%s has no payload", i.Ref)
 	}
@@ -76,6 +88,8 @@ func (i ImageManifest) References() []distribution.Descriptor {
 	switch {
 	case i.SchemaV2Manifest != nil:
 		return i.SchemaV2Manifest.References()
+	case i.OCIManifest != nil:
+		return i.OCIManifest.References()
 	default:
 		return nil
 	}
@@ -84,13 +98,35 @@ func (i ImageManifest) References() []distribution.Descriptor {
 // NewImageManifest returns a new ImageManifest object. The values for Platform
 // are initialized from those in the image
 func NewImageManifest(ref reference.Named, desc ocispec.Descriptor, manifest *schema2.DeserializedManifest) ImageManifest {
+	raw, err := manifest.MarshalJSON()
+	if err != nil {
+		raw = nil
+	}
+
 	return ImageManifest{
 		Ref:              &SerializableNamed{Named: ref},
 		Descriptor:       desc,
+		Raw:              raw,
 		SchemaV2Manifest: manifest,
 	}
 }
 
+// NewOCIImageManifest returns a new ImageManifest object. The values for
+// Platform are initialized from those in the image
+func NewOCIImageManifest(ref reference.Named, desc ocispec.Descriptor, manifest *ocischema.DeserializedManifest) ImageManifest {
+	raw, err := manifest.MarshalJSON()
+	if err != nil {
+		raw = nil
+	}
+
+	return ImageManifest{
+		Ref:         &SerializableNamed{Named: ref},
+		Descriptor:  desc,
+		Raw:         raw,
+		OCIManifest: manifest,
+	}
+}
+
 // SerializableNamed is a reference.Named that can be serialized and deserialized
 // from JSON
 type SerializableNamed struct {

cli/registry/client/fetcher.go

@@ -7,6 +7,7 @@ import (
 	"github.com/docker/cli/cli/manifest/types"
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/ocischema"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/distribution/registry/api/errcode"
@@ -35,6 +36,12 @@ func fetchManifest(ctx context.Context, repo distribution.Repository, ref refere
 			return types.ImageManifest{}, err
 		}
 		return imageManifest, nil
+	case *ocischema.DeserializedManifest:
+		imageManifest, err := pullManifestOCISchema(ctx, ref, repo, *v)
+		if err != nil {
+			return types.ImageManifest{}, err
+		}
+		return imageManifest, nil
 	case *manifestlist.DeserializedManifestList:
 		return types.ImageManifest{}, errors.Errorf("%s is a manifest list", ref)
 	}
@@ -94,6 +101,28 @@ func pullManifestSchemaV2(ctx context.Context, ref reference.Named, repo distrib
 	return types.NewImageManifest(ref, manifestDesc, &mfst), nil
 }
 
+func pullManifestOCISchema(ctx context.Context, ref reference.Named, repo distribution.Repository, mfst ocischema.DeserializedManifest) (types.ImageManifest, error) {
+	manifestDesc, err := validateManifestDigest(ref, mfst)
+	if err != nil {
+		return types.ImageManifest{}, err
+	}
+	configJSON, err := pullManifestSchemaV2ImageConfig(ctx, mfst.Target().Digest, repo)
+	if err != nil {
+		return types.ImageManifest{}, err
+	}
+
+	if manifestDesc.Platform == nil {
+		manifestDesc.Platform = &ocispec.Platform{}
+	}
+
+	// Fill in os and architecture fields from config JSON
+	if err := json.Unmarshal(configJSON, manifestDesc.Platform); err != nil {
+		return types.ImageManifest{}, err
+	}
+
+	return types.NewOCIImageManifest(ref, manifestDesc, &mfst), nil
+}
+
 func pullManifestSchemaV2ImageConfig(ctx context.Context, dgst digest.Digest, repo distribution.Repository) ([]byte, error) {
 	blobs := repo.Blobs(ctx)
 	configJSON, err := blobs.Get(ctx, dgst)
@@ -153,16 +182,21 @@ func pullManifestList(ctx context.Context, ref reference.Named, repo distributio
 		if err != nil {
 			return nil, err
 		}
-		v, ok := manifest.(*schema2.DeserializedManifest)
-		if !ok {
-			return nil, errors.Errorf("unsupported manifest format: %v", v)
-		}
 
 		manifestRef, err := reference.WithDigest(ref, manifestDescriptor.Digest)
 		if err != nil {
 			return nil, err
 		}
-		imageManifest, err := pullManifestSchemaV2(ctx, manifestRef, repo, *v)
+
+		var imageManifest types.ImageManifest
+		switch v := manifest.(type) {
+		case *schema2.DeserializedManifest:
+			imageManifest, err = pullManifestSchemaV2(ctx, manifestRef, repo, *v)
+		case *ocischema.DeserializedManifest:
+			imageManifest, err = pullManifestOCISchema(ctx, manifestRef, repo, *v)
+		default:
+			err = errors.Errorf("unsupported manifest type: %T", manifest)
+		}
 		if err != nil {
 			return nil, err
 		}