feat: support push manifest

Signed-off-by: Yang Kaiyong <yangkaiyong.yky@antgroup.com>

contrib/nydusify/go.mod

@@ -1,6 +1,8 @@
 module github.com/dragonflyoss/nydus/contrib/nydusify
 
-go 1.21
+go 1.23.1
+
+toolchain go1.23.6
 
 require (
 	github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
@@ -37,6 +39,7 @@ require (
 require (
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect
+	github.com/CloudNativeAI/model-spec v0.0.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/Microsoft/hcsshim v0.11.5 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect

contrib/nydusify/go.sum

@@ -4,6 +4,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h
 github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 h1:dIScnXFlF784X79oi7MzVT6GWqr/W1uUt0pB5CsDs9M=
 github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2/go.mod h1:gCLVsLfv1egrcZu+GoJATN5ts75F2s62ih/457eWzOw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/CloudNativeAI/model-spec v0.0.2 h1:uCO86kMk8wwadn8vKs0wT4petig5crByTIngdO3L2cQ=
+github.com/CloudNativeAI/model-spec v0.0.2/go.mod h1:3U/4zubBfbUkW59ATSg41HnkYyKrKUcKFH/cVdoPQnk=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/Microsoft/hcsshim v0.11.5 h1:haEcLNpj9Ka1gd3B3tAEs9CpE0c+1IhoL59w/exYU38=

contrib/nydusify/pkg/external/modctl/modctl.go

@@ -188,7 +188,7 @@ func (handler *Handler) Backend(ctx context.Context) (*backend.Backend, error) {
 }
 
 func (handler *Handler) GetConfig() ([]byte, error) {
-	return json.Marshal(handler.blobConfig)
+	return handler.extractBlobs(handler.blobConfig.Digest)
 }
 
 func (handler *Handler) setBlobs(m *Manifest) {
@@ -211,22 +211,13 @@ func (handler *Handler) setBlobsMap() {
 func (handler *Handler) extractManifest() (*Manifest, error) {
 	tagPath := fmt.Sprintf(MANIFEST_PATH, handler.tag)
 	manifestPath := filepath.Join(handler.root, REPOS_PATH, handler.registryHost, handler.namespace, handler.imageName, tagPath)
-	content, err := os.ReadFile(manifestPath)
+	line, err := os.ReadFile(manifestPath)
 	if err != nil {
 		return nil, errors.Wrap(err, "read manifest digest file failed")
 	}
-
-	line := strings.TrimSpace(string(content))
-	manifestDigest := strings.Split(line, ":")
-	if len(manifestDigest) != 2 {
-		return nil, errors.New("invalid manifest digest file")
-	}
-
-	blobPath := fmt.Sprintf(BLOB_PATH, manifestDigest[0], manifestDigest[1][:2], manifestDigest[1])
-	blobPath = filepath.Join(handler.root, blobPath)
-	content, err = os.ReadFile(blobPath)
+	content, err := handler.extractBlobs(string(line))
 	if err != nil {
-		return nil, errors.Wrap(err, "read manifest blobs file failed")
+		return nil, errors.Wrap(err, "extract blobs failed")
 	}
 
 	var m Manifest
@@ -236,6 +227,23 @@ func (handler *Handler) extractManifest() (*Manifest, error) {
 	return &m, nil
 }
 
+func (handler *Handler) extractBlobs(digest string) ([]byte, error) {
+	line := strings.TrimSpace(digest)
+	digestSplit := strings.Split(line, ":")
+	if len(digestSplit) != 2 {
+		return nil, errors.New("invalid digest string")
+	}
+
+	blobPath := fmt.Sprintf(BLOB_PATH, digestSplit[0], digestSplit[1][:2], digestSplit[1])
+	blobPath = filepath.Join(handler.root, blobPath)
+	content, err := os.ReadFile(blobPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "read  blobs file failed")
+	}
+	return content, nil
+
+}
+
 func (handler *Handler) convertToBlobs(m *Manifest) []backend.Blob {
 	createBlob := func(layer BlobConfig) backend.Blob {
 		digest := strings.Split(layer.Digest, ":")

go.work

@@ -1,4 +1,6 @@
-go 1.21
+go 1.23.1
+
+toolchain go1.23.6
 
 use (
 	./contrib/nydus-overlayfs

smoke/go.mod

@@ -1,6 +1,6 @@
 module github.com/dragonflyoss/nydus/smoke
 
-go 1.21
+go 1.23.1
 
 require (
 	github.com/containerd/containerd v1.7.11

smoke/tests/external_test.go

@@ -13,10 +13,12 @@ import (
 	"testing"
 	"time"
 
+	modelspec "github.com/CloudNativeAI/model-spec/specs-go/v1"
+
 	"github.com/containerd/containerd/content/local"
 	"github.com/containerd/log"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser"
-	parserPkg "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser"
+	"github.com/opencontainers/image-spec/specs-go"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/provider"
@@ -170,25 +172,29 @@ func packWithAttributes(t *testing.T, packOption converter.PackOption, blobDir,
 }
 
 func pushManifest(
-	ctx context.Context, nydusImage parserPkg.Image, bootstrapDiffID digest.Digest, targetRef, workDir, bootstrapName, fsversion string, insecure bool,
+	ctx context.Context, modelCfg modelspec.Model, nydusImage parser.Image, bootstrapDiffID digest.Digest, targetRef, bootstrapTarPath, fsversion string, insecure bool,
 ) error {
 
 	// Push image config
-	config := nydusImage.Config
+	// config := nydusImage.Config
 
-	config.RootFS.DiffIDs = []digest.Digest{}
-	config.RootFS.DiffIDs = append(config.RootFS.DiffIDs, bootstrapDiffID)
+	// add boottrap diffid to modelSpecCfg
+	modelCfg.ModelFS.DiffIDs = []digest.Digest{
+		bootstrapDiffID,
+	}
 
-	configBytes, configDesc, err := makeDesc(config, nydusImage.Manifest.Config)
+	configBytes, configDesc, err := makeDesc(modelCfg, nydusImage.Manifest.Config)
 	if err != nil {
 		return errors.Wrap(err, "make config desc")
 	}
-
+	fmt.Printf("config bytes %s\n", string(configBytes))
+	fmt.Printf("traget_ref %s\n", targetRef)
 	remoter, err := provider.DefaultRemote(targetRef, insecure)
 	if err != nil {
 		return errors.Wrap(err, "create remote")
 	}
 
+	fmt.Printf("config desc digest %s\n", configDesc.Digest)
 	if err := remoter.Push(ctx, *configDesc, true, bytes.NewReader(configBytes)); err != nil {
 		if utils.RetryWithHTTP(err) {
 			remoter.MaybeWithHTTP(err)
@@ -201,13 +207,12 @@ func pushManifest(
 	}
 
 	// Push bootstrap layer
-	bootstrapTarPath := filepath.Join(workDir, bootstrapName)
 	bootstrapTar, err := os.Open(bootstrapTarPath)
 	if err != nil {
 		return errors.Wrap(err, "open bootstrap tar file")
 	}
 
-	bootstrapTarGzPath := filepath.Join(workDir, bootstrapName+".gz")
+	bootstrapTarGzPath := bootstrapTarPath + ".gz"
 	bootstrapTarGz, err := os.Create(bootstrapTarGzPath)
 	if err != nil {
 		return errors.Wrap(err, "create bootstrap tar.gz file")
@@ -248,7 +253,6 @@ func pushManifest(
 	}
 
 	// Push image manifest
-	// 这个manifes 应该是什么？
 	var layers []ocispec.Descriptor
 	layers = append(layers, bootstrapDesc)
 
@@ -259,13 +263,36 @@ func pushManifest(
 	if err != nil {
 		return errors.Wrap(err, "make manifest desc")
 	}
+	fmt.Printf("manifest Bytes %s, digest: %s type: %s \n", string(manifestBytes), manifestDesc.Digest, manifestDesc.MediaType)
 	if err := remoter.Push(ctx, *manifestDesc, false, bytes.NewReader(manifestBytes)); err != nil {
 		return errors.Wrap(err, "push image manifest")
 	}
 
 	return nil
 }
 
+func buildNydusImage() *parser.Image {
+	manifest := ocispec.Manifest{
+		Versioned:    specs.Versioned{SchemaVersion: 2},
+		MediaType:    ocispec.MediaTypeImageManifest,
+		ArtifactType: modelspec.ArtifactTypeModelManifest,
+		Config: ocispec.Descriptor{
+			MediaType: modelspec.MediaTypeModelConfig,
+		},
+		Annotations: map[string]string{
+			"containerd.io/snapshot/nydus-artifact-type": modelspec.ArtifactTypeModelManifest,
+		},
+	}
+	desc := ocispec.Descriptor{
+		MediaType: ocispec.MediaTypeImageManifest,
+	}
+	nydusImage := &parser.Image{
+		Manifest: manifest,
+		Desc:     desc,
+	}
+	return nydusImage
+}
+
 func makeDesc(x interface{}, oldDesc ocispec.Descriptor) ([]byte, *ocispec.Descriptor, error) {
 	data, err := json.MarshalIndent(x, "", "  ")
 	if err != nil {
@@ -352,20 +379,17 @@ func TestModctlExternal(t *testing.T) {
 	})
 	require.Equal(t, []digest.Digest{blobDigest, externalBlobDigest}, actualDigests)
 	fmt.Printf("mergedBootstrap: %s", mergedBootstrap)
-	os.Rename(mergedBootstrap, mergedBootstrap+".tar")
+	bootStrapTarPath := mergedBootstrap + ".tar"
+	os.Rename(mergedBootstrap, bootStrapTarPath)
+
 	cfgBytes, err := handler.GetConfig()
 	require.NoError(t, err)
-	var cfg ocispec.Descriptor
-	err = json.Unmarshal(cfgBytes, &cfg)
+	var modelCfg modelspec.Model
+	err = json.Unmarshal(cfgBytes, &modelCfg)
 	require.NoError(t, err)
-	nydusImage := &parser.Image{
-		Manifest: ocispec.Manifest{
-			MediaType: "application/vnd.oci.image.manifest.v1+json",
-			Config:    cfg,
-		},
-	}
-	err = pushManifest(context.Background(), *nydusImage, *bootstrapDiffID, opt.TargetRef,
-		ctx.Env.WorkDir, mergedBootstrap, ctx.Build.FSVersion, true)
+
+	nydusImage := buildNydusImage()
+	err = pushManifest(context.Background(), modelCfg, *nydusImage, *bootstrapDiffID, opt.TargetRef, bootStrapTarPath, ctx.Build.FSVersion, true)
 	require.NoError(t, err)
 
 	// build manifest