feat(nydusify): support modctl blob's nydus attribute

- copy snapshotter code to internal package. - add modctl handler to produce nydus attribute pattern. - support push manifest to registry in convert Signed-off-by: Yang Kaiyong <yangkaiyong.yky@antgroup.com>
dragonflyoss · Mar 1, 2025 · 24a62f1 · 24a62f1
1 parent f914b91
commit 24a62f1
Show file tree

Hide file tree

Showing 37 changed files with 4,994 additions and 137 deletions.
diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go
@@ -227,6 +227,18 @@ func main() {
 					Usage:    "Target (Nydus) image reference",
 					EnvVars:  []string{"TARGET"},
 				},
+				&cli.StringFlag{
+					Name:    "source-backend-type",
+					Value:   "",
+					Usage:   "Type of storage backend, possible values: 'oss', 's3'",
+					EnvVars: []string{"BACKEND_TYPE"},
+				},
+				&cli.StringFlag{
+					Name:    "source-backend-config",
+					Value:   "",
+					Usage:   "Json configuration string for storage backend",
+					EnvVars: []string{"BACKEND_CONFIG"},
+				},
 				&cli.StringFlag{
 					Name:     "target-suffix",
 					Required: false,
@@ -494,10 +506,12 @@ func main() {
 					WorkDir:        c.String("work-dir"),
 					NydusImagePath: c.String("nydus-image"),
 
-					Source:         c.String("source"),
-					Target:         targetRef,
-					SourceInsecure: c.Bool("source-insecure"),
-					TargetInsecure: c.Bool("target-insecure"),
+					SourceBackendType:   c.String("source-backend-type"),
+					SourceBackendConfig: c.String("source-backend-config"),
+					Source:              c.String("source"),
+					Target:              targetRef,
+					SourceInsecure:      c.Bool("source-insecure"),
+					TargetInsecure:      c.Bool("target-insecure"),
 
 					BackendType:      backendType,
 					BackendConfig:    backendConfig,

diff --git a/contrib/nydusify/go.mod b/contrib/nydusify/go.mod
@@ -1,6 +1,8 @@
 module github.com/dragonflyoss/nydus/contrib/nydusify
 
-go 1.21
+go 1.23.1
+
+toolchain go1.23.6
 
 require (
 	github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
@@ -37,6 +39,7 @@ require (
 require (
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect
+	github.com/CloudNativeAI/model-spec v0.0.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/Microsoft/hcsshim v0.11.5 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect

diff --git a/contrib/nydusify/go.sum b/contrib/nydusify/go.sum
@@ -4,6 +4,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h
 github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 h1:dIScnXFlF784X79oi7MzVT6GWqr/W1uUt0pB5CsDs9M=
 github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2/go.mod h1:gCLVsLfv1egrcZu+GoJATN5ts75F2s62ih/457eWzOw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/CloudNativeAI/model-spec v0.0.2 h1:uCO86kMk8wwadn8vKs0wT4petig5crByTIngdO3L2cQ=
+github.com/CloudNativeAI/model-spec v0.0.2/go.mod h1:3U/4zubBfbUkW59ATSg41HnkYyKrKUcKFH/cVdoPQnk=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/Microsoft/hcsshim v0.11.5 h1:haEcLNpj9Ka1gd3B3tAEs9CpE0c+1IhoL59w/exYU38=

diff --git a/contrib/nydusify/pkg/chunkdict/generator/generator.go b/contrib/nydusify/pkg/chunkdict/generator/generator.go
@@ -20,8 +20,8 @@ import (
 	originprovider "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/provider"
 	"github.com/goharbor/acceleration-service/pkg/remote"
 
-	"github.com/containerd/nydus-snapshotter/pkg/converter"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/converter/provider"
+	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/snapshotter/converter"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils"
 	"github.com/dustin/go-humanize"
 	"github.com/goharbor/acceleration-service/pkg/platformutil"

diff --git a/contrib/nydusify/pkg/committer/commiter.go b/contrib/nydusify/pkg/committer/commiter.go
@@ -22,10 +22,10 @@ import (
 	"github.com/containerd/containerd/content/local"
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/reference/docker"
-	"github.com/containerd/nydus-snapshotter/pkg/converter"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/committer/diff"
 	parserPkg "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/provider"
+	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/snapshotter/converter"
 	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils"
 	"github.com/dustin/go-humanize"
 	"github.com/opencontainers/go-digest"