Merge pull request #473 from yawqi/load-parent-test

nydusify & nydus-image: v6 image conversion support
dragonflyoss · Jun 10, 2022 · aabe14f · aabe14f
2 parents 7be74e9 + 5bb773d
commit aabe14f
Show file tree

Hide file tree

Showing 19 changed files with 486 additions and 132 deletions.
diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go
@@ -210,6 +210,7 @@ func main() {
 				// chosen to make it compatible with the 127 max in graph driver of
 				// docker so that we can pull cache image using docker.
 				&cli.UintFlag{Name: "build-cache-max-records", Value: maxCacheMaxRecords, Usage: "Maximum cache records in cache image", EnvVars: []string{"BUILD_CACHE_MAX_RECORDS"}},
+				&cli.StringFlag{Name: "fs-version", Required: false, Usage: "Version number of nydus image format, possible values: 5, 6", EnvVars: []string{"FS_VERSION"}, Value: "5", DefaultText: "V5 format"},
 			},
 			Action: func(c *cli.Context) error {
 				setupLogLevel(c)
@@ -311,6 +312,7 @@ func main() {
 
 					NydusifyVersion: version,
 					Source:          c.String("source"),
+					FsVersion:       c.String("fs-version"),
 
 					ChunkDict: converter.ChunkDictOpt{
 						Args:     c.String("chunk-dict"),

diff --git a/contrib/nydusify/pkg/build/builder.go b/contrib/nydusify/pkg/build/builder.go
@@ -26,6 +26,7 @@ type BuilderOption struct {
 	// A regular file or fifo into which commands nydus-image to dump contents.
 	BlobPath     string
 	AlignedChunk bool
+	FsVersion    string
 }
 
 type CompactOption struct {
@@ -120,6 +121,8 @@ func (builder *Builder) Run(option BuilderOption) error {
 		option.OutputJSONPath,
 		"--blob",
 		option.BlobPath,
+		"--fs-version",
+		option.FsVersion,
 	)
 
 	if len(option.PrefetchPatterns) > 0 {

diff --git a/contrib/nydusify/pkg/build/workflow.go b/contrib/nydusify/pkg/build/workflow.go
@@ -21,6 +21,7 @@ type WorkflowOption struct {
 	TargetDir        string
 	NydusImagePath   string
 	PrefetchPatterns string
+	FsVersion        string
 }
 
 type Workflow struct {
@@ -118,6 +119,7 @@ func (workflow *Workflow) Build(
 		BlobPath:            blobPath,
 		AlignedChunk:        alignedChunk,
 		ChunkDict:           workflow.ChunkDict,
+		FsVersion:           workflow.FsVersion,
 	}); err != nil {
 		return "", errors.Wrapf(err, "build layer %s", layerDir)
 	}

diff --git a/contrib/nydusify/pkg/checker/checker.go b/contrib/nydusify/pkg/checker/checker.go
@@ -32,6 +32,7 @@ type Opt struct {
 	BackendType    string
 	BackendConfig  string
 	ExpectedArch   string
+	FsVersion      string
 }
 
 // Checker validates Nydus image manifest, bootstrap and mounts filesystem
@@ -105,6 +106,14 @@ func (checker *Checker) Check(ctx context.Context) error {
 		return errors.Wrap(err, "output image information")
 	}
 
+	mode := "cached"
+	digestValidate := true
+	if checker.FsVersion == "6" {
+		mode = "direct"
+		digestValidate = false
+
+	}
+
 	rules := []rule.Rule{
 		&rule.ManifestRule{
 			SourceParsed:  sourceParsed,
@@ -124,14 +133,16 @@ func (checker *Checker) Check(ctx context.Context) error {
 			Source:          checker.Source,
 			SourceMountPath: filepath.Join(checker.WorkDir, "fs/source_mounted"),
 			NydusdConfig: tool.NydusdConfig{
-				NydusdPath:    checker.NydusdPath,
-				BackendType:   checker.BackendType,
-				BackendConfig: checker.BackendConfig,
-				BootstrapPath: filepath.Join(checker.WorkDir, "nydus_bootstrap"),
-				ConfigPath:    filepath.Join(checker.WorkDir, "fs/nydusd_config.json"),
-				BlobCacheDir:  filepath.Join(checker.WorkDir, "fs/nydus_blobs"),
-				MountPath:     filepath.Join(checker.WorkDir, "fs/nydus_mounted"),
-				APISockPath:   filepath.Join(checker.WorkDir, "fs/nydus_api.sock"),
+				NydusdPath:     checker.NydusdPath,
+				BackendType:    checker.BackendType,
+				BackendConfig:  checker.BackendConfig,
+				BootstrapPath:  filepath.Join(checker.WorkDir, "nydus_bootstrap"),
+				ConfigPath:     filepath.Join(checker.WorkDir, "fs/nydusd_config.json"),
+				BlobCacheDir:   filepath.Join(checker.WorkDir, "fs/nydus_blobs"),
+				MountPath:      filepath.Join(checker.WorkDir, "fs/nydus_mounted"),
+				APISockPath:    filepath.Join(checker.WorkDir, "fs/nydus_api.sock"),
+				Mode:           mode,
+				DigestValidate: digestValidate,
 			},
 		},
 	}

diff --git a/contrib/nydusify/pkg/checker/tool/nydusd.go b/contrib/nydusify/pkg/checker/tool/nydusd.go
@@ -30,6 +30,8 @@ type NydusdConfig struct {
 	BlobCacheDir   string
 	APISockPath    string
 	MountPath      string
+	Mode           string
+	DigestValidate bool
 }
 
 // Nydusd runs nydusd binary.
@@ -55,14 +57,14 @@ var configTpl = `
 			}
 		}
 	},
-	"mode": "cached",
+	"mode": "{{.Mode}}",
 	"iostats_files": false,
 	"fs_prefetch": {
 		"enable": {{.EnablePrefetch}},
 		"threads_count": 10,
 		"merging_size": 131072
 	},
-	"digest_validate": true,
+	"digest_validate": {{.DigestValidate}},
 	"enable_xattr": true
 }
 `

diff --git a/contrib/nydusify/pkg/converter/converter.go b/contrib/nydusify/pkg/converter/converter.go
@@ -90,6 +90,7 @@ type Opt struct {
 	Source          string
 
 	ChunkDict ChunkDictOpt
+	FsVersion string
 }
 
 type Converter struct {
@@ -118,6 +119,7 @@ type Converter struct {
 	storageBackend backend.Backend
 
 	chunkDict ChunkDictOpt
+	FsVersion string
 }
 
 func imageRepository(ref string) (string, error) {
@@ -157,6 +159,7 @@ func New(opt Opt) (*Converter, error) {
 		storageBackend: backend,
 
 		chunkDict: opt.ChunkDict,
+		FsVersion: opt.FsVersion,
 	}, nil
 }
 
@@ -211,6 +214,7 @@ func (cvt *Converter) convert(ctx context.Context) (retErr error) {
 		NydusImagePath:   cvt.NydusImagePath,
 		PrefetchPatterns: cvt.PrefetchPatterns,
 		TargetDir:        cvt.WorkDir,
+		FsVersion:        cvt.FsVersion,
 	})
 	if err != nil {
 		return errors.Wrap(err, "Create build flow")

diff --git a/contrib/nydusify/tests/e2e_test.go b/contrib/nydusify/tests/e2e_test.go
@@ -9,74 +9,75 @@ import (
 	"testing"
 )
 
-func testBasicConvert(t *testing.T) {
+func testBasicConvert(t *testing.T, fsVersion string) {
 	registry := NewRegistry(t)
 	defer registry.Destroy(t)
-
 	registry.Build(t, "image-basic")
-	nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "")
+	nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion)
 	nydusify.Convert(t)
 	nydusify.Check(t)
 }
 
-func testConvertWithCache(t *testing.T) {
+func testConvertWithCache(t *testing.T, fsVersion string) {
 	registry := NewRegistry(t)
 	defer registry.Destroy(t)
 
 	registry.Build(t, "image-basic")
-	nydusify1 := NewNydusify(registry, "image-basic", "image-basic-nydus-1", "cache:v1", "")
+	nydusify1 := NewNydusify(registry, "image-basic", "image-basic-nydus-1", "cache:v1", "", fsVersion)
 	nydusify1.Convert(t)
 	nydusify1.Check(t)
-
-	nydusify2 := NewNydusify(registry, "image-basic", "image-basic-nydus-2", "cache:v1", "")
+	nydusify2 := NewNydusify(registry, "image-basic", "image-basic-nydus-2", "cache:v1", "", fsVersion)
 	nydusify2.Convert(t)
 	nydusify2.Check(t)
 
 	registry.Build(t, "image-from-1")
-	nydusify3 := NewNydusify(registry, "image-from-1", "image-from-nydus-1", "cache:v1", "")
+	nydusify3 := NewNydusify(registry, "image-from-1", "image-from-nydus-1", "cache:v1", "", fsVersion)
 	nydusify3.Convert(t)
 	nydusify3.Check(t)
 
 	registry.Build(t, "image-from-2")
-	nydusify4 := NewNydusify(registry, "image-from-2", "image-from-nydus-2", "cache:v1", "")
+	nydusify4 := NewNydusify(registry, "image-from-2", "image-from-nydus-2", "cache:v1", "", fsVersion)
 	nydusify4.Convert(t)
 	nydusify4.Check(t)
 }
 
-func testConvertWithChunkDict(t *testing.T) {
+func testConvertWithChunkDict(t *testing.T, fsVersion string) {
 	registry := NewRegistry(t)
 	defer registry.Destroy(t)
 
 	registry.Build(t, "chunk-dict-1")
 	// build chunk-dict-1 bootstrap
-	nydusify1 := NewNydusify(registry, "chunk-dict-1", "nydus:chunk-dict-1", "", "")
+	nydusify1 := NewNydusify(registry, "chunk-dict-1", "nydus:chunk-dict-1", "", "", fsVersion)
 	nydusify1.Convert(t)
 	nydusify1.Check(t)
 	chunkDictOpt := fmt.Sprintf("bootstrap:registry:%s/%s", registry.Host(), "nydus:chunk-dict-1")
 	// build without build-cache
 	registry.Build(t, "image-basic")
-	nydusify2 := NewNydusify(registry, "image-basic", "nydus:image-basic", "", chunkDictOpt)
+	nydusify2 := NewNydusify(registry, "image-basic", "nydus:image-basic", "", chunkDictOpt, fsVersion)
 	nydusify2.Convert(t)
 	nydusify2.Check(t)
 	// build with build-cache
 	registry.Build(t, "image-from-1")
-	nydusify3 := NewNydusify(registry, "image-from-1", "nydus:image-from-1", "nydus:cache_v1", chunkDictOpt)
+	nydusify3 := NewNydusify(registry, "image-from-1", "nydus:image-from-1", "nydus:cache_v1", chunkDictOpt, fsVersion)
 	nydusify3.Convert(t)
 	nydusify3.Check(t)
 	// change chunk dict
 	registry.Build(t, "chunk-dict-2")
-	nydusify4 := NewNydusify(registry, "chunk-dict-2", "nydus:chunk-dict-2", "", "")
+	nydusify4 := NewNydusify(registry, "chunk-dict-2", "nydus:chunk-dict-2", "", "", fsVersion)
 	nydusify4.Convert(t)
 	nydusify4.Check(t)
 	chunkDictOpt = fmt.Sprintf("bootstrap:registry:%s/%s", registry.Host(), "nydus:chunk-dict-2")
 	registry.Build(t, "image-from-2")
-	nydusify5 := NewNydusify(registry, "image-from-2", "nydus:image-from-2", "nydus:cache_v1", chunkDictOpt)
+	nydusify5 := NewNydusify(registry, "image-from-2", "nydus:image-from-2", "nydus:cache_v1", chunkDictOpt, fsVersion)
 	nydusify5.Convert(t)
 	nydusify5.Check(t)
 }
 
 func TestSmoke(t *testing.T) {
-	testBasicConvert(t)
-	testConvertWithCache(t)
-	testConvertWithChunkDict(t)
+	fsVersions := [2]string{"5", "6"}
+	for _, v := range fsVersions {
+		testBasicConvert(t, v)
+		testConvertWithCache(t, v)
+		testConvertWithChunkDict(t, v)
+	}
 }
diff --git a/contrib/nydusify/tests/image_test.go b/contrib/nydusify/tests/image_test.go
@@ -34,17 +34,18 @@ func transfer(t *testing.T, ref string) {
 	run(t, fmt.Sprintf("docker push %s/%s", host, ref), true)
 }
 
-func convert(t *testing.T, ref string) {
+func convert(t *testing.T, ref string, fsVersion string) {
 	registry := NewRegistry(t)
 	defer registry.Destroy(t)
 	transfer(t, ref)
-	nydusify := NewNydusify(registry, ref, fmt.Sprintf("%s-nydus", ref), "", "")
+	nydusify := NewNydusify(registry, ref, fmt.Sprintf("%s-nydus", ref), "", "", fsVersion)
 	nydusify.Convert(t)
 	nydusify.Check(t)
 }
 
 func TestDockerHubImage(t *testing.T) {
 	for _, ref := range list {
-		convert(t, ref)
+		convert(t, ref, "5")
+		convert(t, ref, "6")
 	}
 }
diff --git a/contrib/nydusify/tests/nydusify.go b/contrib/nydusify/tests/nydusify.go
@@ -42,9 +42,10 @@ type Nydusify struct {
 	backendType   string
 	backendConfig string
 	chunkDictArgs string
+	fsVersion     string
 }
 
-func NewNydusify(registry *Registry, source, target, cache string, chunkDictArgs string) *Nydusify {
+func NewNydusify(registry *Registry, source, target, cache string, chunkDictArgs string, fsVersion string) *Nydusify {
 	host := registry.Host()
 
 	backendType := "registry"
@@ -60,6 +61,9 @@ func NewNydusify(registry *Registry, source, target, cache string, chunkDictArgs
 	if os.Getenv("BACKEND_CONFIG") != "" {
 		backendConfig = os.Getenv("BACKEND_CONFIG")
 	}
+	if len(fsVersion) == 0 {
+		fsVersion = "5"
+	}
 
 	return &Nydusify{
 		Registry:      registry,
@@ -69,6 +73,7 @@ func NewNydusify(registry *Registry, source, target, cache string, chunkDictArgs
 		backendType:   backendType,
 		backendConfig: backendConfig,
 		chunkDictArgs: chunkDictArgs,
+		fsVersion:     fsVersion,
 	}
 }
 
@@ -128,6 +133,7 @@ func (nydusify *Nydusify) Convert(t *testing.T) {
 			Insecure: false,
 			Platform: "linux/amd64",
 		},
+		FsVersion: nydusify.fsVersion,
 	}
 
 	cvt, err := converter.New(opt)
@@ -151,6 +157,7 @@ func (nydusify *Nydusify) Check(t *testing.T) {
 		BackendType:    nydusify.backendType,
 		BackendConfig:  nydusify.backendConfig,
 		ExpectedArch:   "amd64",
+		FsVersion:      nydusify.fsVersion,
 	})
 	assert.Nil(t, err)
 

diff --git a/contrib/nydusify/tests/registry.go b/contrib/nydusify/tests/registry.go
@@ -49,7 +49,8 @@ func NewRegistry(t *testing.T) *Registry {
 }
 
 func (registry *Registry) Destroy(t *testing.T) {
-	run(t, fmt.Sprintf("docker rm -f %s", registry.id), true)
+	run(t, fmt.Sprintf("docker stop  %s", registry.id), true)
+	run(t, fmt.Sprintf("docker rm -f  %s", registry.id), true)
 }
 
 func (registry *Registry) Build(t *testing.T, source string) {