From af8ef5b38f17b0d1c5bffde62a13944db8271fef Mon Sep 17 00:00:00 2001 From: Yadong Ding Date: Fri, 22 Dec 2023 16:19:58 +0800 Subject: [PATCH] nydusify: add unit test for nydusify We had removed the tests files(e2e) in nydusify, we need add the unit tests to improve test coverage. Signed-off-by: Yadong Ding --- contrib/nydusify/cmd/nydusify_test.go | 253 +++++++++++++++++++ contrib/nydusify/pkg/backend/backend_test.go | 62 +++++ contrib/nydusify/pkg/backend/oss_test.go | 133 ++++++++++ contrib/nydusify/pkg/backend/s3_test.go | 115 +++++++++ contrib/nydusify/pkg/packer/artifact.go | 3 +- contrib/nydusify/pkg/packer/artifact_test.go | 31 ++- contrib/nydusify/pkg/packer/backend_test.go | 26 ++ contrib/nydusify/pkg/packer/packer_test.go | 16 ++ contrib/nydusify/pkg/packer/pusher_test.go | 39 ++- contrib/nydusify/pkg/utils/archive_test.go | 17 ++ contrib/nydusify/pkg/utils/utils_test.go | 74 ++++++ 11 files changed, 747 insertions(+), 22 deletions(-) create mode 100644 contrib/nydusify/pkg/backend/backend_test.go create mode 100644 contrib/nydusify/pkg/backend/oss_test.go create mode 100644 contrib/nydusify/pkg/backend/s3_test.go create mode 100644 contrib/nydusify/pkg/packer/backend_test.go diff --git a/contrib/nydusify/cmd/nydusify_test.go b/contrib/nydusify/cmd/nydusify_test.go index 019be92fa2c..5abea7e729c 100644 --- a/contrib/nydusify/cmd/nydusify_test.go +++ b/contrib/nydusify/cmd/nydusify_test.go @@ -6,10 +6,12 @@ package main import ( "encoding/json" + "flag" "os" "testing" "github.com/stretchr/testify/require" + "github.com/urfave/cli/v2" ) func TestIsPossibleValue(t *testing.T) { @@ -35,6 +37,12 @@ func TestAddReferenceSuffix(t *testing.T) { _, err = addReferenceSuffix(source, suffix) require.Error(t, err) require.Contains(t, err.Error(), "invalid source image reference") + + source = "localhost:5000/nginx:latest@sha256:757574c5a2102627de54971a0083d4ecd24eb48fdf06b234d063f19f7bbc22fb" + suffix = "-suffix" + _, err = addReferenceSuffix(source, suffix) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported digested image reference") } func TestParseBackendConfig(t *testing.T) { @@ -65,4 +73,249 @@ func TestParseBackendConfig(t *testing.T) { // Failure situation _, err = parseBackendConfig(configJSON, file.Name()) require.Error(t, err) + + _, err = parseBackendConfig("", "non-existent.json") + require.Error(t, err) +} + +func TestGetBackendConfig(t *testing.T) { + app := &cli.App{ + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "prefixbackend-type", + Value: "", + }, + &cli.StringFlag{ + Name: "prefixbackend-config", + Value: "", + }, + &cli.StringFlag{ + Name: "prefixbackend-config-file", + Value: "", + }, + }, + } + ctx := cli.NewContext(app, nil, nil) + + backendType, backendConfig, err := getBackendConfig(ctx, "prefix", false) + require.NoError(t, err) + require.Empty(t, backendType) + require.Empty(t, backendConfig) + + backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true) + require.Error(t, err) + require.Contains(t, err.Error(), "backend type is empty, please specify option") + require.Empty(t, backendType) + require.Empty(t, backendConfig) + + flagSet := flag.NewFlagSet("test1", flag.PanicOnError) + flagSet.String("prefixbackend-type", "errType", "") + ctx = cli.NewContext(app, flagSet, nil) + backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true) + require.Error(t, err) + require.Contains(t, err.Error(), "backend-type should be one of") + require.Empty(t, backendType) + require.Empty(t, backendConfig) + + flagSet = flag.NewFlagSet("test2", flag.PanicOnError) + flagSet.String("prefixbackend-type", "oss", "") + ctx = cli.NewContext(app, flagSet, nil) + backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true) + require.Error(t, err) + require.Contains(t, err.Error(), "backend configuration is empty, please specify option") + require.Empty(t, backendType) + require.Empty(t, backendConfig) + + configJSON := ` + { + "bucket_name": "test", + "endpoint": "region.oss.com", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "meta_prefix": "meta", + "blob_prefix": "blob" + }` + require.True(t, json.Valid([]byte(configJSON))) + + flagSet = flag.NewFlagSet("test3", flag.PanicOnError) + flagSet.String("prefixbackend-type", "oss", "") + flagSet.String("prefixbackend-config", configJSON, "") + ctx = cli.NewContext(app, flagSet, nil) + backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true) + require.NoError(t, err) + require.Equal(t, "oss", backendType) + require.Equal(t, configJSON, backendConfig) + + file, err := os.CreateTemp("", "nydusify-backend-config-test.json") + require.NoError(t, err) + defer os.RemoveAll(file.Name()) + + _, err = file.WriteString(configJSON) + require.NoError(t, err) + file.Sync() + + flagSet = flag.NewFlagSet("test4", flag.PanicOnError) + flagSet.String("prefixbackend-type", "oss", "") + flagSet.String("prefixbackend-config-file", file.Name(), "") + ctx = cli.NewContext(app, flagSet, nil) + backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true) + require.NoError(t, err) + require.Equal(t, "oss", backendType) + require.Equal(t, configJSON, backendConfig) + + flagSet = flag.NewFlagSet("test5", flag.PanicOnError) + flagSet.String("prefixbackend-type", "oss", "") + flagSet.String("prefixbackend-config", configJSON, "") + flagSet.String("prefixbackend-config-file", file.Name(), "") + ctx = cli.NewContext(app, flagSet, nil) + backendType, backendConfig, err = getBackendConfig(ctx, "prefix", true) + require.Error(t, err) + require.Contains(t, err.Error(), "--backend-config conflicts with --backend-config-file") + require.Empty(t, backendType) + require.Empty(t, backendConfig) +} + +func TestGetTargetReference(t *testing.T) { + app := &cli.App{ + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "target", + Value: "", + }, + &cli.StringFlag{ + Name: "target-suffix", + Value: "", + }, + &cli.StringFlag{ + Name: "source", + Value: "", + }, + }, + } + ctx := cli.NewContext(app, nil, nil) + + target, err := getTargetReference(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "--target or --target-suffix is required") + require.Empty(t, target) + + flagSet := flag.NewFlagSet("test1", flag.PanicOnError) + flagSet.String("target", "testTarget", "") + flagSet.String("target-suffix", "testSuffix", "") + ctx = cli.NewContext(app, flagSet, nil) + target, err = getTargetReference(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "-target conflicts with --target-suffix") + require.Empty(t, target) + + flagSet = flag.NewFlagSet("test2", flag.PanicOnError) + flagSet.String("target-suffix", "-nydus", "") + flagSet.String("source", "localhost:5000/nginx:latest", "") + ctx = cli.NewContext(app, flagSet, nil) + target, err = getTargetReference(ctx) + require.NoError(t, err) + require.Equal(t, "localhost:5000/nginx:latest-nydus", target) + + flagSet = flag.NewFlagSet("test3", flag.PanicOnError) + flagSet.String("target-suffix", "-nydus", "") + flagSet.String("source", "localhost:5000\nginx:latest", "") + ctx = cli.NewContext(app, flagSet, nil) + target, err = getTargetReference(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid source image reference") + require.Empty(t, target) + + flagSet = flag.NewFlagSet("test4", flag.PanicOnError) + flagSet.String("target", "testTarget", "") + ctx = cli.NewContext(app, flagSet, nil) + target, err = getTargetReference(ctx) + require.NoError(t, err) + require.Equal(t, "testTarget", target) +} + +func TestGetCacheReferencet(t *testing.T) { + app := &cli.App{ + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "build-cache", + Value: "", + }, + &cli.StringFlag{ + Name: "build-cache-tag", + Value: "", + }, + }, + } + ctx := cli.NewContext(app, nil, nil) + + cache, err := getCacheReference(ctx, "") + require.NoError(t, err) + require.Empty(t, cache) + + flagSet := flag.NewFlagSet("test1", flag.PanicOnError) + flagSet.String("build-cache", "cache", "") + flagSet.String("build-cache-tag", "cacheTag", "") + ctx = cli.NewContext(app, flagSet, nil) + cache, err = getCacheReference(ctx, "") + require.Error(t, err) + require.Contains(t, err.Error(), "--build-cache conflicts with --build-cache-tag") + require.Empty(t, cache) + + flagSet = flag.NewFlagSet("test2", flag.PanicOnError) + flagSet.String("build-cache-tag", "cacheTag", "errTarget") + ctx = cli.NewContext(app, flagSet, nil) + cache, err = getCacheReference(ctx, "") + require.Error(t, err) + require.Contains(t, err.Error(), "invalid target image reference: invalid reference format") + require.Empty(t, cache) + + flagSet = flag.NewFlagSet("test2", flag.PanicOnError) + flagSet.String("build-cache-tag", "latest-cache", "") + ctx = cli.NewContext(app, flagSet, nil) + cache, err = getCacheReference(ctx, "localhost:5000/nginx:latest") + require.NoError(t, err) + require.Equal(t, "localhost:5000/nginx:latest-cache", cache) +} + +func TestGetPrefetchPatterns(t *testing.T) { + app := &cli.App{ + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "prefetch-dir", + Value: "", + }, + &cli.BoolFlag{ + Name: "prefetch-patterns", + Value: false, + }, + }, + } + ctx := cli.NewContext(app, nil, nil) + + patterns, err := getPrefetchPatterns(ctx) + require.NoError(t, err) + require.Equal(t, "/", patterns) + + flagSet := flag.NewFlagSet("test1", flag.PanicOnError) + flagSet.String("prefetch-dir", "/etc/passwd", "") + ctx = cli.NewContext(app, flagSet, nil) + patterns, err = getPrefetchPatterns(ctx) + require.NoError(t, err) + require.Equal(t, "/etc/passwd", patterns) + + flagSet = flag.NewFlagSet("test2", flag.PanicOnError) + flagSet.String("prefetch-dir", "/etc/passwd", "") + flagSet.Bool("prefetch-patterns", true, "") + ctx = cli.NewContext(app, flagSet, nil) + patterns, err = getPrefetchPatterns(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "--prefetch-dir conflicts with --prefetch-patterns") + require.Empty(t, patterns) + + flagSet = flag.NewFlagSet("test3", flag.PanicOnError) + flagSet.Bool("prefetch-patterns", true, "") + ctx = cli.NewContext(app, flagSet, nil) + patterns, err = getPrefetchPatterns(ctx) + require.NoError(t, err) + require.Equal(t, "/", patterns) } diff --git a/contrib/nydusify/pkg/backend/backend_test.go b/contrib/nydusify/pkg/backend/backend_test.go new file mode 100644 index 00000000000..0fc5223408e --- /dev/null +++ b/contrib/nydusify/pkg/backend/backend_test.go @@ -0,0 +1,62 @@ +package backend + +import ( + "encoding/json" + "testing" + + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/provider" + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils" + "github.com/stretchr/testify/require" +) + +func TestBlobDesc(t *testing.T) { + desc := blobDesc(123456, "205eed24cbec29ad9cb4593a73168ef1803402370a82f7d51ce25646fc2f943a") + require.Equal(t, int64(123456), desc.Size) + require.Equal(t, "sha256:205eed24cbec29ad9cb4593a73168ef1803402370a82f7d51ce25646fc2f943a", desc.Digest.String()) + require.Equal(t, utils.MediaTypeNydusBlob, desc.MediaType) + require.Equal(t, map[string]string{ + utils.LayerAnnotationUncompressed: "sha256:205eed24cbec29ad9cb4593a73168ef1803402370a82f7d51ce25646fc2f943a", + utils.LayerAnnotationNydusBlob: "true", + }, desc.Annotations) +} + +func TestNewBackend(t *testing.T) { + ossConfigJSON := ` + { + "bucket_name": "test", + "endpoint": "region.oss.com", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "object_prefix": "blob" + }` + require.True(t, json.Valid([]byte(ossConfigJSON))) + backend, err := NewBackend("oss", []byte(ossConfigJSON), nil) + require.NoError(t, err) + require.Equal(t, OssBackend, backend.Type()) + + s3ConfigJSON := ` + { + "bucket_name": "test", + "endpoint": "s3.amazonaws.com", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "object_prefix": "blob", + "scheme": "https", + "region": "region1" + }` + require.True(t, json.Valid([]byte(s3ConfigJSON))) + backend, err = NewBackend("s3", []byte(s3ConfigJSON), nil) + require.NoError(t, err) + require.Equal(t, S3backend, backend.Type()) + + testRegistryRemote, err := provider.DefaultRemote("test", false) + require.NoError(t, err) + backend, err = NewBackend("registry", nil, testRegistryRemote) + require.NoError(t, err) + require.Equal(t, RegistryBackend, backend.Type()) + + backend, err = NewBackend("errBackend", nil, testRegistryRemote) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported backend type") + require.Nil(t, backend) +} diff --git a/contrib/nydusify/pkg/backend/oss_test.go b/contrib/nydusify/pkg/backend/oss_test.go new file mode 100644 index 00000000000..3fb58c84ef7 --- /dev/null +++ b/contrib/nydusify/pkg/backend/oss_test.go @@ -0,0 +1,133 @@ +package backend + +import ( + "encoding/json" + "hash/crc64" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func tempOSSBackend() *OSSBackend { + ossConfigJSON := ` + { + "bucket_name": "test", + "endpoint": "region.oss.com", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "object_prefix": "blob" + }` + backend, _ := newOSSBackend([]byte(ossConfigJSON)) + return backend +} + +func TestCalcCrc64ECMA(t *testing.T) { + blobCrc64, err := calcCrc64ECMA("nil") + require.Error(t, err) + require.Contains(t, err.Error(), "calc md5sum") + require.Zero(t, blobCrc64) + + file, err := os.CreateTemp("", "temp") + require.NoError(t, err) + defer os.RemoveAll(file.Name()) + + _, err = file.WriteString("123") + require.NoError(t, err) + file.Sync() + + blobCrc64, err = calcCrc64ECMA(file.Name()) + require.NoError(t, err) + require.Equal(t, crc64.Checksum([]byte("123"), crc64.MakeTable(crc64.ECMA)), blobCrc64) +} + +func TestOSSRemoteID(t *testing.T) { + ossBackend := tempOSSBackend() + id := ossBackend.remoteID("111") + require.Equal(t, "oss://test/blob111", id) +} + +func TestNewOSSBackend(t *testing.T) { + ossConfigJSON1 := ` + { + "bucket_name": "test", + "endpoint": "region.oss.com", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "object_prefix": "blob" + }` + require.True(t, json.Valid([]byte(ossConfigJSON1))) + backend, err := newOSSBackend([]byte(ossConfigJSON1)) + require.NoError(t, err) + require.Equal(t, "test", backend.bucket.BucketName) + require.Equal(t, "blob", backend.objectPrefix) + + ossConfigJSON2 := ` + { + "bucket_name": "test", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "object_prefix": "blob" + }` + require.True(t, json.Valid([]byte(ossConfigJSON2))) + backend, err = newOSSBackend([]byte(ossConfigJSON2)) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid OSS configuration: missing 'endpoint' or 'bucket'") + require.Nil(t, backend) + + ossConfigJSON3 := ` + { + "bucket_name": "test", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "object_prefix": "blob" + }` + require.True(t, json.Valid([]byte(ossConfigJSON3))) + backend, err = newOSSBackend([]byte(ossConfigJSON3)) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid OSS configuration: missing 'endpoint' or 'bucket'") + require.Nil(t, backend) + + ossConfigJSON4 := ` + { + "bucket_name": "t", + "endpoint": "region.oss.com", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "object_prefix": "blob" + }` + require.True(t, json.Valid([]byte(ossConfigJSON4))) + backend, err = newOSSBackend([]byte(ossConfigJSON4)) + require.Error(t, err) + require.Contains(t, err.Error(), "Create bucket") + require.Contains(t, err.Error(), "len is between [3-63],now is") + require.Nil(t, backend) + + ossConfigJSON5 := ` + { + "bucket_name": "AAA", + "endpoint": "region.oss.com", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "object_prefix": "blob" + }` + require.True(t, json.Valid([]byte(ossConfigJSON5))) + backend, err = newOSSBackend([]byte(ossConfigJSON5)) + require.Error(t, err) + require.Contains(t, err.Error(), "Create bucket") + require.Contains(t, err.Error(), "can only include lowercase letters, numbers, and -") + require.Nil(t, backend) + + ossConfigJSON6 := ` + { + "bucket_name": "AAA", + "endpoint": "region.oss.com", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "object_prefix": "blob", + }` + backend, err = newOSSBackend([]byte(ossConfigJSON6)) + require.Error(t, err) + require.Contains(t, err.Error(), "Parse OSS storage backend configuration") + require.Nil(t, backend) +} diff --git a/contrib/nydusify/pkg/backend/s3_test.go b/contrib/nydusify/pkg/backend/s3_test.go new file mode 100644 index 00000000000..be29e433b3b --- /dev/null +++ b/contrib/nydusify/pkg/backend/s3_test.go @@ -0,0 +1,115 @@ +package backend + +import ( + "context" + "encoding/json" + "testing" + + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/stretchr/testify/require" +) + +func tempS3Backend() *S3Backend { + s3ConfigJSON := ` + { + "bucket_name": "test", + "endpoint": "s3.amazonaws.com", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "object_prefix": "blob", + "scheme": "https", + "region": "region1" + }` + backend, _ := newS3Backend([]byte(s3ConfigJSON)) + return backend +} + +func TestS3RemoteID(t *testing.T) { + s3Backend := tempS3Backend() + id := s3Backend.remoteID("111") + require.Equal(t, "https://s3.amazonaws.com/test/111", id) +} + +func TestBlobObjectKey(t *testing.T) { + s3Backend := tempS3Backend() + blobObjectKey := s3Backend.blobObjectKey("111") + require.Equal(t, "blob111", blobObjectKey) +} + +func TestNewS3Backend(t *testing.T) { + s3ConfigJSON1 := ` + { + "bucket_name": "test", + "endpoint": "s3.amazonaws.com", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "object_prefix": "blob", + "scheme": "https", + "region": "region1" + }` + require.True(t, json.Valid([]byte(s3ConfigJSON1))) + backend, err := newS3Backend([]byte(s3ConfigJSON1)) + require.NoError(t, err) + require.Equal(t, "blob", backend.objectPrefix) + require.Equal(t, "test", backend.bucketName) + require.Equal(t, "https://s3.amazonaws.com", backend.endpointWithScheme) + require.Equal(t, "https://s3.amazonaws.com", *backend.client.Options().BaseEndpoint) + testCredentials, err := backend.client.Options().Credentials.Retrieve(context.Background()) + require.NoError(t, err) + realCredentials, err := credentials.NewStaticCredentialsProvider("testAK", "testSK", "").Retrieve(context.Background()) + require.NoError(t, err) + require.Equal(t, testCredentials, realCredentials) + + s3ConfigJSON2 := ` + { + "bucket_name": "test", + "endpoint": "s3.amazonaws.com", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "object_prefix": "blob", + "scheme": "https", + "region": "region1", + }` + backend, err = newS3Backend([]byte(s3ConfigJSON2)) + require.Error(t, err) + require.Contains(t, err.Error(), "parse S3 storage backend configuration") + require.Nil(t, backend) + + s3ConfigJSON3 := ` + { + "bucket_name": "test", + "endpoint": "", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "object_prefix": "blob", + "scheme": "", + "region": "region1" + }` + require.True(t, json.Valid([]byte(s3ConfigJSON3))) + backend, err = newS3Backend([]byte(s3ConfigJSON3)) + require.NoError(t, err) + require.Equal(t, "blob", backend.objectPrefix) + require.Equal(t, "test", backend.bucketName) + require.Equal(t, "https://s3.amazonaws.com", backend.endpointWithScheme) + testCredentials, err = backend.client.Options().Credentials.Retrieve(context.Background()) + require.NoError(t, err) + realCredentials, err = credentials.NewStaticCredentialsProvider("testAK", "testSK", "").Retrieve(context.Background()) + require.NoError(t, err) + require.Equal(t, testCredentials, realCredentials) + + s3ConfigJSON4 := ` + { + "bucket_name": "", + "endpoint": "s3.amazonaws.com", + "access_key_id": "testAK", + "access_key_secret": "testSK", + "object_prefix": "blob", + "scheme": "https", + "region": "" + }` + require.True(t, json.Valid([]byte(s3ConfigJSON4))) + backend, err = newS3Backend([]byte(s3ConfigJSON4)) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid S3 configuration: missing 'bucket_name' or 'region'") + require.Nil(t, backend) +} diff --git a/contrib/nydusify/pkg/packer/artifact.go b/contrib/nydusify/pkg/packer/artifact.go index 53f266348ce..13ec4274270 100644 --- a/contrib/nydusify/pkg/packer/artifact.go +++ b/contrib/nydusify/pkg/packer/artifact.go @@ -32,9 +32,8 @@ func (a Artifact) blobFilePath(imageName string, isDigest bool) string { return filepath.Join(a.OutputDir, imageName) } else if suffix := filepath.Ext(imageName); suffix != "" { return filepath.Join(a.OutputDir, strings.TrimSuffix(imageName, suffix)+".blob") - } else { - return filepath.Join(a.OutputDir, imageName+".blob") } + return filepath.Join(a.OutputDir, imageName+".blob") } func (a Artifact) outputJSONPath() string { diff --git a/contrib/nydusify/pkg/packer/artifact_test.go b/contrib/nydusify/pkg/packer/artifact_test.go index 688acd453f1..989da82c290 100644 --- a/contrib/nydusify/pkg/packer/artifact_test.go +++ b/contrib/nydusify/pkg/packer/artifact_test.go @@ -1,20 +1,31 @@ package packer import ( + "os" "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestArtifactPath(t *testing.T) { - artifact, err := NewArtifact("/tmp") + artifact, err := NewArtifact("") + defer os.RemoveAll("./.nydus-build-output") + require.NoError(t, err) + require.Equal(t, ".nydus-build-output/test.meta", artifact.bootstrapPath("test.meta")) + require.Equal(t, ".nydus-build-output/test.m", artifact.bootstrapPath("test.m")) + require.Equal(t, ".nydus-build-output/test.meta", artifact.bootstrapPath("test")) + require.Equal(t, ".nydus-build-output/test.blob", artifact.blobFilePath("test.meta", false)) + require.Equal(t, ".nydus-build-output/test.blob", artifact.blobFilePath("test.m", false)) + require.Equal(t, ".nydus-build-output/test.blob", artifact.blobFilePath("test", false)) + require.Equal(t, ".nydus-build-output/test", artifact.blobFilePath("test", true)) - assert.Nil(t, err) - assert.Equal(t, artifact.bootstrapPath("test.meta"), "/tmp/test.meta") - assert.Equal(t, artifact.bootstrapPath("test.m"), "/tmp/test.m") - assert.Equal(t, artifact.bootstrapPath("test"), "/tmp/test.meta") - assert.Equal(t, artifact.blobFilePath("test.meta", false), "/tmp/test.blob") - assert.Equal(t, artifact.blobFilePath("test.m", false), "/tmp/test.blob") - assert.Equal(t, artifact.blobFilePath("test", false), "/tmp/test.blob") - assert.Equal(t, artifact.blobFilePath("test", true), "/tmp/test") + artifact, err = NewArtifact("/tmp") + require.NoError(t, err) + require.Equal(t, "/tmp/test.meta", artifact.bootstrapPath("test.meta")) + require.Equal(t, "/tmp/test.m", artifact.bootstrapPath("test.m")) + require.Equal(t, "/tmp/test.meta", artifact.bootstrapPath("test")) + require.Equal(t, "/tmp/test.blob", artifact.blobFilePath("test.meta", false)) + require.Equal(t, "/tmp/test.blob", artifact.blobFilePath("test.m", false)) + require.Equal(t, "/tmp/test.blob", artifact.blobFilePath("test", false)) + require.Equal(t, "/tmp/test", artifact.blobFilePath("test", true)) } diff --git a/contrib/nydusify/pkg/packer/backend_test.go b/contrib/nydusify/pkg/packer/backend_test.go new file mode 100644 index 00000000000..2b9df069e49 --- /dev/null +++ b/contrib/nydusify/pkg/packer/backend_test.go @@ -0,0 +1,26 @@ +package packer + +import ( + "testing" + + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/backend" + "github.com/stretchr/testify/require" +) + +func TestS3BackendConfig(t *testing.T) { + s3BackendConfig := &S3BackendConfig{ + Endpoint: "s3.amazonaws.com", + Scheme: "https", + AccessKeyID: "testAK", + AccessKeySecret: "testSK", + Region: "region1", + BucketName: "test", + MetaPrefix: "meta", + BlobPrefix: "blob", + } + _, err := backend.NewBackend("s3", s3BackendConfig.rawMetaBackendCfg(), nil) + require.NoError(t, err) + _, err = backend.NewBackend("s3", s3BackendConfig.rawBlobBackendCfg(), nil) + require.NoError(t, err) + require.Equal(t, "s3", s3BackendConfig.backendType()) +} diff --git a/contrib/nydusify/pkg/packer/packer_test.go b/contrib/nydusify/pkg/packer/packer_test.go index 45b2ddcaae1..7459666cd03 100644 --- a/contrib/nydusify/pkg/packer/packer_test.go +++ b/contrib/nydusify/pkg/packer/packer_test.go @@ -31,6 +31,22 @@ func TestNew(t *testing.T) { NydusImagePath: filepath.Join(tmpDir, "nydus-image"), }) assert.Nil(t, err) + _, err = New(Opt{ + LogLevel: logrus.InfoLevel, + OutputDir: tmpDir, + NydusImagePath: filepath.Join(tmpDir, "nydus-image"), + BackendConfig: &S3BackendConfig{ + Endpoint: "s3.amazonaws.com", + Scheme: "https", + AccessKeyID: "testAK", + AccessKeySecret: "testSK", + Region: "region1", + BucketName: "test", + MetaPrefix: "meta", + BlobPrefix: "blob", + }, + }) + assert.Nil(t, err) } func copyFile(src, dst string) { diff --git a/contrib/nydusify/pkg/packer/pusher_test.go b/contrib/nydusify/pkg/packer/pusher_test.go index 394720ae7ac..384049c41b4 100644 --- a/contrib/nydusify/pkg/packer/pusher_test.go +++ b/contrib/nydusify/pkg/packer/pusher_test.go @@ -10,8 +10,8 @@ import ( "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/backend" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" ) type mockBackend struct { @@ -46,8 +46,8 @@ func (m *mockBackend) Size(_ string) (int64, error) { func Test_parseBackendConfig(t *testing.T) { cfg, err := ParseBackendConfig("oss", filepath.Join("testdata", "backend-config.json")) - assert.Nil(t, err) - assert.Equal(t, &OssBackendConfig{ + require.NoError(t, err) + require.Equal(t, &OssBackendConfig{ Endpoint: "mock.aliyuncs.com", AccessKeyID: "testid", AccessKeySecret: "testkey", @@ -66,8 +66,8 @@ func Test_parseBackendConfigString(t *testing.T) { "meta_prefix": "test/", "blob_prefix": "" }`) - assert.Nil(t, err) - assert.Equal(t, &OssBackendConfig{ + require.NoError(t, err) + require.Equal(t, &OssBackendConfig{ Endpoint: "mock.aliyuncs.com", AccessKeyID: "testid", AccessKeySecret: "testkey", @@ -87,7 +87,8 @@ func TestPusher_Push(t *testing.T) { os.WriteFile(filepath.Join(tmpDir, "output.json"), content, 0755) artifact, err := NewArtifact(tmpDir) - assert.Nil(t, err) + require.NoError(t, err) + mp := &mockBackend{} pusher := Pusher{ Artifact: artifact, @@ -100,21 +101,20 @@ func TestPusher_Push(t *testing.T) { metaBackend: mp, blobBackend: mp, } - hash := "3093776c78a21e47f0a8b4c80a1f019b1e838fc1ade274209332af1ca5f57090" - assert.Nil(t, err) mp.On("Upload", mock.Anything, "mock.meta", mock.Anything, mock.Anything, mock.Anything).Return(&ocispec.Descriptor{ URLs: []string{"oss://testbucket/testmetaprefix/mock.meta"}, }, nil) mp.On("Upload", mock.Anything, hash, mock.Anything, mock.Anything, mock.Anything).Return(&ocispec.Descriptor{ URLs: []string{"oss://testbucket/testblobprefix/3093776c78a21e47f0a8b4c80a1f019b1e838fc1ade274209332af1ca5f57090"}, }, nil) + res, err := pusher.Push(PushRequest{ Meta: "mock.meta", Blob: hash, }) - assert.Nil(t, err) - assert.Equal( + require.NoError(t, err) + require.Equal( t, PushResult{ RemoteMeta: "oss://testbucket/testmetaprefix/mock.meta", @@ -123,3 +123,22 @@ func TestPusher_Push(t *testing.T) { res, ) } + +func TestNewPusher(t *testing.T) { + tmpDir, tearDown := setUpTmpDir(t) + defer tearDown() + + artifact, err := NewArtifact(tmpDir) + require.NoError(t, err) + _, err = NewPusher(NewPusherOpt{ + Artifact: artifact, + BackendConfig: &OssBackendConfig{ + Endpoint: "region.oss.com", + BucketName: "testbucket", + BlobPrefix: "testblobprefix", + MetaPrefix: "testmetaprefix", + }, + Logger: logrus.New(), + }) + require.NoError(t, err) +} diff --git a/contrib/nydusify/pkg/utils/archive_test.go b/contrib/nydusify/pkg/utils/archive_test.go index 992babe4c7a..cfcc23f1efd 100644 --- a/contrib/nydusify/pkg/utils/archive_test.go +++ b/contrib/nydusify/pkg/utils/archive_test.go @@ -5,10 +5,13 @@ package utils import ( + "context" + "io" "os" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestPackTargzInfo(t *testing.T) { @@ -25,3 +28,17 @@ func TestPackTargzInfo(t *testing.T) { assert.Equal(t, "sha256:6cdd1b26d54d5852fbea95a81cbb25383975b70b4ffad9f9b6d25c7a434a51eb", digest.String()) assert.Equal(t, size, int64(315)) } + +func TestUnpackTargz(t *testing.T) { + file, err := os.CreateTemp("", "nydusify-test") + defer os.RemoveAll(file.Name()) + require.NoError(t, err) + err = os.WriteFile(file.Name(), []byte("123456789"), 0666) + require.NoError(t, err) + reader, err := PackTargz(file.Name(), file.Name(), true) + require.NoError(t, err) + + err = UnpackTargz(context.Background(), "test", io.Reader(reader), false) + defer os.RemoveAll("test") + require.NoError(t, err) +} diff --git a/contrib/nydusify/pkg/utils/utils_test.go b/contrib/nydusify/pkg/utils/utils_test.go index 19179d53774..d8b1b8bf539 100644 --- a/contrib/nydusify/pkg/utils/utils_test.go +++ b/contrib/nydusify/pkg/utils/utils_test.go @@ -6,11 +6,14 @@ package utils import ( "archive/tar" "compress/gzip" + "fmt" "io" + "net/http" "os" "strings" "testing" + "github.com/goharbor/acceleration-service/pkg/driver/nydus/utils" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/require" @@ -31,6 +34,8 @@ func makePlatform(osArch string, nydus bool) *ocispec.Platform { } if nydus { platform.OSFeatures = []string{ManifestOSFeatureNydus} + } else { + platform.OSFeatures = nil } return platform } @@ -75,6 +80,11 @@ func TestMatchNydusPlatform(t *testing.T) { require.Equal(t, MatchNydusPlatform(&desc, "linux", "amd64"), true) require.Equal(t, MatchNydusPlatform(&desc, "windows", "amd64"), false) require.Equal(t, MatchNydusPlatform(&desc, "windows", "arm64"), false) + desc = makeDesc("nydus", makePlatform("linux/amd64", false)) + require.Equal(t, MatchNydusPlatform(&desc, "linux", "arm64"), false) + require.Equal(t, MatchNydusPlatform(&desc, "linux", "amd64"), false) + require.Equal(t, MatchNydusPlatform(&desc, "windows", "amd64"), false) + require.Equal(t, MatchNydusPlatform(&desc, "windows", "arm64"), false) desc = makeDesc("nydus", makePlatform("windows/arm64", true)) require.Equal(t, MatchNydusPlatform(&desc, "windows", "arm64"), true) require.Equal(t, MatchNydusPlatform(&desc, "windows", "amd64"), false) @@ -194,3 +204,67 @@ func TestUnpackFile(t *testing.T) { require.NoError(t, err) defer os.Remove(outputName) } + +func TestHashFile(t *testing.T) { + file, err := os.CreateTemp("", "tempFile") + require.NoError(t, err) + defer os.RemoveAll(file.Name()) + + _, err = file.WriteString("123456") + require.NoError(t, err) + file.Sync() + + hashSum, err := HashFile(file.Name()) + require.NoError(t, err) + require.Len(t, hashSum, 32) +} + +func TestMarshalToDesc(t *testing.T) { + config := ocispec.Image{ + Config: ocispec.ImageConfig{}, + RootFS: ocispec.RootFS{ + Type: "layers", + // Layers from manifest must be match image config. + DiffIDs: []digest.Digest{}, + }, + } + configDesc, configBytes, err := utils.MarshalToDesc(config, ocispec.MediaTypeImageConfig) + require.NoError(t, err) + require.Equal(t, "application/vnd.oci.image.config.v1+json", configDesc.MediaType) + require.Equal(t, "sha256:92ceafbaf24c322b45bfeed6e98e25f735c723ae470442c647569086f74cc032", configDesc.Digest.String()) + require.Equal(t, int64(len(configBytes)), configDesc.Size) +} + +func TestWithRetry(t *testing.T) { + err := WithRetry(func() error { + _, err := http.Get("http://localhost:5000") + return err + }) + require.Contains(t, err.Error(), "connect: connection refused") +} + +func TestRetryWithHTTP(t *testing.T) { + require.True(t, RetryWithHTTP(fmt.Errorf("server gave HTTP response to HTTPS client"))) + require.False(t, RetryWithHTTP(nil)) +} + +func TestGetNydusFsVersionOrDefault(t *testing.T) { + testAnnotations := make(map[string]string) + fsVersion := GetNydusFsVersionOrDefault(testAnnotations, V5) + require.Equal(t, fsVersion, V5) + + fsVersion = GetNydusFsVersionOrDefault(nil, V6) + require.Equal(t, fsVersion, V6) + + testAnnotations[LayerAnnotationNydusFsVersion] = "5" + fsVersion = GetNydusFsVersionOrDefault(testAnnotations, V6) + require.Equal(t, fsVersion, V5) + + testAnnotations[LayerAnnotationNydusFsVersion] = "6" + fsVersion = GetNydusFsVersionOrDefault(testAnnotations, V5) + require.Equal(t, fsVersion, V6) + + testAnnotations[LayerAnnotationNydusFsVersion] = "7" + fsVersion = GetNydusFsVersionOrDefault(testAnnotations, V5) + require.Equal(t, fsVersion, V5) +}