diff --git a/smoke/tests/api_test.go b/smoke/tests/api_test.go new file mode 100644 index 00000000000..3e267627e2f --- /dev/null +++ b/smoke/tests/api_test.go @@ -0,0 +1,254 @@ +// Copyright 2023 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/containerd/containerd/log" + "github.com/containerd/nydus-snapshotter/pkg/converter" + "github.com/stretchr/testify/require" + + "github.com/dragonflyoss/image-service/smoke/tests/texture" + "github.com/dragonflyoss/image-service/smoke/tests/tool" +) + +type APIV1TestSuite struct { +} + +func (a *APIV1TestSuite) start(pt *testing.T) { + pt.Run("TestDaemonStatus", func(t *testing.T) { + t.Parallel() + a.TestDaemonStatus(t) + }) + pt.Run("TestMetrics", func(t *testing.T) { + t.Parallel() + a.TestMetrics(t) + }) + pt.Run("TestPrefetch", func(t *testing.T) { + t.Parallel() + a.TestPrefetch(t) + }) +} + +func (a *APIV1TestSuite) TestDaemonStatus(t *testing.T) { + + ctx := tool.DefaultContext(t) + + ctx.PrepareWorkDir(t) + defer ctx.Destroy(t) + + rootFs := texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "root-fs")) + + rafs := a.rootFsToRafs(t, ctx, rootFs) + + nydusd, err := tool.NewNydusd(tool.NydusdConfig{ + NydusdPath: ctx.Binary.Nydusd, + BootstrapPath: rafs, + ConfigPath: filepath.Join(ctx.Env.WorkDir, "nydusd-config.fusedev.json"), + MountPath: ctx.Env.MountDir, + APISockPath: filepath.Join(ctx.Env.WorkDir, "nydusd-api.sock"), + BackendType: "localfs", + BackendConfig: fmt.Sprintf(`{"dir": "%s"}`, ctx.Env.BlobDir), + EnablePrefetch: ctx.Runtime.EnablePrefetch, + BlobCacheDir: ctx.Env.CacheDir, + CacheType: ctx.Runtime.CacheType, + CacheCompressed: ctx.Runtime.CacheCompressed, + RafsMode: ctx.Runtime.RafsMode, + DigestValidate: false, + }) + require.NoError(t, err) + + err = nydusd.Mount() + require.NoError(t, err) + defer func() { + if err := nydusd.Umount(); err != nil { + log.L.WithError(err).Errorf("umount") + } + }() + + // The implementation of runNydusd() has checked stats, however, + // it's clear of semantic to check stats again. + newCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + select { + case <-tool.CheckReady(newCtx, nydusd.APISockPath): + return + case <-time.After(50 * time.Millisecond): + require.Fail(t, "nydusd status is not RUNNING") + } +} + +func (a *APIV1TestSuite) TestMetrics(t *testing.T) { + + ctx := tool.DefaultContext(t) + + ctx.PrepareWorkDir(t) + defer ctx.Destroy(t) + + rootFs := texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "root-fs")) + + rafs := a.rootFsToRafs(t, ctx, rootFs) + + nydusd, err := tool.NewNydusd(tool.NydusdConfig{ + NydusdPath: ctx.Binary.Nydusd, + BootstrapPath: rafs, + ConfigPath: filepath.Join(ctx.Env.WorkDir, "nydusd-config.fusedev.json"), + MountPath: ctx.Env.MountDir, + APISockPath: filepath.Join(ctx.Env.WorkDir, "nydusd-api.sock"), + BackendType: "localfs", + BackendConfig: fmt.Sprintf(`{"dir": "%s"}`, ctx.Env.BlobDir), + EnablePrefetch: ctx.Runtime.EnablePrefetch, + BlobCacheDir: ctx.Env.CacheDir, + CacheType: ctx.Runtime.CacheType, + CacheCompressed: ctx.Runtime.CacheCompressed, + RafsMode: ctx.Runtime.RafsMode, + DigestValidate: false, + IOStatsFiles: true, + LatestReadFiles: true, + AccessPattern: true, + }) + require.NoError(t, err) + + err = nydusd.Mount() + require.NoError(t, err) + defer func() { + if err := nydusd.Umount(); err != nil { + log.L.WithError(err).Errorf("umount") + } + }() + + err = a.visit(filepath.Join(ctx.Env.MountDir, "file-1")) + require.NoError(t, err) + + gm, err := nydusd.GetGlobalMetrics() + require.NoError(t, err) + require.True(t, gm.FilesAccountEnabled) + require.True(t, gm.MeasureLatency) + require.True(t, gm.AccessPattern) + require.Equal(t, uint64(len("file-1")), gm.DataRead) + require.Equal(t, uint64(1), gm.FOPS[4]) + + err = a.visit(filepath.Join(ctx.Env.MountDir, "dir-1/file-1")) + require.NoError(t, err) + gmNew, err := nydusd.GetGlobalMetrics() + require.NoError(t, err) + require.Equal(t, gm.DataRead+uint64(len("dir-1/file-1")), gmNew.DataRead) + require.Equal(t, gm.FOPS[4]+1, gmNew.FOPS[4]) + + _, err = nydusd.GetFilesMetrics("/") + require.NoError(t, err) + + _, err = nydusd.GetBackendMetrics("/") + require.NoError(t, err) + + _, err = nydusd.GetLatestFileMetrics() + require.NoError(t, err) + + apms, err := nydusd.GetAccessPatternMetrics("/") + require.NoError(t, err) + require.NotEmpty(t, apms) + + apms, err = nydusd.GetAccessPatternMetrics("") + require.NoError(t, err) + require.NotEmpty(t, apms) + + apms, err = nydusd.GetAccessPatternMetrics("poison") + require.NoError(t, err) + require.Empty(t, apms) +} + +func (a *APIV1TestSuite) TestPrefetch(t *testing.T) { + + ctx := tool.DefaultContext(t) + + ctx.PrepareWorkDir(t) + defer ctx.Destroy(t) + + rootFs := texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "root-fs")) + + rafs := a.rootFsToRafs(t, ctx, rootFs) + + config := tool.NydusdConfig{ + NydusdPath: ctx.Binary.Nydusd, + MountPath: ctx.Env.MountDir, + APISockPath: filepath.Join(ctx.Env.WorkDir, "nydusd-api.sock"), + ConfigPath: filepath.Join(ctx.Env.WorkDir, "nydusd-config.fusedev.json"), + } + nydusd, err := tool.NewNydusd(config) + require.NoError(t, err) + + err = nydusd.Mount() + require.NoError(t, err) + defer func() { + if err := nydusd.Umount(); err != nil { + log.L.WithError(err).Errorf("umount") + } + }() + + config.BootstrapPath = rafs + config.MountPath = "/pseudo_fs_1" + config.BackendType = "localfs" + config.BackendConfig = fmt.Sprintf(`{"dir": "%s"}`, ctx.Env.BlobDir) + config.EnablePrefetch = true + config.PrefetchFiles = []string{"/"} + config.BlobCacheDir = ctx.Env.CacheDir + config.CacheType = ctx.Runtime.CacheType + config.CacheCompressed = ctx.Runtime.CacheCompressed + config.RafsMode = ctx.Runtime.RafsMode + err = nydusd.MountByAPI(config) + require.NoError(t, err) + time.Sleep(time.Millisecond * 10) + + bcm, err := nydusd.GetBlobCacheMetrics("") + require.NoError(t, err) + require.Greater(t, bcm.PrefetchDataAmount, uint64(0)) + + _, err = nydusd.GetBlobCacheMetrics("/pseudo_fs_1") + require.NoError(t, err) +} + +func (a *APIV1TestSuite) rootFsToRafs(t *testing.T, ctx *tool.Context, rootFs *tool.Layer) string { + digest := rootFs.Pack(t, + converter.PackOption{ + BuilderPath: ctx.Binary.Builder, + Compressor: ctx.Build.Compressor, + FsVersion: ctx.Build.FSVersion, + ChunkSize: ctx.Build.ChunkSize, + }, + ctx.Env.BlobDir) + _, bootstrap := tool.MergeLayers(t, *ctx, + converter.MergeOption{ + BuilderPath: ctx.Binary.Builder, + }, + []converter.Layer{ + {Digest: digest}, + }) + return bootstrap +} + +func (a *APIV1TestSuite) visit(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + + ioutil.ReadAll(f) + + return nil +} + +func TestAPI(t *testing.T) { + apiV1Tests := APIV1TestSuite{} + apiV1Tests.start(t) +} diff --git a/smoke/tests/tool/nydusd.go b/smoke/tests/tool/nydusd.go index 183675f2195..ee822d1b9d9 100644 --- a/smoke/tests/tool/nydusd.go +++ b/smoke/tests/tool/nydusd.go @@ -10,16 +10,47 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "net" "net/http" "os" "os/exec" + "strings" "text/template" "time" "github.com/pkg/errors" ) +type GlobalMetrics struct { + FilesAccountEnabled bool `json:"files_account_enabled"` + MeasureLatency bool `json:"measure_latency"` + AccessPattern bool `json:"access_pattern_enabled"` + DataRead uint64 `json:"data_read"` + FOPS []uint64 `json:"fop_hits"` +} + +type FileMetrics struct { +} + +type BackendMetrics struct { +} + +type AccessPatternMetrics struct { + Ino uint64 `json:"ino"` + NRRead uint64 `json:"nr_read"` + FirstAccessTimeSecs uint `json:"first_access_time_secs"` + FirstAccessTimeNanos uint64 `json:"first_access_time_nanos"` +} + +type BlobCacheMetrics struct { + PrefetchDataAmount uint64 `json:"prefetch_data_amount"` +} + +type InflightMetrics struct { + Ino uint64 `json:"inode"` +} + type NydusdConfig struct { EnablePrefetch bool NydusdPath string @@ -34,6 +65,10 @@ type NydusdConfig struct { DigestValidate bool CacheType string CacheCompressed bool + IOStatsFiles bool + LatestReadFiles bool + AccessPattern bool + PrefetchFiles []string } type Nydusd struct { @@ -60,14 +95,16 @@ var configTpl = ` } }, "mode": "{{.RafsMode}}", - "iostats_files": false, + "iostats_files": {{.IOStatsFiles}}, "fs_prefetch": { "enable": {{.EnablePrefetch}}, "threads_count": 10, "merging_size": 131072 }, "digest_validate": {{.DigestValidate}}, - "enable_xattr": true + "enable_xattr": true, + "latest_read_files": {{.LatestReadFiles}}, + "access_pattern": {{.AccessPattern}} } ` @@ -80,13 +117,13 @@ func makeConfig(conf NydusdConfig) error { } if err := os.WriteFile(conf.ConfigPath, ret.Bytes(), 0600); err != nil { - return errors.New("write config file for Nydusd") + return errors.Wrapf(err, "write config file for Nydusd") } return nil } -func checkReady(ctx context.Context, sock string) <-chan bool { +func CheckReady(ctx context.Context, sock string) <-chan bool { ready := make(chan bool) transport := &http.Transport{ @@ -143,7 +180,7 @@ func checkReady(ctx context.Context, sock string) <-chan bool { func NewNydusd(conf NydusdConfig) (*Nydusd, error) { if err := makeConfig(conf); err != nil { - return nil, errors.New("create config file for Nydusd") + return nil, errors.Wrap(err, "create config file for Nydusd") } return &Nydusd{ NydusdConfig: conf, @@ -154,17 +191,19 @@ func (nydusd *Nydusd) Mount() error { _ = nydusd.Umount() args := []string{ - "--config", - nydusd.ConfigPath, "--mountpoint", nydusd.MountPath, - "--bootstrap", - nydusd.BootstrapPath, "--apisock", nydusd.APISockPath, "--log-level", "error", } + if len(nydusd.ConfigPath) > 0 { + args = append(args, "--config", nydusd.ConfigPath) + } + if len(nydusd.BootstrapPath) > 0 { + args = append(args, "--bootstrap", nydusd.BootstrapPath) + } cmd := exec.Command(nydusd.NydusdPath, args...) cmd.Stdout = os.Stdout @@ -178,7 +217,7 @@ func (nydusd *Nydusd) Mount() error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - ready := checkReady(ctx, nydusd.APISockPath) + ready := CheckReady(ctx, nydusd.APISockPath) select { case err := <-runErr: @@ -194,6 +233,64 @@ func (nydusd *Nydusd) Mount() error { return nil } +func (nydusd *Nydusd) MountByAPI(config NydusdConfig) error { + + err := makeConfig(config) + if err != nil { + return err + } + f, err := os.Open(config.ConfigPath) + if err != nil { + return err + } + defer f.Close() + rafsConfig, err := ioutil.ReadAll(f) + if err != nil { + return err + } + + nydusdConfig := struct { + Bootstrap string `json:"source"` + RafsConfig string `json:"config"` + FsType string `json:"fs_type"` + PrefetchFiles []string `json:"prefetch_files"` + }{ + Bootstrap: config.BootstrapPath, + RafsConfig: string(rafsConfig), + FsType: "rafs", + PrefetchFiles: config.PrefetchFiles, + } + + transport := &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 5 * time.Second, + } + return dialer.DialContext(ctx, "unix", nydusd.APISockPath) + }, + } + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: transport, + } + + body, err := json.Marshal(nydusdConfig) + if err != nil { + return err + } + _, err = client.Post( + fmt.Sprintf("http://unix/api/v1/mount?mountpoint=%s", config.MountPath), + "application/json", + bytes.NewBuffer(body), + ) + + return err +} + func (nydusd *Nydusd) Umount() error { if _, err := os.Stat(nydusd.MountPath); err == nil { cmd := exec.Command("umount", nydusd.MountPath) @@ -204,3 +301,290 @@ func (nydusd *Nydusd) Umount() error { } return nil } + +func (nydusd *Nydusd) GetGlobalMetrics() (*GlobalMetrics, error) { + + transport := &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 5 * time.Second, + } + return dialer.DialContext(ctx, "unix", nydusd.APISockPath) + }, + } + + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: transport, + } + + resp, err := client.Get(fmt.Sprintf("http://unix%s", "/api/v1/metrics")) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var info GlobalMetrics + if err = json.Unmarshal(body, &info); err != nil { + return nil, err + } + + return &info, nil +} + +func (nydusd *Nydusd) GetFilesMetrics(id string) (map[string]FileMetrics, error) { + transport := &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 5 * time.Second, + } + return dialer.DialContext(ctx, "unix", nydusd.APISockPath) + }, + } + + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: transport, + } + + resp, err := client.Get(fmt.Sprintf("http://unix/api/v1/metrics/files?id=%s", id)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + info := make(map[string]FileMetrics) + if err = json.Unmarshal(body, &info); err != nil { + return nil, err + } + + return info, nil +} + +func (nydusd *Nydusd) GetBackendMetrics(id string) (*BackendMetrics, error) { + transport := &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 5 * time.Second, + } + return dialer.DialContext(ctx, "unix", nydusd.APISockPath) + }, + } + + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: transport, + } + + resp, err := client.Get(fmt.Sprintf("http://unix/api/v1/metrics/backend?id=%s", id)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var info BackendMetrics + if err = json.Unmarshal(body, &info); err != nil { + return nil, err + } + + return &info, nil +} + +func (nydusd *Nydusd) GetLatestFileMetrics() ([][]uint64, error) { + transport := &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 5 * time.Second, + } + return dialer.DialContext(ctx, "unix", nydusd.APISockPath) + }, + } + + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: transport, + } + + resp, err := client.Get("http://unix/api/v1/metrics/files?latest=true") + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var info [][]uint64 + if err = json.Unmarshal(body, &info); err != nil { + return nil, err + } + + return info, nil +} + +func (nydusd *Nydusd) GetAccessPatternMetrics(id string) ([]AccessPatternMetrics, error) { + transport := &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 5 * time.Second, + } + return dialer.DialContext(ctx, "unix", nydusd.APISockPath) + }, + } + + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: transport, + } + + args := "" + if len(id) > 0 { + args += "?id=" + id + } + + resp, err := client.Get(fmt.Sprintf("http://unix/api/v1/metrics/pattern%s", args)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if strings.Contains(string(body), "Pattern(Metrics(Stats(NoCounter)))") { + return nil, nil + } + + var info []AccessPatternMetrics + if err = json.Unmarshal(body, &info); err != nil { + return nil, err + } + + return info, nil +} + +func (nydusd *Nydusd) GetBlobCacheMetrics(id string) (*BlobCacheMetrics, error) { + + transport := &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 5 * time.Second, + } + return dialer.DialContext(ctx, "unix", nydusd.APISockPath) + }, + } + + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: transport, + } + + args := "" + if len(id) > 0 { + args += "?id=" + id + } + + resp, err := client.Get(fmt.Sprintf("http://unix/api/v1/metrics/blobcache%s", args)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var info BlobCacheMetrics + if err = json.Unmarshal(body, &info); err != nil { + return nil, err + } + + return &info, nil +} + +func (nydusd *Nydusd) GetInflightMetrics() (*InflightMetrics, error) { + + transport := &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 5 * time.Second, + } + return dialer.DialContext(ctx, "unix", nydusd.APISockPath) + }, + } + + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: transport, + } + + resp, err := client.Get("http://unix/api/v1/metrics/inflight") + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNoContent { + return nil, nil + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var info InflightMetrics + if err = json.Unmarshal(body, &info); err != nil { + return nil, err + } + + return &info, err +}