From 4a754b33638763fece6b6c91447d7a07e7efdf3f Mon Sep 17 00:00:00 2001 From: xixi2 <2821893735@qq.com> Date: Fri, 30 Aug 2024 15:38:13 +0800 Subject: [PATCH] add log cache evict --- .../app/options/qrm/memory_plugin.go | 50 +++++ pkg/agent/qrm-plugins/memory/consts/consts.go | 1 + .../memory/dynamicpolicy/policy.go | 18 ++ .../handlers/logcache/cache_evictor_linux.go | 85 ++++++++ .../logcache/cache_evictor_linux_test.go | 81 ++++++++ .../logcache/cache_evictor_unsupported.go | 23 +++ .../memory/handlers/logcache/interface.go | 29 +++ .../memory/handlers/logcache/manager.go | 192 ++++++++++++++++++ .../memory/handlers/logcache/manager_test.go | 86 ++++++++ pkg/config/agent/qrm/memory_plugin.go | 23 +++ 10 files changed, 588 insertions(+) create mode 100644 pkg/agent/qrm-plugins/memory/handlers/logcache/cache_evictor_linux.go create mode 100644 pkg/agent/qrm-plugins/memory/handlers/logcache/cache_evictor_linux_test.go create mode 100644 pkg/agent/qrm-plugins/memory/handlers/logcache/cache_evictor_unsupported.go create mode 100644 pkg/agent/qrm-plugins/memory/handlers/logcache/interface.go create mode 100644 pkg/agent/qrm-plugins/memory/handlers/logcache/manager.go create mode 100644 pkg/agent/qrm-plugins/memory/handlers/logcache/manager_test.go diff --git a/cmd/katalyst-agent/app/options/qrm/memory_plugin.go b/cmd/katalyst-agent/app/options/qrm/memory_plugin.go index 64f16160d..2b7abc744 100644 --- a/cmd/katalyst-agent/app/options/qrm/memory_plugin.go +++ b/cmd/katalyst-agent/app/options/qrm/memory_plugin.go @@ -17,6 +17,8 @@ limitations under the License. package qrm import ( + "time" + cliflag "k8s.io/component-base/cli/flag" qrmconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/qrm" @@ -33,6 +35,7 @@ type MemoryOptions struct { OOMPriorityPinnedMapAbsPath string SockMemOptions + LogCacheOptions } type SockMemOptions struct { @@ -43,6 +46,23 @@ type SockMemOptions struct { SetCgroupTCPMemRatio int } +type LogCacheOptions struct { + // EnableEvictingLogCache is used to enable evicting log cache by advise kernel to throw page cache for log files + EnableEvictingLogCache bool + // If the change value of the page cache between two operations exceeds this threshold, then increase the frequency of subsequent eviction operations. + HighThreshold uint64 + // If the change value of the page cache between two operations is lower than this value, then slow down the frequency of subsequent eviction operations. + LowThreshold uint64 + // The minimum time interval between two operations + MinInterval time.Duration + // The maximum time interval between two operations + MaxInterval time.Duration + // The file directory for evicting the log cache + PathList []string + // Keywords for recognizing log files + FileFilters []string +} + func NewMemoryOptions() *MemoryOptions { return &MemoryOptions{ PolicyName: "dynamic", @@ -56,6 +76,15 @@ func NewMemoryOptions() *MemoryOptions { SetGlobalTCPMemRatio: 20, // default: 20% * {host total memory} SetCgroupTCPMemRatio: 100, // default: 100% * {cgroup memory} }, + LogCacheOptions: LogCacheOptions{ + EnableEvictingLogCache: false, + HighThreshold: 30, // default: 30GB + LowThreshold: 5, // default: 5GB + MinInterval: time.Second * 600, + MaxInterval: time.Second * 60 * 60 * 2, + PathList: []string{}, + FileFilters: []string{".*\\.log.*"}, + }, } } @@ -84,6 +113,20 @@ func (o *MemoryOptions) AddFlags(fss *cliflag.NamedFlagSets) { o.SetGlobalTCPMemRatio, "limit global max tcp memory usage") fs.IntVar(&o.SetCgroupTCPMemRatio, "qrm-memory-cgroup-tcpmem-ratio", o.SetCgroupTCPMemRatio, "limit cgroup max tcp memory usage") + fs.BoolVar(&o.EnableEvictingLogCache, "enable-evicting-logcache", + o.EnableEvictingLogCache, "if set true, we will enable log cache eviction") + fs.Uint64Var(&o.HighThreshold, "qrm-memory-logcache-high-threshold", + o.HighThreshold, "high level of evicted cache memory(GB) for log files") + fs.Uint64Var(&o.LowThreshold, "qrm-memory-logcache-low-threshold", + o.LowThreshold, "low level of evicted cache memory(GB) for log files") + fs.DurationVar(&o.MinInterval, "qrm-memory-logcache-min-interval", o.MinInterval, + "the minimum interval for logcache eviction") + fs.DurationVar(&o.MaxInterval, "qrm-memory-logcache-max-interval", o.MaxInterval, + "the maximum interval for logcache eviction") + fs.StringSliceVar(&o.PathList, "qrm-memory-logcache-path-list", o.PathList, + "the absolute path list where files will be checked to evic page cache") + fs.StringSliceVar(&o.FileFilters, "qrm-memory-logcache-file-filters", + o.FileFilters, "string list to filter log files, default to *log*") } func (o *MemoryOptions) ApplyTo(conf *qrmconfig.MemoryQRMPluginConfig) error { @@ -98,5 +141,12 @@ func (o *MemoryOptions) ApplyTo(conf *qrmconfig.MemoryQRMPluginConfig) error { conf.EnableSettingSockMem = o.EnableSettingSockMem conf.SetGlobalTCPMemRatio = o.SetGlobalTCPMemRatio conf.SetCgroupTCPMemRatio = o.SetCgroupTCPMemRatio + conf.EnableEvictingLogCache = o.EnableEvictingLogCache + conf.HighThreshold = o.HighThreshold + conf.LowThreshold = o.LowThreshold + conf.MinInterval = o.MinInterval + conf.MaxInterval = o.MaxInterval + conf.PathList = o.PathList + conf.FileFilters = o.FileFilters return nil } diff --git a/pkg/agent/qrm-plugins/memory/consts/consts.go b/pkg/agent/qrm-plugins/memory/consts/consts.go index 0374f0b4e..bec95e8ec 100644 --- a/pkg/agent/qrm-plugins/memory/consts/consts.go +++ b/pkg/agent/qrm-plugins/memory/consts/consts.go @@ -33,4 +33,5 @@ const ( SetSockMem = MemoryPluginDynamicPolicyName + "_set_sock_mem" CommunicateWithAdvisor = MemoryPluginDynamicPolicyName + "_communicate_with_advisor" DropCache = MemoryPluginDynamicPolicyName + "_drop_cache" + EvictLogCache = MemoryPluginDynamicPolicyName + "_evict_log_cache" ) diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go index 125acb408..4a42d8ed2 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go @@ -41,6 +41,7 @@ import ( "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/dynamicpolicy/memoryadvisor" "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/dynamicpolicy/oom" "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/dynamicpolicy/state" + "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/handlers/logcache" "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/handlers/sockmem" "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util" "github.com/kubewharf/katalyst-core/pkg/agent/utilcomponent/periodicalhandler" @@ -156,6 +157,9 @@ type DynamicPolicy struct { oomPriorityMapPinnedPath string oomPriorityMapLock sync.Mutex oomPriorityMap *ebpf.Map + + enableEvictingLogCache bool + logCacheEvictionManager logcache.Manager } func NewDynamicPolicy(agentCtx *agent.GenericContext, conf *config.Configuration, @@ -219,6 +223,7 @@ func NewDynamicPolicy(agentCtx *agent.GenericContext, conf *config.Configuration extraControlKnobConfigs: extraControlKnobConfigs, // [TODO]: support modifying extraControlKnobConfigs by KCC enableOOMPriority: conf.EnableOOMPriority, oomPriorityMapPinnedPath: conf.OOMPriorityPinnedMapAbsPath, + enableEvictingLogCache: conf.EnableEvictingLogCache, } policyImplement.allocationHandlers = map[string]util.AllocationHandler{ @@ -263,6 +268,9 @@ func NewDynamicPolicy(agentCtx *agent.GenericContext, conf *config.Configuration memoryadvisor.RegisterControlKnobHandler(memoryadvisor.ControlKnowKeyMemoryOffloading, memoryadvisor.ControlKnobHandlerWithChecker(policyImplement.handleAdvisorMemoryOffloading)) + if policyImplement.enableEvictingLogCache { + policyImplement.logCacheEvictionManager = logcache.NewManager(conf, agentCtx.MetaServer) + } return true, &agent.PluginWrapper{GenericPlugin: pluginWrapper}, nil } @@ -392,6 +400,16 @@ func (p *DynamicPolicy) Start() (err error) { } } + if p.enableEvictingLogCache { + general.Infof("evictLogCache enabled") + err := periodicalhandler.RegisterPeriodicalHandlerWithHealthz(memconsts.EvictLogCache, + general.HealthzCheckStateNotReady, qrm.QRMMemoryPluginPeriodicalHandlerGroupName, + p.logCacheEvictionManager.EvictLogCache, 600*time.Second, healthCheckTolerationTimes) + if err != nil { + general.Errorf("evictLogCache failed, err=%v", err) + } + } + go wait.Until(func() { periodicalhandler.ReadyToStartHandlersByGroup(qrm.QRMMemoryPluginPeriodicalHandlerGroupName) }, 5*time.Second, p.stopCh) diff --git a/pkg/agent/qrm-plugins/memory/handlers/logcache/cache_evictor_linux.go b/pkg/agent/qrm-plugins/memory/handlers/logcache/cache_evictor_linux.go new file mode 100644 index 000000000..0c5fdb28f --- /dev/null +++ b/pkg/agent/qrm-plugins/memory/handlers/logcache/cache_evictor_linux.go @@ -0,0 +1,85 @@ +//go:build linux +// +build linux + +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logcache + +import ( + "fmt" + "os" + "syscall" + + "github.com/pkg/errors" + "golang.org/x/sys/unix" +) + +func EvictFileCache(filePath string, fileSizeBytes int64) error { + file, err := openFileWithRetry(filePath, os.O_RDONLY|syscall.O_NOATIME) + if err != nil { + return err + } + + defer func(file *os.File) { + _ = file.Close() + }(file) + + err = unix.Fadvise(int(file.Fd()), 0, fileSizeBytes, unix.FADV_DONTNEED) + if err != nil { + return fmt.Errorf("failed to evict page cache for file %s", filePath) + } + + return nil +} + +func openFileWithRetry(filePath string, flag int) (file *os.File, err error) { + for { + file, err = os.OpenFile(filePath, flag, 0) + if err == nil { + return file, nil + } + + if errors.Is(err, syscall.ENFILE) || errors.Is(err, syscall.EMFILE) { + if err = incrementNoFileRLimit(); err != nil { + break + } + continue + } else if errors.Is(err, syscall.EPERM) { + flag = flag & ^syscall.O_NOATIME + continue + } else { + break + } + } + return file, err +} + +func incrementNoFileRLimit() error { + var rLimit syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) + if err != nil { + return err + } + rLimit.Cur = rLimit.Max + 1 + rLimit.Max = rLimit.Max + 1 + + err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit) + if err != nil { + return err + } + return nil +} diff --git a/pkg/agent/qrm-plugins/memory/handlers/logcache/cache_evictor_linux_test.go b/pkg/agent/qrm-plugins/memory/handlers/logcache/cache_evictor_linux_test.go new file mode 100644 index 000000000..d274459ef --- /dev/null +++ b/pkg/agent/qrm-plugins/memory/handlers/logcache/cache_evictor_linux_test.go @@ -0,0 +1,81 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logcache + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEvictFileCache(t *testing.T) { + t.Parallel() + + testcases := []struct { + name string + filePath string + expectedError bool + }{ + { + name: "empty file path", + filePath: "", + expectedError: true, + }, + { + name: "failed due to permission", + filePath: "writeonly.log", + expectedError: true, + }, + { + name: "file not exist", + filePath: "non-exit.log", + expectedError: true, + }, + } + + for _, testcase := range testcases { + tc := testcase + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + file, err := os.Lstat(tc.filePath) + if err == nil && file.Mode().IsRegular() { + err = EvictFileCache(tc.filePath, file.Size()) + } + + if (err != nil) != tc.expectedError { + t.Errorf("evict file cache error %v, got %v, error: %v", err != nil, tc.expectedError, err) + } + }) + } + + regularFile, err := os.Create("regular.log") + require.NoError(t, err) + + file, err := os.Lstat(regularFile.Name()) + if err == nil && file.Mode().IsRegular() { + err = EvictFileCache(regularFile.Name(), file.Size()) + } + if err != nil { + t.Errorf("evict file cache unexpected error: %v", err) + } + + defer func() { + _ = regularFile.Close() + _ = os.Remove(regularFile.Name()) + }() +} diff --git a/pkg/agent/qrm-plugins/memory/handlers/logcache/cache_evictor_unsupported.go b/pkg/agent/qrm-plugins/memory/handlers/logcache/cache_evictor_unsupported.go new file mode 100644 index 000000000..216b61abd --- /dev/null +++ b/pkg/agent/qrm-plugins/memory/handlers/logcache/cache_evictor_unsupported.go @@ -0,0 +1,23 @@ +//go:build !linux + +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logcache + +func EvictFileCache(_ string, _ int64) error { + return nil +} diff --git a/pkg/agent/qrm-plugins/memory/handlers/logcache/interface.go b/pkg/agent/qrm-plugins/memory/handlers/logcache/interface.go new file mode 100644 index 000000000..fd8b0b219 --- /dev/null +++ b/pkg/agent/qrm-plugins/memory/handlers/logcache/interface.go @@ -0,0 +1,29 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logcache + +import ( + coreconfig "github.com/kubewharf/katalyst-core/pkg/config" + dynamicconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/dynamic" + "github.com/kubewharf/katalyst-core/pkg/metaserver" + "github.com/kubewharf/katalyst-core/pkg/metrics" +) + +type Manager interface { + EvictLogCache(_ *coreconfig.Configuration, _ interface{}, _ *dynamicconfig.DynamicAgentConfiguration, + emitter metrics.MetricEmitter, metaServer *metaserver.MetaServer) +} diff --git a/pkg/agent/qrm-plugins/memory/handlers/logcache/manager.go b/pkg/agent/qrm-plugins/memory/handlers/logcache/manager.go new file mode 100644 index 000000000..41bbeaa3d --- /dev/null +++ b/pkg/agent/qrm-plugins/memory/handlers/logcache/manager.go @@ -0,0 +1,192 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logcache + +import ( + "io/fs" + "path/filepath" + "regexp" + "time" + + "k8s.io/apimachinery/pkg/util/errors" + + memconsts "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/consts" + coreconfig "github.com/kubewharf/katalyst-core/pkg/config" + dynamicconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/dynamic" + "github.com/kubewharf/katalyst-core/pkg/consts" + "github.com/kubewharf/katalyst-core/pkg/metaserver" + "github.com/kubewharf/katalyst-core/pkg/metrics" + "github.com/kubewharf/katalyst-core/pkg/util/general" +) + +func canFilePathMatch(filePath string, patternStrings []string) bool { + for _, str := range patternStrings { + re := regexp.MustCompile(str) + if re.MatchString(filePath) { + return true + } + } + return false +} + +type fileCacheEvictionManager struct { + metaServer *metaserver.MetaServer + + highThresholdGB uint64 + lowThresholdGB uint64 + + minInterval time.Duration + maxInterval time.Duration + curInterval time.Duration + + lastEvictTime *time.Time + + pathList []string + + fileFilters []string +} + +func (e *fileCacheEvictionManager) shouldEvictFile(filePath string) bool { + return canFilePathMatch(filePath, e.fileFilters) +} + +func (e *fileCacheEvictionManager) evictWalk(path string) error { + return filepath.Walk(path, func(path string, file fs.FileInfo, err error) error { + if err != nil { + return err + } + + if !file.Mode().IsRegular() { + return nil + } + + if e.shouldEvictFile(path) && file.Size() > 0 { + if err = EvictFileCache(path, file.Size()); err != nil { + return err + } + } + return nil + }) +} + +func (e *fileCacheEvictionManager) determineNextInterval(evictedGB uint64, elapsedTime time.Duration) { + interval := e.curInterval + if evictedGB > e.highThresholdGB { + interval -= e.minInterval + } else if evictedGB < e.lowThresholdGB { + interval += e.minInterval + } + if interval < elapsedTime { + interval = elapsedTime * 4 + } + + if interval < e.minInterval { + interval = e.minInterval + } else if interval > e.maxInterval { + interval = e.maxInterval + } + + e.curInterval = interval +} + +func (e *fileCacheEvictionManager) getCachedMemoryInGB() uint64 { + m, err := e.metaServer.GetNodeMetric(consts.MetricMemPageCacheSystem) + if err != nil { + return 0 + } + + cachedBytes := uint64(m.Value) + cachedGB := cachedBytes >> 30 + return cachedGB +} + +func (e *fileCacheEvictionManager) waitForNodeMetricsSync() { + time.Sleep(time.Second * 30) +} + +func (e *fileCacheEvictionManager) doEviction() error { + now := time.Now() + + runAtOnce := true + if e.lastEvictTime != nil { + lastEvictTime := *e.lastEvictTime + expectedToRunAt := lastEvictTime.Add(e.curInterval) + if now.Before(expectedToRunAt) { + runAtOnce = false + } + } + + if !runAtOnce { + return nil + } + + beforeEvictedGB := e.getCachedMemoryInGB() + + var errList []error + for _, path := range e.pathList { + if err := e.evictWalk(path); err != nil { + general.Errorf("walk path %s error: %v", path, err) + errList = append(errList, err) + } + } + + elapsedTime := time.Since(now) + + e.waitForNodeMetricsSync() + afterEvictedGB := e.getCachedMemoryInGB() + var evictedGB uint64 = 0 + if beforeEvictedGB > afterEvictedGB { + evictedGB = beforeEvictedGB - afterEvictedGB + } + + e.lastEvictTime = &now + e.determineNextInterval(evictedGB, elapsedTime) + + general.Infof("file cache eviction finished at %v, cost %v, cached memory from %d GB to %d GB, evicted %d GB, will run at %v later", + e.lastEvictTime.String(), elapsedTime.String(), beforeEvictedGB, afterEvictedGB, evictedGB, e.curInterval) + + return errors.NewAggregate(errList) +} + +func (e *fileCacheEvictionManager) EvictLogCache(_ *coreconfig.Configuration, + _ interface{}, _ *dynamicconfig.DynamicAgentConfiguration, emitter metrics.MetricEmitter, metaServer *metaserver.MetaServer, +) { + var err error + defer func() { + _ = general.UpdateHealthzStateByError(memconsts.EvictLogCache, err) + }() + + err = e.doEviction() +} + +func NewManager(conf *coreconfig.Configuration, metaServer *metaserver.MetaServer) Manager { + e := &fileCacheEvictionManager{ + metaServer: metaServer, + highThresholdGB: conf.HighThreshold, + lowThresholdGB: conf.LowThreshold, + minInterval: conf.MinInterval, + maxInterval: conf.MaxInterval, + curInterval: conf.MinInterval, + lastEvictTime: nil, + pathList: conf.PathList, + fileFilters: conf.FileFilters, + } + + general.Infof("log cache manager: highThreshold: %v, lowThreshold: %v, minInterval: %v, maxInterval: %v, pathList: %v", + conf.HighThreshold, conf.LowThreshold, conf.MinInterval, conf.MaxInterval, conf.PathList) + return e +} diff --git a/pkg/agent/qrm-plugins/memory/handlers/logcache/manager_test.go b/pkg/agent/qrm-plugins/memory/handlers/logcache/manager_test.go new file mode 100644 index 000000000..895970eca --- /dev/null +++ b/pkg/agent/qrm-plugins/memory/handlers/logcache/manager_test.go @@ -0,0 +1,86 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logcache + +import ( + "io/ioutil" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + + katalystbase "github.com/kubewharf/katalyst-core/cmd/base" + "github.com/kubewharf/katalyst-core/cmd/katalyst-agent/app/options" + "github.com/kubewharf/katalyst-core/pkg/config" + "github.com/kubewharf/katalyst-core/pkg/metaserver" + "github.com/kubewharf/katalyst-core/pkg/metrics" +) + +func generateTestConfiguration(t *testing.T, checkpointDir, stateFileDir string) *config.Configuration { + conf, err := options.NewOptions().Config() + require.NoError(t, err) + require.NotNil(t, conf) + + conf.GenericSysAdvisorConfiguration.StateFileDirectory = stateFileDir + conf.MetaServerConfiguration.CheckpointManagerDir = checkpointDir + conf.EnableMetricsFetcher = false + conf.HighThreshold = 30 + conf.LowThreshold = 10 + conf.MaxInterval = time.Second * 120 + conf.MinInterval = time.Second + conf.PathList = []string{"."} + conf.FileFilters = []string{".*.log.*"} + return conf +} + +func TestFileCacheEvictionManager_EvictLogCache(t *testing.T) { + t.Parallel() + + ckDir, err := ioutil.TempDir("", "checkpoint") + require.NoError(t, err) + defer func() { _ = os.RemoveAll(ckDir) }() + + sfDir, err := ioutil.TempDir("", "statefile") + require.NoError(t, err) + defer func() { _ = os.RemoveAll(sfDir) }() + + conf := generateTestConfiguration(t, ckDir, sfDir) + + genericCtx, err := katalystbase.GenerateFakeGenericContext([]runtime.Object{}) + require.NoError(t, err) + + metaServer, err := metaserver.NewMetaServer(genericCtx.Client, metrics.DummyMetrics{}, conf) + assert.NoError(t, err) + + mgr := NewManager(conf, metaServer) + + stopCh := make(chan struct{}) + go func() { + time.Sleep(time.Minute * 4) + stopCh <- struct{}{} + }() + + go wait.Until(func() { + var extraConf interface{} + mgr.EvictLogCache(conf, extraConf, conf.DynamicAgentConfiguration, metrics.DummyMetrics{}, metaServer) + }, time.Second, stopCh) + <-stopCh +} diff --git a/pkg/config/agent/qrm/memory_plugin.go b/pkg/config/agent/qrm/memory_plugin.go index 02004d773..e775856ea 100644 --- a/pkg/config/agent/qrm/memory_plugin.go +++ b/pkg/config/agent/qrm/memory_plugin.go @@ -16,6 +16,10 @@ limitations under the License. package qrm +import ( + "time" +) + type MemoryQRMPluginConfig struct { // PolicyName is used to switch between several strategies PolicyName string @@ -36,6 +40,8 @@ type MemoryQRMPluginConfig struct { // SockMemQRMPluginConfig: the configuration for sockmem limitation in cgroup and host level SockMemQRMPluginConfig + // LogCacheQRMPluginConfig: the configuration for logcache evicting + LogCacheQRMPluginConfig } type SockMemQRMPluginConfig struct { @@ -47,6 +53,23 @@ type SockMemQRMPluginConfig struct { SetCgroupTCPMemRatio int } +type LogCacheQRMPluginConfig struct { + // EnableEvictingLogCache is used to enable evicting log cache by advise kernel to throw page cache for log files + EnableEvictingLogCache bool + // If the change value of the page cache between two operations exceeds this threshold, then increase the frequency of subsequent eviction operations. + HighThreshold uint64 + // If the change value of the page cache between two operations is lower than this value, then slow down the frequency of subsequent eviction operations. + LowThreshold uint64 + // The minimum time interval between two operations + MinInterval time.Duration + // The maximum time interval between two operations + MaxInterval time.Duration + // The file directory for evicting the log cache + PathList []string + // Keywords for recognizing log files + FileFilters []string +} + func NewMemoryQRMPluginConfig() *MemoryQRMPluginConfig { return &MemoryQRMPluginConfig{} }