feat: adding support to io.cost setting (#496)

* feat: adding support to io.cost setting

Signed-off-by: Jianyu Sun <sunjianyu@bytedance.com>
Signed-off-by: Robin Lu <robin.lu@bytedance.com>

* feat(io.cost): adding rollback method

Signed-off-by: Robin Lu <robin.lu@bytedance.com>

* iocost: using default io.cost.qos

for now, only HDD was supported.

Signed-off-by: Robin Lu <robin.lu@bytedance.com>

* iocost: using default io.cost.model

default io.cost.model applied to HDD disks

Signed-off-by: Robin Lu <robin.lu@bytedance.com>

---------

Signed-off-by: Jianyu Sun <sunjianyu@bytedance.com>
Signed-off-by: Robin Lu <robin.lu@bytedance.com>

cmd/katalyst-agent/app/options/qrm/io_plugin.go

@@ -28,6 +28,8 @@ type IOOptions struct {
 	WritebackThrottlingOption // option for writeback throttling, it determin the recycling speed of dirty memory.
 	// TO-DO
 	//DirtyThrottlingOption // option for dirty throttling, it determin the global watermark of dirty memory.
+
+	IOCostOption
 }
 
 type WritebackThrottlingOption struct {
@@ -36,6 +38,12 @@ type WritebackThrottlingOption struct {
 	WBTValueSSD      int
 }
 
+type IOCostOption struct {
+	EnableSettingIOCost   bool
+	IOCostQoSConfigFile   string
+	IOCostModelConfigFile string
+}
+
 func NewIOOptions() *IOOptions {
 	return &IOOptions{
 		PolicyName: "static",
@@ -44,6 +52,11 @@ func NewIOOptions() *IOOptions {
 			WBTValueHDD:      75000,
 			WBTValueSSD:      2000,
 		},
+		IOCostOption: IOCostOption{
+			EnableSettingIOCost:   false,
+			IOCostQoSConfigFile:   "",
+			IOCostModelConfigFile: "",
+		},
 	}
 }
 
@@ -58,12 +71,21 @@ func (o *IOOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 		o.WBTValueHDD, "writeback throttling value for HDD")
 	fs.IntVar(&o.WBTValueSSD, "disk-wbt-ssd",
 		o.WBTValueSSD, "writeback throttling value for SSD")
+	fs.BoolVar(&o.EnableSettingIOCost, "enable-io-cost",
+		o.EnableSettingIOCost, "if set it to true, io.cost setting will be executed")
+	fs.StringVar(&o.IOCostQoSConfigFile, "io-cost-qos-config-file",
+		o.IOCostQoSConfigFile, "the absolute path of io.cost.qos qos config file")
+	fs.StringVar(&o.IOCostModelConfigFile, "io-cost-model-config-file",
+		o.IOCostModelConfigFile, "the absolute path of io.cost.model qos config file")
 }
 
 func (o *IOOptions) ApplyTo(conf *qrmconfig.IOQRMPluginConfig) error {
 	conf.PolicyName = o.PolicyName
 	conf.EnableSettingWBT = o.EnableSettingWBT
 	conf.WBTValueHDD = o.WBTValueHDD
 	conf.WBTValueSSD = o.WBTValueSSD
+	conf.EnableSettingIOCost = o.EnableSettingIOCost
+	conf.IOCostQoSConfigFile = o.IOCostQoSConfigFile
+	conf.IOCostModelConfigFile = o.IOCostModelConfigFile
 	return nil
 }

pkg/agent/qrm-plugins/io/handlers/iocost/const.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package iocost
+
+const EnableSetIOCostPeriodicalHandlerName = "SetIOCost"
+
+type DevModel string
+
+const (
+	DevModelDefault      DevModel = "default"
+	DevModelDefaultHDD   DevModel = "default_hdd"
+	DevModelVirtualdisk  DevModel = "virtualdisk"
+	DevModelNbd          DevModel = "nbd"
+	DevModelLoop         DevModel = "loop"
+	DevModelBcache       DevModel = "bcache"
+	DevModelDeviceMapper DevModel = "dm"
+
+	IOStatMetricCostVrate = "cost.vrate"
+
+	MetricNameIOCostVrate = "iocost_vrate"
+
+	queueRotationalFilePattern = "/sys/block/%s/queue/rotational"
+)
+
+type DeviceType int
+
+const (
+	HDD DeviceType = iota
+	SSD
+	Unknown
+)

pkg/agent/qrm-plugins/io/handlers/iocost/iocost_linux.go

@@ -0,0 +1,282 @@
+//go:build linux
+// +build linux
+
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package iocost
+
+import (
+	"fmt"
+	"path/filepath"
+	"strconv"
+	"sync"
+
+	"github.com/kubewharf/katalyst-core/pkg/config"
+	coreconfig "github.com/kubewharf/katalyst-core/pkg/config"
+	dynamicconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/dynamic"
+	"github.com/kubewharf/katalyst-core/pkg/metaserver"
+	"github.com/kubewharf/katalyst-core/pkg/metrics"
+	"github.com/kubewharf/katalyst-core/pkg/util/cgroup/common"
+	cgcommon "github.com/kubewharf/katalyst-core/pkg/util/cgroup/common"
+	"github.com/kubewharf/katalyst-core/pkg/util/cgroup/manager"
+	"github.com/kubewharf/katalyst-core/pkg/util/general"
+)
+
+var (
+	initializeOnce   sync.Once
+	ioCgroupRootPath = cgcommon.GetCgroupRootPath(cgcommon.CgroupSubsysIO)
+)
+
+func applyIOCostModelWithDefault(ioCostModelConfigs map[DevModel]*common.IOCostModelData, devsIDToModel map[string]DevModel) {
+	curDevIDToIOCostModelData, err := manager.GetIOCostModelWithAbsolutePath(ioCgroupRootPath)
+	if err != nil {
+		general.Errorf("GetIOCostModelWithAbsolutePath failed with error: %v", err)
+		return
+	}
+
+	for devID := range devsIDToModel {
+		var expectedModelData, curModelData *common.IOCostModelData
+
+		expectedModelData = ioCostModelConfigs[DevModelDefault]
+		if expectedModelData == nil {
+			general.Errorf("there is no expected io cost Model Data for devID: %s", devID)
+			continue
+		}
+
+		curModelData = curDevIDToIOCostModelData[devID]
+		if curModelData == nil {
+			general.Errorf("there is no current io cost Model Data for devID: %s", devID)
+			continue
+		}
+
+		if (*curModelData) != (*expectedModelData) {
+			err = manager.ApplyIOCostModelWithAbsolutePath(ioCgroupRootPath, devID, expectedModelData)
+			if err != nil {
+				general.Errorf("ApplyIOCostModelWithAbsolutePath for devID: %s, failed with error: %v",
+					devID, err)
+			} else {
+				general.Infof("ApplyIOCostModelWithAbsolutePath for devID: %s successfully:%v ",
+					devID, expectedModelData)
+			}
+		} else {
+			general.Infof("modelData isn't changed, skip ApplyIOCostModelWithAbsolutePath for devID: %s, modelData=%v ",
+				devID, expectedModelData)
+		}
+	}
+}
+
+func reportDevicesIOCostVrate(emitter metrics.MetricEmitter) {
+	devIDToIOStat, err := manager.GetIOStatWithAbsolutePath(ioCgroupRootPath)
+	if err != nil {
+		general.Errorf("GetIOStatWithAbsolutePath failed with error: %v", err)
+		return
+	}
+
+	for devID, ioStat := range devIDToIOStat {
+		if valueStr, found := ioStat[IOStatMetricCostVrate]; found {
+			valueFloat64, err := strconv.ParseFloat(valueStr, 64)
+			if err != nil {
+				general.Errorf("%s value: %s is invalid for devID: %s",
+					IOStatMetricCostVrate, valueStr, devID)
+				continue
+			}
+
+			devName, found, err := getDeviceNameFromID(devID)
+			if err != nil {
+				general.Errorf("getDeviceNameFromID: %s failed with error: %v",
+					devID, err)
+				continue
+			} else if !found {
+				general.Errorf("no device name found for device id: %s", devID)
+				continue
+			}
+
+			_ = emitter.StoreFloat64(MetricNameIOCostVrate, valueFloat64,
+				metrics.MetricTypeNameRaw, metrics.MetricTag{
+					Key: "device_name",
+					Val: devName,
+				})
+		}
+	}
+}
+
+func disableIOCost(conf *config.Configuration) {
+	if !cgcommon.CheckCgroup2UnifiedMode() {
+		return
+	}
+
+	devIDToIOCostQoSData, err := manager.GetIOCostQoSWithAbsolutePath(ioCgroupRootPath)
+	if err != nil {
+		general.Errorf("GetIOCostQoSWithAbsolutePath failed with error: %v in Init", err)
+	}
+
+	disabledIOCostQoSData := &cgcommon.IOCostQoSData{Enable: 0}
+	for devID, ioCostQoSData := range devIDToIOCostQoSData {
+		if ioCostQoSData == nil {
+			general.Warningf("nil ioCostQoSData")
+			continue
+		} else if ioCostQoSData.Enable == 0 {
+			general.Warningf("devID: %s ioCostQoS is already disabled", devID)
+			continue
+		}
+
+		err = manager.ApplyIOCostQoSWithAbsolutePath(ioCgroupRootPath, devID, disabledIOCostQoSData)
+		if err != nil {
+			general.Errorf("ApplyIOCostQoSWithAbsolutePath for devID: %s, failed with error: %v", devID, err)
+		} else {
+			general.Infof("disable ioCostQoS for devID: %s successfully", devID)
+		}
+	}
+}
+
+func applyIOCostQoSWithDefault(
+	ioCostQoSConfigs map[DevModel]*common.IOCostQoSData,
+	devsIDToModel map[string]DevModel,
+) {
+	for devID := range devsIDToModel {
+
+		// checking device type: isHDD?
+		devName, found, err := getDeviceNameFromID(devID)
+		if err != nil {
+			general.Errorf("getDeviceNameFromID: %s failed with error: %v", devID, err)
+			continue
+		} else if !found {
+			general.Errorf("no device name found for device id: %s", devID)
+			continue
+		}
+
+		rotationalFile := filepath.Clean(fmt.Sprintf(queueRotationalFilePattern, devName))
+		deviceType, err := getDeviceType(devName, rotationalFile)
+		if err != nil {
+			general.Errorf("checking device %v failed, error:%v", devName, err)
+			continue
+		}
+
+		var defaultConfig DevModel
+		switch deviceType {
+		case HDD:
+			defaultConfig = DevModelDefaultHDD
+		case Unknown:
+			general.Warningf("for now, only HDD were supported, device:%v.", devName)
+			continue
+		}
+
+		expectedQoSData := ioCostQoSConfigs[defaultConfig]
+		if expectedQoSData == nil {
+			general.Errorf("there is no default io cost QoS Data for devID: %s", devID)
+			continue
+		}
+		err = manager.ApplyIOCostQoSWithAbsolutePath(ioCgroupRootPath, devID, expectedQoSData)
+		if err != nil {
+			general.Errorf("ApplyIOCostQoSWithAbsolutePath for devID: %s, failed with error: %v",
+				devID, err)
+		}
+	}
+}
+
+func applyIOCostConfig(conf *config.Configuration, emitter metrics.MetricEmitter) {
+	if !conf.EnableSettingIOCost {
+		general.Infof("IOCostControl disabled, skip applyIOCostConfig")
+		return
+	} else if conf.IOCostQoSConfigFile == "" || conf.IOCostModelConfigFile == "" {
+		general.Errorf("IOCostQoSConfigFile or IOCostQoSConfigFile not configured")
+		return
+	}
+
+	ioCostQoSConfigs := make(map[DevModel]*common.IOCostQoSData)
+	err := general.LoadJsonConfig(conf.IOCostQoSConfigFile, &ioCostQoSConfigs)
+
+	if err != nil {
+		general.Errorf("load IOCostQoSConfigs failed with error: %v", err)
+		return
+	}
+
+	ioCostModelConfigs := make(map[DevModel]*common.IOCostModelData)
+	err = general.LoadJsonConfig(conf.IOCostModelConfigFile, &ioCostModelConfigs)
+
+	if err != nil {
+		general.Errorf("load IOCostModelConfigs failed with error: %v", err)
+		return
+	}
+
+	var targetDeviceNames []string
+
+	targetDeviceNames, err = getAllDeviceNames()
+
+	if err != nil {
+		general.Errorf("get targetDevices with error: %v", err)
+		return
+	}
+
+	general.Infof("targetDeviceNames: %+v to apply io cost configurations", targetDeviceNames)
+
+	if len(targetDeviceNames) == 0 {
+		general.Warningf("empty targetDeviceNames")
+		return
+	}
+
+	devsIDToModel, err := getDevicesIdToModel(targetDeviceNames)
+
+	if err != nil {
+		general.Errorf("getDevicesIdToModel failed with error: %v", err)
+		return
+	}
+
+	applyIOCostQoSWithDefault(ioCostQoSConfigs, devsIDToModel)
+	applyIOCostModelWithDefault(ioCostModelConfigs, devsIDToModel)
+}
+
+func SetIOCost(conf *coreconfig.Configuration,
+	_ interface{},
+	_ *dynamicconfig.DynamicAgentConfiguration,
+	emitter metrics.MetricEmitter,
+	metaServer *metaserver.MetaServer) {
+	general.Infof("called")
+
+	if conf == nil {
+		general.Errorf("nil extraConf")
+		return
+	} else if emitter == nil {
+		general.Errorf("nil emitter")
+		return
+	} else if metaServer == nil {
+		general.Errorf("nil metaServer")
+		return
+	}
+
+	// EnableSettingIOCost featuregate.
+	if !conf.EnableSettingIOCost {
+		general.Infof("SetIOCost disabled.")
+		// If EnableSettingIOCost was disabled, we should never enable io.cost.
+		initializeOnce.Do(func() {
+			disableIOCost(conf)
+		})
+		return
+	}
+
+	if !cgcommon.CheckCgroup2UnifiedMode() {
+		general.Infof("not in cgv2 environment, skip IOAsyncTaskFunc")
+		return
+	}
+
+	initializeOnce.Do(func() {
+		disableIOCost(conf)
+		applyIOCostConfig(conf, emitter)
+	})
+
+	reportDevicesIOCostVrate(emitter)
+}