Skip to content

Commit

Permalink
feat: adding support to io.cost setting for SSD
Browse files Browse the repository at this point in the history 
Signed-off-by: Robin Lu <robin.lu@bytedance.com>
  • Loading branch information
@lubinszARM
lubinszARM committed Mar 26, 2024
1 parent 2a50e2c commit 61fed04
Show file tree
Hide file tree
Showing 7 changed files with 88 additions and 152 deletions.
17 changes: 6 additions & 11 deletions cmd/katalyst-agent/app/options/qrm/io_plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,9 @@ type WritebackThrottlingOption struct {
}

type IOCostOption struct {
EnableSettingIOCost bool
EnableSettingIOCostHDDOnly bool
IOCostQoSConfigFile string
IOCostModelConfigFile string
EnableSettingIOCost bool
IOCostQoSConfigFile string
IOCostModelConfigFile string
}

func NewIOOptions() *IOOptions {
Expand All @@ -54,10 +53,9 @@ func NewIOOptions() *IOOptions {
WBTValueSSD: 2000,
},
IOCostOption: IOCostOption{
EnableSettingIOCost: false,
EnableSettingIOCostHDDOnly: false,
IOCostQoSConfigFile: "",
IOCostModelConfigFile: "",
EnableSettingIOCost: false,
IOCostQoSConfigFile: "",
IOCostModelConfigFile: "",
},
}
}
Expand All @@ -75,8 +73,6 @@ func (o *IOOptions) AddFlags(fss *cliflag.NamedFlagSets) {
o.WBTValueSSD, "writeback throttling value for SSD")
fs.BoolVar(&o.EnableSettingIOCost, "enable-io-cost",
o.EnableSettingIOCost, "if set it to true, io.cost setting will be executed")
fs.BoolVar(&o.EnableSettingIOCostHDDOnly, "enable-io-cost-hdd-only",
o.EnableSettingIOCostHDDOnly, "if set it to true, only io.cost setting for HDD will be executed")
fs.StringVar(&o.IOCostQoSConfigFile, "io-cost-qos-config-file",
o.IOCostQoSConfigFile, "the absolute path of io.cost.qos qos config file")
fs.StringVar(&o.IOCostModelConfigFile, "io-cost-model-config-file",
Expand All @@ -89,7 +85,6 @@ func (o *IOOptions) ApplyTo(conf *qrmconfig.IOQRMPluginConfig) error {
conf.WBTValueHDD = o.WBTValueHDD
conf.WBTValueSSD = o.WBTValueSSD
conf.EnableSettingIOCost = o.EnableSettingIOCost
conf.EnableSettingIOCostHDDOnly = o.EnableSettingIOCostHDDOnly
conf.IOCostQoSConfigFile = o.IOCostQoSConfigFile
conf.IOCostModelConfigFile = o.IOCostModelConfigFile
return nil
Expand Down
9 changes: 9 additions & 0 deletions pkg/agent/qrm-plugins/io/handlers/iocost/const.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ type DevModel string

const (
DevModelDefault DevModel = "default"
DevModelDefaultHDD DevModel = "default_hdd"
DevModelVirtualdisk DevModel = "virtualdisk"
DevModelNbd DevModel = "nbd"
DevModelLoop DevModel = "loop"
Expand All @@ -34,3 +35,11 @@ const (

queueRotationalFilePattern = "/sys/block/%s/queue/rotational"
)

type DeviceType int

const (
HDD DeviceType = iota
SSD
Unknown
)
62 changes: 31 additions & 31 deletions pkg/agent/qrm-plugins/io/handlers/iocost/iocost_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,57 +139,57 @@ func disableIOCost(conf *config.Configuration) {
continue
}

err = applyIOCostQoSWithAbsolutePath(ioCgroupRootPath, devID, disabledIOCostQoSData, false)
err = manager.ApplyIOCostQoSWithAbsolutePath(ioCgroupRootPath, devID, disabledIOCostQoSData)
if err != nil {
general.Errorf("applyIOCostQoSWithAbsolutePath for devID: %s, failed with error: %v", devID, err)
general.Errorf("ApplyIOCostQoSWithAbsolutePath for devID: %s, failed with error: %v", devID, err)
} else {
general.Infof("disable ioCostQoS for devID: %s successfully", devID)
}
}
}

func applyIOCostQoSWithAbsolutePath(
absCgroupPath string, devID string,
data *common.IOCostQoSData, hddOnly bool) error {
if hddOnly {
func applyIOCostQoSWithDefault(
ioCostQoSConfigs map[DevModel]*common.IOCostQoSData,
devsIDToModel map[string]DevModel,
) {
for devID := range devsIDToModel {

// checking device type: isHDD?
devName, found, err := getDeviceNameFromID(devID)
if err != nil {
return fmt.Errorf("getDeviceNameFromID: %s failed with error: %v", devID, err)
general.Errorf("getDeviceNameFromID: %s failed with error: %v", devID, err)
continue
} else if !found {
return fmt.Errorf("no device name found for device id: %s", devID)
general.Errorf("no device name found for device id: %s", devID)
continue
}

rotationalFile := filepath.Clean(fmt.Sprintf(queueRotationalFilePattern, devName))
hdd, err := isHDD(devName, rotationalFile)
deviceType, err := getDeviceType(devName, rotationalFile)
if err != nil {
return err
}
if !hdd {
return fmt.Errorf("skip none-HDD")
general.Errorf("checking device %v failed, error:%v", devName, err)
continue
}
}
return manager.ApplyIOCostQoSWithAbsolutePath(absCgroupPath, devID, data)
}

func applyIOCostQoSWithDefault(
ioCostQoSConfigs map[DevModel]*common.IOCostQoSData,
devsIDToModel map[string]DevModel,
hddOnly bool,
) {
if !hddOnly {
// TO-DO: for now, only HDD supported
return
}
var defaultConfig DevModel
switch deviceType {
case HDD:
defaultConfig = DevModelDefaultHDD
case SSD:
defaultConfig = DevModelDefault
case Unknown:
general.Errorf("for now, only HDD/SSD were supported, device:%v.", devName)
continue
}

for devID := range devsIDToModel {
expectedQoSData := ioCostQoSConfigs[DevModelDefault]
expectedQoSData := ioCostQoSConfigs[defaultConfig]
if expectedQoSData == nil {
general.Errorf("there is no default io cost QoS Data for devID: %s", devID)
continue
}
err := applyIOCostQoSWithAbsolutePath(ioCgroupRootPath, devID, expectedQoSData, hddOnly)
err = manager.ApplyIOCostQoSWithAbsolutePath(ioCgroupRootPath, devID, expectedQoSData)
if err != nil {
general.Errorf("applyIOCostQoSWithAbsolutePath for devID: %s, failed with error: %v",
general.Errorf("ApplyIOCostQoSWithAbsolutePath for devID: %s, failed with error: %v",
devID, err)
}
}
Expand Down Expand Up @@ -243,7 +243,7 @@ func applyIOCostConfig(conf *config.Configuration, emitter metrics.MetricEmitter
return
}

applyIOCostQoSWithDefault(ioCostQoSConfigs, devsIDToModel, conf.EnableSettingIOCostHDDOnly)
applyIOCostQoSWithDefault(ioCostQoSConfigs, devsIDToModel)
applyIOCostModel(ioCostModelConfigs, devsIDToModel)
}

Expand All @@ -266,7 +266,7 @@ func SetIOCost(conf *coreconfig.Configuration,
}

// EnableSettingIOCost featuregate.
if !conf.EnableSettingIOCost || !conf.EnableSettingIOCostHDDOnly {
if !conf.EnableSettingIOCost {
general.Infof("SetIOCost disabled.")
// If EnableSettingIOCost was disabled, we should never enable io.cost.
initializeOnce.Do(func() {
Expand Down
44 changes: 8 additions & 36 deletions pkg/agent/qrm-plugins/io/handlers/iocost/iocost_linux_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,7 @@ func TestSetIOCost(t *testing.T) {
QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{
IOQRMPluginConfig: &qrm.IOQRMPluginConfig{
IOCostOption: qrm.IOCostOption{
EnableSettingIOCost: true,
EnableSettingIOCostHDDOnly: true,
EnableSettingIOCost: true,
},
},
},
Expand Down Expand Up @@ -176,8 +175,7 @@ func Test_disableIOCost(t *testing.T) {
QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{
IOQRMPluginConfig: &qrm.IOQRMPluginConfig{
IOCostOption: qrm.IOCostOption{
EnableSettingIOCost: true,
EnableSettingIOCostHDDOnly: true,
EnableSettingIOCost: true,
},
},
},
Expand Down Expand Up @@ -281,10 +279,9 @@ func Test_applyIOCostConfig(t *testing.T) {
QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{
IOQRMPluginConfig: &qrm.IOQRMPluginConfig{
IOCostOption: qrm.IOCostOption{
EnableSettingIOCost: true,
EnableSettingIOCostHDDOnly: true,
IOCostModelConfigFile: "/tmp/fakeFile1",
IOCostQoSConfigFile: "/tmp/fakeFile2",
EnableSettingIOCost: true,
IOCostModelConfigFile: "/tmp/fakeFile1",
IOCostQoSConfigFile: "/tmp/fakeFile2",
},
},
},
Expand All @@ -303,10 +300,9 @@ func Test_applyIOCostConfig(t *testing.T) {
QRMPluginsConfiguration: &qrm.QRMPluginsConfiguration{
IOQRMPluginConfig: &qrm.IOQRMPluginConfig{
IOCostOption: qrm.IOCostOption{
EnableSettingIOCost: true,
EnableSettingIOCostHDDOnly: true,
IOCostModelConfigFile: "/tmp/fakeFile1",
IOCostQoSConfigFile: absPath,
EnableSettingIOCost: true,
IOCostModelConfigFile: "/tmp/fakeFile1",
IOCostQoSConfigFile: absPath,
},
},
},
Expand Down Expand Up @@ -380,27 +376,3 @@ func Test_applyIOCostModel(t *testing.T) {
})
}
}

func TestApplyIOCostQoSWithAbsolutePathIsHDDTrue(t *testing.T) {
t.Parallel()
devsIDToModel := map[string]DevModel{
"test": "default",
}
applyIOCostQoSWithDefault(nil, devsIDToModel, false)
applyIOCostQoSWithDefault(nil, devsIDToModel, true)
ioCostQoSData := common.IOCostQoSData{}
err := applyIOCostQoSWithAbsolutePath("absPath", "devID", &ioCostQoSData, true)
assert.Error(t, err)
err = applyIOCostQoSWithAbsolutePath("absPath", "devID", &ioCostQoSData, false)
assert.Error(t, err)
err = applyIOCostQoSWithAbsolutePath("absPath", "8:1", &ioCostQoSData, true)
assert.Error(t, err)
err = applyIOCostQoSWithAbsolutePath("absPath", "254:0", &ioCostQoSData, true)
assert.Error(t, err)
err = applyIOCostQoSWithAbsolutePath("absPath", "254:16", &ioCostQoSData, true)
assert.Error(t, err)
err = applyIOCostQoSWithAbsolutePath("absPath", "11:0", &ioCostQoSData, true)
assert.Error(t, err)
err = applyIOCostQoSWithAbsolutePath("absPath", "259:0", &ioCostQoSData, true)
assert.Error(t, err)
}
52 changes: 13 additions & 39 deletions pkg/agent/qrm-plugins/io/handlers/iocost/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,6 @@ import (
"github.com/kubewharf/katalyst-core/pkg/util/general"
)

const (
containerdConfigFile = "/root/tce/containerd/conf/containerd.toml"
)

func getDevicesIdToModel(deviceNames []string) (map[string]DevModel, error) {
devsIDToModel := make(map[string]DevModel, len(deviceNames))
for _, devName := range deviceNames {
Expand Down Expand Up @@ -107,43 +103,14 @@ func getDeviceNameFromID(targetDevID string) (string, bool, error) {
return "", false, nil
}

func getContainerdRootDir() (string, error) {
lines, err := general.ReadFileIntoLines(containerdConfigFile)
if err != nil {
return "", fmt.Errorf("failed to ReadLines %s, err %v", containerdConfigFile, err)
}

var rootDir string
for _, line := range lines {
cols := strings.Split(line, "=")
if len(cols) != 2 {
continue
}

if strings.TrimSpace(cols[0]) == "root" {
rootDir = strings.TrimSpace(cols[1])
rootDir = strings.TrimPrefix(rootDir, "\"")
rootDir = strings.TrimSuffix(rootDir, "\"")
rootDir = strings.TrimSpace(rootDir)
break
}
}

if rootDir == "" {
return "", fmt.Errorf("failed to find root config in %s", containerdConfigFile)
}

return rootDir, nil
}

func isHDD(deviceName, rotationalFilePath string) (bool, error) {
func getDeviceType(deviceName, rotationalFilePath string) (DeviceType, error) {
/* Check if the device name starts with "sd"
* sd means scsi devices.
* Currently, only HDD/SSD could be scsi device.
*/
// Step1, the device should be scsi device.
if !strings.HasPrefix(deviceName, "sd") {
return false, fmt.Errorf("not scsi disk")
return Unknown, fmt.Errorf("not scsi disk")
}

// Step2, if it is scsi device, then check rotational
Expand All @@ -152,12 +119,19 @@ func isHDD(deviceName, rotationalFilePath string) (bool, error) {
contents, err := ioutil.ReadFile(cleanedRotationalFilePath)
if err != nil {
if os.IsNotExist(err) {
return false, nil
return Unknown, nil
}
return false, err
return Unknown, err
}

// Parse rotational status (1 means rotational, 0 means non-rotational)
rotational := strings.TrimSpace(string(contents))
return rotational == "1", nil
switch rotational {
case "1":
return HDD, nil
case "0":
return SSD, nil
default:
return Unknown, fmt.Errorf("unknown rotational status")
}

}
Loading

0 comments on commit 61fed04

Please sign in to comment.