From 2852ea325f705f971e5ce919bfa88a75e96dca36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulik?= Date: Thu, 16 Jul 2020 20:27:12 +0200 Subject: [PATCH] Add mon groups for resctrl. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "mon_groups" can be created to monitor subsets of tasks in the CTRL_MON group that is their ancestor. More info: https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt Signed-off-by: Paweł Szulik --- libcontainer/intelrdt/intelrdt.go | 262 ++++++++++++++++--------- libcontainer/intelrdt/intelrdt_test.go | 39 ++++ update.go | 1 + 3 files changed, 207 insertions(+), 95 deletions(-) diff --git a/libcontainer/intelrdt/intelrdt.go b/libcontainer/intelrdt/intelrdt.go index 5b19d55a2d7..bf9a42c824a 100644 --- a/libcontainer/intelrdt/intelrdt.go +++ b/libcontainer/intelrdt/intelrdt.go @@ -12,6 +12,8 @@ import ( "strings" "sync" + "github.com/sirupsen/logrus" + "github.com/opencontainers/runc/libcontainer/configs" ) @@ -167,10 +169,17 @@ type IntelRdtManager struct { Config *configs.Config Id string Path string + Type string } const ( - IntelRdtTasks = "tasks" + IntelRdtTasks = "tasks" + MonitoringGroupDirectoryName = "mon_groups" + + // See "Resource alloc and monitor groups" part here: + // https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt + MonitoringGroupType = "MON" + ControlGroupType = "CTRL_MON" ) var ( @@ -548,26 +557,47 @@ func GetIntelRdtPath(id string) (string, error) { return path, nil } +// Get the 'container_id" path in Intel RDT "monitoring group" filesystem. +func getMonGroupIntelRdtPath(id string) (string, error) { + rootPath, err := getIntelRdtRoot() + if err != nil { + return "", err + } + + path := filepath.Join(rootPath, MonitoringGroupDirectoryName, id) + return path, nil +} + // Applies Intel RDT configuration to the process with the specified pid func (m *IntelRdtManager) Apply(pid int) (err error) { - // If intelRdt is not specified in config, we do nothing - if m.Config.IntelRdt == nil { + switch m.Type { + case ControlGroupType: + // If intelRdt is not specified in config, we do nothing + if m.Config.IntelRdt == nil { + return nil + } + rdtData, err := getIntelRdtData(m.Config, pid) + if err != nil && !IsNotFound(err) { + return err + } + + m.mu.Lock() + defer m.mu.Unlock() + path, err := rdtData.join(m.Id) + if err != nil { + return err + } + + m.Path = path return nil - } - d, err := getIntelRdtData(m.Config, pid) - if err != nil && !IsNotFound(err) { - return err - } - m.mu.Lock() - defer m.mu.Unlock() - path, err := d.join(m.Id) - if err != nil { - return err + case MonitoringGroupType: + m.mu.Lock() + defer m.mu.Unlock() + return WriteIntelRdtTasks(m.Path, pid) } - m.Path = path - return nil + return fmt.Errorf("couldn't recognize resctrl type: %v", m.Type) } // Destroys the Intel RDT 'container_id' group @@ -584,94 +614,124 @@ func (m *IntelRdtManager) Destroy() error { // Returns Intel RDT path to save in a state file and to be able to // restore the object later func (m *IntelRdtManager) GetPath() string { + var err error if m.Path == "" { - m.Path, _ = GetIntelRdtPath(m.Id) + switch m.Type { + case ControlGroupType: + m.Path, err = GetIntelRdtPath(m.Id) + if err != nil { + logrus.Errorf("couldn't obtain Resctrl control group path for manager with id %v: %v", m.Id, err) + } + case MonitoringGroupType: + flattedContainerID := strings.Replace(m.Id, "/", "-", -1) + m.Path, err = getMonGroupIntelRdtPath(flattedContainerID) + if err != nil { + logrus.Errorf("couldn't obtain Resctrl monitoring group path for manager with id %v: %v", m.Id, err) + } + } } return m.Path } // Returns statistics for Intel RDT func (m *IntelRdtManager) GetStats() (*Stats, error) { - // If intelRdt is not specified in config - if m.Config.IntelRdt == nil { - return nil, nil - } - m.mu.Lock() defer m.mu.Unlock() stats := NewStats() - rootPath, err := getIntelRdtRoot() - if err != nil { - return nil, err - } - // The read-only L3 cache and memory bandwidth schemata in root - tmpRootStrings, err := getIntelRdtParamString(rootPath, "schemata") - if err != nil { - return nil, err - } - schemaRootStrings := strings.Split(tmpRootStrings, "\n") - - // The L3 cache and memory bandwidth schemata in 'container_id' group - containerPath := m.GetPath() - tmpStrings, err := getIntelRdtParamString(containerPath, "schemata") - if err != nil { - return nil, err - } - schemaStrings := strings.Split(tmpStrings, "\n") + switch m.Type { + case ControlGroupType: + containerPath := m.GetPath() - if IsCatEnabled() { - // The read-only L3 cache information - l3CacheInfo, err := getL3CacheInfo() + err := getMonitoringStats(containerPath, stats) if err != nil { return nil, err } - stats.L3CacheInfo = l3CacheInfo - // The read-only L3 cache schema in root - for _, schemaRoot := range schemaRootStrings { - if strings.Contains(schemaRoot, "L3") { - stats.L3CacheSchemaRoot = strings.TrimSpace(schemaRoot) + // If intelRdt is not specified in config + if m.Config != nil { + if m.Config.IntelRdt == nil { + return nil, nil } - } - // The L3 cache schema in 'container_id' group - for _, schema := range schemaStrings { - if strings.Contains(schema, "L3") { - stats.L3CacheSchema = strings.TrimSpace(schema) + rootPath, err := getIntelRdtRoot() + if err != nil { + return nil, err } - } - } + // The read-only L3 cache and memory bandwidth schemata in root + tmpRootStrings, err := getIntelRdtParamString(rootPath, "schemata") + if err != nil { + return nil, err + } + schemaRootStrings := strings.Split(tmpRootStrings, "\n") - if IsMbaEnabled() { - // The read-only memory bandwidth information - memBwInfo, err := getMemBwInfo() - if err != nil { - return nil, err - } - stats.MemBwInfo = memBwInfo + // The L3 cache and memory bandwidth schemata in 'container_id' group - // The read-only memory bandwidth information - for _, schemaRoot := range schemaRootStrings { - if strings.Contains(schemaRoot, "MB") { - stats.MemBwSchemaRoot = strings.TrimSpace(schemaRoot) + tmpStrings, err := getIntelRdtParamString(containerPath, "schemata") + if err != nil { + return nil, err } - } + schemaStrings := strings.Split(tmpStrings, "\n") + + if IsCatEnabled() { + // The read-only L3 cache information + l3CacheInfo, err := getL3CacheInfo() + if err != nil { + return nil, err + } + stats.L3CacheInfo = l3CacheInfo - // The memory bandwidth schema in 'container_id' group - for _, schema := range schemaStrings { - if strings.Contains(schema, "MB") { - stats.MemBwSchema = strings.TrimSpace(schema) + // The read-only L3 cache schema in root + for _, schemaRoot := range schemaRootStrings { + if strings.Contains(schemaRoot, "L3") { + stats.L3CacheSchemaRoot = strings.TrimSpace(schemaRoot) + } + } + + // The L3 cache schema in 'container_id' group + for _, schema := range schemaStrings { + if strings.Contains(schema, "L3") { + stats.L3CacheSchema = strings.TrimSpace(schema) + } + } + } + + if IsMbaEnabled() { + // The read-only memory bandwidth information + memBwInfo, err := getMemBwInfo() + if err != nil { + return nil, err + } + stats.MemBwInfo = memBwInfo + + // The read-only memory bandwidth information + for _, schemaRoot := range schemaRootStrings { + if strings.Contains(schemaRoot, "MB") { + stats.MemBwSchemaRoot = strings.TrimSpace(schemaRoot) + } + } + + // The memory bandwidth schema in 'container_id' group + for _, schema := range schemaStrings { + if strings.Contains(schema, "MB") { + stats.MemBwSchema = strings.TrimSpace(schema) + } + } } } - } - err = getMonitoringStats(containerPath, stats) - if err != nil { - return nil, err - } + return stats, nil + + case MonitoringGroupType: + path := m.GetPath() - return stats, nil + err := getMonitoringStats(path, stats) + if err != nil { + return nil, err + } + return stats, nil + } + return nil, fmt.Errorf("couldn't obtain stats from: %q resctrl manager of type: %q", m.Id, m.Type) } // Set Intel RDT "resource control" filesystem as configured. @@ -721,34 +781,46 @@ func (m *IntelRdtManager) Set(container *configs.Config) error { // For example, on a two-socket machine, the schema line could be // "MB:0=5000;1=7000" which means 5000 MBps memory bandwidth limit on // socket 0 and 7000 MBps memory bandwidth limit on socket 1. - if container.IntelRdt != nil { - path := m.GetPath() - l3CacheSchema := container.IntelRdt.L3CacheSchema - memBwSchema := container.IntelRdt.MemBwSchema + switch m.Type { + case ControlGroupType: + if container.IntelRdt != nil { + path := m.GetPath() + l3CacheSchema := container.IntelRdt.L3CacheSchema + memBwSchema := container.IntelRdt.MemBwSchema + + // Write a single joint schema string to schemata file + if l3CacheSchema != "" && memBwSchema != "" { + if err := writeFile(path, "schemata", l3CacheSchema+"\n"+memBwSchema); err != nil { + return NewLastCmdError(err) + } + } - // Write a single joint schema string to schemata file - if l3CacheSchema != "" && memBwSchema != "" { - if err := writeFile(path, "schemata", l3CacheSchema+"\n"+memBwSchema); err != nil { - return NewLastCmdError(err) + // Write only L3 cache schema string to schemata file + if l3CacheSchema != "" && memBwSchema == "" { + if err := writeFile(path, "schemata", l3CacheSchema); err != nil { + return NewLastCmdError(err) + } } - } - // Write only L3 cache schema string to schemata file - if l3CacheSchema != "" && memBwSchema == "" { - if err := writeFile(path, "schemata", l3CacheSchema); err != nil { - return NewLastCmdError(err) + // Write only memory bandwidth schema string to schemata file + if l3CacheSchema == "" && memBwSchema != "" { + if err := writeFile(path, "schemata", memBwSchema); err != nil { + return NewLastCmdError(err) + } } } - // Write only memory bandwidth schema string to schemata file - if l3CacheSchema == "" && memBwSchema != "" { - if err := writeFile(path, "schemata", memBwSchema); err != nil { - return NewLastCmdError(err) - } + case MonitoringGroupType: + if _, err := os.Stat(m.GetPath()); err == nil { + return nil + } + if err := os.Mkdir(m.GetPath(), 0755); err != nil { + return err } + return nil } - return nil + return fmt.Errorf("couldn't set configuration for: %q resctrl manager of type: %q", m.Id, m.Type) } func (raw *intelRdtData) join(id string) (string, error) { diff --git a/libcontainer/intelrdt/intelrdt_test.go b/libcontainer/intelrdt/intelrdt_test.go index a19b961b019..08921866dde 100644 --- a/libcontainer/intelrdt/intelrdt_test.go +++ b/libcontainer/intelrdt/intelrdt_test.go @@ -3,6 +3,7 @@ package intelrdt import ( + "errors" "strings" "testing" ) @@ -120,3 +121,41 @@ func TestIntelRdtSetMemBwScSchema(t *testing.T) { t.Fatal("Got the wrong value, set 'schemata' failed.") } } + +func TestIntelRdtManager_GetStats_NotSupported_Type(t *testing.T) { + manager := IntelRdtManager{ + Type: "not_supported", + Id: "id", + } + + _, err := manager.GetStats() + + expectedError := errors.New("couldn't obtain stats from: \"id\" resctrl manager of type: \"not_supported\"") + + if err == nil { + t.Fatalf("Expected error: %v, got nil.", expectedError) + } + + if err.Error() != expectedError.Error() { + t.Fatalf("Expected error: %v but got: %v.", expectedError, err) + } +} + +func Test_IntelRdtManager_Set_NotSupported_Type(t *testing.T) { + manager := IntelRdtManager{ + Type: "not_supported", + Id: "id", + } + + err := manager.Set(nil) + + expectedError := errors.New("couldn't set configuration for: \"id\" resctrl manager of type: \"not_supported\"") + + if err == nil { + t.Fatalf("Expected error: %v, got nil.", expectedError) + } + + if err.Error() != expectedError.Error() { + t.Fatalf("Expected error: %v but got: %v.", expectedError, err) + } +} diff --git a/update.go b/update.go index 2a6a9d8ab4b..f296d41f139 100644 --- a/update.go +++ b/update.go @@ -321,6 +321,7 @@ other options are ignored. Config: &config, Id: container.ID(), Path: state.IntelRdtPath, + Type: intelrdt.ControlGroupType, } if err := intelRdtManager.Apply(state.InitProcessPid); err != nil { return err