Add mon groups for resctrl.

"mon_groups" can be created to monitor subsets of tasks in the CTRL_MON group that is their ancestor. More info: https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt Signed-off-by: Paweł Szulik <pawel.szulik@intel.com>
opencontainers · Aug 25, 2020 · 2852ea3 · 2852ea3
1 parent e5f2eae
commit 2852ea3
Show file tree

Hide file tree

Showing 3 changed files with 207 additions and 95 deletions.
diff --git a/libcontainer/intelrdt/intelrdt.go b/libcontainer/intelrdt/intelrdt.go
@@ -12,6 +12,8 @@ import (
  "strings"
  "sync"
 
+ "github.com/sirupsen/logrus"
+
  "github.com/opencontainers/runc/libcontainer/configs"
 )
 
@@ -167,10 +169,17 @@ type IntelRdtManager struct {
  Config *configs.Config
  Id string
  Path string
+ Type string
 }
 
 const (
- IntelRdtTasks = "tasks"
+ IntelRdtTasks = "tasks"
+ MonitoringGroupDirectoryName = "mon_groups"
+
+ // See "Resource alloc and monitor groups" part here:
+ // https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt
+ MonitoringGroupType = "MON"
+ ControlGroupType = "CTRL_MON"
 )
 
 var (
@@ -548,26 +557,47 @@ func GetIntelRdtPath(id string) (string, error) {
  return path, nil
 }
 
+// Get the 'container_id" path in Intel RDT "monitoring group" filesystem.
+func getMonGroupIntelRdtPath(id string) (string, error) {
+ rootPath, err := getIntelRdtRoot()
+ if err != nil {
+ return "", err
+ }
+
+ path := filepath.Join(rootPath, MonitoringGroupDirectoryName, id)
+ return path, nil
+}
+
 // Applies Intel RDT configuration to the process with the specified pid
 func (m *IntelRdtManager) Apply(pid int) (err error) {
- // If intelRdt is not specified in config, we do nothing
- if m.Config.IntelRdt == nil {
+ switch m.Type {
+ case ControlGroupType:
+ // If intelRdt is not specified in config, we do nothing
+ if m.Config.IntelRdt == nil {
+ return nil
+ }
+ rdtData, err := getIntelRdtData(m.Config, pid)
+ if err != nil && !IsNotFound(err) {
+ return err
+ }
+
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ path, err := rdtData.join(m.Id)
+ if err != nil {
+ return err
+ }
+
+ m.Path = path
  return nil
- }
- d, err := getIntelRdtData(m.Config, pid)
- if err != nil && !IsNotFound(err) {
- return err
- }
 
- m.mu.Lock()
- defer m.mu.Unlock()
- path, err := d.join(m.Id)
- if err != nil {
- return err
+ case MonitoringGroupType:
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ return WriteIntelRdtTasks(m.Path, pid)
  }
 
- m.Path = path
- return nil
+ return fmt.Errorf("couldn't recognize resctrl type: %v", m.Type)
 }
 
 // Destroys the Intel RDT 'container_id' group
@@ -584,94 +614,124 @@ func (m *IntelRdtManager) Destroy() error {
 // Returns Intel RDT path to save in a state file and to be able to
 // restore the object later
 func (m *IntelRdtManager) GetPath() string {
+ var err error
  if m.Path == "" {
- m.Path, _ = GetIntelRdtPath(m.Id)
+ switch m.Type {
+ case ControlGroupType:
+ m.Path, err = GetIntelRdtPath(m.Id)
+ if err != nil {
+ logrus.Errorf("couldn't obtain Resctrl control group path for manager with id %v: %v", m.Id, err)
+ }
+ case MonitoringGroupType:
+ flattedContainerID := strings.Replace(m.Id, "/", "-", -1)
+ m.Path, err = getMonGroupIntelRdtPath(flattedContainerID)
+ if err != nil {
+ logrus.Errorf("couldn't obtain Resctrl monitoring group path for manager with id %v: %v", m.Id, err)
+ }
+ }
  }
  return m.Path
 }
 
 // Returns statistics for Intel RDT
 func (m *IntelRdtManager) GetStats() (*Stats, error) {
- // If intelRdt is not specified in config
- if m.Config.IntelRdt == nil {
- return nil, nil
- }
-
  m.mu.Lock()
  defer m.mu.Unlock()
  stats := NewStats()
 
- rootPath, err := getIntelRdtRoot()
- if err != nil {
- return nil, err
- }
- // The read-only L3 cache and memory bandwidth schemata in root
- tmpRootStrings, err := getIntelRdtParamString(rootPath, "schemata")
- if err != nil {
- return nil, err
- }
- schemaRootStrings := strings.Split(tmpRootStrings, "\n")
-
- // The L3 cache and memory bandwidth schemata in 'container_id' group
- containerPath := m.GetPath()
- tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
- if err != nil {
- return nil, err
- }
- schemaStrings := strings.Split(tmpStrings, "\n")
+ switch m.Type {
+ case ControlGroupType:
+ containerPath := m.GetPath()
 
- if IsCatEnabled() {
- // The read-only L3 cache information
- l3CacheInfo, err := getL3CacheInfo()
+ err := getMonitoringStats(containerPath, stats)
  if err != nil {
  return nil, err
  }
- stats.L3CacheInfo = l3CacheInfo
 
- // The read-only L3 cache schema in root
- for _, schemaRoot := range schemaRootStrings {
- if strings.Contains(schemaRoot, "L3") {
- stats.L3CacheSchemaRoot = strings.TrimSpace(schemaRoot)
+ // If intelRdt is not specified in config
+ if m.Config != nil {
+ if m.Config.IntelRdt == nil {
+ return nil, nil
  }
- }
 
- // The L3 cache schema in 'container_id' group
- for _, schema := range schemaStrings {
- if strings.Contains(schema, "L3") {
- stats.L3CacheSchema = strings.TrimSpace(schema)
+ rootPath, err := getIntelRdtRoot()
+ if err != nil {
+ return nil, err
  }
- }
- }
+ // The read-only L3 cache and memory bandwidth schemata in root
+ tmpRootStrings, err := getIntelRdtParamString(rootPath, "schemata")
+ if err != nil {
+ return nil, err
+ }
+ schemaRootStrings := strings.Split(tmpRootStrings, "\n")
 
- if IsMbaEnabled() {
- // The read-only memory bandwidth information
- memBwInfo, err := getMemBwInfo()
- if err != nil {
- return nil, err
- }
- stats.MemBwInfo = memBwInfo
+ // The L3 cache and memory bandwidth schemata in 'container_id' group
 
- // The read-only memory bandwidth information
- for _, schemaRoot := range schemaRootStrings {
- if strings.Contains(schemaRoot, "MB") {
- stats.MemBwSchemaRoot = strings.TrimSpace(schemaRoot)
+ tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
+ if err != nil {
+ return nil, err
  }
- }
+ schemaStrings := strings.Split(tmpStrings, "\n")
+
+ if IsCatEnabled() {
+ // The read-only L3 cache information
+ l3CacheInfo, err := getL3CacheInfo()
+ if err != nil {
+ return nil, err
+ }
+ stats.L3CacheInfo = l3CacheInfo
 
- // The memory bandwidth schema in 'container_id' group
- for _, schema := range schemaStrings {
- if strings.Contains(schema, "MB") {
- stats.MemBwSchema = strings.TrimSpace(schema)
+ // The read-only L3 cache schema in root
+ for _, schemaRoot := range schemaRootStrings {
+ if strings.Contains(schemaRoot, "L3") {
+ stats.L3CacheSchemaRoot = strings.TrimSpace(schemaRoot)
+ }
+ }
+
+ // The L3 cache schema in 'container_id' group
+ for _, schema := range schemaStrings {
+ if strings.Contains(schema, "L3") {
+ stats.L3CacheSchema = strings.TrimSpace(schema)
+ }
+ }
+ }
+
+ if IsMbaEnabled() {
+ // The read-only memory bandwidth information
+ memBwInfo, err := getMemBwInfo()
+ if err != nil {
+ return nil, err
+ }
+ stats.MemBwInfo = memBwInfo
+
+ // The read-only memory bandwidth information
+ for _, schemaRoot := range schemaRootStrings {
+ if strings.Contains(schemaRoot, "MB") {
+ stats.MemBwSchemaRoot = strings.TrimSpace(schemaRoot)
+ }
+ }
+
+ // The memory bandwidth schema in 'container_id' group
+ for _, schema := range schemaStrings {
+ if strings.Contains(schema, "MB") {
+ stats.MemBwSchema = strings.TrimSpace(schema)
+ }
+ }
  }
  }
- }
 
- err = getMonitoringStats(containerPath, stats)
- if err != nil {
-  return nil, err
- }
+  return stats, nil
+
+ case MonitoringGroupType:
+  path := m.GetPath()
 
- return stats, nil
+ err := getMonitoringStats(path, stats)
+ if err != nil {
+ return nil, err
+ }
+ return stats, nil
+ }
+ return nil, fmt.Errorf("couldn't obtain stats from: %q resctrl manager of type: %q", m.Id, m.Type)
 }
 
 // Set Intel RDT "resource control" filesystem as configured.
@@ -721,34 +781,46 @@ func (m *IntelRdtManager) Set(container *configs.Config) error {
  // For example, on a two-socket machine, the schema line could be
  // "MB:0=5000;1=7000" which means 5000 MBps memory bandwidth limit on
  // socket 0 and 7000 MBps memory bandwidth limit on socket 1.
- if container.IntelRdt != nil {
- path := m.GetPath()
- l3CacheSchema := container.IntelRdt.L3CacheSchema
- memBwSchema := container.IntelRdt.MemBwSchema
+ switch m.Type {
+ case ControlGroupType:
+ if container.IntelRdt != nil {
+ path := m.GetPath()
+ l3CacheSchema := container.IntelRdt.L3CacheSchema
+ memBwSchema := container.IntelRdt.MemBwSchema
+
+ // Write a single joint schema string to schemata file
+ if l3CacheSchema != "" && memBwSchema != "" {
+ if err := writeFile(path, "schemata", l3CacheSchema+"\n"+memBwSchema); err != nil {
+ return NewLastCmdError(err)
+ }
+ }
 
- // Write a single joint schema string to schemata file
- if l3CacheSchema != "" && memBwSchema != "" {
- if err := writeFile(path, "schemata", l3CacheSchema+"\n"+memBwSchema); err != nil {
- return NewLastCmdError(err)
+ // Write only L3 cache schema string to schemata file
+ if l3CacheSchema != "" && memBwSchema == "" {
+ if err := writeFile(path, "schemata", l3CacheSchema); err != nil {
+ return NewLastCmdError(err)
+ }
  }
- }
 
- // Write only L3 cache schema string to schemata file
- if l3CacheSchema != "" && memBwSchema == "" {
- if err := writeFile(path, "schemata", l3CacheSchema); err != nil {
- return NewLastCmdError(err)
+ // Write only memory bandwidth schema string to schemata file
+ if l3CacheSchema == "" && memBwSchema != "" {
+ if err := writeFile(path, "schemata", memBwSchema); err != nil {
+ return NewLastCmdError(err)
+ }
  }
  }
 
- // Write only memory bandwidth schema string to schemata file
- if l3CacheSchema == "" && memBwSchema != "" {
- if err := writeFile(path, "schemata", memBwSchema); err != nil {
- return NewLastCmdError(err)
- }
+ case MonitoringGroupType:
+ if _, err := os.Stat(m.GetPath()); err == nil {
+ return nil
+ }
+ if err := os.Mkdir(m.GetPath(), 0755); err != nil {
+ return err
  }
+ return nil
  }
 
- return nil
+ return fmt.Errorf("couldn't set configuration for: %q resctrl manager of type: %q", m.Id, m.Type)
 }
 
 func (raw *intelRdtData) join(id string) (string, error) {

diff --git a/libcontainer/intelrdt/intelrdt_test.go b/libcontainer/intelrdt/intelrdt_test.go
@@ -3,6 +3,7 @@
 package intelrdt
 
 import (
+ "errors"
  "strings"
  "testing"
 )
@@ -120,3 +121,41 @@ func TestIntelRdtSetMemBwScSchema(t *testing.T) {
  t.Fatal("Got the wrong value, set 'schemata' failed.")
  }
 }
+
+func TestIntelRdtManager_GetStats_NotSupported_Type(t *testing.T) {
+ manager := IntelRdtManager{
+ Type: "not_supported",
+ Id: "id",
+ }
+
+ _, err := manager.GetStats()
+
+ expectedError := errors.New("couldn't obtain stats from: \"id\" resctrl manager of type: \"not_supported\"")
+
+ if err == nil {
+ t.Fatalf("Expected error: %v, got nil.", expectedError)
+ }
+
+ if err.Error() != expectedError.Error() {
+ t.Fatalf("Expected error: %v but got: %v.", expectedError, err)
+ }
+}
+
+func Test_IntelRdtManager_Set_NotSupported_Type(t *testing.T) {
+ manager := IntelRdtManager{
+ Type: "not_supported",
+ Id: "id",
+ }
+
+ err := manager.Set(nil)
+
+ expectedError := errors.New("couldn't set configuration for: \"id\" resctrl manager of type: \"not_supported\"")
+
+ if err == nil {
+ t.Fatalf("Expected error: %v, got nil.", expectedError)
+ }
+
+ if err.Error() != expectedError.Error() {
+ t.Fatalf("Expected error: %v but got: %v.", expectedError, err)
+ }
+}
diff --git a/update.go b/update.go
@@ -321,6 +321,7 @@ other options are ignored.
  Config: &config,
  Id: container.ID(),
  Path: state.IntelRdtPath,
+ Type: intelrdt.ControlGroupType,
  }
  if err := intelRdtManager.Apply(state.InitProcessPid); err != nil {
  return err