diff --git a/libcontainer/SPEC.md b/libcontainer/SPEC.md index 07ebdc12153..ac31bec65ab 100644 --- a/libcontainer/SPEC.md +++ b/libcontainer/SPEC.md @@ -158,32 +158,38 @@ init process will block waiting for the parent to finish setup. ### IntelRdt Intel platforms with new Xeon CPU support Resource Director Technology (RDT). -Cache Allocation Technology (CAT) and Memory Bandwidth Allocation (MBA) are -two sub-features of RDT. +Cache Allocation Technology (CAT), Cache Monitoring Technology (CMT), +Memory Bandwidth Allocation (MBA) and Memory Bandwidth Monitoring (MBM) are +four sub-features of RDT. Cache Allocation Technology (CAT) provides a way for the software to restrict cache allocation to a defined 'subset' of L3 cache which may be overlapping with other 'subsets'. The different subsets are identified by class of service (CLOS) and each CLOS has a capacity bitmask (CBM). +Cache Monitoring Technology (CMT) supports monitoring of the last-level cache (LLC) occupancy +for each running thread simultaneously. + Memory Bandwidth Allocation (MBA) provides indirect and approximate throttle over memory bandwidth for the software. A user controls the resource by -indicating the percentage of maximum memory bandwidth or memory bandwidth limit -in MBps unit if MBA Software Controller is enabled. +indicating the percentage of maximum memory bandwidth or memory bandwidth +limit in MBps unit if MBA Software Controller is enabled. + +Memory Bandwidth Monitoring (MBM) supports monitoring of total and local memory bandwidth +for each running thread simultaneously. -It can be used to handle L3 cache and memory bandwidth resources allocation -for containers if hardware and kernel support Intel RDT CAT and MBA features. +More details about Intel RDT CAT and MBA can be found in the section 17.18 and 17.19, Volume 3 +of Intel Software Developer Manual: +https://software.intel.com/en-us/articles/intel-sdm -In Linux 4.10 kernel or newer, the interface is defined and exposed via +About Intel RDT kernel interface: +In Linux 4.14 kernel or newer, the interface is defined and exposed via "resource control" filesystem, which is a "cgroup-like" interface. Comparing with cgroups, it has similar process management lifecycle and interfaces in a container. But unlike cgroups' hierarchy, it has single level filesystem layout. -CAT and MBA features are introduced in Linux 4.10 and 4.12 kernel via -"resource control" filesystem. - Intel RDT "resource control" filesystem hierarchy: ``` mount -t resctrl resctrl /sys/fs/resctrl @@ -194,25 +200,46 @@ tree /sys/fs/resctrl | | |-- cbm_mask | | |-- min_cbm_bits | | |-- num_closids +| |-- L3_MON +| | |-- max_threshold_occupancy +| | |-- mon_features +| | |-- num_rmids | |-- MB | |-- bandwidth_gran | |-- delay_linear | |-- min_bandwidth | |-- num_closids -|-- ... +|-- mon_groups + |-- + |-- ... + |-- mon_data + |-- mon_L3_00 + |-- llc_occupancy + |-- mbm_local_bytes + |-- mbm_total_bytes + |-- ... + |-- tasks |-- schemata |-- tasks -|-- +|-- |-- ... - |-- schemata + |-- mon_data + |-- mon_L3_00 + |-- llc_occupancy + |-- mbm_local_bytes + |-- mbm_total_bytes + |-- ... |-- tasks + |-- schemata +|-- ... ``` For runc, we can make use of `tasks` and `schemata` configuration for L3 -cache and memory bandwidth resources constraints. +cache and memory bandwidth resources constraints, `mon_data` directory for +CMT and MBM statistics. The file `tasks` has a list of tasks that belongs to this group (e.g., -" group). Tasks can be added to a group by writing the task ID +"" group). Tasks can be added to a group by writing the task ID to the "tasks" file (which will automatically remove them from the previous group to which they belonged). New tasks created by fork(2) and clone(2) are added to the same group as their parent. @@ -224,7 +251,7 @@ L3 cache schema: It has allocation bitmasks/values for L3 cache on each socket, which contains L3 cache id and capacity bitmask (CBM). ``` - Format: "L3:=;=;..." +Format: "L3:=;=;..." ``` For example, on a two-socket machine, the schema line could be "L3:0=ff;1=c0" which means L3 cache id 0's CBM is 0xff, and L3 cache id 1's CBM is 0xc0. @@ -240,7 +267,7 @@ Memory bandwidth schema: It has allocation values for memory bandwidth on each socket, which contains L3 cache id and memory bandwidth. ``` - Format: "MB:=bandwidth0;=bandwidth1;..." +Format: "MB:=bandwidth0;=bandwidth1;..." ``` For example, on a two-socket machine, the schema line could be "MB:0=20;1=70" @@ -251,8 +278,10 @@ that is allocated is also dependent on the CPU model and can be looked up at min_bw + N * bw_gran. Intermediate values are rounded to the next control step available on the hardware. -If MBA Software Controller is enabled through mount option "-o mba_MBps" +If MBA Software Controller is enabled through mount option "-o mba_MBps": +``` mount -t resctrl resctrl -o mba_MBps /sys/fs/resctrl +``` We could specify memory bandwidth in "MBps" (Mega Bytes per second) unit instead of "percentages". The kernel underneath would use a software feedback mechanism or a "Software Controller" which reads the actual bandwidth using @@ -263,11 +292,12 @@ For example, on a two-socket machine, the schema line could be "MB:0=5000;1=7000" which means 5000 MBps memory bandwidth limit on socket 0 and 7000 MBps memory bandwidth limit on socket 1. -For more information about Intel RDT kernel interface: +For more information about Intel RDT kernel interface: https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt -``` + An example for runc: +``` Consider a two-socket machine with two L3 caches where the default CBM is 0x7ff and the max CBM length is 11 bits, and minimum memory bandwidth of 10% with a memory bandwidth granularity of 10%. @@ -281,7 +311,17 @@ maximum memory bandwidth of 20% on socket 0 and 70% on socket 1. "closID": "guaranteed_group", "l3CacheSchema": "L3:0=7f0;1=1f", "memBwSchema": "MB:0=20;1=70" - } + } +} +``` +Another example: +``` +We only want to monitor memory bandwidth and llc occupancy. +"linux": { + "intelRdt": { + "enableMBM": true, + "enableCMT": true + } } ``` diff --git a/libcontainer/configs/config.go b/libcontainer/configs/config.go index 18955cf95c9..55f6ba762df 100644 --- a/libcontainer/configs/config.go +++ b/libcontainer/configs/config.go @@ -197,7 +197,7 @@ type Config struct { NoNewKeyring bool `json:"no_new_keyring"` // IntelRdt specifies settings for Intel RDT group that the container is placed into - // to limit the resources (e.g., L3 cache, memory bandwidth) the container has available + // to limit the resources (e.g., L3 cache, memory bandwidth) the container has available. IntelRdt *IntelRdt `json:"intel_rdt,omitempty"` // RootlessEUID is set when the runc was launched with non-zero EUID. diff --git a/libcontainer/configs/intelrdt.go b/libcontainer/configs/intelrdt.go index f8d951ab8b9..c778b43a948 100644 --- a/libcontainer/configs/intelrdt.go +++ b/libcontainer/configs/intelrdt.go @@ -13,4 +13,12 @@ type IntelRdt struct { // The unit of memory bandwidth is specified in "percentages" by // default, and in "MBps" if MBA Software Controller is enabled. MemBwSchema string `json:"memBwSchema,omitempty"` + + // The flag to indicate if Intel RDT CMT is enabled. CMT (Cache Monitoring Technology) supports monitoring of + // the last-level cache (LLC) occupancy for the container. + EnableCMT bool `json:"enableCMT,omitempty"` + + // The flag to indicate if Intel RDT MBM is enabled. MBM (Memory Bandwidth Monitoring) supports monitoring of + // total and local memory bandwidth for the container. + EnableMBM bool `json:"enableMBM,omitempty"` } diff --git a/libcontainer/configs/intelrdt_test.go b/libcontainer/configs/intelrdt_test.go new file mode 100644 index 00000000000..dec676c6c23 --- /dev/null +++ b/libcontainer/configs/intelrdt_test.go @@ -0,0 +1,42 @@ +package configs_test + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/opencontainers/runc/libcontainer/configs" +) + +func TestUnmarshalIntelRDT(t *testing.T) { + testCases := []struct { + JSON string + Expected configs.IntelRdt + }{ + { + "{\"enableMBM\": true}", + configs.IntelRdt{EnableMBM: true, EnableCMT: false}, + }, + { + "{\"enableMBM\": true,\"enableCMT\": false}", + configs.IntelRdt{EnableMBM: true, EnableCMT: false}, + }, + { + "{\"enableMBM\": false,\"enableCMT\": true}", + configs.IntelRdt{EnableMBM: false, EnableCMT: true}, + }, + } + + for _, tc := range testCases { + got := configs.IntelRdt{} + + err := json.Unmarshal([]byte(tc.JSON), &got) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(tc.Expected, got) { + t.Errorf("expected unmarshalled IntelRDT config %+v, got %+v", tc.Expected, got) + } + } +} diff --git a/libcontainer/configs/validate/validator.go b/libcontainer/configs/validate/validator.go index 2027a37203e..73c62f91772 100644 --- a/libcontainer/configs/validate/validator.go +++ b/libcontainer/configs/validate/validator.go @@ -219,12 +219,18 @@ func intelrdtCheck(config *configs.Config) error { return fmt.Errorf("invalid intelRdt.ClosID %q", config.IntelRdt.ClosID) } - if !intelrdt.IsCATEnabled() && config.IntelRdt.L3CacheSchema != "" { + if config.IntelRdt.L3CacheSchema != "" && !intelrdt.IsCATEnabled() { return errors.New("intelRdt.l3CacheSchema is specified in config, but Intel RDT/CAT is not enabled") } - if !intelrdt.IsMBAEnabled() && config.IntelRdt.MemBwSchema != "" { + if config.IntelRdt.MemBwSchema != "" && !intelrdt.IsMBAEnabled() { return errors.New("intelRdt.memBwSchema is specified in config, but Intel RDT/MBA is not enabled") } + if config.IntelRdt.EnableCMT && !intelrdt.IsCMTEnabled() { + return errors.New("intelRdt.enableCMT is specified in config, but Intel RDT/CMT is not enabled") + } + if config.IntelRdt.EnableMBM && !intelrdt.IsMBMEnabled() { + return errors.New("intelRdt.enableMBM is specified in config, but Intel RDT/MBM is not enabled") + } } return nil diff --git a/libcontainer/container_linux.go b/libcontainer/container_linux.go index c099e458aed..12a20885cd2 100644 --- a/libcontainer/container_linux.go +++ b/libcontainer/container_linux.go @@ -2012,6 +2012,7 @@ func (c *Container) currentState() (*State, error) { if c.intelRdtManager != nil { intelRdtPath = c.intelRdtManager.GetPath() } + state := &State{ BaseState: BaseState{ ID: c.ID(), diff --git a/libcontainer/intelrdt/intelrdt.go b/libcontainer/intelrdt/intelrdt.go index 0f5c457b099..5dfe3283a4c 100644 --- a/libcontainer/intelrdt/intelrdt.go +++ b/libcontainer/intelrdt/intelrdt.go @@ -20,34 +20,38 @@ import ( /* * About Intel RDT features: * Intel platforms with new Xeon CPU support Resource Director Technology (RDT). - * Cache Allocation Technology (CAT) and Memory Bandwidth Allocation (MBA) are - * two sub-features of RDT. + * Cache Allocation Technology (CAT), Cache Monitoring Technology (CMT), + * Memory Bandwidth Allocation (MBA) and Memory Bandwidth Monitoring (MBM) are + * four sub-features of RDT. * * Cache Allocation Technology (CAT) provides a way for the software to restrict * cache allocation to a defined 'subset' of L3 cache which may be overlapping * with other 'subsets'. The different subsets are identified by class of * service (CLOS) and each CLOS has a capacity bitmask (CBM). * + * Cache Monitoring Technology (CMT) supports monitoring of the last-level cache (LLC) occupancy + * for each running thread simultaneously. + * * Memory Bandwidth Allocation (MBA) provides indirect and approximate throttle * over memory bandwidth for the software. A user controls the resource by * indicating the percentage of maximum memory bandwidth or memory bandwidth * limit in MBps unit if MBA Software Controller is enabled. * - * More details about Intel RDT CAT and MBA can be found in the section 17.18 + * Memory Bandwidth Monitoring (MBM) supports monitoring of total and local memory bandwidth + * for each running thread simultaneously. + * + * More details about Intel RDT CAT and MBA can be found in the section 17.18 and 17.19, Volume 3 * of Intel Software Developer Manual: * https://software.intel.com/en-us/articles/intel-sdm * * About Intel RDT kernel interface: - * In Linux 4.10 kernel or newer, the interface is defined and exposed via + * In Linux 4.14 kernel or newer, the interface is defined and exposed via * "resource control" filesystem, which is a "cgroup-like" interface. * * Comparing with cgroups, it has similar process management lifecycle and * interfaces in a container. But unlike cgroups' hierarchy, it has single level * filesystem layout. * - * CAT and MBA features are introduced in Linux 4.10 and 4.12 kernel via - * "resource control" filesystem. - * * Intel RDT "resource control" filesystem hierarchy: * mount -t resctrl resctrl /sys/fs/resctrl * tree /sys/fs/resctrl @@ -66,19 +70,37 @@ import ( * | |-- delay_linear * | |-- min_bandwidth * | |-- num_closids - * |-- ... + * |-- mon_groups + * |-- + * |-- ... + * |-- mon_data + * |-- mon_L3_00 + * |-- llc_occupancy + * |-- mbm_local_bytes + * |-- mbm_total_bytes + * |-- ... + * |-- tasks * |-- schemata * |-- tasks * |-- * |-- ... - * |-- schemata + * |-- mon_data + * |-- mon_L3_00 + * |-- llc_occupancy + * |-- mbm_local_bytes + * |-- mbm_total_bytes + * |-- ... * |-- tasks + * |-- schemata + * |-- ... + * * * For runc, we can make use of `tasks` and `schemata` configuration for L3 - * cache and memory bandwidth resources constraints. + * cache and memory bandwidth resources constraints, `mon_data` directory for + * CMT and MBM statistics. * * The file `tasks` has a list of tasks that belongs to this group (e.g., - * " group). Tasks can be added to a group by writing the task ID + * "" group). Tasks can be added to a group by writing the task ID * to the "tasks" file (which will automatically remove them from the previous * group to which they belonged). New tasks created by fork(2) and clone(2) are * added to the same group as their parent. @@ -89,7 +111,9 @@ import ( * L3 cache schema: * It has allocation bitmasks/values for L3 cache on each socket, which * contains L3 cache id and capacity bitmask (CBM). - * Format: "L3:=;=;..." + * + * Format: "L3:=;=;..." + * * For example, on a two-socket machine, the schema line could be "L3:0=ff;1=c0" * which means L3 cache id 0's CBM is 0xff, and L3 cache id 1's CBM is 0xc0. * @@ -103,7 +127,9 @@ import ( * Memory bandwidth schema: * It has allocation values for memory bandwidth on each socket, which contains * L3 cache id and memory bandwidth. - * Format: "MB:=bandwidth0;=bandwidth1;..." + * + * Format: "MB:=bandwidth0;=bandwidth1;..." + * * For example, on a two-socket machine, the schema line could be "MB:0=20;1=70" * * The minimum bandwidth percentage value for each CPU model is predefined and @@ -128,7 +154,9 @@ import ( * For more information about Intel RDT kernel interface: * https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt * + * * An example for runc: + * * Consider a two-socket machine with two L3 caches where the default CBM is * 0x7ff and the max CBM length is 11 bits, and minimum memory bandwidth of 10% * with a memory bandwidth granularity of 10%. @@ -141,15 +169,27 @@ import ( * "intelRdt": { * "l3CacheSchema": "L3:0=7f0;1=1f", * "memBwSchema": "MB:0=20;1=70" - * } + * } + * } + * + * Another example: + * + * We only want to monitor memory bandwidth and llc occupancy. + * "linux": { + * "intelRdt": { + * "enableMBM": true, + * "enableCMT": true + * } * } + * */ type Manager struct { - mu sync.Mutex - config *configs.Config - id string - path string + mu sync.Mutex + config *configs.Config + id string + path string + monitoringGroup bool } // NewManager returns a new instance of Manager, or nil if the Intel RDT @@ -169,15 +209,26 @@ func NewManager(config *configs.Config, id string, path string) *Manager { // newManager is the same as NewManager, except it does not check if the feature // is actually available. Used by unit tests that mock intelrdt paths. func newManager(config *configs.Config, id string, path string) *Manager { + var monitoringGroup bool + if config.IntelRdt.L3CacheSchema != "" || config.IntelRdt.MemBwSchema != "" || config.IntelRdt.ClosID != "" { + monitoringGroup = false + } else if config.IntelRdt.EnableCMT || config.IntelRdt.EnableMBM { + monitoringGroup = true + } else { + return nil + } + return &Manager{ - config: config, - id: id, - path: path, + config: config, + id: id, + path: path, + monitoringGroup: monitoringGroup, } } const ( - intelRdtTasks = "tasks" + intelRdtTasks = "tasks" + monitoringGroupRoot = "mon_groups" ) var ( @@ -435,12 +486,16 @@ func (m *Manager) getIntelRdtPath() (string, error) { return "", err } - clos := m.id + groupName := m.id if m.config.IntelRdt != nil && m.config.IntelRdt.ClosID != "" { - clos = m.config.IntelRdt.ClosID + groupName = m.config.IntelRdt.ClosID + } + + if m.monitoringGroup { + return filepath.Join(rootPath, monitoringGroupRoot, groupName), nil } - return filepath.Join(rootPath, clos), nil + return filepath.Join(rootPath, groupName), nil } // Applies Intel RDT configuration to the process with the specified pid @@ -518,6 +573,9 @@ func (m *Manager) GetStats() (*Stats, error) { if err != nil { return nil, err } + + containerPath := m.GetPath() + // The read-only L3 cache and memory bandwidth schemata in root tmpRootStrings, err := getIntelRdtParamString(rootPath, "schemata") if err != nil { @@ -525,9 +583,13 @@ func (m *Manager) GetStats() (*Stats, error) { } schemaRootStrings := strings.Split(tmpRootStrings, "\n") + closPath := containerPath + if m.monitoringGroup { + closPath = filepath.Join(containerPath, "../..") + } + // The L3 cache and memory bandwidth schemata in container's clos group - containerPath := m.GetPath() - tmpStrings, err := getIntelRdtParamString(containerPath, "schemata") + tmpStrings, err := getIntelRdtParamString(closPath, "schemata") if err != nil { return nil, err } @@ -579,8 +641,8 @@ func (m *Manager) GetStats() (*Stats, error) { } } - if IsMBMEnabled() || IsCMTEnabled() { - err = getMonitoringStats(containerPath, stats) + if (IsCMTEnabled() && m.config.IntelRdt.EnableCMT) || (IsMBMEnabled() && m.config.IntelRdt.EnableMBM) { + err = getMonitoringStats(containerPath, stats, m.config.IntelRdt.EnableCMT, m.config.IntelRdt.EnableMBM) if err != nil { return nil, err } @@ -637,17 +699,26 @@ func (m *Manager) Set(container *configs.Config) error { // "MB:0=5000;1=7000" which means 5000 MBps memory bandwidth limit on // socket 0 and 7000 MBps memory bandwidth limit on socket 1. if container.IntelRdt != nil { - path := m.GetPath() + l3CacheSchema := container.IntelRdt.L3CacheSchema memBwSchema := container.IntelRdt.MemBwSchema + // Write a single joint schema string to schemata file + if l3CacheSchema != "" || memBwSchema != "" { + // Schema can be set only in Control Group. + if m.monitoringGroup { + return fmt.Errorf("couldn't set IntelRdt l3CacheSchema or memBwSchema for the monitoring group") + } + } + // TODO: verify that l3CacheSchema and/or memBwSchema match the // existing schemata if ClosID has been specified. This is a more // involved than reading the file and doing plain string comparison as // the value written in does not necessarily match what gets read out // (leading zeros, cache id ordering etc). - // Write a single joint schema string to schemata file + // Write a single joint schemata string to schemata file + path := m.GetPath() if l3CacheSchema != "" && memBwSchema != "" { if err := writeFile(path, "schemata", l3CacheSchema+"\n"+memBwSchema); err != nil { return err diff --git a/libcontainer/intelrdt/intelrdt_test.go b/libcontainer/intelrdt/intelrdt_test.go index c127cd8f7c6..baa845483bd 100644 --- a/libcontainer/intelrdt/intelrdt_test.go +++ b/libcontainer/intelrdt/intelrdt_test.go @@ -1,10 +1,14 @@ package intelrdt import ( + "errors" "os" "path/filepath" "strings" + "sync" "testing" + + "github.com/opencontainers/runc/libcontainer/configs" ) func TestIntelRdtSetL3CacheSchema(t *testing.T) { @@ -125,3 +129,50 @@ func TestApply(t *testing.T) { t.Fatalf("unexpected tasks file, expected '1235', got %q", pids) } } + +func TestIntelRdtManagerSetSchemataInMonGroup(t *testing.T) { + helper := NewIntelRdtTestUtil(t) + + intelrdt := Manager{ + mu: sync.Mutex{}, + config: helper.config, + id: "", + path: helper.IntelRdtPath, + monitoringGroup: true, + } + + test := []struct { + l3Schema string + memBwSchema string + }{ + { + "L3:0=f0;1=f", + "", + }, + { + "L3:0=f0;1=f", + "MB:0=20;1=70", + }, + { + "", + "MB:0=20;1=70", + }, + } + + expectedError := errors.New("couldn't set IntelRdt l3CacheSchema or memBwSchema for the monitoring group") + + for _, tc := range test { + err := intelrdt.Set(&configs.Config{IntelRdt: &configs.IntelRdt{ + L3CacheSchema: tc.l3Schema, + MemBwSchema: tc.memBwSchema, + }}) + + if err == nil { + t.Fatalf("Expected error: %v, got nil.", expectedError) + } + + if err.Error() != expectedError.Error() { + t.Fatalf("Expected error: %v but got: %v.", expectedError, err) + } + } +} diff --git a/libcontainer/intelrdt/monitoring.go b/libcontainer/intelrdt/monitoring.go index 82e0002efad..94fe664bc24 100644 --- a/libcontainer/intelrdt/monitoring.go +++ b/libcontainer/intelrdt/monitoring.go @@ -47,7 +47,7 @@ func parseMonFeatures(reader io.Reader) (monFeatures, error) { return monFeatures, scanner.Err() } -func getMonitoringStats(containerPath string, stats *Stats) error { +func getMonitoringStats(containerPath string, stats *Stats, enableCMT bool, enableMBM bool) error { numaFiles, err := os.ReadDir(filepath.Join(containerPath, "mon_data")) if err != nil { return err @@ -59,14 +59,14 @@ func getMonitoringStats(containerPath string, stats *Stats) error { for _, file := range numaFiles { if file.IsDir() { numaPath := filepath.Join(containerPath, "mon_data", file.Name()) - if IsMBMEnabled() { + if enableMBM && IsMBMEnabled() { numaMBMStats, err := getMBMNumaNodeStats(numaPath) if err != nil { return err } mbmStats = append(mbmStats, *numaMBMStats) } - if IsCMTEnabled() { + if enableCMT && IsCMTEnabled() { numaCMTStats, err := getCMTNumaNodeStats(numaPath) if err != nil { return err diff --git a/libcontainer/intelrdt/monitoring_test.go b/libcontainer/intelrdt/monitoring_test.go index 0a89ef2f7b8..8ce8489a208 100644 --- a/libcontainer/intelrdt/monitoring_test.go +++ b/libcontainer/intelrdt/monitoring_test.go @@ -80,7 +80,7 @@ func TestGetMonitoringStats(t *testing.T) { t.Run("Gather monitoring stats", func(t *testing.T) { var stats Stats - err := getMonitoringStats(mockedL3_MON, &stats) + err := getMonitoringStats(mockedL3_MON, &stats, true, true) if err != nil { t.Fatal(err) } diff --git a/libcontainer/specconv/spec_linux.go b/libcontainer/specconv/spec_linux.go index 4b32f286e44..7a96a36fc6f 100644 --- a/libcontainer/specconv/spec_linux.go +++ b/libcontainer/specconv/spec_linux.go @@ -432,6 +432,8 @@ func CreateLibcontainerConfig(opts *CreateOpts) (*configs.Config, error) { L3CacheSchema: spec.Linux.IntelRdt.L3CacheSchema, MemBwSchema: spec.Linux.IntelRdt.MemBwSchema, } + config.IntelRdt.EnableCMT = spec.Linux.IntelRdt.EnableCMT + config.IntelRdt.EnableMBM = spec.Linux.IntelRdt.EnableMBM } } diff --git a/update.go b/update.go index b9a560b242a..7e1c087f17d 100644 --- a/update.go +++ b/update.go @@ -22,6 +22,13 @@ func u64Ptr(i uint64) *uint64 { return &i } func u16Ptr(i uint16) *uint16 { return &i } func boolPtr(b bool) *bool { return &b } +const ( + l3CacheSchemaFlag = "l3-cache-schema" + memBwSchemaFlag = "mem-bw-schema" + enableCMTFlag = "enable-intelrdt-cmt" + enableMBMFlag = "enable-intelrdt-mbm" +) + var updateCommand = cli.Command{ Name: "update", Usage: "update container resource constraints", @@ -119,13 +126,21 @@ other options are ignored. Usage: "Maximum number of pids allowed in the container", }, cli.StringFlag{ - Name: "l3-cache-schema", + Name: l3CacheSchemaFlag, Usage: "The string of Intel RDT/CAT L3 cache schema", }, cli.StringFlag{ - Name: "mem-bw-schema", + Name: memBwSchemaFlag, Usage: "The string of Intel RDT/MBA memory bandwidth schema", }, + cli.BoolFlag{ + Name: enableCMTFlag, + Usage: "Enable Intel RDT/CMT metrics for the container", + }, + cli.BoolFlag{ + Name: enableMBMFlag, + Usage: "Enable Intel RDT/MBM metrics for the container", + }, }, Action: func(context *cli.Context) error { if err := checkArgs(context, 1, exactArgs); err != nil { @@ -301,17 +316,32 @@ other options are ignored. config.Cgroups.Resources.Unified = r.Unified // Update Intel RDT - l3CacheSchema := context.String("l3-cache-schema") - memBwSchema := context.String("mem-bw-schema") - if l3CacheSchema != "" && !intelrdt.IsCATEnabled() { - return errors.New("Intel RDT/CAT: l3 cache schema is not enabled") - } + isL3CacheSchemaFlagSet := context.IsSet(l3CacheSchemaFlag) + l3CacheSchema := context.String(l3CacheSchemaFlag) - if memBwSchema != "" && !intelrdt.IsMBAEnabled() { - return errors.New("Intel RDT/MBA: memory bandwidth schema is not enabled") - } + isMemBwSchemaFlagSet := context.IsSet(memBwSchemaFlag) + memBwSchema := context.String(memBwSchemaFlag) + + isEnableCMTFlagSet := context.IsSet(enableCMTFlag) + enableCMT := context.Bool(enableCMTFlag) + + isEnableMBMFlagSet := context.IsSet(enableMBMFlag) + enableMBM := context.Bool(enableMBMFlag) + + if isL3CacheSchemaFlagSet || isMemBwSchemaFlagSet || isEnableCMTFlagSet || isEnableMBMFlagSet { + if isL3CacheSchemaFlagSet && l3CacheSchema != "" && !intelrdt.IsCATEnabled() { + return errors.New("Intel RDT/CAT: l3 cache schema is not enabled") + } + if isMemBwSchemaFlagSet && memBwSchema != "" && !intelrdt.IsMBAEnabled() { + return errors.New("Intel RDT/MBA: memory bandwidth schema is not enabled") + } + if isEnableCMTFlagSet && enableCMT && !intelrdt.IsCMTEnabled() { + return errors.New("Intel RDT/CMT: CMT is not enabled") + } + if isEnableMBMFlagSet && enableMBM && !intelrdt.IsMBMEnabled() { + return errors.New("Intel RDT/MBM: MBM is not enabled") + } - if l3CacheSchema != "" || memBwSchema != "" { // If intelRdt is not specified in original configuration, we just don't // Apply() to create intelRdt group or attach tasks for this container. // In update command, we could re-enable through IntelRdtManager.Apply() @@ -322,13 +352,28 @@ other options are ignored. return err } config.IntelRdt = &configs.IntelRdt{} + config.IntelRdt.L3CacheSchema = l3CacheSchema + config.IntelRdt.MemBwSchema = memBwSchema + config.IntelRdt.EnableCMT = enableCMT + config.IntelRdt.EnableMBM = enableMBM intelRdtManager := intelrdt.NewManager(&config, container.ID(), state.IntelRdtPath) if err := intelRdtManager.Apply(state.InitProcessPid); err != nil { return err } + } else { + if isL3CacheSchemaFlagSet { + config.IntelRdt.L3CacheSchema = l3CacheSchema + } + if isMemBwSchemaFlagSet { + config.IntelRdt.MemBwSchema = memBwSchema + } + if isEnableCMTFlagSet { + config.IntelRdt.EnableCMT = enableCMT + } + if isEnableMBMFlagSet { + config.IntelRdt.EnableMBM = enableMBM + } } - config.IntelRdt.L3CacheSchema = l3CacheSchema - config.IntelRdt.MemBwSchema = memBwSchema } // XXX(kolyshkin@): currently "runc update" is unable to change