libcontainer/intelrdt: add basic "MON" groups support.

More info: https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt

Signed-off-by: Paweł Szulik <pawel.szulik@intel.com>

go.mod

@@ -12,7 +12,7 @@ require (
  github.com/godbus/dbus/v5 v5.0.5
  github.com/moby/sys/mountinfo v0.4.1
  github.com/mrunalp/fileutils v0.5.0
- github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
+ github.com/opencontainers/runtime-spec v1.0.3-0.20210910115017-0d6cc581aeea
  github.com/opencontainers/selinux v1.9.1
  github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921
  github.com/sirupsen/logrus v1.8.1

go.sum

@@ -36,6 +36,8 @@ github.com/mrunalp/fileutils v0.5.0 h1:NKzVxiH7eSk+OQ4M+ZYW1K6h27RUV3MI6NUTsHhU6
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc=
 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20210910115017-0d6cc581aeea h1:WmF5mV2OwWlHap/Ol8Z+iLZVlvLJrG7PzO/j8vwSLz8=
+github.com/opencontainers/runtime-spec v1.0.3-0.20210910115017-0d6cc581aeea/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/selinux v1.9.1 h1:b4VPEF3O5JLZgdTDBmGepaaIbAo0GqoF6EBRq5f/g3Y=
 github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

libcontainer/SPEC.md

@@ -158,32 +158,38 @@ init process will block waiting for the parent to finish setup.
 ### IntelRdt
 
 Intel platforms with new Xeon CPU support Resource Director Technology (RDT).
-Cache Allocation Technology (CAT) and Memory Bandwidth Allocation (MBA) are
-two sub-features of RDT.
+Cache Allocation Technology (CAT), Cache Monitoring Technology (CMT),
+Memory Bandwidth Allocation (MBA) and Memory Bandwidth Monitoring (MBM) are
+four sub-features of RDT.
 
 Cache Allocation Technology (CAT) provides a way for the software to restrict
 cache allocation to a defined 'subset' of L3 cache which may be overlapping
 with other 'subsets'. The different subsets are identified by class of
 service (CLOS) and each CLOS has a capacity bitmask (CBM).
 
+Cache Monitoring Technology (CMT) supports monitoring of the last-level cache (LLC) occupancy
+for each running thread simultaneously.
+
 Memory Bandwidth Allocation (MBA) provides indirect and approximate throttle
 over memory bandwidth for the software. A user controls the resource by
-indicating the percentage of maximum memory bandwidth or memory bandwidth limit
-in MBps unit if MBA Software Controller is enabled.
+indicating the percentage of maximum memory bandwidth or memory bandwidth
+limit in MBps unit if MBA Software Controller is enabled.
+
+Memory Bandwidth Monitoring (MBM) supports monitoring of total and local memory bandwidth
+for each running thread simultaneously.
 
-It can be used to handle L3 cache and memory bandwidth resources allocation
-for containers if hardware and kernel support Intel RDT CAT and MBA features.
+More details about Intel RDT CAT and MBA can be found in the section 17.18 and 17.19, Volume 3
+of Intel Software Developer Manual:
+https://software.intel.com/en-us/articles/intel-sdm
 
-In Linux 4.10 kernel or newer, the interface is defined and exposed via
+About Intel RDT kernel interface:
+In Linux 4.14 kernel or newer, the interface is defined and exposed via
 "resource control" filesystem, which is a "cgroup-like" interface.
 
 Comparing with cgroups, it has similar process management lifecycle and
 interfaces in a container. But unlike cgroups' hierarchy, it has single level
 filesystem layout.
 
-CAT and MBA features are introduced in Linux 4.10 and 4.12 kernel via
-"resource control" filesystem.
-
 Intel RDT "resource control" filesystem hierarchy:
 ```
 mount -t resctrl resctrl /sys/fs/resctrl
@@ -194,25 +200,46 @@ tree /sys/fs/resctrl
 | | |-- cbm_mask
 | | |-- min_cbm_bits
 | | |-- num_closids
+| |-- L3_MON
+| | |-- max_threshold_occupancy
+| | |-- mon_features
+| | |-- num_rmids
 | |-- MB
 | |-- bandwidth_gran
 | |-- delay_linear
 | |-- min_bandwidth
 | |-- num_closids
-|-- ...
+|-- mon_groups
+ |-- <rmid>
+ |-- ...
+ |-- mon_data
+ |-- mon_L3_00
+ |-- llc_occupancy
+ |-- mbm_local_bytes
+ |-- mbm_total_bytes
+ |-- ...
+ |-- tasks
 |-- schemata
 |-- tasks
-|-- <container_id>
+|-- <clos>
  |-- ...
- |-- schemata
+ |-- mon_data
+ |-- mon_L3_00
+ |-- llc_occupancy
+ |-- mbm_local_bytes
+ |-- mbm_total_bytes
+ |-- ...
  |-- tasks
+ |-- schemata
+|-- ...
 ```
 
 For runc, we can make use of `tasks` and `schemata` configuration for L3
-cache and memory bandwidth resources constraints.
+cache and memory bandwidth resources constraints, `mon_data` directory for
+CMT and MBM statistics.
 
 The file `tasks` has a list of tasks that belongs to this group (e.g.,
-<container_id>" group). Tasks can be added to a group by writing the task ID
+"<clos>" group). Tasks can be added to a group by writing the task ID
 to the "tasks" file (which will automatically remove them from the previous
 group to which they belonged). New tasks created by fork(2) and clone(2) are
 added to the same group as their parent.
@@ -224,7 +251,7 @@ L3 cache schema:
 It has allocation bitmasks/values for L3 cache on each socket, which
 contains L3 cache id and capacity bitmask (CBM).
 ```
- Format: "L3:<cache_id0>=<cbm0>;<cache_id1>=<cbm1>;..."
+Format: "L3:<cache_id0>=<cbm0>;<cache_id1>=<cbm1>;..."
 ```
 For example, on a two-socket machine, the schema line could be "L3:0=ff;1=c0"
 which means L3 cache id 0's CBM is 0xff, and L3 cache id 1's CBM is 0xc0.
@@ -240,7 +267,7 @@ Memory bandwidth schema:
 It has allocation values for memory bandwidth on each socket, which contains
 L3 cache id and memory bandwidth.
 ```
- Format: "MB:<cache_id0>=bandwidth0;<cache_id1>=bandwidth1;..."
+Format: "MB:<cache_id0>=bandwidth0;<cache_id1>=bandwidth1;..."
 ```
 For example, on a two-socket machine, the schema line could be "MB:0=20;1=70"
 
@@ -251,8 +278,10 @@ that is allocated is also dependent on the CPU model and can be looked up at
 min_bw + N * bw_gran. Intermediate values are rounded to the next control
 step available on the hardware.
 
-If MBA Software Controller is enabled through mount option "-o mba_MBps"
+If MBA Software Controller is enabled through mount option "-o mba_MBps":
+```
 mount -t resctrl resctrl -o mba_MBps /sys/fs/resctrl
+```
 We could specify memory bandwidth in "MBps" (Mega Bytes per second) unit
 instead of "percentages". The kernel underneath would use a software feedback
 mechanism or a "Software Controller" which reads the actual bandwidth using
@@ -263,11 +292,12 @@ For example, on a two-socket machine, the schema line could be
 "MB:0=5000;1=7000" which means 5000 MBps memory bandwidth limit on socket 0
 and 7000 MBps memory bandwidth limit on socket 1.
 
-For more information about Intel RDT kernel interface: 
+For more information about Intel RDT kernel interface:
 https://www.kernel.org/doc/Documentation/x86/intel_rdt_ui.txt
 
-```
+
 An example for runc:
+```
 Consider a two-socket machine with two L3 caches where the default CBM is
 0x7ff and the max CBM length is 11 bits, and minimum memory bandwidth of 10%
 with a memory bandwidth granularity of 10%.
@@ -281,7 +311,17 @@ maximum memory bandwidth of 20% on socket 0 and 70% on socket 1.
  "closID": "guaranteed_group",
  "l3CacheSchema": "L3:0=7f0;1=1f",
  "memBwSchema": "MB:0=20;1=70"
- }
+ }
+}
+```
+Another example:
+```
+We only want to monitor memory bandwidth and llc occupancy.
+"linux": {
+ "intelRdt": {
+ "enableMBM": true,
+ "enableCMT": true
+ }
 }
 ```
 

libcontainer/configs/config.go

@@ -199,7 +199,7 @@ type Config struct {
  NoNewKeyring bool `json:"no_new_keyring"`
 
  // IntelRdt specifies settings for Intel RDT group that the container is placed into
- // to limit the resources (e.g., L3 cache, memory bandwidth) the container has available
+ // to limit the resources (e.g., L3 cache, memory bandwidth) the container has available.
  IntelRdt *IntelRdt `json:"intel_rdt,omitempty"`
 
  // RootlessEUID is set when the runc was launched with non-zero EUID.

libcontainer/configs/intelrdt.go

@@ -13,4 +13,12 @@ type IntelRdt struct {
  // The unit of memory bandwidth is specified in "percentages" by
  // default, and in "MBps" if MBA Software Controller is enabled.
  MemBwSchema string `json:"memBwSchema,omitempty"`
+
+ // The flag to indicate if Intel RDT CMT is enabled. CMT (Cache Monitoring Technology) supports monitoring of
+ // the last-level cache (LLC) occupancy for the container.
+ EnableCMT bool `json:"enableCMT,omitempty"`
+
+ // The flag to indicate if Intel RDT MBM is enabled. MBM (Memory Bandwidth Monitoring) supports monitoring of
+ // total and local memory bandwidth for the container.
+ EnableMBM bool `json:"enableMBM,omitempty"`
 }

libcontainer/configs/validate/validator.go

@@ -198,20 +198,24 @@ func (v *ConfigValidator) sysctl(config *configs.Config) error {
 
 func (v *ConfigValidator) intelrdt(config *configs.Config) error {
  if config.IntelRdt != nil {
- if !intelrdt.IsCATEnabled() && !intelrdt.IsMBAEnabled() {
- return errors.New("intelRdt is specified in config, but Intel RDT is not supported or enabled")
+ if config.IntelRdt.L3CacheSchema == "" && config.IntelRdt.MemBwSchema == "" && !config.IntelRdt.EnableCMT && !config.IntelRdt.EnableMBM {
+ return errors.New("intelRdt config is specified but values are empty")
  }
-
  if config.IntelRdt.ClosID == "." || config.IntelRdt.ClosID == ".." || strings.Contains(config.IntelRdt.ClosID, "/") {
  return fmt.Errorf("invalid intelRdt.ClosID %q", config.IntelRdt.ClosID)
  }
-
  if !intelrdt.IsCATEnabled() && config.IntelRdt.L3CacheSchema != "" {
  return errors.New("intelRdt.l3CacheSchema is specified in config, but Intel RDT/CAT is not enabled")
  }
  if !intelrdt.IsMBAEnabled() && config.IntelRdt.MemBwSchema != "" {
  return errors.New("intelRdt.memBwSchema is specified in config, but Intel RDT/MBA is not enabled")
  }
+ if !intelrdt.IsCMTEnabled() && config.IntelRdt.EnableCMT {
+ return errors.New("intelRdt.enableCMT is specified in config, but Intel RDT/CMT is not enabled")
+ }
+ if !intelrdt.IsMBMEnabled() && config.IntelRdt.EnableMBM {
+ return errors.New("intelRdt.enableMBM is specified in config, but Intel RDT/MBM is not enabled")
+ }
  }
 
  return nil

libcontainer/container_linux.go

@@ -1948,6 +1948,7 @@ func (c *linuxContainer) currentState() (*State, error) {
  if c.intelRdtManager != nil {
  intelRdtPath = c.intelRdtManager.GetPath()
  }
+
  state := &State{
  BaseState: BaseState{
  ID: c.ID(),

libcontainer/factory_linux.go

@@ -52,7 +52,7 @@ func InitArgs(args ...string) func(*LinuxFactory) error {
 // containers that use the Intel RDT "resource control" filesystem to
 // create and manage Intel RDT resources (e.g., L3 cache, memory bandwidth).
 func IntelRdtFs(l *LinuxFactory) error {
- if !intelrdt.IsCATEnabled() && !intelrdt.IsMBAEnabled() {
+ if !intelrdt.IsCATEnabled() && !intelrdt.IsMBAEnabled() && !intelrdt.IsCMTEnabled() && !intelrdt.IsMBMEnabled() {
  l.NewIntelRdtManager = nil
  } else {
  l.NewIntelRdtManager = func(config *configs.Config, id string, path string) intelrdt.Manager {