Add mon_group support for resctrl.

cmd/cadvisor.go

@@ -77,6 +77,8 @@ var rawCgroupPrefixWhiteList = flag.String("raw_cgroup_prefix_whitelist", "", "A
 
 var perfEvents = flag.String("perf_events_config", "", "Path to a JSON file containing configuration of perf events to measure. Empty value disabled perf events measuring.")
 
+var resctrlInterval = flag.Duration("resctrl_interval", 0, "Resctrl mon groups updating interval. Zero value disables updating mon groups.")
+
 var (
 	// Metrics to be ignored.
 	// Tcp metrics are ignored by default.
@@ -173,7 +175,7 @@ func main() {
 
 	collectorHttpClient := createCollectorHttpClient(*collectorCert, *collectorKey)
 
-	resourceManager, err := manager.New(memoryStorage, sysFs, housekeepingConfig, includedMetrics, &collectorHttpClient, strings.Split(*rawCgroupPrefixWhiteList, ","), *perfEvents)
+	resourceManager, err := manager.New(memoryStorage, sysFs, housekeepingConfig, includedMetrics, &collectorHttpClient, strings.Split(*rawCgroupPrefixWhiteList, ","), *perfEvents, *resctrlInterval)
 	if err != nil {
 		klog.Fatalf("Failed to create a manager: %s", err)
 	}

cmd/go.mod

@@ -27,6 +27,6 @@ require (
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
 	google.golang.org/api v0.34.0
 	gopkg.in/olivere/elastic.v2 v2.0.12
-	k8s.io/klog/v2 v2.2.0
+	k8s.io/klog/v2 v2.8.0
 	k8s.io/utils v0.0.0-20201110183641-67b214c5f920
 )

cmd/go.sum

@@ -156,8 +156,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
-github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
+github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME=
@@ -822,8 +822,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A=
-k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
+k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

deploy/Dockerfile

@@ -1,6 +1,6 @@
 FROM alpine:3.12 AS build
 
-RUN apk --no-cache add libc6-compat device-mapper findutils zfs build-base linux-headers go bash git wget cmake pkgconfig ndctl-dev && \
+RUN apk --no-cache add libc6-compat device-mapper findutils zfs build-base linux-headers go python3 bash git wget cmake pkgconfig ndctl-dev && \
     apk --no-cache add thin-provisioning-tools --repository http://dl-3.alpinelinux.org/alpine/edge/main/ && \
     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
     rm -rf /var/cache/apk/*
@@ -34,7 +34,7 @@ RUN ./build/build.sh
 FROM alpine:3.12
 MAINTAINER dengnan@google.com vmarmol@google.com vishnuk@google.com jimmidyson@gmail.com stclair@google.com
 
-RUN apk --no-cache add libc6-compat device-mapper findutils zfs ndctl && \
+RUN apk --no-cache add libc6-compat device-mapper findutils zfs ndctl procps && \
     apk --no-cache add thin-provisioning-tools --repository http://dl-3.alpinelinux.org/alpine/edge/main/ && \
     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
     rm -rf /var/cache/apk/*

docs/runtime_options.md

@@ -419,6 +419,11 @@ should be a human readable string that will become a metric name.
 * `cas_count_read` will be measured as uncore non-grouped event on all Integrated Memory Controllers Performance Monitoring Units because of unset `type` field and
 `uncore_imc` prefix.
 
+## Resctrl
+
+```
+--resctrl_interval=0: Resctrl mon groups updating interval. Zero value disables updating mon groups.
+``` 
 
 ## Storage driver specific instructions:
 

go.mod

@@ -46,6 +46,6 @@ require (
 	google.golang.org/grpc v1.27.1
 	google.golang.org/protobuf v1.25.0 // indirect
 	gotest.tools/v3 v3.0.3 // indirect
-	k8s.io/klog/v2 v2.2.0
+	k8s.io/klog/v2 v2.8.0
 	k8s.io/utils v0.0.0-20201110183641-67b214c5f920
 )

go.sum

@@ -89,8 +89,8 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
-github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
+github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -489,8 +489,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A=
-k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
+k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

manager/container.go

@@ -130,6 +130,7 @@ func (cd *containerData) Stop() error {
 	}
 	close(cd.stop)
 	cd.perfCollector.Destroy()
+	cd.resctrlCollector.Destroy()
 	return nil
 }
 
@@ -727,7 +728,7 @@ func (cd *containerData) updateStats() error {
 		return perfStatsErr
 	}
 	if resctrlStatsErr != nil {
-		klog.Errorf("error occurred while collecting resctrl stats for container %s: %s", cInfo.Name, err)
+		klog.Errorf("error occurred while collecting resctrl stats for container %s: %s", cInfo.Name, resctrlStatsErr)
 		return resctrlStatsErr
 	}
 	return customStatsErr

manager/manager.go

@@ -49,7 +49,6 @@ import (
 
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/opencontainers/runc/libcontainer/cgroups/fs2"
-	"github.com/opencontainers/runc/libcontainer/intelrdt"
 
 	"k8s.io/klog/v2"
 	"k8s.io/utils/clock"
@@ -147,7 +146,7 @@ type HouskeepingConfig = struct {
 }
 
 // New takes a memory storage and returns a new manager.
-func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, houskeepingConfig HouskeepingConfig, includedMetricsSet container.MetricSet, collectorHTTPClient *http.Client, rawContainerCgroupPathPrefixWhiteList []string, perfEventsFile string) (Manager, error) {
+func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, houskeepingConfig HouskeepingConfig, includedMetricsSet container.MetricSet, collectorHTTPClient *http.Client, rawContainerCgroupPathPrefixWhiteList []string, perfEventsFile string, resctrlInterval time.Duration) (Manager, error) {
 	if memoryCache == nil {
 		return nil, fmt.Errorf("manager requires memory storage")
 	}
@@ -218,7 +217,7 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, houskeepingConfig
 		return nil, err
 	}
 
-	newManager.resctrlManager, err = resctrl.NewManager(selfContainer)
+	newManager.resctrlManager, err = resctrl.NewManager(resctrlInterval, resctrl.Setup)
 	if err != nil {
 		klog.V(4).Infof("Cannot gather resctrl metrics: %v", err)
 	}
@@ -263,7 +262,7 @@ type manager struct {
 	collectorHTTPClient      *http.Client
 	nvidiaManager            stats.Manager
 	perfManager              stats.Manager
-	resctrlManager           stats.Manager
+	resctrlManager           resctrl.Manager
 	// List of raw container cgroup path prefix whitelist.
 	rawContainerCgroupPathPrefixWhiteList []string
 }
@@ -328,7 +327,7 @@ func (m *manager) Start() error {
 
 func (m *manager) Stop() error {
 	defer m.nvidiaManager.Destroy()
-	defer m.destroyPerfCollectors()
+	defer m.destroyCollectors()
 	// Stop and wait on all quit channels.
 	for i, c := range m.quitChannels {
 		// Send the exit signal and wait on the thread to exit (by closing the channel).
@@ -346,9 +345,10 @@ func (m *manager) Stop() error {
 	return nil
 }
 
-func (m *manager) destroyPerfCollectors() {
+func (m *manager) destroyCollectors() {
 	for _, container := range m.containers {
 		container.perfCollector.Destroy()
+		container.resctrlCollector.Destroy()
 	}
 }
 
@@ -957,14 +957,11 @@ func (m *manager) createContainerLocked(containerName string, watchSource watche
 	}
 
 	if m.includedMetrics.Has(container.ResctrlMetrics) {
-		resctrlPath, err := intelrdt.GetIntelRdtPath(containerName)
+		cont.resctrlCollector, err = m.resctrlManager.GetCollector(containerName, func() ([]string, error) {
+			return cont.getContainerPids(true)
+		}, len(m.machineInfo.Topology))
 		if err != nil {
-			klog.V(4).Infof("Error getting resctrl path: %q", err)
-		} else {
-			cont.resctrlCollector, err = m.resctrlManager.GetCollector(resctrlPath)
-			if err != nil {
-				klog.V(4).Infof("resctrl metrics will not be available for container %s: %s", cont.info.Name, err)
-			}
+			klog.V(4).Infof("resctrl metrics will not be available for container %s: %s", cont.info.Name, err)
 		}
 	}
 
@@ -1006,7 +1003,6 @@ func (m *manager) createContainerLocked(containerName string, watchSource watche
 	if err != nil {
 		return err
 	}
-
 	// Start the container's housekeeping.
 	return cont.Start()
 }

resctrl/collector.go

@@ -1,6 +1,6 @@
 // +build linux
 
-// Copyright 2020 Google Inc. All Rights Reserved.
+// Copyright 2021 Google Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,57 +18,142 @@
 package resctrl
 
 import (
-	info "github.com/google/cadvisor/info/v1"
-	"github.com/google/cadvisor/stats"
+	"fmt"
+	"os"
+	"sync"
+	"time"
+
+	"k8s.io/klog/v2"
 
-	"github.com/opencontainers/runc/libcontainer/configs"
-	"github.com/opencontainers/runc/libcontainer/intelrdt"
+	info "github.com/google/cadvisor/info/v1"
 )
 
 type collector struct {
-	resctrl intelrdt.Manager
-	stats.NoopDestroy
+	id                string
+	interval          time.Duration
+	getContainerPids  func() ([]string, error)
+	resctrlPath       string
+	running           bool
+	numberOfNUMANodes int
+	mu                sync.Mutex
 }
 
-func newCollector(id string, resctrlPath string) *collector {
-	collector := &collector{
-		resctrl: intelrdt.NewManager(
-			&configs.Config{
-				IntelRdt: &configs.IntelRdt{},
-			},
-			id,
-			resctrlPath,
-		),
+func newCollector(id string, getContainerPids func() ([]string, error), interval time.Duration, numberOfNUMANodes int) *collector {
+	return &collector{id: id, interval: interval, getContainerPids: getContainerPids, numberOfNUMANodes: numberOfNUMANodes,
+		mu: sync.Mutex{}}
+}
+
+func (c *collector) setup() error {
+	var err error
+	c.resctrlPath, err = prepareMonitoringGroup(c.id, c.getContainerPids)
+
+	if c.interval != 0 {
+		if err != nil {
+			c.running = false
+			klog.Errorf("Failed to setup container %q resctrl collector: %w \n Trying again in next intervals.", c.id, err)
+		} else {
+			c.running = true
+		}
+		go func() {
+			for {
+				time.Sleep(c.interval)
+				c.mu.Lock()
+				klog.V(5).Infof("Trying to check %q containers control group.", c.id)
+				if c.running {
+					err = c.checkMonitoringGroup()
+					if err != nil {
+						c.running = false
+						klog.Errorf("Failed to check %q resctrl collector control group: %w \n Trying again in next intervals.", c.id, err)
+					}
+				} else {
+					c.resctrlPath, err = prepareMonitoringGroup(c.id, c.getContainerPids)
+					if err != nil {
+						c.running = false
+						klog.Errorf("Failed to setup container %q resctrl collector: %w \n Trying again in next intervals.", c.id, err)
+					} else {
+						c.running = true
+					}
+				}
+				c.mu.Unlock()
+			}
+		}()
+	} else {
+		// There is no interval set, if setup fail, stop.
+		if err != nil {
+			return fmt.Errorf("failed to setup container %q resctrl collector: %w", c.id, err)
+		}
+		c.running = true
 	}
 
-	return collector
+	return nil
 }
 
-func (c *collector) UpdateStats(stats *info.ContainerStats) error {
-	stats.Resctrl = info.ResctrlStats{}
-
-	resctrlStats, err := c.resctrl.GetStats()
+func (c *collector) checkMonitoringGroup() error {
+	newPath, err := prepareMonitoringGroup(c.id, c.getContainerPids)
 	if err != nil {
-		return err
+		return fmt.Errorf("couldn't obtain mon_group path: %v", err)
 	}
 
-	numberOfNUMANodes := len(*resctrlStats.MBMStats)
+	// Check if container moved between control groups.
+	if newPath != c.resctrlPath {
+		err = c.clear()
+		if err != nil {
+			return fmt.Errorf("couldn't clear previous monitoring group: %w", err)
+		}
+		c.resctrlPath = newPath
+	}
 
-	stats.Resctrl.MemoryBandwidth = make([]info.MemoryBandwidthStats, 0, numberOfNUMANodes)
-	stats.Resctrl.Cache = make([]info.CacheStats, 0, numberOfNUMANodes)
+	return nil
+}
 
-	for _, numaNodeStats := range *resctrlStats.MBMStats {
-		stats.Resctrl.MemoryBandwidth = append(stats.Resctrl.MemoryBandwidth,
-			info.MemoryBandwidthStats{
-				TotalBytes: numaNodeStats.MBMTotalBytes,
-				LocalBytes: numaNodeStats.MBMLocalBytes,
-			})
+func (c *collector) UpdateStats(stats *info.ContainerStats) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.running {
+		stats.Resctrl = info.ResctrlStats{}
+
+		resctrlStats, err := getIntelRDTStatsFrom(c.resctrlPath)
+		if err != nil {
+			return err
+		}
+
+		stats.Resctrl.MemoryBandwidth = make([]info.MemoryBandwidthStats, 0, c.numberOfNUMANodes)
+		stats.Resctrl.Cache = make([]info.CacheStats, 0, c.numberOfNUMANodes)
+
+		for _, numaNodeStats := range *resctrlStats.MBMStats {
+			stats.Resctrl.MemoryBandwidth = append(stats.Resctrl.MemoryBandwidth,
+				info.MemoryBandwidthStats{
+					TotalBytes: numaNodeStats.MBMTotalBytes,
+					LocalBytes: numaNodeStats.MBMLocalBytes,
+				})
+		}
+
+		for _, numaNodeStats := range *resctrlStats.CMTStats {
+			stats.Resctrl.Cache = append(stats.Resctrl.Cache,
+				info.CacheStats{LLCOccupancy: numaNodeStats.LLCOccupancy})
+		}
 	}
 
-	for _, numaNodeStats := range *resctrlStats.CMTStats {
-		stats.Resctrl.Cache = append(stats.Resctrl.Cache,
-			info.CacheStats{LLCOccupancy: numaNodeStats.LLCOccupancy})
+	return nil
+}
+
+func (c *collector) Destroy() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.running = false
+	err := c.clear()
+	if err != nil {
+		klog.Errorf("trying to destroy %q resctrl collector but: %v", c.id, err)
 	}
+}
 
+func (c *collector) clear() error {
+	// Not allowed to remove root or undefined resctrl directory.
+	if c.id != rootContainer && c.resctrlPath != "" {
+		err := os.RemoveAll(c.resctrlPath)
+		if err != nil {
+			return fmt.Errorf("couldn't clear mon_group: %v", err)
+		}
+	}
 	return nil
 }