Add metrics for the error count and duration of OVS flow operation on…

… node Signed-off-by: Yuki Tsuboi <ytsuboi@vmware.com>
antrea-io · Jun 28, 2020 · c04db3b · c04db3b
1 parent aafd76e
commit c04db3b
Show file tree

Hide file tree

Showing 3 changed files with 145 additions and 8 deletions.
diff --git a/pkg/agent/metrics/prometheus.go b/pkg/agent/metrics/prometheus.go
@@ -67,6 +67,54 @@ var (
 		Help:           "Flow count for each OVS flow table. The TableID is used as a label.",
 		StabilityLevel: metrics.STABLE,
 	}, []string{"table_id"})
+
+	OVSFlowAddErrorCount = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Name:           "antrea_agent_ovs_flow_add_error_count",
+			Help:           "Number of OVS flow adding errors.",
+			StabilityLevel: metrics.STABLE,
+		},
+	)
+
+	OVSFlowModifyErrorCount = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Name:           "antrea_agent_ovs_flow_modify_error_count",
+			Help:           "Number of OVS flow modifying errors.",
+			StabilityLevel: metrics.STABLE,
+		},
+	)
+
+	OVSFlowDeleteErrorCount = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Name:           "antrea_agent_ovs_flow_delete_error_count",
+			Help:           "Number of OVS flow deleting errors.",
+			StabilityLevel: metrics.STABLE,
+		},
+	)
+
+	OVSFlowAddDuration = metrics.NewSummary(
+		&metrics.SummaryOpts{
+			Name:           "antrea_agent_ovs_flow_add_duration_milliseconds",
+			Help:           "The duration of adding ovs flows",
+			StabilityLevel: metrics.STABLE,
+		},
+	)
+
+	OVSFlowModifyDuration = metrics.NewSummary(
+		&metrics.SummaryOpts{
+			Name:           "antrea_agent_ovs_flow_modify_duration_milliseconds",
+			Help:           "The duration of modifying ovs flows",
+			StabilityLevel: metrics.STABLE,
+		},
+	)
+
+	OVSFlowDeleteDuration = metrics.NewSummary(
+		&metrics.SummaryOpts{
+			Name:           "antrea_agent_ovs_flow_delete_duration_milliseconds",
+			Help:           "The duration of deleting ovs flows",
+			StabilityLevel: metrics.STABLE,
+		},
+	)
 )
 
 func InitializePrometheusMetrics() {
@@ -112,4 +160,22 @@ func InitializePrometheusMetrics() {
 	if err := legacyregistry.Register(OVSFlowCount); err != nil {
 		klog.Error("Failed to register antrea_agent_ovs_flow_count with Prometheus")
 	}
+	if err := legacyregistry.Register(OVSFlowAddErrorCount); err != nil {
+		klog.Error("Failed to register antrea_agent_ovs_flow_add_error_count with Prometheus")
+	}
+	if err := legacyregistry.Register(OVSFlowModifyErrorCount); err != nil {
+		klog.Error("Failed to register antrea_agent_ovs_flow_modify_error_count with Prometheus")
+	}
+	if err := legacyregistry.Register(OVSFlowDeleteErrorCount); err != nil {
+		klog.Error("Failed to register antrea_agent_ovs_flow_delete_error_count with Prometheus")
+	}
+	if err := legacyregistry.Register(OVSFlowAddDuration); err != nil {
+		klog.Error("Failed to register antrea_agent_ovs_flow_add_duration_milliseconds with Prometheus")
+	}
+	if err := legacyregistry.Register(OVSFlowModifyDuration); err != nil {
+		klog.Error("Failed to register antrea_agent_ovs_flow_modify_duration_milliseconds with Prometheus")
+	}
+	if err := legacyregistry.Register(OVSFlowDeleteDuration); err != nil {
+		klog.Error("Failed to register antrea_agent_ovs_flow_delete_duration_milliseconds with Prometheus")
+	}
 }
diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go
@@ -20,8 +20,10 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/vmware-tanzu/antrea/pkg/agent/config"
+	"github.com/vmware-tanzu/antrea/pkg/agent/metrics"
 	"github.com/vmware-tanzu/antrea/pkg/agent/openflow/cookie"
 	"github.com/vmware-tanzu/antrea/pkg/agent/types"
 	binding "github.com/vmware-tanzu/antrea/pkg/ovs/openflow"
@@ -194,31 +196,94 @@ func (c *client) GetTunnelVirtualMAC() net.HardwareAddr {
 }
 
 func (c *client) Add(flow binding.Flow) error {
-	return c.bridge.AddFlowsInBundle([]binding.Flow{flow}, nil, nil)
+	startTime := time.Now()
+	defer func() {
+		d := time.Since(startTime)
+		metrics.OVSFlowAddDuration.Observe(float64(d.Milliseconds()))
+	}()
+	if err := c.bridge.AddFlowsInBundle([]binding.Flow{flow}, nil, nil); err != nil {
+		metrics.OVSFlowAddErrorCount.Inc()
+		return err
+	}
+	return nil
 }
 
 func (c *client) Modify(flow binding.Flow) error {
-	return c.bridge.AddFlowsInBundle(nil, []binding.Flow{flow}, nil)
+	startTime := time.Now()
+	defer func() {
+		d := time.Since(startTime)
+		metrics.OVSFlowModifyDuration.Observe(float64(d.Milliseconds()))
+	}()
+	if err := c.bridge.AddFlowsInBundle(nil, []binding.Flow{flow}, nil); err != nil {
+		metrics.OVSFlowModifyErrorCount.Inc()
+		return err
+	}
+	return nil
 }
 
 func (c *client) Delete(flow binding.Flow) error {
-	return c.bridge.AddFlowsInBundle(nil, nil, []binding.Flow{flow})
+	startTime := time.Now()
+	defer func() {
+		d := time.Since(startTime)
+		metrics.OVSFlowDeleteDuration.Observe(float64(d.Milliseconds()))
+	}()
+	if err := c.bridge.AddFlowsInBundle(nil, nil, []binding.Flow{flow}); err != nil {
+		metrics.OVSFlowDeleteErrorCount.Inc()
+		return err
+	}
+	return nil
 }
 
 func (c *client) AddAll(flows []binding.Flow) error {
-	return c.bridge.AddFlowsInBundle(flows, nil, nil)
+	startTime := time.Now()
+	defer func() {
+		d := time.Since(startTime)
+		metrics.OVSFlowAddDuration.Observe(float64(d.Milliseconds()))
+	}()
+	if err := c.bridge.AddFlowsInBundle(flows, nil, nil); err != nil {
+		metrics.OVSFlowAddErrorCount.Inc()
+		return err
+	}
+	return nil
 }
 
 func (c *client) DeleteAll(flows []binding.Flow) error {
-	return c.bridge.AddFlowsInBundle(nil, nil, flows)
+	startTime := time.Now()
+	defer func() {
+		d := time.Since(startTime)
+		metrics.OVSFlowDeleteDuration.Observe(float64(d.Milliseconds()))
+	}()
+	if err := c.bridge.AddFlowsInBundle(nil, nil, flows); err != nil {
+		metrics.OVSFlowDeleteErrorCount.Inc()
+		return err
+	}
+	return nil
 }
 
 func (c *client) AddOFEntries(ofEntries []binding.OFEntry) error {
-	return c.bridge.AddOFEntriesInBundle(ofEntries, nil, nil)
+	startTime := time.Now()
+	defer func() {
+		d := time.Since(startTime)
+		metrics.OVSFlowAddDuration.Observe(float64(d.Milliseconds()))
+	}()
+	if err := c.bridge.AddOFEntriesInBundle(ofEntries, nil, nil); err != nil {
+		metrics.OVSFlowAddErrorCount.Inc()
+		return err
+	}
+	return nil
 }
 
 func (c *client) DeleteOFEntries(ofEntries []binding.OFEntry) error {
-	return c.bridge.AddOFEntriesInBundle(nil, nil, ofEntries)
+	startTime := time.Now()
+	defer func() {
+		d := time.Since(startTime)
+		metrics.OVSFlowDeleteDuration.Observe(float64(d.Milliseconds()))
+	}()
+	if err := c.bridge.AddOFEntriesInBundle(nil, nil, ofEntries); err != nil {
+		metrics.OVSFlowDeleteErrorCount.Inc()
+		return err
+	}
+	return nil
 }
 
 // defaultFlows generates the default flows of all tables.

diff --git a/test/e2e/prometheus_test.go b/test/e2e/prometheus_test.go
@@ -39,8 +39,14 @@ var antreaAgentMetrics = []string{
 	"antrea_agent_ingress_networkpolicy_rule_count",
 	"antrea_agent_local_pod_count",
 	"antrea_agent_networkpolicy_count",
-	"antrea_agent_ovs_total_flow_count",
 	"antrea_agent_ovs_flow_count",
+	"antrea_agent_ovs_flow_add_error_count",
+	"antrea_agent_ovs_flow_add_duration_milliseconds",
+	"antrea_agent_ovs_flow_modify_duration_milliseconds",
+	"antrea_agent_ovs_flow_modify_error_count",
+	"antrea_agent_ovs_flow_delete_duration_milliseconds",
+	"antrea_agent_ovs_flow_delete_error_count",
+	"antrea_agent_ovs_total_flow_count",
 	"antrea_agent_runtime_info",
 }