Add Configuration for Metrics Cardinality Simplification

Number of metrics produced can be reduced by controlling the depth
or the level using configmap. Available levels are namespace, run
pipelinerun/taskrun or pipeline/task. Also, type of metrics can
be changed from histogram to prometheus gauge/opencensus last value.
Doing this will reduce the load on metrics databases.

config/config-observability.yaml

@@ -54,3 +54,7 @@ data:
     # charge.  If metrics.backend-destination is not Stackdriver, this is
     # ignored.
     metrics.allow-stackdriver-custom-metrics: "false"
+    metrics.taskrun.level: "taskrun"
+    metrics.taskrun.duration-type: "histogram"
+    metrics.pipelinerun.level: "pipelinerun"
+    metrics.pipelinerun.duration-type: "histogram"

pkg/apis/config/metrics.go

@@ -0,0 +1,148 @@
+/*
+Copyright 2021 The Tekton Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	"knative.dev/pkg/metrics"
+)
+
+const (
+	// metricsTaskrunLevel determines to what level to aggregate metrics
+	// for taskrun
+	metricsTaskrunLevelKey = "metrics.taskrun.level"
+
+	// metricsPipelinerunLevel determines to what level to aggregate metrics
+	// for pipelinerun
+	metricsPipelinerunLevelKey = "metrics.pipelinerun.level"
+	// metricsDurationTaskrunType determines what type of
+	// metrics to use for aggregating duration for taskrun
+	metricsDurationTaskrunType = "metrics.taskrun.duration-type"
+	// metricsDurationPipelinerunType determines what type of
+	// metrics to use for aggregating duration for pipelinerun
+	metricsDurationPipelinerunType = "metrics.pipelinerun.duration-type"
+
+	// DefaultTaskrunLevel determines to what level to aggregate metrics
+	// when it isn't specified in configmap
+	// TBD: Change to task in next
+	// relase and taskrun level will be deprecated
+	DefaultTaskrunLevel = "taskrun"
+	// TaskrunLevelAtTaskrun specify that aggregation will be done at
+	// taskrun level
+	TaskrunLevelAtTaskrun = "taskrun"
+	// TaskrunLevelAtTask specify that aggregation will be done at task level
+	TaskrunLevelAtTask = "task"
+	// TaskrunLevelAtNS specify that aggregation will be done at namespace level
+	TaskrunLevelAtNS = "namespace"
+	// DefaultPipelinerunLevel determines to what level to aggregate metrics
+	// when it isn't specified in configmap
+	// TBD: Change to pipeline in next
+	// relase and pipelinerun level will be deprecated
+	DefaultPipelinerunLevel = "pipelinerun"
+	// PipelinerunLevelAtPipelinerun specify that aggregation will be done at
+	// pipelienrun level
+	PipelinerunLevelAtPipelinerun = "pipelinerun"
+	// PipelinerunLevelAtPipeline specify that aggregation will be done at
+	// pipeline level
+	PipelinerunLevelAtPipeline = "pipeline"
+	// PipelinerunLevelAtNS specify that aggregation will be done at
+	// namespace level
+	PipelinerunLevelAtNS = "namespace"
+
+	// DefaultDurationTaskrunType determines what type
+	// of metrics to use when we don't specify one in
+	// configmap
+	DefaultDurationTaskrunType = "histogram"
+	// DurationTaskrunTypeHistogram specify that histogram
+	// type metrics need to be use for Duration of Taskrun
+	DurationTaskrunTypeHistogram = "histogram"
+	// DurationTaskrunTypeHistogram specify that lastValue or
+	// gauge type metrics need to be use for Duration of Taskrun
+	DurationTaskrunTypeLastValue = "lastvalue"
+
+	// DefaultDurationPipelinerunType determines what type
+	// of metrics to use when we don't specify one in
+	// configmap
+	DefaultDurationPipelinerunType = "histogram"
+	// DurationPipelinerunTypeHistogram specify that histogram
+	// type metrics need to be use for Duration of Pipelinerun
+	DurationPipelinerunTypeHistogram = "histogram"
+	// DurationPipelinerunTypeHistogram specify that lastValue or
+	// gauge type metrics need to be use for Duration of Pipelinerun
+	DurationPipelinerunTypeLastValue = "lastvalue"
+)
+
+// Metrics holds the configurations for the metrics
+// +k8s:deepcopy-gen=true
+type Metrics struct {
+	TaskrunLevel            string
+	PipelinerunLevel        string
+	DurationTaskrunType     string
+	DurationPipelinerunType string
+}
+
+// GetArtifactBucketConfigName returns the name of the configmap containing all
+// customizations for the storage bucket.
+func GetMetricsConfigName() string {
+	return metrics.ConfigMapName()
+}
+
+// Equals returns true if two Configs are identical
+func (cfg *Metrics) Equals(other *Metrics) bool {
+	if cfg == nil && other == nil {
+		return true
+	}
+
+	if cfg == nil || other == nil {
+		return false
+	}
+
+	return other.TaskrunLevel == cfg.TaskrunLevel &&
+		other.PipelinerunLevel == cfg.PipelinerunLevel &&
+		other.DurationTaskrunType == cfg.DurationTaskrunType &&
+		other.DurationPipelinerunType == cfg.DurationPipelinerunType
+}
+
+// NewMetricsFromMap returns a Config given a map corresponding to a ConfigMap
+func NewMetricsFromMap(cfgMap map[string]string) (*Metrics, error) {
+	tc := Metrics{
+		TaskrunLevel:            DefaultTaskrunLevel,
+		PipelinerunLevel:        DefaultPipelinerunLevel,
+		DurationTaskrunType:     DefaultDurationTaskrunType,
+		DurationPipelinerunType: DefaultDurationPipelinerunType,
+	}
+
+	if taskrunLevel, ok := cfgMap[metricsTaskrunLevelKey]; ok {
+		tc.TaskrunLevel = taskrunLevel
+	}
+
+	if pipelinerunLevel, ok := cfgMap[metricsPipelinerunLevelKey]; ok {
+		tc.PipelinerunLevel = pipelinerunLevel
+	}
+	if durationTaskrun, ok := cfgMap[metricsDurationTaskrunType]; ok {
+		tc.DurationTaskrunType = durationTaskrun
+	}
+	if durationPipelienrun, ok := cfgMap[metricsDurationPipelinerunType]; ok {
+		tc.DurationPipelinerunType = durationPipelienrun
+	}
+	return &tc, nil
+}
+
+// NewArtifactBucketFromConfigMap returns a Config for the given configmap
+func NewMetricsFromConfigMap(config *corev1.ConfigMap) (*Metrics, error) {
+	return NewMetricsFromMap(config.Data)
+}

pkg/apis/config/metrics_test.go

@@ -0,0 +1,80 @@
+/*
+Copyright 2021 The Tekton Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config_test
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/tektoncd/pipeline/pkg/apis/config"
+	test "github.com/tektoncd/pipeline/pkg/reconciler/testing"
+	"github.com/tektoncd/pipeline/test/diff"
+)
+
+func TestNewMetricsFromConfigMap(t *testing.T) {
+	type testCase struct {
+		expectedConfig *config.Metrics
+		fileName       string
+	}
+
+	testCases := []testCase{
+		{
+			expectedConfig: &config.Metrics{
+				TaskrunLevel:            config.TaskrunLevelAtTaskrun,
+				PipelinerunLevel:        config.PipelinerunLevelAtPipelinerun,
+				DurationTaskrunType:     config.DurationPipelinerunTypeHistogram,
+				DurationPipelinerunType: config.DurationPipelinerunTypeHistogram,
+			},
+			fileName: config.GetMetricsConfigName(),
+		},
+		{
+			expectedConfig: &config.Metrics{
+				TaskrunLevel:            config.TaskrunLevelAtNS,
+				PipelinerunLevel:        config.PipelinerunLevelAtNS,
+				DurationTaskrunType:     config.DurationTaskrunTypeHistogram,
+				DurationPipelinerunType: config.DurationPipelinerunTypeLastValue,
+			},
+			fileName: "config-observability-namespacelevel",
+		},
+	}
+
+	for _, tc := range testCases {
+		verifyConfigFileWithExpectedMetricsConfig(t, tc.fileName, tc.expectedConfig)
+	}
+}
+
+func TestNewMetricsFromEmptyConfigMap(t *testing.T) {
+	MetricsConfigEmptyName := "config-observability-empty"
+	expectedConfig := &config.Metrics{
+		TaskrunLevel:            config.TaskrunLevelAtTaskrun,
+		PipelinerunLevel:        config.PipelinerunLevelAtPipelinerun,
+		DurationTaskrunType:     config.DurationPipelinerunTypeHistogram,
+		DurationPipelinerunType: config.DurationPipelinerunTypeHistogram,
+	}
+	verifyConfigFileWithExpectedMetricsConfig(t, MetricsConfigEmptyName, expectedConfig)
+}
+
+func verifyConfigFileWithExpectedMetricsConfig(t *testing.T, fileName string, expectedConfig *config.Metrics) {
+	cm := test.ConfigMapFromTestFile(t, fileName)
+	if ab, err := config.NewMetricsFromConfigMap(cm); err == nil {
+		if d := cmp.Diff(ab, expectedConfig); d != "" {
+			t.Errorf("Diff:\n%s", diff.PrintWantGot(d))
+		}
+	} else {
+		t.Errorf("NewMetricsFromConfigMap(actual) = %v", err)
+	}
+}

pkg/apis/config/store.go

@@ -31,6 +31,7 @@ type Config struct {
 	FeatureFlags   *FeatureFlags
 	ArtifactBucket *ArtifactBucket
 	ArtifactPVC    *ArtifactPVC
+	Metrics        *Metrics
 }
 
 // FromContext extracts a Config from the provided context.
@@ -52,11 +53,13 @@ func FromContextOrDefaults(ctx context.Context) *Config {
 	featureFlags, _ := NewFeatureFlagsFromMap(map[string]string{})
 	artifactBucket, _ := NewArtifactBucketFromMap(map[string]string{})
 	artifactPVC, _ := NewArtifactPVCFromMap(map[string]string{})
+	metrics, _ := NewMetricsFromMap(map[string]string{})
 	return &Config{
 		Defaults:       defaults,
 		FeatureFlags:   featureFlags,
 		ArtifactBucket: artifactBucket,
 		ArtifactPVC:    artifactPVC,
+		Metrics:        metrics,
 	}
 }
 
@@ -83,6 +86,7 @@ func NewStore(logger configmap.Logger, onAfterStore ...func(name string, value i
 				GetFeatureFlagsConfigName():   NewFeatureFlagsFromConfigMap,
 				GetArtifactBucketConfigName(): NewArtifactBucketFromConfigMap,
 				GetArtifactPVCConfigName():    NewArtifactPVCFromConfigMap,
+				GetMetricsConfigName():        NewMetricsFromConfigMap,
 			},
 			onAfterStore...,
 		),
@@ -115,10 +119,15 @@ func (s *Store) Load() *Config {
 		artifactPVC, _ = NewArtifactPVCFromMap(map[string]string{})
 	}
 
+	metrics := s.UntypedLoad(GetMetricsConfigName())
+	if metrics == nil {
+		metrics, _ = NewMetricsFromMap(map[string]string{})
+	}
 	return &Config{
 		Defaults:       defaults.(*Defaults).DeepCopy(),
 		FeatureFlags:   featureFlags.(*FeatureFlags).DeepCopy(),
 		ArtifactBucket: artifactBucket.(*ArtifactBucket).DeepCopy(),
 		ArtifactPVC:    artifactPVC.(*ArtifactPVC).DeepCopy(),
+		Metrics:        metrics.(*Metrics).DeepCopy(),
 	}
 }

pkg/apis/config/store_test.go

@@ -32,24 +32,28 @@ func TestStoreLoadWithContext(t *testing.T) {
 	featuresConfig := test.ConfigMapFromTestFile(t, "feature-flags-all-flags-set")
 	artifactBucketConfig := test.ConfigMapFromTestFile(t, "config-artifact-bucket")
 	artifactPVCConfig := test.ConfigMapFromTestFile(t, "config-artifact-pvc")
+	metricsConfig := test.ConfigMapFromTestFile(t, "config-observability")
 
 	expectedDefaults, _ := config.NewDefaultsFromConfigMap(defaultConfig)
 	expectedFeatures, _ := config.NewFeatureFlagsFromConfigMap(featuresConfig)
 	expectedArtifactBucket, _ := config.NewArtifactBucketFromConfigMap(artifactBucketConfig)
 	expectedArtifactPVC, _ := config.NewArtifactPVCFromConfigMap(artifactPVCConfig)
+	metrics, _ := config.NewMetricsFromConfigMap(metricsConfig)
 
 	expected := &config.Config{
 		Defaults:       expectedDefaults,
 		FeatureFlags:   expectedFeatures,
 		ArtifactBucket: expectedArtifactBucket,
 		ArtifactPVC:    expectedArtifactPVC,
+		Metrics:        metrics,
 	}
 
 	store := config.NewStore(logtesting.TestLogger(t))
 	store.OnConfigChanged(defaultConfig)
 	store.OnConfigChanged(featuresConfig)
 	store.OnConfigChanged(artifactBucketConfig)
 	store.OnConfigChanged(artifactPVCConfig)
+	store.OnConfigChanged(metricsConfig)
 
 	cfg := config.FromContext(store.ToContext(context.Background()))
 

pkg/apis/config/testdata/config-observability-empty.yaml

@@ -0,0 +1,26 @@
+# Copyright 2019 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-observability
+  namespace: tekton-pipelines
+  labels:
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/part-of: tekton-pipelines
+data:
+  metrics.backend-destination: prometheus
+  metrics.stackdriver-project-id: "<your stackdriver project id>"
+  metrics.allow-stackdriver-custom-metrics: "false"

pkg/apis/config/testdata/config-observability-namespacelevel.yaml

@@ -0,0 +1,30 @@
+# Copyright 2019 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-observability
+  namespace: tekton-pipelines
+  labels:
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/part-of: tekton-pipelines
+data:
+  metrics.backend-destination: prometheus
+  metrics.stackdriver-project-id: "<your stackdriver project id>"
+  metrics.allow-stackdriver-custom-metrics: "false"
+  metrics.taskrun.level: "namespace"
+  metrics.taskrun.duration-type: "histogram"
+  metrics.pipelinerun.level: "namespace"
+  metrics.pipelinerun.duration-type: "lastvalue"