webhook: add options to disable resource_namespace tag in metrics

To add some context, historically, `resource_name` was removed from this
tag list due to its high potential of causing high metrics cardinality.
See [knative#1464][1] for more information.

While that's great, but it might not be sufficient for large scale use
cases where namespaces can be super dynamic (with generateName, too) or
grows fase enough. There is an issue report from
[tektoncd/pipeline#3171][2] which talks about this.

This proposal makes it possible to disable `resource_namespace` tag via
an option function. The default behavior is not changed, so no user
impact if any of existing users rely on this tag. There is no API
contract change either due to the beauty of variadic functions.

Now downstream projects can consume this by override `StatsReporter` in
webhook context options with their own preference. As a caveat here, if
downstream project does choose to override `StatsReporter`, the default
`ReportMetrics` function shouldn't be called by default as they may now
have a different set of tag keys to report. As such, this function is
only called if the default `StatsReporter` is used.

[1]: knative#1464
[2]: tektoncd/pipeline#3171

injection/sharedmain/main.go

@@ -289,9 +289,6 @@ func MainWithConfig(ctx context.Context, component string, cfg *rest.Config, cto
 	// and pass them in.
 	var wh *webhook.Webhook
 	if len(webhooks) > 0 {
-		// Register webhook metrics
-		webhook.RegisterMetrics()
-
 		wh, err = webhook.New(ctx, webhooks)
 		if err != nil {
 			logger.Fatalw("Failed to create webhook", zap.Error(err))

webhook/stats_reporter.go

@@ -71,37 +71,77 @@ type StatsReporter interface {
 	ReportConversionRequest(request *apixv1.ConversionRequest, response *apixv1.ConversionResponse, d time.Duration) error
 }
 
+type options struct {
+	tagsToExclude map[string]struct{}
+}
+
+type Option func(_ *options)
+
+func WithoutTag(tag string) Option {
+	return func(opts *options) {
+		if opts.tagsToExclude == nil {
+			opts.tagsToExclude = make(map[string]struct{})
+		}
+		opts.tagsToExclude[tag] = struct{}{}
+	}
+}
+
 // reporter implements StatsReporter interface
 type reporter struct {
-	ctx context.Context
+	ctx  context.Context
+	opts options
 }
 
 // NewStatsReporter creates a reporter for webhook metrics
-func NewStatsReporter() (StatsReporter, error) {
+func NewStatsReporter(opts ...Option) (StatsReporter, error) {
 	ctx, err := tag.New(
 		context.Background(),
 	)
 	if err != nil {
 		return nil, err
 	}
 
-	return &reporter{ctx: ctx}, nil
+	options := options{}
+	for _, opt := range opts {
+		opt(&options)
+	}
+
+	return &reporter{ctx: ctx, opts: options}, nil
 }
 
 // Captures req count metric, recording the count and the duration
 func (r *reporter) ReportAdmissionRequest(req *admissionv1.AdmissionRequest, resp *admissionv1.AdmissionResponse, d time.Duration) error {
-	ctx, err := tag.New(
-		r.ctx,
-		tag.Insert(requestOperationKey, string(req.Operation)),
-		tag.Insert(kindGroupKey, req.Kind.Group),
-		tag.Insert(kindVersionKey, req.Kind.Version),
-		tag.Insert(kindKindKey, req.Kind.Kind),
-		tag.Insert(resourceGroupKey, req.Resource.Group),
-		tag.Insert(resourceVersionKey, req.Resource.Version),
-		tag.Insert(resourceResourceKey, req.Resource.Resource),
-		tag.Insert(resourceNamespaceKey, req.Namespace),
-		tag.Insert(admissionAllowedKey, strconv.FormatBool(resp.Allowed)),
-	)
+	mutators := []tag.Mutator{}
+
+	if _, ok := r.opts.tagsToExclude[requestOperationKey.Name()]; !ok {
+		mutators = append(mutators, tag.Insert(requestOperationKey, string(req.Operation)))
+	}
+	if _, ok := r.opts.tagsToExclude[kindGroupKey.Name()]; !ok {
+		mutators = append(mutators, tag.Insert(kindGroupKey, req.Kind.Group))
+	}
+	if _, ok := r.opts.tagsToExclude[kindVersionKey.Name()]; !ok {
+		mutators = append(mutators, tag.Insert(kindVersionKey, req.Kind.Version))
+	}
+	if _, ok := r.opts.tagsToExclude[kindKindKey.Name()]; !ok {
+		mutators = append(mutators, tag.Insert(kindKindKey, req.Kind.Kind))
+	}
+	if _, ok := r.opts.tagsToExclude[resourceGroupKey.Name()]; !ok {
+		mutators = append(mutators, tag.Insert(resourceGroupKey, req.Resource.Group))
+	}
+	if _, ok := r.opts.tagsToExclude[resourceVersionKey.Name()]; !ok {
+		mutators = append(mutators, tag.Insert(resourceVersionKey, req.Resource.Version))
+	}
+	if _, ok := r.opts.tagsToExclude[resourceResourceKey.Name()]; !ok {
+		mutators = append(mutators, tag.Insert(resourceResourceKey, req.Resource.Resource))
+	}
+	if _, ok := r.opts.tagsToExclude[resourceNamespaceKey.Name()]; !ok {
+		mutators = append(mutators, tag.Insert(resourceNamespaceKey, req.Namespace))
+	}
+	if _, ok := r.opts.tagsToExclude[admissionAllowedKey.Name()]; !ok {
+		mutators = append(mutators, tag.Insert(admissionAllowedKey, strconv.FormatBool(resp.Allowed)))
+	}
+
+	ctx, err := tag.New(r.ctx, mutators...)
 	if err != nil {
 		return err
 	}
@@ -114,13 +154,22 @@ func (r *reporter) ReportAdmissionRequest(req *admissionv1.AdmissionRequest, res
 
 // Captures req count metric, recording the count and the duration
 func (r *reporter) ReportConversionRequest(req *apixv1.ConversionRequest, resp *apixv1.ConversionResponse, d time.Duration) error {
-	ctx, err := tag.New(
-		r.ctx,
-		tag.Insert(desiredAPIVersionKey, req.DesiredAPIVersion),
-		tag.Insert(resultStatusKey, resp.Result.Status),
-		tag.Insert(resultReasonKey, string(resp.Result.Reason)),
-		tag.Insert(resultCodeKey, strconv.Itoa(int(resp.Result.Code))),
-	)
+	mutators := []tag.Mutator{}
+
+	if _, ok := r.opts.tagsToExclude[desiredAPIVersionKey.Name()]; !ok {
+		mutators = append(mutators, tag.Insert(desiredAPIVersionKey, req.DesiredAPIVersion))
+	}
+	if _, ok := r.opts.tagsToExclude[resultStatusKey.Name()]; !ok {
+		mutators = append(mutators, tag.Insert(resultStatusKey, resp.Result.Status))
+	}
+	if _, ok := r.opts.tagsToExclude[resultReasonKey.Name()]; !ok {
+		mutators = append(mutators, tag.Insert(resultReasonKey, string(resp.Result.Reason)))
+	}
+	if _, ok := r.opts.tagsToExclude[resultCodeKey.Name()]; !ok {
+		mutators = append(mutators, tag.Insert(resultCodeKey, strconv.Itoa(int(resp.Result.Code))))
+	}
+
+	ctx, err := tag.New(r.ctx, mutators...)
 	if err != nil {
 		return err
 	}
@@ -131,8 +180,8 @@ func (r *reporter) ReportConversionRequest(req *apixv1.ConversionRequest, resp *
 	return nil
 }
 
-func RegisterMetrics() {
-	tagKeys := []tag.Key{
+func RegisterMetrics(opts ...Option) {
+	allTags := []tag.Key{
 		requestOperationKey,
 		kindGroupKey,
 		kindVersionKey,
@@ -147,6 +196,18 @@ func RegisterMetrics() {
 		resultReasonKey,
 		resultCodeKey}
 
+	options := options{}
+	for _, opt := range opts {
+		opt(&options)
+	}
+
+	tagKeys := []tag.Key{}
+	for _, tag := range allTags {
+		if _, ok := options.tagsToExclude[tag.Name()]; !ok {
+			tagKeys = append(tagKeys, tag)
+		}
+	}
+
 	if err := view.Register(
 		&view.View{
 			Description: requestCountM.Description(),

webhook/stats_reporter_test.go

@@ -70,6 +70,47 @@ func TestWebhookStatsReporterAdmission(t *testing.T) {
 	metricstest.CheckDistributionData(t, requestLatenciesName, expectedTags, 2, shortTime, longTime)
 }
 
+func TestWebhookStatsReporterAdmissionWithoutNamespaceTag(t *testing.T) {
+	setup(WithoutTag(resourceNamespaceKey.Name()))
+	req := &admissionv1.AdmissionRequest{
+		UID:       "705ab4f5-6393-11e8-b7cc-42010a800002",
+		Kind:      metav1.GroupVersionKind{Group: "autoscaling", Version: "v1", Kind: "Scale"},
+		Resource:  metav1.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"},
+		Name:      "my-deployment",
+		Namespace: "my-namespace",
+		Operation: admissionv1.Update,
+	}
+
+	resp := &admissionv1.AdmissionResponse{
+		UID:     req.UID,
+		Allowed: true,
+	}
+
+	r, _ := NewStatsReporter(WithoutTag(resourceNamespaceKey.Name()))
+
+	shortTime, longTime := 1100.0, 9100.0
+	expectedTags := map[string]string{
+		requestOperationKey.Name(): string(req.Operation),
+		kindGroupKey.Name():        req.Kind.Group,
+		kindVersionKey.Name():      req.Kind.Version,
+		kindKindKey.Name():         req.Kind.Kind,
+		resourceGroupKey.Name():    req.Resource.Group,
+		resourceVersionKey.Name():  req.Resource.Version,
+		resourceResourceKey.Name(): req.Resource.Resource,
+		admissionAllowedKey.Name(): strconv.FormatBool(resp.Allowed),
+	}
+
+	if err := r.ReportAdmissionRequest(req, resp, time.Duration(shortTime)*time.Millisecond); err != nil {
+		t.Fatalf("ReportAdmissionRequest() = %v", err)
+	}
+	if err := r.ReportAdmissionRequest(req, resp, time.Duration(longTime)*time.Millisecond); err != nil {
+		t.Fatalf("ReportAdmissionRequest() = %v", err)
+	}
+
+	metricstest.CheckCountData(t, requestCountName, expectedTags, 2)
+	metricstest.CheckDistributionData(t, requestLatenciesName, expectedTags, 2, shortTime, longTime)
+}
+
 func TestWebhookStatsReporterConversion(t *testing.T) {
 	setup()
 	req := &apixv1.ConversionRequest{
@@ -103,12 +144,12 @@ func TestWebhookStatsReporterConversion(t *testing.T) {
 	metricstest.CheckDistributionData(t, requestLatenciesName, expectedTags, 2, shortTime, longTime)
 }
 
-func setup() {
-	resetMetrics()
+func setup(opts ...Option) {
+	resetMetrics(opts...)
 }
 
 // opencensus metrics carry global state that need to be reset between unit tests
-func resetMetrics() {
+func resetMetrics(opts ...Option) {
 	metricstest.Unregister(requestCountName, requestLatenciesName)
-	RegisterMetrics()
+	RegisterMetrics(opts...)
 }

webhook/webhook.go

@@ -144,6 +144,7 @@ func New(
 	logger := logging.FromContext(ctx)
 
 	if opts.StatsReporter == nil {
+		RegisterMetrics()
 		reporter, err := NewStatsReporter()
 		if err != nil {
 			return nil, err

webhook/webhook_test.go

@@ -32,20 +32,30 @@ import (
 )
 
 func newDefaultOptions() Options {
+	statsReporter, err := NewStatsReporter()
+	if err != nil {
+		panic(err)
+	}
 	return Options{
-		ServiceName: "webhook",
-		Port:        8443,
-		SecretName:  "webhook-certs",
+		ServiceName:   "webhook",
+		Port:          8443,
+		SecretName:    "webhook-certs",
+		StatsReporter: statsReporter,
 	}
 }
 
 func newCustomOptions() Options {
+	statsReporter, err := NewStatsReporter()
+	if err != nil {
+		panic(err)
+	}
 	return Options{
 		ServiceName:           "webhook",
 		Port:                  8443,
 		SecretName:            "webhook-certs",
 		ServerPrivateKeyName:  "tls.key",
 		ServerCertificateName: "tls.crt",
+		StatsReporter:         statsReporter,
 	}
 }