istio: add metric for debouncing (istio#40523)

* istio: add metric for debouncing

This CL adds metric for the delay between a first config change
enters deboucing until the final merged push request is pushed into
the push queue. This time plus the proxy convergence time give us
an upper bound on the total delay between a config change and the
change is pushed to proxies.

Also increased the buckets since logging shows that the debounce time
is pretty long (more than 1 minute).

Change-Id: I3220f9c3188824ea6925151ff6837f91aac5a15a
Reviewed-on: https://gerrit.musta.ch/c/public/istio/+/3512
Reviewed-by: Weibo He <weibo.he@airbnb.com>
Reviewed-by: Ryan Smick <ryan.smick@airbnb.com>
Reviewed-by: Jungho Ahn <jungho.ahn@airbnb.com>

* istio: fix debounceTime typo

Change-Id: I9977c597768360cc3dd485dbf21bd9afdb2f5151
Reviewed-on: https://gerrit.musta.ch/c/public/istio/+/3517
Reviewed-by: Weibo He <weibo.he@airbnb.com>

* istio: handle debounce time entirely in the debounce() function

To address comments in istio#40523,
we will record after push finishes inside the debounce function.
We can actually remove the DebounceStart field in the push context
as a result.

Also change the buckets as discussed in the PR.

Change-Id: I3b7a7860590e7e5ed4f13282b4398527de089c81
Reviewed-on: https://gerrit.musta.ch/c/public/istio/+/3519
Reviewed-by: Jungho Ahn <jungho.ahn@airbnb.com>
Reviewed-by: Weibo He <weibo.he@airbnb.com>

pilot/pkg/xds/discovery.go

@@ -414,9 +414,10 @@ func debounce(ch chan *model.PushRequest, stopCh <-chan struct{}, opts debounceO
 	free := true
 	freeCh := make(chan struct{}, 1)
 
-	push := func(req *model.PushRequest, debouncedEvents int) {
+	push := func(req *model.PushRequest, debouncedEvents int, startDebounce time.Time) {
 		pushFn(req)
 		updateSent.Add(int64(debouncedEvents))
+		debounceTime.Record(time.Since(startDebounce).Seconds())
 		freeCh <- struct{}{}
 	}
 
@@ -437,7 +438,7 @@ func debounce(ch chan *model.PushRequest, stopCh <-chan struct{}, opts debounceO
 						quietTime, eventDelay, req.Full)
 				}
 				free = false
-				go push(req, debouncedEvents)
+				go push(req, debouncedEvents, startDebounce)
 				req = nil
 				debouncedEvents = 0
 			}

pilot/pkg/xds/monitoring.go

@@ -118,6 +118,12 @@ var (
 	ldsSendErrPushes = pushes.With(typeTag.Value("lds_senderr"))
 	rdsSendErrPushes = pushes.With(typeTag.Value("rds_senderr"))
 
+	debounceTime = monitoring.NewDistribution(
+		"pilot_debounce_time",
+		"Delay in seconds between the first config enters debouncing and the merged push request is pushed into the push queue.",
+		[]float64{.01, .1, 1, 3, 5, 10, 20, 30},
+	)
+
 	pushContextInitTime = monitoring.NewDistribution(
 		"pilot_pushcontext_init_seconds",
 		"Total time in seconds Pilot takes to init pushContext.",
@@ -292,6 +298,8 @@ func init() {
 		xdsClients,
 		xdsResponseWriteTimeouts,
 		pushes,
+		debounceTime,
+		pushContextInitTime,
 		pushTime,
 		proxiesConvergeDelay,
 		proxiesQueueTime,