Attempt to fix our probes with mTLS enabled.

config/activator.yaml

@@ -53,14 +53,23 @@ spec:
         - "-stderrthreshold=FATAL"
         readinessProbe:
           httpGet:
-            # The path does not matter, we look for kubelet probe headers.
+            # The path does not matter, we look for the kubelet user-agent
+            # (or our header below)
             path: /healthz
             port: 8012
+            httpHeaders:
+            # Istio with mTLS strips the Kubelet user-agent, so pass a header too.
+            - name: k-kubelet-probe
+              value: "activator"
         livenessProbe:
           httpGet:
             # The path does not matter, we look for kubelet probe headers.
             path: /healthz
             port: 8012
+            httpHeaders:
+            # Istio with mTLS strips the Kubelet user-agent, so pass a header too.
+            - name: k-kubelet-probe
+              value: "activator"
         resources:
           # Request 2x what we saw running e2e
           requests:

pkg/network/network.go

@@ -72,6 +72,10 @@ const (
 	//   User-Agent = "kube-probe/{major-version}.{minor-version}".
 	kubeProbeUAPrefix = "kube-probe/"
 
+	// Istio with mTLS rewrites probes, but their probes pass a different
+	// user-agent.  So we augment the probes with this header.
+	KubeletProbeHeaderName = "K-Kubelet-Probe"
+
 	// DefaultConnTimeout specifies a short default connection timeout
 	// to avoid hitting the issue fixed in
 	// https://github.com/kubernetes/kubernetes/pull/72534 but only
@@ -260,7 +264,8 @@ func checkTemplate(t *template.Template) error {
 
 // IsKubeletProbe returns true if the request is a kubernetes probe.
 func IsKubeletProbe(r *http.Request) bool {
-	return strings.HasPrefix(r.Header.Get("User-Agent"), kubeProbeUAPrefix)
+	return strings.HasPrefix(r.Header.Get("User-Agent"), kubeProbeUAPrefix) ||
+		r.Header.Get(KubeletProbeHeaderName) != ""
 }
 
 // RewriteHostIn removes the `Host` header from the inbound (server) request

pkg/network/network_test.go

@@ -511,6 +511,14 @@ func TestIsKubeletProbe(t *testing.T) {
 	if !IsKubeletProbe(req) {
 		t.Error("kubelet probe but not counted as such")
 	}
+	req.Header.Del("User-Agent")
+	if IsKubeletProbe(req) {
+		t.Error("Not a kubelet probe but counted as such")
+	}
+	req.Header.Set(KubeletProbeHeaderName, "no matter")
+	if !IsKubeletProbe(req) {
+		t.Error("kubelet probe but not counted as such")
+	}
 }
 
 func TestRewriteHost(t *testing.T) {

pkg/reconciler/revision/resources/deploy.go

@@ -77,6 +77,13 @@ func rewriteUserProbe(p *corev1.Probe, userPort int) {
 		// so that we know the queue proxy is ready/live as well.
 		// It doesn't matter to which queue serving port we are forwarding the probe.
 		p.HTTPGet.Port = intstr.FromInt(networking.BackendHTTPPort)
+		// With mTLS enabled, Istio rewrites probes, but doesn't spoof the kubelet
+		// user agent, so we need to inject an extra header to be able to distinguish
+		// between probes and real requests.
+		p.HTTPGet.HTTPHeaders = append(p.HTTPGet.HTTPHeaders, corev1.HTTPHeader{
+			Name:  network.KubeletProbeHeaderName,
+			Value: "queue",
+		})
 	case p.TCPSocket != nil:
 		p.TCPSocket.Port = intstr.FromInt(userPort)
 	}

pkg/reconciler/revision/resources/deploy_test.go

@@ -282,6 +282,19 @@ func withHTTPReadinessProbe(port int) containerOption {
 	})
 }
 
+func withHTTPQPReadinessProbe(c *corev1.Container) {
+	withReadinessProbe(corev1.Handler{
+		HTTPGet: &corev1.HTTPGetAction{
+			Port: intstr.FromInt(networking.BackendHTTPPort),
+			Path: "/",
+			HTTPHeaders: []corev1.HTTPHeader{{
+				Name:  network.KubeletProbeHeaderName,
+				Value: "queue",
+			}},
+		},
+	})(c)
+}
+
 func withExecReadinessProbe(command []string) containerOption {
 	return withReadinessProbe(corev1.Handler{
 		Exec: &corev1.ExecAction{
@@ -506,7 +519,7 @@ func TestMakePodSpec(t *testing.T) {
 		cc: &deployment.Config{},
 		want: podSpec([]corev1.Container{
 			userContainer(
-				withHTTPReadinessProbe(networking.BackendHTTPPort),
+				withHTTPQPReadinessProbe,
 			),
 			queueContainer(
 				withEnvVar("CONTAINER_CONCURRENCY", "0"),
@@ -556,6 +569,10 @@ func TestMakePodSpec(t *testing.T) {
 					HTTPGet: &corev1.HTTPGetAction{
 						Path: "/",
 						Port: intstr.FromInt(networking.BackendHTTPPort),
+						HTTPHeaders: []corev1.HTTPHeader{{
+							Name:  network.KubeletProbeHeaderName,
+							Value: "queue",
+						}},
 					},
 				}),
 			),