Allow setting Queue Proxy resources in ConfigMap

config/core/configmaps/deployment.yaml

@@ -20,7 +20,7 @@ metadata:
   labels:
     serving.knative.dev/release: devel
   annotations:
-    knative.dev/example-checksum: 3d5b261a
+    knative.dev/example-checksum: 1de882eb
 data:
   # This is the Go import path for the binary that is containerized
   # and substituted here.
@@ -47,3 +47,29 @@ data:
     # ProgressDeadline is the duration we wait for the deployment to
     # be ready before considering it failed.
     progressDeadline: "120s"
+
+    # queueSidecarCPURequest is the requests.cpu to set for the queue proxy sidecar container.
+    # If omitted, a default value (currently "25m"), is used.
+    queueSidecarCPURequest: "25m"
+
+    # queueSidecarCPULimit is the limits.cpu to set for the queue proxy sidecar container.
+    # If omitted, no value is specified and the system default is used.
+    queueSidecarCPULimit: "1000m"
+
+    # queueSidecarMemoryRequest is the requests.memory to set for the queue proxy container.
+    # If omitted, no value is specified and the system default is used.
+    queueSidecarMemoryRequest: "400m"
+
+    # queueSidecarMemoryLimit is the limits.memory to set for the queue proxy container.
+    # If omitted, no value is specified and the system default is used.
+    queueSidecarMemoryLimit: "800m"
+
+    # queueSidecarEphemeralStorageRequest is the requests.ephemeral-storage to
+    # set for the queue proxy sidecar container.
+    # If omitted, no value is specified and the system default is used.
+    queueSidecarEphemeralStorageRequest: "512m"
+
+    # queueSidecarEphemeralStorageLimit is the limits.ephemeral-storage to set
+    # for the queue proxy sidecar container.
+    # If omitted, no value is specified and the system default is used.
+    queueSidecarEphemeralStorageLimit: "1024m"

pkg/deployment/config.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/util/sets"
 
 	cm "knative.dev/pkg/configmap"
@@ -38,16 +39,36 @@ const (
 	// ProgressDeadlineSeconds. This does not match the K8s default value of 600s.
 	ProgressDeadlineDefault = 120 * time.Second
 
-	registriesSkippingTagResolvingKey = "registriesSkippingTagResolving"
-
 	// ProgressDeadlineKey is the key to configure deployment progress deadline.
 	ProgressDeadlineKey = "progressDeadline"
+
+	// registriesSkippingTagResolvingKey is the config map key for the set of registries
+	// (e.g. ko.local) where tags should not be resolved to digests.
+	registriesSkippingTagResolvingKey = "registriesSkippingTagResolving"
+
+	// queueSidecar resource request keys.
+	queueSidecarCPURequestKey              = "queueSidecarCPURequest"
+	queueSidecarMemoryRequestKey           = "queueSidecarMemoryRequest"
+	queueSidecarEphemeralStorageRequestKey = "queueSidecarEphemeralStorageRequest"
+
+	// queueSidecar resource limit keys.
+	queueSidecarCPULimitKey              = "queueSidecarCPULimit"
+	queueSidecarMemoryLimitKey           = "queueSidecarMemoryLimit"
+	queueSidecarEphemeralStorageLimitKey = "queueSidecarEphemeralStorageLimit"
+)
+
+var (
+	// QueueSidecarCPURequestDefault is the default request.cpu to set for the
+	// queue sidecar. It is set at 25m for backwards-compatibility since this was
+	// the historic default before the field was operator-settable.
+	QueueSidecarCPURequestDefault = resource.MustParse("25m")
 )
 
 func defaultConfig() *Config {
 	return &Config{
 		ProgressDeadline:               ProgressDeadlineDefault,
 		RegistriesSkippingTagResolving: sets.NewString("ko.local", "dev.local"),
+		QueueSidecarCPURequest:         &QueueSidecarCPURequestDefault,
 	}
 }
 
@@ -59,6 +80,13 @@ func NewConfigFromMap(configMap map[string]string) (*Config, error) {
 		cm.AsString(QueueSidecarImageKey, &nc.QueueSidecarImage),
 		cm.AsDuration(ProgressDeadlineKey, &nc.ProgressDeadline),
 		cm.AsStringSet(registriesSkippingTagResolvingKey, &nc.RegistriesSkippingTagResolving),
+
+		cm.AsQuantity(queueSidecarCPURequestKey, &nc.QueueSidecarCPURequest),
+		cm.AsQuantity(queueSidecarMemoryRequestKey, &nc.QueueSidecarMemoryRequest),
+		cm.AsQuantity(queueSidecarEphemeralStorageRequestKey, &nc.QueueSidecarEphemeralStorageRequest),
+		cm.AsQuantity(queueSidecarCPULimitKey, &nc.QueueSidecarCPULimit),
+		cm.AsQuantity(queueSidecarMemoryLimitKey, &nc.QueueSidecarMemoryLimit),
+		cm.AsQuantity(queueSidecarEphemeralStorageLimitKey, &nc.QueueSidecarEphemeralStorageLimit),
 	); err != nil {
 		return nil, err
 	}
@@ -91,4 +119,24 @@ type Config struct {
 	// ProgressDeadline is the time in seconds we wait for the deployment to
 	// be ready before considering it failed.
 	ProgressDeadline time.Duration
+
+	// QueueSidecarCPURequest is the CPU Request to set for the queue proxy sidecar container
+	QueueSidecarCPURequest *resource.Quantity
+
+	// QueueSidecarCPULimit is the CPU Limit to set for the queue proxy sidecar container
+	QueueSidecarCPULimit *resource.Quantity
+
+	// QueueSidecarMemoryRequest is the Memory Request to set for the queue proxy sidecar container
+	QueueSidecarMemoryRequest *resource.Quantity
+
+	// QueueSidecarMemoryLimit is the Memory Limit to set for the queue proxy sidecar container
+	QueueSidecarMemoryLimit *resource.Quantity
+
+	// QueueSidecarEphemeralStorageRequest is the Ephemeral Storage Request to
+	// set for the queue proxy sidecar container
+	QueueSidecarEphemeralStorageRequest *resource.Quantity
+
+	// QueueSidecarEphemeralStorageLimit is the Ephemeral Storage Limit to set
+	// for the queue proxy sidecar container
+	QueueSidecarEphemeralStorageLimit *resource.Quantity
 }

pkg/deployment/config_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 
@@ -46,6 +47,13 @@ func TestControllerConfigurationFromFile(t *testing.T) {
 		want := defaultConfig()
 		// We require QSI to be explicitly set. So do it here.
 		want.QueueSidecarImage = "ko://knative.dev/serving/cmd/queue"
+
+		// The following are in the example yaml, to show usage,
+		// but default is nil, i.e. inheriting k8s.
+		// So for this test we ignore those, but verify the other fields.
+		got.QueueSidecarCPULimit = nil
+		got.QueueSidecarMemoryRequest, got.QueueSidecarMemoryLimit = nil, nil
+		got.QueueSidecarEphemeralStorageRequest, got.QueueSidecarEphemeralStorageLimit = nil, nil
 		if !cmp.Equal(got, want) {
 			t.Error("Example stanza does not match default, diff(-want,+got):", cmp.Diff(want, got))
 		}
@@ -63,6 +71,7 @@ func TestControllerConfiguration(t *testing.T) {
 		wantConfig: &Config{
 			RegistriesSkippingTagResolving: sets.NewString("ko.local", ""),
 			QueueSidecarImage:              defaultSidecarImage,
+			QueueSidecarCPURequest:         &QueueSidecarCPURequestDefault,
 			ProgressDeadline:               ProgressDeadlineDefault,
 		},
 		data: map[string]string{
@@ -74,6 +83,7 @@ func TestControllerConfiguration(t *testing.T) {
 		wantConfig: &Config{
 			RegistriesSkippingTagResolving: sets.NewString("ko.local", "dev.local"),
 			QueueSidecarImage:              defaultSidecarImage,
+			QueueSidecarCPURequest:         &QueueSidecarCPURequestDefault,
 			ProgressDeadline:               444 * time.Second,
 		},
 		data: map[string]string{
@@ -85,12 +95,35 @@ func TestControllerConfiguration(t *testing.T) {
 		wantConfig: &Config{
 			RegistriesSkippingTagResolving: sets.NewString("ko.local", "ko.dev"),
 			QueueSidecarImage:              defaultSidecarImage,
+			QueueSidecarCPURequest:         &QueueSidecarCPURequestDefault,
 			ProgressDeadline:               ProgressDeadlineDefault,
 		},
 		data: map[string]string{
 			QueueSidecarImageKey:              defaultSidecarImage,
 			registriesSkippingTagResolvingKey: "ko.local,ko.dev",
 		},
+	}, {
+		name: "controller configuration with custom queue sidecar resource request/limits",
+		wantConfig: &Config{
+			RegistriesSkippingTagResolving:      sets.NewString("ko.local", "dev.local"),
+			QueueSidecarImage:                   defaultSidecarImage,
+			ProgressDeadline:                    ProgressDeadlineDefault,
+			QueueSidecarCPURequest:              resourcePtr(resource.MustParse("123m")),
+			QueueSidecarMemoryRequest:           resourcePtr(resource.MustParse("456M")),
+			QueueSidecarEphemeralStorageRequest: resourcePtr(resource.MustParse("789m")),
+			QueueSidecarCPULimit:                resourcePtr(resource.MustParse("987M")),
+			QueueSidecarMemoryLimit:             resourcePtr(resource.MustParse("654m")),
+			QueueSidecarEphemeralStorageLimit:   resourcePtr(resource.MustParse("321M")),
+		},
+		data: map[string]string{
+			QueueSidecarImageKey:                   defaultSidecarImage,
+			queueSidecarCPURequestKey:              "123m",
+			queueSidecarMemoryRequestKey:           "456M",
+			queueSidecarEphemeralStorageRequestKey: "789m",
+			queueSidecarCPULimitKey:                "987M",
+			queueSidecarMemoryLimitKey:             "654m",
+			queueSidecarEphemeralStorageLimitKey:   "321M",
+		},
 	}, {
 		name:    "controller with no side car image",
 		wantErr: true,
@@ -146,3 +179,7 @@ func TestControllerConfiguration(t *testing.T) {
 		})
 	}
 }
+
+func resourcePtr(q resource.Quantity) *resource.Quantity {
+	return &q
+}

pkg/reconciler/revision/resources/deploy_test.go

@@ -83,7 +83,7 @@ var (
 
 	defaultQueueContainer = &corev1.Container{
 		Name:      QueueContainerName,
-		Resources: createQueueResources(make(map[string]string), &corev1.Container{}),
+		Resources: createQueueResources(&deploymentConfig, make(map[string]string), &corev1.Container{}),
 		Ports:     append(queueNonServingPorts, queueHTTPPort),
 		ReadinessProbe: &corev1.Probe{
 			Handler: corev1.Handler{

pkg/reconciler/revision/resources/queue.go

@@ -78,9 +78,35 @@ var (
 	}
 )
 
-func createQueueResources(annotations map[string]string, userContainer *corev1.Container) corev1.ResourceRequirements {
-	resourceRequests := corev1.ResourceList{corev1.ResourceCPU: queueContainerCPU}
+func createQueueResources(cfg *deployment.Config, annotations map[string]string, userContainer *corev1.Container) corev1.ResourceRequirements {
+	resourceRequests := corev1.ResourceList{}
 	resourceLimits := corev1.ResourceList{}
+
+	for _, r := range []struct {
+		Name    corev1.ResourceName
+		Request *resource.Quantity
+		Limit   *resource.Quantity
+	}{{
+		Name:    corev1.ResourceCPU,
+		Request: cfg.QueueSidecarCPURequest,
+		Limit:   cfg.QueueSidecarCPULimit,
+	}, {
+		Name:    corev1.ResourceMemory,
+		Request: cfg.QueueSidecarMemoryRequest,
+		Limit:   cfg.QueueSidecarMemoryLimit,
+	}, {
+		Name:    corev1.ResourceEphemeralStorage,
+		Request: cfg.QueueSidecarEphemeralStorageRequest,
+		Limit:   cfg.QueueSidecarEphemeralStorageLimit,
+	}} {
+		if r.Request != nil {
+			resourceRequests[r.Name] = *r.Request
+		}
+		if r.Limit != nil {
+			resourceLimits[r.Name] = *r.Limit
+		}
+	}
+
 	var requestCPU, limitCPU, requestMemory, limitMemory resource.Quantity
 
 	if resourceFraction, ok := fractionFromPercentage(annotations, serving.QueueSideCarResourcePercentageAnnotation); ok {
@@ -246,7 +272,7 @@ func makeQueueContainer(rev *v1.Revision, loggingConfig *logging.Config, tracing
 	return &corev1.Container{
 		Name:            QueueContainerName,
 		Image:           deploymentConfig.QueueSidecarImage,
-		Resources:       createQueueResources(rev.GetAnnotations(), container),
+		Resources:       createQueueResources(deploymentConfig, rev.GetAnnotations(), container),
 		Ports:           ports,
 		ReadinessProbe:  makeQueueProbe(rp),
 		VolumeMounts:    volumeMounts,