diff --git a/README.md b/README.md index 9685543d34..bbfa9f1a17 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,7 @@ The downside of using an auto-sharded setup comes from the rollout strategy supp For pod metrics, they can be sharded per node with the following flag: -* `--node` +* `--node=$(NODE_NAME)` Each kube-state-metrics pod uses FieldSelector (spec.nodeName) to watch/list pod metrics only on the same node. @@ -276,6 +276,22 @@ spec: fieldPath: spec.nodeName ``` +To track metrics for unassigned pods, you need to add an additional deployment and set `--node=""`, as shown in the following example: + +``` +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:IMAGE_TAG + name: kube-state-metrics + args: + - --resources=pods + - --node="" +``` + Other metrics can be sharded via [Horizontal sharding](#horizontal-sharding). ### Setup diff --git a/examples/daemonsetsharding/deployment-no-node-pods.yaml b/examples/daemonsetsharding/deployment-no-node-pods.yaml new file mode 100644 index 0000000000..6d8fc97492 --- /dev/null +++ b/examples/daemonsetsharding/deployment-no-node-pods.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: kube-state-metrics-pods + app.kubernetes.io/version: 2.10.0 + name: kube-state-metrics-pods + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + template: + metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.10.0 + spec: + automountServiceAccountToken: true + containers: + - args: + - --resources=pods + - --node="" + image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.10.0 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + timeoutSeconds: 5 + name: kube-state-metrics + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 8081 + name: telemetry + readinessProbe: + httpGet: + path: / + port: 8081 + initialDelaySeconds: 5 + timeoutSeconds: 5 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: kube-state-metrics diff --git a/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet b/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet index 4ac16fbbc3..a7c2904123 100644 --- a/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet +++ b/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet @@ -373,6 +373,30 @@ }, ), + deploymentNoNodePods: + local c = ksm.deployment.spec.template.spec.containers[0] { + args: [ + '--resources=pods', + '--node=""', + ], + }; + local shardksmname = ksm.name + "-pods"; + std.mergePatch(ksm.deployment, + { + metadata: { + name: shardksmname, + labels: {'app.kubernetes.io/name': shardksmname} + }, + spec: { + template: { + spec: { + containers: [c], + }, + }, + }, + }, + ), + daemonset: // extending the default container from above local c0 = ksm.deployment.spec.template.spec.containers[0] { diff --git a/pkg/options/options.go b/pkg/options/options.go index 2a60f89cad..c89ef63f43 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -73,6 +73,7 @@ func NewOptions() *Options { MetricAllowlist: MetricSet{}, MetricDenylist: MetricSet{}, MetricOptInList: MetricSet{}, + Node: NodeType{}, AnnotationsAllowList: LabelsAllowList{}, LabelsAllowList: LabelsAllowList{}, } @@ -136,7 +137,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) { o.cmd.Flags().StringVar(&o.TLSConfig, "tls-config", "", "Path to the TLS configuration file") o.cmd.Flags().StringVar(&o.TelemetryHost, "telemetry-host", "::", `Host to expose kube-state-metrics self metrics on.`) o.cmd.Flags().StringVar(&o.Config, "config", "", "Path to the kube-state-metrics options config file") - o.cmd.Flags().StringVar((*string)(&o.Node), "node", "", "Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.") + o.cmd.Flags().Var(&o.Node, "node", "Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.") o.cmd.Flags().Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').") o.cmd.Flags().Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.") o.cmd.Flags().Var(&o.MetricAllowlist, "metric-allowlist", "Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.") @@ -161,7 +162,7 @@ func (o *Options) Usage() { // Validate validates arguments func (o *Options) Validate() error { shardableResource := "pods" - if o.Node == "" { + if o.Node.String() == "" { return nil } for _, x := range o.Resources.AsSlice() { diff --git a/pkg/options/types.go b/pkg/options/types.go index ac0c11275e..9f03ec863b 100644 --- a/pkg/options/types.go +++ b/pkg/options/types.go @@ -18,6 +18,7 @@ package options import ( "errors" + "regexp" "sort" "strings" @@ -105,14 +106,56 @@ func (r *ResourceSet) Type() string { } // NodeType represents a nodeName to query from. -type NodeType string +type NodeType map[string]struct{} + +// Set converts a comma-separated string of nodename into a slice and appends it to the NodeList +func (n *NodeType) Set(value string) error { + s := *n + cols := strings.Split(value, ",") + for _, col := range cols { + col = strings.TrimSpace(col) + if len(col) != 0 { + s[col] = struct{}{} + } + } + return nil +} + +// AsSlice returns the LabelsAllowList in the form of plain string slice. +func (n NodeType) AsSlice() []string { + cols := make([]string, 0, len(n)) + for col := range n { + cols = append(cols, col) + } + return cols +} + +func (n NodeType) String() string { + return strings.Join(n.AsSlice(), ",") +} + +// Type returns a descriptive string about the NodeList type. +func (n *NodeType) Type() string { + return "string" +} // GetNodeFieldSelector returns a nodename field selector. func (n *NodeType) GetNodeFieldSelector() string { - if string(*n) != "" { - return fields.OneTermEqualSelector("spec.nodeName", string(*n)).String() + if nil == n || len(*n) == 0 { + klog.InfoS("Using node type is nil") + return EmptyFieldSelector() } - return EmptyFieldSelector() + pattern := "[^a-zA-Z0-9_,-]+" + re := regexp.MustCompile(pattern) + result := re.ReplaceAllString(n.String(), "") + klog.InfoS("Using node type", "node", result) + return fields.OneTermEqualSelector("spec.nodeName", result).String() + +} + +// NodeValue represents a nodeName to query from. +type NodeValue interface { + GetNodeFieldSelector() string } // EmptyFieldSelector returns an empty field selector. diff --git a/pkg/options/types_test.go b/pkg/options/types_test.go index a1b43a2c0b..4b89f76f4f 100644 --- a/pkg/options/types_test.go +++ b/pkg/options/types_test.go @@ -162,13 +162,30 @@ func TestNodeFieldSelector(t *testing.T) { Wanted string }{ { - Desc: "empty node name", - Node: "", + Desc: "with node name", Wanted: "", }, { Desc: "with node name", - Node: "k8s-node-1", + Node: nil, + Wanted: "", + }, + { + Desc: "empty node name", + Node: NodeType( + map[string]struct{}{ + "": {}, + }, + ), + Wanted: "spec.nodeName=", + }, + { + Desc: "with node name", + Node: NodeType( + map[string]struct{}{ + "k8s-node-1": {}, + }, + ), Wanted: "spec.nodeName=k8s-node-1", }, } @@ -194,43 +211,67 @@ func TestMergeFieldSelectors(t *testing.T) { Desc: "empty DeniedNamespaces", Namespaces: NamespaceList{"default", "kube-system"}, DeniedNamespaces: NamespaceList{}, - Node: "", - Wanted: "", + Node: NodeType( + map[string]struct{}{ + "": {}, + }, + ), + Wanted: "spec.nodeName=", }, { Desc: "all DeniedNamespaces", Namespaces: DefaultNamespaces, DeniedNamespaces: NamespaceList{"some-system"}, - Node: "", - Wanted: "metadata.namespace!=some-system", + Node: NodeType( + map[string]struct{}{ + "": {}, + }, + ), + Wanted: "metadata.namespace!=some-system,spec.nodeName=", }, { Desc: "general case", Namespaces: DefaultNamespaces, DeniedNamespaces: NamespaceList{"case1-system", "case2-system"}, - Node: "", - Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system", + Node: NodeType( + map[string]struct{}{ + "": {}, + }, + ), + Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system,spec.nodeName=", }, { Desc: "empty DeniedNamespaces", Namespaces: NamespaceList{"default", "kube-system"}, DeniedNamespaces: NamespaceList{}, - Node: "k8s-node-1", - Wanted: "spec.nodeName=k8s-node-1", + Node: NodeType( + map[string]struct{}{ + "k8s-node-1": {}, + }, + ), + Wanted: "spec.nodeName=k8s-node-1", }, { Desc: "all DeniedNamespaces", Namespaces: DefaultNamespaces, DeniedNamespaces: NamespaceList{"some-system"}, - Node: "k8s-node-1", - Wanted: "metadata.namespace!=some-system,spec.nodeName=k8s-node-1", + Node: NodeType( + map[string]struct{}{ + "k8s-node-1": {}, + }, + ), + Wanted: "metadata.namespace!=some-system,spec.nodeName=k8s-node-1", }, { Desc: "general case", Namespaces: DefaultNamespaces, DeniedNamespaces: NamespaceList{"case1-system", "case2-system"}, - Node: "k8s-node-1", - Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system,spec.nodeName=k8s-node-1", + Node: NodeType( + map[string]struct{}{ + "k8s-node-1": {}, + }, + ), + Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system,spec.nodeName=k8s-node-1", }, }