Skip to content

Commit

Permalink
add XR state metrics
Browse files Browse the repository at this point in the history
Signed-off-by: ezgidemirel <ezgidemirel91@gmail.com>
  • Loading branch information
ezgidemirel committed Apr 15, 2024
1 parent af65888 commit 7c1aaab
Show file tree
Hide file tree
Showing 8 changed files with 272 additions and 81 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ require (
k8s.io/api v0.29.1
k8s.io/apiextensions-apiserver v0.29.1
k8s.io/apimachinery v0.29.1
k8s.io/apiserver v0.29.1
k8s.io/client-go v0.29.1
k8s.io/component-base v0.29.1
k8s.io/klog/v2 v2.110.1
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,8 @@ k8s.io/apiextensions-apiserver v0.29.1 h1:S9xOtyk9M3Sk1tIpQMu9wXHm5O2MX6Y1kIpPMi
k8s.io/apiextensions-apiserver v0.29.1/go.mod h1:zZECpujY5yTW58co8V2EQR4BD6A9pktVgHhvc0uLfeU=
k8s.io/apimachinery v0.29.1 h1:KY4/E6km/wLBguvCZv8cKTeOwwOBqFNjwJIdMkMbbRc=
k8s.io/apimachinery v0.29.1/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU=
k8s.io/apiserver v0.29.1 h1:e2wwHUfEmMsa8+cuft8MT56+16EONIEK8A/gpBSco+g=
k8s.io/apiserver v0.29.1/go.mod h1:V0EpkTRrJymyVT3M49we8uh2RvXf7fWC5XLB0P3SwRw=
k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A=
k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks=
k8s.io/component-base v0.29.1 h1:MUimqJPCRnnHsskTTjKD+IC1EHBbRCVyi37IoFBrkYw=
Expand Down
3 changes: 2 additions & 1 deletion pkg/controller/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/crossplane/crossplane-runtime/pkg/logging"
"github.com/crossplane/crossplane-runtime/pkg/ratelimiter"
"github.com/crossplane/crossplane-runtime/pkg/reconciler/managed"
"github.com/crossplane/crossplane-runtime/pkg/statemetrics"
)

// DefaultOptions returns a functional set of options with conservative
Expand Down Expand Up @@ -65,7 +66,7 @@ type Options struct {

MetricRecorder managed.MetricRecorder

StateRecorder managed.StateRecorder
StateRecorder statemetrics.StateRecorder
}

// ForControllerRuntime extracts options for controller-runtime.
Expand Down
34 changes: 12 additions & 22 deletions pkg/reconciler/managed/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,25 +61,25 @@ func NewMRMetricRecorder() *MRMetricRecorder {
Name: "managed_resource_time_to_first_reconcile_seconds",
Help: "The time it took for a managed resource to be detected by the controller",
Buckets: kmetrics.ExponentialBuckets(10e-9, 10, 10),
}, []string{"gvk", "claim", "composite"}),
}, []string{"gvk"}),
mrFirstTimeReady: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Subsystem: subSystem,
Name: "managed_resource_first_time_to_readiness_seconds",
Help: "The time it took for a managed resource to become ready first time after creation",
Buckets: []float64{1, 5, 10, 15, 30, 60, 120, 300, 600, 1800, 3600},
}, []string{"gvk", "claim", "composite"}),
}, []string{"gvk"}),
mrDeletion: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Subsystem: subSystem,
Name: "managed_resource_deletion_seconds",
Help: "The time it took for a managed resource to be deleted",
Buckets: []float64{1, 5, 10, 15, 30, 60, 120, 300, 600, 1800, 3600},
}, []string{"gvk", "claim", "composite"}),
}, []string{"gvk"}),
mrDrift: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Subsystem: subSystem,
Name: "managed_resource_drift_seconds",
Help: "ALPHA: How long since the previous successful reconcile when a resource was found to be out of sync; excludes restart of the provider",
Buckets: kmetrics.ExponentialBuckets(10e-9, 10, 10),
}, []string{"gvk", "claim", "composite"}),
}, []string{"gvk"}),
}
}

Expand Down Expand Up @@ -109,7 +109,8 @@ func (r *MRMetricRecorder) recordUnchanged(name string) {

func (r *MRMetricRecorder) recordFirstTimeReconciled(managed resource.Managed) {
if managed.GetCondition(xpv1.TypeSynced).Status == corev1.ConditionUnknown {
r.mrDetected.With(getMRMetricLabels(managed)).Observe(time.Since(managed.GetCreationTimestamp().Time).Seconds())
r.mrDetected.WithLabelValues("gvk", managed.GetObjectKind().GroupVersionKind().String()).
Observe(time.Since(managed.GetCreationTimestamp().Time).Seconds())
r.firstObservation.Store(managed.GetName(), time.Now()) // this is the first time we reconciled on this resource
}
}
Expand All @@ -125,13 +126,15 @@ func (r *MRMetricRecorder) recordDrift(managed resource.Managed) {
return
}

r.mrDrift.With(getMRMetricLabels(managed)).Observe(time.Since(lt).Seconds())
r.mrDrift.WithLabelValues("gvk", managed.GetObjectKind().GroupVersionKind().String()).
Observe(time.Since(lt).Seconds())

r.lastObservation.Store(name, time.Now())
}

func (r *MRMetricRecorder) recordDeleted(managed resource.Managed) {
r.mrDeletion.With(getMRMetricLabels(managed)).Observe(time.Since(managed.GetDeletionTimestamp().Time).Seconds())
r.mrDeletion.WithLabelValues("gvk", managed.GetObjectKind().GroupVersionKind().String()).
Observe(time.Since(managed.GetDeletionTimestamp().Time).Seconds())
}

func (r *MRMetricRecorder) recordFirstTimeReady(managed resource.Managed) {
Expand All @@ -142,7 +145,8 @@ func (r *MRMetricRecorder) recordFirstTimeReady(managed resource.Managed) {
if !ok {
return
}
r.mrFirstTimeReady.With(getMRMetricLabels(managed)).Observe(time.Since(managed.GetCreationTimestamp().Time).Seconds())
r.mrFirstTimeReady.WithLabelValues("gvk", managed.GetObjectKind().GroupVersionKind().String()).
Observe(time.Since(managed.GetCreationTimestamp().Time).Seconds())
r.firstObservation.Delete(managed.GetName())
}
}
Expand Down Expand Up @@ -170,17 +174,3 @@ func (r *NopMetricRecorder) recordDrift(_ resource.Managed) {}
func (r *NopMetricRecorder) recordDeleted(_ resource.Managed) {}

func (r *NopMetricRecorder) recordFirstTimeReady(_ resource.Managed) {}

func getMRMetricLabels(managed resource.Managed) prometheus.Labels {
l := prometheus.Labels{
"gvk": managed.GetObjectKind().GroupVersionKind().String(),
"claim": "",
"composite": managed.GetLabels()["crossplane.io/composite"],
}

if managed.GetLabels()["crossplane.io/claim-namespace"] != "" && managed.GetLabels()["crossplane.io/claim-name"] != "" {
l["claim"] = managed.GetLabels()["crossplane.io/claim-namespace"] + "/" + managed.GetLabels()["crossplane.io/claim-name"]
}

return l
}
7 changes: 4 additions & 3 deletions pkg/reconciler/managed/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
"github.com/crossplane/crossplane-runtime/pkg/logging"
"github.com/crossplane/crossplane-runtime/pkg/meta"
"github.com/crossplane/crossplane-runtime/pkg/resource"
"github.com/crossplane/crossplane-runtime/pkg/statemetrics"
)

const (
Expand Down Expand Up @@ -488,7 +489,7 @@ type Reconciler struct {
log logging.Logger
record event.Recorder
metricRecorder MetricRecorder
stateRecorder StateRecorder
stateRecorder statemetrics.StateRecorder
}

type mrManaged struct {
Expand Down Expand Up @@ -554,7 +555,7 @@ func WithMetricRecorder(recorder MetricRecorder) ReconcilerOption {
}

// WithStateRecorder configures the Reconciler to use the supplied StateRecorder.
func WithStateRecorder(recorder StateRecorder) ReconcilerOption {
func WithStateRecorder(recorder statemetrics.StateRecorder) ReconcilerOption {
return func(r *Reconciler) {
r.stateRecorder = recorder
}
Expand Down Expand Up @@ -718,7 +719,7 @@ func NewReconciler(m manager.Manager, of resource.ManagedKind, o ...ReconcilerOp
log: logging.NewNopLogger(),
record: event.NewNopRecorder(),
metricRecorder: NewNopMetricRecorder(),
stateRecorder: NewNopStateRecorder(),
stateRecorder: statemetrics.NewNopStateRecorder(),
}

for _, ro := range o {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package managed
package statemetrics

import (
"context"
Expand All @@ -31,31 +31,26 @@ import (
"github.com/crossplane/crossplane-runtime/pkg/logging"
)

// A StateRecorder records the state of given GroupVersionKind.
type StateRecorder interface {
Describe(ch chan<- *prometheus.Desc)
Collect(ch chan<- prometheus.Metric)

Record(ctx context.Context, gvk schema.GroupVersionKind)
Run(ctx context.Context, gvk schema.GroupVersionKind)
}
// A MRStateRecorderOption configures a MRStateRecorder.
type MRStateRecorderOption func(*MRStateRecorder)

// A MRStateRecorder records the state of managed resources.
type MRStateRecorder struct {
client client.Client
log logging.Logger
frequency time.Duration
client client.Client
log logging.Logger
interval time.Duration

mrExists *prometheus.GaugeVec
mrReady *prometheus.GaugeVec
mrSynced *prometheus.GaugeVec
}

// NewMRStateRecorder returns a new MRStateRecorder which records the state of managed resources.
func NewMRStateRecorder(client client.Client, log logging.Logger, o ...StateRecorderOption) *MRStateRecorder {
r := &MRStateRecorder{
client: client,
log: log,
func NewMRStateRecorder(client client.Client, log logging.Logger, interval time.Duration) *MRStateRecorder {
return &MRStateRecorder{
client: client,
log: log,
interval: interval,

mrExists: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Subsystem: subSystem,
Expand All @@ -73,23 +68,6 @@ func NewMRStateRecorder(client client.Client, log logging.Logger, o ...StateReco
Help: "The number of managed resources in Synced=True state",
}, []string{"gvk"}),
}

for _, ro := range o {
ro(r)
}

return r
}

// A StateRecorderOption configures a MRStateRecorder.
type StateRecorderOption func(*MRStateRecorder)

// WithFrequency configures the frequency at which the MRStateRecorder
// will record.
func WithFrequency(f time.Duration) StateRecorderOption {
return func(r *MRStateRecorder) {
r.frequency = f
}
}

// Describe sends the super-set of all possible descriptors of metrics
Expand Down Expand Up @@ -145,9 +123,9 @@ func (r *MRStateRecorder) Record(ctx context.Context, gvk schema.GroupVersionKin
r.mrSynced.WithLabelValues(label).Set(numSynced)
}

// Run records state of managed resources with given frequency.
// Run records state of managed resources with given interval.
func (r *MRStateRecorder) Run(ctx context.Context, gvk schema.GroupVersionKind) {
ticker := time.NewTicker(r.frequency)
ticker := time.NewTicker(r.interval)
quit := make(chan struct{})
go func() {
for {
Expand All @@ -161,23 +139,3 @@ func (r *MRStateRecorder) Run(ctx context.Context, gvk schema.GroupVersionKind)
}
}()
}

// A NopStateRecorder does nothing.
type NopStateRecorder struct{}

// NewNopStateRecorder returns a NopStateRecorder that does nothing.
func NewNopStateRecorder() *NopStateRecorder {
return &NopStateRecorder{}
}

// Describe does nothing.
func (r *NopStateRecorder) Describe(_ chan<- *prometheus.Desc) {}

// Collect does nothing.
func (r *NopStateRecorder) Collect(_ chan<- prometheus.Metric) {}

// Record does nothing.
func (r *NopStateRecorder) Record(_ context.Context, _ schema.GroupVersionKind) {}

// Run does nothing.
func (r *NopStateRecorder) Run(_ context.Context, _ schema.GroupVersionKind) {}
56 changes: 56 additions & 0 deletions pkg/statemetrics/state_recorder.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/*
Copyright 2024 The Crossplane Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package statemetrics contains utilities for recording Crossplane resource state metrics.
package statemetrics

import (
"context"

"github.com/prometheus/client_golang/prometheus"
"k8s.io/apimachinery/pkg/runtime/schema"
)

const subSystem = "crossplane"

// A StateRecorder records the state of given GroupVersionKind.
type StateRecorder interface {
Describe(ch chan<- *prometheus.Desc)
Collect(ch chan<- prometheus.Metric)

Record(ctx context.Context, gvk schema.GroupVersionKind)
Run(ctx context.Context, gvk schema.GroupVersionKind)
}

// A NopStateRecorder does nothing.
type NopStateRecorder struct{}

// NewNopStateRecorder returns a NopStateRecorder that does nothing.
func NewNopStateRecorder() *NopStateRecorder {
return &NopStateRecorder{}
}

// Describe does nothing.
func (r *NopStateRecorder) Describe(_ chan<- *prometheus.Desc) {}

// Collect does nothing.
func (r *NopStateRecorder) Collect(_ chan<- prometheus.Metric) {}

// Record does nothing.
func (r *NopStateRecorder) Record(_ context.Context, _ schema.GroupVersionKind) {}

// Run does nothing.
func (r *NopStateRecorder) Run(_ context.Context, _ schema.GroupVersionKind) {}
Loading

0 comments on commit 7c1aaab

Please sign in to comment.