Skip to content

Commit

Permalink
prometheus.operator.* add debug endpoint to view generated scrape c…
Browse files Browse the repository at this point in the history
…onfigs. (#5311)

* add endpoint to debug generated scrape configs

* changelog
  • Loading branch information
captncraig authored Oct 3, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 751a2fa commit 3684eea
Showing 7 changed files with 76 additions and 15 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -132,6 +132,8 @@ Main (unreleased)
- Add new `agent_component_dependencies_wait_seconds` histogram metric and a dashboard panel
that measures how long components wait to be evaluated after their dependency is updated (@thampiotr)

- Add additional endpoint to debug scrape configs generated inside `prometheus.operator.*` components (@captncraig)

- Components evaluation is now performed in parallel, reducing the impact of
slow components potentially blocking the entire telemetry pipeline.
The `agent_component_evaluation_seconds` metric now measures evaluation time
38 changes: 38 additions & 0 deletions component/prometheus/operator/common/component.go
Original file line number Diff line number Diff line change
@@ -3,13 +3,16 @@ package common
import (
"context"
"fmt"
"net/http"
"strings"
"sync"
"time"

"github.com/go-kit/log/level"
"github.com/grafana/agent/component"
"github.com/grafana/agent/component/prometheus/operator"
"github.com/grafana/agent/service/cluster"
"gopkg.in/yaml.v3"
)

type Component struct {
@@ -143,3 +146,38 @@ func (c *Component) reportHealth(err error) {
}
}
}

func (c *Component) Handler() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// very simple path handling
// only responds to `/scrapeConfig/$NS/$NAME`
c.mut.RLock()
man := c.manager
c.mut.RUnlock()
path := strings.Trim(r.URL.Path, "/")
parts := strings.Split(path, "/")
if man == nil || len(parts) != 3 || parts[0] != "scrapeConfig" {
w.WriteHeader(404)
return
}
ns := parts[1]
name := parts[2]
scs := man.getScrapeConfig(ns, name)
if len(scs) == 0 {
w.WriteHeader(404)
return
}
dat, err := yaml.Marshal(scs)
if err != nil {
if _, err = w.Write([]byte(err.Error())); err != nil {
return
}
w.WriteHeader(500)
return
}
_, err = w.Write(dat)
if err != nil {
w.WriteHeader(500)
}
})
}
33 changes: 25 additions & 8 deletions component/prometheus/operator/common/crdmanager.go
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@ import (
"github.com/grafana/agent/component"
"github.com/grafana/agent/component/prometheus"
"github.com/grafana/agent/service/cluster"
"github.com/grafana/agent/service/http"
"github.com/grafana/ckit/shard"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/config"
@@ -215,6 +216,17 @@ func (c *crdManager) DebugInfo() interface{} {
return info
}

func (c *crdManager) getScrapeConfig(ns, name string) []*config.ScrapeConfig {
prefix := fmt.Sprintf("%s/%s/%s", c.kind, ns, name)
matches := []*config.ScrapeConfig{}
for k, v := range c.scrapeConfigs {
if strings.HasPrefix(k, prefix) {
matches = append(matches, v)
}
}
return matches
}

// runInformers starts all the informers that are required to discover CRDs.
func (c *crdManager) runInformers(restConfig *rest.Config, ctx context.Context) error {
scheme := runtime.NewScheme()
@@ -358,6 +370,11 @@ func (c *crdManager) addDebugInfo(ns string, name string, err error) {
} else {
debug.ReconcileError = ""
}
if data, err := c.opts.GetServiceData(http.ServiceName); err == nil {
if hdata, ok := data.(http.Data); ok {
debug.ScrapeConfigsURL = fmt.Sprintf("%s%s/scrapeConfig/%s/%s", hdata.HTTPListenAddr, hdata.HTTPPathForComponent(c.opts.ID), ns, name)
}
}
prefix := fmt.Sprintf("%s/%s/%s", c.kind, ns, name)
c.debugInfo[prefix] = debug
}
@@ -400,13 +417,13 @@ func (c *crdManager) onAddPodMonitor(obj interface{}) {
}
func (c *crdManager) onUpdatePodMonitor(oldObj, newObj interface{}) {
pm := oldObj.(*promopv1.PodMonitor)
c.clearConfigs("podMonitor", pm.Namespace, pm.Name)
c.clearConfigs(pm.Namespace, pm.Name)
c.addPodMonitor(newObj.(*promopv1.PodMonitor))
}

func (c *crdManager) onDeletePodMonitor(obj interface{}) {
pm := obj.(*promopv1.PodMonitor)
c.clearConfigs("podMonitor", pm.Namespace, pm.Name)
c.clearConfigs(pm.Namespace, pm.Name)
if err := c.apply(); err != nil {
level.Error(c.logger).Log("name", pm.Name, "err", err, "msg", "error applying scrape configs after deleting "+c.kind)
}
@@ -450,13 +467,13 @@ func (c *crdManager) onAddServiceMonitor(obj interface{}) {
}
func (c *crdManager) onUpdateServiceMonitor(oldObj, newObj interface{}) {
pm := oldObj.(*promopv1.ServiceMonitor)
c.clearConfigs("serviceMonitor", pm.Namespace, pm.Name)
c.clearConfigs(pm.Namespace, pm.Name)
c.addServiceMonitor(newObj.(*promopv1.ServiceMonitor))
}

func (c *crdManager) onDeleteServiceMonitor(obj interface{}) {
pm := obj.(*promopv1.ServiceMonitor)
c.clearConfigs("serviceMonitor", pm.Namespace, pm.Name)
c.clearConfigs(pm.Namespace, pm.Name)
if err := c.apply(); err != nil {
level.Error(c.logger).Log("name", pm.Name, "err", err, "msg", "error applying scrape configs after deleting "+c.kind)
}
@@ -498,22 +515,22 @@ func (c *crdManager) onAddProbe(obj interface{}) {
}
func (c *crdManager) onUpdateProbe(oldObj, newObj interface{}) {
pm := oldObj.(*promopv1.Probe)
c.clearConfigs("probe", pm.Namespace, pm.Name)
c.clearConfigs(pm.Namespace, pm.Name)
c.addProbe(newObj.(*promopv1.Probe))
}

func (c *crdManager) onDeleteProbe(obj interface{}) {
pm := obj.(*promopv1.Probe)
c.clearConfigs("probe", pm.Namespace, pm.Name)
c.clearConfigs(pm.Namespace, pm.Name)
if err := c.apply(); err != nil {
level.Error(c.logger).Log("name", pm.Name, "err", err, "msg", "error applying scrape configs after deleting "+c.kind)
}
}

func (c *crdManager) clearConfigs(kind, ns, name string) {
func (c *crdManager) clearConfigs(ns, name string) {
c.mut.Lock()
defer c.mut.Unlock()
prefix := fmt.Sprintf("%s/%s/%s", kind, ns, name)
prefix := fmt.Sprintf("%s/%s/%s", c.kind, ns, name)
for k := range c.discoveryConfigs {
if strings.HasPrefix(k, prefix) {
delete(c.discoveryConfigs, k)
3 changes: 2 additions & 1 deletion component/prometheus/operator/podmonitors/operator.go
Original file line number Diff line number Diff line change
@@ -5,13 +5,14 @@ import (
"github.com/grafana/agent/component/prometheus/operator"
"github.com/grafana/agent/component/prometheus/operator/common"
"github.com/grafana/agent/service/cluster"
"github.com/grafana/agent/service/http"
)

func init() {
component.Register(component.Registration{
Name: "prometheus.operator.podmonitors",
Args: operator.Arguments{},
NeedsServices: []string{cluster.ServiceName},
NeedsServices: []string{cluster.ServiceName, http.ServiceName},

Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
return common.New(opts, args, common.KindPodMonitor)
3 changes: 2 additions & 1 deletion component/prometheus/operator/probes/probes.go
Original file line number Diff line number Diff line change
@@ -5,13 +5,14 @@ import (
"github.com/grafana/agent/component/prometheus/operator"
"github.com/grafana/agent/component/prometheus/operator/common"
"github.com/grafana/agent/service/cluster"
"github.com/grafana/agent/service/http"
)

func init() {
component.Register(component.Registration{
Name: "prometheus.operator.probes",
Args: operator.Arguments{},
NeedsServices: []string{cluster.ServiceName},
NeedsServices: []string{cluster.ServiceName, http.ServiceName},

Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
return common.New(opts, args, common.KindProbe)
Original file line number Diff line number Diff line change
@@ -5,13 +5,14 @@ import (
"github.com/grafana/agent/component/prometheus/operator"
"github.com/grafana/agent/component/prometheus/operator/common"
"github.com/grafana/agent/service/cluster"
"github.com/grafana/agent/service/http"
)

func init() {
component.Register(component.Registration{
Name: "prometheus.operator.servicemonitors",
Args: operator.Arguments{},
NeedsServices: []string{cluster.ServiceName},
NeedsServices: []string{cluster.ServiceName, http.ServiceName},

Build: func(opts component.Options, args component.Arguments) (component.Component, error) {
return common.New(opts, args, common.KindServiceMonitor)
9 changes: 5 additions & 4 deletions component/prometheus/operator/types.go
Original file line number Diff line number Diff line change
@@ -71,8 +71,9 @@ type DebugInfo struct {
}

type DiscoveredResource struct {
Namespace string `river:"namespace,attr"`
Name string `river:"name,attr"`
LastReconcile time.Time `river:"last_reconcile,attr,optional"`
ReconcileError string `river:"reconcile_error,attr,optional"`
Namespace string `river:"namespace,attr"`
Name string `river:"name,attr"`
LastReconcile time.Time `river:"last_reconcile,attr,optional"`
ReconcileError string `river:"reconcile_error,attr,optional"`
ScrapeConfigsURL string `river:"scrape_configs_url,attr,optional"`
}

0 comments on commit 3684eea

Please sign in to comment.