Skip to content

Commit 1f23a08

Browse files
anders-swansonanders-swanson
authored
Cache metrics for out of sync scrape intervals (#316)
Signed-off-by: Anders Swanson <anders.swanson@oracle.com>
1 parent 5e0f10b commit 1f23a08

File tree

34 files changed

+131
-76
lines changed

34 files changed

+131
-76
lines changed

‎collector/collector.go

Lines changed: 27 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -248,11 +248,15 @@ func (e *Exporter) scrapeDatabase(ch chan<- prometheus.Metric, errChan chan<- er
248248
metricsToScrape := 0
249249
for _, metric := range e.metricsToScrape.Metric {
250250
metric := metric //https://golang.org/doc/faq#closures_and_goroutines
251-
if !e.isScrapeMetric(tick, metric, d) {
252-
continue
253-
}
251+
isScrapeMetric := e.isScrapeMetric(tick, metric, d)
254252
metricsToScrape++
255253
go func() {
254+
// If the metric doesn't need to be scraped, send the cached values
255+
if !isScrapeMetric {
256+
metric.sendAll(ch)
257+
errChan <- nil
258+
return
259+
}
256260
e.logger.Debug("About to scrape metric",
257261
"Context", metric.Context,
258262
"MetricsDesc", fmt.Sprint(metric.MetricsDesc),
@@ -389,7 +393,7 @@ func hashFile(h hash.Hash, fn string) error {
389393

390394
func (e *Exporter) reloadMetrics() {
391395
// Truncate metricsToScrape
392-
e.metricsToScrape.Metric = []Metric{}
396+
e.metricsToScrape.Metric = []*Metric{}
393397

394398
// Load default metrics
395399
defaultMetrics := e.DefaultMetrics()
@@ -415,29 +419,24 @@ func (e *Exporter) reloadMetrics() {
415419
}
416420

417421
// ScrapeMetric is an interface method to call scrapeGenericValues using Metric struct values
418-
func (e *Exporter) ScrapeMetric(d *Database, ch chan<- prometheus.Metric, m Metric) error {
422+
func (e *Exporter) ScrapeMetric(d *Database, ch chan<- prometheus.Metric, m *Metric) error {
419423
e.logger.Debug("Calling function ScrapeGenericValues()")
420-
queryTimeout := e.getQueryTimeout(m, d)
421-
return e.scrapeGenericValues(d, ch, m.Context, m.Labels, m.MetricsDesc,
422-
m.MetricsType, m.MetricsBuckets, m.FieldToAppend, m.IgnoreZeroResult,
423-
m.Request, queryTimeout)
424+
return e.scrapeGenericValues(d, ch, m)
424425
}
425426

426427
// generic method for retrieving metrics.
427-
func (e *Exporter) scrapeGenericValues(d *Database, ch chan<- prometheus.Metric, context string, labels []string,
428-
metricsDesc map[string]string, metricsType map[string]string, metricsBuckets map[string]map[string]string,
429-
fieldToAppend string, ignoreZeroResult bool, request string, queryTimeout time.Duration) error {
428+
func (e *Exporter) scrapeGenericValues(d *Database, ch chan<- prometheus.Metric, m *Metric) error {
430429
metricsCount := 0
431430
constLabels := d.constLabels(e.constLabels())
432431
e.logger.Debug("labels", constLabels)
433432
genericParser := func(row map[string]string) error {
434433
// Construct labels value
435434
labelsValues := []string{}
436-
for _, label := range labels {
435+
for _, label := range m.Labels {
437436
labelsValues = append(labelsValues, row[label])
438437
}
439438
// Construct Prometheus values to sent back
440-
for metric, metricHelp := range metricsDesc {
439+
for metric, metricHelp := range m.MetricsDesc {
441440
value, ok := e.parseFloat(metric, metricHelp, row)
442441
if !ok {
443442
// Skip invalid metric values
@@ -446,22 +445,22 @@ func (e *Exporter) scrapeGenericValues(d *Database, ch chan<- prometheus.Metric,
446445
e.logger.Debug("Query result",
447446
"value", value)
448447
// If metric do not use a field content in metric's name
449-
if strings.Compare(fieldToAppend, "") == 0 {
448+
if strings.Compare(m.FieldToAppend, "") == 0 {
450449
desc := prometheus.NewDesc(
451-
prometheus.BuildFQName(namespace, context, metric),
450+
prometheus.BuildFQName(namespace, m.Context, metric),
452451
metricHelp,
453-
labels,
452+
m.Labels,
454453
constLabels,
455454
)
456-
if metricsType[strings.ToLower(metric)] == "histogram" {
455+
if m.MetricsType[strings.ToLower(metric)] == "histogram" {
457456
count, err := strconv.ParseUint(strings.TrimSpace(row["count"]), 10, 64)
458457
if err != nil {
459458
e.logger.Error("Unable to convert count value to int (metric=" + metric +
460459
",metricHelp=" + metricHelp + ",value=<" + row["count"] + ">)")
461460
continue
462461
}
463462
buckets := make(map[float64]uint64)
464-
for field, le := range metricsBuckets[metric] {
463+
for field, le := range m.MetricsBuckets[metric] {
465464
lelimit, err := strconv.ParseFloat(strings.TrimSpace(le), 64)
466465
if err != nil {
467466
e.logger.Error("Unable to convert bucket limit value to float (metric=" + metric +
@@ -476,26 +475,26 @@ func (e *Exporter) scrapeGenericValues(d *Database, ch chan<- prometheus.Metric,
476475
}
477476
buckets[lelimit] = counter
478477
}
479-
ch <- prometheus.MustNewConstHistogram(desc, count, value, buckets, labelsValues...)
478+
m.cacheAndSend(ch, prometheus.MustNewConstHistogram(desc, count, value, buckets, labelsValues...))
480479
} else {
481-
ch <- prometheus.MustNewConstMetric(desc, getMetricType(metric, metricsType), value, labelsValues...)
480+
m.cacheAndSend(ch, prometheus.MustNewConstMetric(desc, getMetricType(metric, m.MetricsType), value, labelsValues...))
482481
}
483482
// If no labels, use metric name
484483
} else {
485484
desc := prometheus.NewDesc(
486-
prometheus.BuildFQName(namespace, context, cleanName(row[fieldToAppend])),
485+
prometheus.BuildFQName(namespace, m.Context, cleanName(row[m.FieldToAppend])),
487486
metricHelp,
488487
nil, constLabels,
489488
)
490-
if metricsType[strings.ToLower(metric)] == "histogram" {
489+
if m.MetricsType[strings.ToLower(metric)] == "histogram" {
491490
count, err := strconv.ParseUint(strings.TrimSpace(row["count"]), 10, 64)
492491
if err != nil {
493492
e.logger.Error("Unable to convert count value to int (metric=" + metric +
494493
",metricHelp=" + metricHelp + ",value=<" + row["count"] + ">)")
495494
continue
496495
}
497496
buckets := make(map[float64]uint64)
498-
for field, le := range metricsBuckets[metric] {
497+
for field, le := range m.MetricsBuckets[metric] {
499498
lelimit, err := strconv.ParseFloat(strings.TrimSpace(le), 64)
500499
if err != nil {
501500
e.logger.Error("Unable to convert bucket limit value to float (metric=" + metric +
@@ -510,22 +509,22 @@ func (e *Exporter) scrapeGenericValues(d *Database, ch chan<- prometheus.Metric,
510509
}
511510
buckets[lelimit] = counter
512511
}
513-
ch <- prometheus.MustNewConstHistogram(desc, count, value, buckets)
512+
m.cacheAndSend(ch, prometheus.MustNewConstHistogram(desc, count, value, buckets))
514513
} else {
515-
ch <- prometheus.MustNewConstMetric(desc, getMetricType(metric, metricsType), value)
514+
m.cacheAndSend(ch, prometheus.MustNewConstMetric(desc, getMetricType(metric, m.MetricsType), value))
516515
}
517516
}
518517
metricsCount++
519518
}
520519
return nil
521520
}
522521
e.logger.Debug("Calling function GeneratePrometheusMetrics()")
523-
err := e.generatePrometheusMetrics(d, genericParser, request, queryTimeout)
522+
err := e.generatePrometheusMetrics(d, genericParser, m.Request, e.getQueryTimeout(m, d))
524523
e.logger.Debug("ScrapeGenericValues() - metricsCount: " + strconv.Itoa(metricsCount))
525524
if err != nil {
526525
return err
527526
}
528-
if !ignoreZeroResult && metricsCount == 0 {
527+
if !m.IgnoreZeroResult && metricsCount == 0 {
529528
// a zero result error is returned for caller error identification.
530529
// https://github.com/oracle/oracle-db-appdev-monitoring/issues/168
531530
return newZeroResultError()

‎collector/metrics.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
// isScrapeMetric returns true if a metric should be scraped. Metrics may not be scraped if they have a custom scrape interval,
1414
// and the time since the last scrape is less than the custom scrape interval.
1515
// If there is no tick time or last known tick, the metric is always scraped.
16-
func (e *Exporter) isScrapeMetric(tick *time.Time, metric Metric, d *Database) bool {
16+
func (e *Exporter) isScrapeMetric(tick *time.Time, metric *Metric, d *Database) bool {
1717
if len(metric.Databases) > 0 {
1818
if !slices.Contains(metric.Databases, d.Name) {
1919
return false
@@ -52,7 +52,7 @@ func (e *Exporter) getScrapeInterval(context, scrapeInterval string) (time.Durat
5252
return 0, false
5353
}
5454

55-
func (e *Exporter) getQueryTimeout(metric Metric, d *Database) time.Duration {
55+
func (e *Exporter) getQueryTimeout(metric *Metric, d *Database) time.Duration {
5656
if len(metric.QueryTimeout) > 0 {
5757
qt, err := time.ParseDuration(metric.QueryTimeout)
5858
if err != nil {

‎collector/types.go

Lines changed: 35 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,11 @@ type Exporter struct {
2828
}
2929

3030
type Database struct {
31-
Name string
32-
Up float64
33-
Session *sql.DB
34-
Type float64
35-
Config DatabaseConfig
31+
Name string
32+
Up float64
33+
Session *sql.DB
34+
Type float64
35+
Config DatabaseConfig
3636
}
3737

3838
type Config struct {
@@ -57,28 +57,29 @@ type Config struct {
5757

5858
// Metric is an object description
5959
type Metric struct {
60-
Context string
61-
Labels []string
62-
MetricsDesc map[string]string
63-
MetricsType map[string]string
64-
MetricsBuckets map[string]map[string]string
65-
FieldToAppend string
66-
Request string
67-
IgnoreZeroResult bool
68-
QueryTimeout string
69-
ScrapeInterval string
70-
Databases []string
60+
Context string
61+
Labels []string
62+
MetricsDesc map[string]string
63+
MetricsType map[string]string
64+
MetricsBuckets map[string]map[string]string
65+
FieldToAppend string
66+
Request string
67+
IgnoreZeroResult bool
68+
QueryTimeout string
69+
ScrapeInterval string
70+
Databases []string
71+
PrometheusMetrics map[string]prometheus.Metric
7172
}
7273

7374
// Metrics is a container structure for prometheus metrics
7475
type Metrics struct {
75-
Metric []Metric
76+
Metric []*Metric
7677
}
7778

7879
type ScrapeContext struct {
7980
}
8081

81-
func (m Metric) id(dbname string) string {
82+
func (m *Metric) id(dbname string) string {
8283
builder := strings.Builder{}
8384
builder.WriteString(dbname)
8485
builder.WriteString(m.Context)
@@ -87,3 +88,19 @@ func (m Metric) id(dbname string) string {
8788
}
8889
return builder.String()
8990
}
91+
92+
// sendAll sends all cached metrics to the collector.
93+
func (m *Metric) sendAll(ch chan<- prometheus.Metric) {
94+
for _, metric := range m.PrometheusMetrics {
95+
ch <- metric
96+
}
97+
}
98+
99+
// cacheAndSend caches the metric and sends it to the collector.
100+
func (m *Metric) cacheAndSend(ch chan<- prometheus.Metric, metric prometheus.Metric) {
101+
if len(m.PrometheusMetrics) == 0 {
102+
m.PrometheusMetrics = map[string]prometheus.Metric{}
103+
}
104+
m.PrometheusMetrics[metric.Desc().String()] = metric
105+
ch <- metric
106+
}

‎custom-metrics-example/custom-metrics.toml

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,38 +16,45 @@ from gv$sql
1616
where last_active_time >= sysdate - 5/(24*60)
1717
'''
1818

19+
# User segment queries may return zero rows on certain database configurations
1920
[[metric]]
2021
context = "size_user_segments_top100"
2122
metricsdesc = {table_bytes="Gauge metric with the size of the tables in user segments."}
2223
labels = ["segment_name"]
2324
request = "select * from (select segment_name,sum(bytes) as table_bytes from user_segments where segment_type='TABLE' group by segment_name) order by table_bytes DESC FETCH NEXT 100 ROWS ONLY"
25+
ignorezeroresult = true
2426

2527
[[metric]]
2628
context = "size_user_segments_top100"
2729
metricsdesc = {table_partition_bytes="Gauge metric with the size of the table partition in user segments."}
2830
labels = ["segment_name"]
2931
request = "select * from (select segment_name,sum(bytes) as table_partition_bytes from user_segments where segment_type='TABLE PARTITION' group by segment_name) order by table_partition_bytes DESC FETCH NEXT 100 ROWS ONLY"
32+
ignorezeroresult = true
3033

3134
[[metric]]
3235
context = "size_user_segments_top100"
3336
metricsdesc = {cluster_bytes="Gauge metric with the size of the cluster in user segments."}
3437
labels = ["segment_name"]
3538
request = "select * from (select segment_name,sum(bytes) as cluster_bytes from user_segments where segment_type='CLUSTER' group by segment_name) order by cluster_bytes DESC FETCH NEXT 100 ROWS ONLY"
39+
ignorezeroresult = true
3640

3741
[[metric]]
3842
context = "size_dba_segments_top100"
3943
metricsdesc = {table_bytes="Gauge metric with the size of the tables in user segments."}
4044
labels = ["segment_name"]
4145
request = "select * from (select segment_name,sum(bytes) as table_bytes from dba_segments where segment_type='TABLE' group by segment_name) order by table_bytes DESC FETCH NEXT 100 ROWS ONLY"
46+
ignorezeroresult = true
4247

4348
[[metric]]
4449
context = "size_dba_segments_top100"
4550
metricsdesc = {table_partition_bytes="Gauge metric with the size of the table partition in user segments."}
4651
labels = ["segment_name"]
4752
request = "select * from (select segment_name,sum(bytes) as table_partition_bytes from dba_segments where segment_type='TABLE PARTITION' group by segment_name) order by table_partition_bytes DESC FETCH NEXT 100 ROWS ONLY"
53+
ignorezeroresult = true
4854

4955
[[metric]]
5056
context = "size_dba_segments_top100"
5157
metricsdesc = {cluster_bytes="Gauge metric with the size of the cluster in user segments."}
5258
labels = ["segment_name"]
53-
request = "select * from (select segment_name,sum(bytes) as cluster_bytes from dba_segments where segment_type='CLUSTER' group by segment_name) order by cluster_bytes DESC FETCH NEXT 100 ROWS ONLY"
59+
request = "select * from (select segment_name,sum(bytes) as cluster_bytes from dba_segments where segment_type='CLUSTER' group by segment_name) order by cluster_bytes DESC FETCH NEXT 100 ROWS ONLY"
60+
ignorezeroresult = true

‎docs/404.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<meta charset="UTF-8">
55
<meta name="generator" content="Docusaurus v3.8.1">
66
<title data-rh="true">Page Not Found | Oracle Database Metrics Exporter</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://oracle.github.io/oracle-db-appdev-monitoring/img/logo.png"><meta data-rh="true" name="twitter:image" content="https://oracle.github.io/oracle-db-appdev-monitoring/img/logo.png"><meta data-rh="true" property="og:url" content="https://oracle.github.io/oracle-db-appdev-monitoring/404.html"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_tag" content="default"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docsearch:docusaurus_tag" content="default"><meta data-rh="true" property="og:title" content="Page Not Found | Oracle Database Metrics Exporter"><link data-rh="true" rel="icon" href="/oracle-db-appdev-monitoring/img/favicon-32x32.png"><link data-rh="true" rel="canonical" href="https://oracle.github.io/oracle-db-appdev-monitoring/404.html"><link data-rh="true" rel="alternate" href="https://oracle.github.io/oracle-db-appdev-monitoring/404.html" hreflang="en"><link data-rh="true" rel="alternate" href="https://oracle.github.io/oracle-db-appdev-monitoring/404.html" hreflang="x-default"><link rel="stylesheet" href="/oracle-db-appdev-monitoring/assets/css/styles.d329a656.css">
7-
<script src="/oracle-db-appdev-monitoring/assets/js/runtime~main.f4fbb5be.js" defer="defer"></script>
7+
<script src="/oracle-db-appdev-monitoring/assets/js/runtime~main.17989aec.js" defer="defer"></script>
88
<script src="/oracle-db-appdev-monitoring/assets/js/main.dfeca7cc.js" defer="defer"></script>
99
</head>
1010
<body class="navigation-with-keyboard">

‎docs/assets/js/c539bf3f.35853f87.js

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/assets/js/c539bf3f.7f73b018.js

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎docs/assets/js/runtime~main.17989aec.js

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)