Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature: Add redis search module metrics #953

Merged
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ test:
TEST_REDIS_CLUSTER_PASSWORD_URI="redis://localhost:17006" \
TEST_TILE38_URI="redis://localhost:19851" \
TEST_REDIS_SENTINEL_URI="redis://localhost:26379" \
go test -v -covermode=atomic -cover -race -coverprofile=coverage.txt -p 1 ./...
TEST_REDIS_MODULES_URI="redis://localhost:36379" \
go test -v -covermode=atomic -cover -race -coverprofile=coverage.txt -p 1 ./...

.PHONY: lint
lint:
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ Prometheus uses file watches and all changes to the json file are applied immedi
| redis-only-metrics | REDIS_EXPORTER_REDIS_ONLY_METRICS | Whether to also export go runtime metrics, defaults to false. |
| include-config-metrics | REDIS_EXPORTER_INCL_CONFIG_METRICS | Whether to include all config settings as metrics, defaults to false. |
| include-system-metrics | REDIS_EXPORTER_INCL_SYSTEM_METRICS | Whether to include system metrics like `total_system_memory_bytes`, defaults to false. |
| include-modules-metrics | REDIS_EXPORTER_INCL_MODULES_METRICS | Whether to collect Redis Modules metrics, defaults to false. |
| exclude-latency-histogram-metrics | REDIS_EXPORTER_EXCLUDE_LATENCY_HISTOGRAM_METRICS | Do not try to collect latency histogram metrics (to avoid `WARNING, LOGGED ONCE ONLY: cmd LATENCY HISTOGRAM` error on Redis < v7). |
| redact-config-metrics | REDIS_EXPORTER_REDACT_CONFIG_METRICS | Whether to redact config settings that include potentially sensitive information like passwords. |
| ping-on-connect | REDIS_EXPORTER_PING_ON_CONNECT | Whether to ping the redis instance after connecting and record the duration as a metric, defaults to false. |
Expand Down
5 changes: 5 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,3 +83,8 @@ services:
image: tile38/tile38:latest
ports:
- "19851:9851"

redis-stack:
image: redis/redis-stack-server:7.4.0-v0
Copy link
Contributor Author

@nantiferov nantiferov Sep 19, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For context, 7.4.0-v0 is current version of redis-stack-server and it includes redis 7.4 + modules with recent versions.

ports:
- "36379:6379"
21 changes: 21 additions & 0 deletions exporter/exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ type Options struct {
ClientKeyFile string
CaCertFile string
InclConfigMetrics bool
InclModulesMetrics bool
DisableExportingKeyValues bool
ExcludeLatencyHistogramMetrics bool
RedactConfigMetrics bool
Expand Down Expand Up @@ -267,6 +268,21 @@ func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) {
"server_threads": "server_threads_total",
"long_lock_waits": "long_lock_waits_total",
"current_client_thread": "current_client_thread",

// Redis Modules metrics
// RediSearch module
"search_number_of_indexes": "search_number_of_indexes",
"search_used_memory_indexes": "search_used_memory_indexes_bytes",
oliver006 marked this conversation as resolved.
Show resolved Hide resolved
"search_total_indexing_time": "search_total_indexing_time",
"search_global_idle": "search_global_idle",
"search_global_total": "search_global_total",
"search_bytes_collected": "search_bytes_collected",
"search_total_cycles": "search_total_cycles",
"search_total_ms_run": "search_total_ms_run",
"search_dialect_1": "search_dialect_1",
"search_dialect_2": "search_dialect_2",
"search_dialect_3": "search_dialect_3",
"search_dialect_4": "search_dialect_4",
oliver006 marked this conversation as resolved.
Show resolved Hide resolved
},

metricMapCounters: map[string]string{
Expand Down Expand Up @@ -421,6 +437,7 @@ func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) {
"stream_radix_tree_keys": {txt: `Radix tree keys count"`, lbls: []string{"db", "stream"}},
"stream_radix_tree_nodes": {txt: `Radix tree nodes count`, lbls: []string{"db", "stream"}},
"up": {txt: "Information about the Redis instance"},
"module_info": {txt: "Information about loaded Redis module", lbls: []string{"name", "ver", "api", "filters", "usedby", "using"}},
} {
e.metricDescriptions[k] = newMetricDescr(opts.Namespace, k, desc.txt, desc.lbls)
}
Expand Down Expand Up @@ -698,6 +715,10 @@ func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error {
e.extractTile38Metrics(ch, c)
}

if e.options.InclModulesMetrics {
e.extractModulesMetrics(ch, c)
}

if len(e.options.LuaScript) > 0 {
for filename, script := range e.options.LuaScript {
if err := e.extractLuaScriptMetrics(ch, c, filename, script); err != nil {
Expand Down
1 change: 1 addition & 0 deletions exporter/http_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ func TestSimultaneousMetricsHttpRequests(t *testing.T) {

os.Getenv("TEST_REDIS5_URI"),
os.Getenv("TEST_REDIS6_URI"),
os.Getenv("TEST_REDIS_MODULES_URI"),
oliver006 marked this conversation as resolved.
Show resolved Hide resolved

// tile38 & Cluster need to be last in this list so we can identify them when selected, down in line 229
os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI"),
Expand Down
52 changes: 52 additions & 0 deletions exporter/modules.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package exporter

import (
"strings"

"github.com/gomodule/redigo/redis"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
)

func (e *Exporter) extractModulesMetrics(ch chan<- prometheus.Metric, c redis.Conn) {
info, err := redis.String(doRedisCmd(c, "INFO", "MODULES"))
if err != nil {
log.Errorf("extractSearchMetrics() err: %s", err)
return

Check warning on line 15 in exporter/modules.go

View check run for this annotation

Codecov / codecov/patch

exporter/modules.go#L14-L15

Added lines #L14 - L15 were not covered by tests
}

lines := strings.Split(info, "\r\n")
for _, line := range lines {
log.Debugf("info: %s", line)

split := strings.Split(line, ":")
oliver006 marked this conversation as resolved.
Show resolved Hide resolved
if len(split) != 2 {
continue
}

if split[0] == "module" {
// module format: 'module:name=search,ver=21005,api=1,filters=0,usedby=[],using=[ReJSON],options=[handle-io-errors]'
module := strings.Split(split[1], ",")
oliver006 marked this conversation as resolved.
Show resolved Hide resolved
if len(module) != 7 {
oliver006 marked this conversation as resolved.
Show resolved Hide resolved
continue

Check warning on line 31 in exporter/modules.go

View check run for this annotation

Codecov / codecov/patch

exporter/modules.go#L31

Added line #L31 was not covered by tests
}
e.registerConstMetricGauge(ch, "module_info", 1,
strings.Split(module[0], "=")[1],
strings.Split(module[1], "=")[1],
strings.Split(module[2], "=")[1],
strings.Split(module[3], "=")[1],
strings.Split(module[4], "=")[1],
strings.Split(module[5], "=")[1],
)
continue
}

fieldKey := split[0]
fieldValue := split[1]

if !e.includeMetric(fieldKey) {
continue
}
e.parseAndRegisterConstMetric(ch, fieldKey, fieldValue)
}
}
74 changes: 74 additions & 0 deletions exporter/modules_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package exporter

import (
"os"
"strings"
"testing"

"github.com/prometheus/client_golang/prometheus"
)

func TestModules(t *testing.T) {
if os.Getenv("TEST_REDIS_MODULES_URI") == "" {
t.Skipf("TEST_REDIS_MODULES_URI not set - skipping")
}

tsts := []struct {
addr string
inclModulesMetrics bool
wantModulesMetrics bool
}{
{addr: os.Getenv("TEST_REDIS_MODULES_URI"), inclModulesMetrics: true, wantModulesMetrics: true},
{addr: os.Getenv("TEST_REDIS_MODULES_URI"), inclModulesMetrics: false, wantModulesMetrics: false},
{addr: os.Getenv("TEST_REDIS_URI"), inclModulesMetrics: true, wantModulesMetrics: false},
{addr: os.Getenv("TEST_REDIS_URI"), inclModulesMetrics: false, wantModulesMetrics: false},
}

for _, tst := range tsts {
e, _ := NewRedisExporter(tst.addr, Options{Namespace: "test", InclModulesMetrics: tst.inclModulesMetrics})

chM := make(chan prometheus.Metric)
go func() {
e.Collect(chM)
close(chM)
}()

wantedMetrics := map[string]bool{
"module_info": false,
"search_number_of_indexes": false,
"search_used_memory_indexes_bytes": false,
"search_total_indexing_time": false,
"search_global_idle": false,
"search_global_total": false,
"search_bytes_collected": false,
"search_total_cycles": false,
"search_total_ms_run": false,
"search_dialect_1": false,
"search_dialect_2": false,
"search_dialect_3": false,
"search_dialect_4": false,
}

for m := range chM {
for want := range wantedMetrics {
if strings.Contains(m.Desc().String(), want) {
wantedMetrics[want] = true
}
}
}

if tst.wantModulesMetrics {
for want, found := range wantedMetrics {
if !found {
t.Errorf("%s was *not* found in Redis Modules metrics but expected", want)
}
}
} else if !tst.wantModulesMetrics {
for want, found := range wantedMetrics {
if found {
t.Errorf("%s was *found* in Redis Modules metrics but *not* expected", want)
}
}
}
}
}
2 changes: 2 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@
redisMetricsOnly = flag.Bool("redis-only-metrics", getEnvBool("REDIS_EXPORTER_REDIS_ONLY_METRICS", false), "Whether to also export go runtime metrics")
pingOnConnect = flag.Bool("ping-on-connect", getEnvBool("REDIS_EXPORTER_PING_ON_CONNECT", false), "Whether to ping the redis instance after connecting")
inclConfigMetrics = flag.Bool("include-config-metrics", getEnvBool("REDIS_EXPORTER_INCL_CONFIG_METRICS", false), "Whether to include all config settings as metrics")
inclModulesMetrics = flag.Bool("include-modules-metrics", getEnvBool("REDIS_EXPORTER_INCL_MODULES_METRICS", false), "Whether to collect Redis Modules metrics")

Check warning on line 97 in main.go

View check run for this annotation

Codecov / codecov/patch

main.go#L97

Added line #L97 was not covered by tests
disableExportingKeyValues = flag.Bool("disable-exporting-key-values", getEnvBool("REDIS_EXPORTER_DISABLE_EXPORTING_KEY_VALUES", false), "Whether to disable values of keys stored in redis as labels or not when using check-keys/check-single-key")
excludeLatencyHistogramMetrics = flag.Bool("exclude-latency-histogram-metrics", getEnvBool("REDIS_EXPORTER_EXCLUDE_LATENCY_HISTOGRAM_METRICS", false), "Do not try to collect latency histogram metrics")
redactConfigMetrics = flag.Bool("redact-config-metrics", getEnvBool("REDIS_EXPORTER_REDACT_CONFIG_METRICS", true), "Whether to redact config settings that include potentially sensitive information like passwords")
Expand Down Expand Up @@ -182,6 +183,7 @@
SetClientName: *setClientName,
IsTile38: *isTile38,
IsCluster: *isCluster,
InclModulesMetrics: *inclModulesMetrics,

Check warning on line 186 in main.go

View check run for this annotation

Codecov / codecov/patch

main.go#L186

Added line #L186 was not covered by tests
ExportClientList: *exportClientList,
ExportClientsInclPort: *exportClientPort,
SkipTLSVerification: *skipTLSVerification,
Expand Down
Loading