From 7ffe57fcd7061e348b50025856e492553fe2c14e Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 14 Oct 2024 20:57:37 -0700 Subject: [PATCH] support valkey:// and valkeys:// scheme (#959) * support valkey:// scheme * fix Dockerfile location * test valkey 8 --- .github/workflows/tests.yml | 22 ++++++++++---- Makefile | 12 +++++--- docker-compose.yml | 28 +++++++++++++++++- exporter/exporter.go | 13 +++++++-- exporter/exporter_test.go | 4 +-- exporter/http_test.go | 6 +++- exporter/redis_test.go | 16 +++++++++++ exporter/streams_test.go | 4 +-- exporter/tls_test.go | 57 +++++++++++++++++++++++++++++++++++++ 9 files changed, 145 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7082e00c..50ecb447 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,6 @@ on: jobs: test-stuff: runs-on: ubuntu-latest - steps: - name: Checkout code uses: actions/checkout@v4 @@ -21,6 +20,12 @@ jobs: - name: Set up Docker Compose run: sudo apt-get install docker-compose + # need to do this before we start the services as they need the TLS creds + - name: Create test certs for TLS + run: | + make test-certs + chmod 777 ./contrib/tls/* + - name: Start services run: docker-compose up -d working-directory: ./ @@ -33,6 +38,12 @@ jobs: - name: Install Dependencies run: go mod tidy + - name: Docker logs + run: | + echo "${{ toJson(job) }}" + docker logs "redis_exporter_redis7-tls_1" + docker logs "redis_exporter_valkey8-tls_1" + - name: Run tests env: LOG_LEVEL: "info" @@ -42,10 +53,11 @@ jobs: make test - - name: Run tests - valkey 7 + - name: Run tests - valkey 8 env: LOG_LEVEL: "info" - TEST_REDIS_URI: "redis://localhost:16384" + TEST_REDIS_URI: "redis://localhost:16382" + TEST_VALKEY8_TLS_URI: "valkeys://localhost:16386" TEST_PWD_REDIS_URI: "redis://:redis-password@localhost:16380" run: | go test -v -race -p 1 ./... @@ -120,7 +132,7 @@ jobs: push: false target: alpine tags: user/app:tst - file: docker/Dockerfile + file: Dockerfile build-args: "GOARCH=amd64" - name: Test Docker Image Build - Scratch @@ -129,5 +141,5 @@ jobs: push: false target: scratch-release tags: user/app:tst - file: docker/Dockerfile + file: Dockerfile build-args: "GOARCH=amd64" diff --git a/Makefile b/Makefile index a3968ad7..369a080c 100644 --- a/Makefile +++ b/Makefile @@ -27,17 +27,21 @@ docker-test: $(DOCKER_COMPOSE) -f docker-compose.yml run --rm tests bash -c 'make test' +.PHONY: test-certs +test-certs: + contrib/tls/gen-test-certs.sh + .PHONY: test test: - contrib/tls/gen-test-certs.sh - + TEST_VALKEY7_URI="valkey://localhost:16384" \ + TEST_VALKEY8_URI="valkey://localhost:16382" \ + TEST_VALKEY8_TLS_URI="valkeys://localhost:16386" \ + TEST_REDIS7_TLS_URI="rediss://localhost:16387" \ TEST_REDIS_URI="redis://localhost:16385" \ TEST_REDIS7_URI="redis://localhost:16385" \ TEST_REDIS5_URI="redis://localhost:16383" \ TEST_REDIS6_URI="redis://localhost:16379" \ - TEST_VALKEY7_URI="redis://localhost:16384" \ - TEST_VALKEY8_URI="redis://localhost:16382" \ TEST_REDIS_2_8_URI="redis://localhost:16381" \ TEST_KEYDB01_URI="redis://localhost:16401" \ TEST_KEYDB02_URI="redis://localhost:16402" \ diff --git a/docker-compose.yml b/docker-compose.yml index 493c9b66..d06bfde0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,18 +1,44 @@ services: - redis74: + redis7: image: redis:7.4 command: "redis-server --enable-debug-command yes --protected-mode no" ports: - "16385:6379" - "6379:6379" + redis7-tls: + image: redis:7.4 + volumes: + - ./contrib/tls:/tls + command: | + redis-server --enable-debug-command yes --protected-mode no + --tls-port 6379 --port 0 + --tls-cert-file /tls/redis.crt + --tls-key-file /tls/redis.key + --tls-ca-cert-file /tls/ca.crt + ports: + - "16387:6379" + valkey8: image: valkey/valkey:8 command: "valkey-server --enable-debug-command yes --protected-mode no" ports: - "16382:6379" + valkey8-tls: + image: valkey/valkey:8 + volumes: + - ./contrib/tls:/tls + command: | + valkey-server --enable-debug-command yes --protected-mode no + --tls-port 6379 --port 0 + --tls-cert-file /tls/redis.crt + --tls-key-file /tls/redis.key + --tls-ca-cert-file /tls/ca.crt + ports: + - "16386:6379" + valkey7: image: valkey/valkey:7.2 command: "valkey-server --enable-debug-command yes --protected-mode no" diff --git a/exporter/exporter.go b/exporter/exporter.go index 1796c80e..f0252d4c 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -86,11 +86,20 @@ type Options struct { } // NewRedisExporter returns a new exporter of Redis metrics. -func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) { +func NewRedisExporter(uri string, opts Options) (*Exporter, error) { log.Debugf("NewRedisExporter options: %#v", opts) + switch { + case strings.HasPrefix(uri, "valkey://"): + uri = strings.Replace(uri, "valkey://", "redis://", 1) + case strings.HasPrefix(uri, "valkeys://"): + uri = strings.Replace(uri, "valkeys://", "rediss://", 1) + } + + log.Debugf("NewRedisExporter = using redis uri: %s", uri) + e := &Exporter{ - redisAddr: redisURI, + redisAddr: uri, options: opts, namespace: opts.Namespace, diff --git a/exporter/exporter_test.go b/exporter/exporter_test.go index 6cdec3d0..2a8ef161 100644 --- a/exporter/exporter_test.go +++ b/exporter/exporter_test.go @@ -160,7 +160,7 @@ func setupDBKeys(t *testing.T, uri string) error { } func setupDBKeysCluster(t *testing.T, uri string) error { - e := Exporter{redisAddr: uri} + e, _ := NewRedisExporter(uri, Options{}) c, err := e.connectToRedisCluster() if err != nil { return err @@ -193,7 +193,7 @@ func deleteKeysFromDB(t *testing.T, addr string) error { } func deleteKeysFromDBCluster(addr string) error { - e := Exporter{redisAddr: addr} + e, _ := NewRedisExporter(addr, Options{}) c, err := e.connectToRedisCluster() if err != nil { return err diff --git a/exporter/http_test.go b/exporter/http_test.go index e5924a4c..236b3f95 100644 --- a/exporter/http_test.go +++ b/exporter/http_test.go @@ -168,7 +168,6 @@ func TestHTTPScrapeMetricsEndpoints(t *testing.T) { // labels and label values `redis_mode`, - `standalone`, `cmd="config`, "maxmemory_policy", @@ -229,6 +228,11 @@ func TestSimultaneousMetricsHttpRequests(t *testing.T) { os.Getenv("TEST_REDIS_URI"), os.Getenv("TEST_REDIS_2_8_URI"), + os.Getenv("TEST_REDIS7_URI"), + + os.Getenv("TEST_VALKEY7_URI"), + os.Getenv("TEST_VALKEY8_URI"), + os.Getenv("TEST_KEYDB01_URI"), os.Getenv("TEST_KEYDB02_URI"), diff --git a/exporter/redis_test.go b/exporter/redis_test.go index 5c32d6e4..76515ff3 100644 --- a/exporter/redis_test.go +++ b/exporter/redis_test.go @@ -31,6 +31,22 @@ func TestHostVariations(t *testing.T) { } } +func TestValkeyScheme(t *testing.T) { + host := os.Getenv("TEST_VALKEY8_URI") + + e, _ := NewRedisExporter(host, Options{SkipTLSVerification: true}) + c, err := e.connectToRedis() + if err != nil { + t.Fatalf("connectToRedis() err: %s", err) + } + + if _, err := c.Do("PING", ""); err != nil { + t.Errorf("PING err: %s", err) + } + + c.Close() +} + func TestPasswordProtectedInstance(t *testing.T) { userAddr := os.Getenv("TEST_USER_PWD_REDIS_URI") if userAddr == "" { diff --git a/exporter/streams_test.go b/exporter/streams_test.go index 0454b6d6..78dc230b 100644 --- a/exporter/streams_test.go +++ b/exporter/streams_test.go @@ -106,7 +106,7 @@ func TestStreamsGetStreamInfoUsingValKey7(t *testing.T) { t.Skipf("TEST_VALKEY7_URI not set - skipping") } - addr := os.Getenv("TEST_VALKEY7_URI") + addr := strings.Replace(os.Getenv("TEST_VALKEY7_URI"), "valkey://", "redis://", 1) c, err := redis.DialURL(addr) if err != nil { t.Fatalf("Couldn't connect to %#v: %#v", addr, err) @@ -285,7 +285,7 @@ func TestStreamsScanStreamGroupsUsingValKey7(t *testing.T) { if os.Getenv("TEST_VALKEY7_URI") == "" { t.Skipf("TEST_VALKEY7_URI not set - skipping") } - addr := os.Getenv("TEST_VALKEY7_URI") + addr := strings.Replace(os.Getenv("TEST_VALKEY7_URI"), "valkey://", "redis://", 1) db := dbNumStr c, err := redis.DialURL(addr) diff --git a/exporter/tls_test.go b/exporter/tls_test.go index 0789a217..a490f9fc 100644 --- a/exporter/tls_test.go +++ b/exporter/tls_test.go @@ -1,7 +1,11 @@ package exporter import ( + "os" + "strings" "testing" + + "github.com/prometheus/client_golang/prometheus" ) func TestCreateClientTLSConfig(t *testing.T) { @@ -40,6 +44,59 @@ func TestCreateClientTLSConfig(t *testing.T) { } } +func TestValkeyTLSScheme(t *testing.T) { + for _, host := range []string{ + os.Getenv("TEST_REDIS7_TLS_URI"), + os.Getenv("TEST_VALKEY8_TLS_URI"), + } { + t.Run(host, func(t *testing.T) { + + e, _ := NewRedisExporter(host, + Options{ + SkipTLSVerification: true, + ClientCertFile: "../contrib/tls/redis.crt", + ClientKeyFile: "../contrib/tls/redis.key", + }, + ) + c, err := e.connectToRedis() + if err != nil { + t.Fatalf("connectToRedis() err: %s", err) + } + + if _, err := c.Do("PING", ""); err != nil { + t.Errorf("PING err: %s", err) + } + + c.Close() + + chM := make(chan prometheus.Metric) + go func() { + e.Collect(chM) + close(chM) + }() + + tsts := []struct { + in string + found bool + }{ + {in: "db_keys"}, + {in: "commands_total"}, + {in: "total_connections_received"}, + {in: "used_memory"}, + } + for m := range chM { + desc := m.Desc().String() + for i := range tsts { + if strings.Contains(desc, tsts[i].in) { + tsts[i].found = true + } + } + } + + }) + } +} + func TestCreateServerTLSConfig(t *testing.T) { e := getTestExporter()