From dd2529f2f61a4c4486edb9bf84c7f51da0d6812b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Fri, 12 Aug 2022 16:14:50 +0300 Subject: [PATCH] cache: add Rueidis client support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for the Rueidis client. It has been instrumental for us in avoiding an incident in production from too much network load on the servers. Since we have lots of alerting/recording rules that use historical data i.e. from Thanos Store, it means that without client-side caching the same data is retrieved over and over again. Rueidis sends a PTTL command to Redis after retrieving data from Redis to know what is the TTL. I contemplated sending a design doc for this but the design doc ended up very slim. It's either some like this or: * Adding client-side caching support to go-redis (it's probably a huge undertaking); * Using the `type` field to differentiate clients. I think it's ready to go expect for two things: * There's no way to disable local cache in Rueidis for now due to limitation in the Rueidis library; * There's no way to disable the default TTL in case PTTL fails - we probably don't want to cache sensitive things like index data for longer than we're supposed to. I'll try to work on these things if this whole idea of having different clients looks good to you. Signed-off-by: Giedrius Statkevičius --- docs/components/store.md | 48 +++++++- go.mod | 41 ++++--- go.sum | 18 ++- pkg/cacheutil/rueidis_client.go | 204 ++++++++++++++++++++++++++++++++ pkg/store/cache/factory.go | 26 +++- test/e2e/e2ethanos/services.go | 11 ++ test/e2e/store_gateway_test.go | 24 ++++ 7 files changed, 347 insertions(+), 25 deletions(-) create mode 100644 pkg/cacheutil/rueidis_client.go diff --git a/docs/components/store.md b/docs/components/store.md index 3071e7b928b..ba91d08c6da 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -252,6 +252,8 @@ Thanos Store Gateway supports an index cache to speed up postings and series loo - `in-memory` (*default*) - `memcached` - `redis` + - `RUEIDIS` client + - `GOREDIS` client (*default*) ### In-memory index cache @@ -308,10 +310,24 @@ While the remaining settings are **optional**: ### Redis index cache -The `redis` index cache allows to use [Redis](https://redis.io) as cache backend. This cache type is configured using `--index-cache.config-file` to reference the configuration file or `--index-cache.config` to put yaml config directly: +The `redis` index cache allows to use [Redis](https://redis.io) as cache backend. This cache type is configured using `--index-cache.config-file` to reference the configuration file or `--index-cache.config` to put YAML config directly. + +It supports two types of clients: + +* `GOREDIS` (the default one) +* `RUEIDIS` + +The main difference is that [Rueidis](https://github.com/rueian/rueidis) supports [client-side caching](https://redis.io/docs/manual/client-side-caching/) which is really important. If you have high cardinality metrics then it means that it is no longer necessary to go to remote-object storage to get information about potentially thousands or millions of metrics. + +Another thing is that [Rueidis](https://github.com/rueian/rueidis) only supports Redis 6.x and later. + +In-memory cache's size for [Rueidis](https://github.com/rueian/rueidis) is controlled by option `cachesize`. + +#### go-redis client ```yaml mdox-exec="go run scripts/cfggen/main.go --name=cacheutil.RedisClientConfig" type: REDIS +client: GOREDIS config: addr: "" username: "" @@ -351,6 +367,36 @@ While the remaining settings are **optional**: - `max_set_multi_concurrency`: specifies the maximum number of concurrent SetMulti() operations. - `set_multi_batch_size`: specifies the maximum size per batch for pipeline set. +#### Rueidis client + +```yaml mdox-exec="go run scripts/cfggen/main.go --name=cacheutil.RueidisClientConfig" +type: REDIS +client: RUEIDIS +config: + addrs: [] + username: "" + password: "" + db: 0 + cache_size: 256MB + dial_timeout: 5s + read_timeout: 3s + write_timeout: 3s +``` + +The **required** settings are: + +- `addrs`: redis server addresses. + +While the remaining settings are **optional**: + +- `username`: the username to connect redis, only redis 6.0 and grater need this field. +- `password`: the password to connect redis. +- `db`: the database to be selected after connecting to the server. +- `dial_timeout`: the redis dial timeout. +- `read_timeout`: the redis read timeout. +- `write_timeout`: the redis write timeout. +- `cache_size`: the size of the in-memory cache size for *each* connection to individual specified Redis servers. + ## Caching Bucket Thanos Store Gateway supports a "caching bucket" with [chunks](../design.md#chunk) and metadata caching to speed up loading of [chunks](../design.md#chunk) from TSDB blocks. To configure caching, one needs to use `--store.caching-bucket.config=` or `--store.caching-bucket.config-file=`. diff --git a/go.mod b/go.mod index 1062dd40973..106aae306fa 100644 --- a/go.mod +++ b/go.mod @@ -89,7 +89,7 @@ require ( go.uber.org/automaxprocs v1.5.1 go.uber.org/goleak v1.1.12 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d - golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e + golang.org/x/net v0.0.0-20220630215102-69896b714898 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20220609170525-579cf78fd858 @@ -102,6 +102,27 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +require ( + github.com/OneOfOne/xxhash v1.2.6 // indirect + github.com/elastic/go-sysinfo v1.8.1 // indirect + github.com/elastic/go-windows v1.0.1 // indirect + github.com/go-kit/kit v0.12.0 // indirect + github.com/go-openapi/spec v0.20.5 // indirect + github.com/gogo/googleapis v1.4.0 // indirect + github.com/golang-jwt/jwt/v4 v4.4.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-msgpack v0.5.5 // indirect + github.com/klauspost/cpuid/v2 v2.0.14 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect + github.com/rs/xid v1.4.0 // indirect + github.com/rueian/rueidis v0.0.69 + github.com/spaolacci/murmur3 v1.1.0 // indirect + go.uber.org/zap v1.21.0 // indirect + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect +) + require ( cloud.google.com/go v0.102.0 // indirect cloud.google.com/go/compute v1.7.0 // indirect @@ -118,7 +139,6 @@ require ( github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.32.3 // indirect - github.com/OneOfOne/xxhash v1.2.6 // indirect github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect @@ -138,6 +158,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.16.1 // indirect github.com/aws/smithy-go v1.11.1 // indirect github.com/baidubce/bce-sdk-go v0.9.111 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/chromedp/sysutil v1.0.0 // indirect github.com/clbanning/mxj v1.8.4 // indirect @@ -147,11 +168,8 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/edsrzf/mmap-go v1.1.0 // indirect - github.com/elastic/go-sysinfo v1.8.1 // indirect - github.com/elastic/go-windows v1.0.1 // indirect github.com/fatih/color v1.13.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/go-kit/kit v0.12.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -161,14 +179,11 @@ require ( github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/loads v0.21.1 // indirect - github.com/go-openapi/spec v0.20.5 // indirect github.com/go-openapi/swag v0.21.1 // indirect github.com/go-openapi/validate v0.21.0 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.1.0 // indirect - github.com/gogo/googleapis v1.4.0 // indirect - github.com/golang-jwt/jwt/v4 v4.4.1 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/go-cmp v0.5.8 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -177,10 +192,8 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect github.com/googleapis/gax-go/v2 v2.4.0 // indirect github.com/googleapis/go-type-adapters v1.0.0 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-hclog v0.16.2 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect - github.com/hashicorp/go-msgpack v0.5.5 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/serf v0.9.6 // indirect @@ -188,17 +201,13 @@ require ( github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect - github.com/klauspost/cpuid/v2 v2.0.14 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20210210170715-a8dfcb80d3a7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -208,13 +217,11 @@ require ( github.com/prometheus/common/sigv4 v0.1.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/rivo/uniseg v0.2.0 // indirect - github.com/rs/xid v1.4.0 // indirect github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/sercand/kuberesolver v2.4.0+incompatible // indirect github.com/shirou/gopsutil/v3 v3.21.2 // indirect github.com/sirupsen/logrus v1.8.1 // indirect - github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stretchr/objx v0.4.0 // indirect github.com/tencentyun/cos-go-sdk-v5 v0.7.34 // indirect github.com/tklauser/go-sysconf v0.3.4 // indirect @@ -228,10 +235,8 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.32.0 // indirect go.opentelemetry.io/otel/metric v0.30.0 // indirect go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.21.0 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26 // indirect - golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect golang.org/x/tools v0.1.11 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect google.golang.org/api v0.86.0 // indirect diff --git a/go.sum b/go.sum index 70a4587c012..aed95f155e6 100644 --- a/go.sum +++ b/go.sum @@ -196,8 +196,9 @@ github.com/baidubce/bce-sdk-go v0.9.111 h1:yGgtPpZYUZW4uoVorQ4xnuEgVeddACydlcJKW github.com/baidubce/bce-sdk-go v0.9.111/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -910,6 +911,7 @@ github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/oklog/ulid/v2 v2.0.2/go.mod h1:mtBL0Qe/0HAx6/a4Z30qxVIAL1eQDweXq5lxOEiwQ68= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= @@ -923,6 +925,8 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -930,8 +934,9 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -961,6 +966,7 @@ github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIw github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -1048,6 +1054,8 @@ github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rueian/rueidis v0.0.69 h1:F4tiDGuyPZq4Ia83hU6pBeZibS3CsZ7+8IJPiDSbdWk= +github.com/rueian/rueidis v0.0.69/go.mod h1:FwnfDILF2GETrvXcYFlhIiru/7NmSIm1f+7C5kutO0I= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -1226,6 +1234,7 @@ go.opentelemetry.io/otel/metric v0.30.0 h1:Hs8eQZ8aQgs0U49diZoaS6Uaxw3+bBE3lcMUK go.opentelemetry.io/otel/metric v0.30.0/go.mod h1:/ShZ7+TS4dHzDFmfi1kSXMhMVubNoP0oIaBp70J6UXU= go.opentelemetry.io/otel/sdk v1.7.0 h1:4OmStpcKVOfvDOgCt7UriAPtKolwIhxpnSNI/yK+1B0= go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU= +go.opentelemetry.io/otel/sdk/metric v0.30.0/go.mod h1:8AKFRi5HyvTR0RRty3paN1aMC9HMT+NzcEhw/BLkLX8= go.opentelemetry.io/otel/trace v1.4.0/go.mod h1:uc3eRsqDfWs9R7b92xbQbU42/eTNz4N+gLP8qJCi4aE= go.opentelemetry.io/otel/trace v1.5.0/go.mod h1:sq55kfhjXYr1zVSyexg0w1mpa03AYXR5eyTkB9NPPdE= go.opentelemetry.io/otel/trace v1.7.0 h1:O37Iogk1lEkMRXewVtZ1BBTVn5JEp8GrJvP92bJqC6o= @@ -1395,8 +1404,9 @@ golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220630215102-69896b714898 h1:K7wO6V1IrczY9QOQ2WkVpw4JQSwCd52UsxVEirZUfiw= +golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1535,6 +1545,7 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1543,6 +1554,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/pkg/cacheutil/rueidis_client.go b/pkg/cacheutil/rueidis_client.go new file mode 100644 index 00000000000..d9202aa4b14 --- /dev/null +++ b/pkg/cacheutil/rueidis_client.go @@ -0,0 +1,204 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package cacheutil + +import ( + "context" + "net" + "time" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/rueian/rueidis" + "github.com/thanos-io/thanos/pkg/model" + "gopkg.in/yaml.v2" +) + +// RueidisClient is a wrap of rueidis.Client. +type RueidisClient struct { + client rueidis.Client + config RueidisClientConfig + + logger log.Logger + durationSet prometheus.Observer + durationSetMulti prometheus.Observer + durationGetMulti prometheus.Observer +} + +// NewRueidisClient makes a new RueidisClient. +func NewRueidisClient(logger log.Logger, name string, conf []byte, reg prometheus.Registerer) (*RueidisClient, error) { + config, err := parseRueidisClientConfig(conf) + if err != nil { + return nil, err + } + + return NewRueidisClientWithConfig(logger, name, config, reg) +} + +// NewRueidisClientWithConfig makes a new RedisClient. +func NewRueidisClientWithConfig(logger log.Logger, name string, config RueidisClientConfig, + reg prometheus.Registerer) (*RueidisClient, error) { + + if err := config.validate(); err != nil { + return nil, err + } + + client, err := rueidis.NewClient(rueidis.ClientOption{ + InitAddress: config.Addrs, + ShuffleInit: true, + Username: config.Username, + Password: config.Password, + SelectDB: config.DB, + CacheSizeEachConn: int(config.CacheSize), + Dialer: net.Dialer{Timeout: config.DialTimeout}, + ConnWriteTimeout: config.WriteTimeout, + }) + if err != nil { + return nil, err + } + + if reg != nil { + reg = prometheus.WrapRegistererWith(prometheus.Labels{"name": name}, reg) + } + + c := &RueidisClient{ + client: client, + config: config, + logger: logger, + } + duration := promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + Name: "thanos_redis_operation_duration_seconds", + Help: "Duration of operations against redis.", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 3, 6, 10}, + }, []string{"operation"}) + c.durationSet = duration.WithLabelValues(opSet) + c.durationSetMulti = duration.WithLabelValues(opSetMulti) + c.durationGetMulti = duration.WithLabelValues(opGetMulti) + return c, nil +} + +// SetAsync implement RemoteCacheClient. +func (c *RueidisClient) SetAsync(ctx context.Context, key string, value []byte, ttl time.Duration) error { + start := time.Now() + if err := c.client.Do(ctx, c.client.B().Set().Key(key).Value(rueidis.BinaryString(value)).ExSeconds(int64(ttl.Seconds())).Build()).Error(); err != nil { + level.Warn(c.logger).Log("msg", "failed to set item into redis", "err", err, "key", key, "value_size", len(value)) + return nil + } + c.durationSet.Observe(time.Since(start).Seconds()) + return nil +} + +// SetMulti set multiple keys and value. +func (c *RueidisClient) SetMulti(ctx context.Context, data map[string][]byte, ttl time.Duration) { + if len(data) == 0 { + return + } + start := time.Now() + sets := make(rueidis.Commands, 0, len(data)) + ittl := int64(ttl.Seconds()) + for k, v := range data { + sets = append(sets, c.client.B().Setex().Key(k).Seconds(ittl).Value(rueidis.BinaryString(v)).Build()) + } + for _, resp := range c.client.DoMulti(ctx, sets...) { + if err := resp.Error(); err != nil { + level.Warn(c.logger).Log("msg", "failed to set multi items from redis", "err", err, "items", len(data)) + return + } + } + c.durationSetMulti.Observe(time.Since(start).Seconds()) +} + +// GetMulti implement RemoteCacheClient. +func (c *RueidisClient) GetMulti(ctx context.Context, keys []string) map[string][]byte { + if len(keys) == 0 { + return nil + } + start := time.Now() + results := make(map[string][]byte, len(keys)) + + if c.config.ReadTimeout > 0 { + timeoutCtx, cancel := context.WithTimeout(ctx, c.config.ReadTimeout) + defer cancel() + ctx = timeoutCtx + } + + // TTL is the default one in case PTTL fails. + resps, err := rueidis.MGetCache(c.client, ctx, 8*time.Hour, keys) + if err != nil { + level.Warn(c.logger).Log("msg", "failed to mget items from redis", "err", err, "items", len(resps)) + } + for key, resp := range resps { + if val, err := resp.ToString(); err == nil { + results[key] = stringToBytes(val) + } + } + c.durationGetMulti.Observe(time.Since(start).Seconds()) + return results +} + +// Stop implement RemoteCacheClient. +func (c *RueidisClient) Stop() { + c.client.Close() +} + +// RueidisClientConfig is the config accepted by RueidisClient. +type RueidisClientConfig struct { + // Addrs specifies the addresses of the redis server. + Addrs []string `yaml:"addr"` + + // Size of local RAM cache. If not zero then + // client-side caching is enabled. + CacheSize model.Bytes `yaml:"cache_size"` + + // Use the specified Username to authenticate the current connection + // with one of the connections defined in the ACL list when connecting + // to a Redis 6.0 instance, or greater, that is using the Redis ACL system. + Username string `yaml:"username"` + // Optional password. Must match the password specified in the + // requirepass server configuration option (if connecting to a Redis 5.0 instance, or lower), + // or the User Password when connecting to a Redis 6.0 instance, or greater, + // that is using the Redis ACL system. + Password string `yaml:"password"` + + // DB Database to be selected after connecting to the server. + DB int `yaml:"db"` + + // DialTimeout specifies the client dial timeout. + DialTimeout time.Duration `yaml:"dial_timeout"` + + // ReadTimeout specifies the client read timeout. + ReadTimeout time.Duration `yaml:"read_timeout"` + + // WriteTimeout specifies the client write timeout. + WriteTimeout time.Duration `yaml:"write_timeout"` +} + +// parseRueidisClientConfig unmarshals a buffer into a RueidisClientConfig with default values. +func parseRueidisClientConfig(conf []byte) (RueidisClientConfig, error) { + config := DefaultRueidisClientConfig + if err := yaml.Unmarshal(conf, &config); err != nil { + return RueidisClientConfig{}, err + } + return config, nil +} + +func (c *RueidisClientConfig) validate() error { + if len(c.Addrs) == 0 { + return errors.New("no redis addrs provided") + } + return nil +} + +var ( + // DefaultRedisClientConfig is default Rueidis config. + DefaultRueidisClientConfig = RueidisClientConfig{ + DialTimeout: time.Second * 5, + ReadTimeout: time.Second * 3, + WriteTimeout: time.Second * 3, + CacheSize: 256 * 1024 * 1024, + } +) diff --git a/pkg/store/cache/factory.go b/pkg/store/cache/factory.go index 9b4103a26d8..09485d700fa 100644 --- a/pkg/store/cache/factory.go +++ b/pkg/store/cache/factory.go @@ -24,9 +24,17 @@ const ( REDIS IndexCacheProvider = "REDIS" ) +type IndexCacheClient string + +const ( + REDISRUEIDISCLIENT IndexCacheClient = "RUEIDIS" + GOREDISCLIENT IndexCacheClient = "GOREDIS" +) + // IndexCacheConfig specifies the index cache config. type IndexCacheConfig struct { Type IndexCacheProvider `yaml:"type"` + Client string `yaml:"client"` Config interface{} `yaml:"config"` } @@ -55,10 +63,22 @@ func NewIndexCache(logger log.Logger, confContentYaml []byte, reg prometheus.Reg } case string(REDIS): var redisCache cacheutil.RemoteCacheClient - redisCache, err = cacheutil.NewRedisClient(logger, "index-cache", backendConfig, reg) - if err == nil { - cache, err = NewRemoteIndexCache(logger, redisCache, reg) + + switch strings.ToUpper(string(cacheConfig.Client)) { + case string(REDISRUEIDISCLIENT): + redisCache, err = cacheutil.NewRueidisClient(logger, "index-cache", backendConfig, reg) + if err == nil { + cache, err = NewRemoteIndexCache(logger, redisCache, reg) + } + default: + fallthrough + case string(GOREDISCLIENT): + redisCache, err = cacheutil.NewRedisClient(logger, "index-cache", backendConfig, reg) + if err == nil { + cache, err = NewRemoteIndexCache(logger, redisCache, reg) + } } + default: return nil, errors.Errorf("index cache with type %s is not supported", cacheConfig.Type) } diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 7006b5750b3..5e54c2fa2cb 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -413,6 +413,17 @@ type ReceiveBuilder struct { image string } +func NewRedis(e e2e.Environment, name string) e2e.InstrumentedRunnable { + return e2e.NewInstrumentedRunnable(e, fmt.Sprintf("redis-%s", name)).WithPorts(map[string]int{"redis": 6379}, "redis").Init( + e2e.StartOptions{ + Image: "docker.io/redis:7.0.4-alpine", + Command: e2e.NewCommand("redis-server", "*:6379"), + User: strconv.Itoa(os.Getuid()), + WaitReadyBackoff: &defaultBackoffConfig, + }, + ) +} + func NewReceiveBuilder(e e2e.Environment, name string) *ReceiveBuilder { f := e2e.NewInstrumentedRunnable(e, fmt.Sprintf("receive-%v", name)). WithPorts(map[string]int{"http": 8080, "grpc": 9091, "remote-write": 8081}, "http"). diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index 20c69396c34..7cbe3e7e8dc 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -27,6 +27,7 @@ import ( "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/cacheutil" "github.com/thanos-io/thanos/pkg/promclient" "github.com/thanos-io/thanos/pkg/testutil" "github.com/thanos-io/thanos/pkg/testutil/e2eutil" @@ -35,6 +36,29 @@ import ( const testQuery = "{a=\"1\"}" +func TestRedisClient_Rueidis(t *testing.T) { + t.Parallel() + + e, err := e2e.NewDockerEnvironment("e2e_test_redis_rueidis_client") + testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, e)) + + r := e2ethanos.NewRedis(e, "redis") + testutil.Ok(t, r.Start()) + + rueidisClient, err := cacheutil.NewRueidisClientWithConfig(log.NewLogfmtLogger(os.Stderr), "redis", cacheutil.RueidisClientConfig{ + Addrs: []string{r.Endpoint("redis")}, + }, nil) + testutil.Ok(t, err) + + err = rueidisClient.SetAsync(context.TODO(), "foo", []byte(`bar`), 1*time.Minute) + testutil.Ok(t, err) + + returnedVals := rueidisClient.GetMulti(context.TODO(), []string{"foo"}) + testutil.Equals(t, 1, len(returnedVals)) + testutil.Equals(t, []byte("bar"), returnedVals["foo"]) +} + // TODO(bwplotka): Extend this test to have multiple stores. func TestStoreGateway(t *testing.T) { t.Parallel()