diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index bb5bdd1ebe..dd9256c95e 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -126,7 +126,7 @@ func registerRule(app *extkingpin.App) { cmd.Flag("eval-interval", "The default evaluation interval to use."). Default("30s").DurationVar(&conf.evalInterval) - conf.rwConfig = extflag.RegisterPathOrContent(cmd, "remote-write.config", "YAML config for the remote-write server where samples should be sent to (see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). This automatically enables stateless mode for ruler and no series will be stored in the ruler's TSDB. If an empty config (or file) is provided, the flag is ignored and ruler is run with its own TSDB.", extflag.WithEnvSubstitution()) + conf.rwConfig = extflag.RegisterPathOrContent(cmd, "remote-write.config", "YAML config for the remote-write configurations, that specify servers where samples should be sent to (see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). This automatically enables stateless mode for ruler and no series will be stored in the ruler's TSDB. If an empty config (or file) is provided, the flag is ignored and ruler is run with its own TSDB.", extflag.WithEnvSubstitution()) reqLogDecision := cmd.Flag("log.request.decision", "Deprecation Warning - This flag would be soon deprecated, and replaced with `request.logging-config`. Request Logging for logging the start and end of requests. By default this flag is disabled. LogFinishCall: Logs the finish call of the requests. LogStartAndFinishCall: Logs the start and finish call of the requests. NoLogCall: Disable request logging.").Default("").Enum("NoLogCall", "LogFinishCall", "LogStartAndFinishCall", "") @@ -350,18 +350,21 @@ func runRule( } if len(rwCfgYAML) > 0 { - var rwCfg config.RemoteWriteConfig + var rwCfg struct { + RemoteWriteConfigs []*config.RemoteWriteConfig `yaml:"remote_write,omitempty"` + } if err := yaml.Unmarshal(rwCfgYAML, &rwCfg); err != nil { - return err + return errors.Wrapf(err, "failed to parse remote write config %v", string(rwCfgYAML)) } - walDir := filepath.Join(conf.dataDir, rwCfg.Name) + + walDir := filepath.Join(conf.dataDir, "wal") // flushDeadline is set to 1m, but it is for metadata watcher only so not used here. remoteStore := remote.NewStorage(logger, reg, func() (int64, error) { return 0, nil }, walDir, 1*time.Minute, nil) if err := remoteStore.ApplyConfig(&config.Config{ GlobalConfig: config.DefaultGlobalConfig, - RemoteWriteConfigs: []*config.RemoteWriteConfig{&rwCfg}, + RemoteWriteConfigs: rwCfg.RemoteWriteConfigs, }); err != nil { return errors.Wrap(err, "applying config to remote storage") } diff --git a/docs/components/rule.md b/docs/components/rule.md index 5a451dfaae..915be28828 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -210,7 +210,7 @@ Stateless ruler enables nearly indefinite horizontal scalability. Ruler doesn't The WAL only storage reuses the upstream [Prometheus agent](https://prometheus.io/blog/2021/11/16/agent/) and it is compatible with the old TSDB data. For more design purpose of this mode, please refer to the [proposal](https://thanos.io/tip/proposals-done/202005-scalable-rule-storage.md/). -Stateless mode can be enabled by providing `--remote-write.config` or `--remote-write.config-file` flag. For example: +Stateless mode can be enabled by providing [Prometheus remote write config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write) in file via `--remote-write.config` or inlined `--remote-write.config-file` flag. For example: ```bash thanos rule \ @@ -227,7 +227,27 @@ thanos rule \ --remote-write.config-file 'rw-config.yaml' ``` -The remote write config file is exactly the same as the [Prometheus remote write config format](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). +Where `rw-config.yaml` could look as follows: + +```yaml +remote_write: +- url: http://e2e_test_rule_remote_write-receive-1:8081/api/v1/receive + name: thanos-receiver + follow_redirects: false +- url: https://e2e_test_rule_remote_write-receive-2:443/api/v1/receive + remote_timeout: 30s + follow_redirects: true + queue_config: + capacity: 120000 + max_shards: 50 + min_shards: 1 + max_samples_per_send: 40000 + batch_send_deadline: 5s + min_backoff: 5s + max_backoff: 5m +``` + +You can pass this in file using `--remote-write.config-file=` or inline it using `--remote-write.config=`. **NOTE:** 1. `metadata_config` is not supported in this mode and will be ignored if provided in the remote write configuration. @@ -378,8 +398,9 @@ Flags: --remote-write.config= Alternative to 'remote-write.config-file' flag (mutually exclusive). Content of YAML config - for the remote-write server where samples - should be sent to (see + for the remote-write configurations, that + specify servers where samples should be sent to + (see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). This automatically enables stateless mode for ruler and no series will be stored in the @@ -387,8 +408,9 @@ Flags: provided, the flag is ignored and ruler is run with its own TSDB. --remote-write.config-file= - Path to YAML config for the remote-write server - where samples should be sent to (see + Path to YAML config for the remote-write + configurations, that specify servers where + samples should be sent to (see https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). This automatically enables stateless mode for ruler and no series will be stored in the diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 69667b48d6..480e681461 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -551,11 +551,11 @@ func NewTSDBRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.Aler return newRuler(e, name, ruleSubDir, amCfg, queryCfg, nil) } -func NewStatelessRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg *config.RemoteWriteConfig) (*e2e.InstrumentedRunnable, error) { +func NewStatelessRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) (*e2e.InstrumentedRunnable, error) { return newRuler(e, name, ruleSubDir, amCfg, queryCfg, remoteWriteCfg) } -func newRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg *config.RemoteWriteConfig) (*e2e.InstrumentedRunnable, error) { +func newRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []httpconfig.Config, remoteWriteCfg []*config.RemoteWriteConfig) (*e2e.InstrumentedRunnable, error) { dir := filepath.Join(e.SharedDir(), "data", "rule", name) container := filepath.Join(ContainerSharedDir, "data", "rule", name) @@ -592,7 +592,9 @@ func newRuler(e e2e.Environment, name, ruleSubDir string, amCfg []alert.Alertman "--resend-delay": "5s", } if remoteWriteCfg != nil { - rwCfgBytes, err := yaml.Marshal(remoteWriteCfg) + rwCfgBytes, err := yaml.Marshal(struct { + RemoteWriteConfigs []*config.RemoteWriteConfig `yaml:"remote_write,omitempty"` + }{remoteWriteCfg}) if err != nil { return nil, errors.Wrapf(err, "generate remote write config: %v", remoteWriteCfg) } diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index 1b9023e17e..dfffc57c78 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -492,7 +492,12 @@ func TestRule_CanRemoteWriteData(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(receiver)) rwURL := mustURLParse(t, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write"))) - q, err := e2ethanos.NewQuerierBuilder(e, "1", receiver.InternalEndpoint("grpc")).Build() + receiver2, err := e2ethanos.NewIngestingReceiver(e, "2") + testutil.Ok(t, err) + testutil.Ok(t, e2e.StartAndWaitReady(receiver2)) + rwURL2 := mustURLParse(t, e2ethanos.RemoteWriteEndpoint(receiver2.InternalEndpoint("remote-write"))) + + q, err := e2ethanos.NewQuerierBuilder(e, "1", receiver.InternalEndpoint("grpc"), receiver2.InternalEndpoint("grpc")).Build() testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(q)) r, err := e2ethanos.NewStatelessRuler(e, "1", rulesSubDir, []alert.AlertmanagerConfig{ @@ -515,9 +520,9 @@ func TestRule_CanRemoteWriteData(t *testing.T) { Scheme: "http", }, }, - }, &config.RemoteWriteConfig{ - URL: &common_cfg.URL{URL: rwURL}, - Name: "thanos-receiver", + }, []*config.RemoteWriteConfig{ + {URL: &common_cfg.URL{URL: rwURL}, Name: "thanos-receiver"}, + {URL: &common_cfg.URL{URL: rwURL2}, Name: "thanos-receiver2"}, }) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(r)) @@ -536,6 +541,12 @@ func TestRule_CanRemoteWriteData(t *testing.T) { "receive": "1", "tenant_id": "default-tenant", }, + { + "__name__": "test_absent_metric", + "job": "thanos-receive", + "receive": "2", + "tenant_id": "default-tenant", + }, }) }) }