From 9f73e372b9818f25b3f9f0920de9563990281355 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Tue, 29 Oct 2024 11:59:37 +0530 Subject: [PATCH 1/5] remove configdb --- pkg/configs/client/client.go | 184 ----------- pkg/configs/client/configs_test.go | 54 ---- pkg/configs/userconfig/config.go | 344 --------------------- pkg/configs/userconfig/config_test.go | 252 --------------- pkg/ruler/base/storage.go | 12 - pkg/ruler/rulestore/config.go | 6 +- pkg/ruler/rulestore/configdb/store.go | 135 -------- pkg/ruler/rulestore/configdb/store_test.go | 184 ----------- 8 files changed, 2 insertions(+), 1169 deletions(-) delete mode 100644 pkg/configs/client/client.go delete mode 100644 pkg/configs/client/configs_test.go delete mode 100644 pkg/configs/userconfig/config.go delete mode 100644 pkg/configs/userconfig/config_test.go delete mode 100644 pkg/ruler/rulestore/configdb/store.go delete mode 100644 pkg/ruler/rulestore/configdb/store_test.go diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go deleted file mode 100644 index 44af1bda4f504..0000000000000 --- a/pkg/configs/client/client.go +++ /dev/null @@ -1,184 +0,0 @@ -package client - -import ( - "context" - "crypto/tls" - "encoding/json" - "errors" - "flag" - "fmt" - "net/http" - "net/url" - "time" - - "github.com/go-kit/log/level" - dstls "github.com/grafana/dskit/crypto/tls" - "github.com/grafana/dskit/flagext" - "github.com/grafana/dskit/instrument" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/common/version" - - "github.com/grafana/loki/v3/pkg/configs/userconfig" - util_log "github.com/grafana/loki/v3/pkg/util/log" -) - -var ( - errBadURL = errors.New("configs_api_url is not set or valid") -) - -// Config says where we can find the ruler userconfig. -type Config struct { - ConfigsAPIURL flagext.URLValue `yaml:"configs_api_url"` - ClientTimeout time.Duration `yaml:"client_timeout"` // HTTP timeout duration for requests made to the Weave Cloud configs service. - TLS dstls.ClientConfig `yaml:",inline"` -} - -// RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet -func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { - f.Var(&cfg.ConfigsAPIURL, prefix+"configs.url", "URL of configs API server.") - f.DurationVar(&cfg.ClientTimeout, prefix+"configs.client-timeout", 5*time.Second, "Timeout for requests to Weave Cloud configs service.") - cfg.TLS.RegisterFlagsWithPrefix(prefix+"configs", f) -} - -var configsRequestDuration = instrument.NewHistogramCollector(promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "cortex", - Name: "configs_request_duration_seconds", - Help: "Time spent requesting userconfig.", - Buckets: prometheus.DefBuckets, -}, []string{"operation", "status_code"})) - -// Client is what the ruler and altermanger needs from a config store to process rules. -type Client interface { - // GetRules returns all Cortex configurations from a configs API server - // that have been updated after the given userconfig.ID was last updated. - GetRules(ctx context.Context, since userconfig.ID) (map[string]userconfig.VersionedRulesConfig, error) - - // GetAlerts fetches all the alerts that have changes since since. - GetAlerts(ctx context.Context, since userconfig.ID) (*ConfigsResponse, error) -} - -// New creates a new ConfigClient. -func New(cfg Config) (*ConfigDBClient, error) { - - if cfg.ConfigsAPIURL.URL == nil { - return nil, errBadURL - } - - client := &ConfigDBClient{ - URL: cfg.ConfigsAPIURL.URL, - Timeout: cfg.ClientTimeout, - } - - tlsConfig, err := cfg.TLS.GetTLSConfig() - if err != nil { - return nil, err - } - - if tlsConfig != nil { - client.TLSConfig = tlsConfig - } - return client, nil -} - -// ConfigDBClient allows retrieving recording and alerting rules from the configs server. -type ConfigDBClient struct { - URL *url.URL - Timeout time.Duration - TLSConfig *tls.Config -} - -// GetRules implements Client -func (c ConfigDBClient) GetRules(ctx context.Context, since userconfig.ID) (map[string]userconfig.VersionedRulesConfig, error) { - suffix := "" - if since != 0 { - suffix = fmt.Sprintf("?since=%d", since) - } - endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules%s", c.URL.String(), suffix) - var response *ConfigsResponse - err := instrument.CollectedRequest(ctx, "GetRules", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - response, err = doRequest(endpoint, c.Timeout, c.TLSConfig, since) - return err - }) - if err != nil { - return nil, err - } - configs := map[string]userconfig.VersionedRulesConfig{} - for id, view := range response.Configs { - cfg := view.GetVersionedRulesConfig() - if cfg != nil { - configs[id] = *cfg - } - } - return configs, nil -} - -// GetAlerts implements Client. -func (c ConfigDBClient) GetAlerts(ctx context.Context, since userconfig.ID) (*ConfigsResponse, error) { - suffix := "" - if since != 0 { - suffix = fmt.Sprintf("?since=%d", since) - } - endpoint := fmt.Sprintf("%s/private/api/prom/configs/alertmanager%s", c.URL.String(), suffix) - var response *ConfigsResponse - err := instrument.CollectedRequest(ctx, "GetAlerts", configsRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var err error - response, err = doRequest(endpoint, c.Timeout, c.TLSConfig, since) - return err - }) - return response, err -} - -func doRequest(endpoint string, timeout time.Duration, tlsConfig *tls.Config, since userconfig.ID) (*ConfigsResponse, error) { - req, err := http.NewRequest("GET", endpoint, nil) - if err != nil { - return nil, err - } - - client := &http.Client{Timeout: timeout} - if tlsConfig != nil { - client.Transport = &http.Transport{TLSClientConfig: tlsConfig} - } - - req.Header.Set("User-Agent", fmt.Sprintf("Cortex/%s", version.Version)) - - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Invalid response from configs server: %v", resp.StatusCode) - } - - var config ConfigsResponse - if err := json.NewDecoder(resp.Body).Decode(&config); err != nil { - level.Error(util_log.Logger).Log("msg", "configs: couldn't decode JSON body", "err", err) - return nil, err - } - - config.since = since - return &config, nil -} - -// ConfigsResponse is a response from server for Getuserconfig. -type ConfigsResponse struct { - // The version since which these configs were changed - since userconfig.ID - - // Configs maps user ID to their latest userconfig.View. - Configs map[string]userconfig.View `json:"configs"` -} - -// GetLatestConfigID returns the last config ID from a set of userconfig. -func (c ConfigsResponse) GetLatestConfigID() userconfig.ID { - latest := c.since - for _, config := range c.Configs { - if config.ID > latest { - latest = config.ID - } - } - return latest -} diff --git a/pkg/configs/client/configs_test.go b/pkg/configs/client/configs_test.go deleted file mode 100644 index 64f4b98d202e0..0000000000000 --- a/pkg/configs/client/configs_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package client - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/grafana/loki/v3/pkg/configs/userconfig" -) - -var response = `{ - "configs": { - "2": { - "id": 1, - "config": { - "rules_files": { - "recording.rules": "groups:\n- name: demo-service-alerts\n interval: 15s\n rules:\n - alert: SomethingIsUp\n expr: up == 1\n" - }, - "rule_format_version": "2" - } - } - } -} -` - -func TestDoRequest(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, err := w.Write([]byte(response)) - require.NoError(t, err) - })) - defer server.Close() - - resp, err := doRequest(server.URL, 1*time.Second, nil, 0) - assert.Nil(t, err) - - expected := ConfigsResponse{Configs: map[string]userconfig.View{ - "2": { - ID: 1, - Config: userconfig.Config{ - RulesConfig: userconfig.RulesConfig{ - Files: map[string]string{ - "recording.rules": "groups:\n- name: demo-service-alerts\n interval: 15s\n rules:\n - alert: SomethingIsUp\n expr: up == 1\n", - }, - FormatVersion: userconfig.RuleFormatV2, - }, - }, - }, - }} - assert.Equal(t, &expected, resp) -} diff --git a/pkg/configs/userconfig/config.go b/pkg/configs/userconfig/config.go deleted file mode 100644 index e7d22e033a8ec..0000000000000 --- a/pkg/configs/userconfig/config.go +++ /dev/null @@ -1,344 +0,0 @@ -package userconfig - -import ( - "encoding/json" - "fmt" - "time" - - "github.com/go-kit/log" - "github.com/pkg/errors" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/model/rulefmt" - "github.com/prometheus/prometheus/promql/parser" - "github.com/prometheus/prometheus/rules" - "gopkg.in/yaml.v3" - - util_log "github.com/grafana/loki/v3/pkg/util/log" -) - -// An ID is the ID of a single users's Cortex configuration. When a -// configuration changes, it gets a new ID. -type ID int - -// RuleFormatVersion indicates which Prometheus rule format (v1 vs. v2) to use in parsing. -type RuleFormatVersion int - -const ( - // RuleFormatV1 is the Prometheus 1.x rule format. - RuleFormatV1 RuleFormatVersion = iota - // RuleFormatV2 is the Prometheus 2.x rule format. - RuleFormatV2 RuleFormatVersion = iota -) - -// IsValid returns whether the rules format version is a valid (known) version. -func (v RuleFormatVersion) IsValid() bool { - switch v { - case RuleFormatV1, RuleFormatV2: - return true - default: - return false - } -} - -// MarshalJSON implements json.Marshaler. -func (v RuleFormatVersion) MarshalJSON() ([]byte, error) { - switch v { - case RuleFormatV1: - return json.Marshal("1") - case RuleFormatV2: - return json.Marshal("2") - default: - return nil, fmt.Errorf("unknown rule format version %d", v) - } -} - -// MarshalYAML implements yaml.Marshaler. -func (v RuleFormatVersion) MarshalYAML() (interface{}, error) { - switch v { - case RuleFormatV1: - return yaml.Marshal("1") - case RuleFormatV2: - return yaml.Marshal("2") - default: - return nil, fmt.Errorf("unknown rule format version %d", v) - } -} - -// UnmarshalJSON implements json.Unmarshaler. -func (v *RuleFormatVersion) UnmarshalJSON(data []byte) error { - var s string - if err := json.Unmarshal(data, &s); err != nil { - return err - } - switch s { - case "1": - *v = RuleFormatV1 - case "2": - *v = RuleFormatV2 - default: - return fmt.Errorf("unknown rule format version %q", string(data)) - } - return nil -} - -// UnmarshalYAML implements yaml.Unmarshaler. -func (v *RuleFormatVersion) UnmarshalYAML(unmarshal func(interface{}) error) error { - var s string - if err := unmarshal(&s); err != nil { - return err - } - switch s { - case "1": - *v = RuleFormatV1 - case "2": - *v = RuleFormatV2 - default: - return fmt.Errorf("unknown rule format version %q", s) - } - return nil -} - -// A Config is a Cortex configuration for a single user. -type Config struct { - // RulesFiles maps from a rules filename to file contents. - RulesConfig RulesConfig - TemplateFiles map[string]string - AlertmanagerConfig string -} - -// configCompat is a compatibility struct to support old JSON config blobs -// saved in the config DB that didn't have a rule format version yet and -// just had a top-level field for the rule files. -type configCompat struct { - RulesFiles map[string]string `json:"rules_files" yaml:"rules_files"` - RuleFormatVersion RuleFormatVersion `json:"rule_format_version" yaml:"rule_format_version"` - TemplateFiles map[string]string `json:"template_files" yaml:"template_files"` - AlertmanagerConfig string `json:"alertmanager_config" yaml:"alertmanager_config"` -} - -// MarshalJSON implements json.Marshaler. -func (c Config) MarshalJSON() ([]byte, error) { - compat := &configCompat{ - RulesFiles: c.RulesConfig.Files, - RuleFormatVersion: c.RulesConfig.FormatVersion, - TemplateFiles: c.TemplateFiles, - AlertmanagerConfig: c.AlertmanagerConfig, - } - - return json.Marshal(compat) -} - -// MarshalYAML implements yaml.Marshaler. -func (c Config) MarshalYAML() (interface{}, error) { - compat := &configCompat{ - RulesFiles: c.RulesConfig.Files, - RuleFormatVersion: c.RulesConfig.FormatVersion, - TemplateFiles: c.TemplateFiles, - AlertmanagerConfig: c.AlertmanagerConfig, - } - - return yaml.Marshal(compat) -} - -// UnmarshalJSON implements json.Unmarshaler. -func (c *Config) UnmarshalJSON(data []byte) error { - compat := configCompat{} - if err := json.Unmarshal(data, &compat); err != nil { - return err - } - *c = Config{ - RulesConfig: RulesConfig{ - Files: compat.RulesFiles, - FormatVersion: compat.RuleFormatVersion, - }, - TemplateFiles: compat.TemplateFiles, - AlertmanagerConfig: compat.AlertmanagerConfig, - } - return nil -} - -// UnmarshalYAML implements yaml.Unmarshaler. -func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { - compat := configCompat{} - if err := unmarshal(&compat); err != nil { - return errors.WithStack(err) - } - *c = Config{ - RulesConfig: RulesConfig{ - Files: compat.RulesFiles, - FormatVersion: compat.RuleFormatVersion, - }, - TemplateFiles: compat.TemplateFiles, - AlertmanagerConfig: compat.AlertmanagerConfig, - } - return nil -} - -// View is what's returned from the Weave Cloud configs service -// when we ask for all Cortex configurations. -// -// The configs service is essentially a JSON blob store that gives each -// _version_ of a configuration a unique ID and guarantees that later versions -// have greater IDs. -type View struct { - ID ID `json:"id"` - Config Config `json:"config"` - DeletedAt time.Time `json:"deleted_at"` -} - -// IsDeleted tells you if the config is deleted. -func (v View) IsDeleted() bool { - return !v.DeletedAt.IsZero() -} - -// GetVersionedRulesConfig specializes the view to just the rules config. -func (v View) GetVersionedRulesConfig() *VersionedRulesConfig { - if v.Config.RulesConfig.Files == nil { - return nil - } - return &VersionedRulesConfig{ - ID: v.ID, - Config: v.Config.RulesConfig, - DeletedAt: v.DeletedAt, - } -} - -// RulesConfig is the rules configuration for a particular organization. -type RulesConfig struct { - FormatVersion RuleFormatVersion `json:"format_version"` - Files map[string]string `json:"files"` -} - -// Equal compares two RulesConfigs for equality. -// -// instance Eq RulesConfig -func (c RulesConfig) Equal(o RulesConfig) bool { - if c.FormatVersion != o.FormatVersion { - return false - } - if len(o.Files) != len(c.Files) { - return false - } - for k, v1 := range c.Files { - v2, ok := o.Files[k] - if !ok || v1 != v2 { - return false - } - } - return true -} - -// Parse parses and validates the content of the rule files in a RulesConfig -// according to the passed rule format version. -func (c RulesConfig) Parse() (map[string][]rules.Rule, error) { - switch c.FormatVersion { - case RuleFormatV1: - return nil, fmt.Errorf("version %v isn't supported", c.FormatVersion) - case RuleFormatV2: - return c.parseV2() - default: - return nil, fmt.Errorf("unknown rule format version %v", c.FormatVersion) - } -} - -// ParseFormatted returns the rulefmt map of a users rules configs. It allows -// for rules to be mapped to disk and read by the prometheus rules manager. -func (c RulesConfig) ParseFormatted() (map[string]rulefmt.RuleGroups, error) { - switch c.FormatVersion { - case RuleFormatV1: - return nil, fmt.Errorf("version %v isn't supported", c.FormatVersion) - case RuleFormatV2: - return c.parseV2Formatted() - default: - return nil, fmt.Errorf("unknown rule format version %v", c.FormatVersion) - } -} - -// parseV2 parses and validates the content of the rule files in a RulesConfig -// according to the Prometheus 2.x rule format. -func (c RulesConfig) parseV2Formatted() (map[string]rulefmt.RuleGroups, error) { - ruleMap := map[string]rulefmt.RuleGroups{} - - for fn, content := range c.Files { - rgs, errs := rulefmt.Parse([]byte(content)) - for _, err := range errs { // return just the first error, if any - return nil, err - } - ruleMap[fn] = *rgs - - } - return ruleMap, nil -} - -// parseV2 parses and validates the content of the rule files in a RulesConfig -// according to the Prometheus 2.x rule format. -// -// NOTE: On one hand, we cannot return fully-fledged lists of rules.Group -// here yet, as creating a rules.Group requires already -// passing in rules.ManagerOptions options (which in turn require a -// notifier, appender, etc.), which we do not want to create simply -// for parsing. On the other hand, we should not return barebones -// rulefmt.RuleGroup sets here either, as only a fully-converted rules.Rule -// is able to track alert states over multiple rule evaluations. The caller -// would otherwise have to ensure to convert the rulefmt.RuleGroup only exactly -// once, not for every evaluation (or risk losing alert pending states). So -// it's probably better to just return a set of rules.Rule here. -func (c RulesConfig) parseV2() (map[string][]rules.Rule, error) { - groups := map[string][]rules.Rule{} - - for fn, content := range c.Files { - rgs, errs := rulefmt.Parse([]byte(content)) - if len(errs) > 0 { - return nil, fmt.Errorf("error parsing %s: %v", fn, errs[0]) - } - - for _, rg := range rgs.Groups { - rls := make([]rules.Rule, 0, len(rg.Rules)) - for _, rl := range rg.Rules { - expr, err := parser.ParseExpr(rl.Expr.Value) - if err != nil { - return nil, err - } - - if rl.Alert.Value != "" { - rls = append(rls, rules.NewAlertingRule( - rl.Alert.Value, - expr, - time.Duration(rl.For), - time.Duration(rl.KeepFiringFor), - labels.FromMap(rl.Labels), - labels.FromMap(rl.Annotations), - nil, - "", - true, - log.With(util_log.Logger, "alert", rl.Alert.Value), - )) - continue - } - rls = append(rls, rules.NewRecordingRule( - rl.Record.Value, - expr, - labels.FromMap(rl.Labels), - )) - } - - // Group names have to be unique in Prometheus, but only within one rules file. - groups[rg.Name+";"+fn] = rls - } - } - - return groups, nil -} - -// VersionedRulesConfig is a RulesConfig together with a version. -// `data Versioned a = Versioned { id :: ID , config :: a }` -type VersionedRulesConfig struct { - ID ID `json:"id"` - Config RulesConfig `json:"config"` - DeletedAt time.Time `json:"deleted_at"` -} - -// IsDeleted tells you if the config is deleted. -func (vr VersionedRulesConfig) IsDeleted() bool { - return !vr.DeletedAt.IsZero() -} diff --git a/pkg/configs/userconfig/config_test.go b/pkg/configs/userconfig/config_test.go deleted file mode 100644 index ac81d47e4ee98..0000000000000 --- a/pkg/configs/userconfig/config_test.go +++ /dev/null @@ -1,252 +0,0 @@ -package userconfig - -import ( - "encoding/json" - "fmt" - "strconv" - "strings" - "testing" - "time" - - "github.com/go-kit/log" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/model/rulefmt" - "github.com/prometheus/prometheus/promql/parser" - "github.com/prometheus/prometheus/rules" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" - - util_log "github.com/grafana/loki/v3/pkg/util/log" -) - -var legacyRulesFile = `ALERT TestAlert -IF up == 0 -FOR 5m -LABELS { severity = "critical" } -ANNOTATIONS { - message = "I am a message" -}` - -var ruleFile = `groups: -- name: example - rules: - - alert: TestAlert - expr: up == 0 - for: 5m - labels: - severity: critical - annotations: - message: I am a message` - -func TestUnmarshalJSONLegacyConfigWithMissingRuleFormatVersionSucceeds(t *testing.T) { - actual := Config{} - buf := []byte(`{"rules_files": {"a": "b"}}`) - assert.Nil(t, json.Unmarshal(buf, &actual)) - - expected := Config{ - RulesConfig: RulesConfig{ - Files: map[string]string{ - "a": "b", - }, - FormatVersion: RuleFormatV1, - }, - } - - assert.Equal(t, expected, actual) -} - -func TestUnmarshalYAMLLegacyConfigWithMissingRuleFormatVersionSucceeds(t *testing.T) { - actual := Config{} - buf := []byte(strings.TrimSpace(` -rule_format_version: '1' -rules_files: - a: b -`)) - assert.Nil(t, yaml.Unmarshal(buf, &actual)) - - expected := Config{ - RulesConfig: RulesConfig{ - Files: map[string]string{ - "a": "b", - }, - FormatVersion: RuleFormatV1, - }, - } - - assert.Equal(t, expected, actual) -} - -func TestParseLegacyAlerts(t *testing.T) { - parsed, err := parser.ParseExpr("up == 0") - require.NoError(t, err) - rule := rules.NewAlertingRule( - "TestAlert", - parsed, - 5*time.Minute, - 0, - labels.Labels{ - labels.Label{Name: "severity", Value: "critical"}, - }, - labels.Labels{ - labels.Label{Name: "message", Value: "I am a message"}, - }, - nil, - "", - true, - log.With(util_log.Logger, "alert", "TestAlert"), - ) - - for i, tc := range []struct { - cfg RulesConfig - expected map[string][]rules.Rule - wantErr error - }{ - { - cfg: RulesConfig{ - FormatVersion: RuleFormatV1, - Files: map[string]string{ - "legacy.rules": ` - ALERT TestAlert - IF up == 0 - FOR 5m - LABELS { severity = "critical" } - ANNOTATIONS { - message = "I am a message" - } - `, - }, - }, - expected: map[string][]rules.Rule{ - "legacy.rules": {rule}, - }, - wantErr: fmt.Errorf("version 0 isn't supported"), - }, - { - cfg: RulesConfig{ - FormatVersion: RuleFormatV2, - Files: map[string]string{ - "alerts.yaml": ` -groups: -- name: example - rules: - - alert: TestAlert - expr: up == 0 - for: 5m - labels: - severity: critical - annotations: - message: I am a message -`, - }, - }, - expected: map[string][]rules.Rule{ - "example;alerts.yaml": {rule}, - }, - wantErr: nil, - }, - } { - t.Run(strconv.Itoa(i), func(t *testing.T) { - rules, err := tc.cfg.Parse() - if tc.wantErr != nil { - require.EqualError(t, err, tc.wantErr.Error()) - } else { - require.NoError(t, err) - require.Equal(t, tc.expected, rules) - } - }) - } -} - -func TestParseFormatted(t *testing.T) { - dur, err := model.ParseDuration("5m") - require.NoError(t, err) - - rulesV1 := []rulefmt.RuleNode{ - { - Alert: yaml.Node{Value: "TestAlert"}, - Expr: yaml.Node{Value: "up == 0"}, - For: dur, - Labels: map[string]string{ - "severity": "critical", - }, - Annotations: map[string]string{ - "message": "I am a message", - }, - }, - } - - alertNode := yaml.Node{Line: 4, Column: 12} - alertNode.SetString("TestAlert") - exprNode := yaml.Node{Line: 5, Column: 11} - exprNode.SetString("up == 0") - rulesV2 := []rulefmt.RuleNode{ - { - Alert: alertNode, - Expr: exprNode, - For: dur, - Labels: map[string]string{ - "severity": "critical", - }, - Annotations: map[string]string{ - "message": "I am a message", - }, - }, - } - - for i, tc := range []struct { - cfg RulesConfig - expected map[string]rulefmt.RuleGroups - wantErr error - }{ - { - cfg: RulesConfig{ - FormatVersion: RuleFormatV1, - Files: map[string]string{ - "legacy.rules": legacyRulesFile, - }, - }, - expected: map[string]rulefmt.RuleGroups{ - "legacy.rules": { - Groups: []rulefmt.RuleGroup{ - { - Name: "rg:legacy.rules", - Rules: rulesV1, - }, - }, - }, - }, - wantErr: fmt.Errorf("version 0 isn't supported"), - }, - { - cfg: RulesConfig{ - FormatVersion: RuleFormatV2, - Files: map[string]string{ - "alerts.yaml": ruleFile, - }, - }, - expected: map[string]rulefmt.RuleGroups{ - "alerts.yaml": { - Groups: []rulefmt.RuleGroup{ - { - Name: "example", - Rules: rulesV2, - }, - }, - }, - }, - wantErr: nil, - }, - } { - t.Run(strconv.Itoa(i), func(t *testing.T) { - rules, err := tc.cfg.ParseFormatted() - if tc.wantErr != nil { - require.EqualError(t, err, tc.wantErr.Error()) - } else { - require.NoError(t, err) - require.Equal(t, tc.expected, rules) - } - }) - } -} diff --git a/pkg/ruler/base/storage.go b/pkg/ruler/base/storage.go index 7bbe1d133e571..282247ffa48a4 100644 --- a/pkg/ruler/base/storage.go +++ b/pkg/ruler/base/storage.go @@ -10,10 +10,8 @@ import ( "github.com/prometheus/client_golang/prometheus" promRules "github.com/prometheus/prometheus/rules" - configClient "github.com/grafana/loki/v3/pkg/configs/client" "github.com/grafana/loki/v3/pkg/ruler/rulestore" "github.com/grafana/loki/v3/pkg/ruler/rulestore/bucketclient" - "github.com/grafana/loki/v3/pkg/ruler/rulestore/configdb" "github.com/grafana/loki/v3/pkg/ruler/rulestore/local" "github.com/grafana/loki/v3/pkg/ruler/rulestore/objectclient" "github.com/grafana/loki/v3/pkg/storage" @@ -30,7 +28,6 @@ import ( ) // RuleStoreConfig configures a rule store. -// TODO remove this legacy config in Cortex 1.11. type RuleStoreConfig struct { Type string `yaml:"type"` @@ -124,15 +121,6 @@ func NewLegacyRuleStore(cfg RuleStoreConfig, hedgeCfg hedging.Config, clientMetr // NewRuleStore returns a rule store backend client based on the provided cfg. func NewRuleStore(ctx context.Context, cfg rulestore.Config, cfgProvider bucket.TenantConfigProvider, loader promRules.GroupLoader, logger log.Logger, _ prometheus.Registerer) (rulestore.RuleStore, error) { - if cfg.Backend == configdb.Name { - c, err := configClient.New(cfg.ConfigDB) - if err != nil { - return nil, err - } - - return configdb.NewConfigRuleStore(c), nil - } - if cfg.Backend == local.Name { return local.NewLocalRulesClient(cfg.Local, loader) } diff --git a/pkg/ruler/rulestore/config.go b/pkg/ruler/rulestore/config.go index 812c7d1e5c07a..a9a3864be31d8 100644 --- a/pkg/ruler/rulestore/config.go +++ b/pkg/ruler/rulestore/config.go @@ -6,10 +6,9 @@ import ( "github.com/grafana/dskit/flagext" - "github.com/grafana/loki/v3/pkg/configs/client" - "github.com/grafana/loki/v3/pkg/ruler/rulestore/configdb" "github.com/grafana/loki/v3/pkg/ruler/rulestore/local" "github.com/grafana/loki/v3/pkg/storage/bucket" + "github.com/grafana/loki/v3/pkg/tool/client" ) // Config configures a rule store. @@ -24,8 +23,7 @@ type Config struct { func (cfg *Config) RegisterFlags(f *flag.FlagSet) { prefix := "ruler-storage." - cfg.ExtraBackends = []string{configdb.Name, local.Name} - cfg.ConfigDB.RegisterFlagsWithPrefix(prefix, f) + cfg.ExtraBackends = []string{local.Name} cfg.Local.RegisterFlagsWithPrefix(prefix, f) f.StringVar(&cfg.Backend, prefix+"backend", "filesystem", "Backend storage to use. Supported backends are: s3, gcs, azure, swift, filesystem.") cfg.RegisterFlagsWithPrefix(prefix, f) diff --git a/pkg/ruler/rulestore/configdb/store.go b/pkg/ruler/rulestore/configdb/store.go deleted file mode 100644 index e4a0526386fe4..0000000000000 --- a/pkg/ruler/rulestore/configdb/store.go +++ /dev/null @@ -1,135 +0,0 @@ -package configdb - -import ( - "context" - "errors" - - "github.com/grafana/loki/v3/pkg/configs/client" - "github.com/grafana/loki/v3/pkg/configs/userconfig" - "github.com/grafana/loki/v3/pkg/ruler/rulespb" -) - -const ( - Name = "configdb" -) - -// ConfigRuleStore is a concrete implementation of RuleStore that sources rules from the config service -type ConfigRuleStore struct { - configClient client.Client - since userconfig.ID - ruleGroupList map[string]rulespb.RuleGroupList -} - -func (c *ConfigRuleStore) SupportsModifications() bool { - return false -} - -// NewConfigRuleStore constructs a ConfigRuleStore -func NewConfigRuleStore(c client.Client) *ConfigRuleStore { - return &ConfigRuleStore{ - configClient: c, - since: 0, - ruleGroupList: make(map[string]rulespb.RuleGroupList), - } -} - -func (c *ConfigRuleStore) ListAllUsers(ctx context.Context) ([]string, error) { - m, err := c.ListAllRuleGroups(ctx) - - result := make([]string, 0, len(m)) - for u := range m { - result = append(result, u) - } - - return result, err -} - -// ListAllRuleGroups implements RuleStore -func (c *ConfigRuleStore) ListAllRuleGroups(ctx context.Context) (map[string]rulespb.RuleGroupList, error) { - configs, err := c.configClient.GetRules(ctx, c.since) - - if err != nil { - return nil, err - } - - for user, cfg := range configs { - userRules := rulespb.RuleGroupList{} - if cfg.IsDeleted() { - delete(c.ruleGroupList, user) - continue - } - rMap, err := cfg.Config.ParseFormatted() - if err != nil { - return nil, err - } - for file, rgs := range rMap { - for _, rg := range rgs.Groups { - userRules = append(userRules, rulespb.ToProto(user, file, rg)) - } - } - c.ruleGroupList[user] = userRules - } - - c.since = getLatestConfigID(configs, c.since) - - return c.ruleGroupList, nil -} - -// getLatestConfigID gets the latest configs ID. -// max [latest, max (map getID cfgs)] -func getLatestConfigID(cfgs map[string]userconfig.VersionedRulesConfig, latest userconfig.ID) userconfig.ID { - ret := latest - for _, config := range cfgs { - if config.ID > ret { - ret = config.ID - } - } - return ret -} - -func (c *ConfigRuleStore) ListRuleGroupsForUserAndNamespace(ctx context.Context, userID string, namespace string) (rulespb.RuleGroupList, error) { - r, err := c.ListAllRuleGroups(ctx) - if err != nil { - return nil, err - } - - if namespace == "" { - return r[userID], nil - } - - list := r[userID] - for ix := 0; ix < len(list); { - if list[ix].GetNamespace() != namespace { - list = append(list[:ix], list[ix+1:]...) - } else { - ix++ - } - } - - return list, nil -} - -func (c *ConfigRuleStore) LoadRuleGroups(_ context.Context, _ map[string]rulespb.RuleGroupList) error { - // Since ConfigRuleStore already Loads the rules in the List methods, there is nothing left to do here. - return nil -} - -// GetRuleGroup is not implemented -func (c *ConfigRuleStore) GetRuleGroup(_ context.Context, _, _, _ string) (*rulespb.RuleGroupDesc, error) { - return nil, errors.New("not implemented by the config service rule store") -} - -// SetRuleGroup is not implemented -func (c *ConfigRuleStore) SetRuleGroup(_ context.Context, _, _ string, _ *rulespb.RuleGroupDesc) error { - return errors.New("not implemented by the config service rule store") -} - -// DeleteRuleGroup is not implemented -func (c *ConfigRuleStore) DeleteRuleGroup(_ context.Context, _, _ string, _ string) error { - return errors.New("not implemented by the config service rule store") -} - -// DeleteNamespace is not implemented -func (c *ConfigRuleStore) DeleteNamespace(_ context.Context, _, _ string) error { - return errors.New("not implemented by the config service rule store") -} diff --git a/pkg/ruler/rulestore/configdb/store_test.go b/pkg/ruler/rulestore/configdb/store_test.go deleted file mode 100644 index 4d39581cb6492..0000000000000 --- a/pkg/ruler/rulestore/configdb/store_test.go +++ /dev/null @@ -1,184 +0,0 @@ -package configdb - -import ( - "context" - fmt "fmt" - "testing" - time "time" - - "github.com/stretchr/testify/assert" - - "github.com/grafana/loki/v3/pkg/configs/client" - "github.com/grafana/loki/v3/pkg/configs/userconfig" -) - -var zeroTime time.Time - -type MockClient struct { - cfgs map[string]userconfig.VersionedRulesConfig - err error -} - -func (c *MockClient) GetRules(_ context.Context, _ userconfig.ID) (map[string]userconfig.VersionedRulesConfig, error) { - return c.cfgs, c.err -} - -func (c *MockClient) GetAlerts(_ context.Context, _ userconfig.ID) (*client.ConfigsResponse, error) { - return nil, nil -} - -func Test_ConfigRuleStoreError(t *testing.T) { - mock := &MockClient{ - cfgs: nil, - err: fmt.Errorf("Error"), - } - - store := NewConfigRuleStore(mock) - _, err := store.ListAllRuleGroups(context.Background()) - - assert.Equal(t, mock.err, err, "Unexpected error returned") -} - -func Test_ConfigRuleStoreReturn(t *testing.T) { - id := userconfig.ID(10) - mock := &MockClient{ - cfgs: map[string]userconfig.VersionedRulesConfig{ - "user": { - ID: id, - Config: fakeRuleConfig(), - DeletedAt: zeroTime, - }, - }, - err: nil, - } - - store := NewConfigRuleStore(mock) - rules, _ := store.ListAllRuleGroups(context.Background()) - - assert.Equal(t, 1, len(rules["user"])) - assert.Equal(t, id, store.since) -} - -func Test_ConfigRuleStoreDelete(t *testing.T) { - mock := &MockClient{ - cfgs: map[string]userconfig.VersionedRulesConfig{ - "user": { - ID: 1, - Config: fakeRuleConfig(), - DeletedAt: zeroTime, - }, - }, - err: nil, - } - - store := NewConfigRuleStore(mock) - _, _ = store.ListAllRuleGroups(context.Background()) - - mock.cfgs["user"] = userconfig.VersionedRulesConfig{ - ID: 1, - Config: userconfig.RulesConfig{}, - DeletedAt: time.Unix(0, 1), - } - - rules, _ := store.ListAllRuleGroups(context.Background()) - - assert.Equal(t, 0, len(rules["user"])) -} - -func Test_ConfigRuleStoreAppend(t *testing.T) { - mock := &MockClient{ - cfgs: map[string]userconfig.VersionedRulesConfig{ - "user": { - ID: 1, - Config: fakeRuleConfig(), - DeletedAt: zeroTime, - }, - }, - err: nil, - } - - store := NewConfigRuleStore(mock) - _, _ = store.ListAllRuleGroups(context.Background()) - - delete(mock.cfgs, "user") - mock.cfgs["user2"] = userconfig.VersionedRulesConfig{ - ID: 1, - Config: fakeRuleConfig(), - DeletedAt: zeroTime, - } - - rules, _ := store.ListAllRuleGroups(context.Background()) - - assert.Equal(t, 2, len(rules)) -} - -func Test_ConfigRuleStoreSinceSet(t *testing.T) { - mock := &MockClient{ - cfgs: map[string]userconfig.VersionedRulesConfig{ - "user": { - ID: 1, - Config: fakeRuleConfig(), - DeletedAt: zeroTime, - }, - "user1": { - ID: 10, - Config: fakeRuleConfig(), - DeletedAt: zeroTime, - }, - "user2": { - ID: 100, - Config: fakeRuleConfig(), - DeletedAt: zeroTime, - }, - }, - err: nil, - } - - store := NewConfigRuleStore(mock) - _, _ = store.ListAllRuleGroups(context.Background()) - assert.Equal(t, userconfig.ID(100), store.since) - - delete(mock.cfgs, "user") - delete(mock.cfgs, "user1") - mock.cfgs["user2"] = userconfig.VersionedRulesConfig{ - ID: 50, - Config: fakeRuleConfig(), - DeletedAt: zeroTime, - } - - _, _ = store.ListAllRuleGroups(context.Background()) - assert.Equal(t, userconfig.ID(100), store.since) - - mock.cfgs["user2"] = userconfig.VersionedRulesConfig{ - ID: 101, - Config: fakeRuleConfig(), - DeletedAt: zeroTime, - } - - _, _ = store.ListAllRuleGroups(context.Background()) - assert.Equal(t, userconfig.ID(101), store.since) -} - -func fakeRuleConfig() userconfig.RulesConfig { - return userconfig.RulesConfig{ - FormatVersion: userconfig.RuleFormatV2, - Files: map[string]string{ - "test": ` -# Config no. 1. -groups: -- name: example - rules: - - alert: ScrapeFailed - expr: 'up != 1' - for: 10m - labels: - severity: warning - annotations: - summary: "Scrape of {{$labels.job}} (pod: {{$labels.instance}}) failed." - description: "Prometheus cannot reach the /metrics page on the {{$labels.instance}} pod." - impact: "We have no monitoring data for {{$labels.job}} - {{$labels.instance}}. At worst, it's completely down. At best, we cannot reliably respond to operational issues." - dashboardURL: "$${base_url}/admin/prometheus/targets" -`, - }, - } -} From e9eedceebb4d5d39af8dfedc448bffb7a2e9bb5c Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Tue, 29 Oct 2024 12:49:32 +0530 Subject: [PATCH 2/5] enable ruler_storage --- pkg/loki/loki.go | 5 +++++ pkg/loki/modules.go | 10 +++++++-- pkg/ruler/base/storage.go | 4 ++-- .../rulestore/bucketclient/bucket_client.go | 4 ++-- pkg/ruler/rulestore/config.go | 6 ++---- pkg/storage/bucket/sse_bucket_client.go | 8 +++---- pkg/storage/bucket/user_bucket_client.go | 2 +- pkg/util/limiter/combined_limits.go | 2 ++ pkg/validation/limits.go | 21 +++++++++++++++++++ tools/doc-generator/parse/root_blocks.go | 8 +++++++ 10 files changed, 55 insertions(+), 15 deletions(-) diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index f59218307e7d7..cd12d6654519b 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -92,6 +92,7 @@ type Config struct { Frontend lokifrontend.Config `yaml:"frontend,omitempty"` QueryRange queryrange.Config `yaml:"query_range,omitempty"` Ruler ruler.Config `yaml:"ruler,omitempty"` + RulerStorage rulestore.Config `yaml:"ruler_storage,omitempty" doc:"hidden"` IngesterClient ingester_client.Config `yaml:"ingester_client,omitempty"` IngesterRF1Client ingester_client.Config `yaml:"ingester_rf1_client,omitempty"` Ingester ingester.Config `yaml:"ingester,omitempty"` @@ -179,6 +180,7 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { c.TableManager.RegisterFlags(f) c.Frontend.RegisterFlags(f) c.Ruler.RegisterFlags(f) + c.RulerStorage.RegisterFlags(f) c.Worker.RegisterFlags(f) c.QueryRange.RegisterFlags(f) c.RuntimeConfig.RegisterFlags(f) @@ -262,6 +264,9 @@ func (c *Config) Validate() error { if err := c.Ruler.Validate(); err != nil { errs = append(errs, errors.Wrap(err, "CONFIG ERROR: invalid ruler config")) } + if err := c.RulerStorage.Validate(); err != nil { + errs = append(errs, errors.Wrap(err, "CONFIG ERROR: invalid ruler_storage config")) + } if err := c.Ingester.Validate(); err != nil { errs = append(errs, errors.Wrap(err, "CONFIG ERROR: invalid ingester config")) } diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index 86b5e668167cb..fcfca457f0a30 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -1232,7 +1232,8 @@ func (t *Loki) initRulerStorage() (_ services.Service, err error) { // to determine if it's unconfigured. the following check, however, correctly tests this. // Single binary integration tests will break if this ever drifts legacyReadMode := t.Cfg.LegacyReadTarget && t.Cfg.isTarget(Read) - if (t.Cfg.isTarget(All) || legacyReadMode || t.Cfg.isTarget(Backend)) && t.Cfg.Ruler.StoreConfig.IsDefaults() { + storageNotConfigured := (t.Cfg.StorageConfig.UseThanosObjstore && t.Cfg.RulerStorage.IsDefaults()) || t.Cfg.Ruler.StoreConfig.IsDefaults() + if (t.Cfg.isTarget(All) || legacyReadMode || t.Cfg.isTarget(Backend)) && storageNotConfigured { level.Info(util_log.Logger).Log("msg", "Ruler storage is not configured; ruler will not be started.") return } @@ -1245,7 +1246,12 @@ func (t *Loki) initRulerStorage() (_ services.Service, err error) { } } - t.RulerStorage, err = base_ruler.NewLegacyRuleStore(t.Cfg.Ruler.StoreConfig, t.Cfg.StorageConfig.Hedging, t.ClientMetrics, ruler.GroupLoader{}, util_log.Logger) + if t.Cfg.StorageConfig.UseThanosObjstore { + // TODO: add SSE limits + t.RulerStorage, err = base_ruler.NewRuleStore(context.Background(), t.Cfg.RulerStorage, t.Overrides, ruler.GroupLoader{}, util_log.Logger) + } else { + t.RulerStorage, err = base_ruler.NewLegacyRuleStore(t.Cfg.Ruler.StoreConfig, t.Cfg.StorageConfig.Hedging, t.ClientMetrics, ruler.GroupLoader{}, util_log.Logger) + } return } diff --git a/pkg/ruler/base/storage.go b/pkg/ruler/base/storage.go index 282247ffa48a4..9558cff9ee1ab 100644 --- a/pkg/ruler/base/storage.go +++ b/pkg/ruler/base/storage.go @@ -7,7 +7,6 @@ import ( "github.com/go-kit/log" "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" promRules "github.com/prometheus/prometheus/rules" "github.com/grafana/loki/v3/pkg/ruler/rulestore" @@ -120,10 +119,11 @@ func NewLegacyRuleStore(cfg RuleStoreConfig, hedgeCfg hedging.Config, clientMetr } // NewRuleStore returns a rule store backend client based on the provided cfg. -func NewRuleStore(ctx context.Context, cfg rulestore.Config, cfgProvider bucket.TenantConfigProvider, loader promRules.GroupLoader, logger log.Logger, _ prometheus.Registerer) (rulestore.RuleStore, error) { +func NewRuleStore(ctx context.Context, cfg rulestore.Config, cfgProvider bucket.SSEConfigProvider, loader promRules.GroupLoader, logger log.Logger) (rulestore.RuleStore, error) { if cfg.Backend == local.Name { return local.NewLocalRulesClient(cfg.Local, loader) } + bucketClient, err := bucket.NewClient(ctx, cfg.Backend, cfg.Config, "ruler-storage", logger) if err != nil { return nil, err diff --git a/pkg/ruler/rulestore/bucketclient/bucket_client.go b/pkg/ruler/rulestore/bucketclient/bucket_client.go index a39a8b03532da..89ad69f2e3c62 100644 --- a/pkg/ruler/rulestore/bucketclient/bucket_client.go +++ b/pkg/ruler/rulestore/bucketclient/bucket_client.go @@ -38,11 +38,11 @@ var ( // using the Thanos objstore.Bucket interface type BucketRuleStore struct { bucket objstore.Bucket - cfgProvider bucket.TenantConfigProvider + cfgProvider bucket.SSEConfigProvider logger log.Logger } -func NewBucketRuleStore(bkt objstore.Bucket, cfgProvider bucket.TenantConfigProvider, logger log.Logger) *BucketRuleStore { +func NewBucketRuleStore(bkt objstore.Bucket, cfgProvider bucket.SSEConfigProvider, logger log.Logger) *BucketRuleStore { return &BucketRuleStore{ bucket: bucket.NewPrefixedBucketClient(bkt, rulesPrefix), cfgProvider: cfgProvider, diff --git a/pkg/ruler/rulestore/config.go b/pkg/ruler/rulestore/config.go index a9a3864be31d8..334e43de0917d 100644 --- a/pkg/ruler/rulestore/config.go +++ b/pkg/ruler/rulestore/config.go @@ -8,15 +8,13 @@ import ( "github.com/grafana/loki/v3/pkg/ruler/rulestore/local" "github.com/grafana/loki/v3/pkg/storage/bucket" - "github.com/grafana/loki/v3/pkg/tool/client" ) // Config configures a rule store. type Config struct { bucket.Config `yaml:",inline"` - Backend string `yaml:"backend"` - ConfigDB client.Config `yaml:"configdb"` - Local local.Config `yaml:"local"` + Backend string `yaml:"backend"` + Local local.Config `yaml:"local"` } // RegisterFlags registers the backend storage config. diff --git a/pkg/storage/bucket/sse_bucket_client.go b/pkg/storage/bucket/sse_bucket_client.go index 426522cfcfd1f..04c3d71a68e10 100644 --- a/pkg/storage/bucket/sse_bucket_client.go +++ b/pkg/storage/bucket/sse_bucket_client.go @@ -12,8 +12,8 @@ import ( "github.com/grafana/loki/v3/pkg/storage/bucket/s3" ) -// TenantConfigProvider defines a per-tenant config provider. -type TenantConfigProvider interface { +// SSEConfigProvider defines a per-tenant SSE config provider. +type SSEConfigProvider interface { // S3SSEType returns the per-tenant S3 SSE type. S3SSEType(userID string) string @@ -29,11 +29,11 @@ type TenantConfigProvider interface { type SSEBucketClient struct { userID string bucket objstore.Bucket - cfgProvider TenantConfigProvider + cfgProvider SSEConfigProvider } // NewSSEBucketClient makes a new SSEBucketClient. The cfgProvider can be nil. -func NewSSEBucketClient(userID string, bucket objstore.Bucket, cfgProvider TenantConfigProvider) *SSEBucketClient { +func NewSSEBucketClient(userID string, bucket objstore.Bucket, cfgProvider SSEConfigProvider) *SSEBucketClient { return &SSEBucketClient{ userID: userID, bucket: bucket, diff --git a/pkg/storage/bucket/user_bucket_client.go b/pkg/storage/bucket/user_bucket_client.go index 14926a837b6f9..47fa996195585 100644 --- a/pkg/storage/bucket/user_bucket_client.go +++ b/pkg/storage/bucket/user_bucket_client.go @@ -6,7 +6,7 @@ import ( // NewUserBucketClient returns a bucket client to use to access the storage on behalf of the provided user. // The cfgProvider can be nil. -func NewUserBucketClient(userID string, bucket objstore.Bucket, cfgProvider TenantConfigProvider) objstore.InstrumentedBucket { +func NewUserBucketClient(userID string, bucket objstore.Bucket, cfgProvider SSEConfigProvider) objstore.InstrumentedBucket { // Inject the user/tenant prefix. bucket = NewPrefixedBucketClient(bucket, userID) diff --git a/pkg/util/limiter/combined_limits.go b/pkg/util/limiter/combined_limits.go index f29d703a6c68b..5c98c6bf9383d 100644 --- a/pkg/util/limiter/combined_limits.go +++ b/pkg/util/limiter/combined_limits.go @@ -14,6 +14,7 @@ import ( "github.com/grafana/loki/v3/pkg/ruler" scheduler_limits "github.com/grafana/loki/v3/pkg/scheduler/limits" "github.com/grafana/loki/v3/pkg/storage" + "github.com/grafana/loki/v3/pkg/storage/bucket" ) type CombinedLimits interface { @@ -30,4 +31,5 @@ type CombinedLimits interface { bloomplanner.Limits bloombuilder.Limits pattern.Limits + bucket.SSEConfigProvider } diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index 0f34448c6270c..215ace5c11cd4 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -231,6 +231,12 @@ type Limits struct { PatternIngesterTokenizableJSONFieldsDefault dskit_flagext.StringSliceCSV `yaml:"pattern_ingester_tokenizable_json_fields_default" json:"pattern_ingester_tokenizable_json_fields_default" doc:"hidden"` PatternIngesterTokenizableJSONFieldsAppend dskit_flagext.StringSliceCSV `yaml:"pattern_ingester_tokenizable_json_fields_append" json:"pattern_ingester_tokenizable_json_fields_append" doc:"hidden"` PatternIngesterTokenizableJSONFieldsDelete dskit_flagext.StringSliceCSV `yaml:"pattern_ingester_tokenizable_json_fields_delete" json:"pattern_ingester_tokenizable_json_fields_delete" doc:"hidden"` + + // This config doesn't have a CLI flag registered here because they're registered in + // their own original config struct. + S3SSEType string `yaml:"s3_sse_type" json:"s3_sse_type" doc:"nocli|description=S3 server-side encryption type. Required to enable server-side encryption overrides for a specific tenant. If not set, the default S3 client settings are used."` + S3SSEKMSKeyID string `yaml:"s3_sse_kms_key_id" json:"s3_sse_kms_key_id" doc:"nocli|description=S3 server-side encryption KMS Key ID. Ignored if the SSE type override is not set."` + S3SSEKMSEncryptionContext string `yaml:"s3_sse_kms_encryption_context" json:"s3_sse_kms_encryption_context" doc:"nocli|description=S3 server-side encryption KMS encryption context. If unset and the key ID override is set, the encryption context will not be provided to S3. Ignored if the SSE type override is not set."` } type StreamRetention struct { @@ -1106,6 +1112,21 @@ func (o *Overrides) PatternIngesterTokenizableJSONFieldsDelete(userID string) [] return o.getOverridesForUser(userID).PatternIngesterTokenizableJSONFieldsDelete } +// S3SSEType returns the per-tenant S3 SSE type. +func (o *Overrides) S3SSEType(user string) string { + return o.getOverridesForUser(user).S3SSEType +} + +// S3SSEKMSKeyID returns the per-tenant S3 KMS-SSE key id. +func (o *Overrides) S3SSEKMSKeyID(user string) string { + return o.getOverridesForUser(user).S3SSEKMSKeyID +} + +// S3SSEKMSEncryptionContext returns the per-tenant S3 KMS-SSE encryption context. +func (o *Overrides) S3SSEKMSEncryptionContext(user string) string { + return o.getOverridesForUser(user).S3SSEKMSEncryptionContext +} + func (o *Overrides) getOverridesForUser(userID string) *Limits { if o.tenantLimits != nil { l := o.tenantLimits.TenantLimits(userID) diff --git a/tools/doc-generator/parse/root_blocks.go b/tools/doc-generator/parse/root_blocks.go index f3ee4265596a4..776a0dee1afa7 100644 --- a/tools/doc-generator/parse/root_blocks.go +++ b/tools/doc-generator/parse/root_blocks.go @@ -30,6 +30,7 @@ import ( "github.com/grafana/loki/v3/pkg/querier/queryrange" querier_worker "github.com/grafana/loki/v3/pkg/querier/worker" "github.com/grafana/loki/v3/pkg/ruler" + "github.com/grafana/loki/v3/pkg/ruler/rulestore" "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/scheduler" "github.com/grafana/loki/v3/pkg/storage" @@ -301,6 +302,13 @@ Named store from this example can be used by setting object_store to store-1 in StructType: []reflect.Type{reflect.TypeOf(gcs.Config{})}, Desc: "The gcs_storage_backend block configures the connection to Google Cloud Storage object storage backend.", }, + { + Name: "ruler_storage_config", + StructType: []reflect.Type{reflect.TypeOf(rulestore.Config{})}, + Desc: `The ruler_storage_config configures ruler storage backend. +It uses thanos-io/objstore clients for connecting to object storage backends. This will become the default way of configuring object store clients in future releases. +Currently this is opt-in and takes effect only when ` + "`-use-thanos-objstore` " + "is set to true.", + }, } ) From b4350ea6589979e1519e55cf47223dae6fb99dde Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Tue, 29 Oct 2024 13:08:23 +0530 Subject: [PATCH 3/5] set loader for local store --- pkg/ruler/base/storage.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/ruler/base/storage.go b/pkg/ruler/base/storage.go index 9558cff9ee1ab..068718f5491a6 100644 --- a/pkg/ruler/base/storage.go +++ b/pkg/ruler/base/storage.go @@ -121,6 +121,9 @@ func NewLegacyRuleStore(cfg RuleStoreConfig, hedgeCfg hedging.Config, clientMetr // NewRuleStore returns a rule store backend client based on the provided cfg. func NewRuleStore(ctx context.Context, cfg rulestore.Config, cfgProvider bucket.SSEConfigProvider, loader promRules.GroupLoader, logger log.Logger) (rulestore.RuleStore, error) { if cfg.Backend == local.Name { + if loader == nil { + loader = promRules.FileLoader{} + } return local.NewLocalRulesClient(cfg.Local, loader) } From aa67a1ce1307b6590fbf16d6b9114738e295f413 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Tue, 29 Oct 2024 13:13:25 +0530 Subject: [PATCH 4/5] remove TODO comment --- pkg/loki/modules.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index fcfca457f0a30..8e5c76e8828de 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -1247,7 +1247,6 @@ func (t *Loki) initRulerStorage() (_ services.Service, err error) { } if t.Cfg.StorageConfig.UseThanosObjstore { - // TODO: add SSE limits t.RulerStorage, err = base_ruler.NewRuleStore(context.Background(), t.Cfg.RulerStorage, t.Overrides, ruler.GroupLoader{}, util_log.Logger) } else { t.RulerStorage, err = base_ruler.NewLegacyRuleStore(t.Cfg.Ruler.StoreConfig, t.Cfg.StorageConfig.Hedging, t.ClientMetrics, ruler.GroupLoader{}, util_log.Logger) From 69644510e2001af860e9f7cfb98e00929eca1ddb Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Tue, 29 Oct 2024 13:18:30 +0530 Subject: [PATCH 5/5] make doc --- docs/sources/shared/configuration.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index e6dfa17fb843a..a72b48e8e30ce 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -3844,6 +3844,20 @@ otlp_config: # disables shuffle sharding and tenant is sharded across all partitions. # CLI flag: -limits.ingestion-partition-tenant-shard-size [ingestion_partitions_tenant_shard_size: | default = 0] + +# S3 server-side encryption type. Required to enable server-side encryption +# overrides for a specific tenant. If not set, the default S3 client settings +# are used. +[s3_sse_type: | default = ""] + +# S3 server-side encryption KMS Key ID. Ignored if the SSE type override is not +# set. +[s3_sse_kms_key_id: | default = ""] + +# S3 server-side encryption KMS encryption context. If unset and the key ID +# override is set, the encryption context will not be provided to S3. Ignored if +# the SSE type override is not set. +[s3_sse_kms_encryption_context: | default = ""] ``` ### local_storage_config