From 6998d49f7f469bed1cae655d09d22c8302c80afd Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Wed, 15 Oct 2025 16:11:49 +0900 Subject: [PATCH 1/7] Add-changelog-for-utf8-support Signed-off-by: SungJin1212 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1a8e5ca4e..f94daffb4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ * [FEATURE] Querier: Support for configuring query optimizers and enabling XFunctions in the Thanos engine. #6873 * [FEATURE] Query Frontend: Add support /api/v1/format_query API for formatting queries. #6893 * [FEATURE] Query Frontend: Add support for /api/v1/parse_query API (experimental) to parse a PromQL expression and return it as a JSON-formatted AST (abstract syntax tree). #6978 +* [ENHANCEMENT] Upgrade the Prometheus version to 3.6.0 and add a `-validation.name-validation-scheme` flag to support UTF-8. #7040 * [ENHANCEMENT] Distributor: Emit an error with a 400 status code when empty labels are found before the relabelling or label dropping process. #7052 * [ENHANCEMENT] Parquet Storage: Add support for additional sort columns during Parquet file generation #7003 * [ENHANCEMENT] Modernizes the entire codebase by using go modernize tool. #7005 From 055e6eee18a1a09a20729d792b1b855dbb0305cc Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Thu, 16 Oct 2025 11:42:14 +0900 Subject: [PATCH 2/7] Add a utf8 query test case Signed-off-by: SungJin1212 --- integration/utf8_test.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/integration/utf8_test.go b/integration/utf8_test.go index a7bdffee93..6acbcb5184 100644 --- a/integration/utf8_test.go +++ b/integration/utf8_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/prometheus/common/model" "github.com/prometheus/prometheus/prompb" "github.com/stretchr/testify/require" @@ -105,16 +106,16 @@ overrides: require.NoError(t, s.StartAndWaitReady(cortex)) // user-1 uses legacy validation - user1Client, err := e2ecortex.NewClient(cortex.HTTPEndpoint(), "", "", "", "user-1") + user1Client, err := e2ecortex.NewClient(cortex.HTTPEndpoint(), cortex.HTTPEndpoint(), "", "", "user-1") require.NoError(t, err) // user-2 uses utf8 validation - user2Client, err := e2ecortex.NewClient(cortex.HTTPEndpoint(), "", "", "", "user-2") + user2Client, err := e2ecortex.NewClient(cortex.HTTPEndpoint(), cortex.HTTPEndpoint(), "", "", "user-2") require.NoError(t, err) now := time.Now() - utf8Series, _ := generateSeries("series_1", now, prompb.Label{Name: "test.utf8.metric", Value: "😄"}) + utf8Series, _ := generateSeries("series.1", now, prompb.Label{Name: "test.utf8.metric", Value: "😄"}) legacySeries, _ := generateSeries("series_2", now, prompb.Label{Name: "job", Value: "test"}) res, err := user1Client.Push(legacySeries) @@ -134,4 +135,15 @@ overrides: res, err = user2Client.Push(utf8Series) require.NoError(t, err) require.Equal(t, 200, res.StatusCode) + + // utf8 querying + // c.f. https://prometheus.io/docs/guides/utf8/#querying + query := `{"series.1", "test.utf8.metric"="😄"}` + queryResult, err := user2Client.Query(query, now) + require.NoError(t, err) + require.Equal(t, 200, res.StatusCode) + require.Equal(t, model.ValVector, queryResult.Type()) + vec := queryResult.(model.Vector) + require.Equal(t, 1, len(vec)) + } From 46d3529b97a5371c159951f5475f1a6abe0c81ab Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Fri, 17 Oct 2025 14:09:58 +0900 Subject: [PATCH 3/7] set utf8 as default and add test cases for label apis Signed-off-by: SungJin1212 --- integration/utf8_test.go | 18 ++++++++++++++++++ pkg/cortex/configinit/init.go | 8 -------- pkg/cortex/cortex.go | 1 - 3 files changed, 18 insertions(+), 9 deletions(-) delete mode 100644 pkg/cortex/configinit/init.go diff --git a/integration/utf8_test.go b/integration/utf8_test.go index 6acbcb5184..1273508d66 100644 --- a/integration/utf8_test.go +++ b/integration/utf8_test.go @@ -146,4 +146,22 @@ overrides: vec := queryResult.(model.Vector) require.Equal(t, 1, len(vec)) + // label names + start := now + end := now.Add(time.Minute * 5) + labelNames, err := user2Client.LabelNames(start, end) + require.NoError(t, err) + require.Equal(t, []string{"__name__", "job", "test.utf8.metric"}, labelNames) + + // series + series, err := user2Client.Series([]string{`{"test.utf8.metric"="😄"}`}, start, end) + require.NoError(t, err) + require.Equal(t, 1, len(series)) + require.Equal(t, `{__name__="series.1", test.utf8.metric="😄"}`, series[0].String()) + + // label values + labelValues, err := user2Client.LabelValues("test.utf8.metric", start, end, nil) + require.NoError(t, err) + require.Equal(t, 1, len(labelValues)) + require.Equal(t, model.LabelValue("😄"), labelValues[0]) } diff --git a/pkg/cortex/configinit/init.go b/pkg/cortex/configinit/init.go deleted file mode 100644 index bfb180797b..0000000000 --- a/pkg/cortex/configinit/init.go +++ /dev/null @@ -1,8 +0,0 @@ -package configinit - -import "github.com/prometheus/common/model" - -func init() { - // nolint:staticcheck - model.NameValidationScheme = model.LegacyValidation -} diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index e7575abdce..970a99f452 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -31,7 +31,6 @@ import ( "github.com/cortexproject/cortex/pkg/configs" configAPI "github.com/cortexproject/cortex/pkg/configs/api" "github.com/cortexproject/cortex/pkg/configs/db" - _ "github.com/cortexproject/cortex/pkg/cortex/configinit" "github.com/cortexproject/cortex/pkg/cortex/storage" "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/distributor" From ebbc63b3fcc9272f247d9525fb5b2155145b6736 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Fri, 17 Oct 2025 19:17:55 +0900 Subject: [PATCH 4/7] fix lint Signed-off-by: SungJin1212 --- pkg/distributor/distributor_test.go | 1 - pkg/util/validation/validate_test.go | 1 - 2 files changed, 2 deletions(-) diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index f68cd116aa..d5246e0b8c 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -36,7 +36,6 @@ import ( "google.golang.org/grpc/status" promchunk "github.com/cortexproject/cortex/pkg/chunk/encoding" - _ "github.com/cortexproject/cortex/pkg/cortex/configinit" "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/ha" "github.com/cortexproject/cortex/pkg/ingester" diff --git a/pkg/util/validation/validate_test.go b/pkg/util/validation/validate_test.go index a2e2825a38..52252b4cb4 100644 --- a/pkg/util/validation/validate_test.go +++ b/pkg/util/validation/validate_test.go @@ -16,7 +16,6 @@ import ( "github.com/stretchr/testify/require" "github.com/weaveworks/common/httpgrpc" - _ "github.com/cortexproject/cortex/pkg/cortex/configinit" "github.com/cortexproject/cortex/pkg/cortexpb" util_log "github.com/cortexproject/cortex/pkg/util/log" ) From 3a1c61e54585a4134d4b1c2e337ba8b98958a411 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Fri, 17 Oct 2025 19:59:39 +0900 Subject: [PATCH 5/7] Add alertmanger utf8 e2e test case Signed-off-by: SungJin1212 --- integration/alertmanager_test.go | 58 ++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/integration/alertmanager_test.go b/integration/alertmanager_test.go index f38442b8eb..842919a006 100644 --- a/integration/alertmanager_test.go +++ b/integration/alertmanager_test.go @@ -32,6 +32,64 @@ const simpleAlertmanagerConfig = `route: receivers: - name: dummy` +const utf8AlertmanagerConfig = `route: + receiver: dummy + group_by: [group.test.🙂] +receivers: + - name: dummy` + +func TestAlertmanager_UTF8(t *testing.T) { + s, err := e2e.NewScenario(networkName) + require.NoError(t, err) + defer s.Close() + + flags := mergeFlags(AlertmanagerFlags(), AlertmanagerS3Flags()) + + minio := e2edb.NewMinio(9000, alertsBucketName) + require.NoError(t, s.StartAndWaitReady(minio)) + + alertmanager := e2ecortex.NewAlertmanager( + "alertmanager", + flags, + "", + ) + + require.NoError(t, s.StartAndWaitReady(alertmanager)) + + c, err := e2ecortex.NewClient("", "", alertmanager.HTTPEndpoint(), "", "user-1") + require.NoError(t, err) + + ctx := context.Background() + + err = c.SetAlertmanagerConfig(ctx, utf8AlertmanagerConfig, map[string]string{}) + require.NoError(t, err) + + require.NoError(t, alertmanager.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"cortex_alertmanager_config_last_reload_successful"}, e2e.WaitMissingMetrics)) + require.NoError(t, alertmanager.WaitSumMetricsWithOptions(e2e.Greater(0), []string{"cortex_alertmanager_config_hash"}, e2e.WaitMissingMetrics)) + + silenceId, err := c.CreateSilence(ctx, types.Silence{ + Matchers: amlabels.Matchers{ + {Name: "silences.name.🙂", Value: "silences.value.🙂"}, + }, + Comment: "test silences", + StartsAt: time.Now(), + EndsAt: time.Now().Add(time.Minute), + }) + require.NoError(t, err) + require.NotEmpty(t, silenceId) + require.NoError(t, alertmanager.WaitSumMetrics(e2e.Equals(1), "cortex_alertmanager_silences")) + + err = c.SendAlertToAlermanager(ctx, &model.Alert{ + Labels: model.LabelSet{ + "alert.name.🙂": "alert.value.🙂", + }, + StartsAt: time.Now(), + EndsAt: time.Now().Add(time.Minute), + }) + require.NoError(t, err) + require.NoError(t, alertmanager.WaitSumMetrics(e2e.Equals(1), "cortex_alertmanager_alerts_received_total")) +} + func TestAlertmanager(t *testing.T) { s, err := e2e.NewScenario(networkName) require.NoError(t, err) From c665a41e1f7b40b899c70ca8e688b41f3a9a46a6 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Fri, 17 Oct 2025 20:32:06 +0900 Subject: [PATCH 6/7] Add ruler utf8 e2e test case Signed-off-by: SungJin1212 --- integration/ruler_test.go | 54 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/integration/ruler_test.go b/integration/ruler_test.go index 48bdaff551..3c0b3ee72d 100644 --- a/integration/ruler_test.go +++ b/integration/ruler_test.go @@ -284,6 +284,60 @@ func TestRulerSharding(t *testing.T) { assert.ElementsMatch(t, expectedNames, actualNames) } +func TestRulerAPIUTF8(t *testing.T) { + s, err := e2e.NewScenario(networkName) + require.NoError(t, err) + defer s.Close() + + // Start dependencies. + consul := e2edb.NewConsul() + minio := e2edb.NewMinio(9000, bucketName, rulestoreBucketName) + require.NoError(t, s.StartAndWaitReady(consul, minio)) + + rulerFlags := mergeFlags( + BlocksStorageFlags(), + RulerFlags(), + ) + + // Start ruler. + ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), rulerFlags, "") + require.NoError(t, s.StartAndWaitReady(ruler)) + + c, err := e2ecortex.NewClient("", "", "", ruler.HTTPEndpoint(), "user-1") + require.NoError(t, err) + + groupLabels := map[string]string{ + "group.label.🙂": "val.🙂", + } + ruleLabels := map[string]string{ + "rule.label.🙂": "val.🙂", + } + + interval, _ := model.ParseDuration("1s") + + ruleGroup := rulefmt.RuleGroup{ + Name: "utf8Rule", + Interval: interval, + Rules: []rulefmt.Rule{{ + Alert: "alert.rule", + Expr: "up", + Labels: ruleLabels, + }, { + Record: "record.rule", + Expr: "up", + Labels: ruleLabels, + }}, + Labels: groupLabels, + } + + // Set rule group + err = c.SetRuleGroup(ruleGroup, "namespace") + require.NoError(t, err) + + require.NoError(t, ruler.WaitSumMetrics(e2e.Equals(1), "cortex_ruler_managers_total")) + require.NoError(t, ruler.WaitSumMetrics(e2e.Equals(1), "cortex_ruler_rule_groups_in_store")) +} + func TestRulerAPISharding(t *testing.T) { testRulerAPIWithSharding(t, false) } From a5a75b942b6188225ee1f0820b3d828dac52d763 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Mon, 20 Oct 2025 21:58:29 +0900 Subject: [PATCH 7/7] Add additional information for validation.name-validation-scheme Signed-off-by: SungJin1212 --- docs/configuration/config-file-reference.md | 5 +++-- pkg/util/validation/limits.go | 2 +- schemas/cortex-config-schema.json | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index b8f97501de..5acb365d02 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -4345,8 +4345,9 @@ query_rejection: # list of rule groups to disable [disabled_rule_groups: | default = []] -# Name validation scheme for metric names and label names, Support values are: -# legacy, utf8. +# Name validation scheme for metric names and label names, it is applied when +# the Distributor validates labels and when the Ruler validates its +# ruler_external_labels. Support values are: legacy, utf8. # CLI flag: -validation.name-validation-scheme [name_validation_scheme: | default = legacy] ``` diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 1f2bc794a1..fbe32a81f1 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -358,7 +358,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.AlertmanagerMaxSilencesSizeBytes, "alertmanager.max-silences-size-bytes", 0, "Maximum size of individual silences that a single user can have. 0 = no limit.") _ = l.NameValidationScheme.Set(model.LegacyValidation.String()) - f.Var(&l.NameValidationScheme, "validation.name-validation-scheme", fmt.Sprintf("Name validation scheme for metric names and label names, Support values are: %s.", strings.Join([]string{model.LegacyValidation.String(), model.UTF8Validation.String()}, ", "))) + f.Var(&l.NameValidationScheme, "validation.name-validation-scheme", fmt.Sprintf("Name validation scheme for metric names and label names, it is applied when the Distributor validates labels and when the Ruler validates its ruler_external_labels. Support values are: %s.", strings.Join([]string{model.LegacyValidation.String(), model.UTF8Validation.String()}, ", "))) } // Validate the limits config and returns an error if the validation diff --git a/schemas/cortex-config-schema.json b/schemas/cortex-config-schema.json index 159bfda0a2..21ab770ec8 100644 --- a/schemas/cortex-config-schema.json +++ b/schemas/cortex-config-schema.json @@ -5186,7 +5186,7 @@ }, "name_validation_scheme": { "default": "legacy", - "description": "Name validation scheme for metric names and label names, Support values are: legacy, utf8.", + "description": "Name validation scheme for metric names and label names, it is applied when the Distributor validates labels and when the Ruler validates its ruler_external_labels. Support values are: legacy, utf8.", "type": "number", "x-cli-flag": "validation.name-validation-scheme" },