diff --git a/apis/apps/v1alpha1/configconstraint_types.go b/apis/apps/v1alpha1/configconstraint_types.go index b1bb97b38ed..b652821a438 100644 --- a/apis/apps/v1alpha1/configconstraint_types.go +++ b/apis/apps/v1alpha1/configconstraint_types.go @@ -277,6 +277,15 @@ type IniConfig struct { // sectionName describes ini section. // +optional SectionName string `json:"sectionName,omitempty"` + + // applyAllSection determines whether to support multiple sections in the ini file. + // if set to true, all sections parameter can be updated, e.g: client.default_character_set=utf8mb4 + // +optional + ApplyAllSection *bool `json:"applyAllSection,omitempty"` + + // parametersInSection is a map of section name to parameters. + // +optional + ParametersInSectionAsMap map[string][]string `json:"parametersInSection,omitempty"` } // +genclient @@ -309,3 +318,10 @@ type ConfigConstraintList struct { func init() { SchemeBuilder.Register(&ConfigConstraint{}, &ConfigConstraintList{}) } + +func (in *IniConfig) IsSupportMultiSection() bool { + if in.ApplyAllSection == nil { + return false + } + return *in.ApplyAllSection +} diff --git a/apis/apps/v1alpha1/zz_generated.deepcopy.go b/apis/apps/v1alpha1/zz_generated.deepcopy.go index 5d44e9f848f..e0d665da9c5 100644 --- a/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -3139,7 +3139,7 @@ func (in *FormatterOptions) DeepCopyInto(out *FormatterOptions) { if in.IniConfig != nil { in, out := &in.IniConfig, &out.IniConfig *out = new(IniConfig) - **out = **in + (*in).DeepCopyInto(*out) } } @@ -3240,6 +3240,26 @@ func (in *HorizontalScaling) DeepCopy() *HorizontalScaling { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IniConfig) DeepCopyInto(out *IniConfig) { *out = *in + if in.ApplyAllSection != nil { + in, out := &in.ApplyAllSection, &out.ApplyAllSection + *out = new(bool) + **out = **in + } + if in.ParametersInSectionAsMap != nil { + in, out := &in.ParametersInSectionAsMap, &out.ParametersInSectionAsMap + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IniConfig. diff --git a/config/crd/bases/apps.kubeblocks.io_configconstraints.yaml b/config/crd/bases/apps.kubeblocks.io_configconstraints.yaml index a06e46ae0a6..bd49cebd0fc 100644 --- a/config/crd/bases/apps.kubeblocks.io_configconstraints.yaml +++ b/config/crd/bases/apps.kubeblocks.io_configconstraints.yaml @@ -199,6 +199,19 @@ spec: iniConfig: description: iniConfig represents the ini options. properties: + applyAllSection: + description: 'applyAllSection determines whether to support + multiple sections in the ini file. if set to true, all sections + parameter can be updated, e.g: client.default_character_set=utf8mb4' + type: boolean + parametersInSection: + additionalProperties: + items: + type: string + type: array + description: parametersInSection is a map of section name + to parameters. + type: object sectionName: description: sectionName describes ini section. type: string diff --git a/deploy/helm/crds/apps.kubeblocks.io_configconstraints.yaml b/deploy/helm/crds/apps.kubeblocks.io_configconstraints.yaml index a06e46ae0a6..bd49cebd0fc 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_configconstraints.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_configconstraints.yaml @@ -199,6 +199,19 @@ spec: iniConfig: description: iniConfig represents the ini options. properties: + applyAllSection: + description: 'applyAllSection determines whether to support + multiple sections in the ini file. if set to true, all sections + parameter can be updated, e.g: client.default_character_set=utf8mb4' + type: boolean + parametersInSection: + additionalProperties: + items: + type: string + type: array + description: parametersInSection is a map of section name + to parameters. + type: object sectionName: description: sectionName describes ini section. type: string diff --git a/pkg/configuration/core/config.go b/pkg/configuration/core/config.go index 7c80997db2f..034994876d6 100644 --- a/pkg/configuration/core/config.go +++ b/pkg/configuration/core/config.go @@ -22,6 +22,7 @@ package core import ( "encoding/json" "path" + "slices" "strings" "github.com/StudioSol/set" @@ -233,7 +234,12 @@ func WithFormatterConfig(formatConfig *appsv1alpha1.FormatterConfig) Option { return func(ctx *CfgOpOption) { if formatConfig.Format == appsv1alpha1.Ini && formatConfig.IniConfig != nil { ctx.IniContext = &IniContext{ - SectionName: formatConfig.IniConfig.SectionName, + SectionName: formatConfig.IniConfig.SectionName, + ParametersInSectionAsMap: formatConfig.IniConfig.ParametersInSectionAsMap, + ApplyAllSection: false, + } + if formatConfig.IniConfig.ApplyAllSection != nil { + ctx.IniContext.ApplyAllSection = *formatConfig.IniConfig.ApplyAllSection } } } @@ -241,7 +247,9 @@ func WithFormatterConfig(formatConfig *appsv1alpha1.FormatterConfig) Option { func NestedPrefixField(formatConfig *appsv1alpha1.FormatterConfig) string { if formatConfig != nil && formatConfig.Format == appsv1alpha1.Ini && formatConfig.IniConfig != nil { - return formatConfig.IniConfig.SectionName + if !formatConfig.IniConfig.IsSupportMultiSection() { + return formatConfig.IniConfig.SectionName + } } return "" } @@ -276,7 +284,7 @@ func (c *cfgWrapper) queryAllCfg(jsonpath string, option CfgOpOption) ([]byte, e return util.RetrievalWithJSONPath(tops, jsonpath) } -func (c cfgWrapper) getConfigObject(option CfgOpOption) unstructured.ConfigObject { +func (c *cfgWrapper) getConfigObject(option CfgOpOption) unstructured.ConfigObject { if len(c.v) == 0 { return nil } @@ -289,13 +297,28 @@ func (c cfgWrapper) getConfigObject(option CfgOpOption) unstructured.ConfigObjec } func (c *cfgWrapper) generateKey(paramKey string, option CfgOpOption) string { - if option.IniContext != nil && len(option.IniContext.SectionName) > 0 { - return strings.Join([]string{option.IniContext.SectionName, paramKey}, unstructured.DelimiterDot) + if option.IniContext != nil { + // support special section, e.g: mysql.default-character-set + if strings.Index(paramKey, unstructured.DelimiterDot) > 0 { + return paramKey + } + sectionName := fromIniConfig(paramKey, option.IniContext) + if sectionName != "" { + return strings.Join([]string{sectionName, paramKey}, unstructured.DelimiterDot) + } } - return paramKey } +func fromIniConfig(paramKey string, iniContext *IniContext) string { + for s, params := range iniContext.ParametersInSectionAsMap { + if slices.Contains(params, paramKey) { + return s + } + } + return iniContext.SectionName +} + func FromCMKeysSelector(keys []string) *set.LinkedHashSetString { var cmKeySet *set.LinkedHashSetString if len(keys) > 0 { @@ -310,15 +333,16 @@ func GenerateVisualizedParamsList(configPatch *ConfigPatchInfo, formatConfig *ap } var trimPrefix = NestedPrefixField(formatConfig) + var applyAllSections = isIniCfgAndSupportMultiSection(formatConfig) r := make([]VisualizedParam, 0) - r = append(r, generateUpdateParam(configPatch.UpdateConfig, trimPrefix, sets)...) - r = append(r, generateUpdateKeyParam(configPatch.AddConfig, trimPrefix, AddedType, sets)...) - r = append(r, generateUpdateKeyParam(configPatch.DeleteConfig, trimPrefix, DeletedType, sets)...) + r = append(r, generateUpdateParam(configPatch.UpdateConfig, trimPrefix, sets, applyAllSections)...) + r = append(r, generateUpdateKeyParam(configPatch.AddConfig, trimPrefix, AddedType, sets, applyAllSections)...) + r = append(r, generateUpdateKeyParam(configPatch.DeleteConfig, trimPrefix, DeletedType, sets, applyAllSections)...) return r } -func generateUpdateParam(updatedParams map[string][]byte, trimPrefix string, sets *set.LinkedHashSetString) []VisualizedParam { +func generateUpdateParam(updatedParams map[string][]byte, trimPrefix string, sets *set.LinkedHashSetString, applyAllSections bool) []VisualizedParam { r := make([]VisualizedParam, 0, len(updatedParams)) for key, b := range updatedParams { @@ -330,7 +354,7 @@ func generateUpdateParam(updatedParams map[string][]byte, trimPrefix string, set if err := json.Unmarshal(b, &v); err != nil { return nil } - if params := checkAndFlattenMap(v, trimPrefix); params != nil { + if params := checkAndFlattenMap(v, trimPrefix, applyAllSections); params != nil { r = append(r, VisualizedParam{ Key: key, Parameters: params, @@ -341,18 +365,18 @@ func generateUpdateParam(updatedParams map[string][]byte, trimPrefix string, set return r } -func checkAndFlattenMap(v any, trim string) []ParameterPair { +func checkAndFlattenMap(v any, trim string, applyAllSections bool) []ParameterPair { m := cast.ToStringMap(v) if m != nil && trim != "" { m = cast.ToStringMap(m[trim]) } if m != nil { - return flattenMap(m, "") + return flattenMap(m, "", applyAllSections) } return nil } -func flattenMap(m map[string]interface{}, prefix string) []ParameterPair { +func flattenMap(m map[string]interface{}, prefix string, applyAllSections bool) []ParameterPair { if prefix != "" { prefix += unstructured.DelimiterDot } @@ -362,10 +386,10 @@ func flattenMap(m map[string]interface{}, prefix string) []ParameterPair { fullKey := prefix + k switch m2 := val.(type) { case map[string]interface{}: - r = append(r, flattenMap(m2, fullKey)...) + r = append(r, flattenMap(m2, fullKey, applyAllSections)...) case []interface{}: r = append(r, ParameterPair{ - Key: transArrayFieldName(fullKey), + Key: transArrayFieldName(trimPrimaryKeyName(fullKey, applyAllSections)), Value: util.ToPointer(transJSONString(val)), }) default: @@ -374,7 +398,7 @@ func flattenMap(m map[string]interface{}, prefix string) []ParameterPair { v = util.ToPointer(cast.ToString(val)) } r = append(r, ParameterPair{ - Key: fullKey, + Key: trimPrimaryKeyName(fullKey, applyAllSections), Value: v, }) } @@ -382,14 +406,32 @@ func flattenMap(m map[string]interface{}, prefix string) []ParameterPair { return r } -func generateUpdateKeyParam(files map[string]interface{}, trimPrefix string, updatedType ParameterUpdateType, sets *set.LinkedHashSetString) []VisualizedParam { +func trimPrimaryKeyName(key string, section bool) string { + if !section { + return key + } + + pos := strings.Index(key, ".") + if pos < 0 { + return key + } else { + return key[pos+1:] + } +} + +func generateUpdateKeyParam( + files map[string]interface{}, + trimPrefix string, + updatedType ParameterUpdateType, + sets *set.LinkedHashSetString, + applyAllSections bool) []VisualizedParam { r := make([]VisualizedParam, 0, len(files)) for key, params := range files { if sets != nil && sets.Length() > 0 && !sets.InArray(key) { continue } - if params := checkAndFlattenMap(params, trimPrefix); params != nil { + if params := checkAndFlattenMap(params, trimPrefix, applyAllSections); params != nil { r = append(r, VisualizedParam{ Key: key, Parameters: params, diff --git a/pkg/configuration/core/config_patch_test.go b/pkg/configuration/core/config_patch_test.go index 9fb507d8cb1..13111384f4f 100644 --- a/pkg/configuration/core/config_patch_test.go +++ b/pkg/configuration/core/config_patch_test.go @@ -94,6 +94,33 @@ func TestConfigPatch(t *testing.T) { log.Log.Info("patch : %v", patch) require.False(t, patch.IsModify) } + + { + + require.Nil(t, + cfg.MergeFrom(map[string]interface{}{ + "server-id": 1, + "socket": "/data/mysql/tmp/mysqld.sock2", + "client.port": "6666", + }, NewCfgOptions("", func(ctx *CfgOpOption) { + // filter mysqld + ctx.IniContext = &IniContext{ + SectionName: "mysqld", + ParametersInSectionAsMap: map[string][]string{ + "client": {"port", "socket"}, + }, + } + }))) + content, err := cfg.ToCfgContent() + require.Nil(t, err) + newContent := content[cfg.name] + // CreateMergePatch([]byte(iniConfig), []byte(newContent), cfg.Option) + patch, err := CreateMergePatch([]byte(iniConfig), []byte(newContent), cfg.Option) + require.Nil(t, err) + log.Log.Info("patch : %v", patch) + require.True(t, patch.IsModify) + require.Equal(t, string(patch.UpdateConfig["raw"]), `{"client":{"port":"6666","socket":"/data/mysql/tmp/mysqld.sock2"}}`) + } } func TestYamlConfigPatch(t *testing.T) { diff --git a/pkg/configuration/core/config_test.go b/pkg/configuration/core/config_test.go index aa5af04ebb2..ca8576094dc 100644 --- a/pkg/configuration/core/config_test.go +++ b/pkg/configuration/core/config_test.go @@ -158,9 +158,10 @@ func TestGenerateVisualizedParamsList(t *testing.T) { } var ( - testJSON any - fileUpdatedParams = []byte(`{"mysqld": { "max_connections": "666", "read_buffer_size": "55288" }}`) - testUpdatedParams = []byte(`{"mysqld": { "max_connections": "666", "read_buffer_size": "55288", "delete_params": null }}`) + testJSON any + fileUpdatedParams = []byte(`{"mysqld": { "max_connections": "666", "read_buffer_size": "55288" }}`) + testUpdatedParams = []byte(`{"mysqld": { "max_connections": "666", "read_buffer_size": "55288", "delete_params": null }}`) + testUpdatedParams2 = []byte(`{"mysqld": { "max_connections": "666", "read_buffer_size": "55288", "delete_params": null }, "mysql": { "default-character-set": "utf8mb4"}, "client": { "port": "3306"}}`) ) require.Nil(t, json.Unmarshal(fileUpdatedParams, &testJSON)) @@ -278,6 +279,69 @@ func TestGenerateVisualizedParamsList(t *testing.T) { Value: util.ToPointer("55288"), }}, }}, + }, { + name: "visualizedUpdatedMultiParamsTest", + args: args{ + configPatch: &ConfigPatchInfo{ + IsModify: true, + UpdateConfig: map[string][]byte{"key": testUpdatedParams2}}, + formatConfig: &appsv1alpha1.FormatterConfig{ + Format: appsv1alpha1.Ini, + FormatterOptions: appsv1alpha1.FormatterOptions{IniConfig: &appsv1alpha1.IniConfig{ + SectionName: "mysqld", + ApplyAllSection: util.ToPointer(true), + }}, + }, + }, + want: []VisualizedParam{{ + Key: "key", + UpdateType: UpdatedType, + Parameters: []ParameterPair{ + { + Key: "max_connections", + Value: util.ToPointer("666"), + }, { + Key: "read_buffer_size", + Value: util.ToPointer("55288"), + }, { + Key: "delete_params", + Value: nil, + }, { + Key: "default-character-set", + Value: util.ToPointer("utf8mb4"), + }, { + Key: "port", + Value: util.ToPointer("3306"), + }}, + }}, + }, { + name: "visualizedUpdatedMultiParamsTest", + args: args{ + configPatch: &ConfigPatchInfo{ + IsModify: true, + UpdateConfig: map[string][]byte{"key": testUpdatedParams2}}, + formatConfig: &appsv1alpha1.FormatterConfig{ + Format: appsv1alpha1.Ini, + FormatterOptions: appsv1alpha1.FormatterOptions{IniConfig: &appsv1alpha1.IniConfig{ + SectionName: "mysqld", + }}, + }, + }, + want: []VisualizedParam{{ + Key: "key", + UpdateType: UpdatedType, + Parameters: []ParameterPair{ + { + Key: "max_connections", + Value: util.ToPointer("666"), + }, { + Key: "read_buffer_size", + Value: util.ToPointer("55288"), + }, { + Key: "delete_params", + Value: nil, + }}, + }}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/configuration/core/reconfigure_util.go b/pkg/configuration/core/reconfigure_util.go index 6042a332267..103099fe2b3 100644 --- a/pkg/configuration/core/reconfigure_util.go +++ b/pkg/configuration/core/reconfigure_util.go @@ -106,7 +106,8 @@ func IsUpdateDynamicParameters(cc *appsv1alpha1.ConfigConstraintSpec, cfg *Confi if len(updatedParams) == 0 { return true, nil } - updatedParamsSet := util.NewSet(updatedParams...) + // for ini format, if support multi-section, trim section name + updatedParamsSet := util.NewSet(trimIniSectionName(updatedParams, isIniCfgAndSupportMultiSection(cc.FormatterConfig))...) // if ConfigConstraint has StaticParameters, check updated parameter if len(cc.StaticParameters) > 0 { @@ -133,6 +134,22 @@ func IsUpdateDynamicParameters(cc *appsv1alpha1.ConfigConstraintSpec, cfg *Confi return false, nil } +func trimIniSectionName(updatedParams []string, supportIncMultiSection bool) []string { + if !supportIncMultiSection { + return updatedParams + } + + trimFields := make([]string, len(updatedParams)) + for i, param := range updatedParams { + trimFields[i] = trimPrimaryKeyName(param, true) + } + return trimFields +} + +func isIniCfgAndSupportMultiSection(config *appsv1alpha1.FormatterConfig) bool { + return config != nil && config.IniConfig != nil && config.IniConfig.IsSupportMultiSection() +} + // IsParametersUpdateFromManager checks if the parameters are updated from manager func IsParametersUpdateFromManager(cm *corev1.ConfigMap) bool { annotation := cm.ObjectMeta.Annotations diff --git a/pkg/configuration/core/type.go b/pkg/configuration/core/type.go index f003936d4f8..7cf3ec2aed5 100644 --- a/pkg/configuration/core/type.go +++ b/pkg/configuration/core/type.go @@ -49,6 +49,9 @@ type RawConfig struct { type IniContext struct { SectionName string + + ApplyAllSection bool + ParametersInSectionAsMap map[string][]string } // XMLContext TODO(zt) Support Xml config