Skip to content

Commit

Permalink
feat: Support upate parameters in non-default sections, e.g client or…
Browse files Browse the repository at this point in the history 
… mysql: --set mysql.default-character-set = utf8mb4 (#6453)
  • Loading branch information
@sophon-zt
sophon-zt committed Feb 19, 2024
1 parent 32d6770 commit fca926f
Show file tree
Hide file tree
Showing 9 changed files with 239 additions and 24 deletions.
16 changes: 16 additions & 0 deletions apis/apps/v1alpha1/configconstraint_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,15 @@ type IniConfig struct {
// sectionName describes ini section.
// +optional
SectionName string `json:"sectionName,omitempty"`

// applyAllSection determines whether to support multiple sections in the ini file.
// if set to true, all sections parameter can be updated, e.g: client.default_character_set=utf8mb4
// +optional
ApplyAllSection *bool `json:"applyAllSection,omitempty"`

// parametersInSection is a map of section name to parameters.
// +optional
ParametersInSectionAsMap map[string][]string `json:"parametersInSection,omitempty"`
}

// +genclient
Expand Down Expand Up @@ -309,3 +318,10 @@ type ConfigConstraintList struct {
func init() {
SchemeBuilder.Register(&ConfigConstraint{}, &ConfigConstraintList{})
}

func (in *IniConfig) IsSupportMultiSection() bool {
if in.ApplyAllSection == nil {
return false
}
return *in.ApplyAllSection
}
22 changes: 21 additions & 1 deletion apis/apps/v1alpha1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 13 additions & 0 deletions config/crd/bases/apps.kubeblocks.io_configconstraints.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,19 @@ spec:
iniConfig:
description: iniConfig represents the ini options.
properties:
applyAllSection:
description: 'applyAllSection determines whether to support
multiple sections in the ini file. if set to true, all sections
parameter can be updated, e.g: client.default_character_set=utf8mb4'
type: boolean
parametersInSection:
additionalProperties:
items:
type: string
type: array
description: parametersInSection is a map of section name
to parameters.
type: object
sectionName:
description: sectionName describes ini section.
type: string
Expand Down
13 changes: 13 additions & 0 deletions deploy/helm/crds/apps.kubeblocks.io_configconstraints.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,19 @@ spec:
iniConfig:
description: iniConfig represents the ini options.
properties:
applyAllSection:
description: 'applyAllSection determines whether to support
multiple sections in the ini file. if set to true, all sections
parameter can be updated, e.g: client.default_character_set=utf8mb4'
type: boolean
parametersInSection:
additionalProperties:
items:
type: string
type: array
description: parametersInSection is a map of section name
to parameters.
type: object
sectionName:
description: sectionName describes ini section.
type: string
Expand Down
80 changes: 61 additions & 19 deletions pkg/configuration/core/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ package core
import (
"encoding/json"
"path"
"slices"
"strings"

"github.com/StudioSol/set"
Expand Down Expand Up @@ -233,15 +234,22 @@ func WithFormatterConfig(formatConfig *appsv1alpha1.FormatterConfig) Option {
return func(ctx *CfgOpOption) {
if formatConfig.Format == appsv1alpha1.Ini && formatConfig.IniConfig != nil {
ctx.IniContext = &IniContext{
SectionName: formatConfig.IniConfig.SectionName,
SectionName: formatConfig.IniConfig.SectionName,
ParametersInSectionAsMap: formatConfig.IniConfig.ParametersInSectionAsMap,
ApplyAllSection: false,
}
if formatConfig.IniConfig.ApplyAllSection != nil {
ctx.IniContext.ApplyAllSection = *formatConfig.IniConfig.ApplyAllSection
}
}
}
}

func NestedPrefixField(formatConfig *appsv1alpha1.FormatterConfig) string {
if formatConfig != nil && formatConfig.Format == appsv1alpha1.Ini && formatConfig.IniConfig != nil {
return formatConfig.IniConfig.SectionName
if !formatConfig.IniConfig.IsSupportMultiSection() {
return formatConfig.IniConfig.SectionName
}
}
return ""
}
Expand Down Expand Up @@ -276,7 +284,7 @@ func (c *cfgWrapper) queryAllCfg(jsonpath string, option CfgOpOption) ([]byte, e
return util.RetrievalWithJSONPath(tops, jsonpath)
}

func (c cfgWrapper) getConfigObject(option CfgOpOption) unstructured.ConfigObject {
func (c *cfgWrapper) getConfigObject(option CfgOpOption) unstructured.ConfigObject {
if len(c.v) == 0 {
return nil
}
Expand All @@ -289,13 +297,28 @@ func (c cfgWrapper) getConfigObject(option CfgOpOption) unstructured.ConfigObjec
}

func (c *cfgWrapper) generateKey(paramKey string, option CfgOpOption) string {
if option.IniContext != nil && len(option.IniContext.SectionName) > 0 {
return strings.Join([]string{option.IniContext.SectionName, paramKey}, unstructured.DelimiterDot)
if option.IniContext != nil {
// support special section, e.g: mysql.default-character-set
if strings.Index(paramKey, unstructured.DelimiterDot) > 0 {
return paramKey
}
sectionName := fromIniConfig(paramKey, option.IniContext)
if sectionName != "" {
return strings.Join([]string{sectionName, paramKey}, unstructured.DelimiterDot)
}
}

return paramKey
}

func fromIniConfig(paramKey string, iniContext *IniContext) string {
for s, params := range iniContext.ParametersInSectionAsMap {
if slices.Contains(params, paramKey) {
return s
}
}
return iniContext.SectionName
}

func FromCMKeysSelector(keys []string) *set.LinkedHashSetString {
var cmKeySet *set.LinkedHashSetString
if len(keys) > 0 {
Expand All @@ -310,15 +333,16 @@ func GenerateVisualizedParamsList(configPatch *ConfigPatchInfo, formatConfig *ap
}

var trimPrefix = NestedPrefixField(formatConfig)
var applyAllSections = isIniCfgAndSupportMultiSection(formatConfig)

r := make([]VisualizedParam, 0)
r = append(r, generateUpdateParam(configPatch.UpdateConfig, trimPrefix, sets)...)
r = append(r, generateUpdateKeyParam(configPatch.AddConfig, trimPrefix, AddedType, sets)...)
r = append(r, generateUpdateKeyParam(configPatch.DeleteConfig, trimPrefix, DeletedType, sets)...)
r = append(r, generateUpdateParam(configPatch.UpdateConfig, trimPrefix, sets, applyAllSections)...)
r = append(r, generateUpdateKeyParam(configPatch.AddConfig, trimPrefix, AddedType, sets, applyAllSections)...)
r = append(r, generateUpdateKeyParam(configPatch.DeleteConfig, trimPrefix, DeletedType, sets, applyAllSections)...)
return r
}

func generateUpdateParam(updatedParams map[string][]byte, trimPrefix string, sets *set.LinkedHashSetString) []VisualizedParam {
func generateUpdateParam(updatedParams map[string][]byte, trimPrefix string, sets *set.LinkedHashSetString, applyAllSections bool) []VisualizedParam {
r := make([]VisualizedParam, 0, len(updatedParams))

for key, b := range updatedParams {
Expand All @@ -330,7 +354,7 @@ func generateUpdateParam(updatedParams map[string][]byte, trimPrefix string, set
if err := json.Unmarshal(b, &v); err != nil {
return nil
}
if params := checkAndFlattenMap(v, trimPrefix); params != nil {
if params := checkAndFlattenMap(v, trimPrefix, applyAllSections); params != nil {
r = append(r, VisualizedParam{
Key: key,
Parameters: params,
Expand All @@ -341,18 +365,18 @@ func generateUpdateParam(updatedParams map[string][]byte, trimPrefix string, set
return r
}

func checkAndFlattenMap(v any, trim string) []ParameterPair {
func checkAndFlattenMap(v any, trim string, applyAllSections bool) []ParameterPair {
m := cast.ToStringMap(v)
if m != nil && trim != "" {
m = cast.ToStringMap(m[trim])
}
if m != nil {
return flattenMap(m, "")
return flattenMap(m, "", applyAllSections)
}
return nil
}

func flattenMap(m map[string]interface{}, prefix string) []ParameterPair {
func flattenMap(m map[string]interface{}, prefix string, applyAllSections bool) []ParameterPair {
if prefix != "" {
prefix += unstructured.DelimiterDot
}
Expand All @@ -362,10 +386,10 @@ func flattenMap(m map[string]interface{}, prefix string) []ParameterPair {
fullKey := prefix + k
switch m2 := val.(type) {
case map[string]interface{}:
r = append(r, flattenMap(m2, fullKey)...)
r = append(r, flattenMap(m2, fullKey, applyAllSections)...)
case []interface{}:
r = append(r, ParameterPair{
Key: transArrayFieldName(fullKey),
Key: transArrayFieldName(trimPrimaryKeyName(fullKey, applyAllSections)),
Value: util.ToPointer(transJSONString(val)),
})
default:
Expand All @@ -374,22 +398,40 @@ func flattenMap(m map[string]interface{}, prefix string) []ParameterPair {
v = util.ToPointer(cast.ToString(val))
}
r = append(r, ParameterPair{
Key: fullKey,
Key: trimPrimaryKeyName(fullKey, applyAllSections),
Value: v,
})
}
}
return r
}

func generateUpdateKeyParam(files map[string]interface{}, trimPrefix string, updatedType ParameterUpdateType, sets *set.LinkedHashSetString) []VisualizedParam {
func trimPrimaryKeyName(key string, section bool) string {
if !section {
return key
}

pos := strings.Index(key, ".")
if pos < 0 {
return key
} else {
return key[pos+1:]
}
}

func generateUpdateKeyParam(
files map[string]interface{},
trimPrefix string,
updatedType ParameterUpdateType,
sets *set.LinkedHashSetString,
applyAllSections bool) []VisualizedParam {
r := make([]VisualizedParam, 0, len(files))

for key, params := range files {
if sets != nil && sets.Length() > 0 && !sets.InArray(key) {
continue
}
if params := checkAndFlattenMap(params, trimPrefix); params != nil {
if params := checkAndFlattenMap(params, trimPrefix, applyAllSections); params != nil {
r = append(r, VisualizedParam{
Key: key,
Parameters: params,
Expand Down
27 changes: 27 additions & 0 deletions pkg/configuration/core/config_patch_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,33 @@ func TestConfigPatch(t *testing.T) {
log.Log.Info("patch : %v", patch)
require.False(t, patch.IsModify)
}

{

require.Nil(t,
cfg.MergeFrom(map[string]interface{}{
"server-id": 1,
"socket": "/data/mysql/tmp/mysqld.sock2",
"client.port": "6666",
}, NewCfgOptions("", func(ctx *CfgOpOption) {
// filter mysqld
ctx.IniContext = &IniContext{
SectionName: "mysqld",
ParametersInSectionAsMap: map[string][]string{
"client": {"port", "socket"},
},
}
})))
content, err := cfg.ToCfgContent()
require.Nil(t, err)
newContent := content[cfg.name]
// CreateMergePatch([]byte(iniConfig), []byte(newContent), cfg.Option)
patch, err := CreateMergePatch([]byte(iniConfig), []byte(newContent), cfg.Option)
require.Nil(t, err)
log.Log.Info("patch : %v", patch)
require.True(t, patch.IsModify)
require.Equal(t, string(patch.UpdateConfig["raw"]), `{"client":{"port":"6666","socket":"/data/mysql/tmp/mysqld.sock2"}}`)
}
}

func TestYamlConfigPatch(t *testing.T) {
Expand Down
Loading

0 comments on commit fca926f

Please sign in to comment.