Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(mysql): mysql-monitor-missing-machine-type #6919 #6920

Merged
merged 1 commit into from
Sep 14, 2024
Merged

fix(mysql): mysql-monitor-missing-machine-type #6919 #6920

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 16 additions & 18 deletions dbm-services/mysql/db-tools/dbactuator/pkg/components/peripheraltools/monitor/example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,25 +23,23 @@ func (c *MySQLMonitorComp) Example() interface{} {
SystemDbs: native.DBSys,
ExecUser: "whoru",
ApiUrl: `http://x.x.x.x:yyyy`,
InstancesInfo: []*instanceInfo{
ItemsConfig: map[string]*config.MonitorItem{
"character-consistency": {
Name: "",
Enable: nil,
Schedule: nil,
MachineType: nil,
Role: nil,
},
},
InstancesInfo: []*internal.InstanceInfo{
{
internal.InstanceInfo{
BkBizId: 1,
Ip: "127.0.0.1",
Port: 123,
Role: "master",
ClusterId: 12,
ImmuteDomain: "aaa.bbb.com",
},
map[string]*config.MonitorItem{
"character-consistency": {
Name: "",
Enable: nil,
Schedule: nil,
MachineType: nil,
Role: nil,
},
},
BkBizId: 1,
Ip: "127.0.0.1",
Port: 123,
Role: "master",
ClusterId: 12,
ImmuteDomain: "aaa.bbb.com",
},
},
MachineType: "backend",
Expand Down
3 changes: 2 additions & 1 deletion ...vices/mysql/db-tools/dbactuator/pkg/components/peripheraltools/monitor/exporter_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package monitor
import (
"dbm-services/common/go-pubpkg/logger"
"dbm-services/mysql/db-tools/dbactuator/pkg/components"
"dbm-services/mysql/db-tools/dbactuator/pkg/components/peripheraltools/internal"
"dbm-services/mysql/db-tools/dbactuator/pkg/native"
"dbm-services/mysql/db-tools/dbactuator/pkg/util"
"dbm-services/mysql/db-tools/dbactuator/pkg/util/osutil"
Expand All @@ -21,7 +22,7 @@ func (c *MySQLMonitorComp) GenerateExporterConfig() (err error) {
return nil
}

func generateExporterConfigIns(mmp *MySQLMonitorParam, instance *instanceInfo, rtap *components.RuntimeAccountParam) (err error) {
func generateExporterConfigIns(mmp *MySQLMonitorParam, instance *internal.InstanceInfo, rtap *components.RuntimeAccountParam) (err error) {
exporterConfigPath := filepath.Join(
"/etc",
fmt.Sprintf("exporter_%d.cnf", instance.Port),
Expand Down
18 changes: 7 additions & 11 deletions dbm-services/mysql/db-tools/dbactuator/pkg/components/peripheraltools/monitor/init.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,11 @@ func (c *MySQLMonitorComp) Init() (err error) {

type MySQLMonitorParam struct {
components.Medium
SystemDbs []string `json:"system_dbs"`
ExecUser string `json:"exec_user"`
ApiUrl string `json:"api_url"`
InstancesInfo []*instanceInfo `json:"instances_info"`
MachineType string `json:"machine_type"`
BkCloudId int `json:"bk_cloud_id"`
}

type instanceInfo struct {
internal.InstanceInfo
ItemsConfig map[string]*config.MonitorItem `json:"items_config" yaml:"items_config"`
SystemDbs []string `json:"system_dbs"`
ExecUser string `json:"exec_user"`
ApiUrl string `json:"api_url"`
InstancesInfo []*internal.InstanceInfo `json:"instances_info"`
MachineType string `json:"machine_type"`
BkCloudId int `json:"bk_cloud_id"`
ItemsConfig map[string]*config.MonitorItem `json:"items_config" yaml:"items_config"`
}
14 changes: 11 additions & 3 deletions ...services/mysql/db-tools/dbactuator/pkg/components/peripheraltools/monitor/items_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,33 @@ import (
"dbm-services/common/go-pubpkg/logger"
"dbm-services/mysql/db-tools/dbactuator/pkg/components/peripheraltools/internal"
"dbm-services/mysql/db-tools/dbactuator/pkg/core/cst"
"dbm-services/mysql/db-tools/mysql-monitor/pkg/config"
"fmt"
"path/filepath"
"slices"
"strings"

"golang.org/x/exp/maps"
"gopkg.in/yaml.v2"
)

func (c *MySQLMonitorComp) GenerateItemsConfig() (err error) {
for _, inst := range c.Params.InstancesInfo {
err = generateItemsConfigIns(inst)
err = generateItemsConfigIns(inst, c.Params.ItemsConfig)
if err != nil {
return err
}
}
return nil
}

func generateItemsConfigIns(instance *instanceInfo) (err error) {
b, err := yaml.Marshal(maps.Values(instance.ItemsConfig))
func generateItemsConfigIns(instance *internal.InstanceInfo, itemsConfig map[string]*config.MonitorItem) (err error) {
itemList := maps.Values(itemsConfig)
slices.SortFunc(itemList, func(a, b *config.MonitorItem) int {
return strings.Compare(a.Name, b.Name)
})

b, err := yaml.Marshal(itemList)
if err != nil {
logger.Error(err.Error())
return err
Expand Down
2 changes: 1 addition & 1 deletion ...rvices/mysql/db-tools/dbactuator/pkg/components/peripheraltools/monitor/runtime_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ func (c *MySQLMonitorComp) GenerateRuntimeConfig() (err error) {
return nil
}

func generateRuntimeConfigIns(mmp *MySQLMonitorParam, instance *instanceInfo, rtap *components.RuntimeAccountParam) (err error) {
func generateRuntimeConfigIns(mmp *MySQLMonitorParam, instance *internal.InstanceInfo, rtap *components.RuntimeAccountParam) (err error) {
if instance.BkInstanceId <= 0 {
err = errors.Errorf(
"%s:%d invalid bk_instance_id: %d",
Expand Down
10 changes: 5 additions & 5 deletions dbm-services/mysql/db-tools/mysql-monitor/pkg/config/items_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@ import "slices"

// MonitorItem 监控项
type MonitorItem struct {
Name string `yaml:"name" validate:"required"`
Enable *bool `yaml:"enable" validate:"required"`
Schedule *string `yaml:"schedule"`
MachineType []string `yaml:"machine_type"`
Role []string `yaml:"role"`
Name string `json:"name" yaml:"name" validate:"required"`
Enable *bool `json:"enable" yaml:"enable" validate:"required"`
Schedule *string `json:"schedule" yaml:"schedule"`
MachineType []string `json:"machine_type" yaml:"machine_type"`
Role []string `json:"role" yaml:"role"`
}

// IsEnable 监控项启用
Expand Down
2 changes: 2 additions & 0 deletions dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_ha_standardize_flow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ def _build_proxy_sub(self, ips: Dict) -> SubProcess:
exec_ip=ip,
bk_cloud_id=bk_cloud_id,
get_mysql_payload_func=MysqlActPayload.get_deploy_mysql_monitor_payload.__name__,
cluster={"cluster_ids": self.data["infos"]["cluster_ids"]},
cluster_type=ClusterType.TenDBHA.value,
)
),
Expand Down Expand Up @@ -308,6 +309,7 @@ def _build_storage_sub(self, ips: Dict) -> SubProcess:
exec_ip=ip,
bk_cloud_id=bk_cloud_id,
get_mysql_payload_func=MysqlActPayload.get_deploy_mysql_monitor_payload.__name__,
cluster={"cluster_ids": self.data["infos"]["cluster_ids"]},
cluster_type=ClusterType.TenDBHA.value,
)
),
Expand Down
1 change: 1 addition & 0 deletions dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_cluster_standardize_flow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,7 @@ def _build_single_instance_sub(
exec_ip=ip,
bk_cloud_id=bk_cloud_id,
get_mysql_payload_func=MysqlActPayload.get_deploy_mysql_monitor_payload.__name__,
cluster={"cluster_ids": self.data["infos"]["cluster_ids"]},
cluster_type=ClusterType.TenDBCluster.value,
)
),
Expand Down
1 change: 1 addition & 0 deletions dbm-ui/backend/flow/engine/bamboo/scene/tendbsingle/standardize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,7 @@ def _build_storage_sub(self, ips: Dict) -> SubProcess:
exec_ip=ip,
bk_cloud_id=bk_cloud_id,
get_mysql_payload_func=MysqlActPayload.get_deploy_mysql_monitor_payload.__name__,
cluster={"cluster_ids": self.data["infos"]["cluster_ids"]},
cluster_type=ClusterType.TenDBSingle.value,
)
),
Expand Down
24 changes: 17 additions & 7 deletions dbm-ui/backend/flow/utils/mysql/mysql_act_playload.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from typing import Any, List

from django.conf import settings
from django.db.models import Q
from django.utils.translation import ugettext as _

from backend import env
Expand Down Expand Up @@ -1493,6 +1494,15 @@ def get_deploy_mysql_monitor_payload(self, **kwargs) -> dict:
部署mysql/proxy/spider事件监控程序
"""
mysql_monitor_pkg = Package.get_latest_package(version=MediumEnum.Latest, pkg_type=MediumEnum.MySQLMonitor)
cluster_ids = self.cluster.get("cluster_ids", [])
if not cluster_ids:
cluster_ids = list(
set(
Cluster.objects.filter(
Q(storageinstance__machine__ip=kwargs["ip"]) | Q(proxyinstance__machine__ip=kwargs["ip"])
).values_list("id", flat=True)
)
)

instances_info = []
machine = Machine.objects.get(ip=kwargs["ip"])
Expand All @@ -1513,7 +1523,7 @@ def get_deploy_mysql_monitor_payload(self, **kwargs) -> dict:
logger.info("config_items: {}".format(config_items))

if machine.machine_type == MachineType.PROXY.value:
for instance in ProxyInstance.objects.filter(machine__ip=kwargs["ip"]):
for instance in ProxyInstance.objects.filter(machine__ip=kwargs["ip"], cluster__id__in=cluster_ids):
cluster = instance.cluster.get()
instances_info.append(
{
Expand All @@ -1524,12 +1534,12 @@ def get_deploy_mysql_monitor_payload(self, **kwargs) -> dict:
"immute_domain": cluster.immute_domain,
"bk_instance_id": instance.bk_instance_id,
"db_module_id": instance.db_module_id,
"items_config": config_items["content"], # 监控配置放到实例信息里面, 为后面个性化配置做准备
# "items_config": config_items["content"], # 监控配置放到实例信息里面, 为后面个性化配置做准备
}
)
# 增加对安装spider监控的适配
elif machine.machine_type == MachineType.SPIDER.value:
for instance in ProxyInstance.objects.filter(machine__ip=kwargs["ip"]):
for instance in ProxyInstance.objects.filter(machine__ip=kwargs["ip"], cluster__id__in=cluster_ids):
cluster = instance.cluster.get()
instances_info.append(
{
Expand All @@ -1541,13 +1551,13 @@ def get_deploy_mysql_monitor_payload(self, **kwargs) -> dict:
"immute_domain": cluster.immute_domain,
"bk_instance_id": instance.bk_instance_id,
"db_module_id": instance.db_module_id,
"items_config": config_items["content"], # 监控配置放到实例信息里面, 为后面个性化配置做准备
# "items_config": config_items["content"], # 监控配置放到实例信息里面, 为后面个性化配置做准备
}
)

# 不同角色的mysql实例，role是否传对应的instance_inner_role ？
elif machine.machine_type in (MachineType.BACKEND.value, MachineType.SINGLE.value, MachineType.REMOTE.value):
for instance in StorageInstance.objects.filter(machine__ip=kwargs["ip"]):
for instance in StorageInstance.objects.filter(machine__ip=kwargs["ip"], cluster__id__in=cluster_ids):
cluster = instance.cluster.get()
instances_info.append(
{
Expand All @@ -1559,7 +1569,7 @@ def get_deploy_mysql_monitor_payload(self, **kwargs) -> dict:
"immute_domain": cluster.immute_domain,
"bk_instance_id": instance.bk_instance_id,
"db_module_id": instance.db_module_id,
"items_config": config_items["content"], # 监控配置放到实例信息里面, 为后面个性化配置做准备
# "items_config": config_items["content"], # 监控配置放到实例信息里面, 为后面个性化配置做准备
}
)
else:
Expand All @@ -1579,7 +1589,7 @@ def get_deploy_mysql_monitor_payload(self, **kwargs) -> dict:
"machine_type": machine.machine_type,
"bk_cloud_id": int(self.bk_cloud_id),
"instances_info": instances_info,
# "items_config": config_items["content"],
"items_config": config_items["content"],
},
},
}
Expand Down
Loading