From 9a612092f4086ea1ddccb0339ea0f3a3f5cb5fb8 Mon Sep 17 00:00:00 2001 From: xiaog Date: Thu, 12 Sep 2024 10:23:07 +0800 Subject: [PATCH] =?UTF-8?q?fix(mysql):=20=E4=BF=AE=E5=A4=8D=20proxy=20expo?= =?UTF-8?q?rter=20=E9=85=8D=E7=BD=AE=E6=B8=B2=E6=9F=93=E9=97=AE=E9=A2=98?= =?UTF-8?q?=20#6878?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../assets/migrations/000011_es_data.up.sql | 42 +++++----- dbm-services/common/go-pubpkg/cmutil/file.go | 12 +++ .../common/go-pubpkg/reportlog/report.go | 21 ++--- .../mysql/restore/recover_binlog.go | 5 +- .../mysql_proxy/install_mysql_proxy.go | 3 +- .../peripheraltools/dbbackup/backup_dir.go | 33 +++++++- .../pkg/util/db_table_filter/db_filter.go | 1 - .../src/backupexe/dumper_logical_mysqldump.go | 77 ++++++++++--------- .../pkg/src/dbareport/report_logger.go | 6 +- .../mysql-dbbackup/pkg/src/mysqlconn/conn.go | 11 +++ 10 files changed, 132 insertions(+), 79 deletions(-) diff --git a/dbm-services/common/db-config/assets/migrations/000011_es_data.up.sql b/dbm-services/common/db-config/assets/migrations/000011_es_data.up.sql index 0053a2c0b1..0371596b4e 100644 --- a/dbm-services/common/db-config/assets/migrations/000011_es_data.up.sql +++ b/dbm-services/common/db-config/assets/migrations/000011_es_data.up.sql @@ -34,7 +34,7 @@ INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (264,'es','dbconf','7.14.2','elasticsearch.yml配置','ES-7.14.2','plat,app,cluster','cluster',1,1,0,NULL,5,365,0,'es配置文件','2022-09-05 17:00:13','2023-06-30 17:27:03',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (303,'es','dbconf','7.2.0','elasticsearch.yml配置','ES-7.2.0','plat,app,cluster','cluster',1,1,0,NULL,5,365,0,'es配置文件','2022-09-05 17:00:13','2023-06-30 17:27:03',''); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (298,'es','dbconf','7.7.0','elasticsearch.yml配置','ES-7.7.0','plat,app,cluster','cluster',1,1,0,NULL,5,365,0,'es配置文件','2022-09-05 17:00:13','2023-06-30 17:27:03',''); -INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (358,'es','dbconf','8.15.0','elasticsearch.yml配置','ES-8.15.0','plat,app,cluster','cluster',1,1,0,NULL,5,365,0,'es配置文件','2022-09-05 17:00:13','2023-06-30 17:27:03',NULL); +INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (358,'es','dbconf','8.15.1','elasticsearch.yml配置','ES-8.15.1','plat,app,cluster','cluster',1,1,0,NULL,5,365,0,'es配置文件','2022-09-05 17:00:13','2024-09-09 10:37:14',NULL); /*!50112 SET @disable_bulk_load = IF (@is_rocksdb_supported, 'SET SESSION rocksdb_bulk_load = @old_rocksdb_bulk_load', 'SET @dummy_rocksdb_bulk_load = 0') */; /*!50112 PREPARE s FROM @disable_bulk_load */; /*!50112 EXECUTE s */; @@ -276,26 +276,26 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (20733,'es','dbconf','7.7.0','processors','STRING','{{.Processors}}',NULL,'',2,0,0,0,1,'{{processors}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-01-22 15:10:54',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (20734,'es','dbconf','7.7.0','thread_pool.write.queue_size','STRING','1000',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2022-11-18 11:30:10',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (20735,'es','dbconf','7.7.0','xpack.monitoring.collection.enabled','STRING','false','true | false','ENUM',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2022-11-18 11:30:10',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25436,'es','dbconf','8.15.0','action.destructive_requires_name','STRING','false','true | false','ENUM',1,0,0,0,1,'NULL','NULL','NULL',-1,'NULL',NULL,'2022-09-05 21:31:41','2022-11-18 11:30:10',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25413,'es','dbconf','8.15.0','bootstrap.memory_lock','STRING','true','true | false','ENUM',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2022-11-18 11:30:10',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25415,'es','dbconf','8.15.0','cluster.initial_master_nodes','STRING','{{.ClusterInitialMasterNodes}}',NULL,'',2,0,0,0,1,'{{master_nodename}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-01-22 15:10:54',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25416,'es','dbconf','8.15.0','cluster.name','STRING','{{.ClusterName}}',NULL,'',2,0,0,0,1,'{{cluster_name}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 17:24:58','2024-01-22 11:44:55',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25417,'es','dbconf','8.15.0','cluster.routing.allocation.same_shard.host','STRING','true','true | false','ENUM',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:42','2022-11-18 11:30:10',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25418,'es','dbconf','8.15.0','discovery.seed_hosts','STRING','{{.DiscoverySeedHosts}}',NULL,'',2,0,0,0,1,'{{master_ip}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-01-22 15:10:54',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25419,'es','dbconf','8.15.0','http.max_content_length','STRING','2000mb',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-01-31 07:29:06',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25420,'es','dbconf','8.15.0','http.port','INT','{{.HTTPPort}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-01-22 16:27:26',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25421,'es','dbconf','8.15.0','indices.recovery.max_bytes_per_sec','STRING','200mb',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2022-11-18 11:30:10',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25422,'es','dbconf','8.15.0','network.host','STRING','{{.NetworkHost}}',NULL,'',2,0,0,0,1,'{{host}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-01-22 15:10:54',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25423,'es','dbconf','8.15.0','network.publish_host','STRING','{{.NetworkPublishhost}}',NULL,'',2,0,0,0,1,'{{host}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-01-22 15:10:54',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25424,'es','dbconf','8.15.0','node.attr.rack_id','INT','1',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2022-11-18 11:30:10',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25425,'es','dbconf','8.15.0','node.attr.tag','STRING','{{.NodeAttrTag}}','hot | cold','ENUM',1,0,0,0,1,'hot',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-01-22 16:27:25',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25430,'es','dbconf','8.15.0','node.name','STRING','{{.NodeName}}',NULL,'',2,0,0,0,1,'{{node_name}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:31','2024-01-22 15:10:54',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25433,'es','dbconf','8.15.0','node.processors','STRING','{{.Processors}}',NULL,'',2,0,0,0,1,'{{processors}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-05 22:03:39',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25426,'es','dbconf','8.15.0','node.roles','STRING','{{.NodeRoles}}',NULL,'',2,0,0,0,1,'NULL',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-05 22:11:40',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25431,'es','dbconf','8.15.0','path.data','STRING','{{.PathData}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'数据路径','2022-09-05 21:31:41','2022-11-18 11:30:10',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25432,'es','dbconf','8.15.0','path.logs','STRING','{{.PathLogs}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'日志路径','2022-09-05 21:31:41','2022-11-18 11:30:10',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25434,'es','dbconf','8.15.0','thread_pool.write.queue_size','STRING','1000',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2022-11-18 11:30:10',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25435,'es','dbconf','8.15.0','xpack.monitoring.collection.enabled','STRING','false','true | false','ENUM',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2022-11-18 11:30:10',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25436,'es','dbconf','8.15.1','action.destructive_requires_name','STRING','false','true | false','ENUM',1,0,0,0,1,'NULL','NULL','NULL',-1,'NULL',NULL,'2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25413,'es','dbconf','8.15.1','bootstrap.memory_lock','STRING','true','true | false','ENUM',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25415,'es','dbconf','8.15.1','cluster.initial_master_nodes','STRING','{{.ClusterInitialMasterNodes}}',NULL,'',2,0,0,0,1,'{{master_nodename}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25416,'es','dbconf','8.15.1','cluster.name','STRING','{{.ClusterName}}',NULL,'',2,0,0,0,1,'{{cluster_name}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 17:24:58','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25417,'es','dbconf','8.15.1','cluster.routing.allocation.same_shard.host','STRING','true','true | false','ENUM',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:42','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25418,'es','dbconf','8.15.1','discovery.seed_hosts','STRING','{{.DiscoverySeedHosts}}',NULL,'',2,0,0,0,1,'{{master_ip}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25419,'es','dbconf','8.15.1','http.max_content_length','STRING','2000mb',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25420,'es','dbconf','8.15.1','http.port','INT','{{.HTTPPort}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25421,'es','dbconf','8.15.1','indices.recovery.max_bytes_per_sec','STRING','200mb',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25422,'es','dbconf','8.15.1','network.host','STRING','{{.NetworkHost}}',NULL,'',2,0,0,0,1,'{{host}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25423,'es','dbconf','8.15.1','network.publish_host','STRING','{{.NetworkPublishhost}}',NULL,'',2,0,0,0,1,'{{host}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25424,'es','dbconf','8.15.1','node.attr.rack_id','INT','1',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25425,'es','dbconf','8.15.1','node.attr.tag','STRING','{{.NodeAttrTag}}','hot | cold','ENUM',1,0,0,0,1,'hot',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25430,'es','dbconf','8.15.1','node.name','STRING','{{.NodeName}}',NULL,'',2,0,0,0,1,'{{node_name}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:31','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25433,'es','dbconf','8.15.1','node.processors','STRING','{{.Processors}}',NULL,'',2,0,0,0,1,'{{processors}}',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25426,'es','dbconf','8.15.1','node.roles','STRING','{{.NodeRoles}}',NULL,'',2,0,0,0,1,'NULL',NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25431,'es','dbconf','8.15.1','path.data','STRING','{{.PathData}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'数据路径','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25432,'es','dbconf','8.15.1','path.logs','STRING','{{.PathLogs}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'日志路径','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25434,'es','dbconf','8.15.1','thread_pool.write.queue_size','STRING','1000',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-09 10:40:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25435,'es','dbconf','8.15.1','xpack.monitoring.collection.enabled','STRING','false','true | false','ENUM',1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'es配置','2022-09-05 21:31:41','2024-09-09 10:40:54',0); /*!50112 SET @disable_bulk_load = IF (@is_rocksdb_supported, 'SET SESSION rocksdb_bulk_load = @old_rocksdb_bulk_load', 'SET @dummy_rocksdb_bulk_load = 0') */; /*!50112 PREPARE s FROM @disable_bulk_load */; /*!50112 EXECUTE s */; diff --git a/dbm-services/common/go-pubpkg/cmutil/file.go b/dbm-services/common/go-pubpkg/cmutil/file.go index efcc4a9979..a4a34554b6 100644 --- a/dbm-services/common/go-pubpkg/cmutil/file.go +++ b/dbm-services/common/go-pubpkg/cmutil/file.go @@ -55,6 +55,18 @@ func IsDirectory(path string) bool { return fileInfo.IsDir() } +// IsSymLinkFile 文件是否是软连 +func IsSymLinkFile(path string) (bool, error) { + info, err := os.Lstat(path) + if err != nil { + return false, err + } else if info.Mode()&os.ModeSymlink > 0 { // 是软链 + return true, nil + } else { + return false, nil + } +} + // GetFileSize get file size from os func GetFileSize(path string) int64 { f, err := os.Stat(path) diff --git a/dbm-services/common/go-pubpkg/reportlog/report.go b/dbm-services/common/go-pubpkg/reportlog/report.go index 32d3fb8167..004a65491c 100644 --- a/dbm-services/common/go-pubpkg/reportlog/report.go +++ b/dbm-services/common/go-pubpkg/reportlog/report.go @@ -39,14 +39,19 @@ func defaultLoggerOpt() *LoggerOption { } // NewReporter init reporter for logFile path +// reportDir maybe like '/home/mysql/dbareport/mysql/dbbackup' func NewReporter(reportDir, filename string, logOpt *LoggerOption) (*Reporter, error) { logFilePath := filepath.Join(reportDir, filename) var reporter *Reporter = &Reporter{ log: &log.Logger{}, } + if reportDir == "" { return nil, errors.Errorf("invalid reportDir:%s", reportDir) - } else if !cmutil.IsDirectory(reportDir) { + } else if link, _ := cmutil.IsSymLinkFile(reportDir); link { + _ = os.Remove(reportDir) // /home/mysql/dbareport 不使用软连 + } + if !cmutil.IsDirectory(reportDir) { if err := os.MkdirAll(reportDir, 0755); err != nil { return nil, errors.Wrap(err, "create report path") } @@ -59,19 +64,7 @@ func NewReporter(reportDir, filename string, logOpt *LoggerOption) (*Reporter, e f.Close() } } - /* - statusFile := "binlog_status.log" - statusLogger := &lumberjack.Logger{ - Filename: filepath.Join(viper.GetString("report.filepath"), statusFile), - MaxSize: 5, // MB - MaxBackups: 10, - MaxAge: 30, // days - Compress: true, // disabled by default - } - statusReporter := new(log.Logger) - statusReporter.SetOutput(statusLogger) - reporter.Status = &logPrint{log: statusReporter} - */ + if logOpt == nil { logOpt = defaultLoggerOpt() } diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/recover_binlog.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/recover_binlog.go index fe7e56955b..1034f4bf8f 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/recover_binlog.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/recover_binlog.go @@ -641,8 +641,9 @@ func (r *RecoverBinlog) PreCheck() error { if err = r.buildBinlogOptions(); err != nil { return err } - err = r.checkBinlogFiles() - logger.Warn("check binlog files error: %s. try to get binlog file from recover dir", err.Error()) + if err = r.checkBinlogFiles(); err != nil { + logger.Warn("check binlog files error: %s. try to get binlog file from recover dir", err.Error()) + } if errors.Is(err, ErrorBinlogMissing) { nameParts := strings.Split(r.BinlogFiles[0], ".") if binlogFiles, err := r.GetBinlogFilesFromDir(r.BinlogDir, nameParts[0]+"."); err != nil { diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql_proxy/install_mysql_proxy.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql_proxy/install_mysql_proxy.go index a80a5393af..5a8e76e959 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql_proxy/install_mysql_proxy.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql_proxy/install_mysql_proxy.go @@ -380,6 +380,7 @@ func (i *InstallMySQLProxyComp) initOneProxyAdminAccount(port Port) (err error) // 回档也会调用 install_mysql，但可能不会 install_monitor，为了避免健康误报，这个 install_mysql 阶段也渲染 exporter cnf func (i *InstallMySQLProxyComp) CreateExporterCnf() (err error) { for _, port := range i.InsPorts { + logger.Info("generate exporter config for %d", port) err = i.createExporterCnfIns(port) if err != nil { logger.Error("create exporter cnf failed %s", err.Error()) @@ -396,7 +397,7 @@ func (i *InstallMySQLProxyComp) createExporterCnfIns(port int) (err error) { fmt.Sprintf("exporter_%d.cnf", port), ) - f, err := os.OpenFile(exporterConfigPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + f, err := os.OpenFile(exporterConfigPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) if err != nil { logger.Error(err.Error()) return err diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/peripheraltools/dbbackup/backup_dir.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/peripheraltools/dbbackup/backup_dir.go index 5ad6433d3f..efe3b3b5e4 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/peripheraltools/dbbackup/backup_dir.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/peripheraltools/dbbackup/backup_dir.go @@ -1,10 +1,15 @@ package dbbackup import ( - "dbm-services/common/go-pubpkg/logger" - "dbm-services/mysql/db-tools/dbactuator/pkg/util/osutil" "fmt" "os" + + "github.com/pkg/errors" + + "dbm-services/common/go-pubpkg/cmutil" + "dbm-services/common/go-pubpkg/logger" + "dbm-services/mysql/db-tools/dbactuator/pkg/core/cst" + "dbm-services/mysql/db-tools/dbactuator/pkg/util/osutil" ) func (c *NewDbBackupComp) InitBackupDir() (err error) { @@ -26,5 +31,29 @@ func (c *NewDbBackupComp) InitBackupDir() (err error) { if err != nil { return fmt.Errorf("execute [%s] get an error:%s,%w", cmd, output, err) } + return c.initReportDir() +} + +func (c *NewDbBackupComp) initReportDir() (err error) { + if c.Params.UntarOnly { + logger.Info("untar_only=true do not need initReportDir") + return nil + } + // redis 会污染 /home/mysql/dbareport，建立成软连 + if isLink, _ := cmutil.IsSymLinkFile(cst.DBAReportBase); isLink { + _ = os.Remove(cst.DBAReportBase) + } + reportDir := c.Params.Configs["Public"]["ReportPath"] + if _, err := os.Stat(reportDir); os.IsNotExist(err) { + err := os.MkdirAll(reportDir, 0755) + if err != nil { + return errors.WithMessagef(err, "execute [mkdir -p %s]", reportDir) + } + } + cmd := fmt.Sprintf("chown -R mysql.mysql %s", reportDir) + output, err := osutil.ExecShellCommand(false, cmd) + if err != nil { + return fmt.Errorf("execute [%s] get an error:%s,%w", cmd, output, err) + } return } diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/util/db_table_filter/db_filter.go b/dbm-services/mysql/db-tools/dbactuator/pkg/util/db_table_filter/db_filter.go index 8923835e6b..53f1663d97 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/util/db_table_filter/db_filter.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/util/db_table_filter/db_filter.go @@ -36,7 +36,6 @@ func (c *DbTableFilter) GetDbsByConn(conn *sqlx.Conn) ([]string, error) { if err != nil { return nil, err } - var dbs []string err = conn.SelectContext(context.Background(), &dbs, `SHOW DATABASES`) if err != nil { diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go index 97ce27352c..bec1d319cc 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go @@ -12,7 +12,6 @@ package backupexe import ( "context" - "database/sql" "fmt" "os" "os/exec" @@ -21,10 +20,12 @@ import ( "strings" "time" + "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/spf13/cast" "dbm-services/common/go-pubpkg/cmutil" + "dbm-services/mysql/db-tools/dbactuator/pkg/util/db_table_filter" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport" @@ -37,7 +38,7 @@ type LogicalDumperMysqldump struct { cnf *config.BackupConfig dbbackupHome string backupInfo dbareport.IndexContent // for mysqldump backup - dbConn *sql.DB + dbConn *sqlx.Conn } // initConfig initializes the configuration for the logical dumper[mysqldump] @@ -60,40 +61,45 @@ func (l *LogicalDumperMysqldump) buildArgsTableFilter() (args []string, err erro dbListExclude := strings.Split(l.cnf.LogicalBackup.ExcludeDatabases, ",") tbListExclude := strings.Split(l.cnf.LogicalBackup.ExcludeTables, ",") - if len(l.cnf.LogicalBackup.Databases) > 0 && len(dbList) >= 2 { - if len(l.cnf.LogicalBackup.Tables) == 0 { // 仅有 databases - args = append(args, "--databases") - args = append(args, dbList...) - } else { - return nil, errors.Errorf("mysqldump --tables cannot be used with multi databases") + var dbListFiltered []string + if filter, err := db_table_filter.NewFilter(dbList, tbList, dbListExclude, tbListExclude); err != nil { + return nil, err + } else { + l.dbConn, err = mysqlconn.InitConnx(&l.cnf.Public, context.Background()) + if err != nil { + return nil, err } - } else if len(l.cnf.LogicalBackup.Databases) > 0 && len(dbList) == 1 { // 只有一个 db，可以指定 table - args = append(args, l.cnf.LogicalBackup.Databases) - args = append(args, tbList...) - } else if len(l.cnf.LogicalBackup.ExcludeDatabases) > 0 { - if len(l.cnf.LogicalBackup.ExcludeTables) == 0 { - logger.Log.Info("get database list from db to exclude:") - l.dbConn, err = mysqlconn.InitConn(&l.cnf.Public) - if err != nil { - return nil, err - } - defer func() { - _ = l.dbConn.Close() - }() - if allDatabases, err := mysqlconn.GetDatabases("", l.dbConn); err != nil { - return nil, err - } else { - dbListExclude = append(dbListExclude, "information_schema", "performance_schema") - for _, d := range dbListExclude { - allDatabases = cmutil.StringsRemove(allDatabases, d) - } - args = append(args, "--databases", strings.Join(allDatabases, " ")) - } - } else { - return nil, errors.Errorf("mysqldump exclude is not allowed, exclude-databases=%s, exclude-tables=%s", - dbListExclude, tbListExclude) + defer func() { + _ = l.dbConn.Close() + }() + dbListFiltered, err = filter.GetDbsByConn(l.dbConn) + if err != nil { + return nil, errors.WithMessage(err, "get database from instance") } } + logger.Log.Infof("database filtered:%v, tables:%v, excludeTables:%v", dbListFiltered, tbList, tbListExclude) + + // mysqldump 条件： + // databases, exclude-databases 允许模糊匹配 + // tables 不允许模糊匹配 + // exclude-tables 只能为空 + // 简单说 mysqldump 支持 --database db1 db2 db3, 或者 db1 table1 table2 + if l.cnf.LogicalBackup.Tables == "*" && l.cnf.LogicalBackup.ExcludeTables == "" { + args = append(args, "--databases") + args = append(args, dbListFiltered...) + } else if len(dbListFiltered) == 1 && l.cnf.LogicalBackup.ExcludeTables == "" { // 只有一个 db，可以指定 table + args = append(args, dbListFiltered...) + args = append(args, tbList...) + } else if strings.Contains(l.cnf.LogicalBackup.Tables, "%") { + return nil, errors.Errorf("mysqldump does not support table like %s", l.cnf.LogicalBackup.Tables) + } else if l.cnf.LogicalBackup.ExcludeTables != "" { + return nil, errors.Errorf("mysqldump does not support table exclude %s", l.cnf.LogicalBackup.ExcludeTables) + } else if len(dbListFiltered) >= 2 && len(tbList) > 0 { + return nil, errors.Errorf("mysqldump --tables cannot be used with multi databases %v", dbListFiltered) + } else { + return nil, errors.Errorf("mysqldump exclude not not support, exclude-databases=%s, exclude-tables=%s", + dbListExclude, tbListExclude) + } return args, nil } @@ -169,9 +175,10 @@ func (l *LogicalDumperMysqldump) Execute(enableTimeOut bool) (err error) { "--skip-opt", "--create-options", "--extended-insert", "--quick", "--single-transaction", "--master-data=2", "--max-allowed-packet=1G", "--no-autocommit", - "--set-gtid-purged=off", - //"--default-character-set // + "--hex-blob", + // "--set-gtid-purged=off", // 5.7 需要 } + args = append(args, "--default-character-set", l.cnf.Public.MysqlCharset) if l.cnf.Public.MysqlRole == cst.RoleSlave { args = append(args, []string{ "--dump-slave=2", // will stop slave sql_thread diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/report_logger.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/report_logger.go index 154d820e2f..64ec56820f 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/report_logger.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/report_logger.go @@ -55,18 +55,18 @@ func NewLogReporter(reportDir string) (*ReportLogger, error) { } resultReport, err := reportlog.NewReporter(reportDir, "backup_result.log", &logOpt) if err != nil { - logger.Log.Warn("fail to init resultReporter:%s", err.Error()) + logger.Log.Warn("fail to init resultReporter:", err.Error()) return nil, errors.WithMessage(err, "fail to init resultReporter") } filesReport, err := reportlog.NewReporter(filepath.Join(reportDir, "result"), "dbareport_result.log", &logOpt) if err != nil { - logger.Log.Warn("fail to init statusReporter:%s", err.Error()) + logger.Log.Warn("fail to init statusReporter:", err.Error()) //filesReport.Disable = true return nil, errors.WithMessage(err, "fail to init statusReporter") } statusReport, err := reportlog.NewReporter(filepath.Join(reportDir, "status"), "backup_status.log", &logOpt) if err != nil { - logger.Log.Warn("fail to init statusReporter:%s", err.Error()) + logger.Log.Warn("fail to init statusReporter:", err.Error()) //statusReport.Disable = true return nil, errors.WithMessage(err, "fail to init statusReporter") } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn/conn.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn/conn.go index a417523e8d..7bdb6e6a5a 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn/conn.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn/conn.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/spf13/cast" @@ -37,6 +38,16 @@ func InitConn(cfg *config.Public) (*sql.DB, error) { return db, nil } +// InitConnx create mysql connection of sqlx +func InitConnx(cfg *config.Public, ctx context.Context) (*sqlx.Conn, error) { + if db, err := InitConn(cfg); err != nil { + return nil, err + } else { + dbx := sqlx.NewDb(db, "mysql") + return dbx.Connx(ctx) + } +} + // GetSingleGlobalVar get single global variable func GetSingleGlobalVar(variableName string, dbh *sql.DB) (val string, err error) { var varName, varValue string