feat(backend): 日志创建支持自定义 ES 集群 #2353

TencentBlueKing · Dec 7, 2023 · 4274b37 · 4274b37
1 parent d21ce5b
commit 4274b37
Show file tree

Hide file tree

Showing 26 changed files with 178 additions and 172 deletions.
diff --git a/dbm-services/common/db-dns/dns-api/Dockerfile b/dbm-services/common/db-dns/dns-api/Dockerfile
@@ -1,12 +1,9 @@
 FROM centos:7
-ARG SRV_NAME
-MAINTAINER vincixu vincixu<vincixu@tencent.com>
 
 RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
 RUN echo "Asia/Shanghai" > /etc/timezone
 
 WORKDIR /
-COPY ./$SRV_NAME .
-COPY ./conf ./conf
+COPY ./bk-dnsapi .
 
 ENTRYPOINT ["/bk-dnsapi"]
diff --git a/dbm-services/mysql/db-partition/Makefile b/dbm-services/mysql/db-partition/Makefile
@@ -5,12 +5,7 @@ COMMAND_NAME = partition
 VER = latest
 CURRENT_VERSION = $(VER)
 TEST_VERSION = test-$(VER)
-NAMESPACE = sccmsp
-DH_USER = Ex_vincixu
-DH_PASS = aA456123
-DH_URL = mirrors.tencent.com
 export GOOS = linux
-#export GOOS = darwin
 BUILD_PATH = .
 
 all: build
@@ -21,18 +16,12 @@ build:clean
 
 publish:build
 	docker build --build-arg SRV_NAME=$(COMMAND_NAME) --rm -t $(SRV_NAME):$(CURRENT_VERSION) .
-	docker tag $(SRV_NAME):$(CURRENT_VERSION) $(DH_URL)/${NAMESPACE}/$(SRV_NAME):$(CURRENT_VERSION)
-	docker push $(DH_URL)/${NAMESPACE}/$(SRV_NAME):$(CURRENT_VERSION)
 
 test:build
 	docker build --build-arg SRV_NAME=$(COMMAND_NAME) --rm -t $(SRV_NAME):$(TEST_VERSION) .
-	docker tag $(SRV_NAME):$(TEST_VERSION) $(DH_URL)/${NAMESPACE}/$(SRV_NAME):$(TEST_VERSION)
-	#docker push $(DH_URL)/${NAMESPACE}/$(SRV_NAME):$(TEST_VERSION)
 
 test_mac:build
 	docker build --build-arg SRV_NAME=$(COMMAND_NAME) --rm -t $(SRV_NAME):$(TEST_VERSION) .
-	docker tag $(SRV_NAME):$(TEST_VERSION) $(DH_URL)/${NAMESPACE}/$(SRV_NAME):$(TEST_VERSION)
-	docker push $(DH_URL)/${NAMESPACE}/$(SRV_NAME):$(TEST_VERSION)
 
 curl:
 	echo curl -XGET -H "Content-Type: application/json" http://127.0.0.1:8080/user

diff --git a/dbm-services/mysql/db-priv/Makefile b/dbm-services/mysql/db-priv/Makefile
@@ -5,12 +5,7 @@ COMMAND_NAME = privservice
 VER = latest
 CURRENT_VERSION = release-$(VER)
 TEST_VERSION = test-$(VER)
-NAMESPACE = sccmsp
-DH_USER = Ex_vincixu
-DH_PASS = aA456123
-DH_URL = mirrors.tencent.com
 export GOOS = linux
-#export GOOS = darwin
 BUILD_PATH = .
 
 all: build
@@ -21,18 +16,12 @@ build:clean
 
 publish:build
 	docker build --build-arg SRV_NAME=$(COMMAND_NAME) --rm -t $(SRV_NAME):$(CURRENT_VERSION) .
-	docker tag $(SRV_NAME):$(CURRENT_VERSION) $(DH_URL)/${NAMESPACE}/$(SRV_NAME):$(CURRENT_VERSION)
-	docker push $(DH_URL)/${NAMESPACE}/$(SRV_NAME):$(CURRENT_VERSION)
 
 test:build
 	docker build --build-arg SRV_NAME=$(COMMAND_NAME) --rm -t $(SRV_NAME):$(TEST_VERSION) .
-	docker tag $(SRV_NAME):$(TEST_VERSION) $(DH_URL)/${NAMESPACE}/$(SRV_NAME):$(TEST_VERSION)
-	# docker push $(DH_URL)/${NAMESPACE}/$(SRV_NAME):$(TEST_VERSION)
 
 test_mac:build
 	docker build --build-arg SRV_NAME=$(COMMAND_NAME) --rm -t $(SRV_NAME):$(TEST_VERSION) .
-	docker tag $(SRV_NAME):$(TEST_VERSION) $(DH_URL)/${NAMESPACE}/$(SRV_NAME):$(TEST_VERSION)
-	docker push $(DH_URL)/${NAMESPACE}/$(SRV_NAME):$(TEST_VERSION)
 
 curl:
 	echo curl -XGET -H "Content-Type: application/json" http://127.0.0.1:8080/user

diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/backup_stm_log.json b/dbm-ui/backend/dbm_init/json_files/bklog/backup_stm_log.json
@@ -7,6 +7,16 @@
     "etl_config": "bk_log_json",
     "description": "backup_stm_log",
     "environment": "linux",
+    "data_encoding": "UTF-8",
+    "retention": 7,
+    "es_shards": 3,
+    "target_object_type": "HOST",
+    "target_node_type": "TOPO",
+    "target_nodes": [
+        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
+        {% endfor %}
+    ],
+    {% if storage_cluster_id %}"storage_cluster_id": {{ storage_cluster_id }},{% endif %}
     "params": {
         "paths": [
             "/data/backup_stm/register/*.log"
@@ -749,14 +759,5 @@
             "is_delete": false,
             "is_dimension": true
         }
-    ],
-    "data_encoding": "UTF-8",
-    "retention": 7,
-    "es_shards": 3,
-    "target_object_type": "HOST",
-    "target_node_type": "TOPO",
-    "target_nodes": [
-        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
-        {% endfor %}
     ]
 }
diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/dbm_dbactuator.json b/dbm-ui/backend/dbm_init/json_files/bklog/dbm_dbactuator.json
@@ -5,6 +5,16 @@
     "collector_scenario_id": "row",
     "category_id": "os",
     "etl_config": "bk_log_text",
+    "data_encoding": "UTF-8",
+    "retention": 7,
+    "es_shards": 3,
+    "target_object_type": "HOST",
+    "target_node_type": "TOPO",
+    "target_nodes": [
+        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
+        {% endfor %}
+    ],
+    {% if storage_cluster_id %}"storage_cluster_id": {{ storage_cluster_id }},{% endif %}
     "params": {
         "paths": ["/data/install/dbactuator*/logs/*.log"],
         "conditions": {
@@ -17,14 +27,5 @@
         "tail_files": true,
         "ignore_older": 2678400,
         "max_bytes": 204800
-    },
-    "data_encoding": "UTF-8",
-    "retention": 7,
-    "es_shards": 3,
-    "target_object_type": "HOST",
-    "target_node_type": "TOPO",
-    "target_nodes": [
-        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
-        {% endfor %}
-    ]
+    }
 }
diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/mysql_backup_result.json b/dbm-ui/backend/dbm_init/json_files/bklog/mysql_backup_result.json
@@ -5,6 +5,16 @@
     "collector_scenario_id": "row",
     "category_id": "host_process",
     "etl_config": "bk_log_json",
+    "data_encoding": "UTF-8",
+    "retention": 7,
+    "es_shards": 3,
+    "target_object_type": "HOST",
+    "target_node_type": "TOPO",
+    "target_nodes": [
+        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
+        {% endfor %}
+    ],
+    {% if storage_cluster_id %}"storage_cluster_id": {{ storage_cluster_id }},{% endif %}
     "params": {
         "paths": ["/home/mysql/dbareport/mysql/dbbackup/result/*.log"],
         "conditions": {
@@ -749,14 +759,5 @@
             "is_delete": false,
             "is_dimension": true
         }
-    ],
-    "data_encoding": "UTF-8",
-    "retention": 7,
-    "es_shards": 3,
-    "target_object_type": "HOST",
-    "target_node_type": "TOPO",
-    "target_nodes": [
-        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
-        {% endfor %}
     ]
 }
diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/mysql_binlog_result.json b/dbm-ui/backend/dbm_init/json_files/bklog/mysql_binlog_result.json
@@ -5,6 +5,16 @@
     "collector_scenario_id": "row",
     "category_id": "host_process",
     "etl_config": "bk_log_json",
+    "data_encoding": "UTF-8",
+    "retention": 7,
+    "es_shards": 3,
+    "target_object_type": "HOST",
+    "target_node_type": "TOPO",
+    "target_nodes": [
+        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
+        {% endfor %}
+    ],
+    {% if storage_cluster_id %}"storage_cluster_id": {{ storage_cluster_id }},{% endif %}
     "etl_params": {
         "retain_original_text": true
     },
@@ -381,14 +391,5 @@
         "tail_files": true,
         "ignore_older": 2678400,
         "max_bytes": 204800
-    },
-    "data_encoding": "UTF-8",
-    "retention": 7,
-    "es_shards": 3,
-    "target_object_type": "HOST",
-    "target_node_type": "TOPO",
-    "target_nodes": [
-        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
-        {% endfor %}
-    ]
+    }
 }
diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/mysql_checksum_result.json b/dbm-ui/backend/dbm_init/json_files/bklog/mysql_checksum_result.json
@@ -5,6 +5,16 @@
     "collector_scenario_id": "row",
     "category_id": "host_process",
     "etl_config": "bk_log_text",
+    "data_encoding": "UTF-8",
+    "retention": 7,
+    "es_shards": 3,
+    "target_object_type": "HOST",
+    "target_node_type": "TOPO",
+    "target_nodes": [
+        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
+        {% endfor %}
+    ],
+    {% if storage_cluster_id %}"storage_cluster_id": {{ storage_cluster_id }},{% endif %}
     "params": {
         "paths": ["/home/mysql/dbareport/checksum/*.log"],
         "conditions": {
@@ -17,14 +27,5 @@
         "tail_files": true,
         "ignore_older": 2678400,
         "max_bytes": 204800
-    },
-    "data_encoding": "UTF-8",
-    "retention": 7,
-    "es_shards": 3,
-    "target_object_type": "HOST",
-    "target_node_type": "TOPO",
-    "target_nodes": [
-        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
-        {% endfor %}
-    ]
+    }
 }
diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/mysql_slowlog.json b/dbm-ui/backend/dbm_init/json_files/bklog/mysql_slowlog.json
@@ -5,6 +5,16 @@
     "collector_scenario_id": "section",
     "category_id": "host_process",
     "etl_config": "bk_log_regexp",
+    "data_encoding": "UTF-8",
+    "retention": 7,
+    "es_shards": 3,
+    "target_object_type": "SERVICE",
+    "target_node_type": "TOPO",
+    "target_nodes": [
+        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
+        {% endfor %}
+    ],
+    {% if storage_cluster_id %}"storage_cluster_id": {{ storage_cluster_id }},{% endif %}
     "etl_params": {
         "retain_original_text": true,
         "separator_regexp":"#\\s+User@Host:\\s+(?P<user>\\w+)\\[\\w+\\]\\s+@\\s+(?P<host>(\\w|\\[|\\]|\\.|\\s)+)\\s+Id:\\s+(?P<thread_id>\\d+)\\n#\\s+Query_time:\\s+(?P<query_time>\\w+\\.?\\w+)\\s+Lock_time:\\s+(?P<lock_time>\\w+\\.?\\w+)\\s+Rows_sent:\\s+(?P<rows_sent>\\d+)\\s+Rows_examined:\\s+(?P<rows_examined>\\d+)\\n(use\\s+(?P<use>.*);\\n)?SET\\s+timestamp=(?P<sql_timestamp>\\d+);\\n(?P<sql_text>(.|\\n)*)"
@@ -213,14 +223,5 @@
             },
             {"key": "$item", "value": "scope"}
         ]
-    },
-    "data_encoding": "UTF-8",
-    "retention": 7,
-    "es_shards": 3,
-    "target_object_type": "SERVICE",
-    "target_node_type": "TOPO",
-    "target_nodes": [
-        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
-        {% endfor %}
-    ]
+    }
 }
diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/redis_bigkey.json b/dbm-ui/backend/dbm_init/json_files/bklog/redis_bigkey.json
@@ -9,6 +9,15 @@
     "etl_params": {
         "retain_original_text": true
     },
+    "retention": 7,
+    "es_shards": 3,
+    "target_object_type": "HOST",
+    "target_node_type": "TOPO",
+    "target_nodes": [
+        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
+        {% endfor %}
+    ],
+    {% if storage_cluster_id %}"storage_cluster_id": {{ storage_cluster_id }},{% endif %}
     "params": {
         "paths": [
             "/home/mysql/dbareport/keylifecycle/redis_bigkey_*.log"
@@ -219,14 +228,5 @@
             "is_dimension": false,
             "previous_type": "int"
         }
-    ],
-    "need_assessment": false,
-    "retention": 7,
-    "es_shards": 3,
-    "target_object_type": "HOST",
-    "target_node_type": "TOPO",
-    "target_nodes": [
-        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
-        {% endfor %}
     ]
 }
diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/redis_binlog_backup_result.json b/dbm-ui/backend/dbm_init/json_files/bklog/redis_binlog_backup_result.json
@@ -6,6 +6,16 @@
     "collector_config_name_en": "redis_binlog_backup_result",
     "collector_scenario_id": "row",
     "category_id": "host_process",
+    "need_assessment": false,
+    "retention": 7,
+    "es_shards": 3,
+    "target_object_type": "HOST",
+    "target_node_type": "TOPO",
+    "target_nodes": [
+        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
+        {% endfor %}
+    ],
+    {% if storage_cluster_id %}"storage_cluster_id": {{ storage_cluster_id }},{% endif %}
     "etl_params": {
         "retain_original_text": true
     },
@@ -363,14 +373,5 @@
             "is_dimension": false,
             "previous_type": "string"
         }
-    ],
-    "need_assessment": false,
-    "retention": 7,
-    "es_shards": 3,
-    "target_object_type": "HOST",
-    "target_node_type": "TOPO",
-    "target_nodes": [
-        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
-        {% endfor %}
     ]
 }
diff --git a/dbm-ui/backend/dbm_init/json_files/bklog/redis_cluster_nodes_result.json b/dbm-ui/backend/dbm_init/json_files/bklog/redis_cluster_nodes_result.json
@@ -6,6 +6,15 @@
     "collector_config_name_en": "redis_cluster_nodes_result",
     "collector_scenario_id": "row",
     "category_id": "host_process",
+    "retention": 7,
+    "es_shards": 3,
+    "target_object_type": "HOST",
+    "target_node_type": "TOPO",
+    "target_nodes": [
+        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
+        {% endfor %}
+    ],
+    {% if storage_cluster_id %}"storage_cluster_id": {{ storage_cluster_id }},{% endif %}
     "etl_params": {
         "retain_original_text": true
     },
@@ -165,14 +174,5 @@
             "is_dimension": false,
             "previous_type": "string"
         }
-    ],
-    "need_assessment": false,
-    "retention": 7,
-    "es_shards": 3,
-    "target_object_type": "HOST",
-    "target_node_type": "TOPO",
-    "target_nodes": [
-        {% for node in target_nodes %}{{ node }}{% if not loop.last %}, {% endif %}
-        {% endfor %}
     ]
 }