diff --git a/sql/aliyun_rds.py b/sql/aliyun_rds.py deleted file mode 100644 index e2a9fd0416..0000000000 --- a/sql/aliyun_rds.py +++ /dev/null @@ -1,198 +0,0 @@ -# -*- coding: UTF-8 -*- - -import simplejson as json -import datetime - -from common.utils.aliyun_sdk import Aliyun -from .models import AliyunRdsConfig - - -# 获取SQL慢日志统计 -def slowquery_review(request): - instance_name = request.POST.get("instance_name") - db_name = request.POST.get("db_name") - start_time = request.POST.get("StartTime") - end_time = request.POST.get("EndTime") - limit = request.POST.get("limit") - offset = request.POST.get("offset") - - # 计算页数 - page_number = (int(offset) + int(limit)) / int(limit) - values = {"PageSize": int(limit), "PageNumber": int(page_number)} - # DBName非必传 - if db_name: - values["DBName"] = db_name - - # UTC时间转化成阿里云需求的时间格式 - start_time = "%sZ" % start_time - end_time = "%sZ" % end_time - - # 通过实例名称获取关联的rds实例id - instance_info = AliyunRdsConfig.objects.get(instance__instance_name=instance_name) - # 调用aliyun接口获取SQL慢日志统计 - slowsql = Aliyun(rds=instance_info).DescribeSlowLogs(start_time, end_time, **values) - - # 解决table数据丢失精度、格式化时间 - sql_slow_log = json.loads(slowsql)["Items"]["SQLSlowLog"] - for SlowLog in sql_slow_log: - SlowLog["SQLId"] = str(SlowLog["SQLHASH"]) - SlowLog["CreateTime"] = Aliyun.utc2local( - SlowLog["CreateTime"], utc_format="%Y-%m-%dZ" - ) - - result = { - "total": json.loads(slowsql)["TotalRecordCount"], - "rows": sql_slow_log, - "PageSize": json.loads(slowsql)["PageRecordCount"], - "PageNumber": json.loads(slowsql)["PageNumber"], - } - # 返回查询结果 - return result - - -# 获取SQL慢日志明细 -def slowquery_review_history(request): - instance_name = request.POST.get("instance_name") - start_time = request.POST.get("StartTime") - end_time = request.POST.get("EndTime") - db_name = request.POST.get("db_name") - sql_id = request.POST.get("SQLId") - limit = request.POST.get("limit") - offset = request.POST.get("offset") - - # 计算页数 - page_number = (int(offset) + int(limit)) / int(limit) - values = {"PageSize": int(limit), "PageNumber": int(page_number)} - # SQLId、DBName非必传 - if sql_id: - values["SQLHASH"] = sql_id - if db_name: - values["DBName"] = db_name - - # UTC时间转化成阿里云需求的时间格式 - start_time = datetime.datetime.strptime( - start_time, "%Y-%m-%d" - ).date() - datetime.timedelta(days=1) - start_time = "%sT16:00Z" % start_time - end_time = "%sT15:59Z" % end_time - - # 通过实例名称获取关联的rds实例id - instance_info = AliyunRdsConfig.objects.get(instance__instance_name=instance_name) - # 调用aliyun接口获取SQL慢日志统计 - slowsql = Aliyun(rds=instance_info).DescribeSlowLogRecords( - start_time, end_time, **values - ) - - # 格式化时间\过滤HostAddress - sql_slow_record = json.loads(slowsql)["Items"]["SQLSlowRecord"] - for SlowRecord in sql_slow_record: - SlowRecord["ExecutionStartTime"] = Aliyun.utc2local( - SlowRecord["ExecutionStartTime"], utc_format="%Y-%m-%dT%H:%M:%SZ" - ) - SlowRecord["HostAddress"] = SlowRecord["HostAddress"].split("[")[0] - - result = { - "total": json.loads(slowsql)["TotalRecordCount"], - "rows": sql_slow_record, - "PageSize": json.loads(slowsql)["PageRecordCount"], - "PageNumber": json.loads(slowsql)["PageNumber"], - } - - # 返回查询结果 - return result - - -# 问题诊断--进程列表 -def process_status(request): - instance_name = request.POST.get("instance_name") - command_type = request.POST.get("command_type") - - if command_type is None or command_type == "": - command_type = "Query" - - # 通过实例名称获取关联的rds实例id - instance_info = AliyunRdsConfig.objects.get(instance__instance_name=instance_name) - # 调用aliyun接口获取进程数据 - process_info = Aliyun(rds=instance_info).RequestServiceOfCloudDBA( - "ShowProcessList", {"Language": "zh", "Command": command_type} - ) - - # 提取进程列表 - process_list = json.loads(process_info)["AttrData"] - process_list = json.loads(process_list)["ProcessList"] - - result = {"status": 0, "msg": "ok", "rows": process_list} - - # 返回查询结果 - return result - - -# 问题诊断--通过进程id构建请求id -def create_kill_session(request): - instance_name = request.POST.get("instance_name") - thread_ids = request.POST.get("ThreadIDs") - - result = {"status": 0, "msg": "ok", "data": []} - # 通过实例名称获取关联的rds实例id - instance_info = AliyunRdsConfig.objects.get(instance__instance_name=instance_name) - # 调用aliyun接口获取进程数据 - request_info = Aliyun(rds=instance_info).RequestServiceOfCloudDBA( - "CreateKillSessionRequest", - {"Language": "zh", "ThreadIDs": json.loads(thread_ids)}, - ) - - # 提取进程列表 - request_list = json.loads(request_info)["AttrData"] - - result["data"] = request_list - - # 返回处理结果 - return result - - -# 问题诊断--终止会话 -def kill_session(request): - instance_name = request.POST.get("instance_name") - request_params = request.POST.get("request_params") - - result = {"status": 0, "msg": "ok", "data": []} - # 通过实例名称获取关联的rds实例id - instance_info = AliyunRdsConfig.objects.get(instance__instance_name=instance_name) - # 调用aliyun接口获取终止进程 - request_params = json.loads(request_params) - service_request_param = dict({"Language": "zh"}, **request_params) - kill_result = Aliyun(rds=instance_info).RequestServiceOfCloudDBA( - "ConfirmKillSessionRequest", service_request_param - ) - - # 获取处理结果 - kill_result = json.loads(kill_result)["AttrData"] - - result["data"] = kill_result - - # 返回查询结果 - return result - - -# 问题诊断--空间列表 -def sapce_status(request): - instance_name = request.POST.get("instance_name") - - # 通过实例名称获取关联的rds实例id - instance_info = AliyunRdsConfig.objects.get(instance__instance_name=instance_name) - # 调用aliyun接口获取进程数据 - space_info = Aliyun(rds=instance_info).RequestServiceOfCloudDBA( - "GetSpaceStatForTables", {"Language": "zh", "OrderType": "Data"} - ) - - # 提取进程列表 - space_list = json.loads(space_info)["ListData"] - if space_list: - space_list = json.loads(space_list) - else: - space_list = [] - - result = {"status": 0, "msg": "ok", "rows": space_list} - - # 返回查询结果 - return result diff --git a/sql/db_diagnostic.py b/sql/db_diagnostic.py index dd26729ffa..89bb10bce0 100644 --- a/sql/db_diagnostic.py +++ b/sql/db_diagnostic.py @@ -11,14 +11,8 @@ from sql.engines import get_engine from common.utils.extend_json_encoder import ExtendJSONEncoder, ExtendJSONEncoderBytes from sql.utils.resource_group import user_instances -from .models import AliyunRdsConfig, Instance +from .models import Instance -from .aliyun_rds import ( - process_status as aliyun_process_status, - create_kill_session as aliyun_create_kill_session, - kill_session as aliyun_kill_session, - sapce_status as aliyun_sapce_status, -) logger = logging.getLogger("default") @@ -38,11 +32,7 @@ def process(request): query_engine = get_engine(instance=instance) query_result = None if instance.db_type == "mysql": - # 判断是RDS还是其他实例 - if AliyunRdsConfig.objects.filter(instance=instance, is_enable=True).exists(): - result = aliyun_process_status(request) - else: - query_result = query_engine.processlist(command_type) + query_result = query_engine.processlist(command_type) elif instance.db_type == "mongo": query_result = query_engine.current_op(command_type) @@ -85,11 +75,7 @@ def create_kill_session(request): result = {"status": 0, "msg": "ok", "data": []} query_engine = get_engine(instance=instance) if instance.db_type == "mysql": - # 判断是RDS还是其他实例 - if AliyunRdsConfig.objects.filter(instance=instance, is_enable=True).exists(): - result = aliyun_create_kill_session(request) - else: - result["data"] = query_engine.get_kill_command(json.loads(thread_ids)) + result["data"] = query_engine.get_kill_command(json.loads(thread_ids)) elif instance.db_type == "mongo": kill_command = query_engine.get_kill_command(json.loads(thread_ids)) result["data"] = kill_command @@ -125,11 +111,7 @@ def kill_session(request): engine = get_engine(instance=instance) r = None if instance.db_type == "mysql": - # 判断是RDS还是其他实例 - if AliyunRdsConfig.objects.filter(instance=instance, is_enable=True).exists(): - result = aliyun_kill_session(request) - else: - r = engine.kill(json.loads(thread_ids)) + r = engine.kill(json.loads(thread_ids)) elif instance.db_type == "mongo": r = engine.kill_op(json.loads(thread_ids)) elif instance.db_type == "oracle": @@ -153,7 +135,7 @@ def kill_session(request): # 问题诊断--表空间信息 @permission_required("sql.tablespace_view", raise_exception=True) -def tablesapce(request): +def tablespace(request): instance_name = request.POST.get("instance_name") offset = int(request.POST.get("offset", 0)) limit = int(request.POST.get("limit", 14)) @@ -165,17 +147,9 @@ def tablesapce(request): query_engine = get_engine(instance=instance) if instance.db_type == "mysql": - # 判断是RDS还是其他实例 - if AliyunRdsConfig.objects.filter(instance=instance, is_enable=True).exists(): - result = aliyun_sapce_status(request) - else: - query_result = query_engine.tablesapce(offset, limit) - r = query_engine.tablesapce_num() - total = r.rows[0][0] + query_result = query_engine.tablespace(offset, limit) elif instance.db_type == "oracle": query_result = query_engine.tablespace(offset, limit) - r = query_engine.tablespace_count() - total = r.rows[0][0] else: result = { "status": 1, @@ -187,6 +161,8 @@ def tablesapce(request): if query_result: if not query_result.error: table_space = query_result.to_dict() + r = query_engine.tablespace_count() + total = r[0][0] result = {"status": 0, "msg": "ok", "rows": table_space, "total": total} else: result = {"status": 1, "msg": query_result.error} diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index a4f461bc1f..3223f9492d 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -215,6 +215,12 @@ def set_variable(self, variable_name, variable_value): def get_engine(instance=None): # pragma: no cover """获取数据库操作engine""" if instance.db_type == "mysql": + from sql.models import AliyunRdsConfig + + if AliyunRdsConfig.objects.filter(instance=instance, is_enable=True).exists(): + from .cloud.aliyun_rds import AliyunRDS + + return AliyunRDS(instance=instance) from .mysql import MysqlEngine return MysqlEngine(instance=instance) diff --git a/sql/engines/cloud/__init__.py b/sql/engines/cloud/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sql/engines/cloud/aliyun_rds.py b/sql/engines/cloud/aliyun_rds.py new file mode 100644 index 0000000000..efb13eaf41 --- /dev/null +++ b/sql/engines/cloud/aliyun_rds.py @@ -0,0 +1,188 @@ +# -*- coding: UTF-8 -*- +import traceback +import simplejson as json +import datetime + +from common.utils.aliyun_sdk import Aliyun +from sql.models import AliyunRdsConfig +from sql.engines.mysql import MysqlEngine +from sql.engines.models import ResultSet + + +class AliyunRDS(MysqlEngine): + def __init__(self, instance=None): + super().__init__(instance=instance) + self.instance_name = instance.instance_name + + # 将sql/aliyun_rds.py的函数迁移值此 + def processlist(self, command_type): + if command_type is None or command_type == "": + command_type = "Query" + + # 通过实例名称获取关联的rds实例id + instance_info = AliyunRdsConfig.objects.get( + instance__instance_name=self.instance_name + ) + # 调用aliyun接口获取进程数据 + process_info = Aliyun(rds=instance_info).RequestServiceOfCloudDBA( + "ShowProcessList", {"Language": "zh", "Command": command_type} + ) + + # 提取进程列表 + process_list = json.loads(process_info)["AttrData"] + process_list = json.loads(process_list)["ProcessList"] + + result_set = ResultSet(full_sql="show processlist") + result_set.rows = process_list + + return result_set + + def get_kill_command(self, thread_ids): + # 通过实例名称获取关联的rds实例id + instance_info = AliyunRdsConfig.objects.get( + instance__instance_name=self.instance_name + ) + # 调用aliyun接口获取进程数据 + request_info = Aliyun(rds=instance_info).RequestServiceOfCloudDBA( + "CreateKillSessionRequest", {"Language": "zh", "ThreadIDs": thread_ids} + ) + + request_list = json.loads(request_info)["AttrData"] + kill_sql = str(request_list) + + return kill_sql + + def kill(self, thread_ids): + kill_sql = "" + for i in thread_ids: + kill_sql = kill_sql + f"kill {i};" + result = ResultSet(full_sql=kill_sql) + try: + # 通过实例名称获取关联的rds实例id + instance_info = AliyunRdsConfig.objects.get( + instance__instance_name=self.instance_name + ) + # 调用aliyun接口获取终止进程 + service_request_param = {"Language": "zh"} + kill_result = Aliyun(rds=instance_info).RequestServiceOfCloudDBA( + "ConfirmKillSessionRequest", service_request_param + ) + + # 获取处理结果 + kill_result = json.loads(kill_result)["AttrData"] + + result.rows = kill_result + except Exception as e: + logger.warning( + f"aliyun rds语句执行报错，语句：{kill_sql}，错误信息{traceback.format_exc()}" + ) + result.error = str(e) + return result + + def tablespace(self, offset, limit): + # 通过实例名称获取关联的rds实例id + instance_info = AliyunRdsConfig.objects.get( + instance__instance_name=self.instance_name + ) + # 调用aliyun接口获取进程数据 + space_info = Aliyun(rds=instance_info).RequestServiceOfCloudDBA( + "GetSpaceStatForTables", {"Language": "zh", "OrderType": "Data"} + ) + + # 提取进程列表 + space_list = json.loads(space_info)["ListData"] + if space_list: + space_list = json.loads(space_list) + else: + space_list = [] + + result = ResultSet(full_sql="select * FROM information_schema.tables") + result.rows = space_list + + return result + + # 获取SQL慢日志统计 + def slowquery_review(self, start_time, end_time, db_name, limit, offset): + # 计算页数 + page_number = (int(offset) + int(limit)) / int(limit) + values = {"PageSize": int(limit), "PageNumber": int(page_number)} + # DBName非必传 + if db_name: + values["DBName"] = db_name + + # UTC时间转化成阿里云需求的时间格式 + start_time = "%sZ" % start_time + end_time = "%sZ" % end_time + + # 通过实例名称获取关联的rds实例id + instance_info = AliyunRdsConfig.objects.get( + instance__instance_name=self.instance_name + ) + # 调用aliyun接口获取SQL慢日志统计 + slowsql = Aliyun(rds=instance_info).DescribeSlowLogs( + start_time, end_time, **values + ) + + # 解决table数据丢失精度、格式化时间 + sql_slow_log = json.loads(slowsql)["Items"]["SQLSlowLog"] + for SlowLog in sql_slow_log: + SlowLog["SQLId"] = str(SlowLog["SQLHASH"]) + SlowLog["CreateTime"] = Aliyun.utc2local( + SlowLog["CreateTime"], utc_format="%Y-%m-%dZ" + ) + + result = { + "total": json.loads(slowsql)["TotalRecordCount"], + "rows": sql_slow_log, + "PageSize": json.loads(slowsql)["PageRecordCount"], + "PageNumber": json.loads(slowsql)["PageNumber"], + } + # 返回查询结果 + return result + + # 获取SQL慢日志明细 + def slowquery_review_history( + self, start_time, end_time, db_name, sql_id, limit, offset + ): + # 计算页数 + page_number = (int(offset) + int(limit)) / int(limit) + values = {"PageSize": int(limit), "PageNumber": int(page_number)} + # SQLId、DBName非必传 + if sql_id: + values["SQLHASH"] = sql_id + if db_name: + values["DBName"] = db_name + + # UTC时间转化成阿里云需求的时间格式 + start_time = datetime.datetime.strptime( + start_time, "%Y-%m-%d" + ).date() - datetime.timedelta(days=1) + start_time = "%sT16:00Z" % start_time + end_time = "%sT15:59Z" % end_time + + # 通过实例名称获取关联的rds实例id + instance_info = AliyunRdsConfig.objects.get( + instance__instance_name=self.instance_name + ) + # 调用aliyun接口获取SQL慢日志统计 + slowsql = Aliyun(rds=instance_info).DescribeSlowLogRecords( + start_time, end_time, **values + ) + + # 格式化时间\过滤HostAddress + sql_slow_record = json.loads(slowsql)["Items"]["SQLSlowRecord"] + for SlowRecord in sql_slow_record: + SlowRecord["ExecutionStartTime"] = Aliyun.utc2local( + SlowRecord["ExecutionStartTime"], utc_format="%Y-%m-%dT%H:%M:%SZ" + ) + SlowRecord["HostAddress"] = SlowRecord["HostAddress"].split("[")[0] + + result = { + "total": json.loads(slowsql)["TotalRecordCount"], + "rows": sql_slow_record, + "PageSize": json.loads(slowsql)["PageRecordCount"], + "PageNumber": json.loads(slowsql)["PageNumber"], + } + + # 返回查询结果 + return result diff --git a/sql/engines/mysql.py b/sql/engines/mysql.py index 2b05b6488b..8e55464578 100644 --- a/sql/engines/mysql.py +++ b/sql/engines/mysql.py @@ -744,7 +744,7 @@ def kill(self, thread_ids): kill_sql = kill_sql + row[0] return self.execute("information_schema", kill_sql) - def tablesapce(self, offset=0, row_count=14): + def tablespace(self, offset=0, row_count=14): """获取表空间信息""" sql = """ SELECT @@ -765,7 +765,7 @@ def tablesapce(self, offset=0, row_count=14): ) return self.query("information_schema", sql) - def tablesapce_num(self): + def tablespace_count(self): """获取表空间数量""" sql = """ SELECT count(*) diff --git a/sql/engines/tests.py b/sql/engines/tests.py index 0b9e7078d9..4e6a8f4997 100644 --- a/sql/engines/tests.py +++ b/sql/engines/tests.py @@ -630,17 +630,17 @@ def test_kill(self, _query, _connect, _cursor, _execute): self.assertIsInstance(r, ResultSet) @patch.object(MysqlEngine, "query") - def test_tablesapce(self, _query): + def test_tablespace(self, _query): new_engine = MysqlEngine(instance=self.ins1) _query.return_value = ResultSet() - r = new_engine.tablesapce() + r = new_engine.tablespace() self.assertIsInstance(r, ResultSet) @patch.object(MysqlEngine, "query") - def test_tablesapce_num(self, _query): + def test_tablespace_count(self, _query): new_engine = MysqlEngine(instance=self.ins1) _query.return_value = ResultSet() - r = new_engine.tablesapce_num() + r = new_engine.tablespace_count() self.assertIsInstance(r, ResultSet) @patch.object(MysqlEngine, "query") diff --git a/sql/slowlog.py b/sql/slowlog.py index cda569cba2..ca952951ad 100644 --- a/sql/slowlog.py +++ b/sql/slowlog.py @@ -15,10 +15,6 @@ from common.utils.extend_json_encoder import ExtendJSONEncoder from .models import Instance, SlowQuery, SlowQueryHistory, AliyunRdsConfig -from .aliyun_rds import ( - slowquery_review as aliyun_rds_slowquery_review, - slowquery_review_history as aliyun_rds_slowquery_review_history, -) import logging @@ -29,6 +25,11 @@ @permission_required("sql.menu_slowquery", raise_exception=True) def slowquery_review(request): instance_name = request.POST.get("instance_name") + start_time = request.POST.get("StartTime") + end_time = request.POST.get("EndTime") + db_name = request.POST.get("db_name") + limit = request.POST.get("limit") + offset = request.POST.get("offset") # 服务端权限校验 try: user_instances(request.user, db_type=["mysql"]).get(instance_name=instance_name) @@ -40,13 +41,11 @@ def slowquery_review(request): instance_info = Instance.objects.get(instance_name=instance_name) if AliyunRdsConfig.objects.filter(instance=instance_info, is_enable=True).exists(): # 调用阿里云慢日志接口 - result = aliyun_rds_slowquery_review(request) + query_engine = get_engine(instance=instance_info) + result = query_engine.slowquery_review( + self, start_time, end_time, db_name, limit, offset + ) else: - start_time = request.POST.get("StartTime") - end_time = request.POST.get("EndTime") - db_name = request.POST.get("db_name") - limit = int(request.POST.get("limit")) - offset = int(request.POST.get("offset")) limit = offset + limit search = request.POST.get("search") sortName = str(request.POST.get("sortName")) @@ -115,6 +114,12 @@ def slowquery_review(request): @permission_required("sql.menu_slowquery", raise_exception=True) def slowquery_review_history(request): instance_name = request.POST.get("instance_name") + start_time = request.POST.get("StartTime") + end_time = request.POST.get("EndTime") + db_name = request.POST.get("db_name") + sql_id = request.POST.get("SQLId") + limit = request.POST.get("limit") + offset = request.POST.get("offset") # 服务端权限校验 try: user_instances(request.user, db_type=["mysql"]).get(instance_name=instance_name) @@ -126,14 +131,11 @@ def slowquery_review_history(request): instance_info = Instance.objects.get(instance_name=instance_name) if AliyunRdsConfig.objects.filter(instance=instance_info, is_enable=True).exists(): # 调用阿里云慢日志接口 - result = aliyun_rds_slowquery_review_history(request) + query_engine = get_engine(instance=instance_info) + result = query_engine.slowquery_review_history( + start_time, end_time, db_name, sql_id, limit, offset + ) else: - start_time = request.POST.get("StartTime") - end_time = request.POST.get("EndTime") - db_name = request.POST.get("db_name") - sql_id = request.POST.get("SQLId") - limit = int(request.POST.get("limit")) - offset = int(request.POST.get("offset")) search = request.POST.get("search") sortName = str(request.POST.get("sortName")) sortOrder = str(request.POST.get("sortOrder")).lower() diff --git a/sql/templates/dbdiagnostic.html b/sql/templates/dbdiagnostic.html index 40f0454829..57f981ba01 100644 --- a/sql/templates/dbdiagnostic.html +++ b/sql/templates/dbdiagnostic.html @@ -460,7 +460,7 @@

确定要终止所选会话吗？

escape: true, method: 'post', contentType: "application/x-www-form-urlencoded", - url: "/db_diagnostic/tablesapce/", + url: "/db_diagnostic/tablespace/", striped: true, //是否显示行间隔色 cache: false, //是否使用缓存，默认为true，所以一般情况下需要设置一下这个属性（*） pagination: true, //是否显示分页（*） diff --git a/sql/urls.py b/sql/urls.py index 0488030c73..e2511c79e9 100644 --- a/sql/urls.py +++ b/sql/urls.py @@ -148,7 +148,7 @@ path("db_diagnostic/process/", db_diagnostic.process), path("db_diagnostic/create_kill_session/", db_diagnostic.create_kill_session), path("db_diagnostic/kill_session/", db_diagnostic.kill_session), - path("db_diagnostic/tablesapce/", db_diagnostic.tablesapce), + path("db_diagnostic/tablespace/", db_diagnostic.tablespace), path("db_diagnostic/trxandlocks/", db_diagnostic.trxandlocks), path("db_diagnostic/innodb_trx/", db_diagnostic.innodb_trx), path("archive/list/", archiver.archive_list),