Skip to content

Commit

Permalink
Merge branch 'master' into archery-2fa
Browse files Browse the repository at this point in the history
  • Loading branch information
@nick2wang
nick2wang authored Apr 30, 2022
2 parents 60f9331 + e66c77f commit f8558d9
Show file tree
Hide file tree
Showing 8 changed files with 88 additions and 131 deletions.
27 changes: 9 additions & 18 deletions sql/engines/clickhouse.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,11 +199,10 @@ def explain_check(self, check_result, db_name=None, line=0, statement=''):
if self.server_version >= (21, 1, 2):
explain_result = self.query(db_name=db_name, sql=f"explain ast {statement}")
if explain_result.error:
check_result.is_critical = True
result = ReviewResult(id=line, errlevel=2,
stagestatus='驳回未通过检查SQL',
errormessage=f'explain语法检查错误：{explain_result.error}',
sql=statement)
stagestatus='驳回未通过检查SQL',
errormessage=f'explain语法检查错误：{explain_result.error}',
sql=statement)
return result

def execute_check(self, db_name=None, sql=''):
Expand All @@ -222,14 +221,12 @@ def execute_check(self, db_name=None, sql=''):
statement = statement.rstrip(';')
# 禁用语句
if re.match(r"^select|^show", statement.lower()):
check_result.is_critical = True
result = ReviewResult(id=line, errlevel=2,
stagestatus='驳回不支持语句',
errormessage='仅支持DML和DDL语句，查询语句请使用SQL查询功能！',
sql=statement)
# 高危语句
elif critical_ddl_regex and p.match(statement.strip().lower()):
check_result.is_critical = True
result = ReviewResult(id=line, errlevel=2,
stagestatus='驳回高危SQL',
errormessage='禁止提交匹配' + critical_ddl_regex + '条件的语句！',
Expand All @@ -245,7 +242,6 @@ def execute_check(self, db_name=None, sql=''):
table_exist = self.get_table_engine(table_name)['status']
if table_exist == 1:
if not table_engine.endswith('MergeTree') and table_engine not in ('Merge', 'Distributed'):
check_result.is_critical = True
result = ReviewResult(id=line, errlevel=2,
stagestatus='驳回不支持SQL',
errormessage='ALTER TABLE仅支持*MergeTree，Merge以及Distributed等引擎表！',
Expand All @@ -254,7 +250,6 @@ def execute_check(self, db_name=None, sql=''):
# delete与update语句，实际是alter语句的变种
if re.match(r"^alter\s+table\s+(.+?)\s+(delete|update)\s+", statement.lower()):
if not table_engine.endswith('MergeTree'):
check_result.is_critical = True
result = ReviewResult(id=line, errlevel=2,
stagestatus='驳回不支持SQL',
errormessage='DELETE与UPDATE仅支持*MergeTree引擎表！',
Expand All @@ -264,7 +259,6 @@ def execute_check(self, db_name=None, sql=''):
else:
result = self.explain_check(check_result, db_name, line, statement)
else:
check_result.is_critical = True
result = ReviewResult(id=line, errlevel=2,
stagestatus='表不存在',
errormessage=f'表 {table_name} 不存在！',
Expand All @@ -281,15 +275,13 @@ def execute_check(self, db_name=None, sql=''):
table_exist = self.get_table_engine(table_name)['status']
if table_exist == 1:
if table_engine in ('View', 'File,', 'URL', 'Buffer', 'Null'):
check_result.is_critical = True
result = ReviewResult(id=line, errlevel=2,
stagestatus='驳回不支持SQL',
errormessage='TRUNCATE不支持View,File,URL,Buffer和Null表引擎！',
sql=statement)
else:
result = self.explain_check(check_result, db_name, line, statement)
else:
check_result.is_critical = True
result = ReviewResult(id=line, errlevel=2,
stagestatus='表不存在',
errormessage=f'表 {table_name} 不存在！',
Expand All @@ -310,13 +302,11 @@ def execute_check(self, db_name=None, sql=''):
affected_rows=0,
execute_time=0, )
else:
check_result.is_critical = True
result = ReviewResult(id=line, errlevel=2,
stagestatus='表不存在',
errormessage=f'表 {table_name} 不存在！',
sql=statement)
else:
check_result.is_critical = True
result = ReviewResult(id=line, errlevel=2,
stagestatus='驳回不支持SQL',
errormessage='INSERT语法不正确！',
Expand All @@ -330,12 +320,13 @@ def execute_check(self, db_name=None, sql=''):
if get_syntax_type(statement, parser=False, db_type='mysql') == 'DDL':
check_result.syntax_type = 1
check_result.rows += [result]

# 遇到禁用和高危语句直接返回
if check_result.is_critical:
check_result.error_count += 1
return check_result
line += 1
# 统计警告和错误数量
for r in check_result.rows:
if r.errlevel == 1:
check_result.warning_count += 1
if r.errlevel == 2:
check_result.error_count += 1
return check_result

def execute_workflow(self, workflow):
Expand Down
10 changes: 6 additions & 4 deletions sql/engines/mongo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -430,7 +430,6 @@ def execute_check(self, db_name=None, sql=''):
alert = ""
if is_in:
check_result.error = "文档已经存在"
check_result.error_count += 1
result = ReviewResult(id=line, errlevel=2,
stagestatus='文档已经存在',
errormessage='文档已经存在！',
Expand All @@ -444,7 +443,6 @@ def execute_check(self, db_name=None, sql=''):
methodStr = sql_str.split('(')[0].split('.')[-1].strip() # 最后一个.和括号(之间的字符串作为方法
if methodStr in is_exist_premise_method and not is_in:
check_result.error = "文档不存在"
check_result.error_count += 1
result = ReviewResult(id=line, errlevel=2,
stagestatus='文档不存在',
errormessage=f'文档不存在，不能进行{methodStr}操作！',
Expand Down Expand Up @@ -492,15 +490,13 @@ def execute_check(self, db_name=None, sql=''):
sql=check_sql,
execute_time=0)
else:
check_result.error_count += 1
result = ReviewResult(id=line, errlevel=2,
stagestatus='驳回不支持语句',
errormessage='仅支持DML和DDL语句，如需查询请使用数据库查询功能！',
sql=check_sql)

else:
check_result.error = "语法错误"
check_result.error_count += 1
result = ReviewResult(id=line, errlevel=2,
stagestatus='语法错误',
errormessage='请检查语句的正确性或（）{} },{是否正确匹配！',
Expand All @@ -511,6 +507,12 @@ def execute_check(self, db_name=None, sql=''):
check_result.column_list = ['Result'] # 审核结果的列名
check_result.checked = True
check_result.warning = self.warning
# 统计警告和错误数量
for r in check_result.rows:
if r.errlevel == 1:
check_result.warning_count += 1
if r.errlevel == 2:
check_result.error_count += 1
return check_result

def get_connection(self, db_name=None):
Expand Down
58 changes: 16 additions & 42 deletions sql/engines/mysql.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,8 +297,8 @@ def query_check(self, db_name=None, sql=''):
result['bad_query'] = True
result['msg'] = explain_result.error
# 不应该查看mysql.user表
if re.match('.*(\\s)+(mysql|`mysql`)(\\s)*\\.(\\s)*(user|`user`)((\\s)*|;).*',sql.lower().replace('\n','')) or\
(db_name=="mysql" and re.match('.*(\\s)+(user|`user`)((\\s)*|;).*',sql.lower().replace('\n',''))):
if re.match('.*(\\s)+(mysql|`mysql`)(\\s)*\\.(\\s)*(user|`user`)((\\s)*|;).*', sql.lower().replace('\n', '')) or \
(db_name == "mysql" and re.match('.*(\\s)+(user|`user`)((\\s)*|;).*', sql.lower().replace('\n', ''))):
result['bad_query'] = True
result['msg'] = '您无权查看该表'

Expand Down Expand Up @@ -348,62 +348,36 @@ def execute_check(self, db_name=None, sql=''):
"""上线单执行前的检查, 返回Review set"""
# 进行Inception检查，获取检测结果
try:
inc_check_result = self.inc_engine.execute_check(instance=self.instance, db_name=db_name, sql=sql)
check_result = self.inc_engine.execute_check(instance=self.instance, db_name=db_name, sql=sql)
except Exception as e:
logger.debug(f"{self.inc_engine.name}检测语句报错：错误信息{traceback.format_exc()}")
raise RuntimeError(f"{self.inc_engine.name}检测语句报错，请注意检查系统配置中{self.inc_engine.name}配置，错误信息：\n{e}")

# 判断Inception检测结果
if inc_check_result.error:
logger.debug(f"{self.inc_engine.name}检测语句报错：错误信息{inc_check_result.error}")
raise RuntimeError(f"{self.inc_engine.name}检测语句报错，错误信息：\n{inc_check_result.error}")
if check_result.error:
logger.debug(f"{self.inc_engine.name}检测语句报错：错误信息{check_result.error}")
raise RuntimeError(f"{self.inc_engine.name}检测语句报错，错误信息：\n{check_result.error}")

# 禁用/高危语句检查
check_critical_result = ReviewSet(full_sql=sql)
line = 1
critical_ddl_regex = self.config.get('critical_ddl_regex', '')
p = re.compile(critical_ddl_regex)
check_critical_result.syntax_type = 2 # TODO 工单类型 0、其他 1、DDL，2、DML

for row in inc_check_result.rows:
for row in check_result.rows:
statement = row.sql
# 去除注释
statement = remove_comments(statement, db_type='mysql')
# 禁用语句
if re.match(r"^select", statement.lower()):
check_critical_result.is_critical = True
result = ReviewResult(id=line, errlevel=2,
stagestatus='驳回不支持语句',
errormessage='仅支持DML和DDL语句，查询语句请使用SQL查询功能！',
sql=statement)
check_result.error_count += 1
row.stagestatus = '驳回不支持语句'
row.errlevel = 2
row.errormessage = '仅支持DML和DDL语句，查询语句请使用SQL查询功能！'
# 高危语句
elif critical_ddl_regex and p.match(statement.strip().lower()):
check_critical_result.is_critical = True
result = ReviewResult(id=line, errlevel=2,
stagestatus='驳回高危SQL',
errormessage='禁止提交匹配' + critical_ddl_regex + '条件的语句！',
sql=statement)
# 正常语句
else:
result = ReviewResult(id=line, errlevel=0,
stagestatus='Audit completed',
errormessage='None',
sql=statement,
affected_rows=0,
execute_time=0, )

# 没有找出DDL语句的才继续执行此判断
if check_critical_result.syntax_type == 2:
if get_syntax_type(statement, parser=False, db_type='mysql') == 'DDL':
check_critical_result.syntax_type = 1
check_critical_result.rows += [result]

# 遇到禁用和高危语句直接返回
if check_critical_result.is_critical:
check_critical_result.error_count += 1
return check_critical_result
line += 1
return inc_check_result
check_result.error_count += 1
row.stagestatus = '驳回高危SQL'
row.errlevel = 2
row.errormessage = '禁止提交匹配' + critical_ddl_regex + '条件的语句！'
return check_result

def execute_workflow(self, workflow):
"""执行上线单，返回Review set"""
Expand Down
Loading

0 comments on commit f8558d9

Please sign in to comment.