Skip to content

Commit

Permalink
[Enhancement](audit log) Add print audit log sesssion variable (#38419)
Browse files Browse the repository at this point in the history
## Proposed changes

For the `insert into` statements during group commit load via JDBC.
Printing audit logs can severely impact performance. Therefore, we have
introduced a session variable to control whether to print audit logs. It
is recommended to turn off audit logs only during group commit load via
JDBC.

<!--Describe your changes.-->
  • Loading branch information
Yukang-Lian authored Jul 31, 2024
1 parent 9eb78d6 commit faaa24c
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -136,18 +136,26 @@ private void handleExecute(PrepareStmt prepareStmt, long stmtId) {
executor = new StmtExecutor(ctx, executeStmt);
ctx.setExecutor(executor);
executor.execute();
PrepareStmtContext preparedStmtContext = ConnectContext.get().getPreparedStmt(String.valueOf(stmtId));
if (preparedStmtContext != null) {
stmtStr = executeStmt.toSql();
//For the `insert into` statements during group commit load via JDBC.
//Printing audit logs can severely impact performance.
//Therefore, we have introduced a session variable to control whether to print audit logs.
//It is recommended to turn off audit logs only during group commit load via JDBC.
if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) {
PrepareStmtContext preparedStmtContext = ConnectContext.get().getPreparedStmt(String.valueOf(stmtId));
if (preparedStmtContext != null) {
stmtStr = executeStmt.toSql();
}
}
} catch (Throwable e) {
} catch (Throwable e) {
// Catch all throwable.
// If reach here, maybe doris bug.
LOG.warn("Process one query failed because unknown reason: ", e);
ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR,
e.getClass().getSimpleName() + ", msg: " + e.getMessage());
}
auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true);
if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) {
auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true);
}
}

private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedStatementContext prepCtx) {
Expand Down Expand Up @@ -199,15 +207,19 @@ private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedS
executor = new StmtExecutor(ctx, stmt);
ctx.setExecutor(executor);
executor.execute();
stmtStr = executeStmt.toSql();
} catch (Throwable e) {
if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) {
stmtStr = executeStmt.toSql();
}
} catch (Throwable e) {
// Catch all throwable.
// If reach here, maybe doris bug.
LOG.warn("Process one query failed because unknown reason: ", e);
ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR,
e.getClass().getSimpleName() + ", msg: " + e.getMessage());
}
auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true);
if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) {
auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true);
}
}

// process COM_EXECUTE, parse binary row data
Expand Down
10 changes: 10 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
Original file line number Diff line number Diff line change
Expand Up @@ -473,8 +473,11 @@ public class SessionVariable implements Serializable, Writable {
public static final String EXTERNAL_TABLE_ANALYZE_PART_NUM = "external_table_analyze_part_num";

public static final String ENABLE_STRONG_CONSISTENCY = "enable_strong_consistency_read";

public static final String GROUP_COMMIT = "group_commit";

public static final String ENABLE_PREPARED_STMT_AUDIT_LOG = "enable_prepared_stmt_audit_log";

public static final String PARALLEL_SYNC_ANALYZE_TASK_NUM = "parallel_sync_analyze_task_num";

public static final String TRUNCATE_CHAR_OR_VARCHAR_COLUMNS = "truncate_char_or_varchar_columns";
Expand Down Expand Up @@ -1679,6 +1682,9 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
@VariableMgr.VarAttr(name = GROUP_COMMIT, needForward = true)
public String groupCommit = "off_mode";

@VariableMgr.VarAttr(name = ENABLE_PREPARED_STMT_AUDIT_LOG, needForward = true)
public boolean enablePreparedStmtAuditLog = true;

@VariableMgr.VarAttr(name = INVERTED_INDEX_CONJUNCTION_OPT_THRESHOLD,
description = {"在match_all中求取多个倒排索引的交集时,如果最大的倒排索引中的总数是最小倒排索引中的总数的整数倍,"
+ "则使用跳表来优化交集操作。",
Expand Down Expand Up @@ -4132,6 +4138,10 @@ public String getGroupCommit() {
return groupCommit;
}

public boolean isEnablePreparedStmtAuditLog() {
return enablePreparedStmtAuditLog;
}

public boolean isEnableMaterializedViewRewrite() {
return enableMaterializedViewRewrite;
}
Expand Down

0 comments on commit faaa24c

Please sign in to comment.