Skip to content

Make some debug log settings configurable and change some log level from info to debug to avoid performance bottlenecks #4766

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kangkaisen merged 3 commits into from
Oct 26, 2020
Merged

Make some debug log settings configurable and change some log level from info to debug to avoid performance bottlenecks #4766

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
585c53c
Make some debug log settings configurable and change some log level f…
Oct 19, 2020
5c9fe4d
merge debug log by review
Oct 21, 2020
4eb90d3
merge debug log by review
Oct 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/common/Config.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public class Config extends ConfigBase {
public static String sys_log_dir = PaloFe.DORIS_HOME_DIR + "/log";
@ConfField public static String sys_log_level = "INFO";
@ConfField public static int sys_log_roll_num = 10;
@ConfField public static String[] sys_log_verbose_modules = {};
@ConfField public static String[] sys_log_verbose_modules = {"org.apache.thrift", "org.apache.doris.thrift", "org.apache.doris.http", "org.apache.doris.service.FrontendServiceImpl"};
@ConfField public static String sys_log_roll_interval = "DAY";
@ConfField public static String sys_log_delete_age = "7d";
@Deprecated
Expand Down
14 changes: 1 addition & 13 deletions fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,18 +96,6 @@ public class Log4jConfig extends XmlConfiguration {
" <Logger name=\"audit\" level=\"ERROR\" additivity=\"false\">\n" +
" <AppenderRef ref=\"Auditfile\"/>\n" +
" </Logger>\n" +
" <Logger name=\"org.apache.thrift\" level=\"DEBUG\"> \n" +
" <AppenderRef ref=\"Sys\"/>\n" +
" </Logger>\n" +
" <Logger name=\"org.apache.thrift.transport\" level=\"DEBUG\"> \n" +
" <AppenderRef ref=\"Sys\"/>\n" +
" </Logger>\n" +
" <Logger name=\"org.apache.doris.thrift\" level=\"DEBUG\"> \n" +
" <AppenderRef ref=\"Sys\"/>\n" +
" </Logger>\n" +
" <Logger name=\"org.apache.doris.http\" level=\"DEBUG\"> \n" +
" <AppenderRef ref=\"Sys\"/>\n" +
" </Logger>\n" +
" <!--REPLACED BY AUDIT AND VERBOSE MODULE NAMES-->\n" +
" </Loggers>\n" +
"</Configuration>";
Expand Down Expand Up @@ -254,6 +242,6 @@ public synchronized static Tuple<String, String[], String[]> updateLogging(
if (toReconfig) {
reconfig();
}
return new Tuple<String, String[], String[]>(sysLogLevel, verboseModules, auditModules);
return new Tuple<>(sysLogLevel, verboseModules, auditModules);
}
}
9 changes: 4 additions & 5 deletions fe/fe-core/src/main/java/org/apache/doris/common/ThriftServerEventProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,15 +75,15 @@ public ServerContext createContext(TProtocol input, TProtocol output) {
break;
}
if (tSocket == null) {
LOG.info("fail to get client socket. server type: {}", thriftServer.getType());
LOG.warn("fail to get client socket. server type: {}", thriftServer.getType());
return null;
}
SocketAddress socketAddress = tSocket.getSocket().getRemoteSocketAddress();
InetSocketAddress inetSocketAddress = null;
if (socketAddress instanceof InetSocketAddress) {
inetSocketAddress = (InetSocketAddress) socketAddress;
} else {
LOG.info("fail to get client socket address. server type: {}",
LOG.warn("fail to get client socket address. server type: {}",
thriftServer.getType());
return null;
}
Expand All @@ -93,7 +93,7 @@ public ServerContext createContext(TProtocol input, TProtocol output) {

thriftServer.addConnect(clientAddress);

LOG.info("create thrift context. client: {}", clientAddress);
LOG.debug("create thrift context. client: {}", clientAddress);
return new ThriftServerContext(clientAddress);
}

Expand All @@ -108,7 +108,7 @@ public void deleteContext(ServerContext serverContext, TProtocol input, TProtoco
TNetworkAddress clientAddress = thriftServerContext.getClient();
connectionContext.remove();
thriftServer.removeConnect(clientAddress);
LOG.info("delete thrift context. client: {}", clientAddress);
LOG.debug("delete thrift context. client: {}", clientAddress);
}

@Override
Expand All @@ -119,7 +119,6 @@ public void processContext(ServerContext serverContext, TTransport inputTranspor

ThriftServerContext thriftServerContext = (ThriftServerContext) serverContext;
TNetworkAddress clientAddress = thriftServerContext.getClient();
Preconditions.checkState(serverContext instanceof ThriftServerContext);
connectionContext.set(new ThriftServerContext(clientAddress));
}
}
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/http/BaseAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ public BaseAction(ActionController controller) {
@Override
public void handleRequest(BaseRequest request) throws Exception {
BaseResponse response = new BaseResponse();
LOG.info("receive http request. url={}", request.getRequest().uri());
LOG.debug("receive http request. url={}", request.getRequest().uri());
try {
execute(request, response);
} catch (Exception e) {
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/http/rest/RestBaseAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public RestBaseAction(ActionController controller) {

@Override
public void handleRequest(BaseRequest request) throws Exception {
LOG.info("receive http request. url={}", request.getRequest().uri());
LOG.debug("receive http request. url={}", request.getRequest().uri());
BaseResponse response = new BaseResponse();
try {
execute(request, response);
Expand Down
32 changes: 11 additions & 21 deletions fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@
// Frontend service used to serve all request for this frontend through
// thrift protocol
public class FrontendServiceImpl implements FrontendService.Iface {
private static final Logger LOG = LogManager.getLogger(MasterImpl.class);
private static final Logger LOG = LogManager.getLogger(FrontendServiceImpl.class);
private MasterImpl masterImpl;
private ExecuteEnv exeEnv;

Expand Down Expand Up @@ -377,7 +377,7 @@ public TFetchResourceResult fetchResource() throws TException {
@Deprecated
@Override
public TFeResult miniLoad(TMiniLoadRequest request) throws TException {
LOG.info("receive mini load request: label: {}, db: {}, tbl: {}, backend: {}",
LOG.debug("receive mini load request: label: {}, db: {}, tbl: {}, backend: {}",
request.getLabel(), request.getDb(), request.getTbl(), request.getBackend());

ConnectContext context = new ConnectContext(null);
Expand Down Expand Up @@ -499,7 +499,7 @@ public TFeResult updateMiniEtlTaskStatus(TUpdateMiniEtlTaskStatusRequest request

// update etl task status
TMiniLoadEtlStatusResult statusResult = request.getEtlTaskStatus();
LOG.info("load job id: {}, etl task id: {}, status: {}", jobId, taskId, statusResult);
LOG.debug("load job id: {}, etl task id: {}, status: {}", jobId, taskId, statusResult);
EtlStatus taskStatus = taskInfo.getTaskStatus();
if (taskStatus.setState(statusResult.getEtlState())) {
if (statusResult.isSetCounters()) {
Expand All @@ -517,7 +517,7 @@ public TFeResult updateMiniEtlTaskStatus(TUpdateMiniEtlTaskStatusRequest request

@Override
public TMiniLoadBeginResult miniLoadBegin(TMiniLoadBeginRequest request) throws TException {
LOG.info("receive mini load begin request. label: {}, user: {}, ip: {}",
LOG.debug("receive mini load begin request. label: {}, user: {}, ip: {}",
request.getLabel(), request.getUser(), request.getUserIp());

TMiniLoadBeginResult result = new TMiniLoadBeginResult();
Expand Down Expand Up @@ -577,7 +577,7 @@ public TMasterOpResult forward(TMasterOpRequest params) throws TException {
}

// add this log so that we can track this stmt
LOG.info("receive forwarded stmt {} from FE: {}", params.getStmtId(), clientAddr.getHostname());
LOG.debug("receive forwarded stmt {} from FE: {}", params.getStmtId(), clientAddr.getHostname());
ConnectContext context = new ConnectContext(null);
ConnectProcessor processor = new ConnectProcessor(context);
TMasterOpResult result = processor.proxyExecute(params);
Expand All @@ -604,7 +604,7 @@ private void checkPasswordAndPrivs(String cluster, String user, String passwd, S

@Override
public TFeResult loadCheck(TLoadCheckRequest request) throws TException {
LOG.info("receive load check request. label: {}, user: {}, ip: {}",
LOG.debug("receive load check request. label: {}, user: {}, ip: {}",
request.getLabel(), request.getUser(), request.getUserIp());

TStatus status = new TStatus(TStatusCode.OK);
Expand Down Expand Up @@ -634,9 +634,7 @@ public TFeResult loadCheck(TLoadCheckRequest request) throws TException {
@Override
public TLoadTxnBeginResult loadTxnBegin(TLoadTxnBeginRequest request) throws TException {
String clientAddr = getClientAddrAsString();
LOG.info("receive txn begin request, db: {}, tbl: {}, label: {}, backend: {}",
request.getDb(), request.getTbl(), request.getLabel(), clientAddr);
LOG.debug("txn begin request: {}", request);
LOG.debug("receive txn begin request: {}, backend: {}", request, clientAddr);

TLoadTxnBeginResult result = new TLoadTxnBeginResult();
TStatus status = new TStatus(TStatusCode.OK);
Expand All @@ -645,7 +643,7 @@ public TLoadTxnBeginResult loadTxnBegin(TLoadTxnBeginRequest request) throws TEx
result.setTxnId(loadTxnBeginImpl(request, clientAddr));
} catch (DuplicatedRequestException e) {
// this is a duplicate request, just return previous txn id
LOG.info("duplicate request for stream load. request id: {}, txn: {}", e.getDuplicatedRequestId(), e.getTxnId());
LOG.warn("duplicate request for stream load. request id: {}, txn: {}", e.getDuplicatedRequestId(), e.getTxnId());
result.setTxnId(e.getTxnId());
} catch (LabelAlreadyUsedException e) {
status.setStatusCode(TStatusCode.LABEL_ALREADY_EXISTS);
Expand Down Expand Up @@ -712,9 +710,7 @@ private long loadTxnBeginImpl(TLoadTxnBeginRequest request, String clientIp) thr
@Override
public TLoadTxnCommitResult loadTxnCommit(TLoadTxnCommitRequest request) throws TException {
String clientAddr = getClientAddrAsString();
LOG.info("receive txn commit request. db: {}, tbl: {}, txn id: {}, backend: {}",
request.getDb(), request.getTbl(), request.getTxnId(), clientAddr);
LOG.debug("txn commit request: {}", request);
LOG.debug("receive txn commit request: {}, backend: {}", request, clientAddr);

TLoadTxnCommitResult result = new TLoadTxnCommitResult();
TStatus status = new TStatus(TStatusCode.OK);
Expand Down Expand Up @@ -778,10 +774,7 @@ private boolean loadTxnCommitImpl(TLoadTxnCommitRequest request) throws UserExce
@Override
public TLoadTxnRollbackResult loadTxnRollback(TLoadTxnRollbackRequest request) throws TException {
String clientAddr = getClientAddrAsString();
LOG.info("receive txn rollback request. db: {}, tbl: {}, txn id: {}, reason: {}, backend: {}",
request.getDb(), request.getTbl(), request.getTxnId(), request.getReason(), clientAddr);
LOG.debug("txn rollback request: {}", request);

LOG.debug("receive txn rollback request: {}, backend: {}", request, clientAddr);
TLoadTxnRollbackResult result = new TLoadTxnRollbackResult();
TStatus status = new TStatus(TStatusCode.OK);
result.setStatus(status);
Expand Down Expand Up @@ -827,10 +820,7 @@ private void loadTxnRollbackImpl(TLoadTxnRollbackRequest request) throws UserExc
@Override
public TStreamLoadPutResult streamLoadPut(TStreamLoadPutRequest request) {
String clientAddr = getClientAddrAsString();
LOG.info("receive stream load put request. db:{}, tbl: {}, txn id: {}, load id: {}, backend: {}",
request.getDb(), request.getTbl(), request.getTxnId(), DebugUtil.printId(request.getLoadId()),
clientAddr);
LOG.debug("stream load put request: {}", request);
LOG.debug("receive stream load put request: {}, backend: {}", request, clientAddr);

TStreamLoadPutResult result = new TStreamLoadPutResult();
TStatus status = new TStatus(TStatusCode.OK);
Expand Down