Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport stable/0.26] fix(gateway): not remove stacktraces #6082

Merged
1 commit merged into from
Jan 8, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 28 additions & 17 deletions gateway/src/main/java/io/zeebe/gateway/grpc/GrpcErrorMapper.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,73 +34,83 @@ public StatusRuntimeException mapError(final Throwable error) {
return mapError(error, Loggers.GATEWAY_LOGGER);
}

public StatusRuntimeException mapError(final Throwable error, final Logger logger) {
StatusRuntimeException mapError(final Throwable error, final Logger logger) {
return StatusProto.toStatusRuntimeException(mapErrorToStatus(error, logger));
}

private Status mapErrorToStatus(final Throwable error, final Logger logger) {
return mapErrorToStatus(error, error, logger);
}

private Status mapErrorToStatus(
final Throwable rootError, final Throwable error, final Logger logger) {
final Builder builder = Status.newBuilder();

if (error instanceof ExecutionException) {
return mapErrorToStatus(error.getCause(), logger);
return mapErrorToStatus(rootError, error.getCause(), logger);
} else if (error instanceof BrokerErrorException) {
final Status status =
mapBrokerErrorToStatus(((BrokerErrorException) error).getError(), logger);
mapBrokerErrorToStatus(rootError, ((BrokerErrorException) error).getError(), logger);
builder.mergeFrom(status);
} else if (error instanceof BrokerRejectionException) {
final Status status = mapRejectionToStatus(((BrokerRejectionException) error).getRejection());
builder.mergeFrom(status);
logger.trace("Expected to handle gRPC request, but the broker rejected it", error);
logger.trace("Expected to handle gRPC request, but the broker rejected it", rootError);
} else if (error instanceof TimeoutException) { // can be thrown by transport
builder
.setCode(Code.DEADLINE_EXCEEDED_VALUE)
.setMessage("Time out between gateway and broker: " + error.getMessage());
logger.debug(
"Expected to handle gRPC request, but request timed out between gateway and broker",
error);
rootError);
} else if (error instanceof InvalidBrokerRequestArgumentException) {
builder.setCode(Code.INVALID_ARGUMENT_VALUE).setMessage(error.getMessage());
logger.debug("Expected to handle gRPC request, but broker argument was invalid", error);
logger.debug("Expected to handle gRPC request, but broker argument was invalid", rootError);
} else if (error instanceof MsgpackPropertyException) {
builder.setCode(Code.INVALID_ARGUMENT_VALUE).setMessage(error.getMessage());
logger.debug("Expected to handle gRPC request, but messagepack property was invalid", error);
logger.debug(
"Expected to handle gRPC request, but messagepack property was invalid", rootError);
} else if (error instanceof PartitionNotFoundException) {
builder.setCode(Code.UNAVAILABLE_VALUE).setMessage(error.getMessage());
logger.debug("Expected to handle gRPC request, but request could not be delivered", error);
logger.debug(
"Expected to handle gRPC request, but request could not be delivered", rootError);
} else if (error instanceof RequestRetriesExhaustedException) {
builder.setCode(Code.RESOURCE_EXHAUSTED_VALUE).setMessage(error.getMessage());

// RequestRetriesExhaustedException will sometimes carry suppressed exceptions which can be
// added/mapped as error details to give more information to the user
for (final Throwable suppressed : error.getSuppressed()) {
builder.addDetails(Any.pack(mapErrorToStatus(suppressed, logger)));
builder.addDetails(Any.pack(mapErrorToStatus(rootError, suppressed, logger)));
}

// this error occurs when all partitions have exhausted for requests which have no fixed
// partitions - it will then also occur when back pressure kicks in, leading to a large burst
// of error logs that is, in fact, expected
logger.trace("Expected to handle gRPC request, but all retries have been exhausted", error);
logger.trace(
"Expected to handle gRPC request, but all retries have been exhausted", rootError);
} else if (error instanceof NoTopologyAvailableException) {
builder.setCode(Code.UNAVAILABLE_VALUE).setMessage(error.getMessage());
logger.trace(
"Expected to handle gRPC request, but the gateway does not know any partitions yet",
error);
rootError);
} else if (error instanceof ConnectTimeoutException) {
builder.setCode(Code.UNAVAILABLE_VALUE).setMessage(error.getMessage());
logger.warn(
"Expected to handle gRPC request, but a connection timeout exception occurred", error);
"Expected to handle gRPC request, but a connection timeout exception occurred",
rootError);
} else {
builder
.setCode(Code.INTERNAL_VALUE)
.setMessage(
"Unexpected error occurred during the request processing: " + error.getMessage());
logger.error("Expected to handle gRPC request, but an unexpected error occurred", error);
logger.error("Expected to handle gRPC request, but an unexpected error occurred", rootError);
}

return builder.build();
}

private Status mapBrokerErrorToStatus(final BrokerError error, final Logger logger) {
private Status mapBrokerErrorToStatus(
final Throwable rootError, final BrokerError error, final Logger logger) {
final Builder builder = Status.newBuilder();
String message = error.getMessage();

Expand All @@ -110,20 +120,21 @@ private Status mapBrokerErrorToStatus(final BrokerError error, final Logger logg
break;
case RESOURCE_EXHAUSTED:
builder.setCode(Code.RESOURCE_EXHAUSTED_VALUE);
logger.trace("Target broker is currently overloaded: {}", error);
logger.trace("Target broker is currently overloaded: {}", error, rootError);
break;
case PARTITION_LEADER_MISMATCH:
// return UNAVAILABLE to indicate to the user that retrying might solve the issue, as this
// is usually a transient issue
logger.trace("Target broker was not the leader of the partition: {}", error);
logger.trace("Target broker was not the leader of the partition: {}", error, rootError);
builder.setCode(Code.UNAVAILABLE_VALUE);
break;
default:
// all the following are for cases where retrying (with the same gateway) is not expected
// to solve anything
logger.error(
"Expected to handle gRPC request, but received an internal error from broker: {}",
error);
error,
rootError);
builder.setCode(Code.INTERNAL_VALUE);
message =
String.format(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
import io.zeebe.gateway.impl.broker.response.BrokerError;
import io.zeebe.protocol.record.ErrorCode;
import io.zeebe.util.logging.RecordingAppender;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.core.LogEvent;
Expand Down Expand Up @@ -94,4 +96,23 @@ void shouldAddDetailsForRequestRetriesExhaustedException() throws InvalidProtoco
assertThat(statusDetail.getCode()).isEqualTo(expectedDetail.getCode());
assertThat(statusDetail.getMessage()).isEqualTo(expectedDetail.getMessage());
}

@Test
void shouldLogTimeoutExceptionWithCorrectStacktrace() {
// given
final ExecutionException executionException =
new ExecutionException(new TimeoutException("Timed out after 1s"));

// when
log.setLevel(Level.TRACE);
final StatusRuntimeException statusException = errorMapper.mapError(executionException, logger);

// then
assertThat(statusException.getStatus().getCode()).isEqualTo(Code.DEADLINE_EXCEEDED);
assertThat(recorder.getAppendedEvents()).hasSize(1);
final LogEvent event = recorder.getAppendedEvents().get(0);
assertThat(event.getLevel()).isEqualTo(Level.DEBUG);

assertThat(event.getThrown()).isEqualTo(executionException);
}
}