Skip to content

Commit

Permalink
HBASE-28064:Implement truncate_region command (apache#5462)
Browse files Browse the repository at this point in the history
Signed-off-by: Wellington Chevreuil <wchevreuil@apache.org>
Signed-off-by: Nihal Jain <nihaljain@apache.org>
  • Loading branch information
vaijosh committed Oct 24, 2023
1 parent 33b00b1 commit f3c8c04
Show file tree
Hide file tree
Showing 25 changed files with 865 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -1547,6 +1547,20 @@ default Future<Void> modifyTableAsync(TableName tableName, TableDescriptor td)
return modifyTableAsync(td);
}

/**
* Truncate an individual region.
* @param regionName region to truncate
* @throws IOException if a remote or network exception occurs
*/
void truncateRegion(byte[] regionName) throws IOException;

/**
* Truncate an individual region. Asynchronous operation.
* @param regionName region to truncate
* @throws IOException if a remote or network exception occurs
*/
Future<Void> truncateRegionAsync(byte[] regionName) throws IOException;

/**
* Modify an existing table, more IRB (ruby) friendly version. Asynchronous operation. This means
* that it may be a while before your schema change is updated across all of the table. You can
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -617,6 +617,12 @@ default CompletableFuture<Void> mergeRegions(byte[] nameOfRegionA, byte[] nameOf
*/
CompletableFuture<Void> splitRegion(byte[] regionName, byte[] splitPoint);

/**
* Truncate an individual region.
* @param regionName region to truncate
*/
CompletableFuture<Void> truncateRegion(byte[] regionName);

/**
* Assign an individual region.
* @param regionName Encoded or full name of region to assign.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,11 @@ public CompletableFuture<Void> splitRegion(byte[] regionName, byte[] splitPoint)
return wrap(rawAdmin.splitRegion(regionName, splitPoint));
}

@Override
public CompletableFuture<Void> truncateRegion(byte[] regionName) {
return wrap(rawAdmin.truncateRegion(regionName));
}

@Override
public CompletableFuture<Void> assign(byte[] regionName) {
return wrap(rawAdmin.assign(regionName));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1554,6 +1554,60 @@ private CompletableFuture<Void> split(final RegionInfo hri, byte[] splitPoint) {
return future;
}

@Override
public CompletableFuture<Void> truncateRegion(byte[] regionName) {
CompletableFuture<Void> future = new CompletableFuture<>();
addListener(getRegionLocation(regionName), (location, err) -> {
if (err != null) {
future.completeExceptionally(err);
return;
}
RegionInfo regionInfo = location.getRegion();
if (regionInfo.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
future.completeExceptionally(new IllegalArgumentException(
"Can't truncate replicas directly.Replicas are auto-truncated "
+ "when their primary is truncated."));
return;
}
ServerName serverName = location.getServerName();
if (serverName == null) {
future
.completeExceptionally(new NoServerForRegionException(Bytes.toStringBinary(regionName)));
return;
}
addListener(truncateRegion(regionInfo), (ret, err2) -> {
if (err2 != null) {
future.completeExceptionally(err2);
} else {
future.complete(ret);
}
});
});
return future;
}

private CompletableFuture<Void> truncateRegion(final RegionInfo hri) {
CompletableFuture<Void> future = new CompletableFuture<>();
TableName tableName = hri.getTable();
final MasterProtos.TruncateRegionRequest request;
try {
request = RequestConverter.buildTruncateRegionRequest(hri, ng.getNonceGroup(), ng.newNonce());
} catch (DeserializationException e) {
future.completeExceptionally(e);
return future;
}
addListener(this.procedureCall(tableName, request, MasterService.Interface::truncateRegion,
MasterProtos.TruncateRegionResponse::getProcId,
new TruncateRegionProcedureBiConsumer(tableName)), (ret, err2) -> {
if (err2 != null) {
future.completeExceptionally(err2);
} else {
future.complete(ret);
}
});
return future;
}

@Override
public CompletableFuture<Void> assign(byte[] regionName) {
CompletableFuture<Void> future = new CompletableFuture<>();
Expand Down Expand Up @@ -2789,6 +2843,18 @@ String getOperationType() {
}
}

private static class TruncateRegionProcedureBiConsumer extends TableProcedureBiConsumer {

TruncateRegionProcedureBiConsumer(TableName tableName) {
super(tableName);
}

@Override
String getOperationType() {
return "TRUNCATE_REGION";
}
}

private static class SnapshotProcedureBiConsumer extends TableProcedureBiConsumer {
SnapshotProcedureBiConsumer(TableName tableName) {
super(tableName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1186,6 +1186,17 @@ public static SplitTableRegionRequest buildSplitTableRegionRequest(final RegionI
return builder.build();
}

public static MasterProtos.TruncateRegionRequest
buildTruncateRegionRequest(final RegionInfo regionInfo, final long nonceGroup, final long nonce)
throws DeserializationException {
MasterProtos.TruncateRegionRequest.Builder builder =
MasterProtos.TruncateRegionRequest.newBuilder();
builder.setRegionInfo(ProtobufUtil.toRegionInfo(regionInfo));
builder.setNonceGroup(nonceGroup);
builder.setNonce(nonce);
return builder.build();
}

/**
* Create a protocol buffer AssignRegionRequest
* @return an AssignRegionRequest
Expand Down
16 changes: 16 additions & 0 deletions hbase-protocol-shaded/src/main/protobuf/Master.proto
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,16 @@ message SplitTableRegionResponse {
optional uint64 proc_id = 1;
}

message TruncateRegionRequest {
required RegionInfo region_info = 1;
optional uint64 nonce_group = 2 [default = 0];
optional uint64 nonce = 3 [default = 0];
}

message TruncateRegionResponse {
optional uint64 proc_id = 1;
}

message CreateTableRequest {
required TableSchema table_schema = 1;
repeated bytes split_keys = 2;
Expand Down Expand Up @@ -852,6 +862,12 @@ service MasterService {
rpc SplitRegion(SplitTableRegionRequest)
returns(SplitTableRegionResponse);

/**
* Truncate region
*/
rpc TruncateRegion(TruncateRegionRequest)
returns(TruncateRegionResponse);

/** Deletes a table */
rpc DeleteTable(DeleteTableRequest)
returns(DeleteTableResponse);
Expand Down
8 changes: 8 additions & 0 deletions hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,14 @@ message TruncateTableStateData {
repeated RegionInfo region_info = 5;
}

enum TruncateRegionState {
TRUNCATE_REGION_PRE_OPERATION = 1;
TRUNCATE_REGION_MAKE_OFFLINE = 2;
TRUNCATE_REGION_REMOVE = 3;
TRUNCATE_REGION_MAKE_ONLINE = 4;
TRUNCATE_REGION_POST_OPERATION = 5;
}

enum DeleteTableState {
DELETE_TABLE_PRE_OPERATION = 1;
DELETE_TABLE_REMOVE_FROM_META = 2;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -665,6 +665,46 @@ default void preSplitRegionAction(final ObserverContext<MasterCoprocessorEnviron
final TableName tableName, final byte[] splitRow) throws IOException {
}

/**
* Called before the region is truncated.
* @param c The environment to interact with the framework and master
* @param regionInfo The Region being truncated
*/
@SuppressWarnings("unused")
default void preTruncateRegionAction(final ObserverContext<MasterCoprocessorEnvironment> c,
final RegionInfo regionInfo) {
}

/**
* Called before the truncate region procedure is called.
* @param c The environment to interact with the framework and master
* @param regionInfo The Region being truncated
*/
@SuppressWarnings("unused")
default void preTruncateRegion(final ObserverContext<MasterCoprocessorEnvironment> c,
RegionInfo regionInfo) {
}

/**
* Called after the truncate region procedure is called.
* @param c The environment to interact with the framework and master
* @param regionInfo The Region being truncated
*/
@SuppressWarnings("unused")
default void postTruncateRegion(final ObserverContext<MasterCoprocessorEnvironment> c,
RegionInfo regionInfo) {
}

/**
* Called post the region is truncated.
* @param c The environment to interact with the framework and master
* @param regionInfo The Region To be truncated
*/
@SuppressWarnings("unused")
default void postTruncateRegionAction(final ObserverContext<MasterCoprocessorEnvironment> c,
final RegionInfo regionInfo) {
}

/**
* Called after the region is split.
* @param c the environment to interact with the framework and master
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@
import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
import org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
import org.apache.hadoop.hbase.master.procedure.TruncateRegionProcedure;
import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure;
import org.apache.hadoop.hbase.master.region.MasterRegion;
import org.apache.hadoop.hbase.master.region.MasterRegionFactory;
Expand Down Expand Up @@ -2482,6 +2483,36 @@ protected String getDescription() {
});
}

@Override
public long truncateRegion(final RegionInfo regionInfo, final long nonceGroup, final long nonce)
throws IOException {
checkInitialized();

return MasterProcedureUtil
.submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
@Override
protected void run() throws IOException {
getMaster().getMasterCoprocessorHost().preTruncateRegion(regionInfo);

LOG.info(
getClientIdAuditPrefix() + " truncate region " + regionInfo.getRegionNameAsString());

// Execute the operation asynchronously
ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch(2, 0);
submitProcedure(
new TruncateRegionProcedure(procedureExecutor.getEnvironment(), regionInfo, latch));
latch.await();

getMaster().getMasterCoprocessorHost().postTruncateRegion(regionInfo);
}

@Override
protected String getDescription() {
return "TruncateRegionProcedure";
}
});
}

@Override
public long addColumn(final TableName tableName, final ColumnFamilyDescriptor column,
final long nonceGroup, final long nonce) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -862,6 +862,60 @@ public void call(MasterObserver observer) throws IOException {
});
}

/**
* Invoked just before calling the truncate region procedure
* @param regionInfo region being truncated
*/
public void preTruncateRegion(RegionInfo regionInfo) throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
@Override
public void call(MasterObserver observer) {
observer.preTruncateRegion(this, regionInfo);
}
});
}

/**
* Invoked after calling the truncate region procedure
* @param regionInfo region being truncated
*/
public void postTruncateRegion(RegionInfo regionInfo) throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
@Override
public void call(MasterObserver observer) {
observer.postTruncateRegion(this, regionInfo);
}
});
}

/**
* Invoked just before calling the truncate region procedure
* @param region Region to be truncated
* @param user The user
*/
public void preTruncateRegionAction(final RegionInfo region, User user) throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation(user) {
@Override
public void call(MasterObserver observer) throws IOException {
observer.preTruncateRegionAction(this, region);
}
});
}

/**
* Invoked after calling the truncate region procedure
* @param region Region which was truncated
* @param user The user
*/
public void postTruncateRegionAction(final RegionInfo region, User user) throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation(user) {
@Override
public void call(MasterObserver observer) throws IOException {
observer.postTruncateRegionAction(this, region);
}
});
}

/**
* This will be called before update META step as part of split table region procedure.
* @param user the user
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -876,6 +876,18 @@ public SplitTableRegionResponse splitRegion(final RpcController controller,
}
}

@Override
public MasterProtos.TruncateRegionResponse truncateRegion(RpcController controller,
final MasterProtos.TruncateRegionRequest request) throws ServiceException {
try {
long procId = server.truncateRegion(ProtobufUtil.toRegionInfo(request.getRegionInfo()),
request.getNonceGroup(), request.getNonce());
return MasterProtos.TruncateRegionResponse.newBuilder().setProcId(procId).build();
} catch (IOException ie) {
throw new ServiceException(ie);
}
}

@Override
public ClientProtos.CoprocessorServiceResponse execMasterService(final RpcController controller,
final ClientProtos.CoprocessorServiceRequest request) throws ServiceException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -433,4 +433,13 @@ boolean normalizeRegions(final NormalizeTableFilterParams ntfp, final boolean is
* Flush master local region
*/
void flushMasterStore() throws IOException;

/**
* Truncate region
* @param regionInfo region to be truncated
* @param nonceGroup the nonce group
* @param nonce the nonce
* @return procedure Id
*/
long truncateRegion(RegionInfo regionInfo, long nonceGroup, long nonce) throws IOException;
}
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler;
import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
import org.apache.hadoop.hbase.master.procedure.TruncateRegionProcedure;
import org.apache.hadoop.hbase.master.region.MasterRegion;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
Expand Down Expand Up @@ -1030,6 +1031,11 @@ public SplitTableRegionProcedure createSplitProcedure(final RegionInfo regionToS
return new SplitTableRegionProcedure(getProcedureEnvironment(), regionToSplit, splitKey);
}

public TruncateRegionProcedure createTruncateRegionProcedure(final RegionInfo regionToTruncate)
throws IOException {
return new TruncateRegionProcedure(getProcedureEnvironment(), regionToTruncate);
}

public MergeTableRegionsProcedure createMergeProcedure(RegionInfo... ris) throws IOException {
return new MergeTableRegionsProcedure(getProcedureEnvironment(), ris, false);
}
Expand Down
Loading

0 comments on commit f3c8c04

Please sign in to comment.