diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index 3fc844b1aac9..028fbba2759d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -1535,7 +1535,16 @@ default void modifyTable(TableName tableName, TableDescriptor td) throws IOExcep throw new IllegalArgumentException("the specified table name '" + tableName + "' doesn't match with the HTD one: " + td.getTableName()); } - modifyTable(td); + modifyTable(td, true); + } + + /** + * Modify an existing table, more IRB friendly version. + * @param td modified description of the table + * @throws IOException if a remote or network exception occurs + */ + default void modifyTable(TableDescriptor td, boolean reopenRegions) throws IOException { + get(modifyTableAsync(td, reopenRegions), getSyncWaitTimeout(), TimeUnit.MILLISECONDS); } /** @@ -1544,7 +1553,7 @@ default void modifyTable(TableName tableName, TableDescriptor td) throws IOExcep * @throws IOException if a remote or network exception occurs */ default void modifyTable(TableDescriptor td) throws IOException { - get(modifyTableAsync(td), getSyncWaitTimeout(), TimeUnit.MILLISECONDS); + get(modifyTableAsync(td, true), getSyncWaitTimeout(), TimeUnit.MILLISECONDS); } /** @@ -1559,7 +1568,7 @@ default void modifyTable(TableDescriptor td) throws IOException { * @return the result of the async modify. You can use Future.get(long, TimeUnit) to wait on the * operation to complete * @deprecated since 2.0 version and will be removed in 3.0 version. use - * {@link #modifyTableAsync(TableDescriptor)} + * {@link #modifyTableAsync(TableDescriptor, boolean)} */ @Deprecated default Future modifyTableAsync(TableName tableName, TableDescriptor td) @@ -1568,7 +1577,7 @@ default Future modifyTableAsync(TableName tableName, TableDescriptor td) throw new IllegalArgumentException("the specified table name '" + tableName + "' doesn't match with the HTD one: " + td.getTableName()); } - return modifyTableAsync(td); + return modifyTableAsync(td, true); } /** @@ -1582,7 +1591,7 @@ default Future modifyTableAsync(TableName tableName, TableDescriptor td) * @return the result of the async modify. You can use Future.get(long, TimeUnit) to wait on the * operation to complete */ - Future modifyTableAsync(TableDescriptor td) throws IOException; + Future modifyTableAsync(TableDescriptor td, boolean reopenRegions) throws IOException; /** * Change the store file tracker of the given table. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index 9509a888cc57..1ffeb62f09ec 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -193,7 +193,20 @@ CompletableFuture createTable(TableDescriptor desc, byte[] startKey, byte[ * Modify an existing table, more IRB friendly version. * @param desc modified description of the table */ - CompletableFuture modifyTable(TableDescriptor desc); + default CompletableFuture modifyTable(TableDescriptor desc) { + return modifyTable(desc, true); + } + + /** + * Modify an existing table, more IRB friendly version. + * @param desc description of the table + * @param reopenRegions By default, 'modifyTable' reopens all regions, potentially causing a RIT + * (Region In Transition) storm in large tables. If set to 'false', regions + * will remain unaware of the modification until they are individually + * reopened. Please note that this may temporarily result in configuration + * inconsistencies among regions. + */ + CompletableFuture modifyTable(TableDescriptor desc, boolean reopenRegions); /** * Change the store file tracker of the given table. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 442fb9115563..e19089b24835 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -148,7 +148,12 @@ public CompletableFuture createTable(TableDescriptor desc, byte[][] splitK @Override public CompletableFuture modifyTable(TableDescriptor desc) { - return wrap(rawAdmin.modifyTable(desc)); + return modifyTable(desc, true); + } + + @Override + public CompletableFuture modifyTable(TableDescriptor desc, boolean reopenRegions) { + return wrap(rawAdmin.modifyTable(desc, reopenRegions)); } @Override diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index b4e9390c13e1..08a78da5e9ae 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -392,7 +392,8 @@ public TableDescriptor getDescriptor(TableName tableName) } @Override - public Future modifyTableAsync(TableDescriptor td) throws IOException { + public Future modifyTableAsync(TableDescriptor td, boolean reopenRegions) + throws IOException { ModifyTableResponse response = executeCallable( new MasterCallable(getConnection(), getRpcControllerFactory()) { long nonceGroup = ng.getNonceGroup(); @@ -401,8 +402,8 @@ public Future modifyTableAsync(TableDescriptor td) throws IOException { @Override protected ModifyTableResponse rpcCall() throws Exception { setPriority(td.getTableName()); - ModifyTableRequest request = - RequestConverter.buildModifyTableRequest(td.getTableName(), td, nonceGroup, nonce); + ModifyTableRequest request = RequestConverter.buildModifyTableRequest(td.getTableName(), + td, nonceGroup, nonce, reopenRegions); return master.modifyTable(getRpcController(), request); } }); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 9231c423ce60..7fe410eefcc6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -659,9 +659,14 @@ private CompletableFuture createTable(TableName tableName, CreateTableRequ @Override public CompletableFuture modifyTable(TableDescriptor desc) { + return modifyTable(desc, true); + } + + public CompletableFuture modifyTable(TableDescriptor desc, boolean reopenRegions) { + // TODO fill the request with reopenRegions return this. procedureCall(desc.getTableName(), RequestConverter.buildModifyTableRequest(desc.getTableName(), desc, ng.getNonceGroup(), - ng.newNonce()), + ng.newNonce(), reopenRegions), (s, c, req, done) -> s.modifyTable(c, req, done), (resp) -> resp.getProcId(), new ModifyTableProcedureBiConsumer(this, desc.getTableName())); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index d6956f4e9df3..00ed41aebbc1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -1295,12 +1295,14 @@ public static CreateTableRequest buildCreateTableRequest(final TableDescriptor t * @return a ModifyTableRequest */ public static ModifyTableRequest buildModifyTableRequest(final TableName tableName, - final TableDescriptor tableDesc, final long nonceGroup, final long nonce) { + final TableDescriptor tableDesc, final long nonceGroup, final long nonce, + boolean reopenRegions) { ModifyTableRequest.Builder builder = ModifyTableRequest.newBuilder(); builder.setTableName(ProtobufUtil.toProtoTableName(tableName)); builder.setTableSchema(ProtobufUtil.toTableSchema(tableDesc)); builder.setNonceGroup(nonceGroup); builder.setNonce(nonce); + builder.setReopenRegions(reopenRegions); return builder.build(); } diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index 6ca065dfc942..c471e993f9d1 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -193,6 +193,7 @@ message ModifyTableRequest { required TableSchema table_schema = 2; optional uint64 nonce_group = 3 [default = 0]; optional uint64 nonce = 4 [default = 0]; + optional bool reopen_regions = 5 [default = true]; } message ModifyTableResponse { diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto index 90d0341df648..aeb8ab42166f 100644 --- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto +++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto @@ -82,6 +82,7 @@ message ModifyTableStateData { required TableSchema modified_table_schema = 3; required bool delete_column_family_in_modify = 4; optional bool should_check_descriptor = 5; + optional bool reopen_regions = 6; } enum TruncateTableState { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 95ce05c6ef48..97ca6524fdf2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -2689,6 +2689,13 @@ protected String getDescription() { private long modifyTable(final TableName tableName, final TableDescriptorGetter newDescriptorGetter, final long nonceGroup, final long nonce, final boolean shouldCheckDescriptor) throws IOException { + return modifyTable(tableName, newDescriptorGetter, nonceGroup, nonce, shouldCheckDescriptor, + true); + } + + private long modifyTable(final TableName tableName, + final TableDescriptorGetter newDescriptorGetter, final long nonceGroup, final long nonce, + final boolean shouldCheckDescriptor, final boolean reopenRegions) throws IOException { return MasterProcedureUtil .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) { @Override @@ -2707,7 +2714,7 @@ protected void run() throws IOException { // checks. This will block only the beginning of the procedure. See HBASE-19953. ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch(); submitProcedure(new ModifyTableProcedure(procedureExecutor.getEnvironment(), - newDescriptor, latch, oldDescriptor, shouldCheckDescriptor)); + newDescriptor, latch, oldDescriptor, shouldCheckDescriptor, reopenRegions)); latch.await(); getMaster().getMasterCoprocessorHost().postModifyTable(tableName, oldDescriptor, @@ -2724,14 +2731,14 @@ protected String getDescription() { @Override public long modifyTable(final TableName tableName, final TableDescriptor newDescriptor, - final long nonceGroup, final long nonce) throws IOException { + final long nonceGroup, final long nonce, final boolean reopenRegions) throws IOException { checkInitialized(); return modifyTable(tableName, new TableDescriptorGetter() { @Override public TableDescriptor get() throws IOException { return newDescriptor; } - }, nonceGroup, nonce, false); + }, nonceGroup, nonce, false, reopenRegions); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 468a477288c4..cd31f3fc8a66 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -1470,7 +1470,8 @@ public ModifyTableResponse modifyTable(RpcController controller, ModifyTableRequ throws ServiceException { try { long procId = master.modifyTable(ProtobufUtil.toTableName(req.getTableName()), - ProtobufUtil.toTableDescriptor(req.getTableSchema()), req.getNonceGroup(), req.getNonce()); + ProtobufUtil.toTableDescriptor(req.getTableSchema()), req.getNonceGroup(), req.getNonce(), + req.getReopenRegions()); return ModifyTableResponse.newBuilder().setProcId(procId).build(); } catch (IOException ioe) { throw new ServiceException(ioe); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java index d465a4a67279..4719a63305df 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java @@ -152,8 +152,19 @@ public long truncateTable(final TableName tableName, final boolean preserveSplit * @param tableName The table name * @param descriptor The updated table descriptor */ + default long modifyTable(final TableName tableName, final TableDescriptor descriptor, + final long nonceGroup, final long nonce) throws IOException { + return modifyTable(tableName, descriptor, nonceGroup, nonce, true); + } + + /** + * Modify the descriptor of an existing table + * @param tableName The table name + * @param descriptor The updated table descriptor + * @param reopenRegions Whether to reopen regions after modifying the table descriptor + */ long modifyTable(final TableName tableName, final TableDescriptor descriptor, - final long nonceGroup, final long nonce) throws IOException; + final long nonceGroup, final long nonce, final boolean reopenRegions) throws IOException; /** * Modify the store file tracker of an existing table diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java index f85fb493fdf2..bab3c7e49422 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -33,11 +34,13 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.zksyncer.MetaLocationSyncer; import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerValidationUtils; import org.apache.hadoop.hbase.replication.ReplicationException; +import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; import org.apache.yetus.audience.InterfaceAudience; @@ -56,6 +59,7 @@ public class ModifyTableProcedure extends AbstractStateMachineTableProcedure s = new HashSet<>(Arrays.asList(TableDescriptorBuilder.REGION_REPLICATION, + TableDescriptorBuilder.REGION_MEMSTORE_REPLICATION, RSGroupInfo.TABLE_DESC_PROP_GROUP)); + for (String k : s) { + if ( + isTablePropertyModified(this.unmodifiedTableDescriptor, this.modifiedTableDescriptor, k) + ) { + throw new HBaseIOException( + "Can not modify " + k + " of a table when modification won't reopen regions"); + } + } + } + } + + private boolean isTablePropertyModified(TableDescriptor oldDescriptor, + TableDescriptor newDescriptor, String key) { + String oldV = oldDescriptor.getValue(key); + String newV = newDescriptor.getValue(key); + if (oldV == null && newV == null) { + return false; + } else if (oldV != null && newV != null && oldV.equals(newV)) { + return false; + } + return true; } private void initialize(final TableDescriptor unmodifiedTableDescriptor, @@ -124,8 +185,11 @@ protected Flow executeFromState(final MasterProcedureEnv env, final ModifyTableS setNextState(ModifyTableState.MODIFY_TABLE_PRE_OPERATION); break; case MODIFY_TABLE_PRE_OPERATION: - preModify(env, state); - setNextState(ModifyTableState.MODIFY_TABLE_CLOSE_EXCESS_REPLICAS); + if (reopenRegions) { + setNextState(ModifyTableState.MODIFY_TABLE_CLOSE_EXCESS_REPLICAS); + } else { + setNextState(ModifyTableState.MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR); + } break; case MODIFY_TABLE_CLOSE_EXCESS_REPLICAS: if (isTableEnabled(env)) { @@ -135,7 +199,11 @@ protected Flow executeFromState(final MasterProcedureEnv env, final ModifyTableS break; case MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR: updateTableDescriptor(env); - setNextState(ModifyTableState.MODIFY_TABLE_REMOVE_REPLICA_COLUMN); + if (reopenRegions) { + setNextState(ModifyTableState.MODIFY_TABLE_REMOVE_REPLICA_COLUMN); + } else { + setNextState(ModifyTableState.MODIFY_TABLE_POST_OPERATION); + } break; case MODIFY_TABLE_REMOVE_REPLICA_COLUMN: removeReplicaColumnsIfNeeded(env); @@ -143,7 +211,11 @@ protected Flow executeFromState(final MasterProcedureEnv env, final ModifyTableS break; case MODIFY_TABLE_POST_OPERATION: postModify(env, state); - setNextState(ModifyTableState.MODIFY_TABLE_REOPEN_ALL_REGIONS); + if (reopenRegions) { + setNextState(ModifyTableState.MODIFY_TABLE_REOPEN_ALL_REGIONS); + } else { + return Flow.NO_MORE_STATE; + } break; case MODIFY_TABLE_REOPEN_ALL_REGIONS: if (isTableEnabled(env)) { @@ -238,7 +310,7 @@ protected void serializeStateData(ProcedureStateSerializer serializer) throws IO .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) .setModifiedTableSchema(ProtobufUtil.toTableSchema(modifiedTableDescriptor)) .setDeleteColumnFamilyInModify(deleteColumnFamilyInModify) - .setShouldCheckDescriptor(shouldCheckDescriptor); + .setShouldCheckDescriptor(shouldCheckDescriptor).setReopenRegions(reopenRegions); if (unmodifiedTableDescriptor != null) { modifyTableMsg @@ -260,6 +332,7 @@ protected void deserializeStateData(ProcedureStateSerializer serializer) throws deleteColumnFamilyInModify = modifyTableMsg.getDeleteColumnFamilyInModify(); shouldCheckDescriptor = modifyTableMsg.hasShouldCheckDescriptor() ? modifyTableMsg.getShouldCheckDescriptor() : false; + reopenRegions = modifyTableMsg.hasReopenRegions() ? modifyTableMsg.getReopenRegions() : true; if (modifyTableMsg.hasUnmodifiedTableSchema()) { unmodifiedTableDescriptor = diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java index d78caff9a393..256f6ac8964f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java @@ -252,6 +252,12 @@ public long modifyTable(final TableName tableName, final TableDescriptor descrip return -1; } + @Override + public long modifyTable(TableName tableName, TableDescriptor descriptor, long nonceGroup, + long nonce, boolean reopenRegions) throws IOException { + return -1; + } + @Override public long enableTable(final TableName tableName, final long nonceGroup, final long nonce) throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java index 81ec43402a49..00441442e860 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java @@ -25,20 +25,24 @@ import org.apache.hadoop.hbase.ConcurrentTableModificationException; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.InvalidFamilyOperationException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.CoprocessorDescriptorBuilder; import org.apache.hadoop.hbase.client.PerClientRandomNonceGenerator; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.StepHook; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; @@ -569,4 +573,124 @@ public void run() { t2.join(); assertFalse("Expected ConcurrentTableModificationException.", (t1.exception || t2.exception)); } + + @Test + public void testModifyWillNotReopenRegions() throws IOException { + final boolean reopenRegions = false; + final TableName tableName = TableName.valueOf(name.getMethodName()); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf"); + + // Test 1: Modify table without reopening any regions + TableDescriptor htd = UTIL.getAdmin().getDescriptor(tableName); + TableDescriptor modifiedDescriptor = TableDescriptorBuilder.newBuilder(htd) + .setValue("test" + ".hbase.conf", "test.hbase.conf.value").build(); + long procId1 = ProcedureTestingUtility.submitAndWait(procExec, new ModifyTableProcedure( + procExec.getEnvironment(), modifiedDescriptor, null, htd, false, reopenRegions)); + ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1)); + TableDescriptor currentHtd = UTIL.getAdmin().getDescriptor(tableName); + assertEquals("test.hbase.conf.value", currentHtd.getValue("test.hbase.conf")); + // Regions should not aware of any changes. + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + Assert.assertNull(r.getTableDescriptor().getValue("test.hbase.conf")); + } + // Force regions to reopen + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + getMaster().getAssignmentManager().move(r.getRegionInfo()); + } + // After the regions reopen, ensure that the configuration is updated. + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + assertEquals("test.hbase.conf.value", r.getTableDescriptor().getValue("test.hbase.conf")); + } + + // Test 2: Modifying region replication is not allowed + htd = UTIL.getAdmin().getDescriptor(tableName); + long oldRegionReplication = htd.getRegionReplication(); + modifiedDescriptor = TableDescriptorBuilder.newBuilder(htd).setRegionReplication(3).build(); + try { + ProcedureTestingUtility.submitAndWait(procExec, new ModifyTableProcedure( + procExec.getEnvironment(), modifiedDescriptor, null, htd, false, reopenRegions)); + Assert.fail( + "An exception should have been thrown while modifying region replication properties."); + } catch (HBaseIOException e) { + assertTrue(e.getMessage().contains("Can not modify")); + } + currentHtd = UTIL.getAdmin().getDescriptor(tableName); + // Nothing changed + assertEquals(oldRegionReplication, currentHtd.getRegionReplication()); + + // Test 3: Adding CFs is not allowed + htd = UTIL.getAdmin().getDescriptor(tableName); + modifiedDescriptor = TableDescriptorBuilder.newBuilder(htd) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder("NewCF".getBytes()).build()) + .build(); + try { + ProcedureTestingUtility.submitAndWait(procExec, new ModifyTableProcedure( + procExec.getEnvironment(), modifiedDescriptor, null, htd, false, reopenRegions)); + Assert.fail("Should have thrown an exception while modifying CF!"); + } catch (HBaseIOException e) { + assertTrue(e.getMessage().contains("Cannot add or remove column families")); + } + currentHtd = UTIL.getAdmin().getDescriptor(tableName); + Assert.assertNull(currentHtd.getColumnFamily("NewCF".getBytes())); + + // Test 4: Modifying CF property is allowed + htd = UTIL.getAdmin().getDescriptor(tableName); + modifiedDescriptor = + TableDescriptorBuilder + .newBuilder(htd).modifyColumnFamily(ColumnFamilyDescriptorBuilder + .newBuilder("cf".getBytes()).setCompressionType(Compression.Algorithm.SNAPPY).build()) + .build(); + ProcedureTestingUtility.submitAndWait(procExec, new ModifyTableProcedure( + procExec.getEnvironment(), modifiedDescriptor, null, htd, false, reopenRegions)); + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + Assert.assertEquals(Compression.Algorithm.NONE, + r.getTableDescriptor().getColumnFamily("cf".getBytes()).getCompressionType()); + } + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + getMaster().getAssignmentManager().move(r.getRegionInfo()); + } + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + Assert.assertEquals(Compression.Algorithm.SNAPPY, + r.getTableDescriptor().getColumnFamily("cf".getBytes()).getCompressionType()); + } + + // Test 5: Modifying TTL as part of CF property is allowed + int newTTL = 60000; + htd = UTIL.getAdmin().getDescriptor(tableName); + modifiedDescriptor = TableDescriptorBuilder.newBuilder(htd) + .modifyColumnFamily( + ColumnFamilyDescriptorBuilder.newBuilder("cf".getBytes()).setTimeToLive(newTTL).build()) + .build(); + ProcedureTestingUtility.submitAndWait(procExec, new ModifyTableProcedure( + procExec.getEnvironment(), modifiedDescriptor, null, htd, false, reopenRegions)); + // The default TTL should be set before the change is applied. + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + Assert.assertEquals(Integer.MAX_VALUE, + r.getTableDescriptor().getColumnFamily("cf".getBytes()).getTimeToLive()); + } + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + getMaster().getAssignmentManager().move(r.getRegionInfo()); + } + for (HRegion r : UTIL.getHBaseCluster().getRegions(tableName)) { + Assert.assertEquals(newTTL, + r.getTableDescriptor().getColumnFamily("cf".getBytes()).getTimeToLive()); + } + + // Test 6: Modifying coprocessor is not allowed + htd = UTIL.getAdmin().getDescriptor(tableName); + modifiedDescriptor = + TableDescriptorBuilder.newBuilder(htd).setCoprocessor(CoprocessorDescriptorBuilder + .newBuilder("any.coprocessor.name").setJarPath("fake/path").build()).build(); + try { + ProcedureTestingUtility.submitAndWait(procExec, new ModifyTableProcedure( + procExec.getEnvironment(), modifiedDescriptor, null, htd, false, reopenRegions)); + Assert.fail("Should have thrown an exception while modifying coprocessor!"); + } catch (HBaseIOException e) { + assertTrue(e.getMessage().contains("Can not modify Coprocessor")); + } + currentHtd = UTIL.getAdmin().getDescriptor(tableName); + assertEquals(0, currentHtd.getCoprocessorDescriptors().size()); + } } diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index a6dcb541c71e..95a0fd4636e8 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -707,6 +707,7 @@ def alter(table_name_str, wait = true, *args) # Get table descriptor htd = org.apache.hadoop.hbase.HTableDescriptor.new(@admin.getTableDescriptor(table_name)) hasTableUpdate = false + reopen_regions = true # Process all args args.each do |arg| @@ -716,9 +717,18 @@ def alter(table_name_str, wait = true, *args) # Normalize args to support shortcut delete syntax arg = { METHOD => 'delete', NAME => arg['delete'] } if arg['delete'] + if arg.key?(REOPEN_REGIONS) + if !['true', 'false'].include?(arg[REOPEN_REGIONS].downcase) + raise(ArgumentError, "Invalid 'REOPEN_REGIONS' for non-boolean value.") + end + reopen_regions = JBoolean.valueOf(arg[REOPEN_REGIONS]) + arg.delete(REOPEN_REGIONS) + end + # There are 3 possible options. # 1) Column family spec. Distinguished by having a NAME and no METHOD. method = arg.delete(METHOD) + if method.nil? && arg.key?(NAME) descriptor = hcd(arg, htd) column_name = descriptor.getNameAsString @@ -831,11 +841,23 @@ def alter(table_name_str, wait = true, *args) # Bulk apply all table modifications. if hasTableUpdate - @admin.modifyTable(table_name, htd) + future = @admin.modifyTableAsync(htd, reopen_regions) + if reopen_regions == false + puts("WARNING: You are using REOPEN_REGIONS => 'false' to modify a table, which will + result in inconsistencies in the configuration of online regions and other risks. If you + encounter any issues, use the original 'alter' command to make the modification again!") + end if wait == true - puts 'Updating all regions with the new schema...' - alter_status(table_name_str) + # If the changes(reopen_regions=false) are not getting applied immediately - + # showing the below can be confusing, hence we only show it when reopen_regions is true. + if reopen_regions == true + puts 'Updating all regions with the new schema...' + end + future.get + if reopen_regions == true + alter_status(table_name_str) + end end end end diff --git a/hbase-shell/src/main/ruby/hbase_constants.rb b/hbase-shell/src/main/ruby/hbase_constants.rb index 76631c91cf9f..ee6b8f22f427 100644 --- a/hbase-shell/src/main/ruby/hbase_constants.rb +++ b/hbase-shell/src/main/ruby/hbase_constants.rb @@ -99,6 +99,7 @@ module HBaseConstants VALUE = 'VALUE'.freeze VERSIONS = org.apache.hadoop.hbase.HConstants::VERSIONS VISIBILITY = 'VISIBILITY'.freeze + REOPEN_REGIONS = 'REOPEN_REGIONS'.freeze # aliases ENDKEY = STOPROW diff --git a/hbase-shell/src/main/ruby/shell/commands/alter.rb b/hbase-shell/src/main/ruby/shell/commands/alter.rb index 9d95bffbdda6..cc47c783f161 100644 --- a/hbase-shell/src/main/ruby/shell/commands/alter.rb +++ b/hbase-shell/src/main/ruby/shell/commands/alter.rb @@ -108,6 +108,18 @@ def help hbase> alter 't1', { NAME => 'f1', VERSIONS => 3 }, { MAX_FILESIZE => '134217728' }, { METHOD => 'delete', NAME => 'f2' }, OWNER => 'johndoe', METADATA => { 'mykey' => 'myvalue' } + +You can also use the REOPEN_REGIONS=>'false' to avoid regions RIT, which let the +modification take effect after regions was reopened (Be careful, the regions of +the table may be configured inconsistently If regions are not reopened after the +modification). This can be used for changing CONFIGURATIONS, changing other table +level properties. Few of the actions like changing region replicas, modifying +coprocessors, adding or removing Column Families are not allowed for safety. + + hbase> alter 't1', REOPEN_REGIONS => 'false', CONFIGURATION => {'hbase.hregion.scan + .loadColumnFamiliesOnDemand' => 'true'} + hbase> alter 't1', NAME => 'f1', REOPEN_REGIONS => 'false', TTL => '12000' + EOF end diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java index 05fc3c932dc5..6f0d07c565a8 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java @@ -904,7 +904,7 @@ public Future modifyTableAsync(TableName tableName, TableDescriptor td) { } @Override - public Future modifyTableAsync(TableDescriptor td) { + public Future modifyTableAsync(TableDescriptor td, boolean reopenRegions) { throw new NotImplementedException("modifyTableAsync not supported in ThriftAdmin"); }