From e96df8bf409210d7bdb2503e3637ba85230e1afd Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Mon, 3 Jan 2022 00:35:14 +0800 Subject: [PATCH] HBASE-26641 Split TestMasterFailoverWithProcedures (#3994) Signed-off-by: GeorryHuang --- .../MasterFailoverWithProceduresTestBase.java | 87 +++++ .../TestCreateTableWithMasterFailover.java | 74 ++++ .../TestDeleteTableWithMasterFailover.java | 76 ++++ .../TestDisableTableWithMasterFailover.java | 74 ++++ .../TestEnableTableWithMasterFailover.java | 74 ++++ .../TestMasterFailoverWithProcedures.java | 335 ------------------ .../TestTruncateTableWithMasterFailover.java | 102 ++++++ 7 files changed, 487 insertions(+), 335 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterFailoverWithProceduresTestBase.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableWithMasterFailover.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableWithMasterFailover.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDisableTableWithMasterFailover.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestEnableTableWithMasterFailover.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableWithMasterFailover.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterFailoverWithProceduresTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterFailoverWithProceduresTestBase.java new file mode 100644 index 000000000000..2814ba4cce6c --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterFailoverWithProceduresTestBase.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.StartTestingClusterOption; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class MasterFailoverWithProceduresTestBase { + + private static final Logger LOG = + LoggerFactory.getLogger(MasterFailoverWithProceduresTestBase.class); + + protected static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + + @BeforeClass + public static void setUp() throws Exception { + UTIL.getConfiguration().setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + StartTestingClusterOption option = StartTestingClusterOption.builder().numMasters(2).build(); + UTIL.startMiniCluster(option); + + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, false); + ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, false); + } + + @AfterClass + public static void tearDown() throws Exception { + UTIL.shutdownMiniCluster(); + } + + // ========================================================================== + // Helpers + // ========================================================================== + protected static ProcedureExecutor getMasterProcedureExecutor() { + return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); + } + + protected static Path getRootDir() { + return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); + } + + protected static void testRecoveryAndDoubleExecution(final HBaseTestingUtil testUtil, + final long procId, final int lastStepBeforeFailover) throws Exception { + ProcedureExecutor procExec = + testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor(); + ProcedureTestingUtility.waitProcedure(procExec, procId); + + final Procedure proc = procExec.getProcedure(procId); + for (int i = 0; i < lastStepBeforeFailover; ++i) { + LOG.info("Restart " + i + " exec state: " + proc); + ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); + MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec); + ProcedureTestingUtility.waitProcedure(procExec, procId); + } + ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); + + LOG.info("Trigger master failover"); + MasterProcedureTestingUtility.masterFailover(testUtil); + + procExec = testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor(); + ProcedureTestingUtility.waitProcedure(procExec, procId); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableWithMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableWithMasterFailover.java new file mode 100644 index 000000000000..3d4740a397b6 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableWithMasterFailover.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.ModifyRegionUtils; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.CreateTableState; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestCreateTableWithMasterFailover extends MasterFailoverWithProceduresTestBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestCreateTableWithMasterFailover.class); + + // ========================================================================== + // Test Create Table + // ========================================================================== + @Test + public void testCreateWithFailover() throws Exception { + // TODO: Should we try every step? (master failover takes long time) + // It is already covered by TestCreateTableProcedure + // but without the master restart, only the executor/store is restarted. + // Without Master restart we may not find bug in the procedure code + // like missing "wait" for resources to be available (e.g. RS) + testCreateWithFailoverAtStep(CreateTableState.CREATE_TABLE_ASSIGN_REGIONS.ordinal()); + } + + private void testCreateWithFailoverAtStep(final int step) throws Exception { + final TableName tableName = TableName.valueOf("testCreateWithFailoverAtStep" + step); + + // create the table + ProcedureExecutor procExec = getMasterProcedureExecutor(); + ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, true); + ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, true); + + // Start the Create procedure && kill the executor + byte[][] splitKeys = null; + TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2"); + RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); + long procId = + procExec.submitProcedure(new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); + testRecoveryAndDoubleExecution(UTIL, procId, step); + + MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(), + tableName, regions, "f1", "f2"); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableWithMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableWithMasterFailover.java new file mode 100644 index 000000000000..66990da40fa0 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableWithMasterFailover.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.DeleteTableState; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestDeleteTableWithMasterFailover extends MasterFailoverWithProceduresTestBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestDeleteTableWithMasterFailover.class); + + // ========================================================================== + // Test Delete Table + // ========================================================================== + @Test + public void testDeleteWithFailover() throws Exception { + // TODO: Should we try every step? (master failover takes long time) + // It is already covered by TestDeleteTableProcedure + // but without the master restart, only the executor/store is restarted. + // Without Master restart we may not find bug in the procedure code + // like missing "wait" for resources to be available (e.g. RS) + testDeleteWithFailoverAtStep(DeleteTableState.DELETE_TABLE_UNASSIGN_REGIONS.ordinal()); + } + + private void testDeleteWithFailoverAtStep(final int step) throws Exception { + final TableName tableName = TableName.valueOf("testDeleteWithFailoverAtStep" + step); + + // create the table + byte[][] splitKeys = null; + RegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(), + tableName, splitKeys, "f1", "f2"); + MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(), + tableName, regions, "f1", "f2"); + UTIL.getAdmin().disableTable(tableName); + + ProcedureExecutor procExec = getMasterProcedureExecutor(); + ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, true); + ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, true); + + // Start the Delete procedure && kill the executor + long procId = + procExec.submitProcedure(new DeleteTableProcedure(procExec.getEnvironment(), tableName)); + testRecoveryAndDoubleExecution(UTIL, procId, step); + + MasterProcedureTestingUtility.validateTableDeletion(UTIL.getHBaseCluster().getMaster(), + tableName); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDisableTableWithMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDisableTableWithMasterFailover.java new file mode 100644 index 000000000000..4a54b1c84988 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDisableTableWithMasterFailover.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.DisableTableState; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestDisableTableWithMasterFailover extends MasterFailoverWithProceduresTestBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestDisableTableWithMasterFailover.class); + + // ========================================================================== + // Test Disable Table + // ========================================================================== + @Test + public void testDisableTableWithFailover() throws Exception { + // TODO: Should we try every step? (master failover takes long time) + // It is already covered by TestDisableTableProcedure + // but without the master restart, only the executor/store is restarted. + // Without Master restart we may not find bug in the procedure code + // like missing "wait" for resources to be available (e.g. RS) + testDisableTableWithFailoverAtStep( + DisableTableState.DISABLE_TABLE_MARK_REGIONS_OFFLINE.ordinal()); + } + + private void testDisableTableWithFailoverAtStep(final int step) throws Exception { + final TableName tableName = TableName.valueOf("testDisableTableWithFailoverAtStep" + step); + + // create the table + final byte[][] splitKeys = + new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") }; + MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(), tableName, splitKeys, + "f1", "f2"); + + ProcedureExecutor procExec = getMasterProcedureExecutor(); + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); + + // Start the Delete procedure && kill the executor + long procId = procExec + .submitProcedure(new DisableTableProcedure(procExec.getEnvironment(), tableName, false)); + testRecoveryAndDoubleExecution(UTIL, procId, step); + + MasterProcedureTestingUtility.validateTableIsDisabled(UTIL.getHBaseCluster().getMaster(), + tableName); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestEnableTableWithMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestEnableTableWithMasterFailover.java new file mode 100644 index 000000000000..0e47cf4148eb --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestEnableTableWithMasterFailover.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.EnableTableState; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestEnableTableWithMasterFailover extends MasterFailoverWithProceduresTestBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestEnableTableWithMasterFailover.class); + + // ========================================================================== + // Test Enable Table + // ========================================================================== + @Test + public void testEnableTableWithFailover() throws Exception { + // TODO: Should we try every step? (master failover takes long time) + // It is already covered by TestEnableTableProcedure + // but without the master restart, only the executor/store is restarted. + // Without Master restart we may not find bug in the procedure code + // like missing "wait" for resources to be available (e.g. RS) + testEnableTableWithFailoverAtStep(EnableTableState.ENABLE_TABLE_MARK_REGIONS_ONLINE.ordinal()); + } + + private void testEnableTableWithFailoverAtStep(final int step) throws Exception { + final TableName tableName = TableName.valueOf("testEnableTableWithFailoverAtStep" + step); + + // create the table + final byte[][] splitKeys = + new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") }; + MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(), tableName, splitKeys, + "f1", "f2"); + UTIL.getAdmin().disableTable(tableName); + + ProcedureExecutor procExec = getMasterProcedureExecutor(); + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); + + // Start the Delete procedure && kill the executor + long procId = + procExec.submitProcedure(new EnableTableProcedure(procExec.getEnvironment(), tableName)); + testRecoveryAndDoubleExecution(UTIL, procId, step); + + MasterProcedureTestingUtility.validateTableIsEnabled(UTIL.getHBaseCluster().getMaster(), + tableName); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java deleted file mode 100644 index 90902c89b232..000000000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterFailoverWithProcedures.java +++ /dev/null @@ -1,335 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.procedure; - -import static org.junit.Assert.assertEquals; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseTestingUtil; -import org.apache.hadoop.hbase.StartTestingClusterOption; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.procedure2.Procedure; -import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; -import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; -import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.CommonFSUtils; -import org.apache.hadoop.hbase.util.ModifyRegionUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.CreateTableState; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.DeleteTableState; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.DisableTableState; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.EnableTableState; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.TruncateTableState; - -@Category({MasterTests.class, LargeTests.class}) -public class TestMasterFailoverWithProcedures { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestMasterFailoverWithProcedures.class); - - private static final Logger LOG = LoggerFactory.getLogger(TestMasterFailoverWithProcedures.class); - - protected static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); - - private static void setupConf(Configuration conf) { - // don't waste time retrying with the roll, the test is already slow enough. - conf.setInt(WALProcedureStore.MAX_RETRIES_BEFORE_ROLL_CONF_KEY, 1); - conf.setInt(WALProcedureStore.WAIT_BEFORE_ROLL_CONF_KEY, 0); - conf.setInt(WALProcedureStore.ROLL_RETRIES_CONF_KEY, 1); - conf.setInt(WALProcedureStore.MAX_SYNC_FAILURE_ROLL_CONF_KEY, 1); - conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); - } - - @Before - public void setup() throws Exception { - setupConf(UTIL.getConfiguration()); - // Set master number and use default values for other options. - StartTestingClusterOption option = StartTestingClusterOption.builder().numMasters(2).build(); - UTIL.startMiniCluster(option); - - final ProcedureExecutor procExec = getMasterProcedureExecutor(); - ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, false); - ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, false); - } - - @After - public void tearDown() throws Exception { - try { - UTIL.shutdownMiniCluster(); - } catch (Exception e) { - LOG.warn("failure shutting down cluster", e); - } - } - - // ========================================================================== - // Test Create Table - // ========================================================================== - @Test - public void testCreateWithFailover() throws Exception { - // TODO: Should we try every step? (master failover takes long time) - // It is already covered by TestCreateTableProcedure - // but without the master restart, only the executor/store is restarted. - // Without Master restart we may not find bug in the procedure code - // like missing "wait" for resources to be available (e.g. RS) - testCreateWithFailoverAtStep(CreateTableState.CREATE_TABLE_ASSIGN_REGIONS.ordinal()); - } - - private void testCreateWithFailoverAtStep(final int step) throws Exception { - final TableName tableName = TableName.valueOf("testCreateWithFailoverAtStep" + step); - - // create the table - ProcedureExecutor procExec = getMasterProcedureExecutor(); - ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, true); - ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, true); - - // Start the Create procedure && kill the executor - byte[][] splitKeys = null; - TableDescriptor htd = MasterProcedureTestingUtility.createHTD(tableName, "f1", "f2"); - RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); - long procId = procExec.submitProcedure( - new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); - testRecoveryAndDoubleExecution(UTIL, procId, step); - - MasterProcedureTestingUtility.validateTableCreation( - UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2"); - } - - // ========================================================================== - // Test Delete Table - // ========================================================================== - @Test - public void testDeleteWithFailover() throws Exception { - // TODO: Should we try every step? (master failover takes long time) - // It is already covered by TestDeleteTableProcedure - // but without the master restart, only the executor/store is restarted. - // Without Master restart we may not find bug in the procedure code - // like missing "wait" for resources to be available (e.g. RS) - testDeleteWithFailoverAtStep(DeleteTableState.DELETE_TABLE_UNASSIGN_REGIONS.ordinal()); - } - - private void testDeleteWithFailoverAtStep(final int step) throws Exception { - final TableName tableName = TableName.valueOf("testDeleteWithFailoverAtStep" + step); - - // create the table - byte[][] splitKeys = null; - RegionInfo[] regions = MasterProcedureTestingUtility.createTable( - getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2"); - Path tableDir = CommonFSUtils.getTableDir(getRootDir(), tableName); - MasterProcedureTestingUtility.validateTableCreation( - UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2"); - UTIL.getAdmin().disableTable(tableName); - - ProcedureExecutor procExec = getMasterProcedureExecutor(); - ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, true); - ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, true); - - // Start the Delete procedure && kill the executor - long procId = procExec.submitProcedure( - new DeleteTableProcedure(procExec.getEnvironment(), tableName)); - testRecoveryAndDoubleExecution(UTIL, procId, step); - - MasterProcedureTestingUtility.validateTableDeletion( - UTIL.getHBaseCluster().getMaster(), tableName); - } - - // ========================================================================== - // Test Truncate Table - // ========================================================================== - @Test - public void testTruncateWithFailover() throws Exception { - // TODO: Should we try every step? (master failover takes long time) - // It is already covered by TestTruncateTableProcedure - // but without the master restart, only the executor/store is restarted. - // Without Master restart we may not find bug in the procedure code - // like missing "wait" for resources to be available (e.g. RS) - testTruncateWithFailoverAtStep(true, TruncateTableState.TRUNCATE_TABLE_ADD_TO_META.ordinal()); - } - - private void testTruncateWithFailoverAtStep(final boolean preserveSplits, final int step) - throws Exception { - final TableName tableName = TableName.valueOf("testTruncateWithFailoverAtStep" + step); - - // create the table - final String[] families = new String[] { "f1", "f2" }; - final byte[][] splitKeys = new byte[][] { - Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") - }; - RegionInfo[] regions = MasterProcedureTestingUtility.createTable( - getMasterProcedureExecutor(), tableName, splitKeys, families); - // load and verify that there are rows in the table - MasterProcedureTestingUtility.loadData( - UTIL.getConnection(), tableName, 100, splitKeys, families); - assertEquals(100, UTIL.countRows(tableName)); - // disable the table - UTIL.getAdmin().disableTable(tableName); - - ProcedureExecutor procExec = getMasterProcedureExecutor(); - ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); - - // Start the Truncate procedure && kill the executor - long procId = procExec.submitProcedure( - new TruncateTableProcedure(procExec.getEnvironment(), tableName, preserveSplits)); - testRecoveryAndDoubleExecution(UTIL, procId, step); - - ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); - UTIL.waitUntilAllRegionsAssigned(tableName); - - // validate the table regions and layout - regions = UTIL.getAdmin().getRegions(tableName).toArray(new RegionInfo[0]); - if (preserveSplits) { - assertEquals(1 + splitKeys.length, regions.length); - } else { - assertEquals(1, regions.length); - } - MasterProcedureTestingUtility.validateTableCreation( - UTIL.getHBaseCluster().getMaster(), tableName, regions, families); - - // verify that there are no rows in the table - assertEquals(0, UTIL.countRows(tableName)); - - // verify that the table is read/writable - MasterProcedureTestingUtility.loadData( - UTIL.getConnection(), tableName, 50, splitKeys, families); - assertEquals(50, UTIL.countRows(tableName)); - } - - // ========================================================================== - // Test Disable Table - // ========================================================================== - @Test - public void testDisableTableWithFailover() throws Exception { - // TODO: Should we try every step? (master failover takes long time) - // It is already covered by TestDisableTableProcedure - // but without the master restart, only the executor/store is restarted. - // Without Master restart we may not find bug in the procedure code - // like missing "wait" for resources to be available (e.g. RS) - testDisableTableWithFailoverAtStep( - DisableTableState.DISABLE_TABLE_MARK_REGIONS_OFFLINE.ordinal()); - } - - private void testDisableTableWithFailoverAtStep(final int step) throws Exception { - final TableName tableName = TableName.valueOf("testDisableTableWithFailoverAtStep" + step); - - // create the table - final byte[][] splitKeys = new byte[][] { - Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") - }; - MasterProcedureTestingUtility.createTable( - getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2"); - - ProcedureExecutor procExec = getMasterProcedureExecutor(); - ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); - - // Start the Delete procedure && kill the executor - long procId = procExec.submitProcedure( - new DisableTableProcedure(procExec.getEnvironment(), tableName, false)); - testRecoveryAndDoubleExecution(UTIL, procId, step); - - MasterProcedureTestingUtility.validateTableIsDisabled( - UTIL.getHBaseCluster().getMaster(), tableName); - } - - // ========================================================================== - // Test Enable Table - // ========================================================================== - @Test - public void testEnableTableWithFailover() throws Exception { - // TODO: Should we try every step? (master failover takes long time) - // It is already covered by TestEnableTableProcedure - // but without the master restart, only the executor/store is restarted. - // Without Master restart we may not find bug in the procedure code - // like missing "wait" for resources to be available (e.g. RS) - testEnableTableWithFailoverAtStep( - EnableTableState.ENABLE_TABLE_MARK_REGIONS_ONLINE.ordinal()); - } - - private void testEnableTableWithFailoverAtStep(final int step) throws Exception { - final TableName tableName = TableName.valueOf("testEnableTableWithFailoverAtStep" + step); - - // create the table - final byte[][] splitKeys = new byte[][] { - Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") - }; - MasterProcedureTestingUtility.createTable( - getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2"); - UTIL.getAdmin().disableTable(tableName); - - ProcedureExecutor procExec = getMasterProcedureExecutor(); - ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); - - // Start the Delete procedure && kill the executor - long procId = procExec.submitProcedure( - new EnableTableProcedure(procExec.getEnvironment(), tableName)); - testRecoveryAndDoubleExecution(UTIL, procId, step); - - MasterProcedureTestingUtility.validateTableIsEnabled( - UTIL.getHBaseCluster().getMaster(), tableName); - } - - // ========================================================================== - // Test Helpers - // ========================================================================== - public static void testRecoveryAndDoubleExecution(final HBaseTestingUtil testUtil, - final long procId, final int lastStepBeforeFailover) throws Exception { - ProcedureExecutor procExec = - testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor(); - ProcedureTestingUtility.waitProcedure(procExec, procId); - - final Procedure proc = procExec.getProcedure(procId); - for (int i = 0; i < lastStepBeforeFailover; ++i) { - LOG.info("Restart "+ i +" exec state: " + proc); - ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); - MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec); - ProcedureTestingUtility.waitProcedure(procExec, procId); - } - ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); - - LOG.info("Trigger master failover"); - MasterProcedureTestingUtility.masterFailover(testUtil); - - procExec = testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor(); - ProcedureTestingUtility.waitProcedure(procExec, procId); - ProcedureTestingUtility.assertProcNotFailed(procExec, procId); - } - - // ========================================================================== - // Helpers - // ========================================================================== - private ProcedureExecutor getMasterProcedureExecutor() { - return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); - } - - private Path getRootDir() { - return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableWithMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableWithMasterFailover.java new file mode 100644 index 000000000000..4ee39c9f6641 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableWithMasterFailover.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.TruncateTableState; + +@Category({ MasterTests.class, MediumTests.class }) +public class TestTruncateTableWithMasterFailover extends MasterFailoverWithProceduresTestBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestTruncateTableWithMasterFailover.class); + + // ========================================================================== + // Test Truncate Table + // ========================================================================== + @Test + public void testTruncateWithFailover() throws Exception { + // TODO: Should we try every step? (master failover takes long time) + // It is already covered by TestTruncateTableProcedure + // but without the master restart, only the executor/store is restarted. + // Without Master restart we may not find bug in the procedure code + // like missing "wait" for resources to be available (e.g. RS) + testTruncateWithFailoverAtStep(true, TruncateTableState.TRUNCATE_TABLE_ADD_TO_META.ordinal()); + } + + private void testTruncateWithFailoverAtStep(final boolean preserveSplits, final int step) + throws Exception { + final TableName tableName = TableName.valueOf("testTruncateWithFailoverAtStep" + step); + + // create the table + final String[] families = new String[] { "f1", "f2" }; + final byte[][] splitKeys = + new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") }; + RegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(), + tableName, splitKeys, families); + // load and verify that there are rows in the table + MasterProcedureTestingUtility.loadData(UTIL.getConnection(), tableName, 100, splitKeys, + families); + assertEquals(100, UTIL.countRows(tableName)); + // disable the table + UTIL.getAdmin().disableTable(tableName); + + ProcedureExecutor procExec = getMasterProcedureExecutor(); + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); + + // Start the Truncate procedure && kill the executor + long procId = procExec.submitProcedure( + new TruncateTableProcedure(procExec.getEnvironment(), tableName, preserveSplits)); + testRecoveryAndDoubleExecution(UTIL, procId, step); + + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); + UTIL.waitUntilAllRegionsAssigned(tableName); + + // validate the table regions and layout + regions = UTIL.getAdmin().getRegions(tableName).toArray(new RegionInfo[0]); + if (preserveSplits) { + assertEquals(1 + splitKeys.length, regions.length); + } else { + assertEquals(1, regions.length); + } + MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(), + tableName, regions, families); + + // verify that there are no rows in the table + assertEquals(0, UTIL.countRows(tableName)); + + // verify that the table is read/writable + MasterProcedureTestingUtility.loadData(UTIL.getConnection(), tableName, 50, splitKeys, + families); + assertEquals(50, UTIL.countRows(tableName)); + } +}