Skip to content

HBASE-29386: SnapshotProcedure and EnableTableProcedure can cause a deadlock #7084

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rmdmattingly merged 1 commit into from
Jun 26, 2025
Merged

HBASE-29386: SnapshotProcedure and EnableTableProcedure can cause a deadlock #7084

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 0 additions & 24 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SnapshotProcedure.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,30 +103,6 @@ public TableOperationType getTableOperationType() {
return TableOperationType.SNAPSHOT;
}

@Override
protected LockState acquireLock(MasterProcedureEnv env) {
// AbstractStateMachineTableProcedure acquires exclusive table lock by default,
// but we may need to downgrade it to shared lock for some reasons:
// a. exclusive lock has a negative effect on assigning region. See HBASE-21480 for details.
// b. we want to support taking multiple different snapshots on same table on the same time.
if (env.getProcedureScheduler().waitTableSharedLock(this, getTableName())) {
return LockState.LOCK_EVENT_WAIT;
}
return LockState.LOCK_ACQUIRED;
}

@Override
protected void releaseLock(MasterProcedureEnv env) {
env.getProcedureScheduler().wakeTableSharedLock(this, getTableName());
}

@Override
protected boolean holdLock(MasterProcedureEnv env) {
// In order to avoid enabling/disabling/modifying/deleting table during snapshot,
// we don't release lock during suspend
return true;
}

@Override
protected Flow executeFromState(MasterProcedureEnv env, SnapshotState state)
throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
Expand Down
2 changes: 1 addition & 1 deletion hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,14 @@ static boolean requireTableExclusiveLock(TableProcedureInterface proc) {
case CREATE:
case DELETE:
case DISABLE:
case SNAPSHOT:
case ENABLE:
return true;
case EDIT:
// we allow concurrent edit on the ns family in meta table
return !proc.getTableName().equals(TableProcedureInterface.DUMMY_NAMESPACE_TABLE_NAME);
case READ:
case FLUSH:
case SNAPSHOT:
return false;
// region operations are using the shared-lock on the table
// and then they will grab an xlock on the region.
Expand Down
57 changes: 57 additions & 0 deletions ...test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureConcurrently.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.master.procedure;

import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.EnableTableState.ENABLE_TABLE_MARK_REGIONS_ONLINE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
Expand All @@ -27,6 +28,7 @@
import java.util.List;
import java.util.stream.Collectors;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.hbase.client.SnapshotType;
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
Expand Down Expand Up @@ -136,4 +138,59 @@ public void run() {
SnapshotTestingUtils.confirmSnapshotValid(TEST_UTIL, snapshotProto, TABLE_NAME, CF);
SnapshotTestingUtils.confirmSnapshotValid(TEST_UTIL, snapshotOnSameTableProto, TABLE_NAME, CF);
}

@Test
public void testItFailsIfTableIsNotDisabledOrEnabled() throws Exception {
ProcedureExecutor<MasterProcedureEnv> executor = master.getMasterProcedureExecutor();
MasterProcedureEnv env = executor.getEnvironment();
TEST_UTIL.getAdmin().disableTable(TABLE_NAME);

TestEnableTableProcedure enableTable = new TestEnableTableProcedure(
master.getMasterProcedureExecutor().getEnvironment(), TABLE_NAME);
long enableProcId = executor.submitProcedure(enableTable);
TEST_UTIL.waitFor(60000, () -> {
Procedure<MasterProcedureEnv> proc = executor.getProcedure(enableProcId);
if (proc == null) {
return false;
}
return ((TestEnableTableProcedure) proc).getProcedureState()
== ENABLE_TABLE_MARK_REGIONS_ONLINE;
});

// Using a delayed spy ensures we hit the problem state while the table enable procedure
// is waiting to run
SnapshotProcedure snapshotProc = new SnapshotProcedure(env, snapshotProto);
long snapshotProcId = executor.submitProcedure(snapshotProc);
TEST_UTIL.waitTableEnabled(TABLE_NAME);
// Wait for procedure to run and finish
TEST_UTIL.waitFor(60000, () -> executor.getProcedure(snapshotProcId) != null);
TEST_UTIL.waitFor(60000, () -> executor.getProcedure(snapshotProcId) == null);

SnapshotTestingUtils.confirmSnapshotValid(TEST_UTIL, snapshotProto, TABLE_NAME, CF);
}

// Needs to be publicly accessible for Procedure validation
public static class TestEnableTableProcedure extends EnableTableProcedure {
// Necessary for Procedure validation
public TestEnableTableProcedure() {
}

public TestEnableTableProcedure(MasterProcedureEnv env, TableName tableName) {
super(env, tableName);
}

public MasterProcedureProtos.EnableTableState getProcedureState() {
return getState(stateCount);
}

@Override
protected Flow executeFromState(MasterProcedureEnv env,
MasterProcedureProtos.EnableTableState state) throws InterruptedException {
if (state == ENABLE_TABLE_MARK_REGIONS_ONLINE) {
Thread.sleep(10000);
}

return super.executeFromState(env, state);
}
}
}
3 changes: 1 addition & 2 deletions ...rver/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureRIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,7 @@ public void testTableInMergeWhileTakingSnapshot() throws Exception {
() -> procExec.getProcedure(mergeProcId).getState() == ProcedureState.RUNNABLE);
SnapshotProcedure sp = new SnapshotProcedure(procExec.getEnvironment(), snapshotProto);
long snapshotProcId = procExec.submitProcedure(sp);
TEST_UTIL.waitFor(2000, 1000, () -> procExec.getProcedure(snapshotProcId) != null
Copy link
Contributor

@Apache9 Apache9 Jun 25, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Mind explaining a bit why we need to remove the WAITING_TIMEOUT state check?

Copy link
Contributor Author

@hgromer hgromer Jun 25, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure; the SnapshotProcedure will never reach the state where it is running at the same time that the merge procedures are running because it will now require an exclusive locks of the table to run. So when we kick off the SnapshotProcedure, it will sit in the TableProcedureWaitingQueue as RUNNABLE before being scheduled. Therefore, this code path will never be executed. We can clean this code up too if we'd like

&& procExec.getProcedure(snapshotProcId).getState() == ProcedureState.WAITING_TIMEOUT);
TEST_UTIL.waitFor(2000, 1000, () -> procExec.getProcedure(snapshotProcId) != null);
ProcedureTestingUtility.waitProcedure(procExec, snapshotProcId);
SnapshotTestingUtils.confirmSnapshotValid(TEST_UTIL, snapshotProto, TABLE_NAME, CF);
}
Expand Down