Skip to content

HBASE-22982: region server suspend/resume and graceful rolling restart actions #592

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
meszibalu merged 5 commits into from
Sep 26, 2019
Merged

HBASE-22982: region server suspend/resume and graceful rolling restart actions #592

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e66abf1
HBASE-22982
BukrosSzabolcs Sep 6, 2019
f8b6b0f
HBASE-22982
BukrosSzabolcs Sep 9, 2019
5cc76d1
HBASE-22982
BukrosSzabolcs Sep 18, 2019
a86975a
HBASE-22982
BukrosSzabolcs Sep 25, 2019
6285cb1
HBASE-22982
BukrosSzabolcs Sep 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 48 additions & 34 deletions hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,13 +97,13 @@ public void close() throws IOException {

@Override
public void startRegionServer(String hostname, int port) throws IOException {
LOG.info("Starting RS on: " + hostname);
LOG.info("Starting RS on: {}", hostname);
clusterManager.start(ServiceType.HBASE_REGIONSERVER, hostname, port);
}

@Override
public void killRegionServer(ServerName serverName) throws IOException {
LOG.info("Aborting RS: " + serverName.getServerName());
LOG.info("Aborting RS: {}", serverName.getServerName());
killedRegionServers.add(serverName);
clusterManager.kill(ServiceType.HBASE_REGIONSERVER,
serverName.getHostname(), serverName.getPort());
Expand All @@ -116,7 +116,7 @@ public boolean isKilledRS(ServerName serverName) {

@Override
public void stopRegionServer(ServerName serverName) throws IOException {
LOG.info("Stopping RS: " + serverName.getServerName());
LOG.info("Stopping RS: {}", serverName.getServerName());
clusterManager.stop(ServiceType.HBASE_REGIONSERVER,
serverName.getHostname(), serverName.getPort());
}
Expand All @@ -126,22 +126,36 @@ public void waitForRegionServerToStop(ServerName serverName, long timeout) throw
waitForServiceToStop(ServiceType.HBASE_REGIONSERVER, serverName, timeout);
}

@Override
public void suspendRegionServer(ServerName serverName) throws IOException {
LOG.info("Suspend RS: {}", serverName.getServerName());
clusterManager.suspend(ServiceType.HBASE_REGIONSERVER,
serverName.getHostname(), serverName.getPort());
}

@Override
public void resumeRegionServer(ServerName serverName) throws IOException {
LOG.info("Resume RS: {}", serverName.getServerName());
clusterManager.resume(ServiceType.HBASE_REGIONSERVER,
serverName.getHostname(), serverName.getPort());
}

@Override
public void startZkNode(String hostname, int port) throws IOException {
LOG.info("Starting ZooKeeper node on: " + hostname);
LOG.info("Starting ZooKeeper node on: {}", hostname);
clusterManager.start(ServiceType.ZOOKEEPER_SERVER, hostname, port);
}

@Override
public void killZkNode(ServerName serverName) throws IOException {
LOG.info("Aborting ZooKeeper node on: " + serverName.getServerName());
LOG.info("Aborting ZooKeeper node on: {}", serverName.getServerName());
clusterManager.kill(ServiceType.ZOOKEEPER_SERVER,
serverName.getHostname(), serverName.getPort());
}

@Override
public void stopZkNode(ServerName serverName) throws IOException {
LOG.info("Stopping ZooKeeper node: " + serverName.getServerName());
LOG.info("Stopping ZooKeeper node: {}", serverName.getServerName());
clusterManager.stop(ServiceType.ZOOKEEPER_SERVER,
serverName.getHostname(), serverName.getPort());
}
Expand All @@ -158,21 +172,21 @@ public void waitForZkNodeToStop(ServerName serverName, long timeout) throws IOEx

@Override
public void startDataNode(ServerName serverName) throws IOException {
LOG.info("Starting data node on: " + serverName.getServerName());
LOG.info("Starting data node on: {}", serverName.getServerName());
clusterManager.start(ServiceType.HADOOP_DATANODE,
serverName.getHostname(), serverName.getPort());
}

@Override
public void killDataNode(ServerName serverName) throws IOException {
LOG.info("Aborting data node on: " + serverName.getServerName());
LOG.info("Aborting data node on: {}", serverName.getServerName());
clusterManager.kill(ServiceType.HADOOP_DATANODE,
serverName.getHostname(), serverName.getPort());
}

@Override
public void stopDataNode(ServerName serverName) throws IOException {
LOG.info("Stopping data node on: " + serverName.getServerName());
LOG.info("Stopping data node on: {}", serverName.getServerName());
clusterManager.stop(ServiceType.HADOOP_DATANODE,
serverName.getHostname(), serverName.getPort());
}
Expand All @@ -189,21 +203,21 @@ public void waitForDataNodeToStop(ServerName serverName, long timeout) throws IO

@Override
public void startNameNode(ServerName serverName) throws IOException {
LOG.info("Starting name node on: " + serverName.getServerName());
LOG.info("Starting name node on: {}", serverName.getServerName());
clusterManager.start(ServiceType.HADOOP_NAMENODE, serverName.getHostname(),
serverName.getPort());
}

@Override
public void killNameNode(ServerName serverName) throws IOException {
LOG.info("Aborting name node on: " + serverName.getServerName());
LOG.info("Aborting name node on: {}", serverName.getServerName());
clusterManager.kill(ServiceType.HADOOP_NAMENODE, serverName.getHostname(),
serverName.getPort());
}

@Override
public void stopNameNode(ServerName serverName) throws IOException {
LOG.info("Stopping name node on: " + serverName.getServerName());
LOG.info("Stopping name node on: {}", serverName.getServerName());
clusterManager.stop(ServiceType.HADOOP_NAMENODE, serverName.getHostname(),
serverName.getPort());
}
Expand All @@ -220,7 +234,7 @@ public void waitForNameNodeToStop(ServerName serverName, long timeout) throws IO

private void waitForServiceToStop(ServiceType service, ServerName serverName, long timeout)
throws IOException {
LOG.info("Waiting for service: " + service + " to stop: " + serverName.getServerName());
LOG.info("Waiting for service: {} to stop: {}", service, serverName.getServerName());
long start = System.currentTimeMillis();

while ((System.currentTimeMillis() - start) < timeout) {
Expand All @@ -234,7 +248,7 @@ private void waitForServiceToStop(ServiceType service, ServerName serverName, lo

private void waitForServiceToStart(ServiceType service, ServerName serverName, long timeout)
throws IOException {
LOG.info("Waiting for service: " + service + " to start: " + serverName.getServerName());
LOG.info("Waiting for service: {} to start: ", service, serverName.getServerName());
long start = System.currentTimeMillis();

while ((System.currentTimeMillis() - start) < timeout) {
Expand All @@ -248,19 +262,19 @@ private void waitForServiceToStart(ServiceType service, ServerName serverName, l

@Override
public void startMaster(String hostname, int port) throws IOException {
LOG.info("Starting Master on: " + hostname + ":" + port);
LOG.info("Starting Master on: {}:{}", hostname, port);
clusterManager.start(ServiceType.HBASE_MASTER, hostname, port);
}

@Override
public void killMaster(ServerName serverName) throws IOException {
LOG.info("Aborting Master: " + serverName.getServerName());
LOG.info("Aborting Master: {}", serverName.getServerName());
clusterManager.kill(ServiceType.HBASE_MASTER, serverName.getHostname(), serverName.getPort());
}

@Override
public void stopMaster(ServerName serverName) throws IOException {
LOG.info("Stopping Master: " + serverName.getServerName());
LOG.info("Stopping Master: {}", serverName.getServerName());
clusterManager.stop(ServiceType.HBASE_MASTER, serverName.getHostname(), serverName.getPort());
}

Expand Down Expand Up @@ -294,7 +308,7 @@ public ServerName getServerHoldingRegion(TableName tn, byte[] regionName) throws
regionLoc = locator.getRegionLocation(startKey, true);
}
if (regionLoc == null) {
LOG.warn("Cannot find region server holding region " + Bytes.toStringBinary(regionName));
LOG.warn("Cannot find region server holding region {}", Bytes.toStringBinary(regionName));
return null;
}
return regionLoc.getServerName();
Expand Down Expand Up @@ -338,15 +352,15 @@ protected boolean restoreMasters(ClusterMetrics initial, ClusterMetrics current)
//check whether current master has changed
final ServerName initMaster = initial.getMasterName();
if (!ServerName.isSameAddress(initMaster, current.getMasterName())) {
LOG.info("Restoring cluster - Initial active master : " + initMaster.getAddress() +
" has changed to : " + current.getMasterName().getAddress());
LOG.info("Restoring cluster - Initial active master : {} has changed to : {}",
initMaster.getAddress(), current.getMasterName().getAddress());
// If initial master is stopped, start it, before restoring the state.
// It will come up as a backup master, if there is already an active master.
try {
if (!clusterManager.isRunning(ServiceType.HBASE_MASTER,
initMaster.getHostname(), initMaster.getPort())) {
LOG.info("Restoring cluster - starting initial active master at:"
+ initMaster.getAddress());
LOG.info("Restoring cluster - starting initial active master at:{}",
initMaster.getAddress());
startMaster(initMaster.getHostname(), initMaster.getPort());
}

Expand All @@ -356,11 +370,11 @@ protected boolean restoreMasters(ClusterMetrics initial, ClusterMetrics current)
// 3. Start backup masters
for (ServerName currentBackup : current.getBackupMasterNames()) {
if (!ServerName.isSameAddress(currentBackup, initMaster)) {
LOG.info("Restoring cluster - stopping backup master: " + currentBackup);
LOG.info("Restoring cluster - stopping backup master: {}", currentBackup);
stopMaster(currentBackup);
}
}
LOG.info("Restoring cluster - stopping active master: " + current.getMasterName());
LOG.info("Restoring cluster - stopping active master: {}", current.getMasterName());
stopMaster(current.getMasterName());
waitForActiveAndReadyMaster(); // wait so that active master takes over
} catch (IOException ex) {
Expand All @@ -376,8 +390,8 @@ protected boolean restoreMasters(ClusterMetrics initial, ClusterMetrics current)
if (!clusterManager.isRunning(ServiceType.HBASE_MASTER,
backup.getHostname(),
backup.getPort())) {
LOG.info("Restoring cluster - starting initial backup master: "
+ backup.getAddress());
LOG.info("Restoring cluster - starting initial backup master: {}",
backup.getAddress());
startMaster(backup.getHostname(), backup.getPort());
}
} catch (IOException ex) {
Expand All @@ -401,7 +415,7 @@ protected boolean restoreMasters(ClusterMetrics initial, ClusterMetrics current)
for (ServerName sn:toStart) {
try {
if(!clusterManager.isRunning(ServiceType.HBASE_MASTER, sn.getHostname(), sn.getPort())) {
LOG.info("Restoring cluster - starting initial backup master: " + sn.getAddress());
LOG.info("Restoring cluster - starting initial backup master: {}", sn.getAddress());
startMaster(sn.getHostname(), sn.getPort());
}
} catch (IOException ex) {
Expand All @@ -412,7 +426,7 @@ protected boolean restoreMasters(ClusterMetrics initial, ClusterMetrics current)
for (ServerName sn:toKill) {
try {
if(clusterManager.isRunning(ServiceType.HBASE_MASTER, sn.getHostname(), sn.getPort())) {
LOG.info("Restoring cluster - stopping backup master: " + sn.getAddress());
LOG.info("Restoring cluster - stopping backup master: {}", sn.getAddress());
stopMaster(sn);
}
} catch (IOException ex) {
Expand All @@ -421,8 +435,8 @@ protected boolean restoreMasters(ClusterMetrics initial, ClusterMetrics current)
}
}
if (!deferred.isEmpty()) {
LOG.warn("Restoring cluster - restoring region servers reported "
+ deferred.size() + " errors:");
LOG.warn("Restoring cluster - restoring region servers reported {} errors:",
deferred.size());
for (int i=0; i<deferred.size() && i < 3; i++) {
LOG.warn(Objects.toString(deferred.get(i)));
}
Expand Down Expand Up @@ -464,7 +478,7 @@ protected boolean restoreRegionServers(ClusterMetrics initial, ClusterMetrics cu
try {
if (!clusterManager.isRunning(ServiceType.HBASE_REGIONSERVER, sn.getHostname(),
sn.getPort()) && master.getPort() != sn.getPort()) {
LOG.info("Restoring cluster - starting initial region server: " + sn.getAddress());
LOG.info("Restoring cluster - starting initial region server: {}", sn.getAddress());
startRegionServer(sn.getHostname(), sn.getPort());
}
} catch (IOException ex) {
Expand All @@ -476,16 +490,16 @@ protected boolean restoreRegionServers(ClusterMetrics initial, ClusterMetrics cu
try {
if (clusterManager.isRunning(ServiceType.HBASE_REGIONSERVER, sn.getHostname(),
sn.getPort()) && master.getPort() != sn.getPort()) {
LOG.info("Restoring cluster - stopping initial region server: " + sn.getAddress());
LOG.info("Restoring cluster - stopping initial region server: {}", sn.getAddress());
stopRegionServer(sn);
}
} catch (IOException ex) {
deferred.add(ex);
}
}
if (!deferred.isEmpty()) {
LOG.warn("Restoring cluster - restoring region servers reported "
+ deferred.size() + " errors:");
LOG.warn("Restoring cluster - restoring region servers reported {} errors:",
deferred.size());
for (int i=0; i<deferred.size() && i < 3; i++) {
LOG.warn(Objects.toString(deferred.get(i)));
}
Expand Down
Loading