diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ZNodeClearer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ZNodeClearer.java index 1cde2fa24844..b884669fe645 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ZNodeClearer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ZNodeClearer.java @@ -26,13 +26,9 @@ import java.io.FileWriter; import java.io.IOException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; -import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; -import org.apache.hadoop.hbase.zookeeper.ZNodePaths; import org.apache.yetus.audience.InterfaceAudience; -import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -145,21 +141,6 @@ public static String parseMasterServerName(String rsZnodePath) { return masterServerName; } - /** - * @return true if cluster is configured with master-rs collocation - * @deprecated since 2.4.0, will be removed in 3.0.0. - * @see HBASE-15549 - */ - @Deprecated - private static boolean tablesOnMaster(Configuration conf) { - boolean tablesOnMaster = true; - String confValue = conf.get(BaseLoadBalancer.TABLES_ON_MASTER); - if (confValue != null && confValue.equalsIgnoreCase("none")) { - tablesOnMaster = false; - } - return tablesOnMaster; - } - /** * Delete the master znode if its content (ServerName string) is the same * as the one in the znode file. (env: HBASE_ZNODE_FILE). I case of master-rs @@ -185,15 +166,7 @@ public static boolean clear(Configuration conf) { String znodeFileContent; try { znodeFileContent = ZNodeClearer.readMyEphemeralNodeOnDisk(); - if (ZNodeClearer.tablesOnMaster(conf)) { - // In case of master crash also remove rsZnode since master is also regionserver - ZKUtil.deleteNodeFailSilent(zkw, - ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode, znodeFileContent)); - return MasterAddressTracker.deleteIfEquals(zkw, - ZNodeClearer.parseMasterServerName(znodeFileContent)); - } else { - return MasterAddressTracker.deleteIfEquals(zkw, znodeFileContent); - } + return MasterAddressTracker.deleteIfEquals(zkw, znodeFileContent); } catch (FileNotFoundException fnfe) { // If no file, just keep going -- return success. LOG.warn("Can't find the znode file; presume non-fatal", fnfe); @@ -201,9 +174,6 @@ public static boolean clear(Configuration conf) { } catch (IOException e) { LOG.warn("Can't read the content of the znode file", e); return false; - } catch (KeeperException e) { - LOG.warn("ZooKeeper exception deleting znode", e); - return false; } finally { zkw.close(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 985ba128d731..3c57407bf360 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -605,17 +605,14 @@ protected void login(UserProvider user, String host) throws IOException { } /** - * If configured to put regions on active master, - * wait till a backup master becomes active. - * Otherwise, loop till the server is stopped or aborted. + * Loop till the server is stopped or aborted. */ @Override - protected void waitForMasterActive(){ + protected void waitForMasterActive() { if (maintenanceMode) { return; } - boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(conf); - while (!(tablesOnMaster && activeMaster) && !isStopped() && !isAborted()) { + while (!isStopped() && !isAborted()) { sleeper.sleep(); } } @@ -658,7 +655,7 @@ protected RSRpcServices createRpcServices() throws IOException { protected void configureInfoServer() { infoServer.addUnprivilegedServlet("master-status", "/master-status", MasterStatusServlet.class); infoServer.setAttribute(MASTER, this); - if (LoadBalancer.isTablesOnMaster(conf)) { + if (maintenanceMode) { super.configureInfoServer(); } } @@ -3703,7 +3700,7 @@ public SyncReplicationReplayWALManager getSyncReplicationReplayWALManager() { @Override public Map getWalGroupsReplicationStatus() { - if (!this.isOnline() || !LoadBalancer.isMasterCanHostUserRegions(conf)) { + if (!this.isOnline() || !maintenanceMode) { return new HashMap<>(); } return super.getWalGroupsReplicationStatus(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java index d908aa5ef514..b49ca824236a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java @@ -48,25 +48,6 @@ */ @InterfaceAudience.Private public interface LoadBalancer extends Configurable, Stoppable, ConfigurationObserver { - /** - * Master can carry regions as of hbase-2.0.0. - * By default, it carries no tables. - * TODO: Add any | system as flags to indicate what it can do. - * - * @deprecated since 2.4.0, will be removed in 3.0.0. - * @see HBASE-15549 - */ - @Deprecated - String TABLES_ON_MASTER = "hbase.balancer.tablesOnMaster"; - - /** - * Master carries system tables. - * - * @deprecated since 2.4.0, will be removed in 3.0.0. - * @see HBASE-15549 - */ - @Deprecated - String SYSTEM_TABLES_ON_MASTER = "hbase.balancer.tablesOnMaster.systemTablesOnly"; // Used to signal to the caller that the region(s) cannot be assigned // We deliberately use 'localhost' so the operation will fail fast @@ -164,32 +145,4 @@ Map> retainAssignment(Map r /*Updates balancer status tag reported to JMX*/ void updateBalancerStatus(boolean status); - - /** - * @return true if Master carries regions - * @deprecated since 2.4.0, will be removed in 3.0.0. - * @see HBASE-15549 - */ - @Deprecated - static boolean isTablesOnMaster(Configuration conf) { - return conf.getBoolean(TABLES_ON_MASTER, false); - } - - /** - * @deprecated since 2.4.0, will be removed in 3.0.0. - * @see HBASE-15549 - */ - @Deprecated - static boolean isSystemTablesOnlyOnMaster(Configuration conf) { - return conf.getBoolean(SYSTEM_TABLES_ON_MASTER, false); - } - - /** - * @deprecated since 2.4.0, will be removed in 3.0.0. - * @see HBASE-15549 - */ - @Deprecated - static boolean isMasterCanHostUserRegions(Configuration conf) { - return isTablesOnMaster(conf) && !isSystemTablesOnlyOnMaster(conf); - } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index f5c6441c8d5a..772e1a3de8f3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -459,8 +459,7 @@ protected RpcServerInterface createRpcServer(final Server server, final String name) throws IOException { final Configuration conf = regionServer.getConfiguration(); // RpcServer at HM by default enable ByteBufferPool iff HM having user table region in it - boolean reservoirEnabled = conf.getBoolean(ByteBuffAllocator.ALLOCATOR_POOL_ENABLED_KEY, - LoadBalancer.isMasterCanHostUserRegions(conf)); + boolean reservoirEnabled = conf.getBoolean(ByteBuffAllocator.ALLOCATOR_POOL_ENABLED_KEY, false); try { return RpcServerFactory.createRpcServer(server, name, getServices(), bindAddress, // use final bindAddress for this server. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index c9ebd89e9288..52664143d23d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -45,7 +45,6 @@ import org.apache.hadoop.hbase.RegionMetrics; import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.ServerMetrics; -import org.apache.hadoop.hbase.ServerMetricsBuilder; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.YouAreDeadException; import org.apache.hadoop.hbase.client.AsyncClusterConnection; @@ -736,13 +735,6 @@ private int getMinToStart() { } int minimumRequired = 1; - if (LoadBalancer.isTablesOnMaster(master.getConfiguration()) && - LoadBalancer.isSystemTablesOnlyOnMaster(master.getConfiguration())) { - // If Master is carrying regions it will show up as a 'server', but is not handling user- - // space regions, so we need a second server. - minimumRequired = 2; - } - int minToStart = this.master.getConfiguration().getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, -1); // Ensure we are never less than minimumRequired else stuff won't work. return Math.max(minToStart, minimumRequired); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index 6c2e86d1cdb9..0de4b21b0416 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -27,7 +27,6 @@ import java.util.Comparator; import java.util.Deque; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -37,7 +36,6 @@ import java.util.TreeMap; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Predicate; -import java.util.stream.Collectors; import org.apache.commons.lang3.NotImplementedException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterMetrics; @@ -1034,13 +1032,6 @@ public String toString() { protected ServerName masterServerName; protected MasterServices services; - /** - * @deprecated since 2.4.0, will be removed in 3.0.0. - * @see HBASE-15549 - */ - @Deprecated - protected boolean onlySystemTablesOnMaster; - @Override public void setConf(Configuration conf) { this.config = conf; @@ -1057,15 +1048,13 @@ public void setConf(Configuration conf) { overallSlop = 1; } - this.onlySystemTablesOnMaster = LoadBalancer.isSystemTablesOnlyOnMaster(this.config); - this.rackManager = new RackManager(getConf()); if (useRegionFinder) { regionFinder.setConf(conf); } this.isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable); // Print out base configs. Don't print overallSlop since it for simple balancer exclusively. - LOG.info("slop={}, systemTablesOnMaster={}", this.slop, this.onlySystemTablesOnMaster); + LOG.info("slop={}", this.slop); } protected void setSlop(Configuration conf) { @@ -1073,95 +1062,6 @@ protected void setSlop(Configuration conf) { this.overallSlop = conf.getFloat("hbase.regions.overallSlop", slop); } - /** - * Check if a region belongs to some system table. - * If so, the primary replica may be expected to be put on the master regionserver. - * - * @deprecated since 2.4.0, will be removed in 3.0.0. - * @see HBASE-15549 - */ - @Deprecated - public boolean shouldBeOnMaster(RegionInfo region) { - return this.onlySystemTablesOnMaster && region.getTable().isSystemTable(); - } - - /** - * Balance the regions that should be on master regionserver. - * - * @deprecated since 2.4.0, will be removed in 3.0.0. - * @see HBASE-15549 - */ - @Deprecated - protected List balanceMasterRegions(Map> clusterMap) { - if (masterServerName == null || clusterMap == null || clusterMap.size() <= 1) return null; - List plans = null; - List regions = clusterMap.get(masterServerName); - if (regions != null) { - Iterator keyIt = null; - for (RegionInfo region: regions) { - if (shouldBeOnMaster(region)) continue; - - // Find a non-master regionserver to host the region - if (keyIt == null || !keyIt.hasNext()) { - keyIt = clusterMap.keySet().iterator(); - } - ServerName dest = keyIt.next(); - if (masterServerName.equals(dest)) { - if (!keyIt.hasNext()) { - keyIt = clusterMap.keySet().iterator(); - } - dest = keyIt.next(); - } - - // Move this region away from the master regionserver - RegionPlan plan = new RegionPlan(region, masterServerName, dest); - if (plans == null) { - plans = new ArrayList<>(); - } - plans.add(plan); - } - } - for (Map.Entry> server: clusterMap.entrySet()) { - if (masterServerName.equals(server.getKey())) continue; - for (RegionInfo region: server.getValue()) { - if (!shouldBeOnMaster(region)) continue; - - // Move this region to the master regionserver - RegionPlan plan = new RegionPlan(region, server.getKey(), masterServerName); - if (plans == null) { - plans = new ArrayList<>(); - } - plans.add(plan); - } - } - return plans; - } - - /** - * If master is configured to carry system tables only, in here is - * where we figure what to assign it. - * - * @deprecated since 2.4.0, will be removed in 3.0.0. - * @see HBASE-15549 - */ - @Deprecated - @NonNull - protected Map> assignMasterSystemRegions( - Collection regions, List servers) { - Map> assignments = new TreeMap<>(); - if (this.onlySystemTablesOnMaster) { - if (masterServerName != null && servers.contains(masterServerName)) { - assignments.put(masterServerName, new ArrayList<>()); - for (RegionInfo region : regions) { - if (shouldBeOnMaster(region)) { - assignments.get(masterServerName).add(region); - } - } - } - } - return assignments; - } - @Override public Configuration getConf() { return this.config; @@ -1281,26 +1181,8 @@ protected final boolean idleRegionServerExist(Cluster c){ @Override @NonNull public Map> roundRobinAssignment(List regions, - List servers) throws HBaseIOException { + List servers) throws HBaseIOException { metricsBalancer.incrMiscInvocations(); - Map> assignments = assignMasterSystemRegions(regions, servers); - if (!assignments.isEmpty()) { - servers = new ArrayList<>(servers); - // Guarantee not to put other regions on master - servers.remove(masterServerName); - List masterRegions = assignments.get(masterServerName); - if (!masterRegions.isEmpty()) { - regions = new ArrayList<>(regions); - regions.removeAll(masterRegions); - } - } - /** - * only need assign system table - */ - if (regions.isEmpty()) { - return assignments; - } - int numServers = servers == null ? 0 : servers.size(); if (numServers == 0) { LOG.warn("Wanted to do round robin assignment but no servers to assign to"); @@ -1313,12 +1195,11 @@ public Map> roundRobinAssignment(List r // and balanced. This should also run fast with fewer number of iterations. if (numServers == 1) { // Only one server, nothing fancy we can do here - ServerName server = servers.get(0); - assignments.put(server, new ArrayList<>(regions)); - return assignments; + return Collections.singletonMap(servers.get(0), new ArrayList<>(regions)); } Cluster cluster = createCluster(servers, regions); + Map> assignments = new HashMap<>(); roundRobinAssignment(cluster, regions, servers, assignments); return assignments; } @@ -1374,17 +1255,6 @@ private List findIdleServers(List servers) { public ServerName randomAssignment(RegionInfo regionInfo, List servers) throws HBaseIOException { metricsBalancer.incrMiscInvocations(); - if (servers != null && servers.contains(masterServerName)) { - if (shouldBeOnMaster(regionInfo)) { - return masterServerName; - } - if (!LoadBalancer.isTablesOnMaster(getConf())) { - // Guarantee we do not put any regions on master - servers = new ArrayList<>(servers); - servers.remove(masterServerName); - } - } - int numServers = servers == null ? 0 : servers.size(); if (numServers == 0) { LOG.warn("Wanted to retain assignment but no servers to assign to"); @@ -1428,28 +1298,14 @@ public Map> retainAssignment(Map servers) throws HBaseIOException { // Update metrics metricsBalancer.incrMiscInvocations(); - Map> assignments = assignMasterSystemRegions(regions.keySet(), servers); - if (!assignments.isEmpty()) { - servers = new ArrayList<>(servers); - // Guarantee not to put other regions on master - servers.remove(masterServerName); - List masterRegions = assignments.get(masterServerName); - regions = regions.entrySet().stream().filter(e -> !masterRegions.contains(e.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - if (regions.isEmpty()) { - return assignments; - } - int numServers = servers == null ? 0 : servers.size(); if (numServers == 0) { LOG.warn("Wanted to do retain assignment but no servers to assign to"); return Collections.emptyMap(); } + if (numServers == 1) { // Only one server, nothing fancy we can do here - ServerName server = servers.get(0); - assignments.put(server, new ArrayList<>(regions.keySet())); - return assignments; + return Collections.singletonMap(servers.get(0), new ArrayList<>(regions.keySet())); } // Group all of the old assignments by their hostname. @@ -1458,6 +1314,7 @@ public Map> retainAssignment(Map> assignments = new HashMap<>(); ArrayListMultimap serversByHostname = ArrayListMultimap.create(); for (ServerName server : servers) { assignments.put(server, new ArrayList<>()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java index 4108a5a201d2..493675c96287 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java @@ -27,12 +27,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; - import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.ServerMetrics; import org.apache.hadoop.hbase.ServerName; @@ -43,7 +44,6 @@ import org.apache.hadoop.hbase.favored.FavoredNodesPlan; import org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position; import org.apache.hadoop.hbase.favored.FavoredNodesPromoter; -import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.util.Pair; @@ -114,27 +114,12 @@ public synchronized void setMasterServices(MasterServices masterServices) { @NonNull public Map> roundRobinAssignment(List regions, List servers) throws HBaseIOException { - metricsBalancer.incrMiscInvocations(); - - Set regionSet = Sets.newHashSet(regions); - Map> assignmentMap = assignMasterSystemRegions(regions, servers); - if (!assignmentMap.isEmpty()) { - servers = new ArrayList<>(servers); - // Guarantee not to put other regions on master - servers.remove(masterServerName); - List masterRegions = assignmentMap.get(masterServerName); - if (!masterRegions.isEmpty()) { - for (RegionInfo region: masterRegions) { - regionSet.remove(region); - } - } - } - - if (regionSet.isEmpty()) { - return assignmentMap; + if (regions.isEmpty()) { + return Collections.emptyMap(); } - + Set regionSet = new HashSet<>(regions); + Map> assignmentMap = new HashMap<>(); try { FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(servers, fnm.getRackManager()); @@ -311,19 +296,6 @@ private void assignRegionToAvailableFavoredNode( @Override public ServerName randomAssignment(RegionInfo regionInfo, List servers) throws HBaseIOException { - - if (servers != null && servers.contains(masterServerName)) { - if (shouldBeOnMaster(regionInfo)) { - metricsBalancer.incrMiscInvocations(); - return masterServerName; - } - if (!LoadBalancer.isTablesOnMaster(getConf())) { - // Guarantee we do not put any regions on master - servers = new ArrayList<>(servers); - servers.remove(masterServerName); - } - } - ServerName destination = null; if (!FavoredNodesManager.isFavoredNodeApplicable(regionInfo)) { return super.randomAssignment(regionInfo, servers); @@ -373,7 +345,6 @@ private void updateFavoredNodesForRegion(RegionInfo regionInfo, List @NonNull public Map> retainAssignment(Map regions, List servers) throws HBaseIOException { - Map> assignmentMap = Maps.newHashMap(); Map> result = super.retainAssignment(regions, servers); if (result.isEmpty()) { @@ -381,12 +352,6 @@ public Map> retainAssignment(Map(servers); - servers.remove(masterServerName); - } - // Lets check if favored nodes info is in META, if not generate now. FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(servers, getConf()); helper.initialize(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java index 4435813a96b5..6ec60249de1e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/SimpleLoadBalancer.java @@ -27,7 +27,6 @@ import java.util.NavigableMap; import java.util.Random; import java.util.TreeMap; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.ServerName; @@ -38,6 +37,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hbase.thirdparty.com.google.common.collect.MinMaxPriorityQueue; /** @@ -113,11 +113,8 @@ void setClusterLoad(Map>> clusterLoa Map server2LoadMap = new HashMap<>(); float sum = 0; for (Map.Entry>> clusterEntry : clusterLoad - .entrySet()) { + .entrySet()) { for (Map.Entry> entry : clusterEntry.getValue().entrySet()) { - if (entry.getKey().equals(masterServerName)) { - continue; // we shouldn't include master as potential assignee - } int regionNum = entry.getValue().size(); server2LoadMap.compute(entry.getKey(), (k, v) -> v == null ? regionNum : regionNum + v); sum += regionNum; @@ -255,18 +252,6 @@ private boolean overallNeedsBalance() { @Override public List balanceTable(TableName tableName, Map> loadOfOneTable) { - List regionsToReturn = balanceMasterRegions(loadOfOneTable); - if (regionsToReturn != null || loadOfOneTable == null || loadOfOneTable.size() <= 1) { - return regionsToReturn; - } - if (masterServerName != null && loadOfOneTable.containsKey(masterServerName)) { - if (loadOfOneTable.size() <= 2) { - return null; - } - loadOfOneTable = new HashMap<>(loadOfOneTable); - loadOfOneTable.remove(masterServerName); - } - long startTime = System.currentTimeMillis(); // construct a Cluster object with clusterMap and rest of the @@ -294,7 +279,7 @@ public List balanceTable(TableName tableName, // TODO: Look at data block locality or a more complex load to do this MinMaxPriorityQueue regionsToMove = MinMaxPriorityQueue.orderedBy(rpComparator).create(); - regionsToReturn = new ArrayList<>(); + List regionsToReturn = new ArrayList<>(); // Walk down most loaded, pruning each to the max int serversOverloaded = 0; @@ -322,9 +307,6 @@ public List balanceTable(TableName tableName, hri = regions.get(regions.size() - 1 - i); } i++; - // Don't rebalance special regions. - if (shouldBeOnMaster(hri) - && masterServerName.equals(sal.getServerName())) continue; regionsToMove.add(new RegionPlan(hri, sal.getServerName(), null)); numTaken++; if (numTaken >= numToOffload) break; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index b40779ade232..63795be2be0a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -381,19 +381,6 @@ Cluster.Action nextAction(Cluster cluster) { @Override public synchronized List balanceTable(TableName tableName, Map> loadOfOneTable) { - List plans = balanceMasterRegions(loadOfOneTable); - if (plans != null || loadOfOneTable == null || loadOfOneTable.size() <= 1) { - return plans; - } - - if (masterServerName != null && loadOfOneTable.containsKey(masterServerName)) { - if (loadOfOneTable.size() <= 2) { - return null; - } - loadOfOneTable = new HashMap<>(loadOfOneTable); - loadOfOneTable.remove(masterServerName); - } - // On clusters with lots of HFileLinks or lots of reference files, // instantiating the storefile infos can be quite expensive. // Allow turning this feature off if the locality cost is not going to @@ -485,7 +472,7 @@ public synchronized List balanceTable(TableName tableName, Map currentCost) { - plans = createRegionPlans(cluster); + List plans = createRegionPlans(cluster); LOG.info("Finished computing new load balance plan. Computation took {}" + " to try {} different iterations. Found a solution that moves " + "{} regions; Going from a computed cost of {}" + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 06a90c6d268c..e555439e9a27 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -102,7 +102,6 @@ import org.apache.hadoop.hbase.exceptions.RegionOpeningException; import org.apache.hadoop.hbase.exceptions.UnknownProtocolException; import org.apache.hadoop.hbase.executor.ExecutorService; -import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorConfig; import org.apache.hadoop.hbase.executor.ExecutorType; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.http.InfoServer; @@ -119,7 +118,6 @@ import org.apache.hadoop.hbase.ipc.ServerRpcController; import org.apache.hadoop.hbase.log.HBaseMarkers; import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.master.MasterRpcServicesVersionWrapper; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; @@ -644,7 +642,7 @@ public HRegionServer(final Configuration conf) throws IOException { regionServerAccounting = new RegionServerAccounting(conf); boolean isMasterNotCarryTable = - this instanceof HMaster && !LoadBalancer.isTablesOnMaster(conf); + this instanceof HMaster && !((HMaster) this).isInMaintenanceMode(); // no need to instantiate block cache and mob file cache when master not carry table if (!isMasterNotCarryTable) { @@ -1944,11 +1942,10 @@ public boolean isOnline() { * be hooked up to WAL. */ private void setupWALAndReplication() throws IOException { - boolean isMasterNoTableOrSystemTableOnly = this instanceof HMaster && - !LoadBalancer.isMasterCanHostUserRegions(conf); + boolean isMaster = this instanceof HMaster; WALFactory factory = - new WALFactory(conf, serverName.toString(), this, !isMasterNoTableOrSystemTableOnly); - if (!isMasterNoTableOrSystemTableOnly) { + new WALFactory(conf, serverName.toString(), this, !isMaster); + if (!isMaster) { // TODO Replication make assumptions here based on the default filesystem impl Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME); String logName = AbstractFSWALProvider.getWALDirectoryName(this.serverName.toString()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java index 1de463528ddb..e5bf4b1441d8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java @@ -39,7 +39,6 @@ import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -368,19 +367,11 @@ private void verifyRoundRobinDistribution(RegionLocator regionLocator, int expec } regs.add(loc.getRegion()); } - boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration()); - if (tablesOnMaster) { - // Ignore the master region server, - // which contains less regions by intention. - numRS--; - } float average = (float) expectedRegions / numRS; int min = (int) Math.floor(average); int max = (int) Math.ceil(average); for (List regionList : server2Regions.values()) { - assertTrue( - "numRS=" + numRS + ", min=" + min + ", max=" + max + ", size=" + regionList.size() + - ", tablesOnMaster=" + tablesOnMaster, + assertTrue("numRS=" + numRS + ", min=" + min + ", max=" + max + ", size=" + regionList.size(), regionList.size() == min || regionList.size() == max); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java index 52ebc1686748..572a1d5a6226 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableAdminApi.java @@ -22,22 +22,18 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import java.util.ArrayList; -import java.util.HashMap; + import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletionException; import org.apache.hadoop.hbase.ClientMetaTableAccessor; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; @@ -125,7 +121,6 @@ public void testCreateTableWithRegions() throws Exception { new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 }, new byte[] { 6, 6, 6 }, new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 }, new byte[] { 9, 9, 9 }, }; int expectedRegions = splitKeys.length + 1; - boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration()); createTableWithDefaultConf(tableName, splitKeys); boolean tableAvailable = admin.isTableAvailable(tableName).get(); @@ -173,9 +168,6 @@ public void testCreateTableWithRegions() throws Exception { hri = hris.next().getRegion(); assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8])); assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0); - if (tablesOnMaster) { - verifyRoundRobinDistribution(regions, expectedRegions); - } // Now test using start/end with a number of regions @@ -228,10 +220,6 @@ public void testCreateTableWithRegions() throws Exception { hri = hris.next().getRegion(); assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 })); assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0); - if (tablesOnMaster) { - // This don't work if master is not carrying regions. FIX. TODO. - verifyRoundRobinDistribution(regions, expectedRegions); - } // Try once more with something that divides into something infinite startKey = new byte[] { 0, 0, 0, 0, 0, 0 }; @@ -249,10 +237,6 @@ public void testCreateTableWithRegions() throws Exception { "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), expectedRegions, regions.size()); System.err.println("Found " + regions.size() + " regions"); - if (tablesOnMaster) { - // This don't work if master is not carrying regions. FIX. TODO. - verifyRoundRobinDistribution(regions, expectedRegions); - } // Try an invalid case where there are duplicate split keys splitKeys = new byte[][] { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 }, @@ -266,27 +250,6 @@ public void testCreateTableWithRegions() throws Exception { } } - private void verifyRoundRobinDistribution(List regions, int expectedRegions) { - int numRS = TEST_UTIL.getMiniHBaseCluster().getNumLiveRegionServers(); - - Map> server2Regions = new HashMap<>(); - regions.stream().forEach((loc) -> { - ServerName server = loc.getServerName(); - server2Regions.computeIfAbsent(server, (s) -> new ArrayList<>()).add(loc.getRegion()); - }); - if (numRS >= 2) { - // Ignore the master region server, - // which contains less regions by intention. - numRS--; - } - float average = (float) expectedRegions / numRS; - int min = (int) Math.floor(average); - int max = (int) Math.ceil(average); - server2Regions.values().forEach((regionList) -> { - assertTrue(regionList.size() == min || regionList.size() == max); - }); - } - @Test public void testCreateTableWithOnlyEmptyStartRow() throws Exception { byte[][] splitKeys = new byte[1][]; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreaded.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreaded.java index 23ab43de0746..6c84b0aafdfe 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreaded.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableGetMultiThreaded.java @@ -19,7 +19,6 @@ import static org.apache.hadoop.hbase.HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT; import static org.apache.hadoop.hbase.io.ByteBuffAllocator.MAX_BUFFER_COUNT_KEY; -import static org.apache.hadoop.hbase.master.LoadBalancer.TABLES_ON_MASTER; import static org.junit.Assert.assertEquals; import java.io.IOException; @@ -49,7 +48,6 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.RetryCounter; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -60,6 +58,7 @@ import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.io.Closeables; +import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; /** * Will split the table, and move region randomly when testing. @@ -92,7 +91,6 @@ public static void setUp() throws Exception { } protected static void setUp(MemoryCompactionPolicy memoryCompaction) throws Exception { - TEST_UTIL.getConfiguration().set(TABLES_ON_MASTER, "none"); TEST_UTIL.getConfiguration().setLong(HBASE_CLIENT_META_OPERATION_TIMEOUT, 60000L); TEST_UTIL.getConfiguration().setInt(MAX_BUFFER_COUNT_KEY, 100); TEST_UTIL.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide4.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide4.java index 5c6a98d4b913..c101599709bf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide4.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide4.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNameTestRule; import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint; -import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -1252,12 +1251,11 @@ public void testDuplicateVersions() throws Exception { // test that the same unmanaged connection works with a new // Admin and can connect to the new master; - boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration()); try (Admin admin = conn.getAdmin()) { assertTrue(admin.tableExists(tableName)); assertEquals( admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size(), - SLAVES + (tablesOnMaster ? 1 : 0)); + SLAVES); } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableFavoredNodes.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableFavoredNodes.java index 280c860f7480..3505cd96dc09 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableFavoredNodes.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestTableFavoredNodes.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.favored.FavoredNodesManager; import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.master.ServerManager; -import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; import org.apache.hadoop.hbase.master.balancer.LoadOnlyFavoredStochasticBalancer; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.regionserver.HRegionServer; @@ -96,8 +95,6 @@ public static void setupBeforeClass() throws Exception { LoadOnlyFavoredStochasticBalancer.class, LoadBalancer.class); conf.set(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, "" + SLAVES); - // This helps test if RS get the appropriate FN updates. - conf.set(BaseLoadBalancer.TABLES_ON_MASTER, "none"); TEST_UTIL.startMiniCluster(SLAVES); TEST_UTIL.getMiniHBaseCluster().waitForActiveAndReadyMaster(WAIT_TIMEOUT); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorderMultiBlocks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorderMultiBlocks.java index 0a4a7f513c17..e2e059dddb8a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorderMultiBlocks.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/fs/TestBlockReorderMultiBlocks.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.Region; @@ -118,8 +117,7 @@ public void testHBaseCluster() throws Exception { MiniHBaseCluster hbm = htu.startMiniHBaseCluster(); hbm.waitForActiveAndReadyMaster(); - HRegionServer targetRs = LoadBalancer.isTablesOnMaster(hbm.getConf())? hbm.getMaster(): - hbm.getRegionServer(0); + HRegionServer targetRs = hbm.getRegionServer(0); // We want to have a datanode with the same name as the region server, so // we're going to get the regionservername, and start a new datanode with this name. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java index eb6107b72f7b..10ed83e550bf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java @@ -122,8 +122,7 @@ public void testClusterRequests() throws Exception { @Test public void testDefaultMasterMetrics() throws Exception { MetricsMasterSource masterSource = master.getMasterMetrics().getMetricsSource(); - boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration()); - metricsHelper.assertGauge("numRegionServers", 1 + (tablesOnMaster ? 1 : 0), masterSource); + metricsHelper.assertGauge("numRegionServers", 1, masterSource); metricsHelper.assertGauge("averageLoad", 1, masterSource); metricsHelper.assertGauge("numDeadRegionServers", 0, masterSource); metricsHelper.assertGauge("numDrainingRegionServers", 0, masterSource); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java index 6ac68b300483..578ee1cd060f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetricsWrapper.java @@ -82,8 +82,7 @@ public void testInfo() { assertEquals(master.getMasterStartTime(), info.getStartTime()); assertEquals(master.getMasterCoprocessors().length, info.getCoprocessors().length); assertEquals(master.getServerManager().getOnlineServersList().size(), info.getNumRegionServers()); - int regionServerCount = - NUM_RS + (LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration())? 1: 0); + int regionServerCount = NUM_RS; assertEquals(regionServerCount, info.getNumRegionServers()); String zkServers = info.getZookeeperQuorum(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java index bdeab3d28a25..1f3221197865 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java @@ -121,11 +121,10 @@ public boolean isAborted() { * Test starting master then stopping it before its fully up. */ @Test - public void testStopDuringStart() - throws IOException, KeeperException, InterruptedException { + public void testStopDuringStart() throws IOException, KeeperException, InterruptedException { HMaster master = new HMaster(TESTUTIL.getConfiguration()); master.start(); - // Immediately have it stop. We used hang in assigning meta. + // Immediately have it stop. We used hang in assigning meta. master.stopMaster(); master.join(); } @@ -150,9 +149,10 @@ public void testMasterInitWithObserverModeClientZKQuorum() throws Exception { conf.set(HConstants.CLIENT_ZOOKEEPER_QUORUM, HConstants.LOCALHOST); conf.setInt(HConstants.CLIENT_ZOOKEEPER_CLIENT_PORT, TESTUTIL.getZkCluster().getClientPort() + 1); + // need to enable maintenance mode so we will start master as a region server + conf.setBoolean(HMaster.MAINTENANCE_MODE, true); // settings to allow us not to start additional RS conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); - conf.setBoolean(LoadBalancer.TABLES_ON_MASTER, true); // main setting for this test case conf.setBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE, true); HMaster master = new HMaster(conf); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNotCarryTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNotCarryTable.java index e62766dca359..63a55f3f7c55 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNotCarryTable.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNotCarryTable.java @@ -67,13 +67,6 @@ public static void tearDown() throws Exception { UTIL.shutdownMiniZKCluster(); } - @Test - public void testMasterNotCarryTable() { - // The default config is false - assertFalse(LoadBalancer.isTablesOnMaster(UTIL.getConfiguration())); - assertFalse(LoadBalancer.isSystemTablesOnlyOnMaster(UTIL.getConfiguration())); - } - @Test public void testMasterBlockCache() { // no need to instantiate block cache. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java index fdac676f052b..b6446fd6df10 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java @@ -157,10 +157,6 @@ public void testBulkAssignment() throws Exception { hris.add(RegionInfoBuilder.FIRST_META_REGIONINFO); tmp.add(master); Map> plans = loadBalancer.roundRobinAssignment(hris, tmp); - if (LoadBalancer.isTablesOnMaster(loadBalancer.getConf())) { - assertTrue(plans.get(master).contains(RegionInfoBuilder.FIRST_META_REGIONINFO)); - assertEquals(1, plans.get(master).size()); - } int totalRegion = 0; for (List regions: plans.values()) { totalRegion += regions.size(); @@ -244,12 +240,7 @@ private void testRandomAssignment(int numberOfIdleServers) throws Exception { List allServers = new ArrayList<>(idleServers.size() + 1); allServers.add(ServerName.valueOf("server-" + numberOfIdleServers, 1000, 1L)); allServers.addAll(idleServers); - LoadBalancer balancer = new MockBalancer() { - @Override - public boolean shouldBeOnMaster(RegionInfo region) { - return false; - } - }; + LoadBalancer balancer = new MockBalancer(); Configuration conf = HBaseConfiguration.create(); conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class); balancer.setConf(conf); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredStochasticBalancerPickers.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredStochasticBalancerPickers.java index 4347edf89731..b750344fa5b1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredStochasticBalancerPickers.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestFavoredStochasticBalancerPickers.java @@ -99,7 +99,6 @@ public static void setupBeforeClass() throws Exception { conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 30000); conf.setInt("hbase.master.balancer.stochastic.moveCost", 0); conf.setBoolean("hbase.master.balancer.stochastic.execute.maxSteps", true); - conf.set(BaseLoadBalancer.TABLES_ON_MASTER, "none"); } @Before diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionsOnMasterOptions.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionsOnMasterOptions.java deleted file mode 100644 index 3b1cfeda2518..000000000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRegionsOnMasterOptions.java +++ /dev/null @@ -1,220 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.balancer; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.List; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.StartMiniClusterOption; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.LoadBalancer; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.JVMClusterUtil; -import org.apache.hadoop.hbase.util.Threads; -import org.junit.After; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.TestName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Test options for regions on master; none, system, or any (i.e. master is like any other - * regionserver). Checks how regions are deployed when each of the options are enabled. - * It then does kill combinations to make sure the distribution is more than just for startup. - * NOTE: Regions on Master does not work well. See HBASE-19828. Until addressed, disabling this - * test. - */ -@Ignore -@Category({MediumTests.class}) -public class TestRegionsOnMasterOptions { - - @ClassRule - public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRegionsOnMasterOptions.class); - - private static final Logger LOG = LoggerFactory.getLogger(TestRegionsOnMasterOptions.class); - @Rule public TestName name = new TestName(); - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private Configuration c; - private String tablesOnMasterOldValue; - private String systemTablesOnMasterOldValue; - private static final int SLAVES = 3; - private static final int MASTERS = 2; - // Make the count of REGIONS high enough so I can distingush case where master is only carrying - // system regions from the case where it is carrying any region; i.e. 2 system regions vs more - // if user + system. - private static final int REGIONS = 12; - private static final int SYSTEM_REGIONS = 2; // ns and meta -- no acl unless enabled. - - @Before - public void setup() { - this.c = TEST_UTIL.getConfiguration(); - this.tablesOnMasterOldValue = c.get(LoadBalancer.TABLES_ON_MASTER); - this.systemTablesOnMasterOldValue = c.get(LoadBalancer.SYSTEM_TABLES_ON_MASTER); - } - - @After - public void tearDown() { - unset(LoadBalancer.TABLES_ON_MASTER, this.tablesOnMasterOldValue); - unset(LoadBalancer.SYSTEM_TABLES_ON_MASTER, this.systemTablesOnMasterOldValue); - } - - private void unset(final String key, final String value) { - if (value == null) { - c.unset(key); - } else { - c.set(key, value); - } - } - - @Test - public void testRegionsOnAllServers() throws Exception { - c.setBoolean(LoadBalancer.TABLES_ON_MASTER, true); - c.setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, false); - int rsCount = (REGIONS + SYSTEM_REGIONS)/(SLAVES + 1/*Master*/); - checkBalance(rsCount, rsCount); - } - - @Test - public void testNoRegionOnMaster() throws Exception { - c.setBoolean(LoadBalancer.TABLES_ON_MASTER, false); - c.setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, false); - int rsCount = (REGIONS + SYSTEM_REGIONS)/SLAVES; - checkBalance(0, rsCount); - } - - @Ignore // Fix this. The Master startup doesn't allow Master reporting as a RegionServer, not - // until way late after the Master startup finishes. Needs more work. - @Test - public void testSystemTablesOnMaster() throws Exception { - c.setBoolean(LoadBalancer.TABLES_ON_MASTER, true); - c.setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, true); - // IS THIS SHORT-CIRCUIT RPC? Yes. Here is how it looks currently if I have an exception - // thrown in doBatchMutate inside a Region. - // - // java.lang.Exception - // at org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:3845) - // at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:2972) - // at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2751) - // at org.apache.hadoop.hbase.client.ClientServiceCallable.doMutate(ClientServiceCallable.java:55) - // at org.apache.hadoop.hbase.client.HTable$3.rpcCall(HTable.java:585) - // at org.apache.hadoop.hbase.client.HTable$3.rpcCall(HTable.java:579) - // at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:126) - // at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:106) - // at org.apache.hadoop.hbase.client.HTable.put(HTable.java:589) - // at org.apache.hadoop.hbase.master.TableNamespaceManager.insertIntoNSTable(TableNamespaceManager.java:156) - // at org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure.insertIntoNSTable(CreateNamespaceProcedure.java:222) - // at org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure.executeFromState(CreateNamespaceProcedure.java:76) - // at org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure.executeFromState(CreateNamespaceProcedure.java:40) - // at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:181) - // at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:847) - // at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1440) - // at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1209) - // at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$800(ProcedureExecutor.java:79) - // at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1719) - // - // If I comment out the ConnectionUtils ConnectionImplementation content, I see this: - // - // java.lang.Exception - // at org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:3845) - // at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:2972) - // at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2751) - // at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:41546) - // at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:406) - // at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) - // at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:278) - // at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:258) - - checkBalance(SYSTEM_REGIONS, REGIONS/SLAVES); - } - - private void checkBalance(int masterCount, int rsCount) throws Exception { - StartMiniClusterOption option = StartMiniClusterOption.builder() - .numMasters(MASTERS).numRegionServers(SLAVES).numDataNodes(SLAVES).build(); - MiniHBaseCluster cluster = TEST_UTIL.startMiniCluster(option); - TableName tn = TableName.valueOf(this.name.getMethodName()); - try { - Table t = TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY, REGIONS); - LOG.info("Server: " + cluster.getMaster().getServerManager().getOnlineServersList()); - List regions = cluster.getMaster().getRegions(); - int mActualCount = regions.size(); - if (masterCount == 0 || masterCount == SYSTEM_REGIONS) { - // 0 means no regions on master. - assertEquals(masterCount, mActualCount); - } else { - // This is master as a regionserver scenario. - checkCount(masterCount, mActualCount); - } - // Allow that balance is not exact. FYI, getRegionServerThreads does not include master - // thread though it is a regionserver so we have to check master and then below the - // regionservers. - for (JVMClusterUtil.RegionServerThread rst: cluster.getRegionServerThreads()) { - regions = rst.getRegionServer().getRegions(); - int rsActualCount = regions.size(); - checkCount(rsActualCount, rsCount); - } - HMaster oldMaster = cluster.getMaster(); - cluster.killMaster(oldMaster.getServerName()); - oldMaster.join(); - while (cluster.getMaster() == null || - cluster.getMaster().getServerName().equals(oldMaster.getServerName())) { - Threads.sleep(10); - } - while (!cluster.getMaster().isInitialized()) { - Threads.sleep(10); - } - while (cluster.getMaster().getAssignmentManager(). - computeRegionInTransitionStat().getTotalRITs() > 0) { - Threads.sleep(100); - LOG.info("Waiting on RIT to go to zero before calling balancer..."); - } - LOG.info("Cluster is up; running balancer"); - cluster.getMaster().balance(); - regions = cluster.getMaster().getRegions(); - int mNewActualCount = regions.size(); - if (masterCount == 0 || masterCount == SYSTEM_REGIONS) { - // 0 means no regions on master. After crash, should still be no regions on master. - // If masterCount == SYSTEM_REGIONS, means master only carrying system regions and should - // still only carry system regions post crash. - assertEquals(masterCount, mNewActualCount); - } - } finally { - LOG.info("Running shutdown of cluster"); - TEST_UTIL.shutdownMiniCluster(); - } - } - - private void checkCount(int actual, int expected) { - assertTrue("Actual=" + actual + ", expected=" + expected, - actual >= (expected - 2) && actual <= (expected + 2)); // Lots of slop +/- 2 - } -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSafemodeBringsDownMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSafemodeBringsDownMaster.java index 44c55103b6b5..8df1694265f3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSafemodeBringsDownMaster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSafemodeBringsDownMaster.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -58,7 +57,6 @@ public class TestSafemodeBringsDownMaster { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); - conf.set(BaseLoadBalancer.TABLES_ON_MASTER, "none"); } @BeforeClass diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClusterId.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClusterId.java index 16799a3efcf6..53d1fc57be02 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClusterId.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestClusterId.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.StartMiniClusterOption; import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.CommonFSUtils; @@ -114,8 +113,7 @@ public void testRewritingClusterIdToPB() throws Exception { } TEST_UTIL.startMiniHBaseCluster(); HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); - int expected = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration())? 2: 1; - assertEquals(expected, master.getServerManager().getOnlineServersList().size()); + assertEquals(1, master.getServerManager().getOnlineServersList().size()); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java index 647feec12efe..40919dd2174f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSKilledWhenInitializing.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.master.ServerListener; import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -104,7 +103,7 @@ public void testRSTerminationAfterRegisteringToMasterBeforeCreatingEphemeralNode cluster.getRegionServers().get(i).start(); } // Expected total regionservers depends on whether Master can host regions or not. - int expectedTotalRegionServers = NUM_RS + (LoadBalancer.isTablesOnMaster(conf)? 1: 0); + int expectedTotalRegionServers = NUM_RS; List onlineServersList = null; do { onlineServersList = master.getMaster().getServerManager().getOnlineServersList(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerHostname.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerHostname.java index b69218f1fbfd..af87a2ac718a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerHostname.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerHostname.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.StartMiniClusterOption; -import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.DNS; @@ -113,9 +112,7 @@ public void testRegionServerHostname() throws Exception { try { ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher(); List servers = ZKUtil.listChildrenNoWatch(zkw, zkw.getZNodePaths().rsZNode); - // there would be NUM_RS+1 children - one for the master - assertTrue(servers.size() == - NUM_RS + (LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration())? 1: 0)); + assertEquals(NUM_RS, servers.size()); for (String server : servers) { assertTrue("From zookeeper: " + server + " hostname: " + hostName, server.startsWith(hostName.toLowerCase(Locale.ROOT)+",")); @@ -197,8 +194,7 @@ public void testRegionServerHostnameReportedToMaster() throws Exception { StartMiniClusterOption option = StartMiniClusterOption.builder() .numMasters(NUM_MASTERS).numRegionServers(NUM_RS).numDataNodes(NUM_RS).build(); TEST_UTIL.startMiniCluster(option); - boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration()); - int expectedRS = NUM_RS + (tablesOnMaster? 1: 0); + int expectedRS = NUM_RS; try (ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher()) { List servers = ZKUtil.listChildrenNoWatch(zkw, zkw.getZNodePaths().rsZNode); assertEquals(expectedRS, servers.size()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index d058d08f5945..9db187d0a31b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hbase.regionserver; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -27,7 +26,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -55,7 +53,6 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; @@ -105,13 +102,11 @@ public class TestRegionServerMetrics { private static byte[] qualifier = Bytes.toBytes("qual"); private static byte[] val = Bytes.toBytes("val"); private static Admin admin; - private static boolean TABLES_ON_MASTER; @BeforeClass public static void startCluster() throws Exception { metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class); TEST_UTIL = new HBaseTestingUtility(); - TABLES_ON_MASTER = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration()); conf = TEST_UTIL.getConfiguration(); conf.getLong("hbase.splitlog.max.resubmit", 0); // Make the failure test faster @@ -232,7 +227,7 @@ private void doScan(int n, boolean caching) throws IOException { @Test public void testRegionCount() throws Exception { - metricsHelper.assertGauge("regionCount", TABLES_ON_MASTER ? 1 : 2, serverSource); + metricsHelper.assertGauge("regionCount", 2, serverSource); } @Test @@ -274,11 +269,6 @@ public void testRequestCount() throws Exception { doNGets(10, true); // true = batch metricsRegionServer.getRegionServerWrapper().forceRecompute(); - if (TABLES_ON_MASTER) { - assertCounter("totalRequestCount", requests + 41); - assertCounter("totalRowActionRequestCount", rowActionRequests + 50); - assertCounter("readRequestCount", readRequests + 20); - } assertCounter("writeRequestCount", writeRequests + 30); @@ -286,30 +276,15 @@ public void testRequestCount() throws Exception { doNPuts(30, true); metricsRegionServer.getRegionServerWrapper().forceRecompute(); - if (TABLES_ON_MASTER) { - assertCounter("totalRequestCount", requests + 42); - assertCounter("totalRowActionRequestCount", rowActionRequests + 80); - assertCounter("readRequestCount", readRequests + 20); - } assertCounter("writeRequestCount", writeRequests + 60); doScan(10, false); // test after batch put so we have enough lines metricsRegionServer.getRegionServerWrapper().forceRecompute(); - if (TABLES_ON_MASTER) { - assertCounter("totalRequestCount", requests + 52); - assertCounter("totalRowActionRequestCount", rowActionRequests + 90); - assertCounter("readRequestCount", readRequests + 30); - } assertCounter("writeRequestCount", writeRequests + 60); numScanNext += 10; doScan(10, true); // true = caching metricsRegionServer.getRegionServerWrapper().forceRecompute(); - if (TABLES_ON_MASTER) { - assertCounter("totalRequestCount", requests + 53); - assertCounter("totalRowActionRequestCount", rowActionRequests + 100); - assertCounter("readRequestCount", readRequests + 40); - } assertCounter("writeRequestCount", writeRequests + 60); numScanNext += 1; } @@ -342,7 +317,7 @@ public void testStoreCount() throws Exception { TEST_UTIL.getAdmin().flush(tableName); metricsRegionServer.getRegionServerWrapper().forceRecompute(); - assertGauge("storeCount", TABLES_ON_MASTER ? 1 : 5); + assertGauge("storeCount", 5); assertGauge("storeFileCount", 1); } @@ -423,9 +398,6 @@ public void testScanSize() throws Exception { } numScanNext += NUM_SCAN_NEXT; assertRegionMetrics("scanCount", NUM_SCAN_NEXT); - if (TABLES_ON_MASTER) { - assertCounter("ScanSize_num_ops", numScanNext); - } } } @@ -443,9 +415,6 @@ public void testScanTime() throws Exception { } numScanNext += NUM_SCAN_NEXT; assertRegionMetrics("scanCount", NUM_SCAN_NEXT); - if (TABLES_ON_MASTER) { - assertCounter("ScanTime_num_ops", numScanNext); - } } @Test @@ -457,17 +426,11 @@ public void testScanSizeForSmallScan() throws Exception { for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) { Result result = resultScanners.next(); assertNotNull(result); - if (TABLES_ON_MASTER) { - assertEquals(1, result.size()); - } } assertNull(resultScanners.next()); } numScanNext += NUM_SCAN_NEXT; assertRegionMetrics("scanCount", NUM_SCAN_NEXT); - if (TABLES_ON_MASTER) { - assertCounter("ScanSize_num_ops", numScanNext); - } } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReadRequestMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReadRequestMetrics.java index 1a83dcfe13cb..e433b90a88d9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReadRequestMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReadRequestMetrics.java @@ -58,7 +58,6 @@ import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.RowFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; -import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; @@ -109,9 +108,6 @@ public class TestRegionServerReadRequestMetrics { @BeforeClass public static void setUpOnce() throws Exception { - // Default starts one regionserver only. - TEST_UTIL.getConfiguration().setBoolean(LoadBalancer.TABLES_ON_MASTER, true); - // TEST_UTIL.getConfiguration().setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, true); TEST_UTIL.startMiniCluster(); admin = TEST_UTIL.getAdmin(); serverNames = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)) @@ -146,16 +142,6 @@ private static void testReadRequests(long resultCount, assertEquals(expectedReadRequests, requestsMap.get(Metric.REGION_READ) - requestsMapPrev.get(Metric.REGION_READ)); - boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration()); - if (tablesOnMaster) { - // If NO tables on master, then the single regionserver in this test carries user-space - // tables and the meta table. The first time through, the read will be inflated by meta - // lookups. We don't know which test will be first through since junit randomizes. This - // method is used by a bunch of tests. Just do this check if master is hosting (system) - // regions only. - assertEquals(expectedReadRequests, - requestsMap.get(Metric.SERVER_READ) - requestsMapPrev.get(Metric.SERVER_READ)); - } assertEquals(expectedFilteredReadRequests, requestsMap.get(Metric.FILTERED_REGION_READ) - requestsMapPrev.get(Metric.FILTERED_REGION_READ)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java index b3c2eb622967..e6b84620ec1b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerReportForDuty.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -168,10 +167,8 @@ public void testReportForDutyWithMasterChange() throws Exception { // Start a master and wait for it to become the active/primary master. // Use a random unique port cluster.getConfiguration().setInt(HConstants.MASTER_PORT, HBaseTestingUtility.randomFreePort()); - // master has a rs. defaultMinToStart = 2 - boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(testUtil.getConfiguration()); - cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, tablesOnMaster? 2: 1); - cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, tablesOnMaster? 2: 1); + cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); + cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 1); master = cluster.addMaster(); rs = cluster.addRegionServer(); LOG.debug("Starting master: " + master.getMaster().getServerName()); @@ -198,10 +195,8 @@ public void testReportForDutyWithMasterChange() throws Exception { // Also let it wait for exactly 2 region severs to report in. // TODO: Add handling bindexception. Random port is not enough!!! Flakie test! cluster.getConfiguration().setInt(HConstants.MASTER_PORT, HBaseTestingUtility.randomFreePort()); - cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, - tablesOnMaster? 3: 2); - cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, - tablesOnMaster? 3: 2); + cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 2); + cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2); backupMaster = cluster.addMaster(); LOG.debug("Starting new master: " + backupMaster.getMaster().getServerName()); backupMaster.start(); @@ -211,8 +206,7 @@ public void testReportForDutyWithMasterChange() throws Exception { // Do some checking/asserts here. assertTrue(backupMaster.getMaster().isActiveMaster()); assertTrue(backupMaster.getMaster().isInitialized()); - assertEquals(backupMaster.getMaster().getServerManager().getOnlineServersList().size(), - tablesOnMaster? 3: 2); + assertEquals(backupMaster.getMaster().getServerManager().getOnlineServersList().size(), 2); } @@ -230,12 +224,8 @@ public void testReportForDutyWithRSRpcRetry() throws Exception { cluster.getConfiguration().setInt(HConstants.MASTER_PORT, HBaseTestingUtility.randomFreePort()); // Override the default RS RPC retry interval of 100ms to 300ms cluster.getConfiguration().setLong("hbase.regionserver.rpc.retry.interval", 300); - // master has a rs. defaultMinToStart = 2 - boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(testUtil.getConfiguration()); - cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, - tablesOnMaster ? 2 : 1); - cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, - tablesOnMaster ? 2 : 1); + cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); + cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 1); master = cluster.addMaster(); rs = cluster.addRegionServer(); LOG.debug("Starting master: " + master.getMaster().getServerName()); @@ -263,12 +253,8 @@ public void testReportForDutyWithEnvironmentEdge() throws Exception { cluster.getConfiguration().setInt("hbase.procedure.remote.dispatcher.delay.msec", 0); cluster.getConfiguration().setLong("hbase.regionserver.rpc.retry.interval", 0); - // master has a rs. defaultMinToStart = 2 - boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(testUtil.getConfiguration()); - cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, - tablesOnMaster ? 2 : 1); - cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, - tablesOnMaster ? 2 : 1); + cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); + cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 1); // Inject manual environment edge for clock skew computation between RS and master ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java index f11544f6a7ac..f455ba8d8d1a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -76,7 +76,6 @@ import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.master.MasterRpcServices; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.RegionState.State; @@ -923,14 +922,7 @@ private int ensureTableRegionNotOnSameServerAsMeta(final Admin admin, // hbase:meta We don't want hbase:meta replay polluting our test when we later crash // the table region serving server. int metaServerIndex = cluster.getServerWithMeta(); - boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TESTING_UTIL.getConfiguration()); - if (tablesOnMaster) { - // Need to check master is supposed to host meta... perhaps it is not. - throw new UnsupportedOperationException(); - // TODO: assertTrue(metaServerIndex == -1); // meta is on master now - } - HRegionServer metaRegionServer = tablesOnMaster? - cluster.getMaster(): cluster.getRegionServer(metaServerIndex); + HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex); int tableRegionIndex = cluster.getServerWith(hri.getRegionName()); assertTrue(tableRegionIndex != -1); HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/throttle/TestFlushWithThroughputController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/throttle/TestFlushWithThroughputController.java index 98fe132c10e2..73e36dade00e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/throttle/TestFlushWithThroughputController.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/throttle/TestFlushWithThroughputController.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; @@ -190,12 +189,6 @@ public void testFlushThroughputTuning() throws Exception { // assertion here. assertTrue(regionServer.getFlushPressure() < pressure); Thread.sleep(5000); - boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(hbtu.getConfiguration()); - if (tablesOnMaster) { - // If no tables on the master, this math is off and I'm not sure what it is supposed to be - // when meta is on the regionserver and not on the master. - assertEquals(10L * 1024 * 1024, throughputController.getMaxThroughput(), EPSILON); - } Table table = conn.getTable(tableName); Random rand = new Random(); for (int i = 0; i < 10; i++) {