Skip to content

Commit

Permalink
HBASE-26988 dynamic configuration of loadbalance.bytable (#4384)
Browse files Browse the repository at this point in the history 
Signed-off-by: Andrew Purtell <apurtell@apache.org>

Conflicts:
	hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
	hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
  • Loading branch information
@d-c-manning @apurtell
d-c-manning authored and apurtell committed Apr 29, 2022
1 parent 7b9fb0f commit adc9d07
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 16 deletions.
11 changes: 7 additions & 4 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
"hbase.master.balancer.rejection.buffer.enabled";
public static final boolean DEFAULT_BALANCER_REJECTION_BUFFER_ENABLED = false;

public static final boolean DEFAULT_HBASE_MASTER_LOADBALANCE_BYTABLE = false;

protected static final int MIN_SERVER_BALANCE = 2;
private volatile boolean stopped = false;

Expand All @@ -83,7 +85,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {

protected volatile RegionLocationFinder regionFinder;
protected boolean useRegionFinder;
protected boolean isByTable = false;
protected boolean isByTable = DEFAULT_HBASE_MASTER_LOADBALANCE_BYTABLE;

// slop for regions
protected float slop;
Expand Down Expand Up @@ -576,9 +578,10 @@ protected void loadConf(Configuration conf) {
} else {
regionFinder = null;
}

this.isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable);
LOG.info("slop={}, systemTablesOnMaster={}", this.slop, this.onlySystemTablesOnMaster);
this.isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE,
DEFAULT_HBASE_MASTER_LOADBALANCE_BYTABLE);
// Print out base configs. Don't print overallSlop since it for simple balancer exclusively.
LOG.info("slop={}", this.slop);
}

@Override
Expand Down
30 changes: 18 additions & 12 deletions ...-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,30 +106,36 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {

protected static final String STEPS_PER_REGION_KEY =
"hbase.master.balancer.stochastic.stepsPerRegion";
protected static final int DEFAULT_STEPS_PER_REGION = 800;
protected static final String MAX_STEPS_KEY =
"hbase.master.balancer.stochastic.maxSteps";
protected static final int DEFAULT_MAX_STEPS = 1000000;
protected static final String RUN_MAX_STEPS_KEY =
"hbase.master.balancer.stochastic.runMaxSteps";
protected static final boolean DEFAULT_RUN_MAX_STEPS = false;
protected static final String MAX_RUNNING_TIME_KEY =
"hbase.master.balancer.stochastic.maxRunningTime";
protected static final long DEFAULT_MAX_RUNNING_TIME = 30 * 1000; // 30 seconds.
protected static final String KEEP_REGION_LOADS =
"hbase.master.balancer.stochastic.numRegionLoadsToRemember";
protected static final int DEFAULT_KEEP_REGION_LOADS = 15;
private static final String TABLE_FUNCTION_SEP = "_";
protected static final String MIN_COST_NEED_BALANCE_KEY =
"hbase.master.balancer.stochastic.minCostNeedBalance";
protected static final float DEFAULT_MIN_COST_NEED_BALANCE = 0.025f;
protected static final String COST_FUNCTIONS_COST_FUNCTIONS_KEY =
"hbase.master.balancer.stochastic.additionalCostFunctions";
public static final String OVERALL_COST_FUNCTION_NAME = "Overall";

Map<String, Deque<BalancerRegionLoad>> loads = new HashMap<>();

// values are defaults
private int maxSteps = 1000000;
private boolean runMaxSteps = false;
private int stepsPerRegion = 800;
private long maxRunningTime = 30 * 1000 * 1; // 30 seconds.
private int numRegionLoadsToRemember = 15;
private float minCostNeedBalance = 0.025f;
private int maxSteps = DEFAULT_MAX_STEPS;
private boolean runMaxSteps = DEFAULT_RUN_MAX_STEPS;
private int stepsPerRegion = DEFAULT_STEPS_PER_REGION;
private long maxRunningTime = DEFAULT_MAX_RUNNING_TIME;
private int numRegionLoadsToRemember = DEFAULT_KEEP_REGION_LOADS;
private float minCostNeedBalance = DEFAULT_MIN_COST_NEED_BALANCE;
private boolean isBalancerDecisionRecording = false;
private boolean isBalancerRejectionRecording = false;

Expand Down Expand Up @@ -224,13 +230,13 @@ protected List<CandidateGenerator> createCandidateGenerators() {
@Override
protected void loadConf(Configuration conf) {
super.loadConf(conf);
maxSteps = conf.getInt(MAX_STEPS_KEY, maxSteps);
stepsPerRegion = conf.getInt(STEPS_PER_REGION_KEY, stepsPerRegion);
maxRunningTime = conf.getLong(MAX_RUNNING_TIME_KEY, maxRunningTime);
runMaxSteps = conf.getBoolean(RUN_MAX_STEPS_KEY, runMaxSteps);
maxSteps = conf.getInt(MAX_STEPS_KEY, DEFAULT_MAX_STEPS);
stepsPerRegion = conf.getInt(STEPS_PER_REGION_KEY, DEFAULT_STEPS_PER_REGION);
maxRunningTime = conf.getLong(MAX_RUNNING_TIME_KEY, DEFAULT_MAX_RUNNING_TIME);
runMaxSteps = conf.getBoolean(RUN_MAX_STEPS_KEY, DEFAULT_RUN_MAX_STEPS);

numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, numRegionLoadsToRemember);
minCostNeedBalance = conf.getFloat(MIN_COST_NEED_BALANCE_KEY, minCostNeedBalance);
numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, DEFAULT_KEEP_REGION_LOADS);
minCostNeedBalance = conf.getFloat(MIN_COST_NEED_BALANCE_KEY, DEFAULT_MIN_COST_NEED_BALANCE);
localityCandidateGenerator = new LocalityBasedCandidateGenerator();
localityCost = new ServerLocalityCostFunction(conf);
rackLocalityCost = new RackLocalityCostFunction(conf);
Expand Down

0 comments on commit adc9d07

Please sign in to comment.