Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HBASE-26988 dynamic configuration of loadbalance.bytable #4384

Merged
merged 2 commits into from
Apr 29, 2022
Merged

HBASE-26988 dynamic configuration of loadbalance.bytable #4384

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4cdf322
HBASE-26988 dynamic configuration of loadbalance.bytable
d-c-manning Apr 29, 2022
a89b7cd
HBASE-26988 dynamic configuration of StochasticLoadBalancer
d-c-manning Apr 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,14 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
"hbase.master.balancer.rejection.buffer.enabled";
public static final boolean DEFAULT_BALANCER_REJECTION_BUFFER_ENABLED = false;

public static final boolean DEFAULT_HBASE_MASTER_LOADBALANCE_BYTABLE = false;

protected static final int MIN_SERVER_BALANCE = 2;
private volatile boolean stopped = false;

protected volatile RegionHDFSBlockLocationFinder regionFinder;
protected boolean useRegionFinder;
protected boolean isByTable = false;
protected boolean isByTable = DEFAULT_HBASE_MASTER_LOADBALANCE_BYTABLE;

// slop for regions
protected float slop;
Expand Down Expand Up @@ -415,7 +417,8 @@ protected void loadConf(Configuration conf) {
} else {
regionFinder = null;
}
this.isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable);
this.isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE,
DEFAULT_HBASE_MASTER_LOADBALANCE_BYTABLE);
// Print out base configs. Don't print overallSlop since it for simple balancer exclusively.
LOG.info("slop={}", this.slop);
}
Expand Down
30 changes: 18 additions & 12 deletions ...alancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,30 +104,36 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {

protected static final String STEPS_PER_REGION_KEY =
"hbase.master.balancer.stochastic.stepsPerRegion";
protected static final int DEFAULT_STEPS_PER_REGION = 800;
protected static final String MAX_STEPS_KEY =
"hbase.master.balancer.stochastic.maxSteps";
protected static final int DEFAULT_MAX_STEPS = 1000000;
protected static final String RUN_MAX_STEPS_KEY =
"hbase.master.balancer.stochastic.runMaxSteps";
protected static final boolean DEFAULT_RUN_MAX_STEPS = false;
protected static final String MAX_RUNNING_TIME_KEY =
"hbase.master.balancer.stochastic.maxRunningTime";
protected static final long DEFAULT_MAX_RUNNING_TIME = 30 * 1000; // 30 seconds.
protected static final String KEEP_REGION_LOADS =
"hbase.master.balancer.stochastic.numRegionLoadsToRemember";
protected static final int DEFAULT_KEEP_REGION_LOADS = 15;
private static final String TABLE_FUNCTION_SEP = "_";
protected static final String MIN_COST_NEED_BALANCE_KEY =
"hbase.master.balancer.stochastic.minCostNeedBalance";
protected static final float DEFAULT_MIN_COST_NEED_BALANCE = 0.025f;
protected static final String COST_FUNCTIONS_COST_FUNCTIONS_KEY =
"hbase.master.balancer.stochastic.additionalCostFunctions";
public static final String OVERALL_COST_FUNCTION_NAME = "Overall";

Map<String, Deque<BalancerRegionLoad>> loads = new HashMap<>();

// values are defaults
private int maxSteps = 1000000;
private boolean runMaxSteps = false;
private int stepsPerRegion = 800;
private long maxRunningTime = 30 * 1000 * 1; // 30 seconds.
private int numRegionLoadsToRemember = 15;
private float minCostNeedBalance = 0.025f;
private int maxSteps = DEFAULT_MAX_STEPS;
private boolean runMaxSteps = DEFAULT_RUN_MAX_STEPS;
private int stepsPerRegion = DEFAULT_STEPS_PER_REGION;
private long maxRunningTime = DEFAULT_MAX_RUNNING_TIME;
private int numRegionLoadsToRemember = DEFAULT_KEEP_REGION_LOADS;
private float minCostNeedBalance = DEFAULT_MIN_COST_NEED_BALANCE;

private List<CostFunction> costFunctions; // FindBugs: Wants this protected; IS2_INCONSISTENT_SYNC
// To save currently configed sum of multiplier. Defaulted at 1 for cases that carry high cost
Expand Down Expand Up @@ -216,13 +222,13 @@ protected List<CandidateGenerator> createCandidateGenerators() {
@Override
protected void loadConf(Configuration conf) {
super.loadConf(conf);
maxSteps = conf.getInt(MAX_STEPS_KEY, maxSteps);
stepsPerRegion = conf.getInt(STEPS_PER_REGION_KEY, stepsPerRegion);
maxRunningTime = conf.getLong(MAX_RUNNING_TIME_KEY, maxRunningTime);
runMaxSteps = conf.getBoolean(RUN_MAX_STEPS_KEY, runMaxSteps);
maxSteps = conf.getInt(MAX_STEPS_KEY, DEFAULT_MAX_STEPS);
stepsPerRegion = conf.getInt(STEPS_PER_REGION_KEY, DEFAULT_STEPS_PER_REGION);
maxRunningTime = conf.getLong(MAX_RUNNING_TIME_KEY, DEFAULT_MAX_RUNNING_TIME);
runMaxSteps = conf.getBoolean(RUN_MAX_STEPS_KEY, DEFAULT_RUN_MAX_STEPS);

numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, numRegionLoadsToRemember);
minCostNeedBalance = conf.getFloat(MIN_COST_NEED_BALANCE_KEY, minCostNeedBalance);
numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, DEFAULT_KEEP_REGION_LOADS);
minCostNeedBalance = conf.getFloat(MIN_COST_NEED_BALANCE_KEY, DEFAULT_MIN_COST_NEED_BALANCE);
localityCandidateGenerator = new LocalityBasedCandidateGenerator();
localityCost = new ServerLocalityCostFunction(conf);
rackLocalityCost = new RackLocalityCostFunction(conf);
Expand Down