From ddf0cdced23c1b579f749f772040f16b7425c573 Mon Sep 17 00:00:00 2001 From: Zhuobin Zheng Date: Wed, 15 Dec 2021 14:47:44 +0800 Subject: [PATCH 1/2] Confused StoreTooBusy error message shows above parallelPutToStoreThreadLimit but above parallelPreparePutToStoreThreadLimit actually --- .../hbase/regionserver/throttle/StoreHotnessProtector.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java index b907aa640093..37c2d76e742c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java @@ -140,7 +140,9 @@ public void start(Map> familyMaps) throws RegionTooBusyExcept if (tooBusyStore != null) { String msg = "StoreTooBusy," + this.region.getRegionInfo().getRegionNameAsString() + ":" + tooBusyStore - + " Above parallelPutToStoreThreadLimit(" + this.parallelPutToStoreThreadLimit + ")"; + + " Above parallelPutToStoreThreadLimit(" + this.parallelPutToStoreThreadLimit + ")" + + " or parallelPreparePutToStoreThreadLimit(" + + this.parallelPreparePutToStoreThreadLimit + ")"; LOG.trace(msg); throw new RegionTooBusyException(msg); } From deb68c2471151ab6be86b6acafa09460e7d7df71 Mon Sep 17 00:00:00 2001 From: Zhuobin Zheng Date: Thu, 16 Dec 2021 10:38:06 +0800 Subject: [PATCH 2/2] print exactly limit --- .../throttle/StoreHotnessProtector.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java index 37c2d76e742c..7cb3a24dde76 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java @@ -109,6 +109,8 @@ public void start(Map> familyMaps) throws RegionTooBusyExcept } String tooBusyStore = null; + boolean aboveParallelThreadLimit = false; + boolean aboveParallelPrePutLimit = false; for (Map.Entry> e : familyMaps.entrySet()) { Store store = this.region.getStore(e.getKey()); @@ -123,12 +125,16 @@ public void start(Map> familyMaps) throws RegionTooBusyExcept int preparePutCount = preparePutToStoreMap .computeIfAbsent(e.getKey(), key -> new AtomicInteger()) .incrementAndGet(); - if (store.getCurrentParallelPutCount() > this.parallelPutToStoreThreadLimit - || preparePutCount > this.parallelPreparePutToStoreThreadLimit) { + boolean storeAboveThread = + store.getCurrentParallelPutCount() > this.parallelPutToStoreThreadLimit; + boolean storeAbovePrePut = preparePutCount > this.parallelPreparePutToStoreThreadLimit; + if (storeAboveThread || storeAbovePrePut) { tooBusyStore = (tooBusyStore == null ? store.getColumnFamilyName() : tooBusyStore + "," + store.getColumnFamilyName()); } + aboveParallelThreadLimit |= storeAboveThread; + aboveParallelPrePutLimit |= storeAbovePrePut; if (LOG.isTraceEnabled()) { LOG.trace(store.getColumnFamilyName() + ": preparePutCount=" + preparePutCount @@ -137,12 +143,15 @@ public void start(Map> familyMaps) throws RegionTooBusyExcept } } - if (tooBusyStore != null) { + if (aboveParallelThreadLimit || aboveParallelPrePutLimit) { String msg = "StoreTooBusy," + this.region.getRegionInfo().getRegionNameAsString() + ":" + tooBusyStore - + " Above parallelPutToStoreThreadLimit(" + this.parallelPutToStoreThreadLimit + ")" - + " or parallelPreparePutToStoreThreadLimit(" - + this.parallelPreparePutToStoreThreadLimit + ")"; + + " Above " + + (aboveParallelThreadLimit ? "parallelPutToStoreThreadLimit(" + + this.parallelPutToStoreThreadLimit + ")" : "") + + (aboveParallelThreadLimit && aboveParallelPrePutLimit ? " or " : "") + + (aboveParallelPrePutLimit ? "parallelPreparePutToStoreThreadLimit(" + + this.parallelPreparePutToStoreThreadLimit + ")" : ""); LOG.trace(msg); throw new RegionTooBusyException(msg); }