From 838c7d9170536ad699f5f2e155390567b0e3b4d5 Mon Sep 17 00:00:00 2001 From: bsglz <18031031@qq.com> Date: Thu, 20 Apr 2023 20:48:10 +0800 Subject: [PATCH 1/2] HBASE-27805 The chunk created by mslab may cause memory fragement and lead to fullgc --- hbase-common/src/main/resources/hbase-default.xml | 8 ++++++-- .../org/apache/hadoop/hbase/regionserver/MemStoreLAB.java | 2 +- src/main/asciidoc/_chapters/offheap_read_write.adoc | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index a4ee0c8b20b6..c5279469c959 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -755,8 +755,12 @@ possible configurations would overwhelm and obscure the important. hbase.hregion.memstore.mslab.chunksize - 2097152 - The maximum byte size of a chunk in the MemStoreLAB. Unit: bytes + 2096128 + + The maximum byte size of a chunk in the MemStoreLAB. + Use 2047k as the default because 2m may cause memory fragmentation in some cases. + Unit: bytes + hbase.regionserver.offheap.global.memstore.size diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java index 4edefaf7ca0d..892855979c71 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java @@ -51,7 +51,7 @@ public interface MemStoreLAB { String MSLAB_CLASS_NAME = "hbase.regionserver.mslab.class"; String CHUNK_SIZE_KEY = "hbase.hregion.memstore.mslab.chunksize"; - int CHUNK_SIZE_DEFAULT = 2048 * 1024; + int CHUNK_SIZE_DEFAULT = 2047 * 1024; String INDEX_CHUNK_SIZE_PERCENTAGE_KEY = "hbase.hregion.memstore.mslab.indexchunksize.percent"; float INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT = 0.1f; String MAX_ALLOC_KEY = "hbase.hregion.memstore.mslab.max.allocation"; diff --git a/src/main/asciidoc/_chapters/offheap_read_write.adoc b/src/main/asciidoc/_chapters/offheap_read_write.adoc index bea47c454418..f4288d3ccfd8 100644 --- a/src/main/asciidoc/_chapters/offheap_read_write.adoc +++ b/src/main/asciidoc/_chapters/offheap_read_write.adoc @@ -213,7 +213,8 @@ of off-heap memory that should be used by MSLAB (e.g. `25` would result in 25MB MaxDirectMemorySize property (see <> for more on _HBASE_OFFHEAPSIZE_). The default value of `hbase.regionserver.offheap.global.memstore.size` is 0 which means MSLAB uses onheap, not offheap, chunks by default. -`hbase.hregion.memstore.mslab.chunksize` controls the size of each off-heap chunk. Default is `2097152` (2MB). +`hbase.hregion.memstore.mslab.chunksize` controls the size of each off-heap chunk. Default is `2096128` (that's 2047KB, why not 2MB? +see link:https://issues.apache.org/jira/browse/HBASE-27805[HBASE-27805] for more detail). When a Cell is added to a MemStore, the bytes for that Cell are copied into these off-heap buffers (if `hbase.regionserver.offheap.global.memstore.size` is non-zero) and a Cell POJO will refer to this memory area. This can greatly reduce the on-heap occupancy of the MemStores and reduce the total heap utilization for RegionServers From 87e0692a3a24d98ac243f012e54f289a271d1844 Mon Sep 17 00:00:00 2001 From: bsglz <18031031@qq.com> Date: Sun, 23 Jul 2023 12:12:07 +0800 Subject: [PATCH 2/2] fix review issues --- .../src/main/resources/hbase-default.xml | 8 ++----- .../hbase/regionserver/MemStoreLAB.java | 2 +- .../_chapters/offheap_read_write.adoc | 3 +-- .../asciidoc/_chapters/troubleshooting.adoc | 22 +++++++++++++++++++ 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index c5279469c959..a4ee0c8b20b6 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -755,12 +755,8 @@ possible configurations would overwhelm and obscure the important. hbase.hregion.memstore.mslab.chunksize - 2096128 - - The maximum byte size of a chunk in the MemStoreLAB. - Use 2047k as the default because 2m may cause memory fragmentation in some cases. - Unit: bytes - + 2097152 + The maximum byte size of a chunk in the MemStoreLAB. Unit: bytes hbase.regionserver.offheap.global.memstore.size diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java index 892855979c71..4edefaf7ca0d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java @@ -51,7 +51,7 @@ public interface MemStoreLAB { String MSLAB_CLASS_NAME = "hbase.regionserver.mslab.class"; String CHUNK_SIZE_KEY = "hbase.hregion.memstore.mslab.chunksize"; - int CHUNK_SIZE_DEFAULT = 2047 * 1024; + int CHUNK_SIZE_DEFAULT = 2048 * 1024; String INDEX_CHUNK_SIZE_PERCENTAGE_KEY = "hbase.hregion.memstore.mslab.indexchunksize.percent"; float INDEX_CHUNK_SIZE_PERCENTAGE_DEFAULT = 0.1f; String MAX_ALLOC_KEY = "hbase.hregion.memstore.mslab.max.allocation"; diff --git a/src/main/asciidoc/_chapters/offheap_read_write.adoc b/src/main/asciidoc/_chapters/offheap_read_write.adoc index f4288d3ccfd8..bea47c454418 100644 --- a/src/main/asciidoc/_chapters/offheap_read_write.adoc +++ b/src/main/asciidoc/_chapters/offheap_read_write.adoc @@ -213,8 +213,7 @@ of off-heap memory that should be used by MSLAB (e.g. `25` would result in 25MB MaxDirectMemorySize property (see <> for more on _HBASE_OFFHEAPSIZE_). The default value of `hbase.regionserver.offheap.global.memstore.size` is 0 which means MSLAB uses onheap, not offheap, chunks by default. -`hbase.hregion.memstore.mslab.chunksize` controls the size of each off-heap chunk. Default is `2096128` (that's 2047KB, why not 2MB? -see link:https://issues.apache.org/jira/browse/HBASE-27805[HBASE-27805] for more detail). +`hbase.hregion.memstore.mslab.chunksize` controls the size of each off-heap chunk. Default is `2097152` (2MB). When a Cell is added to a MemStore, the bytes for that Cell are copied into these off-heap buffers (if `hbase.regionserver.offheap.global.memstore.size` is non-zero) and a Cell POJO will refer to this memory area. This can greatly reduce the on-heap occupancy of the MemStores and reduce the total heap utilization for RegionServers diff --git a/src/main/asciidoc/_chapters/troubleshooting.adoc b/src/main/asciidoc/_chapters/troubleshooting.adoc index eb340bf31df8..411b9b8ef6cd 100644 --- a/src/main/asciidoc/_chapters/troubleshooting.adoc +++ b/src/main/asciidoc/_chapters/troubleshooting.adoc @@ -1566,3 +1566,25 @@ then check if you compiled with jdk8 and tried to run it on jdk7. If so, this won't work. Run on jdk8 or recompile with jdk7. See link:https://issues.apache.org/jira/browse/HBASE-10607[HBASE-10607 JDK8 NoSuchMethodError involving ConcurrentHashMap.keySet if running on JRE 7]. + +=== Full gc caused by mslab when using G1 + +The default size of chunk used by mslab is 2MB, when using G1, if heapRegionSize equals 4MB, these chunks are allocated as humongous objects, exclusively allocating one region, then the remaining 2MB become memory fragment. + +Lots of memory fragment may lead to full gc even if the percent of used heap not high enough. + +The G1HeapRegionSize calculated by initial_heap_size and max_heap_size, here are some cases for better understand: + +- xmx=10G -> region size 2M +- xms=10G, xmx=10G -> region size 4M +- xmx=20G -> region size 4M +- xms=20G, xmx=20G -> region size 8M +- xmx=30G -> region size 4M +- xmx=32G -> region size 8M + +You can avoid this problem by reducing the chunk size a bit to 2047KB as below. + +---- +hbase.hregion.memstore.mslab.chunksize 2096128 +---- +