diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index f9e69193622f..854401d7b5e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -166,13 +166,13 @@ public class IoTDBConfig { private double rejectProportion = 0.8; /** The proportion of write memory for memtable */ - private double writeProportionForMemtable = 0.72; + private double writeProportionForMemtable = 0.76; /** The proportion of write memory for compaction */ private double compactionProportion = 0.2; - /** The proportion of write memory for device path cache */ - private double devicePathCacheProportion = 0.04; + /** The proportion of memtable memory for device path cache */ + private double devicePathCacheProportion = 0.05; /** * If memory cost of data region increased more than proportion of {@linkplain diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index c451841d928a..e4c5c243598a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -284,11 +284,27 @@ public void loadProperties(Properties properties) throws BadNodeUrlException, IO .getProperty("flush_proportion", Double.toString(conf.getFlushProportion())) .trim())); - conf.setRejectProportion( + double rejectProportion = Double.parseDouble( properties .getProperty("reject_proportion", Double.toString(conf.getRejectProportion())) - .trim())); + .trim()); + + double devicePathCacheProportion = + Double.parseDouble( + properties + .getProperty( + "device_path_cache_proportion", + Double.toString(conf.getDevicePathCacheProportion())) + .trim()); + + if (rejectProportion + devicePathCacheProportion >= 1) { + LOGGER.warn( + "The sum of write_memory_proportion and device_path_cache_proportion is too large, use default values 0.8 and 0.05."); + } else { + conf.setRejectProportion(rejectProportion); + conf.setDevicePathCacheProportion(devicePathCacheProportion); + } conf.setWriteMemoryVariationReportProportion( Double.parseDouble( @@ -1772,7 +1788,7 @@ private void initStorageEngineAllocate(Properties properties) { writeMemoryProportion += proportionValue; if (proportionValue <= 0) { LOGGER.warn( - "The value of write_memory_proportion is illegal, use default value 18:1:1 ."); + "The value of write_memory_proportion is illegal, use default value 19:1 ."); return; } } @@ -1784,9 +1800,7 @@ private void initStorageEngineAllocate(Properties properties) { (double) Integer.parseInt(writeProportionArray[0].trim()) / writeMemoryProportion; double timePartitionInfoProportion = (double) Integer.parseInt(writeProportionArray[1].trim()) / writeMemoryProportion; - double devicePathCacheProportion = - (double) Integer.parseInt(writeProportionArray[2].trim()) / writeMemoryProportion; - // writeProportionForMemtable = 8/10 * 18/20 = 0.72 default + // writeProportionForMemtable = 8/10 * 19/20 = 0.76 default conf.setWriteProportionForMemtable( writeAllProportionOfStorageEngineMemory * memTableProportion); @@ -1795,10 +1809,6 @@ private void initStorageEngineAllocate(Properties properties) { (long) ((writeAllProportionOfStorageEngineMemory * timePartitionInfoProportion) * storageMemoryTotal)); - - // device path cache default memory is value is 8/10 * 1/20 = 0.04 for StorageEngine - conf.setDevicePathCacheProportion( - writeAllProportionOfStorageEngineMemory * devicePathCacheProportion); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java index 0a189043e111..f444f9bb9e59 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java @@ -41,6 +41,7 @@ private DataNodeDevicePathCache() { .maximumWeight( (long) (config.getAllocateMemoryForStorageEngine() + * config.getWriteProportionForMemtable() * config.getDevicePathCacheProportion())) .weigher( (Weigher) (key, val) -> (PartialPath.estimateSize(val) + 32)) diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties index 5330bdaa44d0..88d1d0f47cfa 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties @@ -181,11 +181,10 @@ data_replication_factor=1 # The parameter form is a:b:c:d, where a, b, c and d are integers. for example: 8:2 , 7:3 # storage_engine_memory_proportion=8:2 -# Memory allocation ratio in writing: Memtable, TimePartitionInfo, DevicePathCache +# Memory allocation ratio in writing: Memtable, TimePartitionInfo # Memtable is the total memory size of all memtables # TimePartitionInfo is the total memory size of last flush time of all data regions -# DevicePathCache is the deviceId cache, keep only one copy of the same deviceId in memory -# write_memory_proportion=18:1:1 +# write_memory_proportion=19:1 # primitive array size (length of each array) in array pool # Datatype: int @@ -210,6 +209,10 @@ data_replication_factor=1 # Datatype: double # reject_proportion=0.8 +# Ratio of memory for the DevicePathCache. DevicePathCache is the deviceId cache, keep only one copy of the same deviceId in memory +# Datatype: double +# device_path_cache_proportion=0.05 + # If memory cost of data region increased more than proportion of allocated memory for write, report to system. The default value is 0.001 # Datatype: double # write_memory_variation_report_proportion=0.001