Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DataNodeDevicePathCache should use free memory of memtable #12034

Merged
merged 3 commits into from
Feb 7, 2024
Merged

DataNodeDevicePathCache should use free memory of memtable #12034

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0f98b93
DataNodeDevicePathCache should use free memory of memtable
HTHou Feb 7, 2024
490cbb6
Update comment
HTHou Feb 7, 2024
620af23
add a check
HTHou Feb 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,13 +166,13 @@ public class IoTDBConfig {
private double rejectProportion = 0.8;

/** The proportion of write memory for memtable */
private double writeProportionForMemtable = 0.72;
private double writeProportionForMemtable = 0.76;

/** The proportion of write memory for compaction */
private double compactionProportion = 0.2;

/** The proportion of write memory for device path cache */
private double devicePathCacheProportion = 0.04;
/** The proportion of memtable memory for device path cache */
private double devicePathCacheProportion = 0.05;

/**
* If memory cost of data region increased more than proportion of {@linkplain
Expand Down
18 changes: 10 additions & 8 deletions iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,14 @@ public void loadProperties(Properties properties) throws BadNodeUrlException, IO
.getProperty("reject_proportion", Double.toString(conf.getRejectProportion()))
.trim()));

conf.setDevicePathCacheProportion(
Double.parseDouble(
properties
.getProperty(
"device_path_cache_proportion",
Double.toString(conf.getDevicePathCacheProportion()))
.trim()));

conf.setWriteMemoryVariationReportProportion(
Double.parseDouble(
properties
Expand Down Expand Up @@ -1772,7 +1780,7 @@ private void initStorageEngineAllocate(Properties properties) {
writeMemoryProportion += proportionValue;
if (proportionValue <= 0) {
LOGGER.warn(
"The value of write_memory_proportion is illegal, use default value 18:1:1 .");
"The value of write_memory_proportion is illegal, use default value 19:1 .");
return;
}
}
Expand All @@ -1784,9 +1792,7 @@ private void initStorageEngineAllocate(Properties properties) {
(double) Integer.parseInt(writeProportionArray[0].trim()) / writeMemoryProportion;
double timePartitionInfoProportion =
(double) Integer.parseInt(writeProportionArray[1].trim()) / writeMemoryProportion;
double devicePathCacheProportion =
(double) Integer.parseInt(writeProportionArray[2].trim()) / writeMemoryProportion;
// writeProportionForMemtable = 8/10 * 18/20 = 0.72 default
// writeProportionForMemtable = 8/10 * 19/20 = 0.76 default
conf.setWriteProportionForMemtable(
writeAllProportionOfStorageEngineMemory * memTableProportion);

Expand All @@ -1795,10 +1801,6 @@ private void initStorageEngineAllocate(Properties properties) {
(long)
((writeAllProportionOfStorageEngineMemory * timePartitionInfoProportion)
* storageMemoryTotal));

// device path cache default memory is value is 8/10 * 1/20 = 0.04 for StorageEngine
conf.setDevicePathCacheProportion(
writeAllProportionOfStorageEngineMemory * devicePathCacheProportion);
}
}
}
Expand Down
1 change: 1 addition & 0 deletions ...va/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ private DataNodeDevicePathCache() {
.maximumWeight(
(long)
(config.getAllocateMemoryForStorageEngine()
* config.getWriteProportionForMemtable()
* config.getDevicePathCacheProportion()))
.weigher(
(Weigher<String, PartialPath>) (key, val) -> (PartialPath.estimateSize(val) + 32))
Expand Down
9 changes: 6 additions & 3 deletions iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,11 +181,10 @@ data_replication_factor=1
# The parameter form is a:b:c:d, where a, b, c and d are integers. for example: 8:2 , 7:3
# storage_engine_memory_proportion=8:2

# Memory allocation ratio in writing: Memtable, TimePartitionInfo, DevicePathCache
# Memory allocation ratio in writing: Memtable, TimePartitionInfo
# Memtable is the total memory size of all memtables
# TimePartitionInfo is the total memory size of last flush time of all data regions
# DevicePathCache is the deviceId cache, keep only one copy of the same deviceId in memory
# write_memory_proportion=18:1:1
# write_memory_proportion=19:1

# primitive array size (length of each array) in array pool
# Datatype: int
Expand All @@ -210,6 +209,10 @@ data_replication_factor=1
# Datatype: double
# reject_proportion=0.8

# Ratio of memory for the DevicePathCache. DevicePathCache is the deviceId cache, keep only one copy of the same deviceId in memory
# Datatype: double
# device_path_cache_proportion=0.05

# If memory cost of data region increased more than proportion of allocated memory for write, report to system. The default value is 0.001
# Datatype: double
# write_memory_variation_report_proportion=0.001
Expand Down
Loading