Skip to content

[2021-07-11] update #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
SpriCoder merged 2 commits into from
Jul 11, 2021
Merged

[2021-07-11] update #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
98bbec7
Improve group by month performance (#3517)
Alima777 Jul 9, 2021
e5c9f9d
[DOC] fix zh QuickStart multi_dir_strategy typo (#3531)
Eights-Li Jul 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/zh/UserGuide/QuickStart/Files.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ IoTDB需要存储的数据分为三类，分别为数据文件、系统文件以

数据文件存储了用户写入IoTDB系统的所有数据。包含TsFile文件和其他文件，可通过[data_dirs配置项](../Appendix/Config-Manual.md)进行配置。

为了更好的支持用户对于磁盘空间扩展等存储需求，IoTDB为TsFile的存储配置增加了多文件目录的存储方式，用户可自主配置多个存储路径作为数据的持久化位置（详情见[data_dirs配置项](../Appendix/Config-Manual.md)），并可以指定或自定义目录选择策略（详情见[mult_dir_strategy配置项](../Appendix/Config-Manual.md)）。
为了更好的支持用户对于磁盘空间扩展等存储需求，IoTDB为TsFile的存储配置增加了多文件目录的存储方式，用户可自主配置多个存储路径作为数据的持久化位置（详情见[data_dirs配置项](../Appendix/Config-Manual.md)），并可以指定或自定义目录选择策略（详情见[multi_dir_strategy配置项](../Appendix/Config-Manual.md)）。

### 系统文件

Expand All @@ -43,7 +43,7 @@ IoTDB需要存储的数据分为三类，分别为数据文件、系统文件以

接下来我们将举一个数据目录配置的例子，来具体说明如何配置数据的存储目录。

IoTDB涉及到的所有数据目录路径有：data_dirs, mult_dir_strategy, base_dir和wal_dir，它们分别涉及的是IoTDB的数据文件、系统文件以及写前日志文件。您可以选择输入路径自行配置，也可以不进行任何操作使用系统默认的配置项。
IoTDB涉及到的所有数据目录路径有：data_dirs, multi_dir_strategy, base_dir和wal_dir，它们分别涉及的是IoTDB的数据文件、数据文件多目录存储策略、系统文件以及写前日志文件。您可以选择输入路径自行配置，也可以不进行任何操作使用系统默认的配置项。

以下我们给出一个用户对五个目录都进行自行配置的例子。

Expand Down
108 changes: 65 additions & 43 deletions tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ public class GroupByMonthFilter extends GroupByFilter {
private int intervalInMo;
private Calendar calendar = Calendar.getInstance();
private static final long MS_TO_MONTH = 30 * 86400_000L;
private int intervalCnt = 0;
/** 10.31 -> 11.30 -> 12.31, not 10.31 -> 11.30 -> 12.30 */
private final long initialStartTime;

Expand All @@ -58,7 +57,7 @@ public GroupByMonthFilter(
if (isSlidingStepByMonth) {
slidingStepsInMo = (int) (slidingStep / MS_TO_MONTH);
}
getNextIntervalAndSlidingStep();
getNthTimeInterval(0);
}

public GroupByMonthFilter(GroupByMonthFilter filter) {
Expand All @@ -67,7 +66,6 @@ public GroupByMonthFilter(GroupByMonthFilter filter) {
isSlidingStepByMonth = filter.isSlidingStepByMonth;
intervalInMo = filter.intervalInMo;
slidingStepsInMo = filter.slidingStepsInMo;
intervalCnt = filter.intervalCnt;
initialStartTime = filter.initialStartTime;
calendar = Calendar.getInstance();
calendar.setTimeInMillis(filter.calendar.getTimeInMillis());
Expand All @@ -76,36 +74,32 @@ public GroupByMonthFilter(GroupByMonthFilter filter) {
// TODO: time descending order
@Override
public boolean satisfy(long time, Object value) {
if (time < startTime || time >= endTime) {
if (time < initialStartTime || time >= endTime) {
return false;
} else if (time - startTime < interval) {
return true;
} else if (time >= startTime && time < startTime + slidingStep) {
return time - startTime < interval;
} else {
this.startTime = calendar.getTimeInMillis();
getNextIntervalAndSlidingStep();
return satisfy(time, value);
long count = getTimePointPosition(time);
getNthTimeInterval(count);
return time - startTime < interval;
}
}

@Override
public boolean satisfyStartEndTime(long startTime, long endTime) {
boolean isSatisfy = satisfyCurrentInterval(startTime, endTime);
if (isSatisfy) {
if (satisfyCurrentInterval(startTime, endTime)) {
return true;
} else {
long beforeStartTime = this.startTime;
int beforeIntervalCnt = this.intervalCnt;
// TODO: optimize to jump but not one by one
while (endTime >= this.startTime && !isSatisfy) {
this.startTime = calendar.getTimeInMillis();
getNextIntervalAndSlidingStep();
isSatisfy = satisfyCurrentInterval(startTime, endTime);
// get the interval which contains the start time
long count = getTimePointPosition(startTime);
getNthTimeInterval(count);
// judge two adjacent intervals
if (satisfyCurrentInterval(startTime, endTime)) {
return true;
} else {
getNthTimeInterval(count + 1);
return satisfyCurrentInterval(startTime, endTime);
}
// recover the initial state
this.intervalCnt = beforeIntervalCnt - 1;
this.startTime = beforeStartTime;
getNextIntervalAndSlidingStep();
return isSatisfy;
}
}

Expand All @@ -118,28 +112,20 @@ private boolean satisfyCurrentInterval(long startTime, long endTime) {
if (endTime < this.startTime || startTime >= this.endTime) {
return false;
} else {
return startTime <= this.startTime || startTime - this.startTime < interval;
return startTime - this.startTime < interval;
}
}

@Override
public boolean containStartEndTime(long startTime, long endTime) {
boolean isContained = isContainedByCurrentInterval(startTime, endTime);
if (isContained) {
if (isContainedByCurrentInterval(startTime, endTime)) {
return true;
} else {
long beforeStartTime = this.startTime;
int beforeIntervalCnt = this.intervalCnt;
while (!isContained && startTime >= this.startTime) {
this.startTime = calendar.getTimeInMillis();
getNextIntervalAndSlidingStep();
isContained = isContainedByCurrentInterval(startTime, endTime);
}
// recover the initial state
this.intervalCnt = beforeIntervalCnt - 1;
this.startTime = beforeStartTime;
getNextIntervalAndSlidingStep();
return isContained;
// get the interval which contains the start time
long count = getTimePointPosition(startTime);
getNthTimeInterval(count);
// judge single interval that contains start time
return isContainedByCurrentInterval(startTime, endTime);
}
}

Expand Down Expand Up @@ -171,16 +157,52 @@ public int hashCode() {
interval, slidingStep, startTime, endTime, isSlidingStepByMonth, isIntervalByMonth);
}

private void getNextIntervalAndSlidingStep() {
intervalCnt++;
if (isIntervalByMonth) {
/** Get the interval that @param time belongs to. */
private long getTimePointPosition(long time) {
long count;
if (isSlidingStepByMonth) {
count = (time - this.initialStartTime) / (slidingStepsInMo * 31 * 86400_000L);
calendar.setTimeInMillis(initialStartTime);
calendar.add(Calendar.MONTH, (int) count * slidingStepsInMo);
while (calendar.getTimeInMillis() < time) {
calendar.setTimeInMillis(initialStartTime);
calendar.add(Calendar.MONTH, (int) (count + 1) * slidingStepsInMo);
if (calendar.getTimeInMillis() > time) {
break;
} else {
count++;
}
}
} else {
count = (time - this.initialStartTime) / slidingStep;
}
return count;
}

/** get the Nth time interval. */
private void getNthTimeInterval(long n) {
// get start time of time interval
if (isSlidingStepByMonth) {
calendar.setTimeInMillis(initialStartTime);
calendar.add(Calendar.MONTH, slidingStepsInMo * (intervalCnt - 1) + intervalInMo);
calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * n));
} else {
calendar.setTimeInMillis(initialStartTime + slidingStep * n);
}
this.startTime = calendar.getTimeInMillis();

// get interval and sliding step
if (isIntervalByMonth) {
if (isSlidingStepByMonth) {
calendar.setTimeInMillis(initialStartTime);
calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * n) + intervalInMo);
} else {
calendar.add(Calendar.MONTH, intervalInMo);
}
this.interval = calendar.getTimeInMillis() - startTime;
}
if (isSlidingStepByMonth) {
calendar.setTimeInMillis(initialStartTime);
calendar.add(Calendar.MONTH, slidingStepsInMo * intervalCnt);
calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * (n + 1)));
this.slidingStep = calendar.getTimeInMillis() - startTime;
}
}
Expand Down
22 changes: 22 additions & 0 deletions tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@ public void TestSatisfy2() {
// 1970-03-01 08:00:00
assertTrue(filter.satisfy(5097600000L, null));

// 1970-12-30 08:00:00
assertTrue(filter.satisfy(31363200000L, null));

// 1970-12-31 23:59:58
assertTrue(filter.satisfy(31507198000L, null));

Expand Down Expand Up @@ -130,6 +133,25 @@ public void TestSatisfy3() {
assertFalse(filter.satisfy(31507199000L, null));
}

/** Test filter with slidingStep = 100 days, and timeInterval = 1 mo */
@Test
public void TestSatisfy4() {
GroupByMonthFilter filter =
new GroupByMonthFilter(MS_TO_MONTH, MS_TO_DAY * 100, 0, END_TIME, false, true);

// 1970-01-01 08:00:00, timezone = GMT+08:00
assertTrue(filter.satisfy(0, null));

// 1970-02-01 07:59:59
assertTrue(filter.satisfy(2678399000L, null));

// 1970-03-01 08:00:00
assertFalse(filter.satisfy(5097600000L, null));

// 1970-05-01 08:00:00
assertTrue(filter.satisfy(10368000000L, null));
}

/** Test filter with slidingStep = 1 month, and timeInterval = 1 day */
@Test
public void TestSatisfyStartEndTime() {
Expand Down