Skip to content

[To rel/0.12] Improve group by month performance #3518

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
qiaojialin merged 6 commits into from
Jul 9, 2021
Merged

[To rel/0.12] Improve group by month performance #3518

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b297adb
improve group by month performance
Alima777 Jul 6, 2021
c28e7d2
update
Alima777 Jul 7, 2021
34eb237
update logic again
Alima777 Jul 7, 2021
f8fa5a0
fix bug
Alima777 Jul 7, 2021
0a96266
fix satisfy bug
Alima777 Jul 7, 2021
4efa856
Fix codesmells
Alima777 Jul 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 65 additions & 43 deletions tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ public class GroupByMonthFilter extends GroupByFilter {
private int intervalInMo;
private Calendar calendar = Calendar.getInstance();
private static final long MS_TO_MONTH = 30 * 86400_000L;
private int intervalCnt = 0;
/** 10.31 -> 11.30 -> 12.31, not 10.31 -> 11.30 -> 12.30 */
private final long initialStartTime;

Expand All @@ -58,7 +57,7 @@ public GroupByMonthFilter(
if (isSlidingStepByMonth) {
slidingStepsInMo = (int) (slidingStep / MS_TO_MONTH);
}
getNextIntervalAndSlidingStep();
getNthTimeInterval(0);
}

public GroupByMonthFilter(GroupByMonthFilter filter) {
Expand All @@ -67,7 +66,6 @@ public GroupByMonthFilter(GroupByMonthFilter filter) {
isSlidingStepByMonth = filter.isSlidingStepByMonth;
intervalInMo = filter.intervalInMo;
slidingStepsInMo = filter.slidingStepsInMo;
intervalCnt = filter.intervalCnt;
initialStartTime = filter.initialStartTime;
calendar = Calendar.getInstance();
calendar.setTimeInMillis(filter.calendar.getTimeInMillis());
Expand All @@ -76,36 +74,32 @@ public GroupByMonthFilter(GroupByMonthFilter filter) {
// TODO: time descending order
@Override
public boolean satisfy(long time, Object value) {
if (time < startTime || time >= endTime) {
if (time < initialStartTime || time >= endTime) {
return false;
} else if (time - startTime < interval) {
return true;
} else if (time >= startTime && time < startTime + slidingStep) {
return time - startTime < interval;
} else {
this.startTime = calendar.getTimeInMillis();
getNextIntervalAndSlidingStep();
return satisfy(time, value);
long count = getTimePointPosition(time);
getNthTimeInterval(count);
return time - startTime < interval;
}
}

@Override
public boolean satisfyStartEndTime(long startTime, long endTime) {
boolean isSatisfy = satisfyCurrentInterval(startTime, endTime);
if (isSatisfy) {
if (satisfyCurrentInterval(startTime, endTime)) {
return true;
} else {
long beforeStartTime = this.startTime;
int beforeIntervalCnt = this.intervalCnt;
// TODO: optimize to jump but not one by one
while (endTime >= this.startTime && !isSatisfy) {
this.startTime = calendar.getTimeInMillis();
getNextIntervalAndSlidingStep();
isSatisfy = satisfyCurrentInterval(startTime, endTime);
// get the interval which contains the start time
long count = getTimePointPosition(startTime);
getNthTimeInterval(count);
// judge two adjacent intervals
if (satisfyCurrentInterval(startTime, endTime)) {
return true;
} else {
getNthTimeInterval(count + 1);
return satisfyCurrentInterval(startTime, endTime);
}
// recover the initial state
this.intervalCnt = beforeIntervalCnt - 1;
this.startTime = beforeStartTime;
getNextIntervalAndSlidingStep();
return isSatisfy;
}
}

Expand All @@ -118,28 +112,20 @@ private boolean satisfyCurrentInterval(long startTime, long endTime) {
if (endTime < this.startTime || startTime >= this.endTime) {
return false;
} else {
return startTime <= this.startTime || startTime - this.startTime < interval;
return startTime - this.startTime < interval;
}
}

@Override
public boolean containStartEndTime(long startTime, long endTime) {
boolean isContained = isContainedByCurrentInterval(startTime, endTime);
if (isContained) {
if (isContainedByCurrentInterval(startTime, endTime)) {
return true;
} else {
long beforeStartTime = this.startTime;
int beforeIntervalCnt = this.intervalCnt;
while (!isContained && startTime >= this.startTime) {
this.startTime = calendar.getTimeInMillis();
getNextIntervalAndSlidingStep();
isContained = isContainedByCurrentInterval(startTime, endTime);
}
// recover the initial state
this.intervalCnt = beforeIntervalCnt - 1;
this.startTime = beforeStartTime;
getNextIntervalAndSlidingStep();
return isContained;
// get the interval which contains the start time
long count = getTimePointPosition(startTime);
getNthTimeInterval(count);
// judge single interval that contains start time
return isContainedByCurrentInterval(startTime, endTime);
}
}

Expand Down Expand Up @@ -171,16 +157,52 @@ public int hashCode() {
interval, slidingStep, startTime, endTime, isSlidingStepByMonth, isIntervalByMonth);
}

private void getNextIntervalAndSlidingStep() {
intervalCnt++;
if (isIntervalByMonth) {
/** Get the interval that @param time belongs to. */
private long getTimePointPosition(long time) {
long count;
if (isSlidingStepByMonth) {
count = (time - this.initialStartTime) / (slidingStepsInMo * 31 * 86400_000L);
calendar.setTimeInMillis(initialStartTime);
calendar.add(Calendar.MONTH, (int) count * slidingStepsInMo);
while (calendar.getTimeInMillis() < time) {
calendar.setTimeInMillis(initialStartTime);
calendar.add(Calendar.MONTH, (int) (count + 1) * slidingStepsInMo);
if (calendar.getTimeInMillis() > time) {
break;
} else {
count++;
}
}
} else {
count = (time - this.initialStartTime) / slidingStep;
}
return count;
}

/** get the Nth time interval. */
private void getNthTimeInterval(long n) {
// get start time of time interval
if (isSlidingStepByMonth) {
calendar.setTimeInMillis(initialStartTime);
calendar.add(Calendar.MONTH, slidingStepsInMo * (intervalCnt - 1) + intervalInMo);
calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * n));
} else {
calendar.setTimeInMillis(initialStartTime + slidingStep * n);
}
this.startTime = calendar.getTimeInMillis();

// get interval and sliding step
if (isIntervalByMonth) {
if (isSlidingStepByMonth) {
calendar.setTimeInMillis(initialStartTime);
calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * n) + intervalInMo);
} else {
calendar.add(Calendar.MONTH, intervalInMo);
}
this.interval = calendar.getTimeInMillis() - startTime;
}
if (isSlidingStepByMonth) {
calendar.setTimeInMillis(initialStartTime);
calendar.add(Calendar.MONTH, slidingStepsInMo * intervalCnt);
calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * (n + 1)));
this.slidingStep = calendar.getTimeInMillis() - startTime;
}
}
Expand Down
22 changes: 22 additions & 0 deletions tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@ public void TestSatisfy2() {
// 1970-03-01 08:00:00
assertTrue(filter.satisfy(5097600000L, null));

// 1970-12-30 08:00:00
assertTrue(filter.satisfy(31363200000L, null));

// 1970-12-31 23:59:58
assertTrue(filter.satisfy(31507198000L, null));

Expand Down Expand Up @@ -130,6 +133,25 @@ public void TestSatisfy3() {
assertFalse(filter.satisfy(31507199000L, null));
}

/** Test filter with slidingStep = 100 days, and timeInterval = 1 mo */
@Test
public void TestSatisfy4() {
GroupByMonthFilter filter =
new GroupByMonthFilter(MS_TO_MONTH, MS_TO_DAY * 100, 0, END_TIME, false, true);

// 1970-01-01 08:00:00, timezone = GMT+08:00
assertTrue(filter.satisfy(0, null));

// 1970-02-01 07:59:59
assertTrue(filter.satisfy(2678399000L, null));

// 1970-03-01 08:00:00
assertFalse(filter.satisfy(5097600000L, null));

// 1970-05-01 08:00:00
assertTrue(filter.satisfy(10368000000L, null));
}

/** Test filter with slidingStep = 1 month, and timeInterval = 1 day */
@Test
public void TestSatisfyStartEndTime() {
Expand Down