Skip to content

[Bug] Unify the timezone #3910

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 3 commits into from
Jun 24, 2020
Merged

[Bug] Unify the timezone #3910

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4a40cc1
fix bug
morningman Jun 19, 2020
f104fb2
add PRC
morningman-cmy Jun 21, 2020
0dd03f8
fix ut
morningman-cmy Jun 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fe/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public class DynamicPartitionProperty {
private int buckets;
private StartOfDate startOfWeek;
private StartOfDate startOfMonth;
private TimeZone tz = TimeUtils.getDefaultTimeZone();
private TimeZone tz = TimeUtils.getSystemTimeZone();

public DynamicPartitionProperty(Map<String, String> properties) {
if (properties != null && !properties.isEmpty()) {
Expand Down
3 changes: 1 addition & 2 deletions fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@
import java.text.SimpleDateFormat;
import java.time.DayOfWeek;
import java.time.Month;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Calendar;
Expand Down Expand Up @@ -207,7 +206,7 @@ private static boolean checkInputDynamicPartitionProperties(Map<String, String>
throw new DdlException("Must assign dynamic_partition.buckets properties");
}
if (Strings.isNullOrEmpty(timeZone)) {
properties.put(DynamicPartitionProperty.TIME_ZONE, ZoneId.systemDefault().toString());
properties.put(DynamicPartitionProperty.TIME_ZONE, TimeUtils.getSystemTimeZone().getID());
}
}
return true;
Expand Down
19 changes: 11 additions & 8 deletions fe/src/main/java/org/apache/doris/common/util/TimeUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.VariableMgr;

import com.google.common.base.Preconditions;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.text.ParseException;
Expand All @@ -54,7 +54,8 @@ public class TimeUtils {
private static final TimeZone TIME_ZONE;

// set CST to +08:00 instead of America/Chicago
public static final ImmutableMap<String, String> timeZoneAliasMap = ImmutableMap.of("CST", DEFAULT_TIME_ZONE);
public static final ImmutableMap<String, String> timeZoneAliasMap = ImmutableMap.of(
"CST", DEFAULT_TIME_ZONE, "PRC", DEFAULT_TIME_ZONE);

// NOTICE: Date formats are not synchronized.
// it must be used as synchronized externally.
Expand Down Expand Up @@ -126,17 +127,19 @@ public static TimeZone getTimeZone() {
return TimeZone.getTimeZone(ZoneId.of(timezone, timeZoneAliasMap));
}

public static TimeZone getDefaultTimeZone() {
return TimeZone.getTimeZone(ZoneId.of(DEFAULT_TIME_ZONE, timeZoneAliasMap));
// return the time zone of current system
public static TimeZone getSystemTimeZone() {
return TimeZone.getTimeZone(ZoneId.of(ZoneId.systemDefault().getId(), timeZoneAliasMap));
}

// get time zone of given zone name, or return system time zone if name is null.
public static TimeZone getOrSystemTimeZone(String timeZone) {
if (timeZone == null) {
timeZone = ZoneId.systemDefault().toString();
return getSystemTimeZone();
}
return TimeZone.getTimeZone(timeZone);
return TimeZone.getTimeZone(ZoneId.of(timeZone, timeZoneAliasMap));
}

public static String longToTimeString(long timeStamp, SimpleDateFormat dateFormat) {
if (timeStamp <= 0L) {
return FeConstants.null_string;
Expand Down
5 changes: 2 additions & 3 deletions fe/src/main/java/org/apache/doris/qe/GlobalVariable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@
package org.apache.doris.qe;

import org.apache.doris.common.Version;

import java.time.ZoneId;
import org.apache.doris.common.util.TimeUtils;

// You can place your global variable in this class with public and VariableMgr.VarAttr annotation.
// You can get this variable from MySQL client with statement `SELECT @@variable_name`,
Expand Down Expand Up @@ -50,7 +49,7 @@ public final class GlobalVariable {

// A string to be executed by the server for each client that connects
@VariableMgr.VarAttr(name = "system_time_zone", flag = VariableMgr.READ_ONLY)
public static String systemTimeZone = ZoneId.systemDefault().normalized().toString();
public static String systemTimeZone = TimeUtils.getSystemTimeZone().getID();

// The amount of memory allocated for caching query results
@VariableMgr.VarAttr(name = "query_cache_size")
Expand Down
3 changes: 2 additions & 1 deletion fe/src/main/java/org/apache/doris/qe/SessionVariable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.qe.VariableMgr.VarAttr;
import org.apache.doris.thrift.TQueryOptions;

Expand Down Expand Up @@ -183,7 +184,7 @@ public class SessionVariable implements Serializable, Writable {

// The current time zone
@VariableMgr.VarAttr(name = TIME_ZONE)
private String timeZone = "CST";
private String timeZone = TimeUtils.getSystemTimeZone().getID();

@VariableMgr.VarAttr(name = PARALLEL_EXCHANGE_INSTANCE_NUM)
private int exchangeInstanceParallel = -1;
Expand Down
52 changes: 25 additions & 27 deletions fe/src/test/java/org/apache/doris/common/util/DynamicPartitionUtilTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,12 @@
import org.junit.Assert;
import org.junit.Test;

import java.sql.Time;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.DateTimeException;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.TemporalAccessor;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;
import java.util.TimeZone;

public class DynamicPartitionUtilTest {

Expand All @@ -63,7 +56,12 @@ private static Map<String, String> getDynamProp(String timeUnit, int start, int

private static ZonedDateTime getZonedDateTimeFromStr(String dateStr) throws DateTimeException {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern(FORMAT);
return LocalDate.parse(dateStr, formatter).atStartOfDay(ZoneId.systemDefault());
return LocalDate.parse(dateStr, formatter).atStartOfDay(
TimeUtils.getOrSystemTimeZone(TimeUtils.DEFAULT_TIME_ZONE).toZoneId());
}

private static TimeZone getCTSTimeZone() {
return TimeUtils.getOrSystemTimeZone(TimeUtils.DEFAULT_TIME_ZONE);
}

@Test
Expand All @@ -75,25 +73,25 @@ public void testGetPartitionRangeString() throws DateTimeException {
String res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), -7,
FORMAT);
Assert.assertEquals("2020-05-18", res);
String partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "DAY");
String partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "DAY");
Assert.assertEquals("20200518", partName);
// 2. 2020-05-25, offset 0
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), 0,
FORMAT);
Assert.assertEquals("2020-05-25", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "DAY");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "DAY");
Assert.assertEquals("20200525", partName);
// 3. 2020-05-25, offset 7
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), 7,
FORMAT);
Assert.assertEquals("2020-06-01", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "DAY");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "DAY");
Assert.assertEquals("20200601", partName);
// 4. 2020-02-28, offset 3
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-02-28"), 3,
FORMAT);
Assert.assertEquals("2020-03-02", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "DAY");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "DAY");
Assert.assertEquals("20200302", partName);

// TimeUnit: WEEK
Expand All @@ -102,63 +100,63 @@ public void testGetPartitionRangeString() throws DateTimeException {
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), 0,
FORMAT);
Assert.assertEquals("2020-05-25", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "WEEK");
Assert.assertEquals("2020_22", partName);

// 2. 2020-05-28, start day: MONDAY, offset 0
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 1, -1));
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-28"), 0,
FORMAT);
Assert.assertEquals("2020-05-25", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "WEEK");
Assert.assertEquals("2020_22", partName);

// 3. 2020-05-25, start day: SUNDAY, offset 0
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 7, -1));
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), 0,
FORMAT);
Assert.assertEquals("2020-05-31", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "WEEK");
Assert.assertEquals("2020_23", partName);

// 4. 2020-05-25, start day: MONDAY, offset -2
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 1, -1));
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), -2,
FORMAT);
Assert.assertEquals("2020-05-11", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "WEEK");
Assert.assertEquals("2020_20", partName);

// 5. 2020-02-29, start day: WED, offset 0
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 3, -1));
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-02-29"), 0,
FORMAT);
Assert.assertEquals("2020-02-26", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "WEEK");
Assert.assertEquals("2020_09", partName);

// 6. 2020-02-29, start day: TUS, offset 1
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 2, -1));
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-02-29"), 1,
FORMAT);
Assert.assertEquals("2020-03-03", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "WEEK");
Assert.assertEquals("2020_10", partName);

// 6. 2020-01-01, start day: MONDAY, offset -1
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 1, -1));
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-01-01"), -1,
FORMAT);
Assert.assertEquals("2019-12-23", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "WEEK");
Assert.assertEquals("2019_52", partName);

// 6. 2020-01-01, start day: MONDAY, offset 0
property = new DynamicPartitionProperty(getDynamProp("WEEK", -3, 3, 1, -1));
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-01-01"), 0,
FORMAT);
Assert.assertEquals("2019-12-30", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "WEEK");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "WEEK");
Assert.assertEquals("2019_53", partName);

// TimeUnit: MONTH
Expand All @@ -167,47 +165,47 @@ public void testGetPartitionRangeString() throws DateTimeException {
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), 0,
FORMAT);
Assert.assertEquals("2020-05-01", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "MONTH");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "MONTH");
Assert.assertEquals("202005", partName);

// 2. 2020-05-25, start day: 26, offset 0
property = new DynamicPartitionProperty(getDynamProp("MONTH", -3, 3, -1, 26));
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), 0,
FORMAT);
Assert.assertEquals("2020-04-26", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "MONTH");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "MONTH");
Assert.assertEquals("202004", partName);

// 3. 2020-05-25, start day: 26, offset -1
property = new DynamicPartitionProperty(getDynamProp("MONTH", -3, 3, -1, 26));
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-05-25"), -1,
FORMAT);
Assert.assertEquals("2020-03-26", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "MONTH");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "MONTH");
Assert.assertEquals("202003", partName);

// 4. 2020-02-29, start day: 26, offset 3
property = new DynamicPartitionProperty(getDynamProp("MONTH", -3, 3, -1, 26));
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-02-29"), 3,
FORMAT);
Assert.assertEquals("2020-05-26", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "MONTH");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "MONTH");
Assert.assertEquals("202005", partName);

// 5. 2020-02-29, start day: 27, offset 0
property = new DynamicPartitionProperty(getDynamProp("MONTH", -3, 3, -1, 27));
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-02-29"), 0,
FORMAT);
Assert.assertEquals("2020-02-27", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "MONTH");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "MONTH");
Assert.assertEquals("202002", partName);

// 6. 2020-02-29, start day: 27, offset -3
property = new DynamicPartitionProperty(getDynamProp("MONTH", -3, 3, -1, 27));
res = DynamicPartitionUtil.getPartitionRangeString(property, getZonedDateTimeFromStr("2020-02-29"), -3,
FORMAT);
Assert.assertEquals("2019-11-27", res);
partName = DynamicPartitionUtil.getFormattedPartitionName(TimeUtils.getDefaultTimeZone(), res, "MONTH");
partName = DynamicPartitionUtil.getFormattedPartitionName(getCTSTimeZone(), res, "MONTH");
Assert.assertEquals("201911", partName);
}

Expand Down
8 changes: 7 additions & 1 deletion fe/src/test/java/org/apache/doris/qe/VariableMgrTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,16 @@ public void testNormal() throws IllegalAccessException, DdlException, NoSuchFiel

SetVar setVar3 = new SetVar(SetType.GLOBAL, "time_zone", new StringLiteral("Asia/Shanghai"));
VariableMgr.setVar(var, setVar3);
Assert.assertEquals("CST", var.getTimeZone());
Assert.assertEquals("Asia/Shanghai", var.getTimeZone());
var = VariableMgr.newSessionVariable();
Assert.assertEquals("Asia/Shanghai", var.getTimeZone());

setVar3 = new SetVar(SetType.GLOBAL, "time_zone", new StringLiteral("CST"));
VariableMgr.setVar(var, setVar3);
Assert.assertEquals("Asia/Shanghai", var.getTimeZone());
var = VariableMgr.newSessionVariable();
Assert.assertEquals("CST", var.getTimeZone());

// Set session variable
setVar = new SetVar(SetType.GLOBAL, "exec_mem_limit", new IntLiteral(1234L));
VariableMgr.setVar(var, setVar);
Expand Down