修复 0.49 版本一些问题

aliyun · Sep 12, 2024 · a07cf9c · a07cf9c
1 parent 3366b2a
commit a07cf9c
Show file tree

Hide file tree

Showing 6 changed files with 50 additions and 21 deletions.
diff --git a/odps-sdk/odps-sdk-commons/src/main/java/com/aliyun/odps/data/AbstractChar.java b/odps-sdk/odps-sdk-commons/src/main/java/com/aliyun/odps/data/AbstractChar.java
@@ -60,6 +60,6 @@ public String toString() {
   }
 
   public int length() {
-    return value.length();
+    return value.codePointCount(0, value.length());
   }
 }
diff --git a/odps-sdk/odps-sdk-commons/src/test/java/com/aliyun/odps/data/ArrayRecordTest.java b/odps-sdk/odps-sdk-commons/src/test/java/com/aliyun/odps/data/ArrayRecordTest.java
@@ -875,4 +875,23 @@ public void testNewDateTypeInMap() {
 
   }
 
+  @Test
+  public void testVarcharOverFlow() {
+    // make sure count char but not count length in varchar and char
+    TableSchema tableSchema = new TableSchema();
+    tableSchema.addColumn(new Column("varchar", TypeInfoFactory.getVarcharTypeInfo(11)));
+    tableSchema.addColumn(new Column("char", TypeInfoFactory.getCharTypeInfo(11)));
+
+    ArrayRecord r = new ArrayRecord(tableSchema);
+    String str = "184了776991\uD873\uDC56";
+    Assert.assertEquals(11, new Varchar(str).length());
+
+    String str2 = "184了776991\uD83D\uDD3A";
+    Assert.assertEquals(11, new Char(str2).length());
+
+    r.set(0, new Varchar(str));
+    r.set(1, new Char(str2));
+
+    System.out.println(r);
+  }
 }
diff --git a/odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/CreateProjectParam.java b/odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/CreateProjectParam.java
@@ -140,6 +140,11 @@ public CreateProjectParam groupName(String groupName) {
     return this;
   }
 
+  public CreateProjectParam defaultCtrlService(String defaultCtrlService) {
+    this.projectModel.defaultCtrlService = defaultCtrlService;
+    return this;
+  }
+
   Project.ProjectModel getProjectModel() {
     return this.projectModel;
   }

diff --git a/odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/Project.java b/odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/Project.java
@@ -29,7 +29,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Optional;
 
 import com.aliyun.odps.commons.transport.Headers;
 import com.aliyun.odps.commons.transport.Response;
@@ -180,6 +179,8 @@ static class ProjectModel {
      */
     StorageTierInfo storageTierInfo;
 
+    @Element(name = "DefaultCtrlService", required = false)
+    String defaultCtrlService;
   }
 
   public static class ExternalProjectProperties {

diff --git a/...dk/odps-sdk-core/src/main/java/com/aliyun/odps/data/converter/ObjectConverterFactory.java b/...dk/odps-sdk-core/src/main/java/com/aliyun/odps/data/converter/ObjectConverterFactory.java
@@ -32,6 +32,7 @@
 import java.time.format.ResolverStyle;
 import java.time.temporal.ChronoField;
 import java.util.Base64;
+import java.util.TimeZone;
 
 import org.apache.commons.codec.binary.Hex;
 
@@ -522,10 +523,11 @@ private static class DateConverter implements OdpsObjectConverter {
                 parsePattern = DEFAULT_PARSE_PATTERN;
             }
             if (!strictMode) {
-                legacyOutputFormatter = getLegacyDateTimeFormatter(outputPattern, OdpsType.DATE);
+                legacyOutputFormatter =
+                    getLegacyDateTimeFormatter(outputPattern, config.timezone, OdpsType.DATE);
             }
-            outputFormatter = getDateTimeFormatter(outputPattern, null, OdpsType.DATE);
-            parseFormatter = getDateTimeFormatter(parsePattern, null, OdpsType.DATE);
+            outputFormatter = getDateTimeFormatter(outputPattern, config.timezone, OdpsType.DATE);
+            parseFormatter = getDateTimeFormatter(parsePattern, config.timezone, OdpsType.DATE);
         }
 
         @Override
@@ -599,7 +601,7 @@ private static class DatetimeConverter implements OdpsObjectConverter {
                 parseFormatter = getDateTimeFormatter(parsePattern, config.timezone, OdpsType.DATETIME);
             }
             if (!strictMode) {
-                legacyOutputFormatter = getLegacyDateTimeFormatter(outputPattern, OdpsType.DATETIME);
+                legacyOutputFormatter = getLegacyDateTimeFormatter(outputPattern, config.timezone, OdpsType.DATETIME);
             }
             outputFormatter = getDateTimeFormatter(outputPattern, config.timezone, OdpsType.DATETIME);
         }
@@ -662,17 +664,17 @@ private static abstract class AbstractTimestampFormatter implements OdpsObjectCo
 
     private static class TimestampConverter extends AbstractTimestampFormatter {
         private final boolean useSqlFormat;
-
+        private final ZoneId timezone;
         TimestampConverter(OdpsRecordConverterBuilder.Config config) {
             this.useSqlFormat = config.useSqlFormat;
+            this.timezone = config.timezone;
             String outputPattern = config.timestampOutputFormat;
             if (outputPattern != null) {
                 outputFormatter =
                     getDateTimeFormatter(outputPattern, config.timezone, OdpsType.TIMESTAMP);
             } else {
                 outputFormatter = outputFormatter.withZone(config.timezone);
             }
-
             String parsePattern = config.timestampParseFormat;
             if (parsePattern != null) {
                 parseFormatter =
@@ -690,9 +692,7 @@ public String format(Object object, TypeInfo typeInfo, OdpsRecordConverter conve
                 formattedStr = outputFormatter.format(i);
             } else if (object instanceof Timestamp) {
                 Timestamp timestamp = (Timestamp) object;
-                formattedStr =
-                    timestamp.getNanos() == 0 ? timestamp.toString().substring(0, 19)
-                                              : timestamp.toString();
+                formattedStr = outputFormatter.format(timestamp.toInstant().atZone(timezone));
             } else if (object instanceof LocalDateTime) {
                 LocalDateTime localDateTime = (LocalDateTime) object;
                 formattedStr = localDateTime.format(outputFormatter);
@@ -732,11 +732,11 @@ private static class TimestampNtzConverter extends AbstractTimestampFormatter {
 
             String outputPattern = config.timestampNtzOutputFormat;
             if (outputPattern != null) {
-                outputFormatter = getDateTimeFormatter(outputPattern, null, OdpsType.TIMESTAMP_NTZ);
+                outputFormatter = getDateTimeFormatter(outputPattern, ZoneId.of("UTC"), OdpsType.TIMESTAMP_NTZ);
             }
             String parsePattern = config.timestampNtzParseFormat;
             if (parsePattern != null) {
-                parseFormatter = getDateTimeFormatter(outputPattern, null, OdpsType.TIMESTAMP_NTZ);
+                parseFormatter = getDateTimeFormatter(outputPattern, ZoneId.of("UTC"), OdpsType.TIMESTAMP_NTZ);
             }
         }
 
@@ -747,10 +747,8 @@ public String format(Object object, TypeInfo typeInfo, OdpsRecordConverter conve
                 Instant i = (Instant) object;
                 formattedStr = i.atZone(zoneId).format(outputFormatter);
             } else if (object instanceof Timestamp) {
-                Timestamp timestamp = (Timestamp) object;
-                formattedStr =
-                    timestamp.getNanos() == 0 ? timestamp.toString().substring(0, 19)
-                                              : timestamp.toString();
+                Instant i = ((Timestamp) object).toInstant();
+                formattedStr = i.atZone(zoneId).format(outputFormatter);
             } else if (object instanceof LocalDateTime) {
                 LocalDateTime localDateTime = (LocalDateTime) object;
                 formattedStr = localDateTime.format(outputFormatter);
@@ -872,12 +870,18 @@ private static DateTimeFormatter getDateTimeFormatter(String pattern, ZoneId zon
         }
     }
 
-    private static SimpleDateFormat getLegacyDateTimeFormatter(String pattern, OdpsType odpsType) {
+    private static SimpleDateFormat getLegacyDateTimeFormatter(String pattern, ZoneId zoneId,
+                                                               OdpsType odpsType) {
         try {
             pattern = pattern.replace("u", "y");
-            return new SimpleDateFormat(pattern);
+            SimpleDateFormat simpleDateFormat = new SimpleDateFormat(pattern);
+            if (zoneId != null) {
+                simpleDateFormat.setTimeZone(TimeZone.getTimeZone(zoneId));
+            }
+            return simpleDateFormat;
         } catch (Exception e) {
-            throw new IllegalArgumentException("DateTime format for " + odpsType + " illegal: " + pattern);
+            throw new IllegalArgumentException(
+                "DateTime format for " + odpsType + " illegal: " + pattern);
         }
     }
 

diff --git a/odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/task/SQLTask.java b/odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/task/SQLTask.java
@@ -700,7 +700,7 @@ public static Instance run(Odps odps, String project, String sql,
    */
   public static Instance run(Odps odps, String project, String sql,
                              String taskName, Map<String, String> hints,
-                             Map<String, String> aliases, int priority) throws OdpsException {
+                             Map<String, String> aliases, Integer priority) throws OdpsException {
     return run(odps, project, sql, taskName, hints, aliases, priority, "sql");
   }