Skip to content

[WIP] HIVE-29383: Iceberg: [V3] Add support for timestamp with nanosecond precession #6258

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ayushtkn wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[WIP] HIVE-29383: Iceberg: [V3] Add support for timestamp with nanosecond precession #6258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TimestampTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.iceberg.Schema;
import org.apache.iceberg.expressions.Expressions;
Expand Down Expand Up @@ -130,6 +132,10 @@ Type convertType(TypeInfo typeInfo, String defaultValue) {
case STRING:
return Types.StringType.get();
case TIMESTAMP:
TimestampTypeInfo ts = (TimestampTypeInfo) typeInfo;
if (ts.getPrecision() == 9) {
return Types.TimestampNanoType.withoutZone();
}
return Types.TimestampType.withoutZone();
case DATE:
return Types.DateType.get();
Expand All @@ -141,6 +147,10 @@ Type convertType(TypeInfo typeInfo, String defaultValue) {
default:
// special case for Timestamp with Local TZ which is only available in Hive3
if ("TIMESTAMPLOCALTZ".equalsIgnoreCase(((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory().name())) {
TimestampLocalTZTypeInfo tz = (TimestampLocalTZTypeInfo) typeInfo;
if (tz.getPrecision() == 9) {
return Types.TimestampNanoType.withZone();
}
return Types.TimestampType.withZone();
}
throw new IllegalArgumentException("Unsupported Hive type (" +
Expand Down
6 changes: 6 additions & 0 deletions iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,12 @@ public static String convertToTypeString(Type type) {
return "timestamp with local time zone";
}
return "timestamp";
case TIMESTAMP_NANO:
Types.TimestampNanoType timestampNanoType = (Types.TimestampNanoType) type;
if (timestampNanoType.shouldAdjustToUTC()) {
return "timestamp with local time zone(9)";
}
return "timestamp(9)";
case FIXED:
case BINARY:
return "binary";
Expand Down
5 changes: 5 additions & 0 deletions ...rc/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,11 @@ public ObjectInspector primitive(Type.PrimitiveType primitiveType) {
case TIMESTAMP:
boolean adjustToUTC = ((Types.TimestampType) primitiveType).shouldAdjustToUTC();
return adjustToUTC ? TIMESTAMP_INSPECTOR_WITH_TZ : TIMESTAMP_INSPECTOR;
case TIMESTAMP_NANO:
boolean adjustUTC = ((Types.TimestampNanoType) primitiveType).shouldAdjustToUTC();
return adjustUTC ?
IcebergTimestampWithZoneObjectInspectorHive3.get(9) :
IcebergTimestampObjectInspectorHive3.get(9);
case TIME:
return IcebergTimeObjectInspector.get();
default:
Expand Down
18 changes: 15 additions & 3 deletions ...rg/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspectorHive3.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,20 +26,32 @@
import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;


public class IcebergTimestampObjectInspectorHive3 extends AbstractPrimitiveJavaObjectInspector
implements TimestampObjectInspector, WriteObjectInspector {

private static final IcebergTimestampObjectInspectorHive3 INSTANCE = new IcebergTimestampObjectInspectorHive3();
private static final IcebergTimestampObjectInspectorHive3 INSTANCE =
new IcebergTimestampObjectInspectorHive3(TypeInfoFactory.timestampTypeInfo);

private static final IcebergTimestampObjectInspectorHive3 NANO_INSTANCE =
new IcebergTimestampObjectInspectorHive3(TypeInfoFactory.nanoTimestampTypeInfo);

public static IcebergTimestampObjectInspectorHive3 get() {
return INSTANCE;
}

private IcebergTimestampObjectInspectorHive3() {
super(TypeInfoFactory.timestampTypeInfo);
public static IcebergTimestampObjectInspectorHive3 get(int precision) {
if (precision == 9) {
return NANO_INSTANCE;
}
return INSTANCE;
}

private IcebergTimestampObjectInspectorHive3(PrimitiveTypeInfo typeInfo) {
super(typeInfo);
}

@Override
Expand Down
17 changes: 14 additions & 3 deletions ...e/iceberg/mr/hive/serde/objectinspector/IcebergTimestampWithZoneObjectInspectorHive3.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;

Expand All @@ -33,14 +34,24 @@ public class IcebergTimestampWithZoneObjectInspectorHive3 extends AbstractPrimit
implements TimestampLocalTZObjectInspector, WriteObjectInspector {

private static final IcebergTimestampWithZoneObjectInspectorHive3 INSTANCE =
new IcebergTimestampWithZoneObjectInspectorHive3();
new IcebergTimestampWithZoneObjectInspectorHive3(TypeInfoFactory.timestampLocalTZTypeInfo);

private static final IcebergTimestampWithZoneObjectInspectorHive3 NANO_INSTANCE =
new IcebergTimestampWithZoneObjectInspectorHive3(TypeInfoFactory.timestampNanoLocalTZTypeInfo);

public static IcebergTimestampWithZoneObjectInspectorHive3 get() {
return INSTANCE;
}

private IcebergTimestampWithZoneObjectInspectorHive3() {
super(TypeInfoFactory.timestampLocalTZTypeInfo);
public static IcebergTimestampWithZoneObjectInspectorHive3 get(int precision) {
if (precision == 9) {
return NANO_INSTANCE;
}
return INSTANCE;
}

private IcebergTimestampWithZoneObjectInspectorHive3(PrimitiveTypeInfo typeInfo) {
super(typeInfo);
}

@Override
Expand Down
40 changes: 40 additions & 0 deletions iceberg/iceberg-handler/src/test/queries/positive/timestamp_ns.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
-- Mask random uuid
--! qt:replace:/(\s+'uuid'=')\S+('\s*)/$1#Masked#$2/
--! qt:replace:/(\s+uuid\s+)\S+/$1#Masked#/
-- Mask random snapshot id
--! qt:replace:/('current-snapshot-id'=')\d+/$1#SnapshotId#/
-- Mask current-snapshot-timestamp-ms
--! qt:replace:/('current-snapshot-timestamp-ms'=')\d+/$1#Masked#/
-- Mask iceberg version
--! qt:replace:/("iceberg-version":")(\w+\s\w+\s\d+\.\d+\.\d+\s\(\w+\s\w+\))/$1#Masked#/
-- Mask added-files-size
--! qt:replace:/(\S\"added-files-size":")(\d+)(")/$1#Masked#$3/
-- Mask total-files-size
--! qt:replace:/(\S\"total-files-size":")(\d+)(")/$1#Masked#$3/

CREATE TABLE t (
ts_us timestamp,
ts_ns timestamp(9),
ts_tz_us timestamp with local time zone,
ts_tz_ns timestamp with local time zone(9)
)
STORED BY ICEBERG
TBLPROPERTIES ('format-version'='3');

INSERT INTO t VALUES (
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789'
);

SELECT ts_ns FROM t ORDER BY ts_ns;
SELECT ts_tz_ns FROM t ORDER BY ts_tz_ns;
SELECT CAST(ts_ns AS STRING) FROM t;
SELECT CAST(ts_tz_ns AS STRING) FROM t;

SELECT * FROM t;

CREATE TABLE tgt STORED BY ICEBERG TBLPROPERTIES ('format-version'='3') AS SELECT * FROM t;

SELECT * FROM tgt;
110 changes: 110 additions & 0 deletions iceberg/iceberg-handler/src/test/results/positive/timestamp_ns.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
PREHOOK: query: CREATE TABLE t (
ts_us timestamp,
ts_ns timestamp(9),
ts_tz_us timestamp with local time zone,
ts_tz_ns timestamp with local time zone(9)
)
STORED BY ICEBERG
TBLPROPERTIES ('format-version'='3')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@t
POSTHOOK: query: CREATE TABLE t (
ts_us timestamp,
ts_ns timestamp(9),
ts_tz_us timestamp with local time zone,
ts_tz_ns timestamp with local time zone(9)
)
STORED BY ICEBERG
TBLPROPERTIES ('format-version'='3')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@t
PREHOOK: query: INSERT INTO t VALUES (
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789'
)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@t
POSTHOOK: query: INSERT INTO t VALUES (
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789'
)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@t
PREHOOK: query: SELECT ts_ns FROM t ORDER BY ts_ns
PREHOOK: type: QUERY
PREHOOK: Input: default@t
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: SELECT ts_ns FROM t ORDER BY ts_ns
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
POSTHOOK: Output: hdfs://### HDFS PATH ###
2025-12-18 10:15:30.123456789
PREHOOK: query: SELECT ts_tz_ns FROM t ORDER BY ts_tz_ns
PREHOOK: type: QUERY
PREHOOK: Input: default@t
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: SELECT ts_tz_ns FROM t ORDER BY ts_tz_ns
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
POSTHOOK: Output: hdfs://### HDFS PATH ###
2025-12-18 10:15:30.123456789 US/Pacific
PREHOOK: query: SELECT CAST(ts_ns AS STRING) FROM t
PREHOOK: type: QUERY
PREHOOK: Input: default@t
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: SELECT CAST(ts_ns AS STRING) FROM t
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
POSTHOOK: Output: hdfs://### HDFS PATH ###
2025-12-18 10:15:30.123456789
PREHOOK: query: SELECT CAST(ts_tz_ns AS STRING) FROM t
PREHOOK: type: QUERY
PREHOOK: Input: default@t
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: SELECT CAST(ts_tz_ns AS STRING) FROM t
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
POSTHOOK: Output: hdfs://### HDFS PATH ###
2025-12-18 10:15:30.123456789 US/Pacific
PREHOOK: query: SELECT * FROM t
PREHOOK: type: QUERY
PREHOOK: Input: default@t
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: SELECT * FROM t
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
POSTHOOK: Output: hdfs://### HDFS PATH ###
2025-12-18 10:15:30.123456 2025-12-18 10:15:30.123456789 2025-12-18 10:15:30.123456 US/Pacific 2025-12-18 10:15:30.123456789 US/Pacific
PREHOOK: query: CREATE TABLE tgt STORED BY ICEBERG TBLPROPERTIES ('format-version'='3') AS SELECT * FROM t
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@t
PREHOOK: Output: database:default
PREHOOK: Output: default@tgt
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: CREATE TABLE tgt STORED BY ICEBERG TBLPROPERTIES ('format-version'='3') AS SELECT * FROM t
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@t
POSTHOOK: Output: database:default
POSTHOOK: Output: default@tgt
POSTHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: Lineage: tgt.ts_ns SIMPLE [(t)t.FieldSchema(name:ts_ns, type:timestamp(9), comment:null), ]
POSTHOOK: Lineage: tgt.ts_tz_ns SIMPLE [(t)t.FieldSchema(name:ts_tz_ns, type:timestamp with local time zone(9), comment:null), ]
POSTHOOK: Lineage: tgt.ts_tz_us SIMPLE [(t)t.FieldSchema(name:ts_tz_us, type:timestamp with local time zone, comment:null), ]
POSTHOOK: Lineage: tgt.ts_us SIMPLE [(t)t.FieldSchema(name:ts_us, type:timestamp, comment:null), ]
PREHOOK: query: SELECT * FROM tgt
PREHOOK: type: QUERY
PREHOOK: Input: default@tgt
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: SELECT * FROM tgt
POSTHOOK: type: QUERY
POSTHOOK: Input: default@tgt
POSTHOOK: Output: hdfs://### HDFS PATH ###
2025-12-18 10:15:30.123456 2025-12-18 10:15:30.123456789 2025-12-18 10:15:30.123456 US/Pacific 2025-12-18 10:15:30.123456789 US/Pacific
11 changes: 9 additions & 2 deletions parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2434,9 +2434,16 @@ primitiveType
| KW_DOUBLE KW_PRECISION? -> TOK_DOUBLE
| KW_DATE -> TOK_DATE
| KW_DATETIME -> TOK_DATETIME
| KW_TIMESTAMP -> TOK_TIMESTAMP
| KW_TIMESTAMPLOCALTZ -> TOK_TIMESTAMPLOCALTZ
| KW_TIMESTAMP KW_WITH KW_LOCAL KW_TIME KW_ZONE -> TOK_TIMESTAMPLOCALTZ
| KW_TIMESTAMP
(
KW_WITH KW_LOCAL KW_TIME KW_ZONE
(LPAREN p=Number RPAREN)?
-> ^(TOK_TIMESTAMPLOCALTZ $p?)
|
(LPAREN p=Number RPAREN)?
-> ^(TOK_TIMESTAMP $p?)
)
// Uncomment to allow intervals as table column types
//| KW_INTERVAL KW_YEAR KW_TO KW_MONTH -> TOK_INTERVAL_YEAR_MONTH
//| KW_INTERVAL KW_DAY KW_TO KW_SECOND -> TOK_INTERVAL_DAY_TIME
Expand Down
12 changes: 10 additions & 2 deletions ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.InputFormat;
Expand Down Expand Up @@ -1160,9 +1161,10 @@ public static void validateColumns(List<FieldSchema> columns, List<FieldSchema>
throw new HiveException("Duplicate column name " + colName
+ " in the table definition.");
}
if (!icebergTable && VARIANT_TYPE_NAME.equalsIgnoreCase(col.getType())) {
if (!icebergTable && isUnsupportedInNonIceberg(col.getType())) {
throw new HiveException(
"Column name " + colName + " cannot be of type 'variant' as it is not supported in non-Iceberg tables.");
"Column name " + colName + " cannot be of type '" + col.getType() + "' as it is not supported in "
+ "non-Iceberg tables.");
}
colNames.add(colName);
}
Expand Down Expand Up @@ -1392,4 +1394,10 @@ public List<VirtualColumn> getVirtualColumns() {

return virtualColumns;
}

private static boolean isUnsupportedInNonIceberg(String columnType) {
return VARIANT_TYPE_NAME.equalsIgnoreCase(columnType) ||
TypeInfoFactory.nanoTimestampTypeInfo.getQualifiedName().equalsIgnoreCase(columnType) ||
TypeInfoFactory.timestampNanoLocalTZTypeInfo.getQualifiedName().equalsIgnoreCase(columnType);
}
}
2 changes: 1 addition & 1 deletion ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,7 @@ public static TypeInfo convertPrimitiveType(RelDataType rType) {
} catch (HiveException e) {
throw new RuntimeException(e);
}
return TypeInfoFactory.getTimestampTZTypeInfo(conf.getLocalTimeZone());
return TypeInfoFactory.getTimestampTZTypeInfo(conf.getLocalTimeZone(), 6);
case INTERVAL_YEAR:
case INTERVAL_MONTH:
case INTERVAL_YEAR_MONTH:
Expand Down
13 changes: 11 additions & 2 deletions ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1067,14 +1067,23 @@ private static String getTypeName(ASTNode node) throws SemanticException {
typeName = varcharTypeInfo.getQualifiedName();
break;
case HiveParser.TOK_TIMESTAMPLOCALTZ:
TimestampLocalTZTypeInfo timestampLocalTZTypeInfo =
TypeInfoFactory.getTimestampTZTypeInfo(null);
int precision = 6;
if (node.getChildCount() > 0) {
precision = Integer.parseInt(node.getChild(0).getText());
}
TimestampLocalTZTypeInfo timestampLocalTZTypeInfo = TypeInfoFactory.getTimestampTZTypeInfo(null, precision);
typeName = timestampLocalTZTypeInfo.getQualifiedName();
break;
case HiveParser.TOK_DECIMAL:
DecimalTypeInfo decTypeInfo = ParseUtils.getDecimalTypeTypeInfo(node);
typeName = decTypeInfo.getQualifiedName();
break;
case HiveParser.TOK_TIMESTAMP:
int prec = 6;
if (node.getChildCount() > 0) {
prec = Integer.parseInt(node.getChild(0).getText());
}
return TypeInfoFactory.getTimestampTypeInfo(prec).getQualifiedName();
default:
typeName = TOKEN_TO_TYPE.get(token);
}
Expand Down
3 changes: 3 additions & 0 deletions ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,9 @@ public void configure(MapredContext context) {

@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
if (tsConvertors[0] instanceof ObjectInspectorConverters.IdentityConverter) {
return arguments[0].get();
}
PrimitiveObjectInspectorConverter.TimestampConverter ts =
(PrimitiveObjectInspectorConverter.TimestampConverter) tsConvertors[0];
ts.setIntToTimestampInSeconds(intToTimestampInSeconds);
Expand Down
2 changes: 2 additions & 0 deletions ql/src/test/queries/clientnegative/timestamp_ns_add_column.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
create table emp(id int);
alter table emp add columns (t timestamp(9));
1 change: 1 addition & 0 deletions ql/src/test/queries/clientnegative/timestamp_ns_non_iceberg.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
CREATE EXTERNAL TABLE nano_timestamp_basic (id INT, t timestamp with local time zone(9));
13 changes: 13 additions & 0 deletions ql/src/test/results/clientnegative/timestamp_ns_add_column.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
PREHOOK: query: create table emp(id int)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@emp
POSTHOOK: query: create table emp(id int)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@emp
PREHOOK: query: alter table emp add columns (t timestamp(9))
PREHOOK: type: ALTERTABLE_ADDCOLS
PREHOOK: Input: default@emp
PREHOOK: Output: default@emp
FAILED: Execution Error, return code 40000 from org.apache.hadoop.hive.ql.ddl.DDLTask. Column name t cannot be of type 'timestamp(9)' as it is not supported in non-Iceberg tables.
Loading