Skip to content

HIVE-29383: Iceberg: [V3] Add support for timestamp with nanosecond precession #6258

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ayushtkn merged 2 commits into from
Jan 22, 2026
Merged

HIVE-29383: Iceberg: [V3] Add support for timestamp with nanosecond precession #6258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bd4a90f
HIVE-29383: Iceberg: [V3] Add support for timestamp with nanosecond p…
ayushtkn Jan 6, 2026
a5d9659
Address Review Comments
ayushtkn Jan 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,7 @@ public enum ErrorMsg {
CATALOG_ALREADY_EXISTS(10444, "Catalog {0} already exists", true),
CATALOG_NOT_EXISTS(10445, "Catalog {0} does not exists:", true),
INVALID_SCHEDULED_QUERY(10446, "Scheduled query {0} does not exist", true),
UNSUPPORTED_TIMESTAMP_PRECISION(10447, "Unsupported value for precision: {0}", true),

//========================== 20000 range starts here ========================//

Expand Down
10 changes: 10 additions & 0 deletions iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TimestampTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.iceberg.Schema;
import org.apache.iceberg.expressions.Expressions;
Expand Down Expand Up @@ -130,6 +132,10 @@ Type convertType(TypeInfo typeInfo, String defaultValue) {
case STRING:
return Types.StringType.get();
case TIMESTAMP:
TimestampTypeInfo ts = (TimestampTypeInfo) typeInfo;
if (ts.getPrecision() == 9) {
return Types.TimestampNanoType.withoutZone();
}
return Types.TimestampType.withoutZone();
case DATE:
return Types.DateType.get();
Expand All @@ -141,6 +147,10 @@ Type convertType(TypeInfo typeInfo, String defaultValue) {
default:
// special case for Timestamp with Local TZ which is only available in Hive3
if ("TIMESTAMPLOCALTZ".equalsIgnoreCase(((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory().name())) {
TimestampLocalTZTypeInfo tz = (TimestampLocalTZTypeInfo) typeInfo;
if (tz.getPrecision() == 9) {
return Types.TimestampNanoType.withZone();
}
return Types.TimestampType.withZone();
}
throw new IllegalArgumentException("Unsupported Hive type (" +
Expand Down
20 changes: 19 additions & 1 deletion iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,9 @@ private static void getDefaultValDiff(Types.NestedField field, Map<String, Strin

if (field.type().isPrimitiveType()) {
Object expectedDefault = HiveSchemaUtil.getDefaultValue(defaultStr, field.type());
if (expectedDefault instanceof Literal<?>) {
expectedDefault = ((Literal<?>) expectedDefault).value();
}
if (!Objects.equals(expectedDefault, field.writeDefault())) {
difference.addDefaultChanged(field, expectedDefault);
}
Expand Down Expand Up @@ -379,6 +382,12 @@ public static String convertToTypeString(Type type) {
return "timestamp with local time zone";
}
return "timestamp";
case TIMESTAMP_NANO:
Types.TimestampNanoType timestampNanoType = (Types.TimestampNanoType) type;
if (timestampNanoType.shouldAdjustToUTC()) {
return "timestamp with local time zone(9)";
}
return "timestamp(9)";
case FIXED:
case BINARY:
return "binary";
Expand Down Expand Up @@ -506,6 +515,15 @@ public static Object convertToWriteType(Object value, Type type) {
DateTimeUtil.timestampFromMicros((Long) value);
}
break;
case TIMESTAMP_NANO:
// Convert nanoseconds since epoch (Long) to LocalDateTime
if (value instanceof Long) {
Types.TimestampNanoType timestampNanoType = (Types.TimestampNanoType) type;
return timestampNanoType.shouldAdjustToUTC() ?
DateTimeUtil.timestamptzFromNanos((Long) value) :
DateTimeUtil.timestampFromNanos((Long) value);
}
break;
default:
// For other types, no conversion needed
return value;
Expand Down Expand Up @@ -536,7 +554,7 @@ public static Object getDefaultValue(String defaultValue, Type type) {
}
return switch (type.typeId()) {
case DATE, TIME, TIMESTAMP, TIMESTAMP_NANO ->
Literal.of(stripQuotes(defaultValue)).to(type).value();
Literal.of(stripQuotes(defaultValue)).to(type);
default -> Conversions.fromPartitionString(type, stripQuotes(defaultValue));
};
}
Expand Down
5 changes: 5 additions & 0 deletions ...rc/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,11 @@ public ObjectInspector primitive(Type.PrimitiveType primitiveType) {
case TIMESTAMP:
boolean adjustToUTC = ((Types.TimestampType) primitiveType).shouldAdjustToUTC();
return adjustToUTC ? TIMESTAMP_INSPECTOR_WITH_TZ : TIMESTAMP_INSPECTOR;
case TIMESTAMP_NANO:
boolean adjustUTC = ((Types.TimestampNanoType) primitiveType).shouldAdjustToUTC();
return adjustUTC ?
IcebergTimestampWithZoneObjectInspectorHive3.get(9) :
IcebergTimestampObjectInspectorHive3.get(9);
case TIME:
return IcebergTimeObjectInspector.get();
default:
Expand Down
18 changes: 15 additions & 3 deletions ...rg/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspectorHive3.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,20 +26,32 @@
import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;


public class IcebergTimestampObjectInspectorHive3 extends AbstractPrimitiveJavaObjectInspector
implements TimestampObjectInspector, WriteObjectInspector {

private static final IcebergTimestampObjectInspectorHive3 INSTANCE = new IcebergTimestampObjectInspectorHive3();
private static final IcebergTimestampObjectInspectorHive3 INSTANCE =
new IcebergTimestampObjectInspectorHive3(TypeInfoFactory.timestampTypeInfo);

private static final IcebergTimestampObjectInspectorHive3 NANO_INSTANCE =
new IcebergTimestampObjectInspectorHive3(TypeInfoFactory.nanoTimestampTypeInfo);

public static IcebergTimestampObjectInspectorHive3 get() {
return INSTANCE;
}

private IcebergTimestampObjectInspectorHive3() {
super(TypeInfoFactory.timestampTypeInfo);
public static IcebergTimestampObjectInspectorHive3 get(int precision) {
if (precision == 9) {
return NANO_INSTANCE;
}
return INSTANCE;
}

private IcebergTimestampObjectInspectorHive3(PrimitiveTypeInfo typeInfo) {
super(typeInfo);
}

@Override
Expand Down
17 changes: 14 additions & 3 deletions ...e/iceberg/mr/hive/serde/objectinspector/IcebergTimestampWithZoneObjectInspectorHive3.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;

Expand All @@ -33,14 +34,24 @@ public class IcebergTimestampWithZoneObjectInspectorHive3 extends AbstractPrimit
implements TimestampLocalTZObjectInspector, WriteObjectInspector {

private static final IcebergTimestampWithZoneObjectInspectorHive3 INSTANCE =
new IcebergTimestampWithZoneObjectInspectorHive3();
new IcebergTimestampWithZoneObjectInspectorHive3(TypeInfoFactory.timestampLocalTZTypeInfo);

private static final IcebergTimestampWithZoneObjectInspectorHive3 NANO_INSTANCE =
new IcebergTimestampWithZoneObjectInspectorHive3(TypeInfoFactory.timestampNanoLocalTZTypeInfo);

public static IcebergTimestampWithZoneObjectInspectorHive3 get() {
return INSTANCE;
}

private IcebergTimestampWithZoneObjectInspectorHive3() {
super(TypeInfoFactory.timestampLocalTZTypeInfo);
public static IcebergTimestampWithZoneObjectInspectorHive3 get(int precision) {
if (precision == 9) {
return NANO_INSTANCE;
}
return INSTANCE;
}

private IcebergTimestampWithZoneObjectInspectorHive3(PrimitiveTypeInfo typeInfo) {
super(typeInfo);
}

@Override
Expand Down
6 changes: 6 additions & 0 deletions ...andler/src/main/java/org/apache/iceberg/mr/mapreduce/HiveIdentityPartitionConverters.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ public static Object convertConstant(Type type, Object value) {
} else {
return new Timestamp(DateTimeUtil.timestampFromMicros((Long) value));
}
case TIMESTAMP_NANO:
if (((Types.TimestampNanoType) type).shouldAdjustToUTC()) {
return DateTimeUtil.timestamptzFromNanos((Long) value).toOffsetTime();
} else {
return new Timestamp(DateTimeUtil.timestampFromNanos((Long) value));
}
case DECIMAL:
if (value.getClass().isAssignableFrom(BigDecimal.class)) {
return HiveDecimal.create((BigDecimal) value);
Expand Down
40 changes: 40 additions & 0 deletions iceberg/iceberg-handler/src/test/queries/positive/timestamp_ns.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
-- Mask random uuid
--! qt:replace:/(\s+'uuid'=')\S+('\s*)/$1#Masked#$2/
--! qt:replace:/(\s+uuid\s+)\S+/$1#Masked#/
-- Mask random snapshot id
--! qt:replace:/('current-snapshot-id'=')\d+/$1#SnapshotId#/
-- Mask current-snapshot-timestamp-ms
--! qt:replace:/('current-snapshot-timestamp-ms'=')\d+/$1#Masked#/
-- Mask iceberg version
--! qt:replace:/("iceberg-version":")(\w+\s\w+\s\d+\.\d+\.\d+\s\(\w+\s\w+\))/$1#Masked#/
-- Mask added-files-size
--! qt:replace:/(\S\"added-files-size":")(\d+)(")/$1#Masked#$3/
-- Mask total-files-size
--! qt:replace:/(\S\"total-files-size":")(\d+)(")/$1#Masked#$3/

CREATE TABLE t (
ts_us timestamp,
ts_ns timestamp(9),
ts_tz_us timestamp with local time zone,
ts_tz_ns timestamp with local time zone(9)
)
STORED BY ICEBERG
TBLPROPERTIES ('format-version'='3');

INSERT INTO t VALUES (
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789'
);

SELECT ts_ns FROM t ORDER BY ts_ns;
SELECT ts_tz_ns FROM t ORDER BY ts_tz_ns;
SELECT CAST(ts_ns AS STRING) FROM t;
SELECT CAST(ts_tz_ns AS STRING) FROM t;

SELECT * FROM t;

CREATE TABLE tgt STORED BY ICEBERG TBLPROPERTIES ('format-version'='3') AS SELECT * FROM t;

SELECT * FROM tgt;
110 changes: 110 additions & 0 deletions iceberg/iceberg-handler/src/test/results/positive/timestamp_ns.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
PREHOOK: query: CREATE TABLE t (
ts_us timestamp,
ts_ns timestamp(9),
ts_tz_us timestamp with local time zone,
ts_tz_ns timestamp with local time zone(9)
)
STORED BY ICEBERG
TBLPROPERTIES ('format-version'='3')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@t
POSTHOOK: query: CREATE TABLE t (
ts_us timestamp,
ts_ns timestamp(9),
ts_tz_us timestamp with local time zone,
ts_tz_ns timestamp with local time zone(9)
)
STORED BY ICEBERG
TBLPROPERTIES ('format-version'='3')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@t
PREHOOK: query: INSERT INTO t VALUES (
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789'
)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@t
POSTHOOK: query: INSERT INTO t VALUES (
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789',
'2025-12-18 10:15:30.123456789'
)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@t
PREHOOK: query: SELECT ts_ns FROM t ORDER BY ts_ns
PREHOOK: type: QUERY
PREHOOK: Input: default@t
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: SELECT ts_ns FROM t ORDER BY ts_ns
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
POSTHOOK: Output: hdfs://### HDFS PATH ###
2025-12-18 10:15:30.123456789
PREHOOK: query: SELECT ts_tz_ns FROM t ORDER BY ts_tz_ns
PREHOOK: type: QUERY
PREHOOK: Input: default@t
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: SELECT ts_tz_ns FROM t ORDER BY ts_tz_ns
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
POSTHOOK: Output: hdfs://### HDFS PATH ###
2025-12-18 10:15:30.123456789 US/Pacific
PREHOOK: query: SELECT CAST(ts_ns AS STRING) FROM t
PREHOOK: type: QUERY
PREHOOK: Input: default@t
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: SELECT CAST(ts_ns AS STRING) FROM t
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
POSTHOOK: Output: hdfs://### HDFS PATH ###
2025-12-18 10:15:30.123456789
PREHOOK: query: SELECT CAST(ts_tz_ns AS STRING) FROM t
PREHOOK: type: QUERY
PREHOOK: Input: default@t
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: SELECT CAST(ts_tz_ns AS STRING) FROM t
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
POSTHOOK: Output: hdfs://### HDFS PATH ###
2025-12-18 10:15:30.123456789 US/Pacific
PREHOOK: query: SELECT * FROM t
PREHOOK: type: QUERY
PREHOOK: Input: default@t
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: SELECT * FROM t
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
POSTHOOK: Output: hdfs://### HDFS PATH ###
2025-12-18 10:15:30.123456 2025-12-18 10:15:30.123456789 2025-12-18 10:15:30.123456 US/Pacific 2025-12-18 10:15:30.123456789 US/Pacific
PREHOOK: query: CREATE TABLE tgt STORED BY ICEBERG TBLPROPERTIES ('format-version'='3') AS SELECT * FROM t
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@t
PREHOOK: Output: database:default
PREHOOK: Output: default@tgt
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: CREATE TABLE tgt STORED BY ICEBERG TBLPROPERTIES ('format-version'='3') AS SELECT * FROM t
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@t
POSTHOOK: Output: database:default
POSTHOOK: Output: default@tgt
POSTHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: Lineage: tgt.ts_ns SIMPLE [(t)t.FieldSchema(name:ts_ns, type:timestamp(9), comment:null), ]
POSTHOOK: Lineage: tgt.ts_tz_ns SIMPLE [(t)t.FieldSchema(name:ts_tz_ns, type:timestamp with local time zone(9), comment:null), ]
POSTHOOK: Lineage: tgt.ts_tz_us SIMPLE [(t)t.FieldSchema(name:ts_tz_us, type:timestamp with local time zone, comment:null), ]
POSTHOOK: Lineage: tgt.ts_us SIMPLE [(t)t.FieldSchema(name:ts_us, type:timestamp, comment:null), ]
PREHOOK: query: SELECT * FROM tgt
PREHOOK: type: QUERY
PREHOOK: Input: default@tgt
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: SELECT * FROM tgt
POSTHOOK: type: QUERY
POSTHOOK: Input: default@tgt
POSTHOOK: Output: hdfs://### HDFS PATH ###
2025-12-18 10:15:30.123456 2025-12-18 10:15:30.123456789 2025-12-18 10:15:30.123456 US/Pacific 2025-12-18 10:15:30.123456789 US/Pacific
11 changes: 9 additions & 2 deletions parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2434,9 +2434,16 @@ primitiveType
| KW_DOUBLE KW_PRECISION? -> TOK_DOUBLE
| KW_DATE -> TOK_DATE
| KW_DATETIME -> TOK_DATETIME
| KW_TIMESTAMP -> TOK_TIMESTAMP
| KW_TIMESTAMPLOCALTZ -> TOK_TIMESTAMPLOCALTZ
| KW_TIMESTAMP KW_WITH KW_LOCAL KW_TIME KW_ZONE -> TOK_TIMESTAMPLOCALTZ
| KW_TIMESTAMP
(
KW_WITH KW_LOCAL KW_TIME KW_ZONE
(LPAREN p=Number RPAREN)?
-> ^(TOK_TIMESTAMPLOCALTZ $p?)
|
(LPAREN p=Number RPAREN)?
-> ^(TOK_TIMESTAMP $p?)
)
// Uncomment to allow intervals as table column types
//| KW_INTERVAL KW_YEAR KW_TO KW_MONTH -> TOK_INTERVAL_YEAR_MONTH
//| KW_INTERVAL KW_DAY KW_TO KW_SECOND -> TOK_INTERVAL_DAY_TIME
Expand Down
12 changes: 10 additions & 2 deletions ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.InputFormat;
Expand Down Expand Up @@ -1160,9 +1161,10 @@ public static void validateColumns(List<FieldSchema> columns, List<FieldSchema>
throw new HiveException("Duplicate column name " + colName
+ " in the table definition.");
}
if (!icebergTable && VARIANT_TYPE_NAME.equalsIgnoreCase(col.getType())) {
if (!icebergTable && isUnsupportedInNonIceberg(col.getType())) {
throw new HiveException(
"Column name " + colName + " cannot be of type 'variant' as it is not supported in non-Iceberg tables.");
"Column name " + colName + " cannot be of type '" + col.getType() + "' as it is not supported in "
+ "non-Iceberg tables.");
}
colNames.add(colName);
}
Expand Down Expand Up @@ -1392,4 +1394,10 @@ public List<VirtualColumn> getVirtualColumns() {

return virtualColumns;
}

private static boolean isUnsupportedInNonIceberg(String columnType) {
return VARIANT_TYPE_NAME.equalsIgnoreCase(columnType) ||
TypeInfoFactory.nanoTimestampTypeInfo.getQualifiedName().equalsIgnoreCase(columnType) ||
TypeInfoFactory.timestampNanoLocalTZTypeInfo.getQualifiedName().equalsIgnoreCase(columnType);
}
}
2 changes: 1 addition & 1 deletion ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,7 @@ public static TypeInfo convertPrimitiveType(RelDataType rType) {
} catch (HiveException e) {
throw new RuntimeException(e);
}
return TypeInfoFactory.getTimestampTZTypeInfo(conf.getLocalTimeZone());
return TypeInfoFactory.getTimestampTZTypeInfo(conf.getLocalTimeZone(), 6);
case INTERVAL_YEAR:
case INTERVAL_MONTH:
case INTERVAL_YEAR_MONTH:
Expand Down
Loading