Skip to content

HIVE-24324: Remove deprecated API usage from Avro #1621

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sunchao merged 5 commits into from
Nov 3, 2020
Merged

HIVE-24324: Remove deprecated API usage from Avro #1621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
96880ef
HIVE_24324: Remove deprecated API usage from Avro
sunchao Oct 28, 2020
e9dcc63
Convert string to int for scale/precision
sunchao Oct 29, 2020
d90c9a5
Parse Integer or String from Avro schema
sunchao Oct 31, 2020
16ad500
Shouldn't accept string in AvroDeserializer
sunchao Oct 31, 2020
a6f4d86
Fix parsing precision & scale
sunchao Nov 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ private Object deserializePrimitive(Object datum, Schema fileSchema, Schema reco

int scale = 0;
try {
scale = fileSchema.getJsonProp(AvroSerDe.AVRO_PROP_SCALE).asInt();
scale = AvroSerdeUtils.getIntFromSchema(fileSchema, AvroSerDe.AVRO_PROP_SCALE);
} catch(Exception ex) {
throw new AvroSerdeException("Failed to obtain scale value from file schema: " + fileSchema, ex);
}
Expand All @@ -294,7 +294,7 @@ private Object deserializePrimitive(Object datum, Schema fileSchema, Schema reco

int maxLength = 0;
try {
maxLength = fileSchema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt();
maxLength = AvroSerdeUtils.getIntFromSchema(fileSchema, AvroSerDe.AVRO_PROP_MAX_LENGTH);
} catch (Exception ex) {
throw new AvroSerdeException("Failed to obtain maxLength value for char field from file schema: " + fileSchema, ex);
}
Expand All @@ -309,7 +309,7 @@ private Object deserializePrimitive(Object datum, Schema fileSchema, Schema reco

maxLength = 0;
try {
maxLength = fileSchema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt();
maxLength = AvroSerdeUtils.getIntFromSchema(fileSchema, AvroSerDe.AVRO_PROP_MAX_LENGTH);
} catch (Exception ex) {
throw new AvroSerdeException("Failed to obtain maxLength value for varchar field from file schema: " + fileSchema, ex);
}
Expand Down
12 changes: 12 additions & 0 deletions serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,18 @@ public static Schema getSchemaFor(URL url) {
}
}

public static int getIntFromSchema(Schema schema, String name) {
Object obj = schema.getObjectProp(name);
if (obj instanceof String) {
return Integer.parseInt((String) obj);
} else if (obj instanceof Integer) {
return (int) obj;
} else {
throw new IllegalArgumentException("Expect integer or string value from property " + name
+ " but found type " + obj.getClass().getName());
}
}

/**
* Called on specific alter table events, removes schema url and schema literal from given tblproperties
* After the change, HMS solely will be responsible for handling the schema
Expand Down
14 changes: 10 additions & 4 deletions serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,8 +136,14 @@ public static TypeInfo generateTypeInfo(Schema schema,
int precision = 0;
int scale = 0;
try {
precision = schema.getJsonProp(AvroSerDe.AVRO_PROP_PRECISION).getIntValue();
scale = schema.getJsonProp(AvroSerDe.AVRO_PROP_SCALE).getIntValue();
Object o = schema.getObjectProp(AvroSerDe.AVRO_PROP_PRECISION);
if (o instanceof Integer) {
precision = (int) o;
}
o = schema.getObjectProp(AvroSerDe.AVRO_PROP_SCALE);
if (o instanceof Integer) {
scale = (int) o;
}
} catch (Exception ex) {
throw new AvroSerdeException("Failed to obtain scale value from file schema: " + schema, ex);
}
Expand All @@ -155,7 +161,7 @@ public static TypeInfo generateTypeInfo(Schema schema,
AvroSerDe.CHAR_TYPE_NAME.equalsIgnoreCase(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) {
int maxLength = 0;
try {
maxLength = schema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt();
maxLength = AvroSerdeUtils.getIntFromSchema(schema, AvroSerDe.AVRO_PROP_MAX_LENGTH);
} catch (Exception ex) {
throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex);
}
Expand All @@ -166,7 +172,7 @@ public static TypeInfo generateTypeInfo(Schema schema,
.equalsIgnoreCase(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) {
int maxLength = 0;
try {
maxLength = schema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt();
maxLength = AvroSerdeUtils.getIntFromSchema(schema, AvroSerDe.AVRO_PROP_MAX_LENGTH);
} catch (Exception ex) {
throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex);
}
Expand Down