diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala index e55c25c4b0c54..c0e3ee582c809 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala @@ -143,6 +143,15 @@ object DeserializerBuildHelper { returnNullable = false) } + def createDeserializerForJavaByteBuffer(path: Expression, returnNullable: Boolean): Expression = { + StaticInvoke( + classOf[java.nio.ByteBuffer], + ObjectType(classOf[java.nio.ByteBuffer]), + "wrap", + path :: Nil, + returnNullable = false) + } + /** * When we build the `deserializer` for an encoder, we set up a lot of "unresolved" stuff * and lost the required data type, which may lead to runtime error if the real type doesn't diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala index c5be3efc6371e..a6c8d0c2a5162 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala @@ -17,8 +17,8 @@ package org.apache.spark.sql.catalyst -import java.beans.{Introspector, PropertyDescriptor} import java.lang.{Iterable => JIterable} +import java.lang.reflect.Method import java.lang.reflect.Type import java.util.{Iterator => JIterator, List => JList, Map => JMap} @@ -106,6 +106,7 @@ object JavaTypeInference { case c: Class[_] if c == classOf[java.sql.Date] => (DateType, true) case c: Class[_] if c == classOf[java.time.Instant] => (TimestampType, true) case c: Class[_] if c == classOf[java.sql.Timestamp] => (TimestampType, true) + case c: Class[_] if c == classOf[java.nio.ByteBuffer] => (BinaryType, true) case _ if typeToken.isArray => val (dataType, nullable) = inferDataType(typeToken.getComponentType, seenTypeSet) @@ -131,28 +132,44 @@ object JavaTypeInference { s"of class $other") } - // TODO: we should only collect properties that have getter and setter. However, some tests - // pass in scala case class as java bean class which doesn't have getter and setter. - val properties = getJavaBeanReadableProperties(other) - val fields = properties.map { property => - val returnType = typeToken.method(property.getReadMethod).getReturnType - val (dataType, nullable) = inferDataType(returnType, seenTypeSet + other) - new StructField(property.getName, dataType, nullable) + val fields = getObjectProperties(other).map { + case (propertyName, getterMethod, setterMethod) => + val (dataType, nullable) = inferDataType( + TypeToken.of(getterMethod.getGenericReturnType), + seenTypeSet + other) + new StructField(propertyName, dataType, nullable) } (new StructType(fields), true) } } - def getJavaBeanReadableProperties(beanClass: Class[_]): Array[PropertyDescriptor] = { - val beanInfo = Introspector.getBeanInfo(beanClass) - beanInfo.getPropertyDescriptors.filterNot(_.getName == "class") - .filterNot(_.getName == "declaringClass") - .filter(_.getReadMethod != null) - } - - private def getJavaBeanReadableAndWritableProperties( - beanClass: Class[_]): Array[PropertyDescriptor] = { - getJavaBeanReadableProperties(beanClass).filter(_.getWriteMethod != null) + /** + * Returns: Array[(porpertyName, getterName, setterName, propertyType)] + * + * Properties of the object are defined by a getter 'propertyType [get]PropertyName()' and a + * setter 'void [set]PropertyName(propertyType value)' functions; where [get]PropertyName is + * the name of the getter function, and [set]PropertyName is the name of the setter function. + */ + def getObjectProperties(beanClass: Class[_]): Array[(String, Method, Method)] = { + def propertyName(getterName: String, setterName: String): String = { + if (getterName == setterName) { + getterName + } else { + if (getterName.startsWith("get") && setterName.startsWith("set") && + getterName.substring(3) == setterName.substring(3)) { + getterName.substring(3) + } else { + null + } + } + } + for { + a <- beanClass.getMethods.filter(method => method.getParameterCount == 0) + b <- beanClass.getMethods.filter(method => method.getReturnType == Void.TYPE && + method.getParameterCount == 1) + if (propertyName(a.getName, b.getName) != null && + a.getReturnType == b.getParameterTypes.head) + } yield (propertyName(a.getName, b.getName), a, b) } private def elementType(typeToken: TypeToken[_]): TypeToken[_] = { @@ -317,24 +334,26 @@ object JavaTypeInference { keyData :: valueData :: Nil, returnNullable = false) + case other if other == classOf[java.nio.ByteBuffer] => + createDeserializerForJavaByteBuffer(path, returnNullable = false) + case other if other.isEnum => createDeserializerForTypesSupportValueOf( createDeserializerForString(path, returnNullable = false), other) case other => - val properties = getJavaBeanReadableAndWritableProperties(other) - val setters = properties.map { p => - val fieldName = p.getName - val fieldType = typeToken.method(p.getReadMethod).getReturnType - val (dataType, nullable) = inferDataType(fieldType) - val newTypePath = walkedTypePath.recordField(fieldType.getType.getTypeName, fieldName) - val setter = expressionWithNullSafety( - deserializerFor(fieldType, addToPath(path, fieldName, dataType, newTypePath), - newTypePath), - nullable = nullable, - newTypePath) - p.getWriteMethod.getName -> setter + val setters = getObjectProperties(other).map { + case (fieldName, getterMethod, setterMethod) => + val fieldType = TypeToken.of(getterMethod.getGenericReturnType) + val (dataType, nullable) = inferDataType(fieldType) + val newTypePath = walkedTypePath.recordField(fieldType.getType.getTypeName, fieldName) + val setter = expressionWithNullSafety( + deserializerFor(fieldType, addToPath(path, fieldName, dataType, newTypePath), + newTypePath), + nullable = nullable, + newTypePath) + setterMethod.getName -> setter }.toMap val newInstance = NewInstance(other, Nil, ObjectType(other), propagateNull = false) @@ -401,6 +420,9 @@ object JavaTypeInference { case c if c == classOf[java.lang.Float] => createSerializerForFloat(inputObject) case c if c == classOf[java.lang.Double] => createSerializerForDouble(inputObject) + case c if c == classOf[java.nio.ByteBuffer] => + createSerializerForJavaByteBuffer(inputObject) + case _ if typeToken.isArray => toCatalystArray(inputObject, typeToken.getComponentType) @@ -427,13 +449,12 @@ object JavaTypeInference { Invoke(inputObject, "name", ObjectType(classOf[String]), returnNullable = false)) case other => - val properties = getJavaBeanReadableAndWritableProperties(other) - val fields = properties.map { p => - val fieldName = p.getName - val fieldType = typeToken.method(p.getReadMethod).getReturnType + val fields = getObjectProperties(other).map { + case (fieldName, getterMethod, setterMethod) => + val fieldType = TypeToken.of(getterMethod.getGenericReturnType) val fieldValue = Invoke( inputObject, - p.getReadMethod.getName, + getterMethod.getName, inferExternalType(fieldType.getRawType)) (fieldName, serializerFor(fieldValue, fieldType)) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala index e035c4be97240..3fc565f4a882e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala @@ -124,6 +124,10 @@ object SerializerBuildHelper { createSerializerForJavaBigInteger(inputObject) } + def createSerializerForJavaByteBuffer(inputObject: Expression): Expression = { + Invoke(inputObject, "array", BinaryType) + } + def createSerializerForPrimitiveArray( inputObject: Expression, dataType: DataType): Expression = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/AvroExample1.java b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/AvroExample1.java new file mode 100644 index 0000000000000..d42992b54233b --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/AvroExample1.java @@ -0,0 +1,977 @@ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package org.apache.spark.sql.catalyst.encoders; + +import org.apache.avro.specific.SpecificData; +import org.apache.avro.message.BinaryMessageEncoder; +import org.apache.avro.message.BinaryMessageDecoder; +import org.apache.avro.message.SchemaStore; + +@SuppressWarnings("all") +@org.apache.avro.specific.AvroGenerated +public class AvroExample1 extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { + private static final long serialVersionUID = -6893577045970753467L; + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"AvroExample1\",\"namespace\":\"org.apache.spark.sql.catalyst.encoders\",\"fields\":[{\"name\":\"mymoney\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"Money\",\"fields\":[{\"name\":\"amount\",\"type\":\"float\",\"default\":0},{\"name\":\"currency\",\"type\":{\"type\":\"enum\",\"name\":\"Currency\",\"symbols\":[\"EUR\",\"USD\",\"BRL\"]},\"default\":\"EUR\"}]}],\"default\":null},{\"name\":\"myfloat\",\"type\":\"float\"},{\"name\":\"mylong\",\"type\":\"long\"},{\"name\":\"myint\",\"type\":\"int\"},{\"name\":\"mydouble\",\"type\":\"double\"},{\"name\":\"myboolean\",\"type\":\"boolean\"},{\"name\":\"mystring\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"mybytes\",\"type\":\"bytes\"},{\"name\":\"myfixed\",\"type\":{\"type\":\"fixed\",\"name\":\"Magic\",\"size\":4}},{\"name\":\"myarray\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}},{\"name\":\"mymap\",\"type\":{\"type\":\"map\",\"values\":\"int\",\"avro.java.string\":\"String\"}}]}"); + public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } + + private static SpecificData MODEL$ = new SpecificData(); + + private static final BinaryMessageEncoder ENCODER = + new BinaryMessageEncoder(MODEL$, SCHEMA$); + + private static final BinaryMessageDecoder DECODER = + new BinaryMessageDecoder(MODEL$, SCHEMA$); + + /** + * Return the BinaryMessageDecoder instance used by this class. + */ + public static BinaryMessageDecoder getDecoder() { + return DECODER; + } + + /** + * Create a new BinaryMessageDecoder instance for this class that uses the specified {@link SchemaStore}. + * @param resolver a {@link SchemaStore} used to find schemas by fingerprint + */ + public static BinaryMessageDecoder createDecoder(SchemaStore resolver) { + return new BinaryMessageDecoder(MODEL$, SCHEMA$, resolver); + } + + /** Serializes this AvroExample1 to a ByteBuffer. */ + public java.nio.ByteBuffer toByteBuffer() throws java.io.IOException { + return ENCODER.encode(this); + } + + /** Deserializes a AvroExample1 from a ByteBuffer. */ + public static AvroExample1 fromByteBuffer( + java.nio.ByteBuffer b) throws java.io.IOException { + return DECODER.decode(b); + } + + @Deprecated public org.apache.spark.sql.catalyst.encoders.Money mymoney; + @Deprecated public float myfloat; + @Deprecated public long mylong; + @Deprecated public int myint; + @Deprecated public double mydouble; + @Deprecated public boolean myboolean; + @Deprecated public java.lang.String mystring; + @Deprecated public java.nio.ByteBuffer mybytes; + @Deprecated public org.apache.spark.sql.catalyst.encoders.Magic myfixed; + @Deprecated public java.util.List myarray; + @Deprecated public java.util.Map mymap; + + /** + * Default constructor. Note that this does not initialize fields + * to their default values from the schema. If that is desired then + * one should use newBuilder(). + */ + public AvroExample1() {} + + /** + * All-args constructor. + * @param mymoney The new value for mymoney + * @param myfloat The new value for myfloat + * @param mylong The new value for mylong + * @param myint The new value for myint + * @param mydouble The new value for mydouble + * @param myboolean The new value for myboolean + * @param mystring The new value for mystring + * @param mybytes The new value for mybytes + * @param myfixed The new value for myfixed + * @param myarray The new value for myarray + * @param mymap The new value for mymap + */ + public AvroExample1(org.apache.spark.sql.catalyst.encoders.Money mymoney, java.lang.Float myfloat, java.lang.Long mylong, java.lang.Integer myint, java.lang.Double mydouble, java.lang.Boolean myboolean, java.lang.String mystring, java.nio.ByteBuffer mybytes, org.apache.spark.sql.catalyst.encoders.Magic myfixed, java.util.List myarray, java.util.Map mymap) { + this.mymoney = mymoney; + this.myfloat = myfloat; + this.mylong = mylong; + this.myint = myint; + this.mydouble = mydouble; + this.myboolean = myboolean; + this.mystring = mystring; + this.mybytes = mybytes; + this.myfixed = myfixed; + this.myarray = myarray; + this.mymap = mymap; + } + + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public java.lang.Object get(int field$) { + switch (field$) { + case 0: return mymoney; + case 1: return myfloat; + case 2: return mylong; + case 3: return myint; + case 4: return mydouble; + case 5: return myboolean; + case 6: return mystring; + case 7: return mybytes; + case 8: return myfixed; + case 9: return myarray; + case 10: return mymap; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value="unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: mymoney = (org.apache.spark.sql.catalyst.encoders.Money)value$; break; + case 1: myfloat = (java.lang.Float)value$; break; + case 2: mylong = (java.lang.Long)value$; break; + case 3: myint = (java.lang.Integer)value$; break; + case 4: mydouble = (java.lang.Double)value$; break; + case 5: myboolean = (java.lang.Boolean)value$; break; + case 6: mystring = (java.lang.String)value$; break; + case 7: mybytes = (java.nio.ByteBuffer)value$; break; + case 8: myfixed = (org.apache.spark.sql.catalyst.encoders.Magic)value$; break; + case 9: myarray = (java.util.List)value$; break; + case 10: mymap = (java.util.Map)value$; break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + /** + * Gets the value of the 'mymoney' field. + * @return The value of the 'mymoney' field. + */ + public org.apache.spark.sql.catalyst.encoders.Money getMymoney() { + return mymoney; + } + + /** + * Sets the value of the 'mymoney' field. + * @param value the value to set. + */ + public void setMymoney(org.apache.spark.sql.catalyst.encoders.Money value) { + this.mymoney = value; + } + + /** + * Gets the value of the 'myfloat' field. + * @return The value of the 'myfloat' field. + */ + public java.lang.Float getMyfloat() { + return myfloat; + } + + /** + * Sets the value of the 'myfloat' field. + * @param value the value to set. + */ + public void setMyfloat(java.lang.Float value) { + this.myfloat = value; + } + + /** + * Gets the value of the 'mylong' field. + * @return The value of the 'mylong' field. + */ + public java.lang.Long getMylong() { + return mylong; + } + + /** + * Sets the value of the 'mylong' field. + * @param value the value to set. + */ + public void setMylong(java.lang.Long value) { + this.mylong = value; + } + + /** + * Gets the value of the 'myint' field. + * @return The value of the 'myint' field. + */ + public java.lang.Integer getMyint() { + return myint; + } + + /** + * Sets the value of the 'myint' field. + * @param value the value to set. + */ + public void setMyint(java.lang.Integer value) { + this.myint = value; + } + + /** + * Gets the value of the 'mydouble' field. + * @return The value of the 'mydouble' field. + */ + public java.lang.Double getMydouble() { + return mydouble; + } + + /** + * Sets the value of the 'mydouble' field. + * @param value the value to set. + */ + public void setMydouble(java.lang.Double value) { + this.mydouble = value; + } + + /** + * Gets the value of the 'myboolean' field. + * @return The value of the 'myboolean' field. + */ + public java.lang.Boolean getMyboolean() { + return myboolean; + } + + /** + * Sets the value of the 'myboolean' field. + * @param value the value to set. + */ + public void setMyboolean(java.lang.Boolean value) { + this.myboolean = value; + } + + /** + * Gets the value of the 'mystring' field. + * @return The value of the 'mystring' field. + */ + public java.lang.String getMystring() { + return mystring; + } + + /** + * Sets the value of the 'mystring' field. + * @param value the value to set. + */ + public void setMystring(java.lang.String value) { + this.mystring = value; + } + + /** + * Gets the value of the 'mybytes' field. + * @return The value of the 'mybytes' field. + */ + public java.nio.ByteBuffer getMybytes() { + return mybytes; + } + + /** + * Sets the value of the 'mybytes' field. + * @param value the value to set. + */ + public void setMybytes(java.nio.ByteBuffer value) { + this.mybytes = value; + } + + /** + * Gets the value of the 'myfixed' field. + * @return The value of the 'myfixed' field. + */ + public org.apache.spark.sql.catalyst.encoders.Magic getMyfixed() { + return myfixed; + } + + /** + * Sets the value of the 'myfixed' field. + * @param value the value to set. + */ + public void setMyfixed(org.apache.spark.sql.catalyst.encoders.Magic value) { + this.myfixed = value; + } + + /** + * Gets the value of the 'myarray' field. + * @return The value of the 'myarray' field. + */ + public java.util.List getMyarray() { + return myarray; + } + + /** + * Sets the value of the 'myarray' field. + * @param value the value to set. + */ + public void setMyarray(java.util.List value) { + this.myarray = value; + } + + /** + * Gets the value of the 'mymap' field. + * @return The value of the 'mymap' field. + */ + public java.util.Map getMymap() { + return mymap; + } + + /** + * Sets the value of the 'mymap' field. + * @param value the value to set. + */ + public void setMymap(java.util.Map value) { + this.mymap = value; + } + + /** + * Creates a new AvroExample1 RecordBuilder. + * @return A new AvroExample1 RecordBuilder + */ + public static org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder newBuilder() { + return new org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder(); + } + + /** + * Creates a new AvroExample1 RecordBuilder by copying an existing Builder. + * @param other The existing builder to copy. + * @return A new AvroExample1 RecordBuilder + */ + public static org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder newBuilder(org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder other) { + return new org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder(other); + } + + /** + * Creates a new AvroExample1 RecordBuilder by copying an existing AvroExample1 instance. + * @param other The existing instance to copy. + * @return A new AvroExample1 RecordBuilder + */ + public static org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder newBuilder(org.apache.spark.sql.catalyst.encoders.AvroExample1 other) { + return new org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder(other); + } + + /** + * RecordBuilder for AvroExample1 instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private org.apache.spark.sql.catalyst.encoders.Money mymoney; + private org.apache.spark.sql.catalyst.encoders.Money.Builder mymoneyBuilder; + private float myfloat; + private long mylong; + private int myint; + private double mydouble; + private boolean myboolean; + private java.lang.String mystring; + private java.nio.ByteBuffer mybytes; + private org.apache.spark.sql.catalyst.encoders.Magic myfixed; + private java.util.List myarray; + private java.util.Map mymap; + + /** Creates a new Builder */ + private Builder() { + super(SCHEMA$); + } + + /** + * Creates a Builder by copying an existing Builder. + * @param other The existing Builder to copy. + */ + private Builder(org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder other) { + super(other); + if (isValidValue(fields()[0], other.mymoney)) { + this.mymoney = data().deepCopy(fields()[0].schema(), other.mymoney); + fieldSetFlags()[0] = true; + } + if (other.hasMymoneyBuilder()) { + this.mymoneyBuilder = org.apache.spark.sql.catalyst.encoders.Money.newBuilder(other.getMymoneyBuilder()); + } + if (isValidValue(fields()[1], other.myfloat)) { + this.myfloat = data().deepCopy(fields()[1].schema(), other.myfloat); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.mylong)) { + this.mylong = data().deepCopy(fields()[2].schema(), other.mylong); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.myint)) { + this.myint = data().deepCopy(fields()[3].schema(), other.myint); + fieldSetFlags()[3] = true; + } + if (isValidValue(fields()[4], other.mydouble)) { + this.mydouble = data().deepCopy(fields()[4].schema(), other.mydouble); + fieldSetFlags()[4] = true; + } + if (isValidValue(fields()[5], other.myboolean)) { + this.myboolean = data().deepCopy(fields()[5].schema(), other.myboolean); + fieldSetFlags()[5] = true; + } + if (isValidValue(fields()[6], other.mystring)) { + this.mystring = data().deepCopy(fields()[6].schema(), other.mystring); + fieldSetFlags()[6] = true; + } + if (isValidValue(fields()[7], other.mybytes)) { + this.mybytes = data().deepCopy(fields()[7].schema(), other.mybytes); + fieldSetFlags()[7] = true; + } + if (isValidValue(fields()[8], other.myfixed)) { + this.myfixed = data().deepCopy(fields()[8].schema(), other.myfixed); + fieldSetFlags()[8] = true; + } + if (isValidValue(fields()[9], other.myarray)) { + this.myarray = data().deepCopy(fields()[9].schema(), other.myarray); + fieldSetFlags()[9] = true; + } + if (isValidValue(fields()[10], other.mymap)) { + this.mymap = data().deepCopy(fields()[10].schema(), other.mymap); + fieldSetFlags()[10] = true; + } + } + + /** + * Creates a Builder by copying an existing AvroExample1 instance + * @param other The existing instance to copy. + */ + private Builder(org.apache.spark.sql.catalyst.encoders.AvroExample1 other) { + super(SCHEMA$); + if (isValidValue(fields()[0], other.mymoney)) { + this.mymoney = data().deepCopy(fields()[0].schema(), other.mymoney); + fieldSetFlags()[0] = true; + } + this.mymoneyBuilder = null; + if (isValidValue(fields()[1], other.myfloat)) { + this.myfloat = data().deepCopy(fields()[1].schema(), other.myfloat); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.mylong)) { + this.mylong = data().deepCopy(fields()[2].schema(), other.mylong); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.myint)) { + this.myint = data().deepCopy(fields()[3].schema(), other.myint); + fieldSetFlags()[3] = true; + } + if (isValidValue(fields()[4], other.mydouble)) { + this.mydouble = data().deepCopy(fields()[4].schema(), other.mydouble); + fieldSetFlags()[4] = true; + } + if (isValidValue(fields()[5], other.myboolean)) { + this.myboolean = data().deepCopy(fields()[5].schema(), other.myboolean); + fieldSetFlags()[5] = true; + } + if (isValidValue(fields()[6], other.mystring)) { + this.mystring = data().deepCopy(fields()[6].schema(), other.mystring); + fieldSetFlags()[6] = true; + } + if (isValidValue(fields()[7], other.mybytes)) { + this.mybytes = data().deepCopy(fields()[7].schema(), other.mybytes); + fieldSetFlags()[7] = true; + } + if (isValidValue(fields()[8], other.myfixed)) { + this.myfixed = data().deepCopy(fields()[8].schema(), other.myfixed); + fieldSetFlags()[8] = true; + } + if (isValidValue(fields()[9], other.myarray)) { + this.myarray = data().deepCopy(fields()[9].schema(), other.myarray); + fieldSetFlags()[9] = true; + } + if (isValidValue(fields()[10], other.mymap)) { + this.mymap = data().deepCopy(fields()[10].schema(), other.mymap); + fieldSetFlags()[10] = true; + } + } + + /** + * Gets the value of the 'mymoney' field. + * @return The value. + */ + public org.apache.spark.sql.catalyst.encoders.Money getMymoney() { + return mymoney; + } + + /** + * Sets the value of the 'mymoney' field. + * @param value The value of 'mymoney'. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder setMymoney(org.apache.spark.sql.catalyst.encoders.Money value) { + validate(fields()[0], value); + this.mymoneyBuilder = null; + this.mymoney = value; + fieldSetFlags()[0] = true; + return this; + } + + /** + * Checks whether the 'mymoney' field has been set. + * @return True if the 'mymoney' field has been set, false otherwise. + */ + public boolean hasMymoney() { + return fieldSetFlags()[0]; + } + + /** + * Gets the Builder instance for the 'mymoney' field and creates one if it doesn't exist yet. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.Money.Builder getMymoneyBuilder() { + if (mymoneyBuilder == null) { + if (hasMymoney()) { + setMymoneyBuilder(org.apache.spark.sql.catalyst.encoders.Money.newBuilder(mymoney)); + } else { + setMymoneyBuilder(org.apache.spark.sql.catalyst.encoders.Money.newBuilder()); + } + } + return mymoneyBuilder; + } + + /** + * Sets the Builder instance for the 'mymoney' field + * @param value The builder instance that must be set. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder setMymoneyBuilder(org.apache.spark.sql.catalyst.encoders.Money.Builder value) { + clearMymoney(); + mymoneyBuilder = value; + return this; + } + + /** + * Checks whether the 'mymoney' field has an active Builder instance + * @return True if the 'mymoney' field has an active Builder instance + */ + public boolean hasMymoneyBuilder() { + return mymoneyBuilder != null; + } + + /** + * Clears the value of the 'mymoney' field. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder clearMymoney() { + mymoney = null; + mymoneyBuilder = null; + fieldSetFlags()[0] = false; + return this; + } + + /** + * Gets the value of the 'myfloat' field. + * @return The value. + */ + public java.lang.Float getMyfloat() { + return myfloat; + } + + /** + * Sets the value of the 'myfloat' field. + * @param value The value of 'myfloat'. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder setMyfloat(float value) { + validate(fields()[1], value); + this.myfloat = value; + fieldSetFlags()[1] = true; + return this; + } + + /** + * Checks whether the 'myfloat' field has been set. + * @return True if the 'myfloat' field has been set, false otherwise. + */ + public boolean hasMyfloat() { + return fieldSetFlags()[1]; + } + + + /** + * Clears the value of the 'myfloat' field. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder clearMyfloat() { + fieldSetFlags()[1] = false; + return this; + } + + /** + * Gets the value of the 'mylong' field. + * @return The value. + */ + public java.lang.Long getMylong() { + return mylong; + } + + /** + * Sets the value of the 'mylong' field. + * @param value The value of 'mylong'. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder setMylong(long value) { + validate(fields()[2], value); + this.mylong = value; + fieldSetFlags()[2] = true; + return this; + } + + /** + * Checks whether the 'mylong' field has been set. + * @return True if the 'mylong' field has been set, false otherwise. + */ + public boolean hasMylong() { + return fieldSetFlags()[2]; + } + + + /** + * Clears the value of the 'mylong' field. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder clearMylong() { + fieldSetFlags()[2] = false; + return this; + } + + /** + * Gets the value of the 'myint' field. + * @return The value. + */ + public java.lang.Integer getMyint() { + return myint; + } + + /** + * Sets the value of the 'myint' field. + * @param value The value of 'myint'. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder setMyint(int value) { + validate(fields()[3], value); + this.myint = value; + fieldSetFlags()[3] = true; + return this; + } + + /** + * Checks whether the 'myint' field has been set. + * @return True if the 'myint' field has been set, false otherwise. + */ + public boolean hasMyint() { + return fieldSetFlags()[3]; + } + + + /** + * Clears the value of the 'myint' field. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder clearMyint() { + fieldSetFlags()[3] = false; + return this; + } + + /** + * Gets the value of the 'mydouble' field. + * @return The value. + */ + public java.lang.Double getMydouble() { + return mydouble; + } + + /** + * Sets the value of the 'mydouble' field. + * @param value The value of 'mydouble'. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder setMydouble(double value) { + validate(fields()[4], value); + this.mydouble = value; + fieldSetFlags()[4] = true; + return this; + } + + /** + * Checks whether the 'mydouble' field has been set. + * @return True if the 'mydouble' field has been set, false otherwise. + */ + public boolean hasMydouble() { + return fieldSetFlags()[4]; + } + + + /** + * Clears the value of the 'mydouble' field. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder clearMydouble() { + fieldSetFlags()[4] = false; + return this; + } + + /** + * Gets the value of the 'myboolean' field. + * @return The value. + */ + public java.lang.Boolean getMyboolean() { + return myboolean; + } + + /** + * Sets the value of the 'myboolean' field. + * @param value The value of 'myboolean'. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder setMyboolean(boolean value) { + validate(fields()[5], value); + this.myboolean = value; + fieldSetFlags()[5] = true; + return this; + } + + /** + * Checks whether the 'myboolean' field has been set. + * @return True if the 'myboolean' field has been set, false otherwise. + */ + public boolean hasMyboolean() { + return fieldSetFlags()[5]; + } + + + /** + * Clears the value of the 'myboolean' field. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder clearMyboolean() { + fieldSetFlags()[5] = false; + return this; + } + + /** + * Gets the value of the 'mystring' field. + * @return The value. + */ + public java.lang.String getMystring() { + return mystring; + } + + /** + * Sets the value of the 'mystring' field. + * @param value The value of 'mystring'. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder setMystring(java.lang.String value) { + validate(fields()[6], value); + this.mystring = value; + fieldSetFlags()[6] = true; + return this; + } + + /** + * Checks whether the 'mystring' field has been set. + * @return True if the 'mystring' field has been set, false otherwise. + */ + public boolean hasMystring() { + return fieldSetFlags()[6]; + } + + + /** + * Clears the value of the 'mystring' field. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder clearMystring() { + mystring = null; + fieldSetFlags()[6] = false; + return this; + } + + /** + * Gets the value of the 'mybytes' field. + * @return The value. + */ + public java.nio.ByteBuffer getMybytes() { + return mybytes; + } + + /** + * Sets the value of the 'mybytes' field. + * @param value The value of 'mybytes'. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder setMybytes(java.nio.ByteBuffer value) { + validate(fields()[7], value); + this.mybytes = value; + fieldSetFlags()[7] = true; + return this; + } + + /** + * Checks whether the 'mybytes' field has been set. + * @return True if the 'mybytes' field has been set, false otherwise. + */ + public boolean hasMybytes() { + return fieldSetFlags()[7]; + } + + + /** + * Clears the value of the 'mybytes' field. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder clearMybytes() { + mybytes = null; + fieldSetFlags()[7] = false; + return this; + } + + /** + * Gets the value of the 'myfixed' field. + * @return The value. + */ + public org.apache.spark.sql.catalyst.encoders.Magic getMyfixed() { + return myfixed; + } + + /** + * Sets the value of the 'myfixed' field. + * @param value The value of 'myfixed'. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder setMyfixed(org.apache.spark.sql.catalyst.encoders.Magic value) { + validate(fields()[8], value); + this.myfixed = value; + fieldSetFlags()[8] = true; + return this; + } + + /** + * Checks whether the 'myfixed' field has been set. + * @return True if the 'myfixed' field has been set, false otherwise. + */ + public boolean hasMyfixed() { + return fieldSetFlags()[8]; + } + + + /** + * Clears the value of the 'myfixed' field. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder clearMyfixed() { + myfixed = null; + fieldSetFlags()[8] = false; + return this; + } + + /** + * Gets the value of the 'myarray' field. + * @return The value. + */ + public java.util.List getMyarray() { + return myarray; + } + + /** + * Sets the value of the 'myarray' field. + * @param value The value of 'myarray'. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder setMyarray(java.util.List value) { + validate(fields()[9], value); + this.myarray = value; + fieldSetFlags()[9] = true; + return this; + } + + /** + * Checks whether the 'myarray' field has been set. + * @return True if the 'myarray' field has been set, false otherwise. + */ + public boolean hasMyarray() { + return fieldSetFlags()[9]; + } + + + /** + * Clears the value of the 'myarray' field. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder clearMyarray() { + myarray = null; + fieldSetFlags()[9] = false; + return this; + } + + /** + * Gets the value of the 'mymap' field. + * @return The value. + */ + public java.util.Map getMymap() { + return mymap; + } + + /** + * Sets the value of the 'mymap' field. + * @param value The value of 'mymap'. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder setMymap(java.util.Map value) { + validate(fields()[10], value); + this.mymap = value; + fieldSetFlags()[10] = true; + return this; + } + + /** + * Checks whether the 'mymap' field has been set. + * @return True if the 'mymap' field has been set, false otherwise. + */ + public boolean hasMymap() { + return fieldSetFlags()[10]; + } + + + /** + * Clears the value of the 'mymap' field. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.AvroExample1.Builder clearMymap() { + mymap = null; + fieldSetFlags()[10] = false; + return this; + } + + @Override + @SuppressWarnings("unchecked") + public AvroExample1 build() { + try { + AvroExample1 record = new AvroExample1(); + if (mymoneyBuilder != null) { + record.mymoney = this.mymoneyBuilder.build(); + } else { + record.mymoney = fieldSetFlags()[0] ? this.mymoney : (org.apache.spark.sql.catalyst.encoders.Money) defaultValue(fields()[0]); + } + record.myfloat = fieldSetFlags()[1] ? this.myfloat : (java.lang.Float) defaultValue(fields()[1]); + record.mylong = fieldSetFlags()[2] ? this.mylong : (java.lang.Long) defaultValue(fields()[2]); + record.myint = fieldSetFlags()[3] ? this.myint : (java.lang.Integer) defaultValue(fields()[3]); + record.mydouble = fieldSetFlags()[4] ? this.mydouble : (java.lang.Double) defaultValue(fields()[4]); + record.myboolean = fieldSetFlags()[5] ? this.myboolean : (java.lang.Boolean) defaultValue(fields()[5]); + record.mystring = fieldSetFlags()[6] ? this.mystring : (java.lang.String) defaultValue(fields()[6]); + record.mybytes = fieldSetFlags()[7] ? this.mybytes : (java.nio.ByteBuffer) defaultValue(fields()[7]); + record.myfixed = fieldSetFlags()[8] ? this.myfixed : (org.apache.spark.sql.catalyst.encoders.Magic) defaultValue(fields()[8]); + record.myarray = fieldSetFlags()[9] ? this.myarray : (java.util.List) defaultValue(fields()[9]); + record.mymap = fieldSetFlags()[10] ? this.mymap : (java.util.Map) defaultValue(fields()[10]); + return record; + } catch (java.lang.Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } + + @SuppressWarnings("unchecked") + private static final org.apache.avro.io.DatumWriter + WRITER$ = (org.apache.avro.io.DatumWriter)MODEL$.createDatumWriter(SCHEMA$); + + @Override public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + WRITER$.write(this, SpecificData.getEncoder(out)); + } + + @SuppressWarnings("unchecked") + private static final org.apache.avro.io.DatumReader + READER$ = (org.apache.avro.io.DatumReader)MODEL$.createDatumReader(SCHEMA$); + + @Override public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + READER$.read(this, SpecificData.getDecoder(in)); + } + +} diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/Currency.java b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/Currency.java new file mode 100644 index 0000000000000..460d625919739 --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/Currency.java @@ -0,0 +1,13 @@ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package org.apache.spark.sql.catalyst.encoders; +@SuppressWarnings("all") +@org.apache.avro.specific.AvroGenerated +public enum Currency { + EUR, USD, BRL ; + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"enum\",\"name\":\"Currency\",\"namespace\":\"org.apache.spark.sql.catalyst.encoders\",\"symbols\":[\"EUR\",\"USD\",\"BRL\"]}"); + public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } +} diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala index 86e43d71e4608..1e3b7b1d60c40 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala @@ -21,6 +21,7 @@ import java.math.BigInteger import java.sql.{Date, Timestamp} import java.util.Arrays +import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import scala.reflect.runtime.universe.TypeTag @@ -308,6 +309,26 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes encodeDecodeTest(Option("abc"), "option of string") encodeDecodeTest(Option.empty[String], "empty option of string") + encodeDecodeTest( + AvroExample1.newBuilder() + .setMyarray(List("Foo", "Bar").asJava) + .setMyboolean(true) + .setMybytes(java.nio.ByteBuffer.wrap("MyBytes".getBytes())) + .setMydouble(2.5) + .setMyfixed(new Magic("magic".getBytes)) + .setMyfloat(25.0F) + .setMyint(100) + .setMylong(10L) + .setMystring("hello") + .setMymap(Map( + "foo" -> new java.lang.Integer(1), + "bar" -> new java.lang.Integer(2)).asJava) + .setMymoney(Money.newBuilder().setAmount(100.0F).setCurrency(Currency.EUR).build()) + .build(), + "Avro encoder with map, array and fixed types")( + Encoders.bean[AvroExample1](classOf[AvroExample1]) + .asInstanceOf[ExpressionEncoder[AvroExample1]]) + productTest(("UDT", new ExamplePoint(0.1, 0.2))) test("nullable of encoder schema") { @@ -410,7 +431,7 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes val plan = LocalRelation(attr).serialize[T].deserialize[T] assertAnalysisSuccess(plan) - val isCorrect = (input, convertedBack) match { + val isCorrect = input == convertedBack || ((input, convertedBack) match { case (b1: Array[Byte], b2: Array[Byte]) => Arrays.equals(b1, b2) case (b1: Array[Int], b2: Array[Int]) => Arrays.equals(b1, b2) case (b1: Array[Array[_]], b2: Array[Array[_]]) => @@ -419,8 +440,8 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes Arrays.equals(b1.asInstanceOf[Array[AnyRef]], b2.asInstanceOf[Array[AnyRef]]) case (left: Comparable[_], right: Comparable[_]) => left.asInstanceOf[Comparable[Any]].compareTo(right) == 0 - case _ => input == convertedBack - } + case _ => false + }) if (!isCorrect) { val types = convertedBack match { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/Magic.java b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/Magic.java new file mode 100644 index 0000000000000..50d14f512a855 --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/Magic.java @@ -0,0 +1,45 @@ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package org.apache.spark.sql.catalyst.encoders; +@SuppressWarnings("all") +@org.apache.avro.specific.FixedSize(4) +@org.apache.avro.specific.AvroGenerated +public class Magic extends org.apache.avro.specific.SpecificFixed { + private static final long serialVersionUID = -7005876684809828227L; + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"fixed\",\"name\":\"Magic\",\"namespace\":\"org.apache.spark.sql.catalyst.encoders\",\"size\":4}"); + public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + + /** Creates a new Magic */ + public Magic() { + super(); + } + + /** + * Creates a new Magic with the given bytes. + * @param bytes The bytes to create the new Magic. + */ + public Magic(byte[] bytes) { + super(bytes); + } + + private static final org.apache.avro.io.DatumWriter + WRITER$ = new org.apache.avro.specific.SpecificDatumWriter(SCHEMA$); + + @Override public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + WRITER$.write(this, org.apache.avro.specific.SpecificData.getEncoder(out)); + } + + private static final org.apache.avro.io.DatumReader + READER$ = new org.apache.avro.specific.SpecificDatumReader(SCHEMA$); + + @Override public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + READER$.read(this, org.apache.avro.specific.SpecificData.getDecoder(in)); + } + +} diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/Money.java b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/Money.java new file mode 100644 index 0000000000000..3bba887b1e29f --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/Money.java @@ -0,0 +1,307 @@ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package org.apache.spark.sql.catalyst.encoders; + +import org.apache.avro.specific.SpecificData; +import org.apache.avro.message.BinaryMessageEncoder; +import org.apache.avro.message.BinaryMessageDecoder; +import org.apache.avro.message.SchemaStore; + +@SuppressWarnings("all") +@org.apache.avro.specific.AvroGenerated +public class Money extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { + private static final long serialVersionUID = 2836018423368943294L; + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Money\",\"namespace\":\"org.apache.spark.sql.catalyst.encoders\",\"fields\":[{\"name\":\"amount\",\"type\":\"float\",\"default\":0},{\"name\":\"currency\",\"type\":{\"type\":\"enum\",\"name\":\"Currency\",\"symbols\":[\"EUR\",\"USD\",\"BRL\"]},\"default\":\"EUR\"}]}"); + public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } + + private static SpecificData MODEL$ = new SpecificData(); + + private static final BinaryMessageEncoder ENCODER = + new BinaryMessageEncoder(MODEL$, SCHEMA$); + + private static final BinaryMessageDecoder DECODER = + new BinaryMessageDecoder(MODEL$, SCHEMA$); + + /** + * Return the BinaryMessageDecoder instance used by this class. + */ + public static BinaryMessageDecoder getDecoder() { + return DECODER; + } + + /** + * Create a new BinaryMessageDecoder instance for this class that uses the specified {@link SchemaStore}. + * @param resolver a {@link SchemaStore} used to find schemas by fingerprint + */ + public static BinaryMessageDecoder createDecoder(SchemaStore resolver) { + return new BinaryMessageDecoder(MODEL$, SCHEMA$, resolver); + } + + /** Serializes this Money to a ByteBuffer. */ + public java.nio.ByteBuffer toByteBuffer() throws java.io.IOException { + return ENCODER.encode(this); + } + + /** Deserializes a Money from a ByteBuffer. */ + public static Money fromByteBuffer( + java.nio.ByteBuffer b) throws java.io.IOException { + return DECODER.decode(b); + } + + @Deprecated public float amount; + @Deprecated public org.apache.spark.sql.catalyst.encoders.Currency currency; + + /** + * Default constructor. Note that this does not initialize fields + * to their default values from the schema. If that is desired then + * one should use newBuilder(). + */ + public Money() {} + + /** + * All-args constructor. + * @param amount The new value for amount + * @param currency The new value for currency + */ + public Money(java.lang.Float amount, org.apache.spark.sql.catalyst.encoders.Currency currency) { + this.amount = amount; + this.currency = currency; + } + + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + // Used by DatumWriter. Applications should not call. + public java.lang.Object get(int field$) { + switch (field$) { + case 0: return amount; + case 1: return currency; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value="unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: amount = (java.lang.Float)value$; break; + case 1: currency = (org.apache.spark.sql.catalyst.encoders.Currency)value$; break; + default: throw new org.apache.avro.AvroRuntimeException("Bad index"); + } + } + + /** + * Gets the value of the 'amount' field. + * @return The value of the 'amount' field. + */ + public java.lang.Float getAmount() { + return amount; + } + + /** + * Sets the value of the 'amount' field. + * @param value the value to set. + */ + public void setAmount(java.lang.Float value) { + this.amount = value; + } + + /** + * Gets the value of the 'currency' field. + * @return The value of the 'currency' field. + */ + public org.apache.spark.sql.catalyst.encoders.Currency getCurrency() { + return currency; + } + + /** + * Sets the value of the 'currency' field. + * @param value the value to set. + */ + public void setCurrency(org.apache.spark.sql.catalyst.encoders.Currency value) { + this.currency = value; + } + + /** + * Creates a new Money RecordBuilder. + * @return A new Money RecordBuilder + */ + public static org.apache.spark.sql.catalyst.encoders.Money.Builder newBuilder() { + return new org.apache.spark.sql.catalyst.encoders.Money.Builder(); + } + + /** + * Creates a new Money RecordBuilder by copying an existing Builder. + * @param other The existing builder to copy. + * @return A new Money RecordBuilder + */ + public static org.apache.spark.sql.catalyst.encoders.Money.Builder newBuilder(org.apache.spark.sql.catalyst.encoders.Money.Builder other) { + return new org.apache.spark.sql.catalyst.encoders.Money.Builder(other); + } + + /** + * Creates a new Money RecordBuilder by copying an existing Money instance. + * @param other The existing instance to copy. + * @return A new Money RecordBuilder + */ + public static org.apache.spark.sql.catalyst.encoders.Money.Builder newBuilder(org.apache.spark.sql.catalyst.encoders.Money other) { + return new org.apache.spark.sql.catalyst.encoders.Money.Builder(other); + } + + /** + * RecordBuilder for Money instances. + */ + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private float amount; + private org.apache.spark.sql.catalyst.encoders.Currency currency; + + /** Creates a new Builder */ + private Builder() { + super(SCHEMA$); + } + + /** + * Creates a Builder by copying an existing Builder. + * @param other The existing Builder to copy. + */ + private Builder(org.apache.spark.sql.catalyst.encoders.Money.Builder other) { + super(other); + if (isValidValue(fields()[0], other.amount)) { + this.amount = data().deepCopy(fields()[0].schema(), other.amount); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.currency)) { + this.currency = data().deepCopy(fields()[1].schema(), other.currency); + fieldSetFlags()[1] = true; + } + } + + /** + * Creates a Builder by copying an existing Money instance + * @param other The existing instance to copy. + */ + private Builder(org.apache.spark.sql.catalyst.encoders.Money other) { + super(SCHEMA$); + if (isValidValue(fields()[0], other.amount)) { + this.amount = data().deepCopy(fields()[0].schema(), other.amount); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.currency)) { + this.currency = data().deepCopy(fields()[1].schema(), other.currency); + fieldSetFlags()[1] = true; + } + } + + /** + * Gets the value of the 'amount' field. + * @return The value. + */ + public java.lang.Float getAmount() { + return amount; + } + + /** + * Sets the value of the 'amount' field. + * @param value The value of 'amount'. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.Money.Builder setAmount(float value) { + validate(fields()[0], value); + this.amount = value; + fieldSetFlags()[0] = true; + return this; + } + + /** + * Checks whether the 'amount' field has been set. + * @return True if the 'amount' field has been set, false otherwise. + */ + public boolean hasAmount() { + return fieldSetFlags()[0]; + } + + + /** + * Clears the value of the 'amount' field. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.Money.Builder clearAmount() { + fieldSetFlags()[0] = false; + return this; + } + + /** + * Gets the value of the 'currency' field. + * @return The value. + */ + public org.apache.spark.sql.catalyst.encoders.Currency getCurrency() { + return currency; + } + + /** + * Sets the value of the 'currency' field. + * @param value The value of 'currency'. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.Money.Builder setCurrency(org.apache.spark.sql.catalyst.encoders.Currency value) { + validate(fields()[1], value); + this.currency = value; + fieldSetFlags()[1] = true; + return this; + } + + /** + * Checks whether the 'currency' field has been set. + * @return True if the 'currency' field has been set, false otherwise. + */ + public boolean hasCurrency() { + return fieldSetFlags()[1]; + } + + + /** + * Clears the value of the 'currency' field. + * @return This builder. + */ + public org.apache.spark.sql.catalyst.encoders.Money.Builder clearCurrency() { + currency = null; + fieldSetFlags()[1] = false; + return this; + } + + @Override + @SuppressWarnings("unchecked") + public Money build() { + try { + Money record = new Money(); + record.amount = fieldSetFlags()[0] ? this.amount : (java.lang.Float) defaultValue(fields()[0]); + record.currency = fieldSetFlags()[1] ? this.currency : (org.apache.spark.sql.catalyst.encoders.Currency) defaultValue(fields()[1]); + return record; + } catch (java.lang.Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } + + @SuppressWarnings("unchecked") + private static final org.apache.avro.io.DatumWriter + WRITER$ = (org.apache.avro.io.DatumWriter)MODEL$.createDatumWriter(SCHEMA$); + + @Override public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + WRITER$.write(this, SpecificData.getEncoder(out)); + } + + @SuppressWarnings("unchecked") + private static final org.apache.avro.io.DatumReader + READER$ = (org.apache.avro.io.DatumReader)MODEL$.createDatumReader(SCHEMA$); + + @Override public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + READER$.read(this, SpecificData.getDecoder(in)); + } + +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala index 08b7521de9573..c7873a90b1e58 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala @@ -760,10 +760,9 @@ object SQLContext { attrs: Seq[AttributeReference]): Iterator[InternalRow] = { def createStructConverter(cls: Class[_], fieldTypes: Seq[DataType]): Any => InternalRow = { val methodConverters = - JavaTypeInference.getJavaBeanReadableProperties(cls).zip(fieldTypes) - .map { case (property, fieldType) => - val method = property.getReadMethod - method -> createConverter(method.getReturnType, fieldType) + JavaTypeInference.getObjectProperties(cls).zip(fieldTypes) + .map { case ((propertyName, getterMethod, setterMethod), fieldType) => + getterMethod -> createConverter(getterMethod.getReturnType, fieldType) } value => if (value == null) {