Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
3b44c59
adding testcase
kevinyu98 Apr 20, 2016
18b4a31
Merge remote-tracking branch 'upstream/master'
kevinyu98 Apr 22, 2016
4f4d1c8
Merge remote-tracking branch 'upstream/master'
kevinyu98 Apr 23, 2016
f5f0cbe
Merge remote-tracking branch 'upstream/master'
kevinyu98 Apr 23, 2016
d8b2edb
Merge remote-tracking branch 'upstream/master'
kevinyu98 Apr 25, 2016
196b6c6
Merge remote-tracking branch 'upstream/master'
kevinyu98 Apr 25, 2016
f37a01e
Merge remote-tracking branch 'upstream/master'
kevinyu98 Apr 27, 2016
bb5b01f
Merge remote-tracking branch 'upstream/master'
kevinyu98 Apr 30, 2016
bde5820
Merge remote-tracking branch 'upstream/master'
kevinyu98 May 4, 2016
5f7cd96
Merge remote-tracking branch 'upstream/master'
kevinyu98 May 10, 2016
893a49a
Merge remote-tracking branch 'upstream/master'
kevinyu98 May 13, 2016
4bbe1fd
Merge remote-tracking branch 'upstream/master'
kevinyu98 May 17, 2016
b2dd795
Merge remote-tracking branch 'upstream/master'
kevinyu98 May 18, 2016
8c3e5da
Merge remote-tracking branch 'upstream/master'
kevinyu98 May 18, 2016
a0eaa40
Merge remote-tracking branch 'upstream/master'
kevinyu98 May 19, 2016
d03c940
Merge remote-tracking branch 'upstream/master'
kevinyu98 May 19, 2016
d728d5e
Merge remote-tracking branch 'upstream/master'
kevinyu98 May 24, 2016
ea104dd
Merge remote-tracking branch 'upstream/master'
kevinyu98 May 25, 2016
6ab1215
Merge remote-tracking branch 'upstream/master'
kevinyu98 May 27, 2016
0c56653
Merge remote-tracking branch 'upstream/master'
kevinyu98 Jun 1, 2016
d7a1874
Merge remote-tracking branch 'upstream/master'
kevinyu98 Jun 1, 2016
85d3500
Merge remote-tracking branch 'upstream/master'
kevinyu98 Jun 2, 2016
c056f91
Merge remote-tracking branch 'upstream/master'
kevinyu98 Jun 3, 2016
0b8189d
Merge remote-tracking branch 'upstream/master'
kevinyu98 Jun 3, 2016
c2ea31d
Merge remote-tracking branch 'upstream/master'
kevinyu98 Jun 6, 2016
a2d3056
Merge remote-tracking branch 'upstream/master'
kevinyu98 Jun 8, 2016
1d451d2
include metadata when writeRelation
kevinyu98 Jun 8, 2016
200a923
address comments for the testcase
kevinyu98 Jun 8, 2016
a47dad4
address test case comments
kevinyu98 Jun 9, 2016
229ca27
address comment
kevinyu98 Jun 9, 2016
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ package object expressions {
implicit class AttributeSeq(val attrs: Seq[Attribute]) extends Serializable {
/** Creates a StructType with a schema matching this `Seq[Attribute]`. */
def toStructType: StructType = {
StructType(attrs.map(a => StructField(a.name, a.dataType, a.nullable)))
StructType(attrs.map(a => StructField(a.name, a.dataType, a.nullable, a.metadata)))
}

// It's possible that `attrs` is a linked list, which can lead to bad O(n^2) loops when
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -625,6 +625,21 @@ class ParquetQuerySuite extends QueryTest with ParquetTest with SharedSQLContext
}
}
}

test("SPARK-15804: write out the metadata to parquet file") {
val df = Seq((1, "abc"), (2, "hello")).toDF("a", "b")
val md = new MetadataBuilder().putString("key", "value").build()
val dfWithmeta = df.select('a, 'b.as("b", md))

withTempPath { dir =>
val path = dir.getCanonicalPath
dfWithmeta.write.parquet(path)

readParquetFile(path) { df =>
assert(df.schema.last.metadata.getString("key") == "value")
}
}
}
}

object TestingUDT {
Expand Down