Skip to content

Commit c2c5ec6

Browse files
authored
[BugFix] Fix the Spark Writer bug when the column name contains a dot(.) (#101)
1 parent 789fe48 commit c2c5ec6

File tree

3 files changed

+4
-4
lines changed

3 files changed

+4
-4
lines changed

spark/src/main/scala/com/alibaba/graphar/utils/IndexGenerator.scala

+2-2
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ object IndexGenerator {
118118
val srcCol = GeneralParams.srcIndexCol;
119119
val indexCol = GeneralParams.vertexIndexCol;
120120
val srcPrimaryKey = GeneralParams.primaryCol;
121-
val trans_df = spark.sql(f"select src_vertex.$indexCol%s as $srcCol%s, edge.* from edge inner join src_vertex on src_vertex.$srcPrimaryKey%s=edge.$srcColumnName%s")
121+
val trans_df = spark.sql(f"select src_vertex.`$indexCol` as `$srcCol`, edge.* from edge inner join src_vertex on src_vertex.`$srcPrimaryKey`=edge.`$srcColumnName`")
122122
// drop the old src id col
123123
trans_df.drop(srcColumnName)
124124
}
@@ -131,7 +131,7 @@ object IndexGenerator {
131131
val dstCol = GeneralParams.dstIndexCol;
132132
val indexCol = GeneralParams.vertexIndexCol;
133133
val dstPrimaryKey = GeneralParams.primaryCol;
134-
val trans_df = spark.sql(f"select dst_vertex.$indexCol%s as $dstCol%s, edges.* from edges inner join dst_vertex on dst_vertex.$dstPrimaryKey%s=edges.$dstColumnName%s")
134+
val trans_df = spark.sql(f"select dst_vertex.`$indexCol` as `$dstCol`, edges.* from edges inner join dst_vertex on dst_vertex.`$dstPrimaryKey`=edges.`$dstColumnName`")
135135
// drop the old dst id col
136136
trans_df.drop(dstColumnName)
137137
}

spark/src/main/scala/com/alibaba/graphar/writer/EdgeWriter.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ class EdgeWriter(prefix: String, edgeInfo: EdgeInfo, adjListType: AdjListType.V
211211
val p_it = propertyGroup.getProperties().iterator
212212
while (p_it.hasNext()) {
213213
val property = p_it.next()
214-
property_list += property.getName()
214+
property_list += "`" + property.getName() + "`"
215215
}
216216
var chunk_index: Long = 0
217217
for (chunk <- chunks) {

spark/src/main/scala/com/alibaba/graphar/writer/VertexWriter.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ class VertexWriter(prefix: String, vertexInfo: VertexInfo, vertexDf: DataFrame)
8282
val it = propertyGroup.getProperties().iterator
8383
while (it.hasNext()) {
8484
val property = it.next()
85-
property_list += property.getName()
85+
property_list += "`" + property.getName() + "`"
8686
}
8787
val pg_df = chunks.select(property_list.map(col): _*)
8888
FileSystem.writeDataFrame(pg_df, propertyGroup.getFile_type(), output_prefix)

0 commit comments

Comments
 (0)