Skip to content

[SPARK-13515] Make FormatNumber work irrespective of locale. #11396

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
lgieron wants to merge 5 commits into from
Closed

[SPARK-13515] Make FormatNumber work irrespective of locale. #11396

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
349c45b
[SPARK-13515] Make FormatNumber work irrespective of locale.
Feb 23, 2016
40e1ec8
SPARK-13515 Improving code formatting/aesthetics.
Feb 26, 2016
aab31f9
SPARK-13515 Clean up imports.
Feb 27, 2016
fef157f
SPARK-13515 Correct formatting as required by lint style checks.
Feb 28, 2016
b3b7b15
SPARK-13515 Improve clarity with variable renaming and extra comments.
Feb 28, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 13 additions & 7 deletions ...catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.apache.spark.sql.catalyst.expressions

import java.text.DecimalFormat
import java.text.{DecimalFormat, DecimalFormatSymbols}
import java.util.{HashMap, Locale, Map => JMap}

import org.apache.spark.sql.catalyst.InternalRow
Expand Down Expand Up @@ -938,8 +938,10 @@ case class FormatNumber(x: Expression, d: Expression)
@transient
private val pattern: StringBuffer = new StringBuffer()

// SPARK-13515: US Locale configures the DecimalFormat object to use a dot ('.')
// as a decimal separator.
@transient
private val numberFormat: DecimalFormat = new DecimalFormat("")
private val numberFormat = new DecimalFormat("", new DecimalFormatSymbols(Locale.US))

override protected def nullSafeEval(xObject: Any, dObject: Any): Any = {
val dValue = dObject.asInstanceOf[Int]
Expand All @@ -962,10 +964,9 @@ case class FormatNumber(x: Expression, d: Expression)
pattern.append("0")
}
}
val dFormat = new DecimalFormat(pattern.toString)
lastDValue = dValue

numberFormat.applyPattern(dFormat.toPattern)
numberFormat.applyLocalizedPattern(pattern.toString)
}

x.dataType match {
Expand All @@ -992,14 +993,20 @@ case class FormatNumber(x: Expression, d: Expression)

val sb = classOf[StringBuffer].getName
val df = classOf[DecimalFormat].getName
val dfs = classOf[DecimalFormatSymbols].getName
val l = classOf[Locale].getName
// SPARK-13515: US Locale configures the DecimalFormat object to use a dot ('.')
// as a decimal separator.
val usLocale = "US"
val lastDValue = ctx.freshName("lastDValue")
val pattern = ctx.freshName("pattern")
val numberFormat = ctx.freshName("numberFormat")
val i = ctx.freshName("i")
val dFormat = ctx.freshName("dFormat")
ctx.addMutableState("int", lastDValue, s"$lastDValue = -100;")
ctx.addMutableState(sb, pattern, s"$pattern = new $sb();")
ctx.addMutableState(df, numberFormat, s"""$numberFormat = new $df("");""")
ctx.addMutableState(df, numberFormat,
s"""$numberFormat = new $df("", new $dfs($l.$usLocale));""")

s"""
if ($d >= 0) {
Expand All @@ -1013,9 +1020,8 @@ case class FormatNumber(x: Expression, d: Expression)
$pattern.append("0");
}
}
$df $dFormat = new $df($pattern.toString());
$lastDValue = $d;
$numberFormat.applyPattern($dFormat.toPattern());
$numberFormat.applyLocalizedPattern($pattern.toString());
}
${ev.value} = UTF8String.fromString($numberFormat.format(${typeHelper(num)}));
} else {
Expand Down