Skip to content

[SPARK-17299] TRIM/LTRIM/RTRIM should not strips characters other than spaces #14924

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
techaddict wants to merge 4 commits into from
Closed

[SPARK-17299] TRIM/LTRIM/RTRIM should not strips characters other than spaces #14924

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
58c2a5d
[SPARK-17299] TRIM/LTRIM/RTRIM should not strips characters other tha…
techaddict Sep 1, 2016
afeb73b
Merge branch 'master' into SPARK-17299
techaddict Sep 2, 2016
554f46c
add tests, remove java trim which trims chars less than 0x20
techaddict Sep 2, 2016
7583070
fix indent
techaddict Sep 2, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -465,9 +465,9 @@ public UTF8String trim() {
int s = 0;
int e = this.numBytes - 1;
// skip all of the space (0x20) in the left side
while (s < this.numBytes && getByte(s) <= 0x20 && getByte(s) >= 0x00) s++;
while (s < this.numBytes && getByte(s) == 0x20) s++;
// skip all of the space (0x20) in the right side
while (e >= 0 && getByte(e) <= 0x20 && getByte(e) >= 0x00) e--;
while (e >= 0 && getByte(e) == 0x20) e--;
if (s > e) {
// empty string
return EMPTY_UTF8;
Expand All @@ -479,7 +479,7 @@ public UTF8String trim() {
public UTF8String trimLeft() {
int s = 0;
// skip all of the space (0x20) in the left side
while (s < this.numBytes && getByte(s) <= 0x20 && getByte(s) >= 0x00) s++;
while (s < this.numBytes && getByte(s) == 0x20) s++;
if (s == this.numBytes) {
// empty string
return EMPTY_UTF8;
Expand All @@ -491,7 +491,7 @@ public UTF8String trimLeft() {
public UTF8String trimRight() {
int e = numBytes - 1;
// skip all of the space (0x20) in the right side
while (e >= 0 && getByte(e) <= 0x20 && getByte(e) >= 0x00) e--;
while (e >= 0 && getByte(e) == 0x20) e--;

if (e < 0) {
// empty string
Expand Down
10 changes: 10 additions & 0 deletions common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,16 @@ public void trims() {
assertEquals(fromString("数据砖头"), fromString("数据砖头").trim());
assertEquals(fromString("数据砖头"), fromString("数据砖头").trimLeft());
assertEquals(fromString("数据砖头"), fromString("数据砖头").trimRight());

char[] charsLessThan0x20 = new char[10];
Arrays.fill(charsLessThan0x20, (char)(' ' - 1));
String stringStartingWithSpace =
new String(charsLessThan0x20) + "hello" + new String(charsLessThan0x20);
assertEquals(fromString(stringStartingWithSpace), fromString(stringStartingWithSpace).trim());
assertEquals(fromString(stringStartingWithSpace),
fromString(stringStartingWithSpace).trimLeft());
assertEquals(fromString(stringStartingWithSpace),
fromString(stringStartingWithSpace).trimRight());
}

@Test
Expand Down
8 changes: 4 additions & 4 deletions ...on/unsafe/src/test/scala/org/apache/spark/unsafe/types/UTF8StringPropertyCheckSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ class UTF8StringPropertyCheckSuite extends FunSuite with GeneratorDrivenProperty
}
}

val whitespaceChar: Gen[Char] = Gen.choose(0x00, 0x20).map(_.toChar)
val whitespaceChar: Gen[Char] = Gen.const(0x20.toChar)
val whitespaceString: Gen[String] = Gen.listOf(whitespaceChar).map(_.mkString)
val randomString: Gen[String] = Arbitrary.arbString.arbitrary

Expand All @@ -107,15 +107,15 @@ class UTF8StringPropertyCheckSuite extends FunSuite with GeneratorDrivenProperty
def lTrim(s: String): String = {
var st = 0
val array: Array[Char] = s.toCharArray
while ((st < s.length) && (array(st) <= ' ')) {
while ((st < s.length) && (array(st) == ' ')) {
st += 1
}
if (st > 0) s.substring(st, s.length) else s
}
def rTrim(s: String): String = {
var len = s.length
val array: Array[Char] = s.toCharArray
while ((len > 0) && (array(len - 1) <= ' ')) {
while ((len > 0) && (array(len - 1) == ' ')) {
len -= 1
}
if (len < s.length) s.substring(0, len) else s
Expand All @@ -127,7 +127,7 @@ class UTF8StringPropertyCheckSuite extends FunSuite with GeneratorDrivenProperty
whitespaceString
) { (start: String, middle: String, end: String) =>
val s = start + middle + end
assert(toUTF8(s).trim() === toUTF8(s.trim()))
assert(toUTF8(s).trim() === toUTF8(rTrim(lTrim(s))))
assert(toUTF8(s).trimLeft() === toUTF8(lTrim(s)))
assert(toUTF8(s).trimRight() === toUTF8(rTrim(s)))
}
Expand Down