NonASCIICharacterChecker should inspect Token.rawText, not Token.text

It is reasonable to enforce a rule that prevents non-ASCII text from appearing
directly in source code. However current implementation also flags use of
unicode escape sequences, which consist of only ASCII chars (e.g. \u1f34e).

NonASCIICharacterChecker should inspect Token.rawText, which represents the
literal source prior to applying unicode escapes. Token.text, which is currently
being used, already has unicode escapes applied, and thus doesn't represent the
actual content of the source code.

Author: latkin

src/main/scala/org/scalastyle/scalariform/NonASCIICharacterChecker.scala

@@ -16,7 +16,6 @@
 
 package org.scalastyle.scalariform
 
-
 import java.util.regex.Pattern
 
 import org.scalastyle.PositionError
@@ -28,13 +27,12 @@ import _root_.scalariform.parser.CompilationUnit
 
 class NonASCIICharacterChecker extends ScalariformChecker {
   val errorKey: String = "non.ascii.character.disallowed"
+  private val asciiPattern = Pattern.compile("""\p{ASCII}+""", Pattern.DOTALL)
 
   override def verify(ast: CompilationUnit): List[ScalastyleError] = {
     ast.tokens.filter(hasNonAsciiChars).map(x => PositionError(x.offset))
   }
 
   private def hasNonAsciiChars(x: Token) =
-    x.text.trim.nonEmpty && !Pattern.compile( """\p{ASCII}+""", Pattern.DOTALL)
-      .matcher(x.text.trim).matches()
-
+    x.rawText.trim.nonEmpty && !asciiPattern.matcher(x.rawText.trim).matches()
 }

src/test/scala/org/scalastyle/scalariform/NonASCIICharacterCheckerTest.scala

@@ -39,8 +39,18 @@ class NonASCIICharacterCheckerTest extends AssertionsForJUnit with CheckerTest {
     assertErrors(List(), source)
   }
 
+  @Test def testStringOK(): Unit = {
+    val source = """
+                   |package foobar
+                   |// non-ascii in string via unicode escape - ok
+                   |class OK {
+                   |  val s = "%s"
+                   |}""".stripMargin.format("\\ud83c\\udf4e")
 
-  @Test def testClassNotOk(): Unit = {
+    assertErrors(List(), source)
+  }
+
+  @Test def testClassNotOK(): Unit = {
     val source = """
                    |package foobar
                    |// \u2190
@@ -49,6 +59,18 @@ class NonASCIICharacterCheckerTest extends AssertionsForJUnit with CheckerTest {
                    |  def `\u21d2` = "test"
                    |}
                    | """.stripMargin
+
     assertErrors(List(columnError(2, 14), columnError(5, 6), columnError(6, 6)), source)
   }
+
+  @Test def testStringNotOK(): Unit = {
+    val source = """
+                   |package foobar
+                   |// non-ascii literal in string - not ok
+                   |class NotOK {
+                   |  val s = "\ud83c\udf4e"
+                   |}""".stripMargin
+
+    assertErrors(List(columnError(5, 10)), source)
+  }
 }