Fix escaped Unicode hex literals parsing (#49)

* Update package set

* Fix escaped Unicode hex literals parsing

The Char type represents a UTF-16 code unit, while String can contain
any UTF-8 code unit. The same logic for escaping Unicode hexadecimal
literals was used by both String and Char literal parsers. This caused
a bug that prevented PureScript source code from having string literals
containing UTF-8 hexadecimal literals representing code units larger
than two bytes.

* Add tests for Unicode hex literals

* Update bench and parse-package-set dependencies

* Update CONTRIBUTORS.md

* Update parse-package-set's test package set.

---------

Co-authored-by: Nathan Faubion <nathan@n-son.com>

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
 
  - uses: purescript-contrib/setup-purescript@main
  with:
- purescript: "0.15.0"
+ purescript: "0.15.8"
  spago: "0.20.9"
  psa: "0.8.2"
  purs-tidy: "latest"

CONTRIBUTORS.md

@@ -19,3 +19,4 @@ By adding your name to the list below, you agree to license your contributions u
 | [@rintcius](https://github.com/rintcius) | Rintcius Blok
 | [@i-am-the-slime](https://github.com/i-am-the-slime) | Mark Eibes
 | [@monoidmusician](https://github.com/MonoidMusician) | Verity Scheel
+| [@turlando](https://github.com/turlando) | Tancredi Orlando

bench/bench.dhall

@@ -10,6 +10,7 @@ in conf // {
  , "control"
  , "effect"
  , "either"
+ , "enums"
  , "foldable-traversable"
  , "free"
  , "functions"

packages.dhall

@@ -1,5 +1,5 @@
 let upstream =
- https://github.com/purescript/package-sets/releases/download/psc-0.15.0-20220507/packages.dhall
- sha256:cf54330f3bc1b25a093b69bff8489180c954b43668c81288901a2ec29a08cc64
+ https://github.com/purescript/package-sets/releases/download/psc-0.15.7-20230401/packages.dhall
+ sha256:d385eeee6ca160c32d7389a1f4f4ee6a05aff95e81373cdc50670b436efa1060
 
 in upstream

parse-package-set/Main.purs

@@ -158,7 +158,7 @@ defaultSpagoDhall :: String
 defaultSpagoDhall = Array.intercalate "\n"
  [ "{ name = \"test-parser\""
  , ", dependencies = [] : List Text"
- , ", packages = https://github.com/purescript/package-sets/releases/download/psc-0.15.0-20220507/packages.dhall sha256:cf54330f3bc1b25a093b69bff8489180c954b43668c81288901a2ec29a08cc64"
+ , ", packages = https://github.com/purescript/package-sets/releases/download/psc-0.15.7-20230401/packages.dhall sha256:d385eeee6ca160c32d7389a1f4f4ee6a05aff95e81373cdc50670b436efa1060"
  , ", sources = [] : List Text"
  , "}"
  ]

parse-package-set/parse-package-set.dhall

@@ -12,6 +12,7 @@ in conf // {
  , "datetime"
  , "effect"
  , "either"
+ , "enums"
  , "exceptions"
  , "filterable"
  , "foldable-traversable"

spago.dhall

@@ -8,6 +8,7 @@
  , "control"
  , "effect"
  , "either"
+ , "enums"
  , "foldable-traversable"
  , "free"
  , "functions"

src/PureScript/CST/Lexer.purs

@@ -13,6 +13,7 @@ import Data.Array.NonEmpty as NonEmptyArray
 import Data.Array.ST as STArray
 import Data.Char as Char
 import Data.Either (Either(..))
+import Data.Enum (toEnum)
 import Data.Foldable (fold, foldl, foldMap)
 import Data.Int (hexadecimal)
 import Data.Int as Int
@@ -23,6 +24,8 @@ import Data.Newtype (unwrap)
 import Data.Number as Number
 import Data.String (Pattern(..), Replacement(..))
 import Data.String as String
+import Data.String.CodePoints (CodePoint)
+import Data.String.CodePoints as SCP
 import Data.String.CodeUnits as SCU
 import Data.String.Regex as Regex
 import Data.String.Regex.Flags (unicode)
@@ -36,6 +39,18 @@ import PureScript.CST.Types (Comment(..), IntValue(..), LineFeed(..), ModuleName
 
 infixr 3 alt as <|>
 
+class IsChar a where
+ fromChar :: Char -> a
+ fromCharCode :: Int -> Maybe a
+
+instance IsChar Char where
+ fromChar = identity
+ fromCharCode = Char.fromCharCode
+
+instance IsChar CodePoint where
+ fromChar = SCP.codePointFromChar
+ fromCharCode = toEnum
+
 data LexResult e a
  = LexFail e String
  | LexSucc a String
@@ -502,29 +517,37 @@ token =
  _ ->
  pure { raw: SCU.singleton ch, char: ch }
 
+ parseEscape
+ :: forall a
+ . IsChar a
+ => Lex (Unit -> ParseError) { raw :: String, char :: a }
  parseEscape = do
  ch <- charAny
  case ch of
  't' ->
- pure { raw: "\\t", char: '\t' }
+ pure { raw: "\\t", char: fromChar '\t' }
  'r' ->
- pure { raw: "\\r", char: '\r' }
+ pure { raw: "\\r", char: fromChar '\r' }
  'n' ->
- pure { raw: "\\n", char: '\n' }
+ pure { raw: "\\n", char: fromChar '\n' }
  '"' ->
- pure { raw: "\\\"", char: '"' }
+ pure { raw: "\\\"", char: fromChar '"' }
  '\'' ->
- pure { raw: "\\'", char: '\'' }
+ pure { raw: "\\'", char: fromChar '\'' }
  '\\' ->
- pure { raw: "\\\\", char: '\\' }
+ pure { raw: "\\\\", char: fromChar '\\' }
  'x' ->
  parseHexEscape
  _ ->
  fail $ LexInvalidCharEscape $ SCU.singleton ch
 
+ parseHexEscape
+ :: forall a
+ . IsChar a
+ => Lex (Unit -> ParseError) { raw :: String, char :: a }
  parseHexEscape = do
  esc <- hexEscapeRegex
- case Char.fromCharCode =<< Int.fromStringAs hexadecimal esc of
+ case fromCharCode =<< Int.fromStringAs hexadecimal esc of
  Just ch ->
  pure { raw: "\\x" <> esc, char: ch }
  Nothing ->
@@ -552,7 +575,7 @@ token =
 
  parseStringEscape = ado
  res <- charBackslash *> parseEscape
- in { raw: res.raw, string: SCU.singleton res.char }
+ in { raw: res.raw, string: SCP.singleton res.char }
 
  parseStringChars = ado
  raw <- stringCharsRegex

test/Main.purs

@@ -232,3 +232,23 @@ main = do
  true
  _ ->
  false
+
+ assertParse "String with Unicode astral code point hex literal"
+ """
+ "\x10ffff"
+ """
+ case _ of
+ ParseSucceeded (ExprString _ _) ->
+ true
+ _ ->
+ false
+
+ assertParse "Unicode astral code point Char hex literal"
+ """
+ '\x10ffff'
+ """
+ case _ of
+ (ParseFailed _ :: RecoveredParserResult Expr) ->
+ true
+ _ ->
+ false