Update other non-Thai characters in newmm

PyThaiNLP · Nov 1, 2023 · 02e9cb5 · 02e9cb5
1 parent fb3e7bb
commit 02e9cb5
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 1 deletion.
diff --git a/pythainlp/tokenize/newmm.py b/pythainlp/tokenize/newmm.py
@@ -44,7 +44,7 @@
 \d+([,\.]\d+)*|    # numbers
 [ \t]+|            # spaces
 \r?\n|             # newlines
-[^\u0E00-\u0E7F]+  # other non-Thai characters
+[^\u0E00-\u0E7F \t]+  # other non-Thai characters
 """
 )
 

diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py
@@ -653,6 +653,15 @@ def test_newmm(self):
                 keep_whitespace=False,
             )
         )
+        self.assertEqual(
+            word_tokenize("(คนไม่เอา)", engine="newmm"), ['(', 'คน', 'ไม่', 'เอา', ')']
+        )
+        self.assertEqual(
+            word_tokenize("กม/ชม", engine="newmm"), ['กม', '/', 'ชม']
+        )
+        self.assertEqual(
+            word_tokenize("สีหน้า(รถ)", engine="newmm"), ['สีหน้า', '(', 'รถ', ')']
+        )
 
     def test_newmm_longtext(self):
         self.assertIsInstance(