Handling more cases with gersam

Yomguithereal · Mar 20, 2020 · 3993939 · 3993939
1 parent b9ba907
commit 3993939
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 3 deletions.
diff --git a/src/tokenizers/words/gersam.js b/src/tokenizers/words/gersam.js
diff --git a/test/tokenizers/words/gersam.js b/test/tokenizers/words/gersam.js
@@ -53,6 +53,16 @@ describe('gersam', function() {
         lang: 'it',
         text: 'O.N.U. La vie.est foutue.',
         tokens: ['O.N.U.', 'La', 'vie', '.', 'est', 'foutue', '.']
+      },
+      {
+        lang: 'en',
+        text: 'Mrs. Langley is back from the market.',
+        tokens: ['Mrs.', 'Langley', 'is', 'back', 'from', 'the', 'market', '.']
+      },
+      {
+        lang: 'fr',
+        text: 'Les É.U. sont nuls.',
+        tokens: ['Les', 'É.U.', 'sont', 'nuls', '.']
       }
     ];