Handling emojis in gersam

Yomguithereal · May 5, 2021 · 7030ea5 · 7030ea5
1 parent 3fcb2e3
commit 7030ea5
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 1 deletion.
diff --git a/src/tokenizers/words/gersam.js b/src/tokenizers/words/gersam.js
diff --git a/test/tokenizers/words/gersam.js b/test/tokenizers/words/gersam.js
@@ -63,6 +63,11 @@ describe('gersam', function() {
         lang: 'fr',
         text: 'Les É.U. sont nuls.',
         tokens: ['Les', 'É.U.', 'sont', 'nuls', '.']
+      },
+      {
+        lang: 'en',
+        text: 'This is a very nice cat 🐱! No?',
+        tokens: ['This', 'is', 'a', 'very', 'nice', 'cat', '🐱', '!', 'No', '?']
       }
     ];