Starting working on gersam

.gitignore

@@ -4,6 +4,7 @@ experiments
 build
 *.log
 *.lock
+*.pl
 .DS_Store
 TODO.md
 

src/tokenizers/words/treebank.js

@@ -125,5 +125,3 @@ export default function tokenize(text) {
 
   return finalize(text);
 }
-
-// FRENCH aujourd'hui & weird apostrophe

test/endpoint.js

@@ -190,6 +190,7 @@ describe('tokenizers', function() {
   });
 
   describe('words', function() {
+    require('./tokenizers/words/gersam.js');
     require('./tokenizers/words/treebank.js');
   });
 });

test/tokenizers/words/gersam.js

@@ -0,0 +1,38 @@
+/**
+ * Talisman tokenizers/words/gersam tests
+ * =======================================
+ *
+ */
+import assert from 'assert';
+import words from '../../../src/tokenizers/words/gersam';
+
+describe('gersam', function() {
+  it('should correctly tokenize words.', function() {
+    const tests = [
+      {
+        lang: 'en',
+        text: 'Good muffins cost $3.88\nin New York.  Please buy me\ntwo of them.\nThanks.',
+        tokens: ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York', '.', 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
+      },
+      {
+        lang: 'en',
+        text: 'They\'ll save and invest more.',
+        tokens: ['They', '\'ll', 'save', 'and', 'invest', 'more', '.']
+      },
+      {
+        lang: 'en',
+        text: 'hi, my name can\'t hello,',
+        tokens: ['hi', ',', 'my', 'name', 'can', '\'t', 'hello', ',']
+      },
+      {
+        lang: 'en',
+        text: '"Hello", Good sir (this is appaling)...',
+        tokens: ['"', 'Hello', '"', ',', 'Good', 'sir', '(', 'this', 'is', 'appaling', ')', '…']
+      }
+    ];
+
+    tests.forEach(function({lang, text, tokens}) {
+      assert.deepEqual(words(lang, text), tokens);
+    });
+  });
+});

test/tokenizers/words/treebank.js

@@ -21,6 +21,10 @@ describe('treebank', function() {
         text: 'hi, my name can\'t hello,',
         tokens: ['hi', ',', 'my', 'name', 'ca', "n't", 'hello', ',']
       },
+      {
+        text: 'O.N.U.',
+        tokens: ['O.N.U', '.']
+      },
       {
         text: '"Hello", Good sir (this is appaling)...',
         tokens: ['``', 'Hello', '\'\'', ',', 'Good', 'sir', '(', 'this', 'is', 'appaling', ')', '...']