Merge pull request #3 from theonion/dumbquotes

Adding elasticsearch char filter for transforming smart quotes into dumb ones
theonion · Mar 26, 2014 · 2b4fc5b · 2b4fc5b
2 parents 78cc8d4 + 02158ae
commit 2b4fc5b
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 1 deletion.
diff --git a/elastimorphic/base.py b/elastimorphic/base.py
@@ -228,6 +228,9 @@ def get_mapping(cls):
                 },
                 "properties": cls.get_mapping_properties(),
                 "dynamic": "strict",
+                "_all": {
+                    "analyzer": "html"
+                }
             }
         }
 

diff --git a/elastimorphic/conf/defaults.py b/elastimorphic/conf/defaults.py
@@ -6,16 +6,29 @@
             "analyzer": {
                 "autocomplete": {
                     "type": "custom",
+                    "char_filter": ["quotes"],
                     "tokenizer": "edge_ngram_tokenizer",
                     "filter": ["asciifolding", "lowercase"]
                 },
                 "html": {
                     "type": "custom",
-                    "char_filter": ["html_strip"],
+                    "char_filter": ["html_strip", "quotes"],
                     "tokenizer": "standard",
                     "filter": ["asciifolding", "lowercase", "stop", "snowball"]
                 }
             },
+            "char_filter": {
+                "quotes": {
+                    "mappings": [
+                        "\u0091=>\u0027",
+                        "\u0092=>\u0027",
+                        "\u2018=>\u0027",
+                        "\u2019=>\u0027",
+                        "\uFF07=>\u0027"
+                    ],
+                    "type": "mapping"
+                },
+            },
             "tokenizer": {
                 "edge_ngram_tokenizer": {
                     "type": "edgeNGram",