Merge pull request #50 from saschazar21/normalize

Added normalize option

README.md

@@ -38,6 +38,7 @@ You can use different handle for the tags, by configuring the `handle` option. `
       /* Can also use deprecated template property.
       "template": "/partials/tag.hbt",
       */
+      "normalize": true,
       "sortBy": "date",
       "reverse": true,
       "skipMetadata": false,
@@ -68,6 +69,10 @@ metalsmith
     // metalsmith-templates plugin. The `template` property is deprecated here
     // as well but still available for use.
     // template:'/partials/tag.hbt',
+    // ------
+    // Normalize special characters like ØçßÜ to their ASCII equivalents ocssü
+    // makes use of the value assigned to the 'slug' property below
+    normalize: true,
     // provide posts sorted by 'date' (optional)
     sortBy: 'date',
     // sort direction (optional)
@@ -97,6 +102,10 @@ metalsmith
   It is possible to use `opts.metadataKey` for defining the name of the global tag list.
   By default it is `'tags'`.
 
+### Normalized characters
+
+  Handle with care. This is to be seen rather as a fallback as it heavily depends on your 'slug' settings. If you use the standard setting provided in this readme (*rfc3986*), you should be good to go.  Should fix [issue #48](https://github.com/totocaster/metalsmith-tags/issues/48).
+
 ## Pagination
 
   Additionally you can paginate your tag pages.  To do so add two additional properties to your configuration object, `pathPage` and `perPage`, and modify `path` to point to the root pagination location:

lib/index.js

@@ -1,3 +1,4 @@
+var debug = require('debug')('metalsmith-tags');
 var slug = require('slug');
 
 /**
@@ -18,6 +19,7 @@ function plugin(opts) {
   opts.layout = opts.layout || 'partials/tag.hbt';
   opts.handle = opts.handle || 'tags';
   opts.metadataKey = opts.metadataKey || 'tags';
+  opts.normalize = opts.normalize || opts.slugify || false;
   opts.sortBy = opts.sortBy || 'title';
   opts.reverse = opts.reverse || false;
   opts.perPage  = opts.perPage || 0;
@@ -102,9 +104,13 @@ function plugin(opts) {
             tag = String(rawTag).trim();
           }
 
+          // If normalize/slugify was set to true, remove all special characters like ÄÖØß...
+          var tag = opts.normalize ? safeTag(tag) : tag;
+          debug(tag);
 
           // Save url safe formatted and display versions of tag data
-          data[opts.handle].push({ name: tag, slug: safeTag(tag)});
+          // Prevent transforming into a safe tag again, if it was done before
+          data[opts.handle].push({ name: tag, slug: opts.normalize ? tag : safeTag(tag)});
 
           // Add each tag to our overall tagList and initialize array if it
           // doesn't exist.

package.json

@@ -23,6 +23,7 @@
   "homepage": "https://github.com/totocaster/metalsmith-tags",
   "devDependencies": {
     "assert-dir-equal": "1.0.1",
+    "debug": "^3.1.0",
     "handlebars": "*",
     "metalsmith": "1.x",
     "metalsmith-layouts": "1.x",

test/fixtures/expected/no-pagination/topics/skol.html

@@ -0,0 +1,10 @@
+<h1>Posts for skol</h1>
+<ul>
+	<li><small>07/02/2014</small> test page 2</li>
+
+	<li><small>07/01/2014</small> test json</li>
+
+	<li><small>02/03/2014</small> test</li>
+
+	<li><small>02/02/2014</small> about</li>
+</ul>

test/fixtures/expected/no-pagination/topics/this-is.html

@@ -1,4 +1,4 @@
-<h1>Posts for this is</h1>
+<h1>Posts for this-is</h1>
 <ul>
 	<li><small>02/03/2014</small> test</li>
 </ul>

test/fixtures/expected/pagination/topics/skol/2/index.html

@@ -0,0 +1,4 @@
+<h1>Posts for skol</h1>
+<ul>
+	<li><small>07/01/2014</small> test json</li>
+</ul><a href="topics/skol/index.html">Previous</a><a href="topics/skol/3/index.html">Next</a>

test/fixtures/expected/pagination/topics/skol/3/index.html

@@ -0,0 +1,4 @@
+<h1>Posts for skol</h1>
+<ul>
+	<li><small>02/03/2014</small> test</li>
+</ul><a href="topics/skol/2/index.html">Previous</a><a href="topics/skol/4/index.html">Next</a>

test/fixtures/expected/pagination/topics/skol/4/index.html

@@ -0,0 +1,4 @@
+<h1>Posts for skol</h1>
+<ul>
+	<li><small>02/02/2014</small> about</li>
+</ul><a href="topics/skol/3/index.html">Previous</a>

test/fixtures/expected/pagination/topics/skol/index.html

@@ -0,0 +1,4 @@
+<h1>Posts for skol</h1>
+<ul>
+	<li><small>07/02/2014</small> test page 2</li>
+</ul><a href="topics/skol/2/index.html">Next</a>

test/fixtures/expected/pagination/topics/this-is/index.html

@@ -1,4 +1,4 @@
-<h1>Posts for this is</h1>
+<h1>Posts for this-is</h1>
 <ul>
 	<li><small>02/03/2014</small> test</li>
 </ul>

test/fixtures/src/about.html

@@ -1,6 +1,6 @@
 ---
 title: about
-tags: hello
+tags: hello, skøl
 date: 2014-02-02T17:39:06.157Z
 ---
 

test/fixtures/src/index.html

@@ -1,6 +1,6 @@
 ---
 title: test
-tags: hello, world, this is, tag
+tags: hello, world, this is, tag, sköl
 date: 2014-03-02T17:39:06.157Z
 ---
 

test/fixtures/src/json.html

@@ -1,7 +1,7 @@
 ---
 {
   "title": "test json",
-  "tags": ["hello", "tag"],
+  "tags": ["hello", "tag", "skol"],
   "date": "2014-01-07T17:39:06.157Z"
 }
 ---

test/fixtures/src/page.html

@@ -1,6 +1,6 @@
 ---
 title: test page 2
-tags: hello, this, tag
+tags: hello, this, tag, skØl
 date: 2014-02-07T17:39:06.157Z
 ---
 

test/index.js

@@ -22,7 +22,13 @@ describe('metalsmith-tags', function() {
       }))
       .build(function(err,files){
         if (err) return done(err);
-        assert.equal(files['index.html'].tags.toString(),[{ name: 'hello', slug: 'hello'}, { name: 'world', slug: 'world'}, { name: 'this is', slug: 'this-is'}, { name: 'tag', slug: 'tag'}].toString());
+        assert.equal(files['index.html'].tags.toString(),[
+          { name: 'hello', slug: 'hello'},
+          { name: 'world', slug: 'world'},
+          { name: 'this is', slug: 'this-is'},
+          { name: 'tag', slug: 'tag'},
+          { name: 'skol', slug: 'skol'},
+        ].toString());
         done();
       });
   });
@@ -42,7 +48,55 @@ describe('metalsmith-tags', function() {
       .build(function(err, files){
         if (err) return done(err);
         var tagListKeys = Object.keys(tagList).sort();
-        assert.deepEqual(tagListKeys, ['hello', 'tag', 'this', 'this is', 'world']);
+        assert.deepEqual(tagListKeys, [
+          'hello',
+          'skol',
+          'skØl',
+          'sköl',
+          'skøl',
+          'tag',
+          'this',
+          'this is',
+          'world',
+        ]);
+        // Ensure every object in the metadata tags array is a data object.
+        tagListKeys.forEach(function(tagName) {
+          var tagPostsArray = tagList[tagName];
+          tagPostsArray.forEach(function(fileData) {
+            assert.equal(typeof fileData, 'object');
+            assert.ok(fileData.stats);
+            assert.ok(fileData.contents);
+            assert.ok(fileData.tags);
+          });
+        });
+        done();
+      });
+  });
+
+  it('should normalize special characters into their ascii equivalent using given slug function', function(done) {
+    var tagList;
+
+    Metalsmith('test/fixtures')
+      .use(tags({
+        handle: 'tags',
+        path: 'topics',
+        normalize: true,
+      }))
+      .use(function(files, metalsmith, done) {
+        tagList = metalsmith.metadata().tags;
+        done();
+      })
+      .build(function(err, files){
+        if (err) return done(err);
+        var tagListKeys = Object.keys(tagList).sort();
+        assert.deepEqual(tagListKeys, [
+          'hello',
+          'skol',
+          'tag',
+          'this',
+          'this-is',
+          'world',
+        ]);
         // Ensure every object in the metadata tags array is a data object.
         tagListKeys.forEach(function(tagName) {
           var tagPostsArray = tagList[tagName];
@@ -63,7 +117,7 @@ describe('metalsmith-tags', function() {
     Metalsmith('test/fixtures')
       .use(tags({
         handle: 'tags',
-        path: 'topics'
+        path: 'topics',
       }))
       .use(function(files, metalsmith, done) {
         tagList = metalsmith.metadata().tags;
@@ -72,13 +126,23 @@ describe('metalsmith-tags', function() {
       .build(function(err, files){
         if (err) return done(err);
         var tagListKeys = Object.keys(tagList).sort();
-        assert.deepEqual(tagListKeys, ['hello', 'tag', 'this', 'this is', 'world']);
+        assert.deepEqual(tagListKeys, [
+          'hello',
+          'skol',
+          'skØl',
+          'sköl',
+          'skøl',
+          'tag',
+          'this',
+          'this is',
+          'world',
+        ]);
         // Ensure every object in the metadata tags array is a data object.
         tagListKeys.forEach(function(tagName) {
           var tagPostsArray = tagList[tagName];
           assert.ok(tagList[tagName].urlSafe);
           assert.equal(typeof tagList[tagName].urlSafe, 'string');
-          assert.equal(slug(tagName), tagList[tagName].urlSafe);
+          assert.equal(slug(tagName, {mode: 'rfc3986'}), tagList[tagName].urlSafe);
         });
         done();
       });
@@ -116,6 +180,7 @@ describe('metalsmith-tags', function() {
         handle: 'tags',
         path: 'topics/:tag.html',
         layout: './tag.hbt',
+        normalize: true,
         sortBy: 'date',
         reverse: true
       }))
@@ -135,6 +200,7 @@ describe('metalsmith-tags', function() {
         pathPage: 'topics/:tag/:num/index.html',
         perPage: 1,
         layout: './tag.hbt',
+        normalize: true,
         sortBy: 'date',
         reverse: true
       }))
@@ -157,7 +223,13 @@ describe('metalsmith-tags', function() {
       }))
       .build(function(err, files) {
         if (err) return done(err);
-        assert.equal(files['index.html'].tags.toString(),[{ name: 'hello', slug: 'HELLO'}, { name: 'world', slug: 'WORLD'}, { name: 'this is', slug: 'THIS IS'}, { name: 'tag', slug: 'TAG'}].toString());
+        assert.equal(files['index.html'].tags.toString(),[
+          { name: 'hello', slug: 'HELLO'},
+          { name: 'world', slug: 'WORLD'},
+          { name: 'this is', slug: 'THIS IS'},
+          { name: 'tag', slug: 'TAG'},
+          { name: 'skol', slug: 'SKOL'},
+        ].toString());
         done();
       });
   })