Merge pull request #20 from doug-martin/master

v0.1.1

Author: doug-martin

README.md

@@ -21,6 +21,11 @@ All methods accept the following `options`
 * `ignoreEmpty=false`: If you wish to ignore empty rows.
 * `delimiter=','`: If your data uses an alternate delimiter such as `;` or `\t`.
    * **NOTE** When specifying an alternate `delimiter` you may only pass in a single character delimeter
+* The following are options for parsing only.
+  * `trim=false`: If you want to trim all values parsed set to true.
+  * `rtrim=false`: If you want to right trim all values parsed set to true.
+  * `ltrim=false`: If you want to left trim all values parsed set to true.
+
 
 **events**
 
@@ -223,22 +228,22 @@ Create a readable stream to read data from.
 ```javascript
 var ws = fs.createWritableStream("my.csv");
 csv
-	.write([
-		["a", "b"],
-		["a1", "b1"],
-		["a2", "b2"]
-	], {headers: true})
-	.pipe(ws);
+   .write([
+       ["a", "b"],
+       ["a1", "b1"],
+       ["a2", "b2"]
+   ], {headers: true})
+   .pipe(ws);
 ```
 
 ```javascript
 var ws = fs.createWritableStream("my.csv");
 csv
-	.write([
-		{a: "a1", b: "b1"},
-		{a: "a2", b: "b2"}
-	], {headers: true})
-	.pipe(ws);
+   .write([
+       {a: "a1", b: "b1"},
+       {a: "a2", b: "b2"}
+   ], {headers: true})
+   .pipe(ws);
 ```
 
 **`writeToStream(stream,arr[, options])`**
@@ -247,20 +252,20 @@ Write an array of values to a `WritableStream`
 
 ```javascript
 csv
-	.writeToStream(fs.createWritableStream("my.csv"), [
-		["a", "b"],
-		["a1", "b1"],
-		["a2", "b2"]
-	], {headers: true});
+   .writeToStream(fs.createWritableStream("my.csv"), [
+       ["a", "b"],
+       ["a1", "b1"],
+       ["a2", "b2"]
+   ], {headers: true});
 ```
 
 ```javascript
 csv
-	.writeToStream(fs.createWritableStream("my.csv"), [
-		{a: "a1", b: "b1"},
-		{a: "a2", b: "b2"}
-	], {headers: true})
-	.pipe(ws);
+   .writeToStream(fs.createWritableStream("my.csv"), [
+       {a: "a1", b: "b1"},
+       {a: "a2", b: "b2"}
+   ], {headers: true})
+   .pipe(ws);
 ```
 
 **`writeToPath(arr[, options])`**
@@ -269,41 +274,41 @@ Write an array of values to the specified path
 
 ```javascript
 csv
-	.writeToPath("my.csv", [
-		["a", "b"],
-		["a1", "b1"],
-		["a2", "b2"]
-	], {headers: true})
-	.on("finish", function(){
-		console.log("done!");
-	});
+   .writeToPath("my.csv", [
+       ["a", "b"],
+       ["a1", "b1"],
+       ["a2", "b2"]
+   ], {headers: true})
+   .on("finish", function(){
+       console.log("done!");
+   });
 ```
 
 ```javascript
 csv
-	.writeToStream("my.csv", [
-		{a: "a1", b: "b1"},
-		{a: "a2", b: "b2"}
-	], {headers: true})
-	.on("finish", function(){
-		console.log("done!");
-	});
+   .writeToStream("my.csv", [
+       {a: "a1", b: "b1"},
+       {a: "a2", b: "b2"}
+   ], {headers: true})
+   .on("finish", function(){
+      console.log("done!");
+   });
 ```
 
 **`writeToString(arr[, options])`**
 
 ```javascript
 csv.writeToString([
-	["a", "b"],
-	["a1", "b1"],
-	["a2", "b2"]
+   ["a", "b"],
+   ["a1", "b1"],
+   ["a2", "b2"]
 ], {headers: true}); //"a,b\na1,b1\na2,b2\n"
 ```
 
 ```javascript
 csv.writeToString([
-	{a: "a1", b: "b1"},
-	{a: "a2", b: "b2"}
+   {a: "a1", b: "b1"},
+   {a: "a2", b: "b2"}
 ], {headers: true}); //"a,b\na1,b1\na2,b2\n"
 ```
 

benchmark/benchmark.js

@@ -1,7 +1,7 @@
 var fastCsv = require("../lib"),
     csv = require("csv"),
     path = require("path"),
-    COUNT = 1000000,
+    COUNT = 20000,
     TEST_FILE = path.resolve(__dirname, "./assets/" + COUNT + ".csv");
 
 

docs/index.html

@@ -185,6 +185,12 @@ <h3>Parsing</h3>
 <li><strong>NOTE</strong> When specifying an alternate <code>delimiter</code> you may only pass in a single character delimeter</li>
 </ul>
 </li>
+<li>The following are options for parsing only.<ul>
+<li><code>trim=false</code>: If you want to trim all values parsed set to true.</li>
+<li><code>rtrim=false</code>: If you want to right trim all values parsed set to true.</li>
+<li><code>ltrim=false</code>: If you want to left trim all values parsed set to true.</li>
+</ul>
+</li>
 </ul>
 <p><strong>events</strong></p>
 <p><code>parse-error</code>: Emitted if there was an error parsing a row.
@@ -332,61 +338,61 @@ <h3>Formatting</h3>
 <p>Create a readable stream to read data from.</p>
 <pre class='prettyprint linenums lang-js'><code class="lang-javascript">var ws = fs.createWritableStream(&quot;my.csv&quot;);
 csv
-    .write([
-        [&quot;a&quot;, &quot;b&quot;],
-        [&quot;a1&quot;, &quot;b1&quot;],
-        [&quot;a2&quot;, &quot;b2&quot;]
-    ], {headers: true})
-    .pipe(ws);</code></pre>
+   .write([
+       [&quot;a&quot;, &quot;b&quot;],
+       [&quot;a1&quot;, &quot;b1&quot;],
+       [&quot;a2&quot;, &quot;b2&quot;]
+   ], {headers: true})
+   .pipe(ws);</code></pre>
 <pre class='prettyprint linenums lang-js'><code class="lang-javascript">var ws = fs.createWritableStream(&quot;my.csv&quot;);
 csv
-    .write([
-        {a: &quot;a1&quot;, b: &quot;b1&quot;},
-        {a: &quot;a2&quot;, b: &quot;b2&quot;}
-    ], {headers: true})
-    .pipe(ws);</code></pre>
+   .write([
+       {a: &quot;a1&quot;, b: &quot;b1&quot;},
+       {a: &quot;a2&quot;, b: &quot;b2&quot;}
+   ], {headers: true})
+   .pipe(ws);</code></pre>
 <p><strong><code>writeToStream(stream,arr[, options])</code></strong></p>
 <p>Write an array of values to a <code>WritableStream</code></p>
 <pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv
-    .writeToStream(fs.createWritableStream(&quot;my.csv&quot;), [
-        [&quot;a&quot;, &quot;b&quot;],
-        [&quot;a1&quot;, &quot;b1&quot;],
-        [&quot;a2&quot;, &quot;b2&quot;]
-    ], {headers: true});</code></pre>
+   .writeToStream(fs.createWritableStream(&quot;my.csv&quot;), [
+       [&quot;a&quot;, &quot;b&quot;],
+       [&quot;a1&quot;, &quot;b1&quot;],
+       [&quot;a2&quot;, &quot;b2&quot;]
+   ], {headers: true});</code></pre>
 <pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv
-    .writeToStream(fs.createWritableStream(&quot;my.csv&quot;), [
-        {a: &quot;a1&quot;, b: &quot;b1&quot;},
-        {a: &quot;a2&quot;, b: &quot;b2&quot;}
-    ], {headers: true})
-    .pipe(ws);</code></pre>
+   .writeToStream(fs.createWritableStream(&quot;my.csv&quot;), [
+       {a: &quot;a1&quot;, b: &quot;b1&quot;},
+       {a: &quot;a2&quot;, b: &quot;b2&quot;}
+   ], {headers: true})
+   .pipe(ws);</code></pre>
 <p><strong><code>writeToPath(arr[, options])</code></strong></p>
 <p>Write an array of values to the specified path</p>
 <pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv
-    .writeToPath(&quot;my.csv&quot;, [
-        [&quot;a&quot;, &quot;b&quot;],
-        [&quot;a1&quot;, &quot;b1&quot;],
-        [&quot;a2&quot;, &quot;b2&quot;]
-    ], {headers: true})
-    .on(&quot;finish&quot;, function(){
-        console.log(&quot;done!&quot;);
-    });</code></pre>
+   .writeToPath(&quot;my.csv&quot;, [
+       [&quot;a&quot;, &quot;b&quot;],
+       [&quot;a1&quot;, &quot;b1&quot;],
+       [&quot;a2&quot;, &quot;b2&quot;]
+   ], {headers: true})
+   .on(&quot;finish&quot;, function(){
+       console.log(&quot;done!&quot;);
+   });</code></pre>
 <pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv
-    .writeToStream(&quot;my.csv&quot;, [
-        {a: &quot;a1&quot;, b: &quot;b1&quot;},
-        {a: &quot;a2&quot;, b: &quot;b2&quot;}
-    ], {headers: true})
-    .on(&quot;finish&quot;, function(){
-        console.log(&quot;done!&quot;);
-    });</code></pre>
+   .writeToStream(&quot;my.csv&quot;, [
+       {a: &quot;a1&quot;, b: &quot;b1&quot;},
+       {a: &quot;a2&quot;, b: &quot;b2&quot;}
+   ], {headers: true})
+   .on(&quot;finish&quot;, function(){
+      console.log(&quot;done!&quot;);
+   });</code></pre>
 <p><strong><code>writeToString(arr[, options])</code></strong></p>
 <pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv.writeToString([
-    [&quot;a&quot;, &quot;b&quot;],
-    [&quot;a1&quot;, &quot;b1&quot;],
-    [&quot;a2&quot;, &quot;b2&quot;]
+   [&quot;a&quot;, &quot;b&quot;],
+   [&quot;a1&quot;, &quot;b1&quot;],
+   [&quot;a2&quot;, &quot;b2&quot;]
 ], {headers: true}); //&quot;a,b\na1,b1\na2,b2\n&quot;</code></pre>
 <pre class='prettyprint linenums lang-js'><code class="lang-javascript">csv.writeToString([
-    {a: &quot;a1&quot;, b: &quot;b1&quot;},
-    {a: &quot;a2&quot;, b: &quot;b2&quot;}
+   {a: &quot;a1&quot;, b: &quot;b1&quot;},
+   {a: &quot;a2&quot;, b: &quot;b2&quot;}
 ], {headers: true}); //&quot;a,b\na1,b1\na2,b2\n&quot;</code></pre>
 <h2>Benchmarks</h2>
 <p><code>Parsing 20000 records AVG over 3 runs</code></p>

lib/index.js

@@ -22,6 +22,11 @@
  * * `ignoreEmpty=false`: If you wish to ignore empty rows.
  * * `delimiter=','`: If your data uses an alternate delimiter such as `;` or `\t`.
  *    * **NOTE** When specifying an alternate `delimiter` you may only pass in a single character delimeter
+ * * The following are options for parsing only.
+ *   * `trim=false`: If you want to trim all values parsed set to true.
+ *   * `rtrim=false`: If you want to right trim all values parsed set to true.
+ *   * `ltrim=false`: If you want to left trim all values parsed set to true.
+ *
  *
  * **events**
  *

lib/parser.js

@@ -1,11 +1,31 @@
-var SINGLE_QUOTE = "'",
+var extended = require("./extended"),
+    trim = extended.trim,
+    trimLeft = extended.trimLeft,
+    trimRight = extended.trimRight,
+    SINGLE_QUOTE = "'",
     DOUBLE_QUOTE = '"';
 
-function createParser(delimiter) {
-
-    var VALUE_REGEXP = new RegExp("([^" + delimiter + "'\"\\s\\\\]*(?:\\s+[^" + delimiter + "'\"\\s\\\\]+)*)"),
+function createParser(options) {
+    options = options || {};
+    var delimiter = options.delimiter || ",",
+        doLtrim = options.ltrim || false,
+        doRtrim = options.rtrim || false,
+        doTrim = options.trim || false,
+        VALUE_REGEXP = new RegExp("([^" + delimiter + "'\"\\s\\\\]*(?:\\s+[^" + delimiter + "'\"\\s\\\\]+)*)"),
         SEARCH_REGEXP = new RegExp("[^\\\\]" + delimiter),
-        ESCAPE_CHAR = "\\";
+        ESCAPE_CHAR = "\\",
+        WHITE_SPACE = /\s/;
+
+    function formatItem(item) {
+        if (doTrim) {
+            item = trim(item);
+        } else if (doLtrim) {
+            item = trimLeft(item);
+        } else if (doRtrim) {
+            item = trimRight(item);
+        }
+        return item;
+    }
 
     function getTokensBetween(str, start, items, cursor) {
         var depth = 0, ret = [];
@@ -35,10 +55,10 @@ function createParser(delimiter) {
                 ++cursor;
             }
         }
-        if (++cursor < str.length && str[cursor].search(delimiter) !== 0) {
+        if (++cursor < str.length && getNextToken(str, cursor).token.search(delimiter) !== 0) {
             throw new Error("Invalid row " + str);
         }
-        items.push(ret.join(""));
+        items.push(formatItem(ret.join("")));
         return ++cursor;
     }
 
@@ -52,7 +72,7 @@ function createParser(delimiter) {
                 nextIndex = searchStr.length - 1;
             }
         }
-        items.push(searchStr.substr(0, nextIndex + 1));
+        items.push(formatItem(searchStr.substr(0, nextIndex + 1)));
         return cursor + (nextIndex + 2);
     }
 
@@ -69,20 +89,39 @@ function createParser(delimiter) {
         return findNextToken(line, items, cursor);
     }
 
+    function getNextToken(line, cursor) {
+        var l = line.length, ret, token;
+        do {
+            token = line[cursor];
+            if (token === delimiter || !WHITE_SPACE.test(token)) {
+                ret = token;
+            } else {
+                token = null;
+            }
+
+        } while (!token && cursor++ < l);
+        if (!token) {
+            throw new Error("Invalid row " + line);
+        }
+        return {token: token, cursor: cursor};
+    }
+
     return function parseLine(line) {
-        var i = 0, l = line.length, items = [], token;
+        var i = 0, l = line.length, items = [], token, nextToken;
         while (i < l) {
-            token = line[i];
+            nextToken = getNextToken(line, i);
+            token = nextToken.token;
             if (token === delimiter) {
                 items.push("");
                 i++;
             } else if (token === SINGLE_QUOTE) {
-                i = parseSingleQuoteItem(line, items, i);
+                i = parseSingleQuoteItem(line, items, nextToken.cursor);
             } else if (token === DOUBLE_QUOTE) {
-                i = parseDoubleQuoteItem(line, items, i);
+                i = parseDoubleQuoteItem(line, items, nextToken.cursor);
             } else {
                 i = parseItem(line, items, i);
             }
+
         }
         return items;
     };

lib/parser_stream.js

@@ -27,7 +27,8 @@ function ParserStream(options) {
     } else {
         delimiter = DEFAULT_DELIMITER;
     }
-    this.parser = createParser(delimiter);
+    options.delimiter = delimiter;
+    this.parser = createParser(options);
     this._headers = options.headers;
     this._ignoreEmpty = options.ignoreEmpty;
     return this;

package.json

@@ -1,6 +1,6 @@
 {
     "name": "fast-csv",
-    "version": "0.1.0",
+    "version": "0.1.1",
     "description": "CSV parser for node.js",
     "main": "index.js",
     "scripts": {