Skip to content

Commit

Permalink
fix #128 ; add done event& hooks; removed parserMgr and parser; upda…
Browse files Browse the repository at this point in the history
…ted readme; use lodash.set now
  • Loading branch information
Keyang committed Dec 30, 2016
1 parent 271901d commit e3801c2
Show file tree
Hide file tree
Showing 18 changed files with 602 additions and 301 deletions.
55 changes: 45 additions & 10 deletions libs/core/Converter.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ var defParam=require("./defParam");
var csvline=require("./csvline");
var fileline=require("./fileline");
var dataToCSVLine=require("./dataToCSVLine");
var fileLineToCSVLine=require("./fileLineToCSVLine");
var linesToJson=require("./linesToJson");
var CSVError=require("./CSVError");
var workerMgr=require("./workerMgr");
Expand Down Expand Up @@ -41,7 +42,8 @@ function Converter(params,options) {
this._csvTransf=null;
this.finalResult=[];
// this.on("data", function() {});
this.on("error", function() {});
this.on("error", emitDone(this));
this.on("end", emitDone(this));
this.initWorker();
process.nextTick(function(){
if (this._needEmitFinalResult === null){
Expand Down Expand Up @@ -69,6 +71,13 @@ function Converter(params,options) {
return this;
}
util.inherits(Converter, Transform);
function emitDone(conv){
return function(err){
process.nextTick(function(){
conv.emit('done',err)
})
}
}
Converter.prototype._transform = function(data, encoding, cb) {
if (this.param.toArrayString && this.started === false) {
this.started = true;
Expand All @@ -94,21 +103,38 @@ Converter.prototype.setPartialData=function(d){
}
Converter.prototype.processData=function(data,cb){
var params=this.param;
var fileLines=fileline(data,this.param)
if (this.preProcessLine && typeof this.preProcessLine === "function"){
fileLines.lines=this._preProcessLines(fileLines.lines,this.lastIndex)
}
if (!params._headers){ //header is not inited. init header
this.processHead(data,cb);
this.processHead(fileLines,cb);
}else{
if (params.workerNum<=1){
var lines=dataToCSVLine(data,params);
var lines=fileLineToCSVLine(fileLines,params);
this.setPartialData(lines.partial);
var jsonArr=linesToJson(lines.lines,params,this.recordNum);
this.processResult(jsonArr)
this.lastIndex+=jsonArr.length;
this.recordNum+=jsonArr.length;
cb();
}else{
this.workerProcess(data,cb);
this.workerProcess(fileLines,cb);
}
}
}
Converter.prototype._preProcessLines=function(lines,startIdx){
var rtn=[]
for (var i=0;i<lines.length;i++){
var result=this.preProcessLine(lines[i],startIdx+i+1)
if (typeof result ==="string"){
rtn.push(result)
}else{
rtn.push(lines[i])
this.emit("error",new Error("preProcessLine should return a string but got: "+JSON.stringify(result)))
}
}
return rtn
}
Converter.prototype.initWorker=function(){
var workerNum=this.param.workerNum-1;
Expand All @@ -117,14 +143,22 @@ Converter.prototype.initWorker=function(){
this.workerMgr.initWorker(workerNum,this.param);
}
}
Converter.prototype.preRawData=function(func){
this.preProcessRaw=func;
return this;
}
Converter.prototype.preFileLine=function(func){
this.preProcessLine=func;
return this;
}
/**
* workerpRocess does not support embeded multiple lines.
*/

Converter.prototype.workerProcess=function(data,cb){
Converter.prototype.workerProcess=function(fileLine,cb){
var self=this;
var line=fileline(data,this.param)
var eol=this.getEol(data)
var line=fileLine
var eol=this.getEol()
this.setPartialData(line.partial)
this.workerMgr.sendWorker(line.lines.join(eol)+eol,this.lastIndex,cb,function(results,lastIndex){
var cur=self.sequenceBuffer[0];
Expand Down Expand Up @@ -154,10 +188,10 @@ Converter.prototype.workerProcess=function(data,cb){
});
this.lastIndex+=line.lines.length;
}
Converter.prototype.processHead=function(data,cb){
Converter.prototype.processHead=function(fileLine,cb){
var params=this.param;
if (!params._headers){ //header is not inited. init header
var lines=dataToCSVLine(data,params);
var lines=fileLineToCSVLine(fileLine,params);
this.setPartialData(lines.partial);
if (params.noheader){
if (params.headers){
Expand Down Expand Up @@ -198,6 +232,7 @@ Converter.prototype.processResult=function(result){
// this.lastIndex+=result.length;
// cb();
}

Converter.prototype.emitResult=function(r){
var index=r.index;
var row=r.row;
Expand Down Expand Up @@ -320,7 +355,7 @@ Converter.prototype.getEol = function(data) {
this.param.eol=eol;
}

return this.param.eol;
return this.param.eol || eol;
};
Converter.prototype.fromFile = function(filePath, cb) {
var fs = require('fs');
Expand Down
8 changes: 7 additions & 1 deletion libs/core/defParam.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,13 @@ module.exports=function(params){
flatKeys: false, // Don't interpret dots and square brackets in header fields as nested object or array identifiers at all.
maxRowLength: 0, //the max character a csv row could have. 0 means infinite. If max number exceeded, parser will emit "error" of "row_exceed". if a possibly corrupted csv data provided, give it a number like 65535 so the parser wont consume memory. default: 0
checkColumn: false, //whether check column number of a row is the same as headers. If column number mismatched headers number, an error of "mismatched_column" will be emitted.. default: false
escape:'"' //escape char for quoted column
escape:'"', //escape char for quoted column

/**below are internal params */
_headerType:[],
_headerTitle:[],
_headerFlag:[],
_headers:null
};
if (!params){
params={};
Expand Down
15 changes: 15 additions & 0 deletions libs/core/fileLineToCSVLine.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
var csvline=require("./csvline");
/**
* Convert data chunk to csv lines with cols
* @param {[type]} data [description]
* @param {[type]} params [description]
* @return {[type]} {lines:[[col1,col2,col3]],partial:String}
*/
module.exports=function(fileLine,params){
var lines=fileLine.lines;
var csvLines=csvline(lines,params);
return {
lines:csvLines.lines,
partial:csvLines.partial+fileLine.partial
}
}
4 changes: 2 additions & 2 deletions libs/core/index.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
module.exports=constructor;
module.exports.Converter = require("./Converter.js");
module.exports.Parser = require("./parser.js");
module.exports.parserMgr = require("./parserMgr.js");
// module.exports.Parser = require("./parser.js");
// module.exports.parserMgr = require("./parserMgr.js");


function constructor(param,options){
Expand Down
Loading

0 comments on commit e3801c2

Please sign in to comment.