initial commit

Author: cemayan

.gitignore

@@ -0,0 +1,9 @@
+**/logs
+**/*.log
+node_modules
+.vscode
+.idea
+.DS_Store
+.cache-loader
+dist
+input

.npmignore

@@ -0,0 +1,14 @@
+.*.swp
+._*
+.DS_Store
+.git
+.hg
+.npmrc
+.lock-wscript
+.svn
+.wafpickle-*
+config.gypi
+CVS
+npm-debug.log
+src
+input

README.md

@@ -0,0 +1,105 @@
+# Async Chunk Reader
+
+This library allows you to read large amounts of data in chunks.
+
+## Install
+```console
+npm install --save async-chunk-reader
+```
+
+## API
+---
+
+**init(parameters : InitParameters)**
+
+> input : InitParameters
+
+- **chunk_size** :   String 
+- **input_file** : String | Stream
+- **encoding** : String
+
+**get()**
+
+> output : Async Iterator
+
+
+## Import
+
+### with require : 
+```javascript
+const reader = require('async-chunk-reader')
+```
+
+### with import : 
+```javascript
+import * as reader from "async-chunk-reader"
+```
+
+## Usage
+---
+
+### with path : 
+
+```javascript
+async function main(){
+
+    const x = await reader
+        .init({
+            chunk_size: 100000,
+            input_file: 'input/mobile_network_201805.csv.gz'
+        })
+        .get()
+
+    for await(let chunk of  x){
+        console.log(chunk.length)
+    }
+}
+
+main();
+
+```
+
+### with stream : 
+
+```javascript
+async function main(){
+
+    const x = await reader
+        .init({
+            input_file:  process.stdin
+        })
+        .get()
+
+    for await(let chunk of  x){
+        console.log(chunk.length)
+    }
+}
+
+main();
+
+```
+
+
+### with string : 
+
+```javascript
+async function main(){
+
+    const x = await reader
+        .init({
+            input_file: "Some string"
+        })
+        .get()
+
+    for await(let chunk of  x){
+        console.log(chunk.length)
+    }
+}
+
+main();
+
+```
+
+
+
+

package-lock.json

@@ -0,0 +1,35 @@
+{
+  "name": "async-chunk-reader",
+  "version": "0.1.1",
+  "description": "Async chunk reader",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc"
+  },
+  "homepage": "https://github.com/cemayan/async-chunk-reader",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/cemayan/async-chunk-reader.git"
+  },
+  "bugs": {
+    "url": "https://github.com/cemayan/async-chunk-reader/issues"
+  },
+  "keywords": [
+    "chunk",
+    "iterator",
+    "stream-reader",
+    "async-reader",
+    "stream"
+  ],
+  "license": "MIT",
+  "author": {
+    "name": "Cem Ayan",
+    "email": "cem_ayan@outlook.com",
+    "url": "http://github.com/cemayan"
+  },
+  "devDependencies": {
+    "typescript": "^3.2.2",
+    "@types/node": "^10.12.18"
+  }
+}

package.json

@@ -0,0 +1,95 @@
+import fs from "fs";
+import path from "path"
+import readline from "readline"
+import zlib from "zlib"
+import { Stream } from "stream";
+import events from "events";
+
+const gunzip = zlib.createGunzip();
+
+export interface InitParameters{
+    chunk_size : number,
+    encoding : string,
+    input_file : String|Stream,
+}
+
+export interface ReaderParameters{
+    input_file : String|Stream
+}
+
+
+events.EventEmitter.defaultMaxListeners = 1000;
+const default_global_settings = {chunk_size : 10000, encoding : "utf8" };
+
+export  function init(parameters : InitParameters){
+    default_global_settings.chunk_size = parameters.chunk_size;
+    default_global_settings.encoding = parameters.encoding;
+
+    return { get(){
+
+        let fileStream;
+
+        if(typeof(parameters.input_file) == "string"){
+            if(path.extname(<string>parameters.input_file)==".gz"){
+                fileStream = fs.createReadStream(<string>parameters.input_file).pipe(gunzip);
+            }  
+            else{
+                fileStream = fs.createReadStream(<string>parameters.input_file);
+            }
+
+            fileStream.on('error', (err) => {
+               throw err;
+            })
+                
+            return  readLineByStream(fileStream); 
+        }  
+        else if(parameters.input_file instanceof Stream ){
+            fileStream = parameters.input_file;
+
+            fileStream.on('error', (err) => {
+                throw err;
+             })
+
+            return  readLineByStream(fileStream); 
+        }
+        return this;
+
+      } 
+    }
+}
+
+
+function readLineByStream(fileStream) {
+
+    return {
+        [Symbol.iterator]() {
+            return this;
+        },
+        next() {
+            const rl = readline.createInterface({
+                input: fileStream,
+            });
+        
+            let async_itr =  rl[Symbol.asyncIterator]();    
+            let data =  readMoreData(async_itr);
+            return {value:data, done : false}         
+        }
+    }
+}
+
+async function  readMoreData(asyncIterator : AsyncIterator<any>):Promise<Array<any>>{
+
+    let count = 0;
+    let arr:Array<object> = [];
+
+    while(count < default_global_settings.chunk_size){
+        const resolved_obj  =  await asyncIterator.next();
+        if(!resolved_obj.done){
+            arr.push(resolved_obj);
+        }
+        count++;
+    }
+    return  arr;
+}
+
+

src/index.ts

@@ -0,0 +1,26 @@
+{
+    "version": "3.2.2",
+    "compilerOptions": {
+      "module": "commonjs",
+      "lib": ["es2015", "es2016","esnext.asynciterable"],
+      "target": "es6",
+      "types": ["node"],
+      "skipLibCheck": true,
+      "strictNullChecks": true,
+      "alwaysStrict": true,
+      "noImplicitThis": true,
+      "forceConsistentCasingInFileNames": true,
+      "noImplicitReturns": true,
+      "noUnusedLocals": true,
+      "experimentalDecorators": true,
+      "sourceMap": true,
+      "esModuleInterop": true,
+      "rootDir": "src",
+      "outDir": "dist"
+    },
+    "exclude": [
+      "**/node_modules",
+      //"**/*.d.ts"
+    ]
+  }
+