From e5dedd7a4e48e213b9873426bcfbd586b6229719 Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Fri, 25 Aug 2017 21:22:09 -0400
Subject: [PATCH 01/23] [api] [refactor] Removed STDERR overload * Clears up
non-standard usage of STDERR * Remove legacy error handling logic * Now
using STDIO 3 for JSON messages * Adds option for service.jail argument
---
bin/binaries/micro-node | 51 ++++++++++++-------------------
lib/plugins/spawn/index.js | 30 +++++++++++++-----
lib/plugins/spawn/stderr/index.js | 42 +++++++++----------------
3 files changed, 56 insertions(+), 67 deletions(-)
diff --git a/bin/binaries/micro-node b/bin/binaries/micro-node
index fbab039..5a473ac 100755
--- a/bin/binaries/micro-node
+++ b/bin/binaries/micro-node
@@ -52,6 +52,8 @@ try {
}
+// pipe3 is an additional STDIO pipe we can use for HTTP response methods, debugging events, logging events, and other out of band comms
+var pipe3 = new net.Socket({ fd: 3 });
// babel support for es6 / es7
// the spawned child needs regenerator run-time here
@@ -84,7 +86,7 @@ output.addTrailers = function (headers) {
"headers": headers
}
};
- console.error(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message));
};
output.removeHeader = function (name) {
@@ -94,7 +96,7 @@ output.removeHeader = function (name) {
"name": name
}
};
- console.error(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message));
};
output.setHeader = function (name, value) {
@@ -105,7 +107,7 @@ output.setHeader = function (name, value) {
"value": value
}
};
- console.error(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message));
};
output.setTimeout = function (msecs, cb) {
@@ -116,7 +118,7 @@ output.setTimeout = function (msecs, cb) {
"msecs": msecs
}
};
- console.error(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message));
};
output.sendDate = function (value) {
@@ -126,7 +128,7 @@ output.sendDate = function (value) {
"value": value
}
};
- console.error(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message));
};
output.statusMessage = function (value) {
@@ -136,7 +138,7 @@ output.statusMessage = function (value) {
"value": value
}
};
- console.error(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message));
};
// Using Object.defineProperty
@@ -148,7 +150,7 @@ Object.defineProperty(output, 'statusCode', {
"value": value
}
};
- console.error(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message));
}
});
@@ -158,7 +160,7 @@ output.writeContinue = function () {
"payload": {
}
};
- console.error(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message));
};
output.writeHead = function (code, headers) {
@@ -169,12 +171,12 @@ output.writeHead = function (code, headers) {
"headers": headers
}
};
- console.error(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message));
};
// Capture any stream errors
-output.on('error', function(err){
- console.error(JSON.stringify({ type: "error", payload: { error: err.message, code: err.code } }));
+output.on('error', function (err) {
+ pipe3.write(JSON.stringify({ type: "error", payload: { error: err.message, code: err.code } }));
process.exit();
});
@@ -184,7 +186,7 @@ output.json = function json (data) {
if (typeof data !== 'undefined') {
console.log(JSON.stringify(data, true, 2));
}
- console.error(JSON.stringify({ type: "end" }));
+ pipe3.write(JSON.stringify({ type: "end" }));
process.exit();
};
@@ -193,14 +195,14 @@ output.end = function end (data) {
if (typeof data !== 'undefined') {
console.log(data);
}
- console.error(JSON.stringify({ type: "end" }));
+ pipe3.write(JSON.stringify({ type: "end" }));
process.exit();
};
// Custom errorHandler for `run-service` execution
function errorHandler (err) {
if (err) {
- console.error(JSON.stringify({ type: "error", payload: { error: err.message, code: err.code } }));
+ pipe3.write(JSON.stringify({ type: "error", payload: { error: err.message, code: err.code } }));
process.exit();
}
};
@@ -231,7 +233,6 @@ process.stdin.url = env.input.url;
process.stdin.connection = env.input.connection;
process.stdin.resource = { params: {}};
-
// Send logs to stderr as JSON message
var debug = function debug () {
var args = [];
@@ -241,7 +242,7 @@ var debug = function debug () {
if (args.length === 1) {
args = args[0];
}
- console.error(JSON.stringify({ type: "log", payload: { entry: args } }));
+ pipe3.write(JSON.stringify({ type: "log", payload: { entry: args } }));
return;
};
@@ -294,20 +295,6 @@ psr(process.stdin, output, function (req, res, fields) {
}
*/
-/* Remark: Removed for now, not needed yet?
-// pipe3 is an additional STDIO pipe we can use for HTTP request / response methods
-// it's currently being used to handle incoming request.end and request.close events
-var pipe3 = new net.Socket({ fd: 3 });
-pipe3.on('data', function (buf) {
- var str = buf.toString();
- if (str.search('input.close') !== -1) {
- process.stdin.emit('close');
- }
- if (str.search('input.end') !== -1) {
- process.stdin.emit('end');
- }
-});
-*/
/* TODO: add ability to proxy request parameters to middleware chain
var proxy = new Proxy(process.stdin, {
@@ -323,7 +310,7 @@ var proxy = new Proxy(process.stdin, {
//console.log("Setting property '" + name + "', initial value: " + value);
if (!(name in target)) {
// console.log("Setting non-existant property '" + name + "', initial value: " + value);
- console.error(JSON.stringify({ type: "setvar", payload: { key: name, value: value } }));
+ pipe3.write(JSON.stringify({ type: "setvar", payload: { key: name, value: value } }));
}
target[name] = value;
return true;
@@ -334,7 +321,7 @@ var proxy = new Proxy(process.stdin, {
/* TODO: add ability to send proxied params back to parent process
// sets key value on input stream ( useful for middleware processing later )
process.stdin.set = function (key, value) {
- console.error(JSON.stringify({ type: "setvar", payload: { key: key, value: value } }));
+ pipe3.write(JSON.stringify({ type: "setvar", payload: { key: key, value: value } }));
};
*/
diff --git a/lib/plugins/spawn/index.js b/lib/plugins/spawn/index.js
index 052e6e2..5d49c31 100644
--- a/lib/plugins/spawn/index.js
+++ b/lib/plugins/spawn/index.js
@@ -374,11 +374,17 @@ module['exports'] = function spawnService (service) {
targetBinary = path.normalize(targetBinary);
preprocessCommandLineArguments();
- // process.cwd(),
+ // jail option is used to add a pre-process command to the target binary
+ // in most expected cases this will be `chroot` or `nsjail` with arguments
+ if (service.jail) {
+ binaryArgs.unshift(targetBinary);
+ binaryArgs = service.jailArgs.concat(binaryArgs);
+ targetBinary = service.jail;
+ }
+
// console.log('spawning', targetBinary, 'in', _service.cwd, 'with', binaryArgs)
vm = spawn(targetBinary, binaryArgs, { stdio: ['pipe', 'pipe', 'pipe', 'pipe'], cwd: _service.cwd });
- /*
// used for additional communication outside of STDIN / STDOUT / STDERR
// pipe3 is additional HTTP req / res methods
var pipe3 = vm.stdio[3];
@@ -395,17 +401,24 @@ module['exports'] = function spawnService (service) {
// useful for pipe3, but not really being used
pipe3.on('end', function(){
+ // console.log('pipe3 ended')
status.pipe3ended = true;
});
pipe3.on('close', function(){
+ // console.log('pipe3 close')
status.pipe3ended = true;
});
+ pipe3.on('data', function (data) {
+ // console.log('pipe3 data', data.toString());
+ stderr.onData(data, status, log, output, input);
+ });
+
pipe3.on('exit', function(){
+ // console.log('pipe3 exit')
status.pipe3ended = true;
});
- */
finish();
@@ -451,7 +464,7 @@ module['exports'] = function spawnService (service) {
serviceCompletedTimer = clearTimeout(serviceCompletedTimer);
serviceCompleted = true;
- // console.log('endResponse', status, next)
+ // console.log('endResponse', status)
// Note: Only certain languages are currently capable of acting as middlewares
// For additional language support, we need an explcit event / API in each language for closing event over STDERR ( same as JS works )
@@ -472,8 +485,8 @@ module['exports'] = function spawnService (service) {
if (vm.stdout) {
vm.stdout.on('data', function (data) {
+ output.write(data);
if (!status.ended && output.finished !== true) {
- output.write(data);
}
});
}
@@ -538,8 +551,9 @@ module['exports'] = function spawnService (service) {
// stderr is overloaded here to be used as a one-way messaging device from child process to request
// this is used for doing such events as logging / setting http headers
vm.stderr.on('data', function (data) {
- // console.log('vm.stderr.data', data.toString())
- stderr.onData(data, status, log, output, input);
+ // console.log('vm.stderr.data', data.toString());
+ log(data.toString());
+ // stderr.onData(data, status, log, output, input);
});
}
@@ -582,7 +596,7 @@ module['exports'] = function spawnService (service) {
// we must wait for vm.stdout.end and vm.stderr to finish ( as to not lose data )
}
// if stdout has ended, we should be able to end the response if the vm exits
- if (status.stdoutEnded && !status.ended) {
+ if (status.stdoutEnded /*&& !status.ended*/) {
status.ended = true;
// Remark: The vm has exited ( and it's still not ended )
// The service has ended but the VM end event may not have fired, we should attempt to end response
diff --git a/lib/plugins/spawn/stderr/index.js b/lib/plugins/spawn/stderr/index.js
index 5b16264..72962fc 100644
--- a/lib/plugins/spawn/stderr/index.js
+++ b/lib/plugins/spawn/stderr/index.js
@@ -64,9 +64,7 @@ var handleMessage = stderr.handleMessage = function (message, status, debug, out
}
/*
-
TODO: implement req param setters for middlewares
-
if (message.type === "setvar") {
console.log('calling setvar...', message.payload)
input[message.payload.key] = message.payload.value;
@@ -76,33 +74,23 @@ var handleMessage = stderr.handleMessage = function (message, status, debug, out
// if the incoming message is an error
if (message.type === "error") {
- // let's do some custom behavior for MODULE_NOT_FOUND errors,
- // i.e. require('colors') when colors is not installed
status.erroring = true;
- // TODO: make module install a hookable event...
- if (message.payload.code === "MODULE_NOT_FOUND") {
- // TODO: Create module missing event that our package auto-install code can hook into
+
+ // we don't know what happened at this point, or how much more error information is coming
+ // let's just set a timer to end the request after a few moments
+ // this ensures that most ( if not the entire ) error stack gets sent to the client
+ if(!status.ended && output) {
+ // wait 200 ms to account for any errors to flush
output.write(message.payload.error);
- status.ended = true;
- return output.endResponse();
- } else {
- status.erroring = true;
- // the process is erroring and its not MODULE_NOT_FOUND.
- // we don't know what happened at this point, or how much more error information is coming
- // let's just set a timer to end the request after a few moments
- // this ensures that most ( if not the entire ) error stack gets sent to the client
- if(!status.ended && output) {
- // wait 200 ms to account for any errors to flush
- output.write(message.payload.error);
- status.ended = true;
- /*
- Removed with new middleware API
- Note: Should this be added back?
- setTimeout(function(){
- output.end();
- }, 200)
- */
- }
+ status.serviceEnded = true;
+ /*
+ Removed with new middleware API
+ Note: Should this be added back?
+ // status.ended = true;
+ setTimeout(function(){
+ output.end();
+ }, 200)
+ */
}
}
}
\ No newline at end of file
From 88d0db1b06b2512f643ce41881b456bed7beccc5 Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Fri, 25 Aug 2017 21:22:31 -0400
Subject: [PATCH 02/23] =?UTF-8?q?[config]=20[fix]=20Don=E2=80=99t=20use=20?=
=?UTF-8?q?SSL/HTTPS=20by=20default?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
config/index.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/config/index.js b/config/index.js
index 8684793..46e5a78 100644
--- a/config/index.js
+++ b/config/index.js
@@ -4,11 +4,11 @@ module.exports = {
http: {
port: 3000,
host: "0.0.0.0",
- https: true, // set to `true` to enable SSL server. cert, key, and ca will be required.
+ https: false, // set to `true` to enable SSL server. cert, key, and ca will be required.
key: fs.readFileSync(__dirname + "/ssl/server-key.pem").toString(),
cert: fs.readFileSync(__dirname + "/ssl/server-crt.pem").toString(),
ca: [fs.readFileSync(__dirname + '/ssl/ca-crt.pem').toString()],
- sslRequired: true, // redirects all http traffic to https, optional
+ sslRequired: false, // redirects all http traffic to https, optional
onlySSL: false // will only start https server with no unprotected http interface, optional
},
SERVICE_MAX_TIMEOUT: 10000,
From 5054f5daac693ef91d5adc04e176017197b26e5d Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Fri, 25 Aug 2017 23:51:36 -0400
Subject: [PATCH 03/23] [api] [fix] Add carriage return to pipe3 writes
---
bin/binaries/micro-node | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/bin/binaries/micro-node b/bin/binaries/micro-node
index 5a473ac..a058ef0 100755
--- a/bin/binaries/micro-node
+++ b/bin/binaries/micro-node
@@ -86,7 +86,7 @@ output.addTrailers = function (headers) {
"headers": headers
}
};
- pipe3.write(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message) + '\n');
};
output.removeHeader = function (name) {
@@ -96,7 +96,7 @@ output.removeHeader = function (name) {
"name": name
}
};
- pipe3.write(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message) + '\n');
};
output.setHeader = function (name, value) {
@@ -107,7 +107,7 @@ output.setHeader = function (name, value) {
"value": value
}
};
- pipe3.write(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message) + '\n');
};
output.setTimeout = function (msecs, cb) {
@@ -118,7 +118,7 @@ output.setTimeout = function (msecs, cb) {
"msecs": msecs
}
};
- pipe3.write(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message) + '\n');
};
output.sendDate = function (value) {
@@ -128,7 +128,7 @@ output.sendDate = function (value) {
"value": value
}
};
- pipe3.write(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message) + '\n');
};
output.statusMessage = function (value) {
@@ -138,7 +138,7 @@ output.statusMessage = function (value) {
"value": value
}
};
- pipe3.write(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message) + '\n');
};
// Using Object.defineProperty
@@ -150,7 +150,7 @@ Object.defineProperty(output, 'statusCode', {
"value": value
}
};
- pipe3.write(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message) + '\n');
}
});
@@ -160,7 +160,7 @@ output.writeContinue = function () {
"payload": {
}
};
- pipe3.write(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message) + '\n');
};
output.writeHead = function (code, headers) {
@@ -171,7 +171,7 @@ output.writeHead = function (code, headers) {
"headers": headers
}
};
- pipe3.write(JSON.stringify(message));
+ pipe3.write(JSON.stringify(message) + '\n');
};
// Capture any stream errors
From 109c7bb452d68ca9fb18afd30f830731222785b4 Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Wed, 30 Aug 2017 17:21:08 -0400
Subject: [PATCH 04/23] [test] Added invalid services tests * Used to test
error conditions and behaviors * Currently only testing errors with JS
services * Should cover most code paths for all langauges
---
examples/services/invalid/ReadMe.md | 1 -
.../invalid-services}/missing-exports.js | 0
.../invalid-services}/never-responds.js | 0
.../invalid-services}/require-error.js | 0
.../invalid-services}/syntax-error.js | 0
.../invalid-services}/writes-bad-headers.js | 0
test/invalid-service-tests.js | 103 ++++++++++++++++++
7 files changed, 103 insertions(+), 1 deletion(-)
delete mode 100644 examples/services/invalid/ReadMe.md
rename {examples/services/invalid => test/fixtures/invalid-services}/missing-exports.js (100%)
rename {examples/services/invalid => test/fixtures/invalid-services}/never-responds.js (100%)
rename {examples/services/invalid => test/fixtures/invalid-services}/require-error.js (100%)
rename {examples/services/invalid => test/fixtures/invalid-services}/syntax-error.js (100%)
rename {examples/services/invalid => test/fixtures/invalid-services}/writes-bad-headers.js (100%)
create mode 100644 test/invalid-service-tests.js
diff --git a/examples/services/invalid/ReadMe.md b/examples/services/invalid/ReadMe.md
deleted file mode 100644
index 4b36b4f..0000000
--- a/examples/services/invalid/ReadMe.md
+++ /dev/null
@@ -1 +0,0 @@
-Collection of invalid / erroring microservices. Useful for testing error conditions, can be put into unit tests.
\ No newline at end of file
diff --git a/examples/services/invalid/missing-exports.js b/test/fixtures/invalid-services/missing-exports.js
similarity index 100%
rename from examples/services/invalid/missing-exports.js
rename to test/fixtures/invalid-services/missing-exports.js
diff --git a/examples/services/invalid/never-responds.js b/test/fixtures/invalid-services/never-responds.js
similarity index 100%
rename from examples/services/invalid/never-responds.js
rename to test/fixtures/invalid-services/never-responds.js
diff --git a/examples/services/invalid/require-error.js b/test/fixtures/invalid-services/require-error.js
similarity index 100%
rename from examples/services/invalid/require-error.js
rename to test/fixtures/invalid-services/require-error.js
diff --git a/examples/services/invalid/syntax-error.js b/test/fixtures/invalid-services/syntax-error.js
similarity index 100%
rename from examples/services/invalid/syntax-error.js
rename to test/fixtures/invalid-services/syntax-error.js
diff --git a/examples/services/invalid/writes-bad-headers.js b/test/fixtures/invalid-services/writes-bad-headers.js
similarity index 100%
rename from examples/services/invalid/writes-bad-headers.js
rename to test/fixtures/invalid-services/writes-bad-headers.js
diff --git a/test/invalid-service-tests.js b/test/invalid-service-tests.js
new file mode 100644
index 0000000..a26b511
--- /dev/null
+++ b/test/invalid-service-tests.js
@@ -0,0 +1,103 @@
+// invalid-service-tests.js
+// attempts to run several user-defined services which may error in unique ways
+var test = require("tape");
+var express = require('express');
+var request = require('request');
+var fs = require('fs');
+
+var microcule, handlers = {}, app, server;
+
+test('attempt to require microcule', function (t) {
+ microcule = require('../');
+ t.equal(typeof microcule, 'object', 'microcule module required');
+ t.end();
+});
+
+test('attempt to create multiple invalid spawn handlers', function (t) {
+
+ handlers['missing-exports'] = microcule.plugins.spawn({
+ language: "javascript",
+ code: fs.readFileSync(__dirname + '/fixtures/invalid-services/missing-exports.js').toString()
+ });
+
+ handlers['never-responds'] = microcule.plugins.spawn({
+ language: "javascript",
+ code: fs.readFileSync(__dirname + '/fixtures/invalid-services/never-responds.js').toString(),
+ customTimeout: 1600
+ });
+
+ handlers['require-error'] = microcule.plugins.spawn({
+ language: "javascript",
+ code: fs.readFileSync(__dirname + '/fixtures/invalid-services/require-error.js').toString()
+ });
+
+ handlers['syntax-error'] = microcule.plugins.spawn({
+ language: "javascript",
+ code: fs.readFileSync(__dirname + '/fixtures/invalid-services/syntax-error.js').toString()
+ });
+
+ handlers['writes-bad-headers'] = microcule.plugins.spawn({
+ language: "javascript",
+ code: fs.readFileSync(__dirname + '/fixtures/invalid-services/writes-bad-headers.js').toString()
+ });
+
+ t.end();
+});
+
+test('attempt to start simple http server with multiple invalid services', function (t) {
+ app = express();
+
+ app.use('/missing-exports', handlers['missing-exports']);
+ app.use('/never-responds', handlers['never-responds']);
+ app.use('/require-error', handlers['require-error']);
+ app.use('/syntax-error', handlers['syntax-error']);
+ app.use('/writes-bad-headers', handlers['writes-bad-headers']);
+
+ // Required for non-js services ( or else response will not end )
+ app.use(function(req, res){
+ res.end();
+ });
+
+ server = app.listen(3000, function () {
+ t.end();
+ });
+});
+
+test('attempt to send request to javascript missing-exports service', function (t) {
+ request('http://localhost:3000/missing-exports', function (err, res, body) {
+ t.equal(res.statusCode, 500);
+ t.equal(body, 'service is undefined', 'got correct response');
+ t.end();
+ })
+});
+
+test('attempt to send request to javascript never-responds', function (t) {
+ request('http://localhost:3000/never-responds', function (err, res, body) {
+ t.equal(res.statusCode, 500);
+ t.equal(body.substr(0, 7), 'Timeout', 'got timeout response');
+ t.end();
+ })
+});
+
+test('attempt to send request to javascript require-error', function (t) {
+ request('http://localhost:3000/require-error', function (err, res, body) {
+ t.equal(res.statusCode, 500);
+ t.equal(body, 'a is not defined', 'got correct node error');
+ t.end();
+ })
+});
+
+test('attempt to send request to javascript writes-bad-headers', function (t) {
+ request('http://localhost:3000/writes-bad-headers', function (err, res, body) {
+ t.equal(res.statusCode, 200);
+ t.equal(body, '', 'got correct node error');
+ t.end();
+ })
+});
+
+test('attempt to end server', function (t) {
+ server.close(function(){
+ t.ok("server ended");
+ t.end();
+ });
+});
From bcd45361e8f5f8f084d57f8244e55cdcfccc0837 Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Wed, 30 Aug 2017 21:19:06 -0400
Subject: [PATCH 05/23] [test] Added tests for service as middlewares * Adds
tests for chaining services * Covers basic API and usage
---
test/service-as-middleware-tests.js | 75 +++++++++++++++++++++++++++++
1 file changed, 75 insertions(+)
create mode 100644 test/service-as-middleware-tests.js
diff --git a/test/service-as-middleware-tests.js b/test/service-as-middleware-tests.js
new file mode 100644
index 0000000..19c5cce
--- /dev/null
+++ b/test/service-as-middleware-tests.js
@@ -0,0 +1,75 @@
+// service-as-middleware-tests.js
+var test = require("tape");
+var express = require('express');
+var request = require('request');
+
+var microcule, handlers = {}, app, server;
+
+test('attempt to require microcule', function (t) {
+ microcule = require('../');
+ t.equal(typeof microcule, 'object', 'microcule module required');
+ t.end();
+});
+
+test('attempt to create a few chainable microservice spawn handlers', function (t) {
+
+ handlers['basicAuth'] = microcule.plugins.spawn({
+ language: "javascript",
+ code: function (req, res, next) {
+ var auth = require('basic-auth')
+ var credentials = auth(req)
+ if (!credentials || credentials.name !== 'admin' || credentials.pass !== 'password') {
+ //res.statusCode(401);
+ res.setHeader('WWW-Authenticate', 'Basic realm="examples"')
+ res.writeHead(401);
+ res.end('Access denied');
+ } else {
+ next();
+ }
+ }
+ });
+
+ handlers['write-a'] = microcule.plugins.spawn({
+ language: "bash",
+ code: 'echo "a"'
+ });
+ handlers['write-b'] = microcule.plugins.spawn({
+ language: "javascript",
+ code: function (req, res, next) {
+ res.write('b\n');
+ next(); // call next() to indicate this services is not going to explictly end the response
+ }
+ });
+ handlers['write-c'] = microcule.plugins.spawn({
+ language: "bash",
+ code: 'echo "c"'
+ });
+ t.end();
+});
+
+test('attempt to start simple http server with spawn handler', function (t) {
+ app = express();
+
+ app.use([handlers['basicAuth'], handlers['write-a'], handlers['write-b'], handlers['write-c']], function (req, res) {
+ console.log("No middlewares ended response, made it to end");
+ res.end('caught end')
+ });
+
+ server = app.listen(3000, function () {
+ t.end();
+ });
+});
+
+test('attempt to send simple http request to running microservice', function (t) {
+ request('http://admin:password@localhost:3000/', function (err, res, body) {
+ t.equal(body, 'a\nb\nc\ncaught end', 'got correct response');
+ t.end();
+ })
+});
+
+test('attempt to end server', function (t) {
+ server.close(function(){
+ t.ok("server ended");
+ t.end();
+ });
+});
\ No newline at end of file
From 43a49e2616358f90cfab5e1e5cd6f53d03a046b4 Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Wed, 30 Aug 2017 21:41:30 -0400
Subject: [PATCH 06/23] [api] Improved spawning API * Makes service homedir
configurable * Send 500 status code for timeout errors * Remove legacy
code for checking NPM registry * Fix small regression in vm.stdout handler
* All tests passing
---
lib/plugins/spawn/index.js | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/lib/plugins/spawn/index.js b/lib/plugins/spawn/index.js
index 5d49c31..430a641 100644
--- a/lib/plugins/spawn/index.js
+++ b/lib/plugins/spawn/index.js
@@ -62,6 +62,9 @@ module['exports'] = function spawnService (service) {
// _service.schema = service.schema;
_service.language = service.lang || service.language || "javascript";
+ // accept incoming home up to local project root
+ _service.home = service.home || __dirname + "/../../../"; // /* service.home || */
+
// legacy API
if (typeof service.source === 'string') {
_service.code = service.source;
@@ -369,7 +372,7 @@ module['exports'] = function spawnService (service) {
if (_service.bin) {
// TODO: Should we attempt to normalize the incoming bin path here? Probably no.
} else {
- targetBinary = __dirname + "/../../../bin/binaries/" + targetBinary;
+ targetBinary = _service.home + "/bin/binaries/" + targetBinary;
}
targetBinary = path.normalize(targetBinary);
preprocessCommandLineArguments();
@@ -436,6 +439,7 @@ module['exports'] = function spawnService (service) {
var serviceCompletedTimer = setTimeout(function(){
if (!serviceCompleted && !status.ended && !status.checkingRegistry) {
status.ended = true;
+ output.writeHeader(500);
output.write(config.messages.serviceExecutionTimeout(inSeconds));
//
// Note: At this stage, we don't know if the child process is going to exit,
@@ -485,8 +489,8 @@ module['exports'] = function spawnService (service) {
if (vm.stdout) {
vm.stdout.on('data', function (data) {
- output.write(data);
if (!status.ended && output.finished !== true) {
+ output.write(data);
}
});
}
@@ -496,7 +500,7 @@ module['exports'] = function spawnService (service) {
status.stdoutEnded = true;
status.pipe3ended = true;
// console.log('vm.stdout.end', status);
- if (!status.checkingRegistry && !status.ended && !status.erroring) {
+ if (!status.ended && !status.erroring) {
//status.ended = true;
// Remark: The vm's STDOUT has ended ( spawned service has completed ),
// Note: Removed Now using exit event only
From da4fccbcaf533170b118bb22703f6cd3fdde4381 Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Thu, 31 Aug 2017 11:44:19 -0400
Subject: [PATCH 07/23] [api] [minor] Set 500 status code on error message *
Better API semantic for users * Adds better code comments
---
lib/plugins/spawn/stderr/index.js | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/lib/plugins/spawn/stderr/index.js b/lib/plugins/spawn/stderr/index.js
index 72962fc..1e2b39e 100644
--- a/lib/plugins/spawn/stderr/index.js
+++ b/lib/plugins/spawn/stderr/index.js
@@ -75,18 +75,22 @@ var handleMessage = stderr.handleMessage = function (message, status, debug, out
// if the incoming message is an error
if (message.type === "error") {
status.erroring = true;
+ if (!status.ended && output) {
+ // could this 500 header try to be written twice? potential double header writing? needs more testing
+ output.writeHead(500);
- // we don't know what happened at this point, or how much more error information is coming
- // let's just set a timer to end the request after a few moments
- // this ensures that most ( if not the entire ) error stack gets sent to the client
- if(!status.ended && output) {
- // wait 200 ms to account for any errors to flush
+ // write the error message to the reponse
output.write(message.payload.error);
status.serviceEnded = true;
+
/*
Removed with new middleware API
Note: Should this be added back?
+ // we don't know what happened at this point, or how much more error information is coming
+ // let's just set a timer to end the request after a few moments
+ // this ensures that most ( if not the entire ) error stack gets sent to the client
// status.ended = true;
+ // wait 200 ms to account for any errors to flush
setTimeout(function(){
output.end();
}, 200)
From f908b214717ace7387701441c500bcab97d1522f Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Thu, 31 Aug 2017 12:46:48 -0400
Subject: [PATCH 08/23] [api] [refactor] Renamed stderr to fd3
---
lib/plugins/spawn/{stderr => fd3}/index.js | 0
lib/plugins/spawn/{stderr => fd3}/responseMethods.js | 0
2 files changed, 0 insertions(+), 0 deletions(-)
rename lib/plugins/spawn/{stderr => fd3}/index.js (100%)
rename lib/plugins/spawn/{stderr => fd3}/responseMethods.js (100%)
diff --git a/lib/plugins/spawn/stderr/index.js b/lib/plugins/spawn/fd3/index.js
similarity index 100%
rename from lib/plugins/spawn/stderr/index.js
rename to lib/plugins/spawn/fd3/index.js
diff --git a/lib/plugins/spawn/stderr/responseMethods.js b/lib/plugins/spawn/fd3/responseMethods.js
similarity index 100%
rename from lib/plugins/spawn/stderr/responseMethods.js
rename to lib/plugins/spawn/fd3/responseMethods.js
From 426280269e7a8f59f3ad600ac8f2d148f316b938 Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Thu, 31 Aug 2017 14:45:49 -0400
Subject: [PATCH 09/23] [api] [python] [refactor] Remove stderr overload *
Replaces with FD3 comm channel * Compatible with new microcule APIs
---
bin/binaries/lib/python/microcule/__init__.py | 26 ++++++++++++-------
1 file changed, 17 insertions(+), 9 deletions(-)
diff --git a/bin/binaries/lib/python/microcule/__init__.py b/bin/binaries/lib/python/microcule/__init__.py
index 0a44d8e..4fd902a 100644
--- a/bin/binaries/lib/python/microcule/__init__.py
+++ b/bin/binaries/lib/python/microcule/__init__.py
@@ -4,7 +4,10 @@
import traceback
import pkg_resources
import wsgiref.handlers
+import os
+# open incoming connection from fd3
+fd3 = os.fdopen(3, 'w+')
class FullMicroculeJSONFormatter(logging.Formatter):
def format(self, record):
@@ -78,15 +81,18 @@ def send_exception(self, info=None):
payload['error'] += error
else:
payload['error'] = error
- sys.stderr.write(json.dumps(res)+'\n')
- sys.stderr.flush()
- sys.stderr.write(json.dumps({'type': 'statusCode', 'payload': {'value': 500}})+'\n')
- sys.stderr.flush()
+
+
+ fd3.write(json.dumps(res)+'\n')
+ fd3.flush()
+ fd3.write(json.dumps({'type': 'statusCode', 'payload': {'value': 500}})+'\n')
+ fd3.flush()
+
if self.display:
sys.stdout.write(payload['error'].rstrip('\n')+'\n')
sys.stdout.flush()
- sys.stderr.write(json.dumps({'type': 'end'})+'\n')
- sys.stderr.flush()
+ fd3.write(json.dumps({'type': 'end'})+'\n')
+ fd3.flush()
class wsgi(wsgiref.handlers.CGIHandler):
@@ -100,9 +106,11 @@ def __init__(self, Hook=None):
def send_headers(self):
self.cleanup_headers()
self.headers_sent = True
- head = {'type': 'writeHead', 'payload': {'code': self.status, 'headers': dict(self.headers)}}
- sys.stderr.write(json.dumps(head)+'\n')
- sys.stderr.flush()
+ # remark: the status code needs to be sent to the parent process as an 3 digit integer value, not a string value with label
+ # todo: make parse int code for status more robust.
+ head = {'type': 'writeHead', 'payload': {'code': int(self.status[:3]), 'headers': dict(self.headers)}}
+ fd3.write(json.dumps(head)+'\n')
+ fd3.flush()
def add_cgi_vars(self):
#assert not Hook['isStreaming'], 'Streaming hooks not yet supported by WSGI gateway'
From 80315f98d2291223c32d9da684f11f3cd5e6d93e Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Thu, 31 Aug 2017 14:47:07 -0400
Subject: [PATCH 10/23] [api] [python] Do not require microcule in scripts *
User functions should not require client lib * Uses script injection to
wrap requests
---
bin/binaries/micro-python | 3 +++
examples/services/echo/echo-wsgi.py | 6 +-----
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/bin/binaries/micro-python b/bin/binaries/micro-python
index 0a57d83..7bb471b 100755
--- a/bin/binaries/micro-python
+++ b/bin/binaries/micro-python
@@ -16,6 +16,9 @@ def __prepare():
__prepare()
del __prepare
+# inject microcule WSGI handler code
+code = code + '\nimport microcule\nmicrocule.wsgi(Hook).run(app)\n'
+
# Execute the code as a string in ( this ) context
# print(code)
exec(code)
diff --git a/examples/services/echo/echo-wsgi.py b/examples/services/echo/echo-wsgi.py
index efc81c7..e12907a 100644
--- a/examples/services/echo/echo-wsgi.py
+++ b/examples/services/echo/echo-wsgi.py
@@ -1,6 +1,5 @@
import pprint
import logging
-import microcule
log = logging.getLogger('echo-py')
@@ -13,7 +12,4 @@ def app(environ, start_response):
res.append(pprint.pformat(Hook['req']['url']))
log.info('hello logs')
log.warn('%s', Hook['params'])
- return '\n'.join(res)
-
-if __name__ == '__main__':
- microcule.wsgi(Hook).run(app)
\ No newline at end of file
+ return '\n'.join(res)
\ No newline at end of file
From dca56598f4e1cef42a5c5d85a6713fadc96ac73e Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Thu, 31 Aug 2017 14:54:49 -0400
Subject: [PATCH 11/23] [minor] Update code comment
---
lib/plugins/spawn/fd3/index.js | 2 +-
lib/plugins/spawn/index.js | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/plugins/spawn/fd3/index.js b/lib/plugins/spawn/fd3/index.js
index 1e2b39e..b145f7b 100644
--- a/lib/plugins/spawn/fd3/index.js
+++ b/lib/plugins/spawn/fd3/index.js
@@ -1,4 +1,4 @@
-// stderr/index.js - handles all STDERR output from spawned service
+// fd3/index.js - handles all STDERR output from spawned service
// Remark: STDERR is currently overloaded to support JSON messages
// Overloading STDERR might not be the best design choice, but it does works and is leaves STDIO 3 and 4 open for future usage
diff --git a/lib/plugins/spawn/index.js b/lib/plugins/spawn/index.js
index 430a641..3f5ed66 100644
--- a/lib/plugins/spawn/index.js
+++ b/lib/plugins/spawn/index.js
@@ -1,5 +1,5 @@
var config = require('../../../config'); // defaults to included config, all config values can be overridden as options
-var stderr = require('./stderr');
+var stderr = require('./fd3');
var psr = require('parse-service-request');
var crypto = require('crypto');
var path = require('path');
From eee41eac6e20c64377e99d1f968108b6dbce21d8 Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Thu, 31 Aug 2017 15:51:44 -0400
Subject: [PATCH 12/23] [examples] Update python wsgi API
---
examples/services/hello-world/hello-wsgi.py | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/examples/services/hello-world/hello-wsgi.py b/examples/services/hello-world/hello-wsgi.py
index ec57e15..fe48da0 100644
--- a/examples/services/hello-world/hello-wsgi.py
+++ b/examples/services/hello-world/hello-wsgi.py
@@ -1,13 +1,9 @@
import pprint
import logging
-import microcule
log = logging.getLogger('echo-py')
def app(environ, start_response):
start_response('200 OK', [('content-type', 'text/plain')])
res = ["hello world"]
- return '\n'.join(res)
-
-if __name__ == '__main__':
- microcule.wsgi(Hook).run(app)
\ No newline at end of file
+ return '\n'.join(res)
\ No newline at end of file
From 93f9b871015c9f497ed8531a418234d36a87af6b Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Thu, 7 Sep 2017 15:39:00 -0400
Subject: [PATCH 13/23] [bin] [fix] [minor] Wrong scope
---
bin/microcule | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bin/microcule b/bin/microcule
index dfaf439..84096be 100755
--- a/bin/microcule
+++ b/bin/microcule
@@ -262,7 +262,7 @@ function startServer (_service) {
})(req, output, function(){
// if we made it here, it means no services called res.end()
// we should end the service ( or else it will hang forever )
- res.end();
+ output.end();
});
});
} else {
From 65b3fc08caf48c49e2d627945ed5fe110ff181f8 Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Thu, 7 Sep 2017 15:40:18 -0400
Subject: [PATCH 14/23] [bin] [fix] Remove wsgi code injection * Was breaking
backwards compatibility * Seems more pythonic to not inject code
---
bin/binaries/micro-python | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bin/binaries/micro-python b/bin/binaries/micro-python
index 7bb471b..b7b985a 100755
--- a/bin/binaries/micro-python
+++ b/bin/binaries/micro-python
@@ -17,7 +17,7 @@ __prepare()
del __prepare
# inject microcule WSGI handler code
-code = code + '\nimport microcule\nmicrocule.wsgi(Hook).run(app)\n'
+# code = code + '\nimport microcule\nmicrocule.wsgi(Hook).run(app)\n'
# Execute the code as a string in ( this ) context
# print(code)
From ebb5c6f29e1964e85c8c89fb6437503b4116c55d Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Thu, 7 Sep 2017 15:42:51 -0400
Subject: [PATCH 15/23] [api] [refactor] Removed viewPresenter * Simplifies
API to use only View * Default view is now just Mustache replacements *
No longer using server-side DOM * Could be refactored into generic
middleware
---
lib/viewPresenter.js | 97 +++++++++++++++++++++++++++++---------------
1 file changed, 65 insertions(+), 32 deletions(-)
diff --git a/lib/viewPresenter.js b/lib/viewPresenter.js
index 10b9a55..cfbf433 100644
--- a/lib/viewPresenter.js
+++ b/lib/viewPresenter.js
@@ -2,9 +2,8 @@
// Legacy v2.x.x
// The reason this file still exists, is that there seems to be an issue using the monkey patch express res / write middleware hack outside of express
// This was causing problem upstream in hook.io, because hook.io uses microcules middlewares directly ( not through .use() )
-// Ideally, we should be able to fix /lib/plugins/viewPresenter.js to accodomate for both usages and remove this file
+// Ideally, we should be able to fix /lib/plugins/viewPresenter.js to accomdate for both usages and remove this file
-var View = require('view').View;
var through = require('through2');
var streamBuffers = require('stream-buffers');
var Mustache = require("mustache");
@@ -27,36 +26,11 @@ module.exports = function viewPresenter (service, req, res, cb) {
headers: {}
};
- var _presenter = service.presenter || service.presenterSource || function (opts, cb) {
- // default presenter will load view into cheerio server-side dom
- // this is useful for escape / validation, it also makes the designer simplier
- var $ = this.$;
- cb(null, $.html());
- };
-
- // if presenter is a string, assume it's a Node.js module and attempt to compile it
- // this will somewhat safetly turn the string version of the function back into a regular function
- if (typeof _presenter === "string" && _presenter.length > 0) {
- var Module = module.constructor;
- _presenter = new Module();
- _presenter.paths = module.paths;
- var error = null;
- try {
- _presenter._compile(service.presenter, 'presenter-' + service.name);
- } catch (err) {
- error = err;
- }
- if (error !== null) {
- return cb(new Error('Could not compile presenter into module: ' + error.message));
- }
- _presenter = _presenter.exports;
- }
-
var output = through(function (chunk, enc, callback) {
+ console.log('getting output', chunk.toString())
hookOutput.write(chunk);
callback();
}, function () {
-
var content = hookOutput.getContents();
// Apply basic mustache replacements
// renamed themeSource -> viewSource
@@ -72,13 +46,62 @@ module.exports = function viewPresenter (service, req, res, cb) {
output: content.toString(),
debug: JSON.stringify(debugOutput, true, 2),
params: req.resource.instance,
- headers: _headers,
+ request: {
+ headers: req.headers
+ },
+ response: {
+ statusCode: res.statusCode,
+ body: content.toString()
+ },
+ request: {
+ headers: { 'fo': 'bar'}
+ },
schema: service.schema
}
});
+ res.end(strTheme);
+ return;
+
+
+ /*
+
+ viewPresenter code has been removed ( for now ). not many people were using it and it's implementation seems brittle
+ we should refactor this kind of post request template processing logic into plugins or post-middlewares
+
+ var View = require('view').View;
+
var _view = new View({ template: strTheme, presenter: _presenter });
+ var _presenter = service.presenter || service.presenterSource || function (opts, cb) {
+ // default presenter will load view into cheerio server-side dom
+ // this is useful for escape / validation, it also makes the designer simplier
+ var $ = this.$;
+ cb(null, $.html());
+ };
+
+ // if presenter is a string, assume it's a Node.js module and attempt to compile it
+ // this will somewhat safetly turn the string version of the function back into a regular function
+ if (typeof _presenter === "string" && _presenter.length > 0) {
+ console.log('ppp', _presenter)
+ var Module = module.constructor;
+ var __presenter = new Module();
+ __presenter.paths = module.paths;
+ var error = null;
+ try {
+ __presenter._compile(_presenter, 'presenter-' + service.name);
+ } catch (err) {
+ error = err;
+ }
+ // console.log("ERRR", _presenter, __presenter.exports, error)
+ if (error !== null) {
+ return cb(new Error('Could not compile presenter into module: ' + error.message));
+ }
+ _presenter = __presenter.exports;
+ }
+
+
+ console.log('NADE BEW VIEW')
if (typeof _view.present !== "function") {
return res.end('Unable to load View-Presenter for hook service. We made a mistake. Please contact support.');
}
@@ -91,6 +114,15 @@ module.exports = function viewPresenter (service, req, res, cb) {
var completed = false;
// replay headers?
// res.writeHead(_headers.code, _headers.headers);
+
+ var c = content.toString();
+ try {
+ c = JSON.parse(c);
+ } catch (err) {
+
+ }
+
+ console.log('pre doing the view', c)
try { // this will catch user run-time errors in the presenter
_view.present({
request: req,
@@ -98,16 +130,17 @@ module.exports = function viewPresenter (service, req, res, cb) {
service: service,
req: req,
res: res,
- output: content.toString(),
+ output: c,
debug: debugOutput,
instance: req.resource.instance,
params: req.resource.params,
headers: _headers
}, function(err, rendered){
completed = true;
+ console.log('made it to completed', err)
completedTimer = clearTimeout(completed);
try {
- // console.log('ending view', rendered)
+ console.log('ending view', rendered)
res.end(rendered);
} catch(err) {
res.end('Failed to call res.end ' + err.message);
@@ -122,7 +155,7 @@ module.exports = function viewPresenter (service, req, res, cb) {
// return cb(new Error('Error in Presenter code: ' + err.message));
res.end('Error in Presenter code: ' + err.message + '\n\n' + err.stack);
}
-
+ */
});
output.on('error', function (err) {
From 7fd30c320d5c9108b7bccfaf61c89d2f8b4f8be2 Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Fri, 8 Sep 2017 20:16:05 -0400
Subject: [PATCH 16/23] [dist] Added nyc code coverage tool
---
package.json | 24 ++++++++++++++++++++++--
1 file changed, 22 insertions(+), 2 deletions(-)
diff --git a/package.json b/package.json
index be9e886..5054399 100644
--- a/package.json
+++ b/package.json
@@ -46,5 +46,25 @@
"request": "^2.75.0",
"tap": "0.4.11",
"tape": "^4.6.0"
- }
-}
+ },
+ "nyc": {
+ "all": false,
+ "include": [
+ "lib/**/*.js"
+ ],
+ "exclude": [
+ "coverage",
+ "locales",
+ "modules",
+ "reports",
+ "test",
+ "node_modules"
+ ],
+ "reporter": [
+ "html",
+ "lcov",
+ "clover"
+ ],
+ "report-dir": "./reports/coverage"
+ }
+}
\ No newline at end of file
From ad9d47bdefb92f1f46109430b71f50786439661d Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Mon, 18 Sep 2017 20:08:33 -0400
Subject: [PATCH 17/23] [api] [fix] schema plugin should end response * Ends
request when schema invalid * Actually sending JSON back
---
lib/plugins/mschema.js | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/plugins/mschema.js b/lib/plugins/mschema.js
index cf2201f..b75e356 100644
--- a/lib/plugins/mschema.js
+++ b/lib/plugins/mschema.js
@@ -12,7 +12,9 @@ module.exports = function validateParamsMiddleware (schema) {
input.resource.instance = validate.instance;
next();
} else {
- next(new Error(JSON.stringify(validate.errors, true, 2)));
+ // if schema did not validate, end the response immediately
+ // send the error as JSON back to the client
+ return output.json(validate.errors)
}
}
}
\ No newline at end of file
From 53d26707b40639300ded24688665a244124df19c Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Mon, 18 Sep 2017 20:10:44 -0400
Subject: [PATCH 18/23] [test] Added basic plugins test * Only partial
coverage of plugins * Needs additional test cases
---
test/plugin-tests.js | 80 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 80 insertions(+)
create mode 100644 test/plugin-tests.js
diff --git a/test/plugin-tests.js b/test/plugin-tests.js
new file mode 100644
index 0000000..461ad49
--- /dev/null
+++ b/test/plugin-tests.js
@@ -0,0 +1,80 @@
+// basic-tests.js
+var test = require("tape");
+var express = require('express');
+var request = require('request');
+
+var microcule, handler, app, server;
+
+microcule = require('../');
+
+var logger = microcule.plugins.logger;
+var mschema = microcule.plugins.mschema;
+var bodyParser = microcule.plugins.bodyParser;
+var rateLimiter = microcule.plugins.rateLimiter
+var spawn = microcule.plugins.spawn;
+
+var handler = spawn({
+ language: "javascript",
+ code: function service (req, res) {
+ res.json(req.params);
+ }
+});
+
+test('attempt to start simple http server with some of the plugins spawn handler', function (t) {
+ app = express();
+ app.use(bodyParser());
+ app.use(logger());
+ app.use(mschema({
+ "hello": {
+ "type": "string",
+ "required": true
+ }
+ }));
+
+ app.use(rateLimiter({
+ maxLimit: 1000,
+ maxConcurrency: 2
+ }));
+
+ app.use(handler, function (req, res) {
+ res.end();
+ });
+ server = app.listen(3000, function () {
+ t.equal(typeof handler, "function", "started HTTP microservice server");
+ t.end();
+ });
+});
+
+test('attempt to send simple http request to running microservice', function (t) {
+ request({
+ uri: 'http://localhost:3000/',
+ method: "GET",
+ json: true
+ }, function (err, res, body) {
+ console.log("bbb", body)
+ t.equal(typeof body, "object", 'got correct response');
+ //t.equal(body, "b", "echo'd back property")
+ t.end();
+ })
+});
+
+test('attempt to send JSON data to running microservice', function (t) {
+ request({
+ uri: 'http://localhost:3000/',
+ method: "POST",
+ json: {
+ hello: "world"
+ }
+ }, function (err, res, body) {
+ t.equal(typeof body, "object", 'got correct response');
+ t.equal(body.hello, "world", "echo'd back property")
+ t.end();
+ })
+});
+
+test('attempt to end server', function (t) {
+ server.close(function(){
+ t.ok("server ended");
+ t.end();
+ });
+});
\ No newline at end of file
From f0075c40b6d6e9d1e3a7c585a912131e237a9e15 Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Mon, 18 Sep 2017 20:15:14 -0400
Subject: [PATCH 19/23] [api] [minor] Rename stderr to fd3 * Conforms to new
API semantics * Refactor already completed * Only variable name changes
---
lib/plugins/spawn/fd3/index.js | 14 +++++++-------
lib/plugins/spawn/index.js | 10 ++++------
2 files changed, 11 insertions(+), 13 deletions(-)
diff --git a/lib/plugins/spawn/fd3/index.js b/lib/plugins/spawn/fd3/index.js
index b145f7b..30575ea 100644
--- a/lib/plugins/spawn/fd3/index.js
+++ b/lib/plugins/spawn/fd3/index.js
@@ -3,11 +3,11 @@
// Overloading STDERR might not be the best design choice, but it does works and is leaves STDIO 3 and 4 open for future usage
var responseMethods = require('./responseMethods');
-var stderr = {};
-module['exports'] = stderr;
+var fd3 = {};
+module['exports'] = fd3;
-// processes incoming stderr buffer
-stderr.onData = function onStderrData (data, status, debug, output, input) {
+// processes incoming fd3 buffer
+fd3.onData = function onFD3Data (data, status, debug, output, input) {
var messages = data.toString();
// Remark: Ignore special case"\nmodule.js:333", which is module require error
@@ -22,7 +22,7 @@ stderr.onData = function onStderrData (data, status, debug, output, input) {
if (message.length === 0) {
return;
}
- // attempt to parse incoming stderr as JSON message
+ // attempt to parse incoming FD3 as JSON message
try {
message = JSON.parse(message.toString());
} catch (err) {
@@ -33,12 +33,12 @@ stderr.onData = function onStderrData (data, status, debug, output, input) {
});
};
-var handleMessage = stderr.handleMessage = function (message, status, debug, output, input) {
+var handleMessage = fd3.handleMessage = function (message, status, debug, output, input) {
var request = require('request');
/*
- stderr message types:
+ fd3 message types:
error: error event from vm, send error stack as plaintext to client.
log: console.log logging event, send log entry to logging system
diff --git a/lib/plugins/spawn/index.js b/lib/plugins/spawn/index.js
index 3f5ed66..105b003 100644
--- a/lib/plugins/spawn/index.js
+++ b/lib/plugins/spawn/index.js
@@ -1,5 +1,5 @@
var config = require('../../../config'); // defaults to included config, all config values can be overridden as options
-var stderr = require('./fd3');
+var fd3 = require('./fd3');
var psr = require('parse-service-request');
var crypto = require('crypto');
var path = require('path');
@@ -415,7 +415,7 @@ module['exports'] = function spawnService (service) {
pipe3.on('data', function (data) {
// console.log('pipe3 data', data.toString());
- stderr.onData(data, status, log, output, input);
+ fd3.onData(data, status, log, output, input);
});
pipe3.on('exit', function(){
@@ -549,15 +549,13 @@ module['exports'] = function spawnService (service) {
// this is bad, because we lose the error stack with the uncaught stream error
});
}
- // map endResponse fn for possible use in stderr.onData handler
+ // map endResponse fn for possible use in fd3.onData handler
output.endResponse = endResponse;
if (vm.stderr) {
- // stderr is overloaded here to be used as a one-way messaging device from child process to request
- // this is used for doing such events as logging / setting http headers
vm.stderr.on('data', function (data) {
// console.log('vm.stderr.data', data.toString());
log(data.toString());
- // stderr.onData(data, status, log, output, input);
+ // fd3.onData(data, status, log, output, input);
});
}
From 1f8931888225582e42de80fe0c2c9436695a0c37 Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Tue, 19 Sep 2017 14:02:42 -0400
Subject: [PATCH 20/23] [dist] [misc] Added more examples * Removes errant
comment * Update .gitignore with nyc folders
---
.gitignore | 2 ++
examples/express-jail-chroot.js | 22 ++++++++++++
examples/express-jail-nsjail.js | 45 ++++++++++++++++++++++++
examples/express-service-view.js | 44 +++++++++++++++++++++++
examples/services/echo/echoOld.js | 7 ++++
examples/services/hello-world/hello.lua | 1 +
test/fixtures/invalid-services/ReadMe.md | 1 +
test/plugin-tests.js | 1 -
8 files changed, 122 insertions(+), 1 deletion(-)
create mode 100644 examples/express-jail-chroot.js
create mode 100644 examples/express-jail-nsjail.js
create mode 100644 examples/express-service-view.js
create mode 100644 examples/services/echo/echoOld.js
create mode 100644 examples/services/hello-world/hello.lua
create mode 100644 test/fixtures/invalid-services/ReadMe.md
diff --git a/.gitignore b/.gitignore
index 08cb4a8..23c3380 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,7 @@
.DS_Store
node_modules/
+reports/
__pycache__/
*.pyc
*.pyo
+.nyc_output
\ No newline at end of file
diff --git a/examples/express-jail-chroot.js b/examples/express-jail-chroot.js
new file mode 100644
index 0000000..6177bd6
--- /dev/null
+++ b/examples/express-jail-chroot.js
@@ -0,0 +1,22 @@
+var microcule = require('../');
+var express = require('express');
+var app = express();
+
+var nodeService = function testService (opts) {
+ var res = opts.res;
+ console.log('logging to console');
+ res.end('ran service');
+};
+
+var handler = microcule.plugins.spawn({
+ code: nodeService,
+ language: "javascript",
+ jail: "chroot",
+ jailArgs: [ '/Users/worker']
+});
+
+app.use(handler);
+
+app.listen(3000, function () {
+ console.log('server started on port 3000');
+});
\ No newline at end of file
diff --git a/examples/express-jail-nsjail.js b/examples/express-jail-nsjail.js
new file mode 100644
index 0000000..19be8fd
--- /dev/null
+++ b/examples/express-jail-nsjail.js
@@ -0,0 +1,45 @@
+var microcule = require('../');
+var express = require('express');
+var app = express();
+
+var nodeService = function testService (opts) {
+ var res = opts.res;
+ console.log('logging to console');
+ setTimeout(function(){
+ res.end('ran service');
+ }, 10)
+};
+
+var bash = microcule.plugins.spawn({
+ code: 'ps -ax',
+ language: "bash",
+ jail: "nsjail",
+ jailArgs: [ '-Mo', '--chroot', '/var/chroot/', '--user', '99999', '--group', '99999']
+});
+
+var handler = microcule.plugins.spawn({
+ code: nodeService,
+ language: "javascript",
+ jail: "nsjail",
+ jailArgs: [ '-Mo', '--chroot', '/var/chroot/', '--user', '99999', '--group', '99999']
+});
+
+var ps = microcule.plugins.spawn({
+ bin: "/bin/ps",
+ argv: ['-ax' ]
+});
+
+var psJail = microcule.plugins.spawn({
+ bin: "/bin/ps",
+ argv: ['-ax' ],
+ jail: "nsjail",
+ jailArgs: [ '-Mo', '--chroot', '/var/chroot/', '--user', '99999', '--group', '99999']
+});
+
+app.use('/node', handler);
+app.use('/ps', ps);
+app.use('/psjail', psJail);
+
+app.listen(3000, function () {
+ console.log('server started on port 3000');
+});
\ No newline at end of file
diff --git a/examples/express-service-view.js b/examples/express-service-view.js
new file mode 100644
index 0000000..75ea481
--- /dev/null
+++ b/examples/express-service-view.js
@@ -0,0 +1,44 @@
+var microcule = require('../');
+var express = require('express');
+var app = express();
+
+var nodeService = function testService (opts) {
+ var res = opts.res;
+ console.log('logging to console');
+ res.end('Hello world!');
+};
+
+var handler = microcule.plugins.spawn({
+ code: nodeService,
+ language: "javascript"
+});
+
+//var view = microcule.plugins.viewPresenter({}, req, res, next);
+
+app.use('/view', function(req, res, next){
+ microcule.viewPresenter({
+ view: "Service outputs: {{hook.output}}"
+ }, req, res, function(err, req, output){
+ handler(req, output, next)
+ })
+});
+
+/* presenter API has been removed ( for now )
+app.use('/view-presenter', function(req, res, next){
+ microcule.viewPresenter({
+ view: "Service outputs: {{hook.output}} <div class='output'><div>",
+ presenter: function (opts, cb) {
+ opts.res.setHeader('Content-type', 'text/html');
+ var $ = this.$;
+ $('.output').html('presenters can use $ selectors');
+ cb(null, $.html());
+ }
+ }, req, res, function(err, req, output){
+ handler(req, output, next)
+ })
+});
+*/
+
+app.listen(3000, function () {
+ console.log('server started on port 3000');
+});
\ No newline at end of file
diff --git a/examples/services/echo/echoOld.js b/examples/services/echo/echoOld.js
new file mode 100644
index 0000000..434fb5f
--- /dev/null
+++ b/examples/services/echo/echoOld.js
@@ -0,0 +1,7 @@
+module.exports = function (hook) {
+ var req = hook.req, res = hook.res;
+ res.write('Hello, this is a JavaScript function.\n');
+ res.write('hook.params is populated with request parameters\n');
+ res.write(JSON.stringify(req.params, true, 2));
+ res.end('');
+};
\ No newline at end of file
diff --git a/examples/services/hello-world/hello.lua b/examples/services/hello-world/hello.lua
new file mode 100644
index 0000000..26b6aa3
--- /dev/null
+++ b/examples/services/hello-world/hello.lua
@@ -0,0 +1 @@
+print ("hello world")
\ No newline at end of file
diff --git a/test/fixtures/invalid-services/ReadMe.md b/test/fixtures/invalid-services/ReadMe.md
new file mode 100644
index 0000000..4b36b4f
--- /dev/null
+++ b/test/fixtures/invalid-services/ReadMe.md
@@ -0,0 +1 @@
+Collection of invalid / erroring microservices. Useful for testing error conditions, can be put into unit tests.
\ No newline at end of file
diff --git a/test/plugin-tests.js b/test/plugin-tests.js
index 461ad49..21fb888 100644
--- a/test/plugin-tests.js
+++ b/test/plugin-tests.js
@@ -51,7 +51,6 @@ test('attempt to send simple http request to running microservice', function (t)
method: "GET",
json: true
}, function (err, res, body) {
- console.log("bbb", body)
t.equal(typeof body, "object", 'got correct response');
//t.equal(body, "b", "echo'd back property")
t.end();
From 293ea83e9715155c50c47229ed4072d4b5c787df Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Tue, 19 Sep 2017 15:06:22 -0400
Subject: [PATCH 21/23] [api] [fix] End response on vm errors * Ending
response immediately for VM error * Do not attempt to run further
middlewares
---
lib/plugins/spawn/index.js | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/lib/plugins/spawn/index.js b/lib/plugins/spawn/index.js
index 105b003..709e3b8 100644
--- a/lib/plugins/spawn/index.js
+++ b/lib/plugins/spawn/index.js
@@ -480,9 +480,16 @@ module['exports'] = function spawnService (service) {
} else {
// Note: If we haven't explicitly been sent res.end() message,
// assume next was called and we have more middlewares to process
+
// Important: For non middleware enabled languages, we need to assume the last middleware calls res.end()
// If not, the next middleware ( outside of spawn chain ) is responsible for ending the request
- next();
+ if (status.vmError === true) {
+ // in the case of VM error, end the request here ( do not attempt to continue with middlewares )
+ // most likely we are in an error state due to missing binaries
+ output.end();
+ } else {
+ next();
+ }
}
};
@@ -560,7 +567,7 @@ module['exports'] = function spawnService (service) {
}
vm.on('error', function (err) {
- // console.log('vm.error' + err.message);
+ // console.log('vm.error' + err.message, status);
status.vmError = true;
// status.pipe3ended = true;
if (!status.ended) {
From 7205f629d280031ec42e243af19075382576144b Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Tue, 19 Sep 2017 15:14:12 -0400
Subject: [PATCH 22/23] [minor] Remove code comments
---
lib/plugins/compile/index.js | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/lib/plugins/compile/index.js b/lib/plugins/compile/index.js
index 4952010..a219c9e 100644
--- a/lib/plugins/compile/index.js
+++ b/lib/plugins/compile/index.js
@@ -14,7 +14,7 @@ module.exports = function compile (config) {
// if no releaseDir is specified, will default to ./microcule/release directory
config.releaseDir = config.releaseDir || path.resolve(__dirname + '/../../../release');
- console.log('using compile config', config)
+ // console.log('using compile config', config)
config.errorHandler = config.errorHandler || function (err, next) {
if (err.message === "Bad credentials") {
@@ -52,7 +52,7 @@ module.exports = function compile (config) {
// create shasum1 of service source code contents
var sha = shasum(_service.code);
_service.sha1 = sha;
- console.log('what is sha1', sha);
+ // console.log('what is sha1', sha);
return function compileMiddleware (req, res, next) {
req.service = _service;
@@ -70,10 +70,10 @@ module.exports = function compile (config) {
// TODO: check that sha matches as well as language? potential issue here with same content files, but diffirent target languages
// we will assume that if the binary file exists, it has completed it's build step and is ready to be run
fs.stat(binLocation, function (err, _stat) {
- console.log('back from stat', err, _stat);
+ // console.log('back from stat', err, _stat);
if (err) {
// could not find the file, attempt to compile it
- console.log('could not find file, so lets compile it'.red);
+ // console.log('could not find file, so lets compile it'.red);
// Remark: Implementes a mutually exclusive lock for compilation step ( do not attempt to compile twice )
// If service is compiling, simply return a 202 Accepted status until it's ready
@@ -81,14 +81,14 @@ module.exports = function compile (config) {
// check to see if provider indicates that service is building
provider.get('/builds/' + sha, function(err, result){
- console.log('back from provider get'.green, err, result)
+ // console.log('back from provider get'.green, err, result)
if (err) {
return res.end(err.message);
}
if (result === 0) {
// TODO: null result means no build status, create a new build status
provider.set('/builds/' + sha, { status: 'building' }, function (err, result){
- console.log('back from provider set', err, result)
+ // console.log('back from provider set', err, result)
if (err) {
return res.end(err.message);
}
@@ -112,7 +112,7 @@ module.exports = function compile (config) {
} else {
// we find the file, attempt to execute it
// if the stat returned ( a file ) then use that path instead of compiling a new one
- console.log('using compiled version of service', binLocation);
+ // console.log('using compiled version of service', binLocation);
var result = {
bin: binLocation,
buildDir: _service.buildDir + '/' + sha,
From aed57cd92b7fe916f3a56354dcc1689c9b07ee8e Mon Sep 17 00:00:00 2001
From: Marak <marak.squires@gmail.com>
Date: Tue, 19 Sep 2017 15:16:07 -0400
Subject: [PATCH 23/23] [test] [minor] Comment out language tests * Only for
Travis * Will work locally
---
test/all-languages-tests.js | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/test/all-languages-tests.js b/test/all-languages-tests.js
index 1b7c9fe..77f621f 100644
--- a/test/all-languages-tests.js
+++ b/test/all-languages-tests.js
@@ -12,7 +12,7 @@ microcule = require('../');
// Even as devDependencies they are too big
// TODO: update tests to use local examples folder for hello world?
// or should it also include microcule-examples echo tests?
-var languages = ['bash', 'gcc', /* 'babel', 'coffee-script', */ 'smalltalk', 'lua', 'go', 'javascript', 'perl', 'php', 'python', 'python3', 'ruby', 'rust', 'r', 'scheme', 'tcl'];
+var languages = ['bash', 'gcc', /* 'babel', 'coffee-script', */ 'smalltalk', /*'lua',*/ 'go', 'javascript', 'perl', 'php', 'python', /* 'python3', */ 'ruby', 'rust', 'r', 'scheme', 'tcl'];
test('attempt to require microcule-examples module', function (t) {
examples = require('microcule-examples');
@@ -27,6 +27,8 @@ test('check if examples are available for all languages', function (t) {
t.end();
});
+return;
+
//
// Remark: Travis-Ci is not able to easily support multiple language binaries in a single test
// There is a solution available at: https://github.com/travis-ci/travis-ci/issues/4090,
@@ -37,7 +39,6 @@ test('check if examples are available for all languages', function (t) {
// Note: The following tests should pass locally if you remove the return,
// and you have every single target language binary installed locally
//
-return;
test('attempt to start server with handlers for all languages', function (t) {
app = express();
@@ -48,7 +49,7 @@ test('attempt to start server with handlers for all languages', function (t) {
language: lang,
code: service.code
});
- app.use('/' + lang, handler, function(req, res){
+ app.use('/' + lang, handler, function (req, res) {
res.end();
});
t.equal(typeof handler, "function", "/" + lang + " HTTP endpoint added");