caffe loader and single shot multibox detector example with ms coco

Author: justadudewhohacks

cc/core/Mat.cc

@@ -21,8 +21,10 @@ NAN_MODULE_INIT(Mat::Init) {
 	Nan::SetAccessor(ctor->InstanceTemplate(), Nan::New("dims").ToLocalChecked(), Mat::GetDims);
 	Nan::SetAccessor(ctor->InstanceTemplate(), Nan::New("depth").ToLocalChecked(), Mat::GetDepth);
 	Nan::SetAccessor(ctor->InstanceTemplate(), Nan::New("empty").ToLocalChecked(), Mat::GetIsEmpty);
+	Nan::SetAccessor(ctor->InstanceTemplate(), Nan::New("sizes").ToLocalChecked(), Mat::GetSizes);
 
 	Nan::SetMethod(ctor, "eye", Eye);
+	Nan::SetPrototypeMethod(ctor, "flattenFloat", FlattenFloat);
 
 	Nan::SetPrototypeMethod(ctor, "at", At);
 	Nan::SetPrototypeMethod(ctor, "atRaw", AtRaw);
@@ -220,6 +222,28 @@ NAN_METHOD(Mat::New) {
 	FF_RETURN(info.Holder());
 }
 
+NAN_METHOD(Mat::Eye) {
+	FF_METHOD_CONTEXT("Mat::Eye");
+	FF_ARG_INT(0, int rows);
+	FF_ARG_INT(1, int cols);
+	FF_ARG_INT(2, int type);
+	FF_OBJ jsEyeMat = FF_NEW_INSTANCE(Mat::constructor);
+	FF_UNWRAP_MAT_AND_GET(jsEyeMat) = cv::Mat::eye(cv::Size(cols, rows), type);
+	FF_RETURN(jsEyeMat);
+}
+
+// squash multidimensional Mat into 2D Mat
+// TODO: figure out how to deal with multidim Mats
+NAN_METHOD(Mat::FlattenFloat) {
+	FF_METHOD_CONTEXT("Mat::To2DFloat");
+	FF_ARG_INT(0, int rows);
+	FF_ARG_INT(1, int cols);
+
+	cv::Mat matSelf = FF_UNWRAP_MAT_AND_GET(info.This());
+	cv::Mat mat2D(rows, cols, CV_32F, matSelf.ptr<float>());
+	FF_RETURN(Mat::Converter::wrap(mat2D));
+}
+
 NAN_METHOD(Mat::At) {
 	FF_METHOD_CONTEXT("Mat::At");
 	cv::Mat matSelf = FF_UNWRAP_MAT_AND_GET(info.This());
@@ -251,16 +275,6 @@ NAN_METHOD(Mat::At) {
 	FF_RETURN(jsVal);
 }
 
-NAN_METHOD(Mat::Eye) {
-	FF_METHOD_CONTEXT("Mat::Eye");
-	FF_ARG_INT(0, int rows);
-	FF_ARG_INT(1, int cols);
-	FF_ARG_INT(2, int type);
-	FF_OBJ jsEyeMat = FF_NEW_INSTANCE(Mat::constructor);
-	FF_UNWRAP_MAT_AND_GET(jsEyeMat) = cv::Mat::eye(cv::Size(cols, rows), type);
-	FF_RETURN(jsEyeMat);
-}
-
 NAN_METHOD(Mat::AtRaw) {
 	FF_METHOD_CONTEXT("Mat::AtRaw");
 	cv::Mat matSelf = FF_UNWRAP_MAT_AND_GET(info.This());

cc/core/Mat.h

@@ -16,6 +16,7 @@ class Mat : public Nan::ObjectWrap {
 	static NAN_METHOD(New);
 
 	static NAN_METHOD(Eye);
+	static NAN_METHOD(FlattenFloat);
 
 	static FF_GETTER(Mat, GetRows, mat.rows);
 	static FF_GETTER(Mat, GetCols, mat.cols);
@@ -24,6 +25,14 @@ class Mat : public Nan::ObjectWrap {
 	static FF_GETTER(Mat, GetDims, mat.dims);
 	static FF_GETTER(Mat, GetDepth, mat.depth());
 	static FF_GETTER(Mat, GetIsEmpty, mat.empty());
+	static NAN_GETTER(GetSizes) {
+		cv::Mat m = Converter::unwrap(info.This());
+		std::vector<int> sizes;
+		for (int s = 0; s < m.dims; s++) {
+			sizes.push_back(m.size[s]);
+		}
+		info.GetReturnValue().Set(IntArrayConverter::wrap(sizes));
+	};
 
 	static NAN_METHOD(At);
 	static NAN_METHOD(AtRaw);

cc/modules/dnn/dnn.cc

@@ -15,6 +15,8 @@ NAN_MODULE_INIT(Dnn::Init) {
 
 	Nan::SetMethod(target, "readNetFromTensorflow", ReadNetFromTensorflow);
 	Nan::SetMethod(target, "readNetFromTensorflowAsync", ReadNetFromTensorflowAsync);
+	Nan::SetMethod(target, "readNetFromCaffe", ReadNetFromCaffe);
+	Nan::SetMethod(target, "readNetFromCaffeAsync", ReadNetFromCaffeAsync);
 	Nan::SetMethod(target, "blobFromImage", BlobFromImage);
 	Nan::SetMethod(target, "blobFromImageAsync", BlobFromImageAsync);
 	Nan::SetMethod(target, "blobFromImages", BlobFromImages);
@@ -57,6 +59,51 @@ NAN_METHOD(Dnn::ReadNetFromTensorflowAsync) {
 	FF_WORKER_ASYNC("Dnn::ReadNetFromTensorflowAsync", ReadNetFromTensorflowWorker, worker);
 }
 
+
+struct Dnn::ReadNetFromCaffeWorker : public SimpleWorker {
+public:
+	std::string prototxt;
+	std::string modelFile = "";
+
+	cv::dnn::Net net;
+
+	const char* execute() {
+		net = cv::dnn::readNetFromCaffe(prototxt, modelFile);
+		if (net.empty()) {
+			return std::string("failed to prototxt: " + prototxt + ", modelFile: " + modelFile).data();
+		}
+		return "";
+	}
+
+	v8::Local<v8::Value> getReturnValue() {
+		return Net::Converter::wrap(net);
+	}
+
+	bool unwrapRequiredArgs(Nan::NAN_METHOD_ARGS_TYPE info) {
+		return (
+			StringConverter::arg(0, &prototxt, info)
+		);
+	}
+
+	bool unwrapOptionalArgs(Nan::NAN_METHOD_ARGS_TYPE info) {
+		return (
+			StringConverter::optArg(1, &modelFile, info)
+		);
+	}
+};
+
+NAN_METHOD(Dnn::ReadNetFromCaffe) {
+	ReadNetFromCaffeWorker worker;
+	FF_WORKER_SYNC("Dnn::ReadNetFromCaffe", worker);
+	info.GetReturnValue().Set(worker.getReturnValue());
+}
+
+NAN_METHOD(Dnn::ReadNetFromCaffeAsync) {
+	ReadNetFromCaffeWorker worker;
+	FF_WORKER_ASYNC("Dnn::ReadNetFromCaffeAsync", ReadNetFromCaffeWorker, worker);
+}
+
+
 struct Dnn::BlobFromImageWorker : public SimpleWorker {
 public:
 	bool isSingleImage;

cc/modules/dnn/dnn.h

@@ -12,15 +12,17 @@ class Dnn {
 	static NAN_METHOD(ReadNetFromTensorflow);
 	static NAN_METHOD(ReadNetFromTensorflowAsync);
 
-#if CV_VERSION_MINOR > 2
+	struct ReadNetFromCaffeWorker;
+	static NAN_METHOD(ReadNetFromCaffe);
+	static NAN_METHOD(ReadNetFromCaffeAsync);
+
 	struct BlobFromImageWorker;
 	static NAN_METHOD(BlobFromImage);
 	static NAN_METHOD(BlobFromImageAsync);
 
 	struct BlobFromImagesWorker;
 	static NAN_METHOD(BlobFromImages);
 	static NAN_METHOD(BlobFromImagesAsync);
-#endif
 };
 
 #endif

data/cars.jpeg

@@ -9,7 +9,8 @@ Mat {
   channels: Number,
   depth: Number,
   dims: Number,
-  empty: Boolean
+  empty: Boolean,
+  sizes: [Number]
 }
 ```
 

data/dishes.jpg

@@ -14,6 +14,21 @@ Net : readNetFromTensorflow(String modelPath)
 readNetFromTensorflowAsync(String modelPath, callback(Error err, Net net))
 ```
 
+<a name="readNetFromCaffe"></a>
+
+### readNetFromCaffe
+``` javascript
+Net : readNetFromCaffe(String prototxt, String modelPath = '')
+```
+
+<a name="readNetFromCaffeAsync"></a>
+
+### readNetFromCaffeAsync
+``` javascript
+readNetFromCaffeAsync(String prototxt, callback(Error err, Net net))
+readNetFromCaffeAsync(String prototxt, String modelPath, callback(Error err, Net net))
+```
+
 <a name="blobFromImage"></a>
 
 ### blobFromImage

doc/core/Mat.md

@@ -0,0 +1,83 @@
+module.exports = [
+  'background',
+  'person',
+  'bicycle',
+  'car',
+  'motorcycle',
+  'airplane',
+  'bus',
+  'train',
+  'truck',
+  'boat',
+  'traffic light',
+  'fire hydrant',
+  'stop sign',
+  'parking meter',
+  'bench',
+  'bird',
+  'cat',
+  'dog',
+  'horse',
+  'sheep',
+  'cow',
+  'elephant',
+  'bear',
+  'zebra',
+  'giraffe',
+  'backpack',
+  'umbrella',
+  'handbag',
+  'tie',
+  'suitcase',
+  'frisbee',
+  'skis',
+  'snowboard',
+  'sports ball',
+  'kite',
+  'baseball bat',
+  'baseball glove',
+  'skateboard',
+  'surfboard',
+  'tennis racket',
+  'bottle',
+  'wine glass',
+  'cup',
+  'fork',
+  'knife',
+  'spoon',
+  'bowl',
+  'banana',
+  'apple',
+  'sandwich',
+  'orange',
+  'broccoli',
+  'carrot',
+  'hot dog',
+  'pizza',
+  'donut',
+  'cake',
+  'chair',
+  'couch',
+  'potted plant',
+  'bed',
+  'dining table',
+  'toilet',
+  'tv',
+  'laptop',
+  'mouse',
+  'remote',
+  'keyboard',
+  'cell phone',
+  'microwave',
+  'oven',
+  'toaster',
+  'sink',
+  'refrigerator',
+  'book',
+  'clock',
+  'vase',
+  'scissors',
+  'teddy bear',
+  'hair drier',
+  'toothbrush'
+];

doc/dnn/dnn.md

@@ -0,0 +1,136 @@
+const cv = require('../');
+const fs = require('fs');
+const path = require('path');
+const classNames = require('./dnnCocoClassNames');
+
+if (!cv.xmodules.dnn) {
+  return console.log('exiting: opencv4nodejs compiled without dnn module');
+}
+
+// replace with path where you unzipped inception model
+const ssdcocoModelPath = '../data/dnn/coco-SSD_300x300'
+
+const prototxt = path.resolve(ssdcocoModelPath, 'deploy.prototxt');
+const modelFile = path.resolve(ssdcocoModelPath, 'VGG_coco_SSD_300x300_iter_400000.caffemodel');
+
+if (!fs.existsSync(prototxt) || !fs.existsSync(modelFile)) {
+  console.log('exiting: could not find ssdcoco model');
+  console.log('download the model from: https://drive.google.com/file/d/0BzKzrI_SkD1_dUY1Ml9GRTFpUWc/view');
+  return;
+}
+
+// initialize ssdcoco model from prototxt and modelFile
+const net = cv.readNetFromCaffe(prototxt, modelFile);
+
+const classifyImg = (img) => {
+  const white = new cv.Vec(255, 255, 255);
+  // ssdcoco model works with 300 x 300 images
+  const imgResized = img.resize(300, 300);
+
+  // network accepts blobs as input
+  const inputBlob = cv.blobFromImage(imgResized);
+  net.setInput(inputBlob);
+
+  // forward pass input through entire network, will return
+  // classification result as 1x1xNxM Mat
+  let outputBlob = net.forward();
+  // extract NxM Mat
+  outputBlob = outputBlob.flattenFloat(outputBlob.sizes[2], outputBlob.sizes[3]);
+
+  const results = Array(outputBlob.rows).fill(0)
+    .map((res, i) => {
+      const className = classNames[outputBlob.at(i, 1)];
+      const confidence = outputBlob.at(i, 2);
+      const topLeft = new cv.Point(
+        outputBlob.at(i, 3) * img.cols,
+        outputBlob.at(i, 6) * img.rows
+      );
+      const bottomRight = new cv.Point(
+        outputBlob.at(i, 5) * img.cols,
+        outputBlob.at(i, 4) * img.rows
+      );
+
+      return ({
+        className,
+        confidence,
+        topLeft,
+        bottomRight
+      })
+    });
+
+    return results;
+};
+
+const makeDrawClassDetections = predictions => (drawImg, className, getColor, thickness = 2) => {
+  predictions
+    .filter(p => p.className === className)
+    .forEach(p => {
+      drawImg.drawRectangle(
+        p.topLeft,
+        p.bottomRight,
+        getColor(),
+        { thickness }
+      );
+    });
+  return drawImg;
+};
+
+const runDetectDishesExample = () => {
+  const img = cv.imread('../data/dishes.jpg');
+  const minConfidence = 0.2;
+
+  const predictions = classifyImg(img).filter(res => res.confidence > minConfidence);
+
+  const drawClassDetections = makeDrawClassDetections(predictions);
+
+  const classColors = {
+    fork: new cv.Vec(0, 255, 0),
+    bowl: new cv.Vec(255, 0, 0),
+    'wine glass': new cv.Vec(0, 0, 255),
+    cup: new cv.Vec(0, 255, 255)
+  };
+
+  const legendLeftTop = new cv.Point(550, 20);
+  Object.keys(classColors).forEach((className, i) => {
+    const color = classColors[className];
+
+    // const draw legend
+    const offY = i * 30;
+    img.drawCircle(
+      legendLeftTop.add(new cv.Point(0, offY)),
+      3,
+      color,
+      { thickness: 4 }
+    );
+    img.putText(
+      className,
+      legendLeftTop.add(new cv.Point(20, offY + 8)),
+      cv.FONT_ITALIC,
+      0.8,
+      color,
+      { thickness: 2 }
+    );
+
+    // draw detections
+    drawClassDetections(img, className, () => color);
+  });
+
+  cv.imshowWait('img', img);
+};
+
+const runDetectPeopleExample = () => {
+  const img = cv.imread('../data/cars.jpeg');
+  const minConfidence = 0.4;
+
+  const predictions = classifyImg(img).filter(res => res.confidence > minConfidence);
+
+  const drawClassDetections = makeDrawClassDetections(predictions);
+
+  const getRandomColor = () => new cv.Vec(Math.random() * 255, Math.random() * 255, 255);
+
+  drawClassDetections(img, 'car', getRandomColor);
+  cv.imshowWait('img', img);
+};
+
+runDetectDishesExample();
+runDetectPeopleExample();

examples/dnnCocoClassNames.js

@@ -7,7 +7,7 @@ if (!cv.xmodules.dnn) {
 }
 
 // replace with path where you unzipped inception model
-const inceptionModelPath = '../data/dnn'
+const inceptionModelPath = '../data/dnn/tf-inception'
 
 const modelFile = path.resolve(inceptionModelPath, 'tensorflow_inception_graph.pb');
 const classNamesFile = path.resolve(inceptionModelPath, 'imagenet_comp_graph_label_strings.txt');