Add offline module

.gitignore

@@ -14,6 +14,7 @@ training/lstm/data/t
 dist
 /dist_examples
 .yarn
+src/offline/models
 
 website/translated_docs
 website/build/

package.json

@@ -12,7 +12,8 @@
     "test": "jest --config tests/jest.config.js",
     "upload-examples": "node scripts/uploadExamples.js",
     "update-p5-version": "node scripts/updateP5Version.js",
-    "update-readme": "node scripts/updateReadme.js"
+    "update-readme": "node scripts/updateReadme.js",
+    "fetch-model-files": "node scripts/fetchModelFiles.js"
   },
   "files": [
     "dist"
@@ -45,6 +46,7 @@
     "postinstall-postinstall": "^2.1.0",
     "prettier": "2.8.8",
     "rimraf": "^5.0.5",
+    "tar": "^7.4.3",
     "terser-webpack-plugin": "^5.3.10",
     "webpack": "^5.76.1",
     "webpack-cli": "^5.0.1",

scripts/fetchModelFiles.js

@@ -0,0 +1,175 @@
+const fs = require("fs");
+const tar = require("tar");
+const path = require("path");
+const rimraf = require("rimraf");
+const { Readable } = require("stream");
+
+const outputDir = "src/offline/models";
+const tmpDir = "src/offline/models/tmp";
+
+/**
+ * URLs of the models to fetch.
+ */
+const modelURLs = {
+  HandPose: {
+    detectorLite:
+      "https://www.kaggle.com/api/v1/models/mediapipe/handpose-3d/tfJs/detector-lite/1/download",
+    landmarkLite:
+      "https://www.kaggle.com/api/v1/models/mediapipe/handpose-3d/tfJs/landmark-lite/1/download",
+  },
+};
+
+/**
+ * Fetch a compressed model from a given URL and save it to a given output directory.
+ * @param {string} url - The URL of the .tar.gz file to fetch.
+ * @param {string} outputDir - The directory path to save the fetched file.
+ */
+async function fetchCompressedModel(url, outputDir) {
+  if (!fs.existsSync(outputDir)) {
+    fs.mkdirSync(outputDir, { recursive: true });
+  }
+
+  const res = await fetch(url);
+
+  const fileStream = fs.createWriteStream(
+    path.resolve(outputDir, "model.tar.gz"),
+    { flags: "w" }
+  );
+  Readable.fromWeb(res.body).pipe(fileStream);
+
+  return new Promise((resolve, reject) => {
+    fileStream.on("finish", resolve);
+    fileStream.on("error", reject);
+  });
+}
+
+/**
+ * Unzips a compressed file to a given output.
+ * @param {string} filePath
+ * @param {string} outputDir
+ */
+function unzip(filePath, outputDir) {
+  if (!fs.existsSync(outputDir)) {
+    fs.mkdirSync(outputDir, { recursive: true });
+  }
+
+  tar.x({
+    file: filePath,
+    cwd: outputDir,
+    gzip: true,
+    sync: true,
+  });
+}
+
+/**
+ * Convert a JSON file to a JS file.
+ * Pad the JSON content with `export default` so it could be imported as a module.
+ * @param {string} jsonPath - Path to the JSON file to convert to JS.
+ * @param {string} outputDir - The directory path to save the JS representation of the JSON file.
+ * @param {string} outputName - The name of the output JS file.
+ */
+function jsonToJs(jsonPath, outputDir, outputName) {
+  if (!fs.existsSync(outputDir)) {
+    fs.mkdirSync(outputDir, { recursive: true });
+  }
+  // Read the model.json file
+  const content = fs.readFileSync(jsonPath, "utf-8");
+  // Pad the content with export default
+  const padded = `const modelJson=${content}; export default modelJson;`;
+  // Write the content to a js file
+  fs.writeFileSync(path.resolve(outputDir, outputName), padded);
+}
+
+/**
+ * Create a JS file from a binary file.
+ * The binary file is converted to a Uint8Array so it could be written in a JS file.
+ * Add `export default` to the Uint8Array so it could be imported as a module.
+ * @param {string} binaryPath - Path to the binary file to convert to JS.
+ * @param {string} outputDir - The directory path to save the JS representation of the binary file.
+ * @param {string} outputName - The name of the output JS file.
+ */
+function binaryToJs(binaryPath, outputDir, outputName) {
+  if (!fs.existsSync(outputDir)) {
+    fs.mkdirSync(outputDir, { recursive: true });
+  }
+  // Read the model.bin file
+  const content = fs.readFileSync(binaryPath);
+  // Convert the binary file to a Uint8Array so it could be written in a js file
+  const arrayBuffer = content.buffer.slice(
+    content.byteOffset,
+    content.byteOffset + content.byteLength
+  );
+  const uint8Array = new Uint8Array(arrayBuffer);
+  // Write the Uint8Array to a js file
+  fs.writeFileSync(
+    path.resolve(outputDir, outputName),
+    `const modelBin=new Uint8Array([${uint8Array}]); export default modelBin;`
+  );
+}
+/**
+ * Download the model binary files and convert them to JS files.
+ * The function will make a JS file for the model.json and model.bin files.
+ * The files are converted to JS so they could be bundled into the ml5.js library.
+ *
+ * @param {string} url - The URL of the .tar.gz file to fetch.
+ * @param {string} outputDir - The directory path to save the JS representation of the model.json and model.bin files.
+ * @returns
+ */
+async function makeJsModelFiles(url, ml5ModelName, modelName) {
+  // temporary directory for processing the model files
+  const modelTmpDir = path.resolve(tmpDir, ml5ModelName, modelName);
+  const modelDir = path.resolve(outputDir, ml5ModelName, modelName);
+
+  await fetchCompressedModel(url, modelTmpDir);
+  unzip(path.resolve(modelTmpDir, "model.tar.gz"), modelTmpDir);
+
+  // Convert model.json to model.json.js
+  jsonToJs(path.resolve(modelTmpDir, "model.json"), modelDir, "model.json.js");
+
+  // Convert all model.bin files to model.bin.js
+  const binFiles = fs
+    .readdirSync(modelTmpDir)
+    .filter((file) => file.endsWith(".bin"));
+
+  binFiles.forEach((binFile) => {
+    binaryToJs(path.resolve(modelTmpDir, binFile), modelDir, `${binFile}.js`);
+  });
+}
+
+/**
+ * Remove the temporary directory.
+ */
+function cleanup() {
+  rimraf.sync(tmpDir);
+}
+
+/**
+ * Check if the model files (js) already
+ * @param {string} ml5Model
+ * @param {string} model
+ * @returns
+ */
+function modelJsExists(ml5Model, model) {
+  const hasDir = fs.existsSync(path.resolve(outputDir, ml5Model, model));
+  if (!hasDir) return false;
+
+  const files = fs.readdirSync(path.resolve(outputDir, ml5Model, model));
+  const hasJson = files.includes("model.json.js");
+  const hasBin = files.some((file) => file.endsWith(".bin.js"));
+  return hasJson && hasBin;
+}
+
+/**
+ * Point of entry to the script.
+ */
+async function main() {
+  for (ml5Model in modelURLs) {
+    for (model in modelURLs[ml5Model]) {
+      if (!modelJsExists(ml5Model, model)) {
+        await makeJsModelFiles(modelURLs[ml5Model][model], ml5Model, model);
+      }
+    }
+  }
+  cleanup();
+}
+main();

src/HandPose/index.js

@@ -149,6 +149,10 @@ class HandPose {
       "handPose"
     );
 
+    if (this.loadOfflineModel) {
+      this.loadOfflineModel(modelConfig);
+    }
+
     // Load the Tensorflow.js detector instance
     await tf.ready();
     this.model = await handPoseDetection.createDetector(pipeline, modelConfig);
@@ -308,3 +312,4 @@ const handPose = (...inputs) => {
 };
 
 export default handPose;
+export { HandPose };

src/offline/HandPose/index.js

@@ -0,0 +1,66 @@
+import { uint8ArrayToFile } from "../../utils/io.js";
+import landmarkLiteJson from "../models/HandPose/landmarkLite/model.json.js";
+import landmarkLiteBinArray from "../models/HandPose/landmarkLite/group1-shard1of1.bin.js";
+import detectorLiteJson from "../models/HandPose/detectorLite/model.json.js";
+import detectorLiteBinArray from "../models/HandPose/detectorLite/group1-shard1of1.bin.js";
+
+/**
+ * Define the loadOfflineModel function.
+ * The function will inject the model URLs into the config object.
+ * This function will be called by HandPose during initialization.
+ * @param {Object} configObject - The configuration object to mutate.
+ */
+function loadOfflineModel(configObject) {
+  let landmarkJson;
+  let landmarkBinArray;
+  let detectorJson;
+  let detectorBinArray;
+
+  // Select the correct model to load based on the config object.
+  if (configObject.modelType === "lite") {
+    landmarkJson = landmarkLiteJson;
+    landmarkBinArray = landmarkLiteBinArray;
+    detectorJson = detectorLiteJson;
+    detectorBinArray = detectorLiteBinArray;
+  }
+
+  // Convert the binary data to a file object.
+  const landmarkBinFile = uint8ArrayToFile(
+    landmarkBinArray,
+    "group1-shard1of1.bin"
+  );
+  const detectorBinFile = uint8ArrayToFile(
+    detectorBinArray,
+    "group1-shard1of1.bin"
+  );
+
+  // Give the detector model binary data a URL.
+  const landmarkBinURL = URL.createObjectURL(landmarkBinFile);
+  const detectorBinURL = URL.createObjectURL(detectorBinFile);
+
+  // Change the path to the binary file in the model json data.
+  landmarkJson.weightsManifest[0].paths[0] = landmarkBinURL.split("/").pop();
+  detectorJson.weightsManifest[0].paths[0] = detectorBinURL.split("/").pop();
+
+  // Convert the json data to file objects.
+  const landmarkJsonFile = new File(
+    [JSON.stringify(landmarkJson)],
+    "model.json",
+    { type: "application/json" }
+  );
+  const detectorJsonFile = new File(
+    [JSON.stringify(detectorJson)],
+    "model.json",
+    { type: "application/json" }
+  );
+
+  // Give the json data URLs.
+  const landmarkJsonURL = URL.createObjectURL(landmarkJsonFile);
+  const detectorJsonURL = URL.createObjectURL(detectorJsonFile);
+
+  // Inject the URLs into the config object.
+  configObject.landmarkModelUrl = landmarkJsonURL;
+  configObject.detectorModelUrl = detectorJsonURL;
+}
+
+export default loadOfflineModel;

src/offline/index.js

@@ -0,0 +1,8 @@
+import ml5 from "../index";
+
+import { HandPose } from "../HandPose";
+import loadHandPoseOfflineModel from "./HandPose";
+
+HandPose.prototype.loadOfflineModel = loadHandPoseOfflineModel;
+
+export default ml5;

src/utils/io.js

@@ -35,4 +35,15 @@ const loadFile = async (path, callback) =>
       throw error;
     });
 
-export { saveBlob, loadFile };
+/**
+ * Convert a Uint8Array to a binary File object.
+ * @param {Uint8Array} uint8Array - The Uint8Array to convert to a file.
+ * @param {string} fileName - The name of the file.
+ * @returns {File} A file object.
+ */
+function uint8ArrayToFile(uint8Array, fileName) {
+  const blob = new Blob([uint8Array], { type: "application/octet-stream" });
+  return new File([blob], fileName, { type: "application/octet-stream" });
+}
+
+export { saveBlob, loadFile, uint8ArrayToFile };

webpack.config.js

@@ -5,15 +5,19 @@ const TerserPlugin = require("terser-webpack-plugin");
 
 const commonConfig = {
   context: __dirname,
-  entry: "./src/index.js",
+  entry: {
+    ml5: "./src/index.js",
+    "ml5-offline": "./src/offline/index.js",
+  },
   output: {
-    filename: "ml5.js",
+    filename: "[name].js",
     path: resolve(__dirname, "dist"),
     library: {
       name: "ml5",
       type: "umd",
       export: "default",
     },
+    globalObject: "this",
   },
 };
 
@@ -48,18 +52,20 @@ const developmentConfig = {
   resolve: {
     fallback: {
       fs: false,
-      util: false
+      util: false,
     },
-  }
+  },
 };
 
 const productionConfig = {
   mode: "production",
+  devtool: "source-map",
   entry: {
     ml5: "./src/index.js",
     "ml5.min": "./src/index.js",
+    "ml5-offline": "./src/offline/index.js",
+    "ml5-offline.min": "./src/offline/index.js",
   },
-  devtool: "source-map",
   output: {
     publicPath: "/",
     filename: "[name].js",
@@ -68,18 +74,17 @@ const productionConfig = {
     minimize: true,
     minimizer: [
       new TerserPlugin({
-        include: "ml5.min.js",
-        exclude: "ml5.js",
+        include: /\.min\.js$/,
         extractComments: false,
       }),
     ],
   },
   resolve: {
     fallback: {
       fs: false,
-      util: false
+      util: false,
     },
-  }
+  },
 };
 
 module.exports = function (env, args) {