feat: add in-memory API for streaming files to destination asar (#360)

* feat: adding `async function createPackageFromStreams(dest: string, filestreams: Filestream[])` to allow streaming files directly from in-memory into the asar. The stream order is used as the insertion order. Consolidated logic for generating Pickle `out` `WriteableStream` and creating unpacked symlinks

* Refactor Type definitions for handling streaming to asar

* Update src/filesystem.ts

Co-authored-by: Kevin Cui <[email protected]>

---------

Co-authored-by: Kevin Cui <[email protected]>

Author: mmaietta

.mocharc.js

@@ -4,4 +4,5 @@ module.exports = {
   file: './test/mocha.setup.js', // setup file before everything else loads
   'forbid-only': process.env.CI ?? false, // make sure no `test.only` is merged into `main`
   reporter: 'spec',
+  spec: 'test/**/*-spec.js',
 };

src/asar.ts

@@ -9,7 +9,7 @@ import {
   FilesystemLinkEntry,
 } from './filesystem';
 import * as disk from './disk';
-import { crawl as crawlFilesystem, determineFileType } from './crawlfs';
+import { CrawledFileType, crawl as crawlFilesystem, determineFileType } from './crawlfs';
 import { IOptions } from './types/glob';
 
 /**
@@ -176,7 +176,13 @@ export async function createPackageFromFiles(
           options.unpackDir,
         );
         files.push({ filename, unpack: shouldUnpack });
-        return filesystem.insertFile(filename, shouldUnpack, file, options);
+        return filesystem.insertFile(
+          filename,
+          () => fs.createReadStream(filename),
+          shouldUnpack,
+          file,
+          options,
+        );
       case 'link':
         shouldUnpack = shouldUnpackPath(
           path.relative(src, filename),
@@ -209,6 +215,101 @@ export async function createPackageFromFiles(
   return next(names.shift());
 }
 
+export type AsarStream = {
+  /**
+    Relative path to the file or directory from within the archive
+  */
+  path: string;
+  /**
+    Function that returns a read stream for a file.
+    Note: this is called multiple times per "file", so a new NodeJS.ReadableStream needs to be created each time
+  */
+  streamGenerator: () => NodeJS.ReadableStream;
+  /**
+    Whether the file/link should be unpacked
+  */
+  unpacked: boolean;
+  stat: CrawledFileType['stat'];
+};
+export type AsarDirectory = Pick<AsarStream, 'path' | 'unpacked'> & {
+  type: 'directory';
+};
+export type AsarSymlinkStream = AsarStream & {
+  type: 'link';
+  symlink: string;
+};
+export type AsarFileStream = AsarStream & {
+  type: 'file';
+};
+export type AsarStreamType = AsarDirectory | AsarFileStream | AsarSymlinkStream;
+
+/**
+ * Create an ASAR archive from a list of streams.
+ *
+ * @param dest - Archive filename (& path).
+ * @param streams - List of streams to be piped in-memory into asar filesystem. Insertion order is preserved.
+ */
+export async function createPackageFromStreams(dest: string, streams: AsarStreamType[]) {
+  // We use an ambiguous root `src` since we're piping directly from a stream and the `filePath` for the stream is already relative to the src/root
+  const src = '.';
+
+  const filesystem = new Filesystem(src);
+  const files: disk.BasicStreamArray = [];
+  const links: disk.BasicStreamArray = [];
+
+  const handleFile = async function (stream: AsarStreamType) {
+    const { path: destinationPath, type } = stream;
+    const filename = path.normalize(destinationPath);
+    switch (type) {
+      case 'directory':
+        filesystem.insertDirectory(filename, stream.unpacked);
+        break;
+      case 'file':
+        files.push({
+          filename,
+          streamGenerator: stream.streamGenerator,
+          link: undefined,
+          mode: stream.stat.mode,
+          unpack: stream.unpacked,
+        });
+        return filesystem.insertFile(filename, stream.streamGenerator, stream.unpacked, {
+          type: 'file',
+          stat: stream.stat,
+        });
+      case 'link':
+        links.push({
+          filename,
+          streamGenerator: stream.streamGenerator,
+          link: stream.symlink,
+          mode: stream.stat.mode,
+          unpack: stream.unpacked,
+        });
+        filesystem.insertLink(filename, stream.unpacked, stream.symlink);
+        break;
+    }
+    return Promise.resolve();
+  };
+
+  const insertsDone = async function () {
+    await fs.mkdirp(path.dirname(dest));
+    return disk.streamFilesystem(dest, filesystem, { files, links });
+  };
+
+  const streamQueue = streams.slice();
+
+  const next = async function (stream?: AsarStreamType) {
+    if (!stream) {
+      return insertsDone();
+    }
+
+    await handleFile(stream);
+
+    return next(streamQueue.shift());
+  };
+
+  return next(streamQueue.shift());
+}
+
 export function statFile(
   archivePath: string,
   filename: string,
@@ -322,6 +423,7 @@ export default {
   createPackage,
   createPackageWithOptions,
   createPackageFromFiles,
+  createPackageFromStreams,
   statFile,
   getRawHeader,
   listPackage,

src/crawlfs.ts

@@ -10,7 +10,7 @@ const glob = promisify(_glob);
 
 export type CrawledFileType = {
   type: 'file' | 'directory' | 'link';
-  stat: Stats;
+  stat: Pick<Stats, 'mode' | 'size'>;
   transformed?: {
     path: string;
     stat: Stats;

src/disk.ts

@@ -3,6 +3,11 @@ import fs from './wrapped-fs';
 import { Pickle } from './pickle';
 import { Filesystem, FilesystemFileEntry } from './filesystem';
 import { CrawledFileType } from './crawlfs';
+import { Stats } from 'fs';
+import { promisify } from 'util';
+import * as stream from 'stream';
+
+const pipeline = promisify(stream.pipeline);
 
 let filesystemCache: Record<string, Filesystem | undefined> = Object.create(null);
 
@@ -19,12 +24,10 @@ async function copyFile(dest: string, src: string, filename: string) {
 }
 
 async function streamTransformedFile(
-  originalFilename: string,
+  stream: NodeJS.ReadableStream,
   outStream: NodeJS.WritableStream,
-  transformed: CrawledFileType['transformed'],
 ) {
   return new Promise<void>((resolve, reject) => {
-    const stream = fs.createReadStream(transformed ? transformed.path : originalFilename);
     stream.pipe(outStream, { end: false });
     stream.on('error', reject);
     stream.on('end', () => resolve());
@@ -35,15 +38,29 @@ export type InputMetadata = {
   [property: string]: CrawledFileType;
 };
 
-export type BasicFilesArray = { filename: string; unpack: boolean }[];
-
-export type FilesystemFilesAndLinks = { files: BasicFilesArray; links: BasicFilesArray };
+export type BasicFilesArray = {
+  filename: string;
+  unpack: boolean;
+}[];
+
+export type BasicStreamArray = {
+  filename: string;
+  streamGenerator: () => NodeJS.ReadableStream; // this is called multiple times per file
+  mode: Stats['mode'];
+  unpack: boolean;
+  link: string | undefined; // only for symlinks, should refactor as part of larger project refactor in follow-up PR
+}[];
+
+export type FilesystemFilesAndLinks<T extends BasicFilesArray | BasicStreamArray> = {
+  files: T;
+  links: T;
+};
 
 const writeFileListToStream = async function (
   dest: string,
   filesystem: Filesystem,
   out: NodeJS.WritableStream,
-  lists: FilesystemFilesAndLinks,
+  lists: FilesystemFilesAndLinks<BasicFilesArray>,
   metadata: InputMetadata,
 ) {
   const { files, links } = lists;
@@ -53,50 +70,55 @@ const writeFileListToStream = async function (
       const filename = path.relative(filesystem.getRootPath(), file.filename);
       await copyFile(`${dest}.unpacked`, filesystem.getRootPath(), filename);
     } else {
-      await streamTransformedFile(file.filename, out, metadata[file.filename].transformed);
+      const transformed = metadata[file.filename].transformed;
+      const stream = fs.createReadStream(transformed ? transformed.path : file.filename);
+      await streamTransformedFile(stream, out);
     }
   }
-  const unpackedSymlinks = links.filter((f) => f.unpack);
-  for (const file of unpackedSymlinks) {
+  for (const file of links.filter((f) => f.unpack)) {
     // the symlink needs to be recreated outside in .unpacked
     const filename = path.relative(filesystem.getRootPath(), file.filename);
     const link = await fs.readlink(file.filename);
-    // if symlink is within subdirectories, then we need to recreate dir structure
-    await fs.mkdirp(path.join(`${dest}.unpacked`, path.dirname(filename)));
-    // create symlink within unpacked dir
-    await fs.symlink(link, path.join(`${dest}.unpacked`, filename)).catch(async (error) => {
-      if (error.code === 'EPERM' && error.syscall === 'symlink') {
-        throw new Error(
-          'Could not create symlinks for unpacked assets. On Windows, consider activating Developer Mode to allow non-admin users to create symlinks by following the instructions at https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development.',
-        );
-      }
-      throw error;
-    });
+    await createSymlink(dest, filename, link);
   }
   return out.end();
 };
 
 export async function writeFilesystem(
   dest: string,
   filesystem: Filesystem,
-  lists: FilesystemFilesAndLinks,
+  lists: FilesystemFilesAndLinks<BasicFilesArray>,
   metadata: InputMetadata,
 ) {
-  const headerPickle = Pickle.createEmpty();
-  headerPickle.writeString(JSON.stringify(filesystem.getHeader()));
-  const headerBuf = headerPickle.toBuffer();
+  const out = await createFilesystemWriteStream(filesystem, dest);
+  return writeFileListToStream(dest, filesystem, out, lists, metadata);
+}
 
-  const sizePickle = Pickle.createEmpty();
-  sizePickle.writeUInt32(headerBuf.length);
-  const sizeBuf = sizePickle.toBuffer();
+export async function streamFilesystem(
+  dest: string,
+  filesystem: Filesystem,
+  lists: FilesystemFilesAndLinks<BasicStreamArray>,
+) {
+  const out = await createFilesystemWriteStream(filesystem, dest);
 
-  const out = fs.createWriteStream(dest);
-  await new Promise<void>((resolve, reject) => {
-    out.on('error', reject);
-    out.write(sizeBuf);
-    return out.write(headerBuf, () => resolve());
-  });
-  return writeFileListToStream(dest, filesystem, out, lists, metadata);
+  const { files, links } = lists;
+  for await (const file of files) {
+    // the file should not be packed into archive
+    if (file.unpack) {
+      const targetFile = path.join(`${dest}.unpacked`, file.filename);
+      await fs.mkdirp(path.dirname(targetFile));
+      const writeStream = fs.createWriteStream(targetFile, { mode: file.mode });
+      await pipeline(file.streamGenerator(), writeStream);
+    } else {
+      await streamTransformedFile(file.streamGenerator(), out);
+    }
+  }
+
+  for (const file of links.filter((f) => f.unpack && f.link)) {
+    // the symlink needs to be recreated outside in .unpacked
+    await createSymlink(dest, file.filename, file.link!);
+  }
+  return out.end();
 }
 
 export interface FileRecord extends FilesystemFileEntry {
@@ -187,3 +209,35 @@ export function readFileSync(filesystem: Filesystem, filename: string, info: Fil
   }
   return buffer;
 }
+
+async function createFilesystemWriteStream(filesystem: Filesystem, dest: string) {
+  const headerPickle = Pickle.createEmpty();
+  headerPickle.writeString(JSON.stringify(filesystem.getHeader()));
+  const headerBuf = headerPickle.toBuffer();
+
+  const sizePickle = Pickle.createEmpty();
+  sizePickle.writeUInt32(headerBuf.length);
+  const sizeBuf = sizePickle.toBuffer();
+
+  const out = fs.createWriteStream(dest);
+  await new Promise<void>((resolve, reject) => {
+    out.on('error', reject);
+    out.write(sizeBuf);
+    return out.write(headerBuf, () => resolve());
+  });
+  return out;
+}
+
+async function createSymlink(dest: string, filepath: string, link: string) {
+  // if symlink is within subdirectories, then we need to recreate dir structure
+  await fs.mkdirp(path.join(`${dest}.unpacked`, path.dirname(filepath)));
+  // create symlink within unpacked dir
+  await fs.symlink(link, path.join(`${dest}.unpacked`, filepath)).catch(async (error) => {
+    if (error.code === 'EPERM' && error.syscall === 'symlink') {
+      throw new Error(
+        'Could not create symlinks for unpacked assets. On Windows, consider activating Developer Mode to allow non-admin users to create symlinks by following the instructions at https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development.',
+      );
+    }
+    throw error;
+  });
+}

src/filesystem.ts

@@ -108,6 +108,7 @@ export class Filesystem {
 
   async insertFile(
     p: string,
+    streamGenerator: () => NodeJS.ReadableStream,
     shouldUnpack: boolean,
     file: CrawledFileType,
     options: {
@@ -119,7 +120,7 @@ export class Filesystem {
     if (shouldUnpack || dirNode.unpacked) {
       node.size = file.stat.size;
       node.unpacked = true;
-      node.integrity = await getFileIntegrity(p);
+      node.integrity = await getFileIntegrity(streamGenerator());
       return Promise.resolve();
     }
 
@@ -130,9 +131,8 @@ export class Filesystem {
       const tmpdir = await fs.mkdtemp(path.join(os.tmpdir(), 'asar-'));
       const tmpfile = path.join(tmpdir, path.basename(p));
       const out = fs.createWriteStream(tmpfile);
-      const readStream = fs.createReadStream(p);
 
-      await pipeline(readStream, transformed, out);
+      await pipeline(streamGenerator(), transformed, out);
       file.transformed = {
         path: tmpfile,
         stat: await fs.lstat(tmpfile),
@@ -149,18 +149,14 @@ export class Filesystem {
 
     node.size = size;
     node.offset = this.offset.toString();
-    node.integrity = await getFileIntegrity(p);
+    node.integrity = await getFileIntegrity(streamGenerator());
     if (process.platform !== 'win32' && file.stat.mode & 0o100) {
       node.executable = true;
     }
     this.offset += BigInt(size);
   }
 
-  insertLink(p: string, shouldUnpack: boolean) {
-    const symlink = fs.readlinkSync(p);
-    // /var => /private/var
-    const parentPath = fs.realpathSync(path.dirname(p));
-    const link = path.relative(fs.realpathSync(this.src), path.join(parentPath, symlink));
+  insertLink(p: string, shouldUnpack: boolean, link: string = this.resolveLink(p)) {
     if (link.startsWith('..')) {
       throw new Error(`${p}: file "${link}" links out of the package`);
     }
@@ -173,6 +169,14 @@ export class Filesystem {
     return link;
   }
 
+  private resolveLink(p: string) {
+    const symlink = fs.readlinkSync(p);
+    // /var/tmp => /private/var
+    const parentPath = fs.realpathSync(path.dirname(p));
+    const link = path.relative(fs.realpathSync(this.src), path.join(parentPath, symlink));
+    return link;
+  }
+
   listFiles(options?: { isPack: boolean }) {
     const files: string[] = [];