feat: allow providing properties (such as unpack) in ordering input file (#350)

* Part 1 for electron/universal#117 with updated unit test

* add additional helper function

* use wrapped-fs for writeFile

* `yarn prettier:write`

* swap to for-of loop and extract helper function `isAlreadySorted` to make it more readable

Author: mmaietta

src/asar.ts

@@ -33,6 +33,73 @@ function isUnpackedDir(dirPath: string, pattern: string, unpackDirs: string[]) {
   }
 }
 
+export type FileProperties = {
+  filepath: string;
+  properties: {
+    unpack?: boolean;
+  };
+};
+
+async function getFileOrdering(
+  options: CreateOptions,
+  src: string,
+  filenames: string[],
+): Promise<FileProperties[]> {
+  if (!options.ordering) {
+    return filenames.map<FileProperties>((filepath) => ({ filepath, properties: {} }));
+  }
+  const orderingMap: FileProperties[] = (await fs.readFile(options.ordering))
+    .toString()
+    .split('\n')
+    .map<FileProperties>((line) => {
+      line = line.trim();
+      const config: FileProperties = { filepath: line, properties: {} };
+      const colonIndex = line.indexOf(':');
+      if (colonIndex > -1) {
+        config.filepath = line.substring(0, colonIndex); // file path
+        const props = line.substring(colonIndex + 1); // props on other side of the `:`
+        config.properties = props.length > 2 ? JSON.parse(props) : {}; // file properties
+      }
+      if (config.filepath.startsWith('/')) {
+        config.filepath = config.filepath.slice(1);
+      }
+      return config;
+    });
+
+  const ordering: FileProperties[] = [];
+  for (const config of orderingMap) {
+    const pathComponents = config.filepath.split(path.sep);
+    let str = src;
+    for (const pathComponent of pathComponents) {
+      str = path.join(str, pathComponent);
+      ordering.push({ filepath: str, properties: config.properties });
+    }
+  }
+
+  let missing = 0;
+  const total = filenames.length;
+
+  const fileOrderingSorted: FileProperties[] = [];
+  const isAlreadySorted = (file: string) =>
+    fileOrderingSorted.findIndex((config) => file === config.filepath) > -1;
+
+  for (const config of ordering) {
+    if (!isAlreadySorted(config.filepath) && filenames.includes(config.filepath)) {
+      fileOrderingSorted.push(config);
+    }
+  }
+
+  for (const file of filenames) {
+    if (!isAlreadySorted(file)) {
+      fileOrderingSorted.push({ filepath: file, properties: {} });
+      missing += 1;
+    }
+  }
+
+  console.log(`Ordering file has ${((total - missing) / total) * 100}% coverage.`);
+  return fileOrderingSorted;
+}
+
 export async function createPackage(src: string, dest: string) {
   return createPackageWithOptions(src, dest, {});
 }
@@ -84,54 +151,10 @@ export async function createPackageFromFiles(
   const links: disk.BasicFilesArray = [];
   const unpackDirs: string[] = [];
 
-  let filenamesSorted: string[] = [];
-  if (options.ordering) {
-    const orderingFiles = (await fs.readFile(options.ordering))
-      .toString()
-      .split('\n')
-      .map((line) => {
-        if (line.includes(':')) {
-          line = line.split(':').pop()!;
-        }
-        line = line.trim();
-        if (line.startsWith('/')) {
-          line = line.slice(1);
-        }
-        return line;
-      });
-
-    const ordering: string[] = [];
-    for (const file of orderingFiles) {
-      const pathComponents = file.split(path.sep);
-      let str = src;
-      for (const pathComponent of pathComponents) {
-        str = path.join(str, pathComponent);
-        ordering.push(str);
-      }
-    }
-
-    let missing = 0;
-    const total = filenames.length;
-
-    for (const file of ordering) {
-      if (!filenamesSorted.includes(file) && filenames.includes(file)) {
-        filenamesSorted.push(file);
-      }
-    }
-
-    for (const file of filenames) {
-      if (!filenamesSorted.includes(file)) {
-        filenamesSorted.push(file);
-        missing += 1;
-      }
-    }
+  const filenamesSorted: FileProperties[] = await getFileOrdering(options, src, filenames);
 
-    console.log(`Ordering file has ${((total - missing) / total) * 100}% coverage.`);
-  } else {
-    filenamesSorted = filenames;
-  }
-
-  const handleFile = async function (filename: string) {
+  const handleFile = async function (config: FileProperties) {
+    const filename = config.filepath;
     if (!metadata[filename]) {
       const fileType = await determineFileType(filename);
       if (!fileType) {
@@ -146,6 +169,9 @@ export async function createPackageFromFiles(
       unpack: string | undefined,
       unpackDir: string | undefined,
     ) {
+      if (config.properties.unpack != null) {
+        return config.properties.unpack;
+      }
       let shouldUnpack = false;
       if (unpack) {
         shouldUnpack = minimatch(filename, unpack, { matchBase: true });
@@ -190,12 +216,12 @@ export async function createPackageFromFiles(
 
   const names = filenamesSorted.slice();
 
-  const next = async function (name?: string) {
-    if (!name) {
+  const next = async function (config?: FileProperties) {
+    if (!config) {
       return insertsDone();
     }
 
-    await handleFile(name);
+    await handleFile(config);
     return next(names.shift());
   };
 

test/cli-spec.js

@@ -190,9 +190,17 @@ describe('command line interface', function () {
     );
   });
   it('should unpack static framework with all underlying symlinks unpacked', async () => {
-    const { tmpPath } = createSymlinkApp('app');
+    const { tmpPath, buildOrderingData } = createSymlinkApp('ordered-app');
+    const orderingPath = path.join(tmpPath, '../ordered-app-ordering.txt');
+
+    // this is functionally the same as `-unpack *.txt --unpack-dir var`
+    const data = buildOrderingData((filepath) => ({
+      unpack: filepath.endsWith('.txt') || filepath.includes('var'),
+    }));
+    await fs.writeFile(orderingPath, data);
+
     await execAsar(
-      `p ${tmpPath} tmp/packthis-with-symlink.asar --unpack *.txt --unpack-dir var --exclude-hidden`,
+      `p ${tmpPath} tmp/packthis-with-symlink.asar --ordering=${orderingPath} --exclude-hidden`,
     );
 
     assert.ok(fs.existsSync('tmp/packthis-with-symlink.asar.unpacked/private/var/file.txt'));

test/util/createSymlinkApp.js

@@ -26,5 +26,33 @@ module.exports = (testName) => {
   const appPath = path.join(varPath, 'app');
   fs.mkdirpSync(appPath);
   fs.symlinkSync('../file.txt', path.join(appPath, 'file.txt'));
-  return { appPath, tmpPath, varPath };
+
+  const ordering = walk(tmpPath).map((filepath) => filepath.substring(tmpPath.length)); // convert to paths relative to root
+
+  return {
+    appPath,
+    tmpPath,
+    varPath,
+    // helper function for generating the `ordering.txt` file data
+    buildOrderingData: (getProps) =>
+      ordering.reduce((prev, curr) => {
+        return `${prev}${curr}:${JSON.stringify(getProps(curr))}\n`;
+      }, ''),
+  };
+};
+
+// returns a list of all directories, files, and symlinks. Automates testing `ordering` logic easy.
+const walk = (root) => {
+  const getPaths = (filepath, filter) =>
+    fs
+      .readdirSync(filepath, { withFileTypes: true })
+      .filter((dirent) => filter(dirent))
+      .map(({ name }) => path.join(filepath, name));
+
+  const dirs = getPaths(root, (dirent) => dirent.isDirectory());
+  const files = dirs.map((dir) => walk(dir)).flat();
+  return files.concat(
+    dirs,
+    getPaths(root, (dirent) => dirent.isFile() || dirent.isSymbolicLink()),
+  );
 };