Perf: run cpu intensive task with worker

Author: SukkaW

Build/build-reject-domainset.js

@@ -33,7 +33,7 @@ const buildRejectDomainSet = task(__dirname, async () => {
   let shouldStop = false;
 
   const [gorhill] = await Promise.all([
-    getGorhillPublicSuffixPromise,
+    getGorhillPublicSuffixPromise(),
     // Parse from remote hosts & domain lists
     ...HOSTS.map(entry => processHosts(entry[0], entry[1]).then(hosts => {
       hosts.forEach(host => {

Build/index.js

@@ -9,14 +9,54 @@ const { buildTelegramCIDR } = require('./build-telegram-cidr');
 const { buildChnCidr } = require('./build-chn-cidr');
 const { buildSpeedtestDomainSet } = require('./build-speedtest-domainset');
 const { buildInternalCDNDomains } = require('./build-internal-cdn-rules');
-const { buildInternalReverseChnCIDR } = require('./build-internal-reverse-chn-cidr');
 const { buildInternalChnDomains } = require('./build-internal-chn-domains');
 const { buildDomesticRuleset } = require('./build-domestic-ruleset');
 const { validate } = require('./validate-domainset');
 
 const { buildPublicHtml } = require('./build-public');
 
+const { Worker } = require('jest-worker');
+
+/**
+ * @template T
+ * @typedef {import('jest-worker').Worker & { __sukka_worker_name: string } & T} WithWorker
+ */
+
+/**
+ * @template T
+ * @param {string} path
+ * @returns {WithWorker<T>}
+ */
+const requireWorker = (path) => {
+  const _worker = /** @type {WithWorker<T>} */ (new Worker(
+    require.resolve(path),
+    {
+      numWorkers: 1,
+      maxRetries: 0,
+      enableWorkerThreads: true
+    }
+  ));
+  _worker.getStderr().pipe(process.stderr);
+  _worker.getStdout().pipe(process.stdout);
+  _worker.__sukka_worker_name = path;
+  return _worker;
+};
+
+/**
+ * @template T
+ * @param {WithWorker<T>} worker
+ */
+const endWorker = async (worker) => {
+  const { forceExited } = worker.end();
+  if (forceExited && worker.__sukka_worker_name) {
+    console.log(worker.__sukka_worker_name, 'forceExited');
+  }
+};
+
 (async () => {
+  const buildInternalReverseChnCIDRWorker = /** @type {WithWorker<import('./build-internal-reverse-chn-cidr')>} */ (requireWorker('./build-internal-reverse-chn-cidr'));
+  const { buildInternalReverseChnCIDR } = buildInternalReverseChnCIDRWorker;
+
   // download-previous-build
   const downloadPreviousBuildPromise = downloadPreviousBuild();
   const downloadPublicSuffixListPromise = downloadPublicSuffixList();
@@ -77,6 +117,7 @@ const { buildPublicHtml } = require('./build-public');
 
   await Promise.all([
     buildPublicHtml(),
-    validate()
+    validate(),
+    endWorker(buildInternalReverseChnCIDRWorker)
   ]);
 })();

Build/lib/get-gorhill-publicsuffix.js

@@ -30,5 +30,9 @@ const getGorhillPublicSuffix = async () => {
   return gorhill;
 };
 
-const getGorhillPublicSuffixPromise = getGorhillPublicSuffix();
-module.exports.getGorhillPublicSuffixPromise = getGorhillPublicSuffixPromise;
+/** @type {Promise<import('gorhill-publicsuffixlist').default | null>} */
+let gorhillPublicSuffixPromise = null;
+module.exports.getGorhillPublicSuffixPromise = () => {
+  gorhillPublicSuffixPromise ||= getGorhillPublicSuffix();
+  return gorhillPublicSuffixPromise;
+};

Build/lib/is-domain-loose.js

@@ -10,10 +10,10 @@ module.exports.isDomainLoose = (domain) => {
 };
 
 /**
- * @param {string} domain
+ * @param {string | null | undefined} domain
  */
 module.exports.normalizeDomain = (domain) => {
-  if (domain == null) {
+  if (!domain) {
     return null;
   }
 

package.json

@@ -21,6 +21,7 @@
     "cidr-tools-wasm": "^0.0.11",
     "fs-extra": "^11.1.1",
     "gorhill-publicsuffixlist": "github:gorhill/publicsuffixlist.js",
+    "jest-worker": "^29.7.0",
     "mnemonist": "^0.39.5",
     "path-scurry": "^1.10.1",
     "picocolors": "^1.0.0",