Skip to content

Commit 70b69be

Browse files
mikrostewmikrostew
committed
run processFile() in parallel, using a workerpool
1 parent f2b63c7 commit 70b69be

File tree

7 files changed

+193
-66
lines changed

7 files changed

+193
-66
lines changed

‎index.js‎

Lines changed: 36 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ var mkdirp = require('mkdirp');
1111
var srcURL = require('source-map-url');
1212
var MatcherCollection = require('matcher-collection');
1313
var debug = require('debug')('broccoli-uglify-sourcemap');
14+
var queue = require('async-promise-queue');
15+
var workerpool = require('workerpool');
16+
17+
var processFile = require('./lib/process-file');
1418

1519
module.exports = UglifyWriter;
1620

@@ -19,6 +23,8 @@ UglifyWriter.prototype.constructor = UglifyWriter;
1923

2024
const silent = process.argv.indexOf('--silent') !== -1;
2125

26+
const worker = queue.async.asyncify((doWork) => doWork());
27+
2228
function UglifyWriter (inputNodes, options) {
2329
if (!(this instanceof UglifyWriter)) {
2430
return new UglifyWriter(inputNodes, options);
@@ -34,6 +40,14 @@ function UglifyWriter (inputNodes, options) {
3440
},
3541
});
3642

43+
// consumers of this plugin can opt-in to async and concurrent behavior
44+
// TODO docs in the README
45+
this.async = (this.options.async === true);
46+
this.concurrency = this.options.concurrency || Number(process.env.JOBS) || Math.max(require('os').cpus().length - 1, 1);
47+
48+
// create a worker pool using an external worker script
49+
this.pool = workerpool.pool(path.join(__dirname, 'lib', 'worker.js'), { maxWorkers: this.concurrency });
50+
3751
this.inputNodes = inputNodes;
3852

3953
var exclude = this.options.exclude;
@@ -53,6 +67,9 @@ var MatchNothing = {
5367
UglifyWriter.prototype.build = function () {
5468
var writer = this;
5569

70+
// when options.async === true, allow processFile() operations to complete asynchronously
71+
var pendingWork = [];
72+
5673
this.inputPaths.forEach(function(inputPath) {
5774
walkSync(inputPath).forEach(function(relativePath) {
5875
if (relativePath.slice(-1) === '/') {
@@ -64,7 +81,15 @@ UglifyWriter.prototype.build = function () {
6481
mkdirp.sync(path.dirname(outFile));
6582

6683
if (relativePath.slice(-3) === '.js' && !writer.excludes.match(relativePath)) {
67-
writer.processFile(inFile, outFile, relativePath, writer.outputPath);
84+
// wrap this in a function so it doesn't actually run yet, and can be throttled
85+
var uglifyOperation = function() {
86+
return writer.processFile(inFile, outFile, relativePath, writer.outputPath);
87+
};
88+
if (writer.async) {
89+
pendingWork.push(uglifyOperation);
90+
return;
91+
}
92+
return uglifyOperation();
6893
} else if (relativePath.slice(-4) === '.map') {
6994
if (writer.excludes.match(relativePath.slice(0, -4) + '.js')) {
7095
// ensure .map files for excluded JS paths are also copied forward
@@ -77,70 +102,18 @@ UglifyWriter.prototype.build = function () {
77102
});
78103
});
79104

80-
return this.outputPath;
105+
return queue(worker, pendingWork, writer.concurrency)
106+
.then((/* results */) => {
107+
// files are finished processing, shut down the workers
108+
writer.pool.terminate();
109+
return writer.outputPath;
110+
});
81111
};
82112

83113
UglifyWriter.prototype.processFile = function(inFile, outFile, relativePath, outDir) {
84-
var src = fs.readFileSync(inFile, 'utf-8');
85-
var mapName = path.basename(outFile).replace(/\.js$/,'') + '.map';
86-
87-
var mapDir;
88-
if (this.options.sourceMapDir) {
89-
mapDir = path.join(outDir, this.options.sourceMapDir);
90-
} else {
91-
mapDir = path.dirname(path.join(outDir, relativePath));
92-
}
93-
94-
let options = defaults({}, this.options.uglify);
95-
if (options.sourceMap) {
96-
let filename = path.basename(inFile);
97-
let url = this.options.sourceMapDir ? '/' + path.join(this.options.sourceMapDir, mapName) : mapName;
98-
99-
let sourceMap = { filename, url };
100-
101-
if (srcURL.existsIn(src)) {
102-
let url = srcURL.getFrom(src);
103-
let sourceMapPath = path.join(path.dirname(inFile), url);
104-
if (fs.existsSync(sourceMapPath)) {
105-
sourceMap.content = JSON.parse(fs.readFileSync(sourceMapPath));
106-
} else if (!silent) {
107-
console.warn(`[WARN] (broccoli-uglify-sourcemap) "${url}" referenced in "${relativePath}" could not be found`);
108-
}
109-
}
110-
111-
options = defaults(options, { sourceMap });
112-
}
113-
114-
var start = new Date();
115-
debug('[starting]: %s %dKB', relativePath, (src.length / 1000));
116-
var result = UglifyJS.minify(src, options);
117-
var end = new Date();
118-
var total = end - start;
119-
if (total > 20000 && !silent) {
120-
console.warn('[WARN] (broccoli-uglify-sourcemap) Minifying: `' + relativePath + '` took: ' + total + 'ms (more than 20,000ms)');
121-
}
122-
123-
if (result.error) {
124-
result.error.filename = relativePath;
125-
throw result.error;
126-
}
127-
128-
debug('[finished]: %s %dKB in %dms', relativePath, (result.code.length / 1000), total);
129-
130-
if (options.sourceMap) {
131-
var newSourceMap = JSON.parse(result.map);
132-
133-
newSourceMap.sources = newSourceMap.sources.map(function(path){
134-
// If out output file has the same name as one of our original
135-
// sources, they will shadow eachother in Dev Tools. So instead we
136-
// alter the reference to the upstream file.
137-
if (path === relativePath) {
138-
path = path.replace(/\.js$/, '-orig.js');
139-
}
140-
return path;
141-
});
142-
mkdirp.sync(mapDir);
143-
fs.writeFileSync(path.join(mapDir, mapName), JSON.stringify(newSourceMap));
114+
// don't run this in the workerpool if concurrency is disabled (can set JOBS <= 1)
115+
if (this.async && this.concurrency > 1) {
116+
return this.pool.exec('processFileParallel', [inFile, outFile, relativePath, outDir, silent, this.options]);
144117
}
145-
fs.writeFileSync(outFile, result.code);
118+
return processFile(inFile, outFile, relativePath, outDir, silent, this.options);
146119
};

‎lib/process-file.js‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
'use strict';
2+
3+
var debug = require('debug')('broccoli-uglify-sourcemap');
4+
var defaults = require('lodash.defaultsdeep');
5+
var fs = require('fs');
6+
var mkdirp = require('mkdirp');
7+
var path = require('path');
8+
var srcURL = require('source-map-url');
9+
10+
var Promise = require('rsvp').Promise;
11+
var UglifyJS = require('uglify-es');
12+
13+
14+
// each of these is a string, so that's good
15+
module.exports = function processFile(inFile, outFile, relativePath, outDir, silent, _options) {
16+
return new Promise((resolve, reject) => {
17+
18+
var src = fs.readFileSync(inFile, 'utf-8');
19+
var mapName = path.basename(outFile).replace(/\.js$/,'') + '.map';
20+
21+
var mapDir;
22+
if (_options.sourceMapDir) {
23+
mapDir = path.join(outDir, _options.sourceMapDir);
24+
} else {
25+
mapDir = path.dirname(path.join(outDir, relativePath));
26+
}
27+
28+
let options = defaults({}, _options.uglify);
29+
if (options.sourceMap) {
30+
let filename = path.basename(inFile);
31+
let url = _options.sourceMapDir ? '/' + path.join(_options.sourceMapDir, mapName) : mapName;
32+
33+
let sourceMap = { filename, url };
34+
35+
if (srcURL.existsIn(src)) {
36+
let url = srcURL.getFrom(src);
37+
let sourceMapPath = path.join(path.dirname(inFile), url);
38+
if (fs.existsSync(sourceMapPath)) {
39+
sourceMap.content = JSON.parse(fs.readFileSync(sourceMapPath));
40+
} else if (!silent) {
41+
console.warn(`[WARN] (broccoli-uglify-sourcemap) "${url}" referenced in "${relativePath}" could not be found`);
42+
}
43+
}
44+
45+
options = defaults(options, { sourceMap });
46+
}
47+
48+
var start = new Date();
49+
debug('[starting]: %s %dKB', relativePath, (src.length / 1000));
50+
var result = UglifyJS.minify(src, options);
51+
var end = new Date();
52+
var total = end - start;
53+
if (total > 20000 && !silent) {
54+
console.warn('[WARN] (broccoli-uglify-sourcemap) Minifying: `' + relativePath + '` took: ' + total + 'ms (more than 20,000ms)');
55+
}
56+
57+
if (result.error) {
58+
result.error.filename = relativePath;
59+
reject(result.error);
60+
return;
61+
}
62+
63+
debug('[finished]: %s %dKB in %dms', relativePath, (result.code.length / 1000), total);
64+
65+
if (options.sourceMap) {
66+
var newSourceMap = JSON.parse(result.map);
67+
68+
newSourceMap.sources = newSourceMap.sources.map(function(path){
69+
// If out output file has the same name as one of our original
70+
// sources, they will shadow eachother in Dev Tools. So instead we
71+
// alter the reference to the upstream file.
72+
if (path === relativePath) {
73+
path = path.replace(/\.js$/, '-orig.js');
74+
}
75+
return path;
76+
});
77+
mkdirp.sync(mapDir);
78+
fs.writeFileSync(path.join(mapDir, mapName), JSON.stringify(newSourceMap));
79+
}
80+
fs.writeFileSync(outFile, result.code);
81+
resolve();
82+
});
83+
};

‎lib/worker.js‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
'use strict';
2+
3+
var workerpool = require('workerpool');
4+
5+
var processFile = require('./process-file');
6+
7+
// TODO - with an option to disable this parallelism
8+
function processFileParallel(inFile, outFile, relativePath, outDir, silent, _options) {
9+
return processFile(inFile, outFile, relativePath, outDir, silent, _options);
10+
}
11+
12+
// create worker and register public functions
13+
workerpool.worker({
14+
processFileParallel
15+
});

‎package.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
"license": "MIT",
1010
"author": "Edward Faulkner <[email protected]>",
1111
"files": [
12-
"index.js"
12+
"index.js",
13+
"lib"
1314
],
1415
"main": "index.js",
1516
"repository": {
@@ -22,6 +23,7 @@
2223
"test:watch": "jest --watchAll"
2324
},
2425
"dependencies": {
26+
"async-promise-queue": "^1.0.4",
2527
"broccoli-plugin": "^1.2.1",
2628
"debug": "^3.1.0",
2729
"lodash.defaultsdeep": "^4.6.0",
@@ -30,7 +32,8 @@
3032
"source-map-url": "^0.4.0",
3133
"symlink-or-copy": "^1.0.1",
3234
"uglify-es": "^3.1.3",
33-
"walk-sync": "^0.3.2"
35+
"walk-sync": "^0.3.2",
36+
"workerpool": "^2.3.0"
3437
},
3538
"devDependencies": {
3639
"babel-jest": "^21.2.0",
@@ -53,6 +56,7 @@
5356
"modulePathIgnorePatterns": [
5457
"<rootDir>/tmp"
5558
],
59+
"testEnvironment": "node",
5660
"testMatch": [
5761
"<rootDir>/test/test.js"
5862
],

‎test/__snapshots__/test.js.snap‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,25 @@ Object {
7979
}
8080
`;
8181

82+
exports[`broccoli-uglify-sourcemap generates expected output async 1`] = `
83+
Object {
84+
"inside": Object {
85+
"with-upstream-sourcemap.js": "function meaningOfLife(){throw new Error(42)}function boom(){throw new Error(\\"boom\\")}function somethingElse(){throw new Error(\\"somethign else\\")}function fourth(){throw new Error(\\"fourth\\")}function third(){throw new Error(\\"oh no\\")}
86+
//# sourceMappingURL=with-upstream-sourcemap.map",
87+
"with-upstream-sourcemap.map": "{\\"version\\":3,\\"sources\\":[\\"/inner/first.js\\",\\"/inner/second.js\\",\\"/other/fourth.js\\",\\"/other/third.js\\"],\\"names\\":[\\"meaningOfLife\\",\\"Error\\",\\"boom\\",\\"somethingElse\\",\\"fourth\\",\\"third\\"],\\"mappings\\":\\"AAAA,SAAAA,gBAEA,MAAA,IAAAC,MADA,IAIA,SAAAC,OACA,MAAA,IAAAD,MAAA,QCNA,SAAAE,gBACA,MAAA,IAAAF,MAAA,kBCEA,SAAAG,SACA,MAAA,IAAAH,MAAA,UCJA,SAAAI,QACA,MAAA,IAAAJ,MAAA\\",\\"file\\":\\"with-upstream-sourcemap.js\\",\\"sourcesContent\\":[\\"function meaningOfLife() {\\\\n var thisIsALongLocal = 42;\\\\n throw new Error(thisIsALongLocal);\\\\n}\\\\n\\\\nfunction boom() {\\\\n throw new Error('boom');\\\\n}\\\\n\\",\\"function somethingElse() {\\\\n throw new Error(\\\\\\"somethign else\\\\\\");\\\\n}\\\\n\\",\\"\\\\n// Hello world\\\\n\\\\nfunction fourth(){\\\\n throw new Error('fourth');\\\\n}\\\\n\\",\\"function third(){\\\\n throw new Error(\\\\\\"oh no\\\\\\");\\\\n}\\\\n\\"]}",
88+
},
89+
"no-upstream-sourcemap.js": "function meaningOfLife(){throw new Error(42)}function boom(){throw new Error(\\"boom\\")}function somethingElse(){throw new Error(\\"somethign else\\")}function fourth(){throw new Error(\\"fourth\\")}function third(){throw new Error(\\"oh no\\")}
90+
//# sourceMappingURL=no-upstream-sourcemap.map",
91+
"no-upstream-sourcemap.map": "{\\"version\\":3,\\"sources\\":[\\"0\\"],\\"names\\":[\\"meaningOfLife\\",\\"Error\\",\\"boom\\",\\"somethingElse\\",\\"fourth\\",\\"third\\"],\\"mappings\\":\\"AACA,SAASA,gBAEP,MAAM,IAAIC,MADa,IAIzB,SAASC,OACP,MAAM,IAAID,MAAM,QAGlB,SAASE,gBACP,MAAM,IAAIF,MAAM,kBAMlB,SAASG,SACP,MAAM,IAAIH,MAAM,UAGlB,SAASI,QACP,MAAM,IAAIJ,MAAM\\",\\"file\\":\\"no-upstream-sourcemap.js\\"}",
92+
"something.css": "#id {
93+
background: white;
94+
}",
95+
"with-broken-sourcemap.js": "function meaningOfLife(){throw new Error(42)}
96+
//# sourceMappingURL=with-broken-sourcemap.map",
97+
"with-broken-sourcemap.map": "{\\"version\\":3,\\"sources\\":[\\"0\\"],\\"names\\":[\\"meaningOfLife\\",\\"Error\\"],\\"mappings\\":\\"AAAA,SAASA,gBAEP,MAAM,IAAIC,MADa\\",\\"file\\":\\"with-broken-sourcemap.js\\"}",
98+
}
99+
`;
100+
82101
exports[`broccoli-uglify-sourcemap supports alternate sourcemap location 1`] = `
83102
Object {
84103
"inside": Object {

‎test/test.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,14 @@ describe('broccoli-uglify-sourcemap', function() {
2525
expect(builder.read()).toMatchSnapshot();
2626
});
2727

28+
it('generates expected output async', async function() {
29+
builder = createBuilder(new uglify(fixtures, { async: true }));
30+
31+
await builder.build();
32+
33+
expect(builder.read()).toMatchSnapshot();
34+
});
35+
2836
it('can handle ES6 code', async function() {
2937
input.write({
3038
'es6.js': `class Foo {

‎yarn.lock‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,13 @@ astral-regex@^1.0.0:
129129
version "1.0.0"
130130
resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-1.0.0.tgz#6c8c3fb827dd43ee3918f27b82782ab7658a6fd9"
131131

132+
async-promise-queue@^1.0.4:
133+
version "1.0.4"
134+
resolved "https://registry.yarnpkg.com/async-promise-queue/-/async-promise-queue-1.0.4.tgz#308baafbc74aff66a0bb6e7f4a18d4fe8434440c"
135+
dependencies:
136+
async "^2.4.1"
137+
debug "^2.6.8"
138+
132139
async@^1.4.0:
133140
version "1.5.2"
134141
resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a"
@@ -139,6 +146,12 @@ async@^2.1.4:
139146
dependencies:
140147
lodash "^4.14.0"
141148

149+
async@^2.4.1:
150+
version "2.6.0"
151+
resolved "https://registry.yarnpkg.com/async/-/async-2.6.0.tgz#61a29abb6fcc026fea77e56d1c6ec53a795951f4"
152+
dependencies:
153+
lodash "^4.14.0"
154+
142155
asynckit@^0.4.0:
143156
version "0.4.0"
144157
resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
@@ -624,6 +637,12 @@ debug@^2.2.0:
624637
dependencies:
625638
ms "0.7.2"
626639

640+
debug@^2.6.8:
641+
version "2.6.9"
642+
resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
643+
dependencies:
644+
ms "2.0.0"
645+
627646
debug@^3.1.0:
628647
version "3.1.0"
629648
resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
@@ -1924,7 +1943,7 @@ oauth-sign@~0.8.1:
19241943
version "0.8.2"
19251944
resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43"
19261945

1927-
object-assign@^4.1.0:
1946+
object-assign@4.1.1, object-assign@^4.1.0:
19281947
version "4.1.1"
19291948
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
19301949

@@ -2780,6 +2799,12 @@ worker-farm@^1.3.1:
27802799
errno "^0.1.4"
27812800
xtend "^4.0.1"
27822801

2802+
workerpool@^2.3.0:
2803+
version "2.3.0"
2804+
resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-2.3.0.tgz#86c5cbe946b55e7dc9d12b1936c8801a6e2d744d"
2805+
dependencies:
2806+
object-assign "4.1.1"
2807+
27832808
wrap-ansi@^2.0.0:
27842809
version "2.1.0"
27852810
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85"

0 commit comments

Comments
 (0)