Skip to content

Commit 644b7b6

Browse files
stefanpennerstefanpenner
authored
Merge pull request #63 from mikrostew/parallel-processing
Run uglify in parallel, using a workerpool
2 parents f2b63c7 + 77bfdb7 commit 644b7b6

File tree

8 files changed

+270
-66
lines changed

8 files changed

+270
-66
lines changed

‎index.js‎

Lines changed: 42 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ var mkdirp = require('mkdirp');
1111
var srcURL = require('source-map-url');
1212
var MatcherCollection = require('matcher-collection');
1313
var debug = require('debug')('broccoli-uglify-sourcemap');
14+
var queue = require('async-promise-queue');
15+
var workerpool = require('workerpool');
16+
17+
var processFile = require('./lib/process-file');
1418

1519
module.exports = UglifyWriter;
1620

@@ -19,6 +23,8 @@ UglifyWriter.prototype.constructor = UglifyWriter;
1923

2024
const silent = process.argv.indexOf('--silent') !== -1;
2125

26+
const worker = queue.async.asyncify((doWork) => doWork());
27+
2228
function UglifyWriter (inputNodes, options) {
2329
if (!(this instanceof UglifyWriter)) {
2430
return new UglifyWriter(inputNodes, options);
@@ -34,6 +40,14 @@ function UglifyWriter (inputNodes, options) {
3440
},
3541
});
3642

43+
// consumers of this plugin can opt-in to async and concurrent behavior
44+
// TODO docs in the README
45+
this.async = (this.options.async === true);
46+
this.concurrency = this.options.concurrency || Number(process.env.JOBS) || Math.max(require('os').cpus().length - 1, 1);
47+
48+
// create a worker pool using an external worker script
49+
this.pool = workerpool.pool(path.join(__dirname, 'lib', 'worker.js'), { maxWorkers: this.concurrency });
50+
3751
this.inputNodes = inputNodes;
3852

3953
var exclude = this.options.exclude;
@@ -53,6 +67,9 @@ var MatchNothing = {
5367
UglifyWriter.prototype.build = function () {
5468
var writer = this;
5569

70+
// when options.async === true, allow processFile() operations to complete asynchronously
71+
var pendingWork = [];
72+
5673
this.inputPaths.forEach(function(inputPath) {
5774
walkSync(inputPath).forEach(function(relativePath) {
5875
if (relativePath.slice(-1) === '/') {
@@ -64,7 +81,15 @@ UglifyWriter.prototype.build = function () {
6481
mkdirp.sync(path.dirname(outFile));
6582

6683
if (relativePath.slice(-3) === '.js' && !writer.excludes.match(relativePath)) {
67-
writer.processFile(inFile, outFile, relativePath, writer.outputPath);
84+
// wrap this in a function so it doesn't actually run yet, and can be throttled
85+
var uglifyOperation = function() {
86+
return writer.processFile(inFile, outFile, relativePath, writer.outputPath);
87+
};
88+
if (writer.async) {
89+
pendingWork.push(uglifyOperation);
90+
return;
91+
}
92+
return uglifyOperation();
6893
} else if (relativePath.slice(-4) === '.map') {
6994
if (writer.excludes.match(relativePath.slice(0, -4) + '.js')) {
7095
// ensure .map files for excluded JS paths are also copied forward
@@ -77,70 +102,24 @@ UglifyWriter.prototype.build = function () {
77102
});
78103
});
79104

80-
return this.outputPath;
105+
return queue(worker, pendingWork, writer.concurrency)
106+
.then((/* results */) => {
107+
// files are finished processing, shut down the workers
108+
writer.pool.terminate();
109+
return writer.outputPath;
110+
})
111+
.catch((e) => {
112+
// make sure to shut down the workers on error
113+
writer.pool.terminate();
114+
throw e;
115+
});
81116
};
82117

83118
UglifyWriter.prototype.processFile = function(inFile, outFile, relativePath, outDir) {
84-
var src = fs.readFileSync(inFile, 'utf-8');
85-
var mapName = path.basename(outFile).replace(/\.js$/,'') + '.map';
86-
87-
var mapDir;
88-
if (this.options.sourceMapDir) {
89-
mapDir = path.join(outDir, this.options.sourceMapDir);
90-
} else {
91-
mapDir = path.dirname(path.join(outDir, relativePath));
92-
}
93-
94-
let options = defaults({}, this.options.uglify);
95-
if (options.sourceMap) {
96-
let filename = path.basename(inFile);
97-
let url = this.options.sourceMapDir ? '/' + path.join(this.options.sourceMapDir, mapName) : mapName;
98-
99-
let sourceMap = { filename, url };
100-
101-
if (srcURL.existsIn(src)) {
102-
let url = srcURL.getFrom(src);
103-
let sourceMapPath = path.join(path.dirname(inFile), url);
104-
if (fs.existsSync(sourceMapPath)) {
105-
sourceMap.content = JSON.parse(fs.readFileSync(sourceMapPath));
106-
} else if (!silent) {
107-
console.warn(`[WARN] (broccoli-uglify-sourcemap) "${url}" referenced in "${relativePath}" could not be found`);
108-
}
109-
}
110-
111-
options = defaults(options, { sourceMap });
112-
}
113-
114-
var start = new Date();
115-
debug('[starting]: %s %dKB', relativePath, (src.length / 1000));
116-
var result = UglifyJS.minify(src, options);
117-
var end = new Date();
118-
var total = end - start;
119-
if (total > 20000 && !silent) {
120-
console.warn('[WARN] (broccoli-uglify-sourcemap) Minifying: `' + relativePath + '` took: ' + total + 'ms (more than 20,000ms)');
121-
}
122-
123-
if (result.error) {
124-
result.error.filename = relativePath;
125-
throw result.error;
126-
}
127-
128-
debug('[finished]: %s %dKB in %dms', relativePath, (result.code.length / 1000), total);
129-
130-
if (options.sourceMap) {
131-
var newSourceMap = JSON.parse(result.map);
132-
133-
newSourceMap.sources = newSourceMap.sources.map(function(path){
134-
// If out output file has the same name as one of our original
135-
// sources, they will shadow eachother in Dev Tools. So instead we
136-
// alter the reference to the upstream file.
137-
if (path === relativePath) {
138-
path = path.replace(/\.js$/, '-orig.js');
139-
}
140-
return path;
141-
});
142-
mkdirp.sync(mapDir);
143-
fs.writeFileSync(path.join(mapDir, mapName), JSON.stringify(newSourceMap));
119+
// don't run this in the workerpool if concurrency is disabled (can set JOBS <= 1)
120+
if (this.async && this.concurrency > 1) {
121+
// each of these arguments is a string, which can be sent to the worker process as-is
122+
return this.pool.exec('processFileParallel', [inFile, outFile, relativePath, outDir, silent, this.options]);
144123
}
145-
fs.writeFileSync(outFile, result.code);
124+
return processFile(inFile, outFile, relativePath, outDir, silent, this.options);
146125
};

‎lib/process-file.js‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
'use strict';
2+
3+
var debug = require('debug')('broccoli-uglify-sourcemap');
4+
var defaults = require('lodash.defaultsdeep');
5+
var fs = require('fs');
6+
var mkdirp = require('mkdirp');
7+
var path = require('path');
8+
var srcURL = require('source-map-url');
9+
10+
var Promise = require('rsvp').Promise;
11+
var UglifyJS = require('uglify-es');
12+
13+
14+
module.exports = function processFile(inFile, outFile, relativePath, outDir, silent, _options) {
15+
var src = fs.readFileSync(inFile, 'utf-8');
16+
var mapName = path.basename(outFile).replace(/\.js$/,'') + '.map';
17+
18+
var mapDir;
19+
if (_options.sourceMapDir) {
20+
mapDir = path.join(outDir, _options.sourceMapDir);
21+
} else {
22+
mapDir = path.dirname(path.join(outDir, relativePath));
23+
}
24+
25+
let options = defaults({}, _options.uglify);
26+
if (options.sourceMap) {
27+
let filename = path.basename(inFile);
28+
let url = _options.sourceMapDir ? '/' + path.join(_options.sourceMapDir, mapName) : mapName;
29+
30+
let sourceMap = { filename, url };
31+
32+
if (srcURL.existsIn(src)) {
33+
let url = srcURL.getFrom(src);
34+
let sourceMapPath = path.join(path.dirname(inFile), url);
35+
if (fs.existsSync(sourceMapPath)) {
36+
sourceMap.content = JSON.parse(fs.readFileSync(sourceMapPath));
37+
} else if (!silent) {
38+
console.warn(`[WARN] (broccoli-uglify-sourcemap) "${url}" referenced in "${relativePath}" could not be found`);
39+
}
40+
}
41+
42+
options = defaults(options, { sourceMap });
43+
}
44+
45+
var start = new Date();
46+
debug('[starting]: %s %dKB', relativePath, (src.length / 1000));
47+
var result = UglifyJS.minify(src, options);
48+
var end = new Date();
49+
var total = end - start;
50+
if (total > 20000 && !silent) {
51+
console.warn('[WARN] (broccoli-uglify-sourcemap) Minifying: `' + relativePath + '` took: ' + total + 'ms (more than 20,000ms)');
52+
}
53+
54+
if (result.error) {
55+
result.error.filename = relativePath;
56+
throw result.error;
57+
}
58+
59+
debug('[finished]: %s %dKB in %dms', relativePath, (result.code.length / 1000), total);
60+
61+
if (options.sourceMap) {
62+
var newSourceMap = JSON.parse(result.map);
63+
64+
newSourceMap.sources = newSourceMap.sources.map(function(path){
65+
// If out output file has the same name as one of our original
66+
// sources, they will shadow eachother in Dev Tools. So instead we
67+
// alter the reference to the upstream file.
68+
if (path === relativePath) {
69+
path = path.replace(/\.js$/, '-orig.js');
70+
}
71+
return path;
72+
});
73+
mkdirp.sync(mapDir);
74+
fs.writeFileSync(path.join(mapDir, mapName), JSON.stringify(newSourceMap));
75+
}
76+
fs.writeFileSync(outFile, result.code);
77+
};

‎lib/worker.js‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
'use strict';
2+
3+
var workerpool = require('workerpool');
4+
5+
var processFile = require('./process-file');
6+
7+
// TODO - with an option to disable this parallelism
8+
function processFileParallel(inFile, outFile, relativePath, outDir, silent, _options) {
9+
return processFile(inFile, outFile, relativePath, outDir, silent, _options);
10+
}
11+
12+
// create worker and register public functions
13+
workerpool.worker({
14+
processFileParallel
15+
});

‎package.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
"license": "MIT",
1010
"author": "Edward Faulkner <[email protected]>",
1111
"files": [
12-
"index.js"
12+
"index.js",
13+
"lib"
1314
],
1415
"main": "index.js",
1516
"repository": {
@@ -22,6 +23,7 @@
2223
"test:watch": "jest --watchAll"
2324
},
2425
"dependencies": {
26+
"async-promise-queue": "^1.0.4",
2527
"broccoli-plugin": "^1.2.1",
2628
"debug": "^3.1.0",
2729
"lodash.defaultsdeep": "^4.6.0",
@@ -30,7 +32,8 @@
3032
"source-map-url": "^0.4.0",
3133
"symlink-or-copy": "^1.0.1",
3234
"uglify-es": "^3.1.3",
33-
"walk-sync": "^0.3.2"
35+
"walk-sync": "^0.3.2",
36+
"workerpool": "^2.3.0"
3437
},
3538
"devDependencies": {
3639
"babel-jest": "^21.2.0",
@@ -53,6 +56,7 @@
5356
"modulePathIgnorePatterns": [
5457
"<rootDir>/tmp"
5558
],
59+
"testEnvironment": "node",
5660
"testMatch": [
5761
"<rootDir>/test/test.js"
5862
],

‎test/__snapshots__/test.js.snap‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,50 @@ Object {
7979
}
8080
`;
8181

82+
exports[`broccoli-uglify-sourcemap generates expected output async 1`] = `
83+
Object {
84+
"inside": Object {
85+
"with-upstream-sourcemap.js": "function meaningOfLife(){throw new Error(42)}function boom(){throw new Error(\\"boom\\")}function somethingElse(){throw new Error(\\"somethign else\\")}function fourth(){throw new Error(\\"fourth\\")}function third(){throw new Error(\\"oh no\\")}
86+
//# sourceMappingURL=with-upstream-sourcemap.map",
87+
"with-upstream-sourcemap.map": "{\\"version\\":3,\\"sources\\":[\\"/inner/first.js\\",\\"/inner/second.js\\",\\"/other/fourth.js\\",\\"/other/third.js\\"],\\"names\\":[\\"meaningOfLife\\",\\"Error\\",\\"boom\\",\\"somethingElse\\",\\"fourth\\",\\"third\\"],\\"mappings\\":\\"AAAA,SAAAA,gBAEA,MAAA,IAAAC,MADA,IAIA,SAAAC,OACA,MAAA,IAAAD,MAAA,QCNA,SAAAE,gBACA,MAAA,IAAAF,MAAA,kBCEA,SAAAG,SACA,MAAA,IAAAH,MAAA,UCJA,SAAAI,QACA,MAAA,IAAAJ,MAAA\\",\\"file\\":\\"with-upstream-sourcemap.js\\",\\"sourcesContent\\":[\\"function meaningOfLife() {\\\\n var thisIsALongLocal = 42;\\\\n throw new Error(thisIsALongLocal);\\\\n}\\\\n\\\\nfunction boom() {\\\\n throw new Error('boom');\\\\n}\\\\n\\",\\"function somethingElse() {\\\\n throw new Error(\\\\\\"somethign else\\\\\\");\\\\n}\\\\n\\",\\"\\\\n// Hello world\\\\n\\\\nfunction fourth(){\\\\n throw new Error('fourth');\\\\n}\\\\n\\",\\"function third(){\\\\n throw new Error(\\\\\\"oh no\\\\\\");\\\\n}\\\\n\\"]}",
88+
},
89+
"no-upstream-sourcemap.js": "function meaningOfLife(){throw new Error(42)}function boom(){throw new Error(\\"boom\\")}function somethingElse(){throw new Error(\\"somethign else\\")}function fourth(){throw new Error(\\"fourth\\")}function third(){throw new Error(\\"oh no\\")}
90+
//# sourceMappingURL=no-upstream-sourcemap.map",
91+
"no-upstream-sourcemap.map": "{\\"version\\":3,\\"sources\\":[\\"0\\"],\\"names\\":[\\"meaningOfLife\\",\\"Error\\",\\"boom\\",\\"somethingElse\\",\\"fourth\\",\\"third\\"],\\"mappings\\":\\"AACA,SAASA,gBAEP,MAAM,IAAIC,MADa,IAIzB,SAASC,OACP,MAAM,IAAID,MAAM,QAGlB,SAASE,gBACP,MAAM,IAAIF,MAAM,kBAMlB,SAASG,SACP,MAAM,IAAIH,MAAM,UAGlB,SAASI,QACP,MAAM,IAAIJ,MAAM\\",\\"file\\":\\"no-upstream-sourcemap.js\\"}",
92+
"something.css": "#id {
93+
background: white;
94+
}",
95+
"with-broken-sourcemap.js": "function meaningOfLife(){throw new Error(42)}
96+
//# sourceMappingURL=with-broken-sourcemap.map",
97+
"with-broken-sourcemap.map": "{\\"version\\":3,\\"sources\\":[\\"0\\"],\\"names\\":[\\"meaningOfLife\\",\\"Error\\"],\\"mappings\\":\\"AAAA,SAASA,gBAEP,MAAM,IAAIC,MADa\\",\\"file\\":\\"with-broken-sourcemap.js\\"}",
98+
}
99+
`;
100+
101+
exports[`broccoli-uglify-sourcemap on error rejects with BuildError 1`] = `Object {}`;
102+
103+
exports[`broccoli-uglify-sourcemap on error rejects with BuildError async 1`] = `Object {}`;
104+
105+
exports[`broccoli-uglify-sourcemap on error shuts down the workerpool 1`] = `Object {}`;
106+
107+
exports[`broccoli-uglify-sourcemap shuts down the workerpool 1`] = `
108+
Object {
109+
"inside": Object {
110+
"with-upstream-sourcemap.js": "function meaningOfLife(){throw new Error(42)}function boom(){throw new Error(\\"boom\\")}function somethingElse(){throw new Error(\\"somethign else\\")}function fourth(){throw new Error(\\"fourth\\")}function third(){throw new Error(\\"oh no\\")}
111+
//# sourceMappingURL=with-upstream-sourcemap.map",
112+
"with-upstream-sourcemap.map": "{\\"version\\":3,\\"sources\\":[\\"/inner/first.js\\",\\"/inner/second.js\\",\\"/other/fourth.js\\",\\"/other/third.js\\"],\\"names\\":[\\"meaningOfLife\\",\\"Error\\",\\"boom\\",\\"somethingElse\\",\\"fourth\\",\\"third\\"],\\"mappings\\":\\"AAAA,SAAAA,gBAEA,MAAA,IAAAC,MADA,IAIA,SAAAC,OACA,MAAA,IAAAD,MAAA,QCNA,SAAAE,gBACA,MAAA,IAAAF,MAAA,kBCEA,SAAAG,SACA,MAAA,IAAAH,MAAA,UCJA,SAAAI,QACA,MAAA,IAAAJ,MAAA\\",\\"file\\":\\"with-upstream-sourcemap.js\\",\\"sourcesContent\\":[\\"function meaningOfLife() {\\\\n var thisIsALongLocal = 42;\\\\n throw new Error(thisIsALongLocal);\\\\n}\\\\n\\\\nfunction boom() {\\\\n throw new Error('boom');\\\\n}\\\\n\\",\\"function somethingElse() {\\\\n throw new Error(\\\\\\"somethign else\\\\\\");\\\\n}\\\\n\\",\\"\\\\n// Hello world\\\\n\\\\nfunction fourth(){\\\\n throw new Error('fourth');\\\\n}\\\\n\\",\\"function third(){\\\\n throw new Error(\\\\\\"oh no\\\\\\");\\\\n}\\\\n\\"]}",
113+
},
114+
"no-upstream-sourcemap.js": "function meaningOfLife(){throw new Error(42)}function boom(){throw new Error(\\"boom\\")}function somethingElse(){throw new Error(\\"somethign else\\")}function fourth(){throw new Error(\\"fourth\\")}function third(){throw new Error(\\"oh no\\")}
115+
//# sourceMappingURL=no-upstream-sourcemap.map",
116+
"no-upstream-sourcemap.map": "{\\"version\\":3,\\"sources\\":[\\"0\\"],\\"names\\":[\\"meaningOfLife\\",\\"Error\\",\\"boom\\",\\"somethingElse\\",\\"fourth\\",\\"third\\"],\\"mappings\\":\\"AACA,SAASA,gBAEP,MAAM,IAAIC,MADa,IAIzB,SAASC,OACP,MAAM,IAAID,MAAM,QAGlB,SAASE,gBACP,MAAM,IAAIF,MAAM,kBAMlB,SAASG,SACP,MAAM,IAAIH,MAAM,UAGlB,SAASI,QACP,MAAM,IAAIJ,MAAM\\",\\"file\\":\\"no-upstream-sourcemap.js\\"}",
117+
"something.css": "#id {
118+
background: white;
119+
}",
120+
"with-broken-sourcemap.js": "function meaningOfLife(){throw new Error(42)}
121+
//# sourceMappingURL=with-broken-sourcemap.map",
122+
"with-broken-sourcemap.map": "{\\"version\\":3,\\"sources\\":[\\"0\\"],\\"names\\":[\\"meaningOfLife\\",\\"Error\\"],\\"mappings\\":\\"AAAA,SAASA,gBAEP,MAAM,IAAIC,MADa\\",\\"file\\":\\"with-broken-sourcemap.js\\"}",
123+
}
124+
`;
125+
82126
exports[`broccoli-uglify-sourcemap supports alternate sourcemap location 1`] = `
83127
Object {
84128
"inside": Object {

‎test/fixtures-error/no-upstream-sourcemap.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
/* This file has an error. */
2+
var i =
3+

0 commit comments

Comments
 (0)