diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b797974 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.bz2 binary +*.gz binary diff --git a/HISTORY.md b/HISTORY.md index 1eacafa..aae988f 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,8 @@ +unreleased +========== + + * Send precompressed variant of content based on `Accept-Encoding` + 0.15.1 / 2017-03-04 =================== diff --git a/README.md b/README.md index 0c8d11d..ca36733 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,50 @@ Provide a max-age in milliseconds for http caching, defaults to 0. This can also be a string accepted by the [ms](https://www.npmjs.org/package/ms#readme) module. +##### precompressed + +Precompressed files are extra static files that are compressed before +they are requested, as opposed to compressing on the fly. Compressing +files once offline (for example during site build) allows using +stronger compression methods and both reduces latency and lowers cpu +usage when serving files. + +The `precompressed` option enables or disables serving of precompressed +content variants. The option defaults to `false`, if set to `true` checks +for existence of gzip compressed files with `.gz` extensions. + +Example scenario: + +The file `site.css` has both `site.css.gz` and `site.css.bz2` +precompressed versions available in the same directory. The server is configured +to serve both `.bz2` and `.gz` files in that prefence order. +When a request comes with an `Accept-Encoding` header with value `gzip, bz2` +requesting `site.css` the contents of `site.css.bz2` is sent instead and +a header `Content-Encoding` with value `br` is added to the response. +In addition a `Vary: Accept-Encoding` header is added to response allowing +caching proxies to work correctly. + +Custom configuration: + +It is also possible to customize the searched file extensions and header +values (used with Accept-Encoding and Content-Encoding headers) by specifying +them explicitly in an array in the preferred priority order. For example: +`[{encoding: 'bzip2', extension: '.bz2'}, {encoding: 'gzip', extension: '.gz'}]`. + +Compression tips: + * Precompress at least all static `js`, `css` and `svg` files. + * Precompress using both brotli (supported by Firefox and Chrome) and + gzip encoders. Brotli compresses generally 15-20% better than gzip. + * Use zopfli for gzip compression for and extra 5% benefit for all browsers. + +Performance of serving static files is lower due to extra stats – worst case +20% with 1 byte files to loopback client. Compared to on-the-fly compression +the precompression is still a large win. + +##### encodingNegotiatorOptions + +Allows configuring the [encoding negotation options](https://github.com/jshttp/negotiator#sort-options). + ##### root Serve files relative to `path`. diff --git a/index.js b/index.js index 3756841..1560257 100644 --- a/index.js +++ b/index.js @@ -30,6 +30,8 @@ var path = require('path') var statuses = require('statuses') var Stream = require('stream') var util = require('util') +var vary = require('vary') +var Negotiator = require('negotiator') /** * Path function references. @@ -153,6 +155,25 @@ function SendStream (req, path, options) { ? normalizeList(opts.extensions, 'extensions option') : [] + if (Array.isArray(opts.precompressed)) { + if (opts.precompressed.length > 0) { + this._precompressionFormats = opts.precompressed + this._precompressionEncodings = this._precompressionFormats.map(function (format) { return format.encoding }) + this._precompressionEncodings.push('identity') + } + } else if (opts.precompressed) { + this._precompressionFormats = [{ encoding: 'gzip', extension: '.gz' }] + this._precompressionEncodings = ['gzip', 'identity'] + } + + this._precompressionFormats = opts.precompressionFormats !== undefined + ? opts.precompressionFormats + : this._precompressionFormats + + this._encodingNegotiatorOptions = opts.encodingNegotiatorOptions !== undefined + ? opts.encodingNegotiatorOptions + : { sortPreference: 'clientThenServer' } + this._index = opts.index !== undefined ? normalizeList(opts.index, 'index option') : ['index.html'] @@ -360,6 +381,33 @@ SendStream.prototype.isPreconditionFailure = function isPreconditionFailure () { return false } +/** + * Return the array of file precompressed file extensions to serve in preference order. + * + * @return {Array} + * @api private + */ + +SendStream.prototype.getAcceptEncodingExtensions = function () { + var self = this + var negotiatedEncodings = new Negotiator(this.req).encodings(self._precompressionEncodings, this._encodingNegotiatorOptions) + var accepted = [] + for (var e = 0; e < negotiatedEncodings.length; e++) { + var encoding = negotiatedEncodings[e] + if (encoding === 'identity') { + break + } + for (var f = 0; f < self._precompressionFormats.length; f++) { + var format = self._precompressionFormats[f] + if (format.encoding === encoding) { + accepted.push(format.extension) + break + } + } + } + return accepted +} + /** * Strip content-* header fields. * @@ -611,8 +659,10 @@ SendStream.prototype.pipe = function pipe (res) { * @api public */ -SendStream.prototype.send = function send (path, stat) { - var len = stat.size +SendStream.prototype.send = function send (path, stat, contentPath, contentStat) { + contentStat = contentStat || stat + contentPath = contentPath || path + var len = contentStat.size var options = this.options var opts = {} var res = this.res @@ -626,7 +676,7 @@ SendStream.prototype.send = function send (path, stat) { return } - debug('pipe "%s"', path) + debug('pipe "%s"', contentPath) // set header fields this.setHeader(path, stat) @@ -712,7 +762,7 @@ SendStream.prototype.send = function send (path, stat) { return } - this.stream(path, opts) + this.stream(contentPath, opts) } /** @@ -733,8 +783,7 @@ SendStream.prototype.sendFile = function sendFile (path) { } if (err) return self.onStatError(err) if (stat.isDirectory()) return self.redirect(path) - self.emit('file', path, stat) - self.send(path, stat) + checkPrecompressionAndSendFile(path, stat) }) function next (err) { @@ -750,10 +799,49 @@ SendStream.prototype.sendFile = function sendFile (path) { fs.stat(p, function (err, stat) { if (err) return next(err) if (stat.isDirectory()) return next() - self.emit('file', p, stat) - self.send(p, stat) + checkPrecompressionAndSendFile(p, stat) }) } + + function checkPrecompressionAndSendFile (p, stat) { + self.emit('file', p, stat) + if (!self._precompressionFormats) return self.send(p, stat) + + var state = { + contents: [], + extensionsToCheck: self._precompressionFormats.length + } + + self._precompressionFormats.forEach(function (format) { + debug('stat "%s%s"', p, format.extension) + fs.stat(p + format.extension, function onstat (err, contentStat) { + if (!err) state.contents.push({ext: format.extension, encoding: format.encoding, contentStat: contentStat}) + if (--state.extensionsToCheck === 0) sendPreferredContent(p, stat, state.contents) + }) + }) + } + + function sendPreferredContent (p, stat, contents) { + if (contents.length) { + vary(self.res, 'Accept-Encoding') + } + + var preferredContent + var extensions = self.getAcceptEncodingExtensions() + for (var e = 0; e < extensions.length && !preferredContent; e++) { + for (var c = 0; c < contents.length; c++) { + if (extensions[e] === contents[c].ext) { + preferredContent = contents[c] + break + } + } + } + + if (!preferredContent) return self.send(p, stat) + + self.res.setHeader('Content-Encoding', preferredContent.encoding) + self.send(p, stat, p + preferredContent.ext, preferredContent.contentStat) + } } /** diff --git a/package.json b/package.json index 8bd72a7..22e2848 100644 --- a/package.json +++ b/package.json @@ -26,9 +26,11 @@ "http-errors": "~1.6.1", "mime": "1.3.4", "ms": "0.7.2", + "negotiator": "jshttp/negotiator#6038bf698c522c1883a1113c834e53256b35584f", "on-finished": "~2.3.0", "range-parser": "~1.2.0", - "statuses": "~1.3.1" + "statuses": "~1.3.1", + "vary": "~1.1.0" }, "devDependencies": { "after": "0.8.2", diff --git a/test/fixtures/name.html.bz2 b/test/fixtures/name.html.bz2 new file mode 100644 index 0000000..b22473e Binary files /dev/null and b/test/fixtures/name.html.bz2 differ diff --git a/test/fixtures/name.html.gz b/test/fixtures/name.html.gz new file mode 100644 index 0000000..c0aa1e0 Binary files /dev/null and b/test/fixtures/name.html.gz differ diff --git a/test/send.js b/test/send.js index 8efe9c5..4a2e47f 100644 --- a/test/send.js +++ b/test/send.js @@ -1232,6 +1232,113 @@ describe('send(file, options)', function () { }) }) + describe('precompressed', function () { + it('should not include vary header when no precompressed variants exist', function (done) { + request(createServer({precompressed: true, root: fixtures})) + .get('/name.txt') + .set('Accept-Encoding', 'gzip') + .expect(shouldNotHaveHeader('Vary')) + .expect(shouldNotHaveHeader('Content-Encoding')) + .expect(200, done) + }) + + it('should include vary header when precompressed variants exist even when accept-encoding not present', function (done) { + request(createServer({precompressed: true, root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', '') + .expect('Content-Length', '11') + .expect(shouldNotHaveHeader('Content-Encoding')) + .expect('Content-Type', 'text/html; charset=UTF-8') + .expect('Vary', 'Accept-Encoding', done) + }) + + it('should prefer server encoding order (bzip2,gzip) when present with equal weight in accept-encoding', function (done) { + request(createServer({precompressed: [{encoding: 'bzip2', extension: '.bz2'}, {encoding: 'gzip', extension: '.gz'}], root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', 'gzip, deflate, bzip2') + .expect('Vary', 'Accept-Encoding') + .expect('Content-Encoding', 'bzip2') + .expect('Content-Type', 'text/html; charset=UTF-8') + .expect('Content-Length', '50', done) + }) + + it('should prefer server encoding order (gzip,bzip2) when present with equal weight in accept-encoding', function (done) { + request(createServer({precompressed: [{encoding: 'gzip', extension: '.gz'}, {encoding: 'bzip2', extension: '.bz2'}], root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', 'bzip2, deflate, gzip') + .expect('Vary', 'Accept-Encoding') + .expect('Content-Encoding', 'gzip') + .expect('Content-Type', 'text/html; charset=UTF-8') + .expect('Content-Length', '31', done) + }) + + it('should send gzip when preferred in accept-encoding', function (done) { + request(createServer({precompressed: true, root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', ' gzip , deflate') + .expect('Vary', 'Accept-Encoding') + .expect('Content-Encoding', 'gzip') + .expect('Content-Type', 'text/html; charset=UTF-8') + .expect('Content-Length', '31', done) + }) + + it('should not send gzip when no-gzip encoding is used', function (done) { + request(createServer({precompressed: true, root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', 'no-gzip, deflate') + .expect('Content-Length', '11') + .expect('Vary', 'Accept-Encoding', done) + }) + + it('should consider empty array of precompressed configuration as disabled', function (done) { + request(createServer({precompressed: [], root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', 'gzip') + .expect(shouldNotHaveHeader('Content-Encoding')) + .expect('Content-Length', '11', done) + }) + + it('should append to existing Vary header', function (done) { + request(http.createServer(function (req, res) { + res.setHeader('Vary', 'custom') + send(req, req.url, {precompressed: true, root: fixtures}) + .pipe(res) + })) + .get('/name.html') + .expect('Vary', 'custom, Accept-Encoding', done) + }) + + it('should honour accept-encoding quality values', function (done) { + request(createServer({precompressed: true, root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', 'gzip;q=0.9, deflate;q=1, bzip2;q=0.1') + .expect('Vary', 'Accept-Encoding') + .expect('Content-Encoding', 'gzip') + .expect('Content-Type', 'text/html; charset=UTF-8') + .expect('Content-Length', '31', done) + }) + + it('should return no encoding if identity encoding preferred in accept-encoding', function (done) { + request(createServer({precompressed: true, root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', 'gzip;q=0.8, identity') + .expect('Vary', 'Accept-Encoding') + .expect(shouldNotHaveHeader('Content-Encoding')) + .expect('Content-Type', 'text/html; charset=UTF-8') + .expect('Content-Length', '11', done) + }) + + it('should return server preferred format for accept-encoding *', function (done) { + request(createServer({precompressed: [{encoding: 'bzip2', extension: '.bz2'}, {encoding: 'gzip', extension: '.gz'}], root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', '*;q=0.9; gzip;q=0.8') + .expect('Vary', 'Accept-Encoding') + .expect('Content-Encoding', 'bzip2') + .expect('Content-Type', 'text/html; charset=UTF-8') + .expect('Content-Length', '50', done) + }) + }) + describe('index', function () { it('should reject numbers', function (done) { request(createServer({root: fixtures, index: 42}))