Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 85ad87c

Browse files
author
Mikko Tiihonen
committedJul 31, 2016
Add support for precompressed (gzip) content
1 parent d6dd3b9 commit 85ad87c

File tree

8 files changed

+245
-9
lines changed

8 files changed

+245
-9
lines changed
 

‎.gitattributes

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*.bz2 binary
2+
*.gz binary

‎HISTORY.md

+5
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
unreleased
2+
==========
3+
4+
* Send precompressed variant of content based on `Accept-Encoding`
5+
16
0.14.1 / 2016-06-09
27
===================
38

‎README.md

+36
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,42 @@ Provide a max-age in milliseconds for http caching, defaults to 0.
9696
This can also be a string accepted by the
9797
[ms](https://www.npmjs.org/package/ms#readme) module.
9898

99+
##### precompressed
100+
101+
Precompressed files are extra static files that are compressed before
102+
they are requested, as opposed to compressing on the fly. Compressing
103+
files once offline (for example during site build) allows using
104+
stronger compression methods and both reduces latency and lowers cpu
105+
usage when serving files.
106+
107+
The `precompressed` option enables or disables serving of precompressed
108+
content variants. The option defaults to `false`, if set to `true` checks
109+
for existence of gzip compressed files with `.gz` extensions.
110+
111+
Example scenario:
112+
113+
The file `site.css` has both `site.css.gz` and `site.css.bz2`
114+
precompressed versions available in the same directory. The server is configured
115+
to serve both `.bz2` and `.gz` files in that prefence order.
116+
When a request comes with an `Accept-Encoding` header with value `gzip, bz2`
117+
requesting `site.css` the contents of `site.css.bz2` is sent instead and
118+
a header `Content-Encoding` with value `br` is added to the response.
119+
In addition a `Vary: Accept-Encoding` header is added to response allowing
120+
caching proxies to work correctly.
121+
122+
Custom configuration:
123+
124+
It is also possible to customize the searched file extensions and header
125+
values (used with Accept-Encoding and Content-Encoding headers) by specifying
126+
them explicitly in an array in the preferred priority order. For example:
127+
`[{encoding: 'bzip2', extension: '.bz2'}, {encoding: 'gzip', extension: '.gz'}]`.
128+
129+
Compression tips:
130+
* Precompress at least all static `js`, `css` and `svg` files.
131+
* Precompress using both brotli (supported by Firefox and Chrome) and
132+
gzip encoders. Brotli compresses generally 15-20% better than gzip.
133+
* Use zopfli for gzip compression for and extra 5% benefit for all browsers.
134+
99135
##### root
100136

101137
Serve files relative to `path`.

‎index.js

+92-8
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ var path = require('path')
3030
var statuses = require('statuses')
3131
var Stream = require('stream')
3232
var util = require('util')
33+
var vary = require('vary')
34+
var Negotiator = require('negotiator')
3335

3436
/**
3537
* Path function references.
@@ -146,6 +148,21 @@ function SendStream (req, path, options) {
146148
? normalizeList(opts.extensions, 'extensions option')
147149
: []
148150

151+
if (Array.isArray(opts.precompressed)) {
152+
if (opts.precompressed.length > 0) {
153+
this._precompressionFormats = opts.precompressed
154+
this._precompressionEncodings = this._precompressionFormats.map(function (format) { return format.encoding; })
155+
this._precompressionEncodings.push('identity')
156+
}
157+
} else if (opts.precompressed) {
158+
this._precompressionFormats = [{encoding:'gzip', extension:'.gz'}]
159+
this._precompressionEncodings = ['gzip', 'identity']
160+
}
161+
162+
this._precompressionFormats = opts.precompressionFormats !== undefined
163+
? opts.precompressionFormats
164+
: this._precompressionFormats
165+
149166
this._index = opts.index !== undefined
150167
? normalizeList(opts.index, 'index option')
151168
: ['index.html']
@@ -319,6 +336,33 @@ SendStream.prototype.isConditionalGET = function isConditionalGET () {
319336
this.req.headers['if-modified-since']
320337
}
321338

339+
/**
340+
* Return the array of file precompressed file extensions to serve in preference order.
341+
*
342+
* @return {Array}
343+
* @api private
344+
*/
345+
346+
SendStream.prototype.getAcceptEncodingExtensions = function() {
347+
var self = this
348+
var negotiatedEncodings = new Negotiator(this.req).encodings(self._precompressionEncodings)
349+
var accepted = []
350+
for (var e = 0; e < negotiatedEncodings.length; e++) {
351+
var encoding = negotiatedEncodings[e]
352+
if (encoding === 'identity') {
353+
break
354+
}
355+
for (var f = 0; f < self._precompressionFormats.length; f++) {
356+
var format = self._precompressionFormats[f]
357+
if (format.encoding === encoding) {
358+
accepted.push(format.extension)
359+
break
360+
}
361+
}
362+
}
363+
return accepted;
364+
}
365+
322366
/**
323367
* Strip content-* header fields.
324368
*
@@ -558,8 +602,10 @@ SendStream.prototype.pipe = function pipe (res) {
558602
* @api public
559603
*/
560604

561-
SendStream.prototype.send = function send (path, stat) {
562-
var len = stat.size
605+
SendStream.prototype.send = function send (path, stat, contentPath, contentStat) {
606+
contentStat = contentStat || stat
607+
contentPath = contentPath || path
608+
var len = contentStat.size;
563609
var options = this.options
564610
var opts = {}
565611
var res = this.res
@@ -573,7 +619,7 @@ SendStream.prototype.send = function send (path, stat) {
573619
return
574620
}
575621

576-
debug('pipe "%s"', path)
622+
debug('pipe "%s"', contentPath)
577623

578624
// set header fields
579625
this.setHeader(path, stat)
@@ -652,7 +698,7 @@ SendStream.prototype.send = function send (path, stat) {
652698
return
653699
}
654700

655-
this.stream(path, opts)
701+
this.stream(contentPath, opts)
656702
}
657703

658704
/**
@@ -673,8 +719,7 @@ SendStream.prototype.sendFile = function sendFile (path) {
673719
}
674720
if (err) return self.onStatError(err)
675721
if (stat.isDirectory()) return self.redirect(self.path)
676-
self.emit('file', path, stat)
677-
self.send(path, stat)
722+
checkPrecompressionAndSendFile(path, stat)
678723
})
679724

680725
function next (err) {
@@ -690,10 +735,49 @@ SendStream.prototype.sendFile = function sendFile (path) {
690735
fs.stat(p, function (err, stat) {
691736
if (err) return next(err)
692737
if (stat.isDirectory()) return next()
693-
self.emit('file', p, stat)
694-
self.send(p, stat)
738+
checkPrecompressionAndSendFile(p, stat)
739+
})
740+
}
741+
742+
function checkPrecompressionAndSendFile(p, stat) {
743+
self.emit('file', p, stat)
744+
if (!self._precompressionFormats) return self.send(p, stat)
745+
746+
var state = {
747+
contents: [],
748+
extensionsToCheck: self._precompressionFormats.length
749+
}
750+
751+
self._precompressionFormats.forEach(function (format) {
752+
debug('stat "%s%s"', p, format.extension);
753+
fs.stat(p + format.extension, function onstat(err, contentStat) {
754+
if (!err) state.contents.push({ext: format.extension, encoding: format.encoding, contentStat: contentStat})
755+
if (--state.extensionsToCheck == 0) sendPreferredContent(p, stat, state.contents)
756+
})
695757
})
696758
}
759+
760+
function sendPreferredContent(p, stat, contents) {
761+
if (contents.length) {
762+
vary(self.res, 'Accept-Encoding')
763+
}
764+
765+
var preferredContent
766+
var extensions = self.getAcceptEncodingExtensions()
767+
for (var e = 0; e < extensions.length && !preferredContent; e++) {
768+
for (var c = 0; c < contents.length; c++) {
769+
if (extensions[e] === contents[c].ext) {
770+
preferredContent = contents[c]
771+
break
772+
}
773+
}
774+
}
775+
776+
if (!preferredContent) return self.send(p, stat)
777+
778+
self.res.setHeader('Content-Encoding', preferredContent.encoding)
779+
self.send(p, stat, p + preferredContent.ext, preferredContent.contentStat)
780+
}
697781
}
698782

699783
/**

‎package.json

+3-1
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,11 @@
2424
"http-errors": "~1.5.0",
2525
"mime": "1.3.4",
2626
"ms": "0.7.1",
27+
"negotiator": "jshttp/negotiator#d9907aec0585476d9a0c4271e464f7c6e4633049",
2728
"on-finished": "~2.3.0",
2829
"range-parser": "~1.2.0",
29-
"statuses": "~1.3.0"
30+
"statuses": "~1.3.0",
31+
"vary": "~1.1.0"
3032
},
3133
"devDependencies": {
3234
"after": "0.8.1",

‎test/fixtures/name.html.bz2

50 Bytes
Binary file not shown.

‎test/fixtures/name.html.gz

31 Bytes
Binary file not shown.

‎test/send.js

+107
Original file line numberDiff line numberDiff line change
@@ -1090,6 +1090,113 @@ describe('send(file, options)', function () {
10901090
})
10911091
})
10921092

1093+
describe('precompressed', function () {
1094+
it('should not include vary header when no precompressed variants exist', function (done) {
1095+
request(createServer({precompressed: true, root: fixtures}))
1096+
.get('/name.txt')
1097+
.set('Accept-Encoding', 'gzip')
1098+
.expect(shouldNotHaveHeader('Vary'))
1099+
.expect(shouldNotHaveHeader('Content-Encoding'))
1100+
.expect(200, done)
1101+
})
1102+
1103+
it('should include vary header when precompressed variants exist even when accept-encoding not present', function (done) {
1104+
request(createServer({precompressed: true, root: fixtures}))
1105+
.get('/name.html')
1106+
.set('Accept-Encoding', '')
1107+
.expect('Content-Length', '11')
1108+
.expect(shouldNotHaveHeader('Content-Encoding'))
1109+
.expect('Content-Type', 'text/html; charset=UTF-8')
1110+
.expect('Vary', 'Accept-Encoding', done)
1111+
})
1112+
1113+
it('should prefer server encoding order (bzip2,gzip) when present with equal weight in accept-encoding', function (done) {
1114+
request(createServer({precompressed: [{encoding: 'bzip2', extension: '.bz2'}, {encoding: 'gzip', extension: '.gz'}], root: fixtures}))
1115+
.get('/name.html')
1116+
.set('Accept-Encoding', 'gzip, deflate, bzip2')
1117+
.expect('Vary', 'Accept-Encoding')
1118+
.expect('Content-Encoding', 'bzip2')
1119+
.expect('Content-Type', 'text/html; charset=UTF-8')
1120+
.expect('Content-Length', '50', done)
1121+
})
1122+
1123+
it('should prefer server encoding order (gzip,bzip2) when present with equal weight in accept-encoding', function (done) {
1124+
request(createServer({precompressed: [{encoding: 'gzip', extension: '.gz'}, {encoding: 'bzip2', extension: '.bz2'}], root: fixtures}))
1125+
.get('/name.html')
1126+
.set('Accept-Encoding', 'bzip2, deflate, gzip')
1127+
.expect('Vary', 'Accept-Encoding')
1128+
.expect('Content-Encoding', 'gzip')
1129+
.expect('Content-Type', 'text/html; charset=UTF-8')
1130+
.expect('Content-Length', '31', done)
1131+
})
1132+
1133+
it('should send gzip when preferred in accept-encoding', function (done) {
1134+
request(createServer({precompressed: true, root: fixtures}))
1135+
.get('/name.html')
1136+
.set('Accept-Encoding', ' gzip , deflate')
1137+
.expect('Vary', 'Accept-Encoding')
1138+
.expect('Content-Encoding', 'gzip')
1139+
.expect('Content-Type', 'text/html; charset=UTF-8')
1140+
.expect('Content-Length', '31', done)
1141+
})
1142+
1143+
it('should not send gzip when no-gzip encoding is used', function (done) {
1144+
request(createServer({precompressed: true, root: fixtures}))
1145+
.get('/name.html')
1146+
.set('Accept-Encoding', 'no-gzip, deflate')
1147+
.expect('Content-Length', '11')
1148+
.expect('Vary', 'Accept-Encoding', done)
1149+
})
1150+
1151+
it('should consider empty array of precompressed configuration as disabled', function (done) {
1152+
request(createServer({precompressed: [], root: fixtures}))
1153+
.get('/name.html')
1154+
.set('Accept-Encoding', 'gzip')
1155+
.expect(shouldNotHaveHeader('Content-Encoding'))
1156+
.expect('Content-Length', '11', done)
1157+
})
1158+
1159+
it('should append to existing Vary header', function (done) {
1160+
request(http.createServer(function (req, res) {
1161+
res.setHeader('Vary', 'custom')
1162+
send(req, req.url, {precompressed: true, root: fixtures})
1163+
.pipe(res)
1164+
}))
1165+
.get('/name.html')
1166+
.expect('Vary', 'custom, Accept-Encoding', done)
1167+
})
1168+
1169+
it('should honour accept-encoding quality values', function (done) {
1170+
request(createServer({precompressed: true, root: fixtures}))
1171+
.get('/name.html')
1172+
.set('Accept-Encoding', 'gzip;q=0.9, deflate;q=1, bzip2;q=0.1')
1173+
.expect('Vary', 'Accept-Encoding')
1174+
.expect('Content-Encoding', 'gzip')
1175+
.expect('Content-Type', 'text/html; charset=UTF-8')
1176+
.expect('Content-Length', '31', done)
1177+
})
1178+
1179+
it('should return no encoding if identity encoding preferred in accept-encoding', function (done) {
1180+
request(createServer({precompressed: true, root: fixtures}))
1181+
.get('/name.html')
1182+
.set('Accept-Encoding', 'gzip;q=0.8, identity')
1183+
.expect('Vary', 'Accept-Encoding')
1184+
.expect(shouldNotHaveHeader('Content-Encoding'))
1185+
.expect('Content-Type', 'text/html; charset=UTF-8')
1186+
.expect('Content-Length', '11', done)
1187+
})
1188+
1189+
it('should return server preferred format for accept-encoding *', function (done) {
1190+
request(createServer({precompressed: [{encoding: 'bzip2', extension: '.bz2'}, {encoding: 'gzip', extension: '.gz'}], root: fixtures}))
1191+
.get('/name.html')
1192+
.set('Accept-Encoding', '*;q=0.9; gzip;q=0.8')
1193+
.expect('Vary', 'Accept-Encoding')
1194+
.expect('Content-Encoding', 'bzip2')
1195+
.expect('Content-Type', 'text/html; charset=UTF-8')
1196+
.expect('Content-Length', '50', done)
1197+
})
1198+
})
1199+
10931200
describe('index', function () {
10941201
it('should reject numbers', function (done) {
10951202
request(createServer({root: fixtures, index: 42}))

0 commit comments

Comments
 (0)