diff --git a/.gitignore b/.gitignore index 4793235a5..fbf4bba9a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ test-report.xml junit.xml coverage/ yarn-error.log +.vscode \ No newline at end of file diff --git a/package.json b/package.json index cfe9403e6..432c96477 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "prettier": "^1.7.0" }, "dependencies": { - "long": "^3.2.0" + "long": "^4.0.0" }, "lint-staged": { "*.js": [ diff --git a/src/protocol/decoder.js b/src/protocol/decoder.js index b7c7416fd..f7cad260c 100644 --- a/src/protocol/decoder.js +++ b/src/protocol/decoder.js @@ -5,6 +5,9 @@ const INT16_SIZE = 2 const INT32_SIZE = 4 const INT64_SIZE = 8 +const MOST_SIGNIFICANT_BIT = 0x80 // 128 +const OTHER_BITS = 0x7f // 127 + module.exports = class Decoder { static int32Size() { return INT32_SIZE @@ -120,6 +123,42 @@ module.exports = class Decoder { return array } + readSignedVarInt32() { + let currentByte + let result = 0 + let i = 0 + + do { + currentByte = this.buffer[this.offset++] + result += (currentByte & OTHER_BITS) << i + i += 7 + } while (currentByte >= MOST_SIGNIFICANT_BIT) + + return this.decodeZigZag(result) + } + + decodeZigZag(value) { + return (value >>> 1) ^ -(value & 1) + } + + readSignedVarInt64() { + let currentByte + let result = Long.fromInt(0) + let i = 0 + + do { + currentByte = this.buffer[this.offset++] + result = result.add(Long.fromInt(currentByte & OTHER_BITS).shiftLeft(i)) + i += 7 + } while (currentByte >= MOST_SIGNIFICANT_BIT) + + return this.decodeZigZag64(result) + } + + decodeZigZag64(longValue) { + return longValue.shiftRightUnsigned(1).xor(longValue.and(Long.fromInt(1)).negate()) + } + slice(size) { return new Decoder(this.buffer.slice(this.offset, this.offset + size)) } diff --git a/src/protocol/encoder.js b/src/protocol/encoder.js index 144380060..4c4f2df5a 100644 --- a/src/protocol/encoder.js +++ b/src/protocol/encoder.js @@ -5,6 +5,11 @@ const INT16_SIZE = 2 const INT32_SIZE = 4 const INT64_SIZE = 8 +const MOST_SIGNIFICANT_BIT = 0x80 // 128 +const OTHER_BITS = 0x7f // 127 +const UNSIGNED_INT32_MAX_NUMBER = 0xffffff80 +const UNSIGNED_INT64_MAX_NUMBER = Long.fromBytes([-1, -1, -1, -1, -1, -1, -1, -128]) + module.exports = class Encoder { constructor() { this.buffer = Buffer.alloc(0) @@ -109,6 +114,51 @@ module.exports = class Encoder { return this } + // Based on: + // https://github.com/addthis/stream-lib/blob/master/src/main/java/com/clearspring/analytics/util/Varint.java#L106 + writeSignedVarInt32(value) { + const byteArray = [] + let encodedValue = this.encodeZigZag(value) + + while ((encodedValue & UNSIGNED_INT32_MAX_NUMBER) !== 0) { + byteArray.push((encodedValue & OTHER_BITS) | MOST_SIGNIFICANT_BIT) + encodedValue >>>= 7 + } + + byteArray.push(encodedValue & OTHER_BITS) + this.buffer = Buffer.concat([this.buffer, Buffer.from(byteArray)]) + return this + } + + encodeZigZag(value) { + return (value << 1) ^ (value >> 31) + } + + writeSignedVarInt64(value) { + const byteArray = [] + let longValue = this.encodeZigZag64(value) + + while (longValue.and(UNSIGNED_INT64_MAX_NUMBER).notEquals(Long.fromInt(0))) { + byteArray.push( + longValue + .and(OTHER_BITS) + .or(MOST_SIGNIFICANT_BIT) + .toInt() + ) + longValue = longValue.shiftRightUnsigned(7) + } + + byteArray.push(longValue.toInt()) + + this.buffer = Buffer.concat([this.buffer, Buffer.from(byteArray)]) + return this + } + + encodeZigZag64(value) { + const longValue = Long.fromValue(value) + return longValue.shiftLeft(1).xor(longValue.shiftRight(63)) + } + size() { return Buffer.byteLength(this.buffer) } diff --git a/src/protocol/encoder.spec.js b/src/protocol/encoder.spec.js new file mode 100644 index 000000000..3d7f6bb80 --- /dev/null +++ b/src/protocol/encoder.spec.js @@ -0,0 +1,176 @@ +const Long = require('long') + +const Encoder = require('./encoder') +const Decoder = require('./decoder') + +const MAX_SAFE_POSITIVE_SIGNED_INT = 2147483647 +const MIN_SAFE_NEGATIVE_SIGNED_INT = -2147483648 + +describe('Protocol > Encoder', () => { + const signed32 = number => new Encoder().writeSignedVarInt32(number).buffer + const decode32 = buffer => new Decoder(buffer).readSignedVarInt32() + + const signed64 = number => new Encoder().writeSignedVarInt64(number).buffer + const decode64 = buffer => new Decoder(buffer).readSignedVarInt64() + + const B = (...args) => Buffer.from(args) + const L = value => Long.fromString(`${value}`) + + describe('varint', () => { + test('encode signed int32 numbers', () => { + expect(signed32(0)).toEqual(B(0x00)) + expect(signed32(1)).toEqual(B(0x02)) + expect(signed32(63)).toEqual(B(0x7e)) + expect(signed32(64)).toEqual(B(0x80, 0x01)) + expect(signed32(8191)).toEqual(B(0xfe, 0x7f)) + expect(signed32(8192)).toEqual(B(0x80, 0x80, 0x01)) + expect(signed32(1048575)).toEqual(B(0xfe, 0xff, 0x7f)) + expect(signed32(1048576)).toEqual(B(0x80, 0x80, 0x80, 0x01)) + expect(signed32(134217727)).toEqual(B(0xfe, 0xff, 0xff, 0x7f)) + expect(signed32(134217728)).toEqual(B(0x80, 0x80, 0x80, 0x80, 0x01)) + + expect(signed32(-1)).toEqual(B(0x01)) + expect(signed32(-64)).toEqual(B(0x7f)) + expect(signed32(-65)).toEqual(B(0x81, 0x01)) + expect(signed32(-8192)).toEqual(B(0xff, 0x7f)) + expect(signed32(-8193)).toEqual(B(0x81, 0x80, 0x01)) + expect(signed32(-1048576)).toEqual(B(0xff, 0xff, 0x7f)) + expect(signed32(-1048577)).toEqual(B(0x81, 0x80, 0x80, 0x01)) + expect(signed32(-134217728)).toEqual(B(0xff, 0xff, 0xff, 0x7f)) + expect(signed32(-134217729)).toEqual(B(0x81, 0x80, 0x80, 0x80, 0x01)) + }) + + test('encode signed int32 boundaries', () => { + expect(signed32(MAX_SAFE_POSITIVE_SIGNED_INT)).toEqual(B(0xfe, 0xff, 0xff, 0xff, 0x0f)) + expect(signed32(MIN_SAFE_NEGATIVE_SIGNED_INT)).toEqual(B(0xff, 0xff, 0xff, 0xff, 0x0f)) + }) + + test('decode int32 numbers', () => { + expect(decode32(signed32(0))).toEqual(0) + expect(decode32(signed32(1))).toEqual(1) + expect(decode32(signed32(63))).toEqual(63) + expect(decode32(signed32(64))).toEqual(64) + expect(decode32(signed32(8191))).toEqual(8191) + expect(decode32(signed32(8192))).toEqual(8192) + expect(decode32(signed32(1048575))).toEqual(1048575) + expect(decode32(signed32(1048576))).toEqual(1048576) + expect(decode32(signed32(134217727))).toEqual(134217727) + expect(decode32(signed32(134217728))).toEqual(134217728) + + expect(decode32(signed32(-1))).toEqual(-1) + expect(decode32(signed32(-64))).toEqual(-64) + expect(decode32(signed32(-65))).toEqual(-65) + expect(decode32(signed32(-8192))).toEqual(-8192) + expect(decode32(signed32(-8193))).toEqual(-8193) + expect(decode32(signed32(-1048576))).toEqual(-1048576) + expect(decode32(signed32(-1048577))).toEqual(-1048577) + expect(decode32(signed32(-134217728))).toEqual(-134217728) + expect(decode32(signed32(-134217729))).toEqual(-134217729) + }) + + test('decode signed int32 boundaries', () => { + expect(decode32(signed32(MAX_SAFE_POSITIVE_SIGNED_INT))).toEqual(MAX_SAFE_POSITIVE_SIGNED_INT) + expect(decode32(signed32(MIN_SAFE_NEGATIVE_SIGNED_INT))).toEqual(MIN_SAFE_NEGATIVE_SIGNED_INT) + }) + }) + + describe('varlong', () => { + test('encode signed int64 number', () => { + expect(signed64(0)).toEqual(B(0x00)) + expect(signed64(1)).toEqual(B(0x02)) + expect(signed64(63)).toEqual(B(0x7e)) + expect(signed64(64)).toEqual(B(0x80, 0x01)) + expect(signed64(8191)).toEqual(B(0xfe, 0x7f)) + expect(signed64(8192)).toEqual(B(0x80, 0x80, 0x01)) + expect(signed64(1048575)).toEqual(B(0xfe, 0xff, 0x7f)) + expect(signed64(1048576)).toEqual(B(0x80, 0x80, 0x80, 0x01)) + expect(signed64(134217727)).toEqual(B(0xfe, 0xff, 0xff, 0x7f)) + expect(signed64(134217728)).toEqual(B(0x80, 0x80, 0x80, 0x80, 0x01)) + expect(signed64(MAX_SAFE_POSITIVE_SIGNED_INT)).toEqual(B(0xfe, 0xff, 0xff, 0xff, 0x0f)) + expect(signed64(L('17179869183'))).toEqual(B(0xfe, 0xff, 0xff, 0xff, 0x7f)) + expect(signed64(L('17179869184'))).toEqual(B(0x80, 0x80, 0x80, 0x80, 0x80, 0x01)) + expect(signed64(L('2199023255551'))).toEqual(B(0xfe, 0xff, 0xff, 0xff, 0xff, 0x7f)) + expect(signed64(L('2199023255552'))).toEqual(B(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01)) + expect(signed64(L('281474976710655'))).toEqual(B(0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f)) + expect(signed64(L('281474976710656'))).toEqual( + B(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01) + ) + expect(signed64(L('36028797018963967'))).toEqual( + B(0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f) + ) + expect(signed64(L('36028797018963968'))).toEqual( + B(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01) + ) + expect(signed64(L('4611686018427387903'))).toEqual( + B(0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f) + ) + expect(signed64(L('4611686018427387904'))).toEqual( + B(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01) + ) + expect(signed64(Long.MAX_VALUE)).toEqual( + B(0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01) + ) + + expect(signed64(-1)).toEqual(B(0x01)) + expect(signed64(-64)).toEqual(B(0x7f)) + expect(signed64(-65)).toEqual(B(0x81, 0x01)) + expect(signed64(-8192)).toEqual(B(0xff, 0x7f)) + expect(signed64(-8193)).toEqual(B(0x81, 0x80, 0x01)) + expect(signed64(-1048576)).toEqual(B(0xff, 0xff, 0x7f)) + expect(signed64(-1048577)).toEqual(B(0x81, 0x80, 0x80, 0x01)) + expect(signed64(-134217728)).toEqual(B(0xff, 0xff, 0xff, 0x7f)) + expect(signed64(-134217729)).toEqual(B(0x81, 0x80, 0x80, 0x80, 0x01)) + expect(signed64(MIN_SAFE_NEGATIVE_SIGNED_INT)).toEqual(B(0xff, 0xff, 0xff, 0xff, 0x0f)) + expect(signed64(L('-17179869184'))).toEqual(B(0xff, 0xff, 0xff, 0xff, 0x7f)) + expect(signed64(L('-17179869185'))).toEqual(B(0x81, 0x80, 0x80, 0x80, 0x80, 0x01)) + expect(signed64(L('-2199023255552'))).toEqual(B(0xff, 0xff, 0xff, 0xff, 0xff, 0x7f)) + expect(signed64(L('-2199023255553'))).toEqual(B(0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01)) + expect(signed64(L('-281474976710656'))).toEqual(B(0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f)) + expect(signed64(L('-281474976710657'))).toEqual( + B(0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 1) + ) + expect(signed64(L('-36028797018963968'))).toEqual( + B(0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f) + ) + expect(signed64(L('-36028797018963969'))).toEqual( + B(0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01) + ) + expect(signed64(L('-4611686018427387904'))).toEqual( + B(0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f) + ) + expect(signed64(L('-4611686018427387905'))).toEqual( + B(0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01) + ) + expect(signed64(Long.MIN_VALUE)).toEqual( + B(0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01) + ) + }) + + test('decode signed int64 number', () => { + expect(decode64(signed64(0))).toEqual(L(0)) + expect(decode64(signed64(1))).toEqual(L(1)) + expect(decode64(signed64(63))).toEqual(L(63)) + expect(decode64(signed64(64))).toEqual(L(64)) + expect(decode64(signed64(8191))).toEqual(L(8191)) + expect(decode64(signed64(8192))).toEqual(L(8192)) + expect(decode64(signed64(1048575))).toEqual(L(1048575)) + expect(decode64(signed64(1048576))).toEqual(L(1048576)) + expect(decode64(signed64(134217727))).toEqual(L(134217727)) + expect(decode64(signed64(134217728))).toEqual(L(134217728)) + expect(decode64(signed64(MAX_SAFE_POSITIVE_SIGNED_INT))).toEqual( + L(MAX_SAFE_POSITIVE_SIGNED_INT) + ) + expect(decode64(signed64(L('17179869183')))).toEqual(L('17179869183')) + expect(decode64(signed64(L('17179869184')))).toEqual(L('17179869184')) + expect(decode64(signed64(L('2199023255551')))).toEqual(L('2199023255551')) + expect(decode64(signed64(L('2199023255552')))).toEqual(L('2199023255552')) + expect(decode64(signed64(L('281474976710655')))).toEqual(L('281474976710655')) + expect(decode64(signed64(L('281474976710656')))).toEqual(L('281474976710656')) + expect(decode64(signed64(L('36028797018963967')))).toEqual(L('36028797018963967')) + expect(decode64(signed64(L('36028797018963968')))).toEqual(L('36028797018963968')) + expect(decode64(signed64(L('4611686018427387903')))).toEqual(L('4611686018427387903')) + expect(decode64(signed64(L('4611686018427387904')))).toEqual(L('4611686018427387904')) + expect(decode64(signed64(Long.MAX_VALUE))).toEqual(Long.MAX_VALUE) + }) + }) +}) diff --git a/yarn.lock b/yarn.lock index 4a563c90c..b43899d38 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2287,9 +2287,9 @@ log-update@^1.0.2: ansi-escapes "^1.0.0" cli-cursor "^1.0.2" -long@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/long/-/long-3.2.0.tgz#d821b7138ca1cb581c172990ef14db200b5c474b" +long@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" longest@^1.0.1: version "1.0.1"