Merge pull request #61 from tulios/add-support-to-protocol-varint

Add support to protocol varint and varlong

.gitignore

@@ -4,3 +4,4 @@ test-report.xml
 junit.xml
 coverage/
 yarn-error.log
+.vscode

package.json

@@ -47,7 +47,7 @@
     "prettier": "^1.7.0"
   },
   "dependencies": {
-    "long": "^3.2.0"
+    "long": "^4.0.0"
   },
   "lint-staged": {
     "*.js": [

src/protocol/decoder.js

@@ -5,6 +5,9 @@ const INT16_SIZE = 2
 const INT32_SIZE = 4
 const INT64_SIZE = 8
 
+const MOST_SIGNIFICANT_BIT = 0x80 // 128
+const OTHER_BITS = 0x7f // 127
+
 module.exports = class Decoder {
   static int32Size() {
     return INT32_SIZE
@@ -120,6 +123,42 @@ module.exports = class Decoder {
     return array
   }
 
+  readSignedVarInt32() {
+    let currentByte
+    let result = 0
+    let i = 0
+
+    do {
+      currentByte = this.buffer[this.offset++]
+      result += (currentByte & OTHER_BITS) << i
+      i += 7
+    } while (currentByte >= MOST_SIGNIFICANT_BIT)
+
+    return this.decodeZigZag(result)
+  }
+
+  decodeZigZag(value) {
+    return (value >>> 1) ^ -(value & 1)
+  }
+
+  readSignedVarInt64() {
+    let currentByte
+    let result = Long.fromInt(0)
+    let i = 0
+
+    do {
+      currentByte = this.buffer[this.offset++]
+      result = result.add(Long.fromInt(currentByte & OTHER_BITS).shiftLeft(i))
+      i += 7
+    } while (currentByte >= MOST_SIGNIFICANT_BIT)
+
+    return this.decodeZigZag64(result)
+  }
+
+  decodeZigZag64(longValue) {
+    return longValue.shiftRightUnsigned(1).xor(longValue.and(Long.fromInt(1)).negate())
+  }
+
   slice(size) {
     return new Decoder(this.buffer.slice(this.offset, this.offset + size))
   }

src/protocol/encoder.js

@@ -5,6 +5,11 @@ const INT16_SIZE = 2
 const INT32_SIZE = 4
 const INT64_SIZE = 8
 
+const MOST_SIGNIFICANT_BIT = 0x80 // 128
+const OTHER_BITS = 0x7f // 127
+const UNSIGNED_INT32_MAX_NUMBER = 0xffffff80
+const UNSIGNED_INT64_MAX_NUMBER = Long.fromBytes([-1, -1, -1, -1, -1, -1, -1, -128])
+
 module.exports = class Encoder {
   constructor() {
     this.buffer = Buffer.alloc(0)
@@ -109,6 +114,51 @@ module.exports = class Encoder {
     return this
   }
 
+  // Based on:
+  // https://github.com/addthis/stream-lib/blob/master/src/main/java/com/clearspring/analytics/util/Varint.java#L106
+  writeSignedVarInt32(value) {
+    const byteArray = []
+    let encodedValue = this.encodeZigZag(value)
+
+    while ((encodedValue & UNSIGNED_INT32_MAX_NUMBER) !== 0) {
+      byteArray.push((encodedValue & OTHER_BITS) | MOST_SIGNIFICANT_BIT)
+      encodedValue >>>= 7
+    }
+
+    byteArray.push(encodedValue & OTHER_BITS)
+    this.buffer = Buffer.concat([this.buffer, Buffer.from(byteArray)])
+    return this
+  }
+
+  encodeZigZag(value) {
+    return (value << 1) ^ (value >> 31)
+  }
+
+  writeSignedVarInt64(value) {
+    const byteArray = []
+    let longValue = this.encodeZigZag64(value)
+
+    while (longValue.and(UNSIGNED_INT64_MAX_NUMBER).notEquals(Long.fromInt(0))) {
+      byteArray.push(
+        longValue
+          .and(OTHER_BITS)
+          .or(MOST_SIGNIFICANT_BIT)
+          .toInt()
+      )
+      longValue = longValue.shiftRightUnsigned(7)
+    }
+
+    byteArray.push(longValue.toInt())
+
+    this.buffer = Buffer.concat([this.buffer, Buffer.from(byteArray)])
+    return this
+  }
+
+  encodeZigZag64(value) {
+    const longValue = Long.fromValue(value)
+    return longValue.shiftLeft(1).xor(longValue.shiftRight(63))
+  }
+
   size() {
     return Buffer.byteLength(this.buffer)
   }

src/protocol/encoder.spec.js

@@ -0,0 +1,176 @@
+const Long = require('long')
+
+const Encoder = require('./encoder')
+const Decoder = require('./decoder')
+
+const MAX_SAFE_POSITIVE_SIGNED_INT = 2147483647
+const MIN_SAFE_NEGATIVE_SIGNED_INT = -2147483648
+
+describe('Protocol > Encoder', () => {
+  const signed32 = number => new Encoder().writeSignedVarInt32(number).buffer
+  const decode32 = buffer => new Decoder(buffer).readSignedVarInt32()
+
+  const signed64 = number => new Encoder().writeSignedVarInt64(number).buffer
+  const decode64 = buffer => new Decoder(buffer).readSignedVarInt64()
+
+  const B = (...args) => Buffer.from(args)
+  const L = value => Long.fromString(`${value}`)
+
+  describe('varint', () => {
+    test('encode signed int32 numbers', () => {
+      expect(signed32(0)).toEqual(B(0x00))
+      expect(signed32(1)).toEqual(B(0x02))
+      expect(signed32(63)).toEqual(B(0x7e))
+      expect(signed32(64)).toEqual(B(0x80, 0x01))
+      expect(signed32(8191)).toEqual(B(0xfe, 0x7f))
+      expect(signed32(8192)).toEqual(B(0x80, 0x80, 0x01))
+      expect(signed32(1048575)).toEqual(B(0xfe, 0xff, 0x7f))
+      expect(signed32(1048576)).toEqual(B(0x80, 0x80, 0x80, 0x01))
+      expect(signed32(134217727)).toEqual(B(0xfe, 0xff, 0xff, 0x7f))
+      expect(signed32(134217728)).toEqual(B(0x80, 0x80, 0x80, 0x80, 0x01))
+
+      expect(signed32(-1)).toEqual(B(0x01))
+      expect(signed32(-64)).toEqual(B(0x7f))
+      expect(signed32(-65)).toEqual(B(0x81, 0x01))
+      expect(signed32(-8192)).toEqual(B(0xff, 0x7f))
+      expect(signed32(-8193)).toEqual(B(0x81, 0x80, 0x01))
+      expect(signed32(-1048576)).toEqual(B(0xff, 0xff, 0x7f))
+      expect(signed32(-1048577)).toEqual(B(0x81, 0x80, 0x80, 0x01))
+      expect(signed32(-134217728)).toEqual(B(0xff, 0xff, 0xff, 0x7f))
+      expect(signed32(-134217729)).toEqual(B(0x81, 0x80, 0x80, 0x80, 0x01))
+    })
+
+    test('encode signed int32 boundaries', () => {
+      expect(signed32(MAX_SAFE_POSITIVE_SIGNED_INT)).toEqual(B(0xfe, 0xff, 0xff, 0xff, 0x0f))
+      expect(signed32(MIN_SAFE_NEGATIVE_SIGNED_INT)).toEqual(B(0xff, 0xff, 0xff, 0xff, 0x0f))
+    })
+
+    test('decode int32 numbers', () => {
+      expect(decode32(signed32(0))).toEqual(0)
+      expect(decode32(signed32(1))).toEqual(1)
+      expect(decode32(signed32(63))).toEqual(63)
+      expect(decode32(signed32(64))).toEqual(64)
+      expect(decode32(signed32(8191))).toEqual(8191)
+      expect(decode32(signed32(8192))).toEqual(8192)
+      expect(decode32(signed32(1048575))).toEqual(1048575)
+      expect(decode32(signed32(1048576))).toEqual(1048576)
+      expect(decode32(signed32(134217727))).toEqual(134217727)
+      expect(decode32(signed32(134217728))).toEqual(134217728)
+
+      expect(decode32(signed32(-1))).toEqual(-1)
+      expect(decode32(signed32(-64))).toEqual(-64)
+      expect(decode32(signed32(-65))).toEqual(-65)
+      expect(decode32(signed32(-8192))).toEqual(-8192)
+      expect(decode32(signed32(-8193))).toEqual(-8193)
+      expect(decode32(signed32(-1048576))).toEqual(-1048576)
+      expect(decode32(signed32(-1048577))).toEqual(-1048577)
+      expect(decode32(signed32(-134217728))).toEqual(-134217728)
+      expect(decode32(signed32(-134217729))).toEqual(-134217729)
+    })
+
+    test('decode signed int32 boundaries', () => {
+      expect(decode32(signed32(MAX_SAFE_POSITIVE_SIGNED_INT))).toEqual(MAX_SAFE_POSITIVE_SIGNED_INT)
+      expect(decode32(signed32(MIN_SAFE_NEGATIVE_SIGNED_INT))).toEqual(MIN_SAFE_NEGATIVE_SIGNED_INT)
+    })
+  })
+
+  describe('varlong', () => {
+    test('encode signed int64 number', () => {
+      expect(signed64(0)).toEqual(B(0x00))
+      expect(signed64(1)).toEqual(B(0x02))
+      expect(signed64(63)).toEqual(B(0x7e))
+      expect(signed64(64)).toEqual(B(0x80, 0x01))
+      expect(signed64(8191)).toEqual(B(0xfe, 0x7f))
+      expect(signed64(8192)).toEqual(B(0x80, 0x80, 0x01))
+      expect(signed64(1048575)).toEqual(B(0xfe, 0xff, 0x7f))
+      expect(signed64(1048576)).toEqual(B(0x80, 0x80, 0x80, 0x01))
+      expect(signed64(134217727)).toEqual(B(0xfe, 0xff, 0xff, 0x7f))
+      expect(signed64(134217728)).toEqual(B(0x80, 0x80, 0x80, 0x80, 0x01))
+      expect(signed64(MAX_SAFE_POSITIVE_SIGNED_INT)).toEqual(B(0xfe, 0xff, 0xff, 0xff, 0x0f))
+      expect(signed64(L('17179869183'))).toEqual(B(0xfe, 0xff, 0xff, 0xff, 0x7f))
+      expect(signed64(L('17179869184'))).toEqual(B(0x80, 0x80, 0x80, 0x80, 0x80, 0x01))
+      expect(signed64(L('2199023255551'))).toEqual(B(0xfe, 0xff, 0xff, 0xff, 0xff, 0x7f))
+      expect(signed64(L('2199023255552'))).toEqual(B(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01))
+      expect(signed64(L('281474976710655'))).toEqual(B(0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f))
+      expect(signed64(L('281474976710656'))).toEqual(
+        B(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01)
+      )
+      expect(signed64(L('36028797018963967'))).toEqual(
+        B(0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f)
+      )
+      expect(signed64(L('36028797018963968'))).toEqual(
+        B(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01)
+      )
+      expect(signed64(L('4611686018427387903'))).toEqual(
+        B(0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f)
+      )
+      expect(signed64(L('4611686018427387904'))).toEqual(
+        B(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01)
+      )
+      expect(signed64(Long.MAX_VALUE)).toEqual(
+        B(0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01)
+      )
+
+      expect(signed64(-1)).toEqual(B(0x01))
+      expect(signed64(-64)).toEqual(B(0x7f))
+      expect(signed64(-65)).toEqual(B(0x81, 0x01))
+      expect(signed64(-8192)).toEqual(B(0xff, 0x7f))
+      expect(signed64(-8193)).toEqual(B(0x81, 0x80, 0x01))
+      expect(signed64(-1048576)).toEqual(B(0xff, 0xff, 0x7f))
+      expect(signed64(-1048577)).toEqual(B(0x81, 0x80, 0x80, 0x01))
+      expect(signed64(-134217728)).toEqual(B(0xff, 0xff, 0xff, 0x7f))
+      expect(signed64(-134217729)).toEqual(B(0x81, 0x80, 0x80, 0x80, 0x01))
+      expect(signed64(MIN_SAFE_NEGATIVE_SIGNED_INT)).toEqual(B(0xff, 0xff, 0xff, 0xff, 0x0f))
+      expect(signed64(L('-17179869184'))).toEqual(B(0xff, 0xff, 0xff, 0xff, 0x7f))
+      expect(signed64(L('-17179869185'))).toEqual(B(0x81, 0x80, 0x80, 0x80, 0x80, 0x01))
+      expect(signed64(L('-2199023255552'))).toEqual(B(0xff, 0xff, 0xff, 0xff, 0xff, 0x7f))
+      expect(signed64(L('-2199023255553'))).toEqual(B(0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01))
+      expect(signed64(L('-281474976710656'))).toEqual(B(0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f))
+      expect(signed64(L('-281474976710657'))).toEqual(
+        B(0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 1)
+      )
+      expect(signed64(L('-36028797018963968'))).toEqual(
+        B(0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f)
+      )
+      expect(signed64(L('-36028797018963969'))).toEqual(
+        B(0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01)
+      )
+      expect(signed64(L('-4611686018427387904'))).toEqual(
+        B(0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f)
+      )
+      expect(signed64(L('-4611686018427387905'))).toEqual(
+        B(0x81, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01)
+      )
+      expect(signed64(Long.MIN_VALUE)).toEqual(
+        B(0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01)
+      )
+    })
+
+    test('decode signed int64 number', () => {
+      expect(decode64(signed64(0))).toEqual(L(0))
+      expect(decode64(signed64(1))).toEqual(L(1))
+      expect(decode64(signed64(63))).toEqual(L(63))
+      expect(decode64(signed64(64))).toEqual(L(64))
+      expect(decode64(signed64(8191))).toEqual(L(8191))
+      expect(decode64(signed64(8192))).toEqual(L(8192))
+      expect(decode64(signed64(1048575))).toEqual(L(1048575))
+      expect(decode64(signed64(1048576))).toEqual(L(1048576))
+      expect(decode64(signed64(134217727))).toEqual(L(134217727))
+      expect(decode64(signed64(134217728))).toEqual(L(134217728))
+      expect(decode64(signed64(MAX_SAFE_POSITIVE_SIGNED_INT))).toEqual(
+        L(MAX_SAFE_POSITIVE_SIGNED_INT)
+      )
+      expect(decode64(signed64(L('17179869183')))).toEqual(L('17179869183'))
+      expect(decode64(signed64(L('17179869184')))).toEqual(L('17179869184'))
+      expect(decode64(signed64(L('2199023255551')))).toEqual(L('2199023255551'))
+      expect(decode64(signed64(L('2199023255552')))).toEqual(L('2199023255552'))
+      expect(decode64(signed64(L('281474976710655')))).toEqual(L('281474976710655'))
+      expect(decode64(signed64(L('281474976710656')))).toEqual(L('281474976710656'))
+      expect(decode64(signed64(L('36028797018963967')))).toEqual(L('36028797018963967'))
+      expect(decode64(signed64(L('36028797018963968')))).toEqual(L('36028797018963968'))
+      expect(decode64(signed64(L('4611686018427387903')))).toEqual(L('4611686018427387903'))
+      expect(decode64(signed64(L('4611686018427387904')))).toEqual(L('4611686018427387904'))
+      expect(decode64(signed64(Long.MAX_VALUE))).toEqual(Long.MAX_VALUE)
+    })
+  })
+})

yarn.lock

@@ -2287,9 +2287,9 @@ log-update@^1.0.2:
     ansi-escapes "^1.0.0"
     cli-cursor "^1.0.2"
 
-long@^3.2.0:
-  version "3.2.0"
-  resolved "https://registry.yarnpkg.com/long/-/long-3.2.0.tgz#d821b7138ca1cb581c172990ef14db200b5c474b"
+long@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28"
 
 longest@^1.0.1:
   version "1.0.1"