rocksdb: commit overwrites full state

index.js

@@ -454,7 +454,10 @@ class Hypercore extends EventEmitter {
 
   get flushedLength () {
     if (this.opened === false) return 0
-    return this.state === this.core.state ? this.core.tree.length : this.state.treeLength
+    if (this.state === this.core.state) return this.core.tree.length
+    const flushed = this.state.flushedLength()
+
+    return flushed === -1 ? this.state.tree.length : flushed
   }
 
   /**

lib/bit-interlude.js

@@ -1,17 +1,16 @@
-const assert = require('nanoassert')
 const b4a = require('b4a')
 const quickbit = require('./compat').quickbit
 
 module.exports = class BitInterlude {
   constructor (bitfield) {
-    this.drop = false
     this.bitfield = bitfield
     this.ranges = []
   }
 
   contiguousLength (from) {
-    if (this.drop && this.ranges.length > 0 && this.ranges[0].start < from) {
-      return this.ranges[0].start
+    for (const r of this.ranges) {
+      if (r.start > from) break
+      if (!r.value && r.start <= from) return r.start
     }
 
     // TODO: be smarter
@@ -38,25 +37,35 @@ module.exports = class BitInterlude {
         continue
       }
 
-      return this.drop === false
+      return r.value
     }
 
     return this.bitfield.get(index)
   }
 
   setRange (start, end, value) {
-    assert(this.drop !== value || this.ranges.length === 0)
-    assert(value === true || this.ranges.length === 0)
-
-    this.drop = value === false
+    if (start === end) return
 
     let r = null
 
     for (let i = 0; i < this.ranges.length; i++) {
       r = this.ranges[i]
 
       // if already inside, stop
-      if (r.start <= start && end <= r.end) return
+      if (r.start <= start && end <= r.end) {
+        if (value === r.value) return
+
+        const lower = { start: r.start, end: start, value: r.value }
+        const upper = { start: end, end: r.end, value: r.value }
+
+        let remove = 1
+
+        if (lower.start < lower.end) this.ranges.splice(i++, remove--, lower)
+        this.ranges.splice(i++, remove, { start, end, value })
+        if (upper.start < upper.end) this.ranges.splice(i, 0, upper)
+
+        return
+      }
 
       // we wanna overun the interval
       if (start > r.end) {
@@ -65,25 +74,32 @@ module.exports = class BitInterlude {
 
       // we overran but this interval is ending after us, move it back
       if (end >= r.start && end <= r.end) {
-        r.start = start
+        r.start = r.value === value ? start : end
+        if (r.value !== value) this.ranges.splice(i, 0, { start, end, value })
         return
       }
 
       // we overran but our start is contained in this interval, move start back
       if (start >= r.start && start <= r.end) {
+        if (r.value !== value) {
+          this.ranges.splice(++i, 0, { start, end, value })
+          r.end = start
+          return
+        }
+
         start = r.start
       }
 
       let remove = 0
 
       for (let j = i; j < this.ranges.length; j++) {
         const n = this.ranges[j]
-        if (n.start > end) break
+        if (n.start > end || n.value !== value) break
         if (n.start <= end && n.end > end) end = n.end
         remove++
       }
 
-      this.ranges.splice(i, remove, { start, end })
+      this.ranges.splice(i, remove, { start, end, value })
       return
     }
 
@@ -97,41 +113,49 @@ module.exports = class BitInterlude {
       if (r.end > start) return
     }
 
-    this.ranges.push({ start, end })
+    this.ranges.push({ start, end, value })
   }
 
   flush (writer, debug) {
-    if (!this.ranges.length) return { ranges: [], drop: this.drop }
+    if (!this.ranges.length) return []
 
     let index = this.ranges[0].start
     const final = this.ranges[this.ranges.length - 1].end
 
     let i = 0
 
     while (index < final) {
-      const page = this.bitfield.getPage(index, this.drop === false)
-      const buf = b4a.allocUnsafe(page.bitfield.byteLength)
+      const page = this.bitfield.getBitfield(index) // read only
+      const pageIndex = page ? page.index : this.bitfield.getPageIndex(index)
+
+      const buf = b4a.allocUnsafe(this.bitfield.getPageByteLength())
 
       const view = new DataView(
         buf.buffer,
         buf.byteOffset,
         buf.byteLength
       )
 
-      for (let i = 0; i < page.bitfield.length; i++) {
-        view.setUint32(i * 4, page.bitfield[i], true)
+      if (page) {
+        for (let i = 0; i < page.bitfield.length; i++) {
+          view.setUint32(i * 4, page.bitfield[i], true)
+        }
       }
 
-      const last = (page.index + 1) * (buf.byteLength << 3)
-      const offset = page.index * buf.byteLength << 3
+      const last = (pageIndex + 1) * (buf.byteLength << 3)
+      const offset = pageIndex * buf.byteLength << 3
+
+      let hasValue = false
 
       while (i < this.ranges.length) {
-        const { start, end } = this.ranges[i]
+        const { start, end, value } = this.ranges[i]
+
+        if (!hasValue && value) hasValue = true
 
         const from = start < index ? index : start
         const to = end < last ? end : last
 
-        quickbit.fill(buf, this.drop === false, from - offset, to - offset)
+        quickbit.fill(buf, value, from - offset, to - offset)
 
         index = to
 
@@ -140,12 +164,9 @@ module.exports = class BitInterlude {
         i++
       }
 
-      writer.putBitfieldPage(page.index, buf)
+      if (page || hasValue) writer.putBitfieldPage(pageIndex, buf)
     }
 
-    return {
-      ranges: this.ranges,
-      drop: this.drop
-    }
+    return this.ranges
   }
 }

lib/bitfield.js

@@ -171,6 +171,9 @@ class BitfieldSegment {
 }
 
 module.exports = class Bitfield {
+  static BITS_PER_PAGE = BITS_PER_PAGE
+  static BYTES_PER_PAGE = BYTES_PER_PAGE
+
   constructor (buffer) {
     this.resumed = !!(buffer && buffer.byteLength >= 0)
 
@@ -239,8 +242,7 @@ module.exports = class Bitfield {
   }
 
   getBitfield (index) {
-    const j = index & (BITS_PER_PAGE - 1)
-    const i = (index - j) / BITS_PER_PAGE
+    const i = this.getPageIndex(index)
 
     const p = this._pages.get(i)
     return p || null
@@ -270,9 +272,17 @@ module.exports = class Bitfield {
     return p ? p.get(j) : false
   }
 
-  getPage (index, create) {
+  getPageByteLength () {
+    return BYTES_PER_PAGE
+  }
+
+  getPageIndex (index) {
     const j = index & (BITS_PER_PAGE - 1)
-    const i = (index - j) / BITS_PER_PAGE
+    return (index - j) / BITS_PER_PAGE
+  }
+
+  getPage (index, create) {
+    const i = this.getPageIndex(index)
 
     let p = this._pages.get(i)
 
@@ -453,9 +463,9 @@ module.exports = class Bitfield {
   //   })
   // }
 
-  onupdate ({ ranges, drop }) {
-    for (const r of ranges) {
-      this.setRange(r.start, r.end - r.start, drop === false)
+  onupdate (ranges) {
+    for (const { start, end, value } of ranges) {
+      this.setRange(start, end - start, value)
     }
   }
 

lib/copy-prologue.js

@@ -2,6 +2,7 @@ const crypto = require('hypercore-crypto')
 const flat = require('flat-tree')
 const b4a = require('b4a')
 const quickbit = require('quickbit-universal')
+const Bitfield = require('./bitfield')
 
 const MAX_BATCH_USED = 4 * 1024 * 1024
 const MIN_BATCH_USED = 512 * 1024
@@ -192,11 +193,11 @@ function prologueToTree (prologue) {
 }
 
 function getBitfieldPage (index) {
-  return Math.floor(index / 32768)
+  return Math.floor(index / Bitfield.BITS_PER_PAGE)
 }
 
 function getBitfieldOffset (index) {
-  return index & 32767
+  return index & (Bitfield.BITS_PER_PAGE - 1)
 }
 
 function walkTree (stack, target, batch) {