Improve bitfield.setRange() (#113)

* Improve `bitfield.setRange()`

* Clean up remote bitfield as well

* Use `bits.fill()`

* Typo

* Add `remoteBitfield.setRange()`

Author: kasperisager

lib/bitfield.js

@@ -1,7 +1,6 @@
-// TODO: needs massive improvements obvs
-
 const BigSparseArray = require('big-sparse-array')
 const b4a = require('b4a')
+const bits = require('bits-to-bytes')
 
 class FixedBitfield {
   constructor (index, bitfield) {
@@ -11,26 +10,17 @@ class FixedBitfield {
   }
 
   get (index) {
-    const j = index & 31
-    const i = (index - j) / 32
-
-    return i < this.bitfield.length && (this.bitfield[i] & (1 << j)) !== 0
+    return bits.get(this.bitfield, index)
   }
 
   set (index, val) {
-    const j = index & 31
-    const i = (index - j) / 32
-    const v = this.bitfield[i]
-
-    if (val === ((v & (1 << j)) !== 0)) return false
-
-    const u = val
-      ? v | (1 << j)
-      : v ^ (1 << j)
-
-    if (u === v) return false
+    return bits.set(this.bitfield, index, val)
+  }
 
-    this.bitfield[i] = u
+  setRange (start, length, val) {
+    // Using fill instead of setRange is ~2 orders of magnitude faster, but does
+    // have the downside of not being able to tell if any bits actually changed.
+    bits.fill(this.bitfield, val, start, start + length)
     return true
   }
 }
@@ -44,7 +34,11 @@ module.exports = class Bitfield {
     this.resumed = !!(buf && buf.byteLength >= 4)
 
     const all = this.resumed
-      ? new Uint32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4))
+      ? new Uint32Array(
+          buf.buffer,
+          buf.byteOffset,
+          Math.floor(buf.byteLength / 4)
+        )
       : new Uint32Array(1024)
 
     for (let i = 0; i < all.length; i += 1024) {
@@ -55,33 +49,52 @@ module.exports = class Bitfield {
   }
 
   get (index) {
-    const j = index & 32767
-    const i = (index - j) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
+
     const p = this.pages.get(i)
 
     return p ? p.get(j) : false
   }
 
   set (index, val) {
-    const j = index & 32767
-    const i = (index - j) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
 
     let p = this.pages.get(i)
 
-    if (!p) {
-      if (!val) return
+    if (!p && val) {
       p = this.pages.set(i, new FixedBitfield(i, new Uint32Array(1024)))
     }
 
-    if (!p.set(j, val) || p.dirty) return
-
-    p.dirty = true
-    this.unflushed.push(p)
+    if (p && p.set(j, val) && !p.dirty) {
+      p.dirty = true
+      this.unflushed.push(p)
+    }
   }
 
   setRange (start, length, val) {
-    for (let i = 0; i < length; i++) {
-      this.set(start + i, val)
+    let j = start & (this.pageSize - 1)
+    let i = (start - j) / this.pageSize
+
+    while (length > 0) {
+      let p = this.pages.get(i)
+
+      if (!p && val) {
+        p = this.pages.set(i, new FixedBitfield(i, new Uint32Array(1024)))
+      }
+
+      const end = Math.min(j + length, this.pageSize)
+      const range = end - j
+
+      if (p && p.setRange(j, range, val) && !p.dirty) {
+        p.dirty = true
+        this.unflushed.push(p)
+      }
+
+      j = 0
+      i++
+      length -= range
     }
   }
 
@@ -120,7 +133,12 @@ module.exports = class Bitfield {
       let error = null
 
       for (const page of this.unflushed) {
-        const buf = b4a.from(page.bitfield.buffer, page.bitfield.byteOffset, page.bitfield.byteLength)
+        const buf = b4a.from(
+          page.bitfield.buffer,
+          page.bitfield.byteOffset,
+          page.bitfield.byteLength
+        )
+
         page.dirty = false
         this.storage.write(page.index * 4096, buf, done)
       }
@@ -151,7 +169,7 @@ module.exports = class Bitfield {
 }
 
 function ensureSize (uint32, size) {
-  if (uint32.length === size) return uint32
+  if (uint32.byteLength === size) return uint32
   const a = new Uint32Array(1024)
   a.set(uint32, 0)
   return a

lib/remote-bitfield.js

@@ -1,24 +1,45 @@
 const BigSparseArray = require('big-sparse-array')
+const bits = require('bits-to-bytes')
 
 module.exports = class RemoteBitfield {
   constructor () {
+    this.pageSize = 32768
     this.pages = new BigSparseArray()
   }
 
   get (index) {
-    const r = index & 32767
-    const i = (index - r) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
+
     const p = this.pages.get(i)
 
-    return p ? (p[r >>> 5] & (1 << (r & 31))) !== 0 : false
+    return p ? bits.get(p, j) : false
   }
 
   set (index, val) {
-    const r = index & 32767
-    const i = (index - r) / 32768
+    const j = index & (this.pageSize - 1)
+    const i = (index - j) / this.pageSize
+
     const p = this.pages.get(i) || this.pages.set(i, new Uint32Array(1024))
 
-    if (val) p[r >>> 5] |= (1 << (r & 31))
-    else p[r >>> 5] &= ~(1 << (r & 31))
+    bits.set(p, j, val)
+  }
+
+  setRange (start, length, val) {
+    let j = start & (this.pageSize - 1)
+    let i = (start - j) / this.pageSize
+
+    while (length > 0) {
+      const p = this.pages.get(i) || this.pages.set(i, new Uint32Array(1024))
+
+      const end = Math.min(j + length, this.pageSize)
+      const range = end - j
+
+      bits.fill(p, val, j, end)
+
+      j = 0
+      i++
+      length -= range
+    }
   }
 }

lib/replicator.js

@@ -577,9 +577,7 @@ class Peer {
   onrange ({ drop, start, length }) {
     const has = drop === false
 
-    for (const end = start + length; start < end; start++) {
-      this.remoteBitfield.set(start, has)
-    }
+    this.remoteBitfield.setRange(start, length, has)
 
     if (drop === false) this._update()
   }

package.json

@@ -35,6 +35,7 @@
     "@hyperswarm/secret-stream": "^6.0.0",
     "b4a": "^1.1.0",
     "big-sparse-array": "^1.0.2",
+    "bits-to-bytes": "^1.2.0",
     "codecs": "^3.0.0",
     "compact-encoding": "^2.5.0",
     "crc32-universal": "^1.0.1",