fix(jsparser): Parsing time for big nested arrays is now linear

Author: Ruben Bridgewater

benchmark/index.js

@@ -63,7 +63,7 @@ for (i = 0; i < arraySize; i++) {
 var arrayBuffer = new Buffer(array)
 
 var bigArraySize = 1000
-var bigArrayChunks = [new Buffer('*' + bigArraySize * 2)]
+var bigArrayChunks = [new Buffer('*1\r\n*1\r\n*' + bigArraySize * 2)]
 for (i = 0; i < bigArraySize; i++) {
   size = (Math.random() * 10000 | 0)
   if (i % 2) {
@@ -251,27 +251,27 @@ suite.add('JS PARSER BUF: * array', function () {
   parserBuffer.execute(arrayBuffer)
 })
 
-// BIG ARRAYS
+// BIG NESTED ARRAYS
 
-suite.add('\nHIREDIS: * big array', function () {
+suite.add('\nHIREDIS: * big nested array', function () {
   for (var i = 0; i < bigArrayChunks.length; i++) {
     parserHiRedis.execute(bigArrayChunks[i])
   }
 })
 
-suite.add('HIREDIS BUF: * big array', function () {
+suite.add('HIREDIS BUF: * big nested array', function () {
   for (var i = 0; i < bigArrayChunks.length; i++) {
     parserHiRedisBuffer.execute(bigArrayChunks[i])
   }
 })
 
-suite.add('JS PARSER: * big array', function () {
+suite.add('JS PARSER: * big nested array', function () {
   for (var i = 0; i < bigArrayChunks.length; i++) {
     parser.execute(bigArrayChunks[i])
   }
 })
 
-suite.add('JS PARSER BUF: * big array', function () {
+suite.add('JS PARSER BUF: * big nested array', function () {
   for (var i = 0; i < bigArrayChunks.length; i++) {
     parserBuffer.execute(bigArrayChunks[i])
   }

changelog.md

@@ -1,3 +1,9 @@
+## v.2.4.0 - xx Jan, 2017
+
+Bugfixes
+
+-  Parsing time for big nested arrays is now linear
+
 ## v.2.3.0 - 25 Nov, 2016
 
 Features

lib/parser.js

@@ -209,13 +209,36 @@ function parseArray (parser) {
   return parseArrayElements(parser, responses, 0)
 }
 
+/**
+ * Push a partly parsed array to the stack
+ *
+ * @param parser
+ * @param elem
+ * @param i
+ * @returns {undefined}
+ */
+function pushArrayCache (parser, elem, pos) {
+  parser.arrayCache.push(elem)
+  parser.arrayPos.push(pos)
+}
+
 /**
  * Parse chunked redis array response
  * @param parser
  * @returns {*}
  */
 function parseArrayChunks (parser) {
-  return parseArrayElements(parser, parser.arrayCache, parser.arrayPos)
+  var tmp = parser.arrayCache.pop()
+  var pos = parser.arrayPos.pop()
+  if (parser.arrayCache.length) {
+    var res = parseArrayChunks(parser)
+    if (!res) {
+      pushArrayCache(parser, tmp, pos)
+      return
+    }
+    tmp[pos++] = res
+  }
+  return parseArrayElements(parser, tmp, pos)
 }
 
 /**
@@ -230,16 +253,15 @@ function parseArrayElements (parser, responses, i) {
   while (i < responses.length) {
     var offset = parser.offset
     if (parser.offset >= bufferLength) {
-      parser.arrayCache = responses
-      parser.arrayPos = i
-      parser.offset = offset
+      pushArrayCache(parser, responses, i)
       return
     }
     var response = parseType(parser, parser.buffer[parser.offset++])
     if (response === undefined) {
-      parser.arrayCache = responses
-      parser.arrayPos = i
-      parser.offset = offset
+      if (!parser.arrayCache.length) {
+        parser.offset = offset
+      }
+      pushArrayCache(parser, responses, i)
       return
     }
     responses[i] = response
@@ -324,8 +346,8 @@ function JavascriptRedisParser (options) {
   this.bigOffset = 0
   this.totalChunkSize = 0
   this.bufferCache = []
-  this.arrayCache = null
-  this.arrayPos = 0
+  this.arrayCache = []
+  this.arrayPos = []
 }
 
 /**
@@ -439,31 +461,29 @@ JavascriptRedisParser.prototype.execute = function execute (buffer) {
     buffer.copy(newBuffer, remainingLength, 0, buffer.length)
     this.buffer = newBuffer
     this.offset = 0
-    if (this.arrayCache) {
+    if (this.arrayCache.length) {
       arr = parseArrayChunks(this)
       if (!arr) {
         return
       }
       this.returnReply(arr)
-      this.arrayCache = null
     }
   } else if (this.totalChunkSize + buffer.length >= this.bigStrSize) {
     this.bufferCache.push(buffer)
-    if (this.optionReturnBuffers === false && !this.arrayCache) {
+    if (this.optionReturnBuffers === false && !this.arrayCache.length) {
       this.returnReply(concatBulkString(this))
       this.buffer = buffer
     } else {
       this.buffer = concatBuffer(this, this.totalChunkSize + buffer.length)
       this.offset = 0
-      if (this.arrayCache) {
+      if (this.arrayCache.length) {
         arr = parseArrayChunks(this)
         if (!arr) {
           this.bigStrSize = 0
           this.bufferCache = []
           return
         }
         this.returnReply(arr)
-        this.arrayCache = null
       }
     }
     this.bigStrSize = 0
@@ -479,7 +499,7 @@ JavascriptRedisParser.prototype.execute = function execute (buffer) {
     var type = this.buffer[this.offset++]
     var response = parseType(this, type)
     if (response === undefined) {
-      if (!this.arrayCache) {
+      if (!this.arrayCache.length) {
         this.offset = offset
       }
       return