Initial main commit

Author: Ruben Bridgewater

.gitignore

@@ -1,27 +1,4 @@
-# Logs
 logs
 *.log
-
-# Runtime data
-pids
-*.pid
-*.seed
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
 coverage
-
-# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (http://nodejs.org/api/addons.html)
-build/Release
-
-# Dependency directory
-# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
 node_modules

.jshintignore

@@ -0,0 +1,4 @@
+node_modules/**
+coverage/**
+**.md
+**.log

.jshintrc

@@ -0,0 +1,20 @@
+{
+    "eqeqeq": true, // Prohibits the use of == and != in favor of === and !==
+    "noarg": true, // Prohibit use of `arguments.caller` and `arguments.callee`
+    "undef": true, // Require all non-global variables be declared before they are used.
+    "unused": "vars", // Warn unused variables, but not unused params
+    "strict": true, // Require `use strict` pragma in every file.
+    "nonbsp": true, // don't allow non utf-8 pages to break
+    "forin": true, // don't allow not filtert for in loops
+    "freeze": true, // prohibit overwriting prototypes of native objects
+    "maxdepth": 4,
+    "latedef": true,
+    "maxparams": 3,
+
+    // Environment options
+    "node": true, // Enable globals available when code is running inside of the NodeJS runtime environment.
+    "mocha": true,
+
+    // Relaxing options
+    "boss": true // Accept statements like `while (key = keys.pop()) {}`
+}

.travis

@@ -0,0 +1,7 @@
+language: node_js
+sudo: false
+node_js:
+  - "0.10"
+  - "0.12"
+  - "4.0"
+  - "5.0"

README.md

@@ -1,4 +1,104 @@
-# node-redis-parser
-Javascript Redis protocol (RESP) parser
+[![Build Status](https://travis-ci.org/NodeRedis/redis-parser.png?branch=master)](https://travis-ci.org/NodeRedis/redis-parser)
 
-WIP
+# redis-parser
+
+A high performance redis parser solution built for [node_redis](https://github.com/NodeRedis/node_redis) and [ioredis](https://github.com/ioredis/luin).
+
+## Install
+
+Install with [NPM](https://npmjs.org/):
+
+```
+npm install redis-parser
+```
+
+## Usage
+
+```js
+new Parser(options);
+```
+
+### Possible options
+
+`returnReply`: *function*; mandatory
+`returnError`: *function*; mandatory
+`returnFatalError`: *function*; optional, defaults to the returnError function
+`returnBuffers`: *boolean*; optional, defaults to false
+`name`: *javascript|hiredis*; optional, defaults to hiredis and falls back to the js parser if not available
+`context`: *A class instance that the return functions get bound to*; optional
+
+### Example
+
+```js
+var Parser = require("redis-parser");
+
+function Library () {}
+
+Library.prototype.returnReply = function (reply) { ... }
+Library.prototype.returnError = function (err) { ... }
+Library.prototype.returnFatalError = function (err) { ... }
+
+var lib = new Library();
+
+var parser = new Parser({
+    returnReply: returnReply,
+    returnError: returnError,
+    returnFatalError: returnFatalError,
+    context: lib
+}); // This returns either a hiredis or the js parser instance depending on what's available
+
+Library.prototype.streamHandler = function () {
+    this.stream.on('data', function (buffer) {
+        // Here the data (e.g. `new Buffer('$5\r\nHello\r\n'`)) is passed to the parser and the result is passed to either function depending on the provided data.
+        // All [RESP](http://redis.io/topics/protocol) data will be properly parsed by the parser.
+        parser.execute(buffer);
+    });
+};
+```
+You do not have to use the context variable, but can also bind the function while passing them to the option object.
+
+And if you want to return buffers instead of strings, you can do this by adding the returnBuffers option.
+
+```js
+// Same functions as in the first example
+
+var parser = new Parser({
+    returnReply: returnReply.bind(lib),
+    returnError: returnError.bind(lib),
+    returnFatalError: returnFatalError.bind(lib),
+    returnBuffers: true // All strings are returned as buffer e.g. <Buffer 48 65 6c 6c 6f>
+});
+
+// The streamHandler as above
+```
+
+## Further info
+
+The [hiredis](https://github.com/redis/hiredis) parser is still the fasted parser for
+Node.js and therefor used as default in redis-parser if the hiredis parser is available.
+
+Otherwise the pure js NodeRedis parser is choosen that is almost as fast as the
+hiredis parser besides some situations in which it'll be a bit slower.
+
+## Contribute
+
+The js parser is already optimized but there are likely further optimizations possible.
+Besides running the tests you'll also have to run the change at least against the node_redis benchmark suite and post the improvement in the PR.
+If you want to write a own parser benchmark, that would also be great!
+
+```
+npm install
+npm test
+
+# Run node_redis benchmark (let's guess you cloned node_redis in another folder)
+cd ../redis
+npm install
+npm run benchmark parser=javascript > old.log
+# Replace the changed parser in the node_modules
+npm run benchmark parser=javascript > new.log
+node benchmarks/diff_multi_bench_output.js old.log new.log > improvement.log
+```
+
+## License
+
+[MIT](./LICENSE

index.js

@@ -0,0 +1,3 @@
+'use strict';
+
+module.exports = require('./lib/parser');

lib/hiredis.js

@@ -0,0 +1,36 @@
+'use strict';
+
+var hiredis = require('hiredis');
+
+function HiredisReplyParser(returnBuffers) {
+    this.name = 'hiredis';
+    this.reader = new hiredis.Reader({
+        return_buffers: returnBuffers
+    });
+}
+
+HiredisReplyParser.prototype.parseData = function () {
+    try {
+        return this.reader.get();
+    } catch (err) {
+        // Protocol errors land here
+        this.returnFatalError(err);
+        return void 0;
+    }
+};
+
+HiredisReplyParser.prototype.execute = function (data) {
+    this.reader.feed(data);
+    var reply = this.parseData();
+
+    while (reply !== undefined) {
+        if (reply && reply.name === 'Error') {
+            this.returnError(reply);
+        } else {
+            this.returnReply(reply);
+        }
+        reply = this.parseData();
+    }
+};
+
+module.exports = HiredisReplyParser;

lib/javascript.js

@@ -0,0 +1,171 @@
+'use strict';
+
+var util = require('util');
+
+function JavascriptReplyParser(return_buffers) {
+    this.name = 'javascript';
+    this.buffer = new Buffer(0);
+    this.offset = 0;
+    this.bigStrSize = 0;
+    this.chunksSize = 0;
+    this.buffers = [];
+    this.type = 0;
+    this.protocolError = false;
+    this.offsetCache = 0;
+    if (return_buffers) {
+        this.handleReply = function (start, end) {
+            return this.buffer.slice(start, end);
+        };
+    }
+}
+
+JavascriptReplyParser.prototype.handleReply = function (start, end) {
+    return this.buffer.toString('utf-8', start, end);
+};
+
+function IncompleteReadBuffer(message) {
+    this.name = 'IncompleteReadBuffer';
+    this.message = message;
+}
+util.inherits(IncompleteReadBuffer, Error);
+
+JavascriptReplyParser.prototype.parseResult = function (type) {
+    var start = 0,
+        end = 0,
+        packetHeader = 0,
+        reply;
+
+    if (type === 36) { // $
+        packetHeader = this.parseHeader();
+        // Packets with a size of -1 are considered null
+        if (packetHeader === -1) {
+            return null;
+        }
+        end = this.offset + packetHeader;
+        start = this.offset;
+        if (end + 2 > this.buffer.length) {
+            this.buffers.push(this.offsetCache === 0 ? this.buffer : this.buffer.slice(this.offsetCache));
+            this.chunksSize = this.buffers[0].length;
+            // Include the packetHeader delimiter
+            this.bigStrSize = packetHeader + 2;
+            throw new IncompleteReadBuffer('Wait for more data.');
+        }
+        // Set the offset to after the delimiter
+        this.offset = end + 2;
+        return this.handleReply(start, end);
+    } else if (type === 58) { // :
+        // Up to the delimiter
+        end = this.packetEndOffset();
+        start = this.offset;
+        // Include the delimiter
+        this.offset = end + 2;
+        // Return the coerced numeric value
+        return +this.buffer.toString('ascii', start, end);
+    } else if (type === 43) { // +
+        end = this.packetEndOffset();
+        start = this.offset;
+        this.offset = end + 2;
+        return this.handleReply(start, end);
+    } else if (type === 42) { // *
+        packetHeader = this.parseHeader();
+        if (packetHeader === -1) {
+            return null;
+        }
+        reply = [];
+        for (var i = 0; i < packetHeader; i++) {
+            if (this.offset >= this.buffer.length) {
+                throw new IncompleteReadBuffer('Wait for more data.');
+            }
+            reply.push(this.parseResult(this.buffer[this.offset++]));
+        }
+        return reply;
+    } else if (type === 45) { // -
+        end = this.packetEndOffset();
+        start = this.offset;
+        this.offset = end + 2;
+        return new Error(this.buffer.toString('utf-8', start, end));
+    } else {
+        return void 0;
+    }
+};
+
+JavascriptReplyParser.prototype.execute = function (buffer) {
+    if (this.chunksSize !== 0) {
+        if (this.bigStrSize > this.chunksSize + buffer.length) {
+            this.buffers.push(buffer);
+            this.chunksSize += buffer.length;
+            return;
+        }
+        this.buffers.push(buffer);
+        this.buffer = Buffer.concat(this.buffers, this.chunksSize + buffer.length);
+        this.buffers = [];
+        this.bigStrSize = 0;
+        this.chunksSize = 0;
+    } else if (this.offset >= this.buffer.length) {
+        this.buffer = buffer;
+    } else {
+        this.buffer = Buffer.concat([this.buffer.slice(this.offset), buffer]);
+    }
+    this.offset = 0;
+    this.protocolError = true;
+    this.run();
+};
+
+JavascriptReplyParser.prototype.tryParsing = function () {
+    try {
+        return this.parseResult(this.type);
+    } catch (err) {
+        // Catch the error (not enough data), rewind if it's an array,
+        // and wait for the next packet to appear
+        this.offset = this.offsetCache;
+        this.protocolError = false;
+        return void 0;
+    }
+};
+
+JavascriptReplyParser.prototype.run = function (buffer) {
+    // Set a rewind point. If a failure occurs, wait for the next execute()/append() and try again
+    this.offsetCache = this.offset;
+    this.type = this.buffer[this.offset++];
+    var reply = this.tryParsing();
+
+    while (reply !== undefined) {
+        if (this.type === 45) { // Errors -
+            this.returnError(reply);
+        } else {
+            this.returnReply(reply); // Strings + // Integers : // Bulk strings $ // Arrays *
+        }
+        this.offsetCache = this.offset;
+        this.type = this.buffer[this.offset++];
+        reply = this.tryParsing();
+    }
+    if (this.type !== undefined && this.protocolError === true) {
+        // Reset the buffer so the parser can handle following commands properly
+        this.buffer = new Buffer(0);
+        this.returnFatalError(new Error('Protocol error, got ' + JSON.stringify(String.fromCharCode(this.type)) + ' as reply type byte'));
+    }
+};
+
+JavascriptReplyParser.prototype.parseHeader = function () {
+    var end   = this.packetEndOffset(),
+        value = this.buffer.toString('ascii', this.offset, end) | 0;
+
+    this.offset = end + 2;
+    return value;
+};
+
+JavascriptReplyParser.prototype.packetEndOffset = function () {
+    var offset = this.offset,
+        len = this.buffer.length - 1;
+
+    while (this.buffer[offset] !== 0x0d && this.buffer[offset + 1] !== 0x0a) {
+        offset++;
+
+        if (offset >= len) {
+            throw new IncompleteReadBuffer('Did not see LF after NL reading multi bulk count (' + offset + ' => ' + this.buffer.length + ', ' + this.offset + ')');
+        }
+    }
+    return offset;
+};
+
+module.exports = JavascriptReplyParser;