Fix isses raised by dougwilson.

*  Split Lexer.js out of Template.js so that tests can fetch that
   directly instead of conditionally exporting the lexer.
*  Also split the tests for Lexer.js out of the tests for Template.js
*  Template no longer does its own Array escaping.  All escaping is
   delegated to either SqlString.escape or SqlString.escapeId.

* Now provides SqlString.identifier which produces a { toSqlString: ... }
  value similar to that produced by SqlString.raw.
* Changed SqlString.escapeId to pass through values like those produced
  by SqlString.identifier.

Still TODO: update documentation.

*  Template.Identifier and Template.SqlFragment use `instanceof` in
   dodgy ways and are redundant.
*  null/undefined do not interpolate well
*  Buffers do not interpolate properly inside quoted sections.
   Now uses Buffer.toString('binary')
*  Lexer recognizes in one place that -- comments need to have a
   following space per dev.mysql.com/doc/refman/5.7/en/ansi-diff-comments.html
   but a later does not handle that caveat.
*  There was no way to pass (stringifyObjects, timeZone) options to
   interpolations.  Changes enable
   sql({ timeZone })`...`

Author: mikesamuel

lib/SqlString.js

@@ -14,6 +14,10 @@ var CHARS_ESCAPE_MAP    = {
   '\''   : '\\\'',
   '\\'   : '\\\\'
 };
+var ONE_ID_PATTERN = '^`(?:[^`]|``)+`$'  // TODO(mikesamuel): Should allow ``?
+// One or more Identifiers separated by dots.
+var QUALIFIED_ID_REGEXP = new RegExp(
+  '^' + ONE_ID_PATTERN + '(?:[.]' + ONE_ID_PATTERN + ')*$');
 
 SqlString.escapeId = function escapeId(val, forbidQualified) {
   if (Array.isArray(val)) {
@@ -24,6 +28,14 @@ SqlString.escapeId = function escapeId(val, forbidQualified) {
     }
 
     return sql;
+  } else if (val && typeof val.toSqlString === 'function') {
+    // If it corresponds to an identifier token, let it through.
+    var sqlString = val.toSqlString();
+    if (QUALIFIED_ID_REGEXP.test(sqlString)) {
+      return sqlString;
+    } else {
+      throw new TypeError();
+    }
   } else if (forbidQualified) {
     return '`' + String(val).replace(ID_GLOBAL_REGEXP, '``') + '`';
   } else {
@@ -140,7 +152,7 @@ SqlString.dateToString = function dateToString(date, timeZone) {
       dt.setTime(dt.getTime() + (tz * 60000));
     }
 
-    year       = dt.getUTCFullYear();
+    year        = dt.getUTCFullYear();
     month       = dt.getUTCMonth() + 1;
     day         = dt.getUTCDate();
     hour        = dt.getUTCHours();
@@ -187,6 +199,17 @@ SqlString.raw = function raw(sql) {
   };
 };
 
+SqlString.identifier = function identifier(id, forbidQualified) {
+  if (typeof id !== 'string') {
+    throw new TypeError('argument id must be a string');
+  }
+
+  var idToken = SqlString.escapeId(id, forbidQualified);
+  return {
+    toSqlString: function toSqlString() { return idToken; }
+  };
+};
+
 function escapeString(val) {
   var chunkIndex = CHARS_GLOBAL_REGEXP.lastIndex = 0;
   var escapedVal = '';

lib/Template.js

@@ -27,28 +27,4 @@ try {
   module.exports.calledAsTemplateTagQuick = function (firstArg, nArgs) {
     return false;
   };
-
-  function stringWrapper() {
-    function TypedString(content) {
-      if (!(this instanceof TypedString)) {
-        return new TypedString(content);
-      }
-      this.content = String(content);
-    }
-    TypedString.prototype.toString = function () {
-      return this.content;
-    };
-    return TypedString;
-  }
-
-  /**
-   * @param {string} content
-   * @constructor
-   */
-  module.exports.Identifier = stringWrapper();
-  /**
-   * @param {string} content
-   * @constructor
-   */
-  module.exports.Fragment = stringWrapper();
 }

lib/es6/Lexer.js

@@ -0,0 +1,107 @@
+// A simple lexer for SQL.
+// SQL has many divergent dialects with subtly different
+// conventions for string escaping and comments.
+// This just attempts to roughly tokenize MySQL's specific variant.
+// See also
+// https://www.w3.org/2005/05/22-SPARQL-MySQL/sql_yacc
+// https://github.com/twitter/mysql/blob/master/sql/sql_lex.cc
+// https://dev.mysql.com/doc/refman/5.7/en/string-literals.html
+
+// "--" followed by whitespace starts a line comment
+// "#"
+// "/*" starts an inline comment ended at first "*/"
+// \N means null
+// Prefixed strings x'...' is a hex string,  b'...' is a binary string, ....
+// '...', "..." are strings.  `...` escapes identifiers.
+// doubled delimiters and backslash both escape
+// doubled delimiters work in `...` identifiers
+
+exports.makeLexer = makeLexer;
+
+const WS = '[\\t\\r\\n ]';
+const PREFIX_BEFORE_DELIMITER = new RegExp(
+  '^(?:' +
+    (
+      // Comment
+      // https://dev.mysql.com/doc/refman/5.7/en/comments.html
+      // https://dev.mysql.com/doc/refman/5.7/en/ansi-diff-comments.html
+      '--(?=' + WS + ')[^\\r\\n]*' +
+      '|#[^\\r\\n]*' +
+      '|/[*][\\s\\S]*?[*]/'
+    ) +
+    '|' +
+    (
+      // Run of non-comment non-string starts
+      '(?:[^\'"`\\-/#]|-(?!-' + WS + ')|/(?![*]))'
+    ) +
+    ')*');
+const DELIMITED_BODIES = {
+  '\'' : /^(?:[^'\\]|\\[\s\S]|'')*/,
+  '"'  : /^(?:[^"\\]|\\[\s\S]|"")*/,
+  '`'  : /^(?:[^`\\]|\\[\s\S]|``)*/
+};
+
+/**
+ * Template tag that creates a new Error with a message.
+ * @param {!Array.<string>} strs a valid TemplateObject.
+ * @return {string} A message suitable for the Error constructor.
+ */
+function msg (strs, ...dyn) {
+  let message = String(strs[0]);
+  for (let i = 0; i < dyn.length; ++i) {
+    message += JSON.stringify(dyn[i]) + strs[i + 1];
+  }
+  return message;
+}
+
+/**
+ * Returns a stateful function that can be fed chunks of input and
+ * which returns a delimiter context.
+ *
+ * @return {!function (string) : string}
+ *    a stateful function that takes a string of SQL text and
+ *    returns the context after it.  Subsequent calls will assume
+ *    that context.
+ */
+function makeLexer () {
+  let errorMessage = null;
+  let delimiter = null;
+  return (text) => {
+    if (errorMessage) {
+      // Replay the error message if we've already failed.
+      throw new Error(errorMessage);
+    }
+    text = String(text);
+    while (text) {
+      const pattern = delimiter
+        ? DELIMITED_BODIES[delimiter]
+        : PREFIX_BEFORE_DELIMITER;
+      const match = pattern.exec(text);
+      if (!match) {
+        throw new Error(
+          errorMessage = msg`Failed to lex starting at ${text}`);
+      }
+      let nConsumed = match[0].length;
+      if (text.length > nConsumed) {
+        const chr = text.charAt(nConsumed);
+        if (delimiter) {
+          if (chr === delimiter) {
+            delimiter = null;
+            ++nConsumed;
+          } else {
+            throw new Error(
+              errorMessage = msg`Expected ${chr} at ${text}`);
+          }
+        } else if (Object.hasOwnProperty.call(DELIMITED_BODIES, chr)) {
+          delimiter = chr;
+          ++nConsumed;
+        } else {
+          throw new Error(
+            errorMessage = msg`Expected delimiter at ${text}`);
+        }
+      }
+      text = text.substring(nConsumed);
+    }
+    return delimiter;
+  };
+}

lib/es6/README.md

@@ -0,0 +1,4 @@
+The source files herein use ES6 features and are loaded optimistically.
+
+Calls that `require` them from source files in the parent directory
+should be prepared for parsing to fail on EcmaScript engines.