refactor: split consume(regex) as consumeType() (#569)

saschanaz · web-flow · commit cd65a26f83bd · 2021-04-20T15:28:26.000+02:00
Fixes #313
diff --git a/lib/productions/argument.js b/lib/productions/argument.js
@@ -36,7 +36,9 @@ export class Argument extends Base {
     if (!tokens.optional) {
       tokens.variadic = tokeniser.consume("...");
     }
-    tokens.name = tokeniser.consume("identifier", ...argumentNameKeywords);
+    tokens.name =
+      tokeniser.consumeType("identifier") ||
+      tokeniser.consume(...argumentNameKeywords);
     if (!tokens.name) {
       return tokeniser.unconsume(start_position);
     }
diff --git a/lib/productions/attribute.js b/lib/productions/attribute.js
@@ -39,7 +39,8 @@ export class Attribute extends Base {
       type_with_extended_attributes(tokeniser, "attribute-type") ||
       tokeniser.error("Attribute lacks a type");
     tokens.name =
-      tokeniser.consume("identifier", "async", "required") ||
+      tokeniser.consumeType("identifier") ||
+      tokeniser.consume("async", "required") ||
       tokeniser.error("Attribute lacks a name");
     tokens.termination =
       tokeniser.consume(";") ||
diff --git a/lib/productions/callback.js b/lib/productions/callback.js
@@ -16,7 +16,7 @@ export class CallbackFunction extends Base {
       new CallbackFunction({ source: tokeniser.source, tokens })
     );
     tokens.name =
-      tokeniser.consume("identifier") ||
+      tokeniser.consumeType("identifier") ||
       tokeniser.error("Callback lacks a name");
     tokeniser.current = ret.this;
     tokens.assign =
diff --git a/lib/productions/constant.js b/lib/productions/constant.js
@@ -22,7 +22,7 @@ export class Constant extends Base {
     let idlType = primitive_type(tokeniser);
     if (!idlType) {
       const base =
-        tokeniser.consume("identifier") ||
+        tokeniser.consumeType("identifier") ||
         tokeniser.error("Const lacks a type");
       idlType = new Type({ source: tokeniser.source, tokens: { base } });
     }
@@ -31,7 +31,8 @@ export class Constant extends Base {
     }
     idlType.type = "const-type";
     tokens.name =
-      tokeniser.consume("identifier") || tokeniser.error("Const lacks a name");
+      tokeniser.consumeType("identifier") ||
+      tokeniser.error("Const lacks a name");
     tokens.assign =
       tokeniser.consume("=") || tokeniser.error("Const lacks value assignment");
     tokens.value =
diff --git a/lib/productions/container.js b/lib/productions/container.js
@@ -11,7 +11,7 @@ function inheritance(tokeniser) {
     return {};
   }
   const inheritance =
-    tokeniser.consume("identifier") ||
+    tokeniser.consumeType("identifier") ||
     tokeniser.error("Inheritance lacks a type");
   return { colon, inheritance };
 }
@@ -26,7 +26,7 @@ export class Container extends Base {
   static parse(tokeniser, instance, { type, inheritable, allowedMembers }) {
     const { tokens } = instance;
     tokens.name =
-      tokeniser.consume("identifier") ||
+      tokeniser.consumeType("identifier") ||
       tokeniser.error(`Missing name in ${instance.type}`);
     tokeniser.current = instance;
     instance = autoParenter(instance);
diff --git a/lib/productions/default.js b/lib/productions/default.js
@@ -12,15 +12,16 @@ export class Default extends Base {
     }
     const def =
       const_value(tokeniser) ||
-      tokeniser.consume("string", "null", "[", "{") ||
+      tokeniser.consumeType("string") ||
+      tokeniser.consume("null", "[", "{") ||
       tokeniser.error("No value for default");
     const expression = [def];
-    if (def.type === "[") {
+    if (def.value === "[") {
       const close =
         tokeniser.consume("]") ||
         tokeniser.error("Default sequence value must be empty");
       expression.push(close);
-    } else if (def.type === "{") {
+    } else if (def.value === "{") {
       const close =
         tokeniser.consume("}") ||
         tokeniser.error("Default dictionary value must be empty");
diff --git a/lib/productions/enum.js b/lib/productions/enum.js
@@ -7,7 +7,7 @@ class EnumValue extends Token {
    * @param {import("../tokeniser").Tokeniser} tokeniser
    */
   static parse(tokeniser) {
-    const value = tokeniser.consume("string");
+    const value = tokeniser.consumeType("string");
     if (value) {
       return new EnumValue({ source: tokeniser.source, tokens: { value } });
     }
@@ -46,7 +46,8 @@ export class Enum extends Base {
       return;
     }
     tokens.name =
-      tokeniser.consume("identifier") || tokeniser.error("No name for enum");
+      tokeniser.consumeType("identifier") ||
+      tokeniser.error("No name for enum");
     const ret = autoParenter(new Enum({ source: tokeniser.source, tokens }));
     tokeniser.current = ret.this;
     tokens.open = tokeniser.consume("{") || tokeniser.error("Bodyless enum");
@@ -55,7 +56,7 @@ export class Enum extends Base {
       allowDangler: true,
       listName: "enumeration",
     });
-    if (tokeniser.probe("string")) {
+    if (tokeniser.probeType("string")) {
       tokeniser.error("No comma between enum values");
     }
     tokens.close =
diff --git a/lib/productions/extended-attributes.js b/lib/productions/extended-attributes.js
@@ -58,7 +58,7 @@ class ExtendedAttributeParameters extends Base {
       new ExtendedAttributeParameters({ source: tokeniser.source, tokens })
     );
     if (tokens.assign) {
-      tokens.secondaryName = tokeniser.consume(...extAttrValueSyntax);
+      tokens.secondaryName = tokeniser.consumeType(...extAttrValueSyntax);
     }
     tokens.open = tokeniser.consume("(");
     if (tokens.open) {
@@ -122,7 +122,7 @@ export class SimpleExtendedAttribute extends Base {
    * @param {import("../tokeniser").Tokeniser} tokeniser
    */
   static parse(tokeniser) {
-    const name = tokeniser.consume("identifier");
+    const name = tokeniser.consumeType("identifier");
     if (name) {
       return new SimpleExtendedAttribute({
         source: tokeniser.source,
diff --git a/lib/productions/field.js b/lib/productions/field.js
@@ -21,7 +21,7 @@ export class Field extends Base {
       type_with_extended_attributes(tokeniser, "dictionary-type") ||
       tokeniser.error("Dictionary member lacks a type");
     tokens.name =
-      tokeniser.consume("identifier") ||
+      tokeniser.consumeType("identifier") ||
       tokeniser.error("Dictionary member lacks a name");
     ret.default = Default.parse(tokeniser);
     if (tokens.required && ret.default)
diff --git a/lib/productions/helpers.js b/lib/productions/helpers.js
@@ -49,14 +49,9 @@ export function list(tokeniser, { parser, allowDangler, listName = "list" }) {
  * @param {import("../tokeniser").Tokeniser} tokeniser
  */
 export function const_value(tokeniser) {
-  return tokeniser.consume(
-    "true",
-    "false",
-    "Infinity",
-    "-Infinity",
-    "NaN",
-    "decimal",
-    "integer"
+  return (
+    tokeniser.consumeType("decimal", "integer") ||
+    tokeniser.consume("true", "false", "Infinity", "-Infinity", "NaN")
   );
 }
 
@@ -67,23 +62,26 @@ export function const_value(tokeniser) {
  */
 export function const_data({ type, value }) {
   switch (type) {
+    case "decimal":
+    case "integer":
+      return { type: "number", value };
+    case "string":
+      return { type: "string", value: value.slice(1, -1) };
+  }
+
+  switch (value) {
     case "true":
     case "false":
-      return { type: "boolean", value: type === "true" };
+      return { type: "boolean", value: value === "true" };
     case "Infinity":
     case "-Infinity":
-      return { type: "Infinity", negative: type.startsWith("-") };
+      return { type: "Infinity", negative: value.startsWith("-") };
     case "[":
       return { type: "sequence", value: [] };
     case "{":
       return { type: "dictionary" };
-    case "decimal":
-    case "integer":
-      return { type: "number", value };
-    case "string":
-      return { type: "string", value: value.slice(1, -1) };
     default:
-      return { type };
+      return { type: value };
   }
 }
 
diff --git a/lib/productions/includes.js b/lib/productions/includes.js
@@ -8,7 +8,7 @@ export class Includes extends Base {
    * @param {import("../tokeniser").Tokeniser} tokeniser
    */
   static parse(tokeniser) {
-    const target = tokeniser.consume("identifier");
+    const target = tokeniser.consumeType("identifier");
     if (!target) {
       return;
     }
@@ -19,7 +19,7 @@ export class Includes extends Base {
       return;
     }
     tokens.mixin =
-      tokeniser.consume("identifier") ||
+      tokeniser.consumeType("identifier") ||
       tokeniser.error("Incomplete includes statement");
     tokens.termination =
       tokeniser.consume(";") ||
diff --git a/lib/productions/operation.js b/lib/productions/operation.js
@@ -33,7 +33,8 @@ export class Operation extends Base {
     }
     ret.idlType =
       return_type(tokeniser) || tokeniser.error("Missing return type");
-    tokens.name = tokeniser.consume("identifier", "includes");
+    tokens.name =
+      tokeniser.consumeType("identifier") || tokeniser.consume("includes");
     tokens.open =
       tokeniser.consume("(") || tokeniser.error("Invalid operation");
     ret.arguments = argument_list(tokeniser);
diff --git a/lib/productions/token.js b/lib/productions/token.js
@@ -10,7 +10,7 @@ export class Token extends Base {
    */
   static parser(tokeniser, type) {
     return () => {
-      const value = tokeniser.consume(type);
+      const value = tokeniser.consumeType(type);
       if (value) {
         return new Token({ source: tokeniser.source, tokens: { value } });
       }
diff --git a/lib/productions/type.js b/lib/productions/type.js
@@ -31,8 +31,8 @@ function generic_type(tokeniser, typeName) {
   );
   ret.tokens.open =
     tokeniser.consume("<") ||
-    tokeniser.error(`No opening bracket after ${base.type}`);
-  switch (base.type) {
+    tokeniser.error(`No opening bracket after ${base.value}`);
+  switch (base.value) {
     case "Promise": {
       if (tokeniser.probe("["))
         tokeniser.error("Promise type cannot have extended attribute");
@@ -47,7 +47,7 @@ function generic_type(tokeniser, typeName) {
     case "ObservableArray": {
       const subtype =
         type_with_extended_attributes(tokeniser, typeName) ||
-        tokeniser.error(`Missing ${base.type} subtype`);
+        tokeniser.error(`Missing ${base.value} subtype`);
       ret.subtype.push(subtype);
       break;
     }
@@ -72,10 +72,10 @@ function generic_type(tokeniser, typeName) {
       break;
     }
   }
-  if (!ret.idlType) tokeniser.error(`Error parsing generic type ${base.type}`);
+  if (!ret.idlType) tokeniser.error(`Error parsing generic type ${base.value}`);
   ret.tokens.close =
     tokeniser.consume(">") ||
-    tokeniser.error(`Missing closing bracket after ${base.type}`);
+    tokeniser.error(`Missing closing bracket after ${base.value}`);
   return ret.this;
 }
 
@@ -97,11 +97,9 @@ function type_suffix(tokeniser, obj) {
 function single_type(tokeniser, typeName) {
   let ret = generic_type(tokeniser, typeName) || primitive_type(tokeniser);
   if (!ret) {
-    const base = tokeniser.consume(
-      "identifier",
-      ...stringTypes,
-      ...typeNameKeywords
-    );
+    const base =
+      tokeniser.consumeType("identifier") ||
+      tokeniser.consume(...stringTypes, ...typeNameKeywords);
     if (!base) {
       return;
     }
diff --git a/lib/productions/typedef.js b/lib/productions/typedef.js
@@ -21,7 +21,7 @@ export class Typedef extends Base {
       type_with_extended_attributes(tokeniser, "typedef-type") ||
       tokeniser.error("Typedef lacks a type");
     tokens.name =
-      tokeniser.consume("identifier") ||
+      tokeniser.consumeType("identifier") ||
       tokeniser.error("Typedef lacks a name");
     tokeniser.current = ret.this;
     tokens.termination =
diff --git a/lib/tokeniser.js b/lib/tokeniser.js
@@ -156,7 +156,7 @@ function tokenise(str) {
               syntaxError(tokens, lastIndex, null, message)
             );
           } else if (nonRegexTerminals.includes(token.value)) {
-            token.type = token.value;
+            token.type = "inline";
           }
         }
       }
@@ -167,7 +167,7 @@ function tokenise(str) {
     for (const punctuation of punctuations) {
       if (str.startsWith(punctuation, lastCharIndex)) {
         tokens.push({
-          type: punctuation,
+          type: "inline",
           value: punctuation,
           trivia,
           line,
@@ -242,25 +242,47 @@ export class Tokeniser {
   /**
    * @param {string} type
    */
-  probe(type) {
+  probeType(type) {
     return (
       this.source.length > this.position &&
       this.source[this.position].type === type
     );
   }
 
+  /**
+   * @param {string} value
+   */
+  probe(value) {
+    return (
+      this.probeType("inline") && this.source[this.position].value === value
+    );
+  }
+
   /**
    * @param  {...string} candidates
    */
-  consume(...candidates) {
+  consumeType(...candidates) {
     for (const type of candidates) {
-      if (!this.probe(type)) continue;
+      if (!this.probeType(type)) continue;
       const token = this.source[this.position];
       this.position++;
       return token;
     }
   }
 
+  /**
+   * @param  {...string} candidates
+   */
+  consume(...candidates) {
+    if (!this.probeType("inline")) return;
+    const token = this.source[this.position];
+    for (const value of candidates) {
+      if (token.value !== value) continue;
+      this.position++;
+      return token;
+    }
+  }
+
   /**
    * @param {number} position
    */
diff --git a/lib/webidl2.js b/lib/webidl2.js
@@ -83,7 +83,7 @@ function parseByTokens(tokeniser, options) {
       autoParenter(def).extAttrs = ea;
       defs.push(def);
     }
-    const eof = consume("eof");
+    const eof = tokeniser.consumeType("eof");
     if (options.concrete) {
       defs.push(eof);
     }

Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,9 @@ export class Argument extends Base {`
`36`	`36`	`if (!tokens.optional) {`
`37`	`37`	`tokens.variadic = tokeniser.consume("...");`
`38`	`38`	`}`
`39`		`- tokens.name = tokeniser.consume("identifier", ...argumentNameKeywords);`
	`39`	`+ tokens.name =`
	`40`	`+ tokeniser.consumeType("identifier") \|\|`
	`41`	`+ tokeniser.consume(...argumentNameKeywords);`
`40`	`42`	`if (!tokens.name) {`
`41`	`43`	`return tokeniser.unconsume(start_position);`
`42`	`44`	`}`