Bugfix for an apparent infinite loop in grammar. (#163)

vpetrovykh · 1st1 · commit 1b72e04ac7fe · 2018-10-04T15:56:26.000-04:00
This is a case of semantically equivalent regex not being computationally equivalent. The issue also relates to how negative character classes (e.g. `[^abc]`) are treated. In the end I had to replace something of the form `(A | B)*` with `A* (BA*)*` to avoid grinding the regex engine to a halt. Issue #150.
diff --git a/grammars/MagicPython.cson b/grammars/MagicPython.cson
@@ -942,7 +942,7 @@ repository:
               {{ | }}
               | (?:
                 {
-                  \\w*? (\\.[[:alpha:]_]\\w*? | \\[[^\\]'"]+\\])*?
+                  \\w* (\\.[[:alpha:]_]\\w* | \\[[^\\]'"]+\\])*
                   (![rsa])?
                   ( : \\w? [<>=^]? [-+ ]? \\#?
                     \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )?
@@ -964,13 +964,11 @@ repository:
           (?x)
             (
               {
-                \\w*? (\\.[[:alpha:]_]\\w*? | \\[[^\\]'"]+\\])*?
+                \\w* (\\.[[:alpha:]_]\\w* | \\[[^\\]'"]+\\])*
                 (![rsa])?
                 (:)
-                  (
-                    [^'"{}\\n]+?
-                    |
-                    \\{ [^'"}\\n]*? \\}
+                  [^'"{}\\n]* (?:
+                    \\{ [^'"}\\n]*? \\} [^'"{}\\n]*
                   )*
               }
             )
diff --git a/grammars/MagicPython.tmLanguage b/grammars/MagicPython.tmLanguage
@@ -1444,7 +1444,7 @@
     {{ | }}
     | (?:
       {
-        \w*? (\.[[:alpha:]_]\w*? | \[[^\]'"]+\])*?
+        \w* (\.[[:alpha:]_]\w* | \[[^\]'"]+\])*
         (![rsa])?
         ( : \w? [&lt;&gt;=^]? [-+ ]? \#?
           \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )?
@@ -1477,13 +1477,11 @@
             <string>(?x)
   (
     {
-      \w*? (\.[[:alpha:]_]\w*? | \[[^\]'"]+\])*?
+      \w* (\.[[:alpha:]_]\w* | \[[^\]'"]+\])*
       (![rsa])?
       (:)
-        (
-          [^'"{}\n]+?
-          |
-          \{ [^'"}\n]*? \}
+        [^'"{}\n]* (?:
+          \{ [^'"}\n]*? \} [^'"{}\n]*
         )*
     }
   )
diff --git a/grammars/src/MagicPython.syntax.yaml b/grammars/src/MagicPython.syntax.yaml
@@ -791,7 +791,7 @@ repository:
               {{ | }}
               | (?:
                 {
-                  \w*? (\.[[:alpha:]_]\w*? | \[[^\]'"]+\])*?
+                  \w* (\.[[:alpha:]_]\w* | \[[^\]'"]+\])*
                   (![rsa])?
                   ( : \w? [<>=^]? [-+ ]? \#?
                     \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )?
@@ -818,13 +818,11 @@ repository:
           (?x)
             (
               {
-                \w*? (\.[[:alpha:]_]\w*? | \[[^\]'"]+\])*?
+                \w* (\.[[:alpha:]_]\w* | \[[^\]'"]+\])*
                 (![rsa])?
                 (:)
-                  (
-                    [^'"{}\n]+?
-                    |
-                    \{ [^'"}\n]*? \}
+                  [^'"{}\n]* (?:
+                    \{ [^'"}\n]*? \} [^'"{}\n]*
                   )*
               }
             )
diff --git a/test/atom-spec/python-spec.js b/test/atom-spec/python-spec.js
@@ -12853,6 +12853,150 @@ describe("Grammar Tests", function() {
       expect(tokens[3][13].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
     });
 
+  it("test/strings/bug1.py", 
+    function() {
+      tokens = grammar.tokenizeLines("# issue 150\nrecord = {\n    \"a\": {k: str(v) for k, v in foo if \"\"}\n}")
+      expect(tokens[0][0].value).toBe("#");
+      expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]);
+      expect(tokens[0][1].value).toBe(" issue 150");
+      expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
+      expect(tokens[1][0].value).toBe("record");
+      expect(tokens[1][0].scopes).toEqual(["source.python"]);
+      expect(tokens[1][1].value).toBe(" ");
+      expect(tokens[1][1].scopes).toEqual(["source.python"]);
+      expect(tokens[1][2].value).toBe("=");
+      expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]);
+      expect(tokens[1][3].value).toBe(" ");
+      expect(tokens[1][3].scopes).toEqual(["source.python"]);
+      expect(tokens[1][4].value).toBe("{");
+      expect(tokens[1][4].scopes).toEqual(["source.python","punctuation.definition.dict.begin.python"]);
+      expect(tokens[2][0].value).toBe("    ");
+      expect(tokens[2][0].scopes).toEqual(["source.python"]);
+      expect(tokens[2][1].value).toBe("\"");
+      expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]);
+      expect(tokens[2][2].value).toBe("a");
+      expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.single.python"]);
+      expect(tokens[2][3].value).toBe("\"");
+      expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]);
+      expect(tokens[2][4].value).toBe(":");
+      expect(tokens[2][4].scopes).toEqual(["source.python","punctuation.separator.dict.python"]);
+      expect(tokens[2][5].value).toBe(" ");
+      expect(tokens[2][5].scopes).toEqual(["source.python"]);
+      expect(tokens[2][6].value).toBe("{");
+      expect(tokens[2][6].scopes).toEqual(["source.python","punctuation.definition.dict.begin.python"]);
+      expect(tokens[2][7].value).toBe("k");
+      expect(tokens[2][7].scopes).toEqual(["source.python"]);
+      expect(tokens[2][8].value).toBe(":");
+      expect(tokens[2][8].scopes).toEqual(["source.python","punctuation.separator.dict.python"]);
+      expect(tokens[2][9].value).toBe(" ");
+      expect(tokens[2][9].scopes).toEqual(["source.python"]);
+      expect(tokens[2][10].value).toBe("str");
+      expect(tokens[2][10].scopes).toEqual(["source.python","meta.function-call.python","support.type.python"]);
+      expect(tokens[2][11].value).toBe("(");
+      expect(tokens[2][11].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]);
+      expect(tokens[2][12].value).toBe("v");
+      expect(tokens[2][12].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]);
+      expect(tokens[2][13].value).toBe(")");
+      expect(tokens[2][13].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]);
+      expect(tokens[2][14].value).toBe(" ");
+      expect(tokens[2][14].scopes).toEqual(["source.python"]);
+      expect(tokens[2][15].value).toBe("for");
+      expect(tokens[2][15].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[2][16].value).toBe(" ");
+      expect(tokens[2][16].scopes).toEqual(["source.python"]);
+      expect(tokens[2][17].value).toBe("k");
+      expect(tokens[2][17].scopes).toEqual(["source.python"]);
+      expect(tokens[2][18].value).toBe(",");
+      expect(tokens[2][18].scopes).toEqual(["source.python","punctuation.separator.element.python"]);
+      expect(tokens[2][19].value).toBe(" ");
+      expect(tokens[2][19].scopes).toEqual(["source.python"]);
+      expect(tokens[2][20].value).toBe("v");
+      expect(tokens[2][20].scopes).toEqual(["source.python"]);
+      expect(tokens[2][21].value).toBe(" ");
+      expect(tokens[2][21].scopes).toEqual(["source.python"]);
+      expect(tokens[2][22].value).toBe("in");
+      expect(tokens[2][22].scopes).toEqual(["source.python","keyword.operator.logical.python"]);
+      expect(tokens[2][23].value).toBe(" ");
+      expect(tokens[2][23].scopes).toEqual(["source.python"]);
+      expect(tokens[2][24].value).toBe("foo");
+      expect(tokens[2][24].scopes).toEqual(["source.python"]);
+      expect(tokens[2][25].value).toBe(" ");
+      expect(tokens[2][25].scopes).toEqual(["source.python"]);
+      expect(tokens[2][26].value).toBe("if");
+      expect(tokens[2][26].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[2][27].value).toBe(" ");
+      expect(tokens[2][27].scopes).toEqual(["source.python"]);
+      expect(tokens[2][28].value).toBe("\"");
+      expect(tokens[2][28].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]);
+      expect(tokens[2][29].value).toBe("\"");
+      expect(tokens[2][29].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]);
+      expect(tokens[2][30].value).toBe("}");
+      expect(tokens[2][30].scopes).toEqual(["source.python","punctuation.definition.dict.end.python"]);
+      expect(tokens[3][0].value).toBe("}");
+      expect(tokens[3][0].scopes).toEqual(["source.python","punctuation.definition.dict.end.python"]);
+    });
+
+  it("test/strings/bug2.py", 
+    function() {
+      tokens = grammar.tokenizeLines("# issue 150\ncmd = \"git-clang-format --style=\\\"{{BasedOnStyle: Google, ColumnLimit: 100, IndentWidth: 2, \" \\\n \"AlignConsecutiveAssignments: true}}\\\" {COMMIT_SHA} -- ./**/*.proto > {OUTPUT}\".format(")
+      expect(tokens[0][0].value).toBe("#");
+      expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]);
+      expect(tokens[0][1].value).toBe(" issue 150");
+      expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
+      expect(tokens[1][0].value).toBe("cmd");
+      expect(tokens[1][0].scopes).toEqual(["source.python"]);
+      expect(tokens[1][1].value).toBe(" ");
+      expect(tokens[1][1].scopes).toEqual(["source.python"]);
+      expect(tokens[1][2].value).toBe("=");
+      expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]);
+      expect(tokens[1][3].value).toBe(" ");
+      expect(tokens[1][3].scopes).toEqual(["source.python"]);
+      expect(tokens[1][4].value).toBe("\"");
+      expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]);
+      expect(tokens[1][5].value).toBe("git-clang-format --style=");
+      expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python"]);
+      expect(tokens[1][6].value).toBe("\\\"");
+      expect(tokens[1][6].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]);
+      expect(tokens[1][7].value).toBe("{{");
+      expect(tokens[1][7].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]);
+      expect(tokens[1][8].value).toBe("BasedOnStyle: Google, ColumnLimit: 100, IndentWidth: 2, ");
+      expect(tokens[1][8].scopes).toEqual(["source.python","string.quoted.single.python"]);
+      expect(tokens[1][9].value).toBe("\"");
+      expect(tokens[1][9].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]);
+      expect(tokens[1][10].value).toBe(" ");
+      expect(tokens[1][10].scopes).toEqual(["source.python"]);
+      expect(tokens[1][11].value).toBe("\\");
+      expect(tokens[1][11].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]);
+      expect(tokens[1][12].value).toBe("");
+      expect(tokens[1][12].scopes).toEqual(["source.python"]);
+      expect(tokens[2][0].value).toBe(" ");
+      expect(tokens[2][0].scopes).toEqual(["source.python"]);
+      expect(tokens[2][1].value).toBe("\"");
+      expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]);
+      expect(tokens[2][2].value).toBe("AlignConsecutiveAssignments: true");
+      expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.single.python"]);
+      expect(tokens[2][3].value).toBe("}}");
+      expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]);
+      expect(tokens[2][4].value).toBe("\\\"");
+      expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]);
+      expect(tokens[2][5].value).toBe(" ");
+      expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.single.python"]);
+      expect(tokens[2][6].value).toBe("{COMMIT_SHA}");
+      expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]);
+      expect(tokens[2][7].value).toBe(" -- ./**/*.proto > ");
+      expect(tokens[2][7].scopes).toEqual(["source.python","string.quoted.single.python"]);
+      expect(tokens[2][8].value).toBe("{OUTPUT}");
+      expect(tokens[2][8].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]);
+      expect(tokens[2][9].value).toBe("\"");
+      expect(tokens[2][9].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]);
+      expect(tokens[2][10].value).toBe(".");
+      expect(tokens[2][10].scopes).toEqual(["source.python","punctuation.separator.period.python"]);
+      expect(tokens[2][11].value).toBe("format");
+      expect(tokens[2][11].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]);
+      expect(tokens[2][12].value).toBe("(");
+      expect(tokens[2][12].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]);
+    });
+
   it("test/strings/bytes1.py", 
     function() {
       tokens = grammar.tokenizeLines("a = b\"\"\"\nmultiline \"binary\" string \\\n\n    \\xf1 \\u1234aaaa \\U1234aaaa\n\n    \\N{BLACK SPADE SUIT}\n\"\"\"")
diff --git a/test/strings/bug1.py b/test/strings/bug1.py
@@ -0,0 +1,47 @@
+# issue 150
+record = {
+    "a": {k: str(v) for k, v in foo if ""}
+}
+
+
+
+
+#             : comment.line.number-sign.python, punctuation.definition.comment.python, source.python
+ issue 150    : comment.line.number-sign.python, source.python
+record        : source.python
+              : source.python
+=             : keyword.operator.assignment.python, source.python
+              : source.python
+{             : punctuation.definition.dict.begin.python, source.python
+              : source.python
+"             : punctuation.definition.string.begin.python, source.python, string.quoted.single.python
+a             : source.python, string.quoted.single.python
+"             : punctuation.definition.string.end.python, source.python, string.quoted.single.python
+:             : punctuation.separator.dict.python, source.python
+              : source.python
+{             : punctuation.definition.dict.begin.python, source.python
+k             : source.python
+:             : punctuation.separator.dict.python, source.python
+              : source.python
+str           : meta.function-call.python, source.python, support.type.python
+(             : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python
+v             : meta.function-call.arguments.python, meta.function-call.python, source.python
+)             : meta.function-call.python, punctuation.definition.arguments.end.python, source.python
+              : source.python
+for           : keyword.control.flow.python, source.python
+              : source.python
+k             : source.python
+,             : punctuation.separator.element.python, source.python
+              : source.python
+v             : source.python
+              : source.python
+in            : keyword.operator.logical.python, source.python
+              : source.python
+foo           : source.python
+              : source.python
+if            : keyword.control.flow.python, source.python
+              : source.python
+"             : punctuation.definition.string.begin.python, source.python, string.quoted.single.python
+"             : punctuation.definition.string.end.python, source.python, string.quoted.single.python
+}             : punctuation.definition.dict.end.python, source.python
+}             : punctuation.definition.dict.end.python, source.python
diff --git a/test/strings/bug2.py b/test/strings/bug2.py
@@ -0,0 +1,35 @@
+# issue 150
+cmd = "git-clang-format --style=\"{{BasedOnStyle: Google, ColumnLimit: 100, IndentWidth: 2, " \
+ "AlignConsecutiveAssignments: true}}\" {COMMIT_SHA} -- ./**/*.proto > {OUTPUT}".format(
+
+
+
+
+#             : comment.line.number-sign.python, punctuation.definition.comment.python, source.python
+ issue 150    : comment.line.number-sign.python, source.python
+cmd           : source.python
+              : source.python
+=             : keyword.operator.assignment.python, source.python
+              : source.python
+"             : punctuation.definition.string.begin.python, source.python, string.quoted.single.python
+git-clang-format --style= : source.python, string.quoted.single.python
+\"            : constant.character.escape.python, source.python, string.quoted.single.python
+{{            : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python
+BasedOnStyle: Google, ColumnLimit: 100, IndentWidth: 2,  : source.python, string.quoted.single.python
+"             : punctuation.definition.string.end.python, source.python, string.quoted.single.python
+              : source.python
+\             : punctuation.separator.continuation.line.python, source.python
+              : source.python
+              : source.python
+"             : punctuation.definition.string.begin.python, source.python, string.quoted.single.python
+AlignConsecutiveAssignments: true : source.python, string.quoted.single.python
+}}            : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python
+\"            : constant.character.escape.python, source.python, string.quoted.single.python
+              : source.python, string.quoted.single.python
+{COMMIT_SHA}  : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python
+ -- ./**/*.proto >  : source.python, string.quoted.single.python
+{OUTPUT}      : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python
+"             : punctuation.definition.string.end.python, source.python, string.quoted.single.python
+.             : punctuation.separator.period.python, source.python
+format        : meta.function-call.generic.python, meta.function-call.python, source.python
+(             : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python

Original file line number	Diff line number	Diff line change
`@@ -942,7 +942,7 @@ repository:`
`942`	`942`	`{{ \| }}`
`943`	`943`	`\| (?:`
`944`	`944`	`{`
`945`		`- \\w? (\\.[[:alpha:]_]\\w? \| \\[[^\\]'"]+\\])*?`
	`945`	`+ \\w* (\\.[[:alpha:]_]\\w* \| \\[[^\\]'"]+\\])*`
`946`	`946`	`(![rsa])?`
`947`	`947`	`( : \\w? [<>=^]? [-+ ]? \\#?`
`948`	`948`	`\\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )?`
`@@ -964,13 +964,11 @@ repository:`
`964`	`964`	`(?x)`
`965`	`965`	`(`
`966`	`966`	`{`
`967`		`- \\w? (\\.[[:alpha:]_]\\w? \| \\[[^\\]'"]+\\])*?`
	`967`	`+ \\w* (\\.[[:alpha:]_]\\w* \| \\[[^\\]'"]+\\])*`
`968`	`968`	`(![rsa])?`
`969`	`969`	`(:)`
`970`		`- (`
`971`		`- [^'"{}\\n]+?`
`972`		`- \|`
`973`		`- \\{ [^'"}\\n]*? \\}`
	`970`	`+ [^'"{}\\n]* (?:`
	`971`	`+ \\{ [^'"}\\n]? \\} [^'"{}\\n]`
`974`	`972`	`)*`
`975`	`973`	`}`
`976`	`974`	`)`
Original file line number	Diff line number	Diff line change
`@@ -1444,7 +1444,7 @@`
`1444`	`1444`	`{{ \| }}`
`1445`	`1445`	`\| (?:`
`1446`	`1446`	`{`
`1447`		`- \w? (\.[[:alpha:]_]\w? \| \[[^\]'"]+\])*?`
	`1447`	`+ \w* (\.[[:alpha:]_]\w* \| \[[^\]'"]+\])*`
`1448`	`1448`	`(![rsa])?`
`1449`	`1449`	`( : \w? [<>=^]? [-+ ]? \#?`
`1450`	`1450`	`\d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )?`
`@@ -1477,13 +1477,11 @@`
`1477`	`1477`	`<string>(?x)`
`1478`	`1478`	`(`
`1479`	`1479`	`{`
`1480`		`- \w? (\.[[:alpha:]_]\w? \| \[[^\]'"]+\])*?`
	`1480`	`+ \w* (\.[[:alpha:]_]\w* \| \[[^\]'"]+\])*`
`1481`	`1481`	`(![rsa])?`
`1482`	`1482`	`(:)`
`1483`		`- (`
`1484`		`- [^'"{}\n]+?`
`1485`		`- \|`
`1486`		`- \{ [^'"}\n]*? \}`
	`1483`	`+ [^'"{}\n]* (?:`
	`1484`	`+ \{ [^'"}\n]? \} [^'"{}\n]`
`1487`	`1485`	`)*`
`1488`	`1486`	`}`
`1489`	`1487`	`)`
Original file line number	Diff line number	Diff line change
`@@ -791,7 +791,7 @@ repository:`
`791`	`791`	`{{ \| }}`
`792`	`792`	`\| (?:`
`793`	`793`	`{`
`794`		`- \w? (\.[[:alpha:]_]\w? \| \[[^\]'"]+\])*?`
	`794`	`+ \w* (\.[[:alpha:]_]\w* \| \[[^\]'"]+\])*`
`795`	`795`	`(![rsa])?`
`796`	`796`	`( : \w? [<>=^]? [-+ ]? \#?`
`797`	`797`	`\d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )?`
`@@ -818,13 +818,11 @@ repository:`
`818`	`818`	`(?x)`
`819`	`819`	`(`
`820`	`820`	`{`
`821`		`- \w? (\.[[:alpha:]_]\w? \| \[[^\]'"]+\])*?`
	`821`	`+ \w* (\.[[:alpha:]_]\w* \| \[[^\]'"]+\])*`
`822`	`822`	`(![rsa])?`
`823`	`823`	`(:)`
`824`		`- (`
`825`		`- [^'"{}\n]+?`
`826`		`- \|`
`827`		`- \{ [^'"}\n]*? \}`
	`824`	`+ [^'"{}\n]* (?:`
	`825`	`+ \{ [^'"}\n]? \} [^'"{}\n]`
`828`	`826`	`)*`
`829`	`827`	`}`
`830`	`828`	`)`