Update the way "in" is scoped.

The "in" keyword is scoped as either `keyword.operator.logical.python`
or as `keyword.control.flow.python` depending on whether it appears as
part of a "for ... in" construct.

Fixes: #193

Author: vpetrovykh

grammars/MagicPython.cson

@@ -46,6 +46,9 @@ repository:
       {
         include: "#function-declaration"
       }
+      {
+        include: "#generator"
+      }
       {
         include: "#statement-keyword"
       }
@@ -328,6 +331,9 @@ repository:
       {
         include: "#lambda"
       }
+      {
+        include: "#generator"
+      }
       {
         include: "#illegal-operator"
       }
@@ -1318,6 +1324,25 @@ repository:
         include: "#expression"
       }
     ]
+  generator:
+    comment: '''
+      Match "for ... in" construct used in generators and for loops to
+      correctly identify the "in" as a control flow keyword.
+      
+    '''
+    begin: "for"
+    beginCaptures:
+      "0":
+        name: "keyword.control.flow.python"
+    end: "in"
+    endCaptures:
+      "0":
+        name: "keyword.control.flow.python"
+    patterns: [
+      {
+        include: "#expression"
+      }
+    ]
   "function-declaration":
     name: "meta.function.python"
     begin: '''

grammars/MagicPython.tmLanguage

@@ -69,6 +69,10 @@
             <key>include</key>
             <string>#function-declaration</string>
           </dict>
+          <dict>
+            <key>include</key>
+            <string>#generator</string>
+          </dict>
           <dict>
             <key>include</key>
             <string>#statement-keyword</string>
@@ -501,6 +505,10 @@ it's probably control flow like:
             <key>include</key>
             <string>#lambda</string>
           </dict>
+          <dict>
+            <key>include</key>
+            <string>#generator</string>
+          </dict>
           <dict>
             <key>include</key>
             <string>#illegal-operator</string>
@@ -2046,6 +2054,40 @@ E.g. "arr[idx](args)"
           </dict>
         </array>
       </dict>
+      <key>generator</key>
+      <dict>
+        <key>comment</key>
+        <string>Match "for ... in" construct used in generators and for loops to
+correctly identify the "in" as a control flow keyword.
+</string>
+        <key>begin</key>
+        <string>for</string>
+        <key>beginCaptures</key>
+        <dict>
+          <key>0</key>
+          <dict>
+            <key>name</key>
+            <string>keyword.control.flow.python</string>
+          </dict>
+        </dict>
+        <key>end</key>
+        <string>in</string>
+        <key>endCaptures</key>
+        <dict>
+          <key>0</key>
+          <dict>
+            <key>name</key>
+            <string>keyword.control.flow.python</string>
+          </dict>
+        </dict>
+        <key>patterns</key>
+        <array>
+          <dict>
+            <key>include</key>
+            <string>#expression</string>
+          </dict>
+        </array>
+      </dict>
       <key>function-declaration</key>
       <dict>
         <key>name</key>

grammars/src/MagicPython.syntax.yaml

@@ -221,6 +221,7 @@ repository:
       - include: '#import'
       - include: '#class-declaration'
       - include: '#function-declaration'
+      - include: '#generator'
       - include: '#statement-keyword'
       - include: '#assignment-operator'
       - include: '#decorator'
@@ -399,6 +400,7 @@ repository:
       - include: '#regexp'
       - include: '#string'
       - include: '#lambda'
+      - include: '#generator'
       - include: '#illegal-operator'
       - include: '#operator'
       - include: '#curly-braces'
@@ -1040,6 +1042,19 @@ repository:
     patterns:
       - include: '#expression'
 
+  generator:
+    comment: |
+      Match "for ... in" construct used in generators and for loops to
+      correctly identify the "in" as a control flow keyword.
+    begin: for
+    beginCaptures:
+      '0': {name: keyword.control.flow.python}
+    end: in
+    endCaptures:
+      '0': {name: keyword.control.flow.python}
+    patterns:
+      - include: '#expression'
+
   function-declaration:
     name: meta.function.python
     begin: |

test/atom-spec/python-spec.js

@@ -5865,7 +5865,7 @@ describe("Grammar Tests", function() {
       expect(tokens[0][10].value).toBe(" ");
       expect(tokens[0][10].scopes).toEqual(["source.python"]);
       expect(tokens[0][11].value).toBe("in");
-      expect(tokens[0][11].scopes).toEqual(["source.python","keyword.operator.logical.python"]);
+      expect(tokens[0][11].scopes).toEqual(["source.python","keyword.control.flow.python"]);
       expect(tokens[0][12].value).toBe(" ");
       expect(tokens[0][12].scopes).toEqual(["source.python"]);
       expect(tokens[0][13].value).toBe("range");
@@ -6147,7 +6147,7 @@ describe("Grammar Tests", function() {
 
   it("test/expressions/keywords.py", 
     function() {
-      tokens = grammar.tokenizeLines("as async await continue del assert break finally for\nfrom elif else if import except pass raise\nreturn try while with\n\nnonlocal global class def")
+      tokens = grammar.tokenizeLines("as async await continue del assert break finally\nfrom elif else if import except pass raise\nreturn try while with\n\nnonlocal global class def\n\nfor")
       expect(tokens[0][0].value).toBe("as");
       expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]);
       expect(tokens[0][1].value).toBe(" ");
@@ -6178,10 +6178,6 @@ describe("Grammar Tests", function() {
       expect(tokens[0][13].scopes).toEqual(["source.python"]);
       expect(tokens[0][14].value).toBe("finally");
       expect(tokens[0][14].scopes).toEqual(["source.python","keyword.control.flow.python"]);
-      expect(tokens[0][15].value).toBe(" ");
-      expect(tokens[0][15].scopes).toEqual(["source.python"]);
-      expect(tokens[0][16].value).toBe("for");
-      expect(tokens[0][16].scopes).toEqual(["source.python","keyword.control.flow.python"]);
       expect(tokens[1][0].value).toBe("from");
       expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]);
       expect(tokens[1][1].value).toBe(" ");
@@ -6240,6 +6236,10 @@ describe("Grammar Tests", function() {
       expect(tokens[4][4].scopes).toEqual(["source.python","storage.type.class.python"]);
       expect(tokens[4][5].value).toBe(" def");
       expect(tokens[4][5].scopes).toEqual(["source.python","storage.type.function.python"]);
+      expect(tokens[5][0].value).toBe("");
+      expect(tokens[5][0].scopes).toEqual(["source.python"]);
+      expect(tokens[6][0].value).toBe("for");
+      expect(tokens[6][0].scopes).toEqual(["source.python","keyword.control.flow.python"]);
     });
 
   it("test/expressions/special1.py", 
@@ -7988,7 +7988,7 @@ describe("Grammar Tests", function() {
       expect(tokens[3][12].value).toBe(" ");
       expect(tokens[3][12].scopes).toEqual(["source.python"]);
       expect(tokens[3][13].value).toBe("in");
-      expect(tokens[3][13].scopes).toEqual(["source.python","keyword.operator.logical.python"]);
+      expect(tokens[3][13].scopes).toEqual(["source.python","keyword.control.flow.python"]);
       expect(tokens[3][14].value).toBe(" ");
       expect(tokens[3][14].scopes).toEqual(["source.python"]);
       expect(tokens[3][15].value).toBe("b");
@@ -8735,7 +8735,7 @@ describe("Grammar Tests", function() {
       expect(tokens[4][7].value).toBe(" ");
       expect(tokens[4][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]);
       expect(tokens[4][8].value).toBe("in");
-      expect(tokens[4][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.logical.python"]);
+      expect(tokens[4][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.control.flow.python"]);
       expect(tokens[5][0].value).toBe("                            ");
       expect(tokens[5][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]);
       expect(tokens[5][1].value).toBe("(");
@@ -12165,7 +12165,7 @@ describe("Grammar Tests", function() {
       expect(tokens[0][9].value).toBe(" ");
       expect(tokens[0][9].scopes).toEqual(["source.python"]);
       expect(tokens[0][10].value).toBe("in");
-      expect(tokens[0][10].scopes).toEqual(["source.python","keyword.operator.logical.python"]);
+      expect(tokens[0][10].scopes).toEqual(["source.python","keyword.control.flow.python"]);
       expect(tokens[0][11].value).toBe(" ");
       expect(tokens[0][11].scopes).toEqual(["source.python"]);
       expect(tokens[0][12].value).toBe("b");
@@ -12190,6 +12190,69 @@ describe("Grammar Tests", function() {
       expect(tokens[3][3].scopes).toEqual(["source.python","constant.numeric.dec.python"]);
     });
 
+  it("test/statements/for2.py", 
+    function() {
+      tokens = grammar.tokenizeLines("for a, b, c in [2 in q, 2 in w]:\n    pass")
+      expect(tokens[0][0].value).toBe("for");
+      expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[0][1].value).toBe(" ");
+      expect(tokens[0][1].scopes).toEqual(["source.python"]);
+      expect(tokens[0][2].value).toBe("a");
+      expect(tokens[0][2].scopes).toEqual(["source.python"]);
+      expect(tokens[0][3].value).toBe(",");
+      expect(tokens[0][3].scopes).toEqual(["source.python","punctuation.separator.element.python"]);
+      expect(tokens[0][4].value).toBe(" ");
+      expect(tokens[0][4].scopes).toEqual(["source.python"]);
+      expect(tokens[0][5].value).toBe("b");
+      expect(tokens[0][5].scopes).toEqual(["source.python"]);
+      expect(tokens[0][6].value).toBe(",");
+      expect(tokens[0][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]);
+      expect(tokens[0][7].value).toBe(" ");
+      expect(tokens[0][7].scopes).toEqual(["source.python"]);
+      expect(tokens[0][8].value).toBe("c");
+      expect(tokens[0][8].scopes).toEqual(["source.python"]);
+      expect(tokens[0][9].value).toBe(" ");
+      expect(tokens[0][9].scopes).toEqual(["source.python"]);
+      expect(tokens[0][10].value).toBe("in");
+      expect(tokens[0][10].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+      expect(tokens[0][11].value).toBe(" ");
+      expect(tokens[0][11].scopes).toEqual(["source.python"]);
+      expect(tokens[0][12].value).toBe("[");
+      expect(tokens[0][12].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]);
+      expect(tokens[0][13].value).toBe("2");
+      expect(tokens[0][13].scopes).toEqual(["source.python","constant.numeric.dec.python"]);
+      expect(tokens[0][14].value).toBe(" ");
+      expect(tokens[0][14].scopes).toEqual(["source.python"]);
+      expect(tokens[0][15].value).toBe("in");
+      expect(tokens[0][15].scopes).toEqual(["source.python","keyword.operator.logical.python"]);
+      expect(tokens[0][16].value).toBe(" ");
+      expect(tokens[0][16].scopes).toEqual(["source.python"]);
+      expect(tokens[0][17].value).toBe("q");
+      expect(tokens[0][17].scopes).toEqual(["source.python"]);
+      expect(tokens[0][18].value).toBe(",");
+      expect(tokens[0][18].scopes).toEqual(["source.python","punctuation.separator.element.python"]);
+      expect(tokens[0][19].value).toBe(" ");
+      expect(tokens[0][19].scopes).toEqual(["source.python"]);
+      expect(tokens[0][20].value).toBe("2");
+      expect(tokens[0][20].scopes).toEqual(["source.python","constant.numeric.dec.python"]);
+      expect(tokens[0][21].value).toBe(" ");
+      expect(tokens[0][21].scopes).toEqual(["source.python"]);
+      expect(tokens[0][22].value).toBe("in");
+      expect(tokens[0][22].scopes).toEqual(["source.python","keyword.operator.logical.python"]);
+      expect(tokens[0][23].value).toBe(" ");
+      expect(tokens[0][23].scopes).toEqual(["source.python"]);
+      expect(tokens[0][24].value).toBe("w");
+      expect(tokens[0][24].scopes).toEqual(["source.python"]);
+      expect(tokens[0][25].value).toBe("]");
+      expect(tokens[0][25].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]);
+      expect(tokens[0][26].value).toBe(":");
+      expect(tokens[0][26].scopes).toEqual(["source.python","punctuation.separator.colon.python"]);
+      expect(tokens[1][0].value).toBe("    ");
+      expect(tokens[1][0].scopes).toEqual(["source.python"]);
+      expect(tokens[1][1].value).toBe("pass");
+      expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]);
+    });
+
   it("test/statements/from1.py", 
     function() {
       tokens = grammar.tokenizeLines("from ... import foo as bar\nraise Exception('done') from exc\nyield from foo")
@@ -13271,7 +13334,7 @@ describe("Grammar Tests", function() {
       expect(tokens[2][21].value).toBe(" ");
       expect(tokens[2][21].scopes).toEqual(["source.python"]);
       expect(tokens[2][22].value).toBe("in");
-      expect(tokens[2][22].scopes).toEqual(["source.python","keyword.operator.logical.python"]);
+      expect(tokens[2][22].scopes).toEqual(["source.python","keyword.control.flow.python"]);
       expect(tokens[2][23].value).toBe(" ");
       expect(tokens[2][23].scopes).toEqual(["source.python"]);
       expect(tokens[2][24].value).toBe("foo");

test/expressions/expr4.py

@@ -13,7 +13,7 @@
               : source.python
 i             : source.python
               : source.python
-in            : keyword.operator.logical.python, source.python
+in            : keyword.control.flow.python, source.python
               : source.python
 range         : meta.function-call.python, source.python, support.function.builtin.python
 (             : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python

test/expressions/keywords.py

@@ -1,9 +1,11 @@
-as async await continue del assert break finally for
+as async await continue del assert break finally
 from elif else if import except pass raise
 return try while with
 
 nonlocal global class def
 
+for
+
 
 
 as            : keyword.control.import.python, source.python
@@ -21,8 +23,6 @@
 break         : keyword.control.flow.python, source.python
               : source.python
 finally       : keyword.control.flow.python, source.python
-              : source.python
-for           : keyword.control.flow.python, source.python
 from          : keyword.control.import.python, source.python
               : source.python
 elif          : keyword.control.flow.python, source.python
@@ -52,3 +52,5 @@
               : source.python
 class         : source.python, storage.type.class.python
  def          : source.python, storage.type.function.python
+              : source.python
+for           : keyword.control.flow.python, source.python

test/functions/async1.py

@@ -36,7 +36,7 @@ def           : meta.function.python, source.python, storage.type.function.pytho
               : source.python
 c             : source.python
               : source.python
-in            : keyword.operator.logical.python, source.python
+in            : keyword.control.flow.python, source.python
               : source.python
 b             : source.python
 :             : punctuation.separator.colon.python, source.python

test/functions/decl4.py

@@ -38,7 +38,7 @@ def           : meta.function.python, source.python, storage.type.function.pytho
               : meta.function.parameters.python, meta.function.python, source.python
 val           : meta.function.parameters.python, meta.function.python, source.python
               : meta.function.parameters.python, meta.function.python, source.python
-in            : keyword.operator.logical.python, meta.function.parameters.python, meta.function.python, source.python
+in            : keyword.control.flow.python, meta.function.parameters.python, meta.function.python, source.python
                              : meta.function.parameters.python, meta.function.python, source.python
 (             : meta.function.parameters.python, meta.function.python, punctuation.parenthesis.begin.python, source.python
 x             : meta.function.parameters.python, meta.function.python, source.python

test/statements/for1.py

@@ -15,7 +15,7 @@
               : source.python
 c             : source.python
               : source.python
-in            : keyword.operator.logical.python, source.python
+in            : keyword.control.flow.python, source.python
               : source.python
 b             : source.python
 :             : punctuation.separator.colon.python, source.python

test/statements/for2.py

@@ -0,0 +1,35 @@
+for a, b, c in [2 in q, 2 in w]:
+    pass
+
+
+
+
+for           : keyword.control.flow.python, source.python
+              : source.python
+a             : source.python
+,             : punctuation.separator.element.python, source.python
+              : source.python
+b             : source.python
+,             : punctuation.separator.element.python, source.python
+              : source.python
+c             : source.python
+              : source.python
+in            : keyword.control.flow.python, source.python
+              : source.python
+[             : punctuation.definition.list.begin.python, source.python
+2             : constant.numeric.dec.python, source.python
+              : source.python
+in            : keyword.operator.logical.python, source.python
+              : source.python
+q             : source.python
+,             : punctuation.separator.element.python, source.python
+              : source.python
+2             : constant.numeric.dec.python, source.python
+              : source.python
+in            : keyword.operator.logical.python, source.python
+              : source.python
+w             : source.python
+]             : punctuation.definition.list.end.python, source.python
+:             : punctuation.separator.colon.python, source.python
+              : source.python
+pass          : keyword.control.flow.python, source.python