Fix tokenizer to tokenize T_NUM_STRING and T_VAR_NAME correctly

roblourens · roblourens · commit 5cff92320d2d · 2017-08-08T17:23:02.000-07:00
diff --git a/src/PhpTokenizer.php b/src/PhpTokenizer.php
@@ -289,11 +289,13 @@ public static function getTokensArrayFromContent(
         T_OBJECT_CAST       => TokenKind::ObjectCastToken,
         T_STRING_CAST       => TokenKind::StringCastToken,
         T_UNSET_CAST        => TokenKind::UnsetCastToken,
+
         T_START_HEREDOC     => TokenKind::HeredocStart,
         T_END_HEREDOC       => TokenKind::HeredocEnd,
-        T_STRING_VARNAME    => TokenKind::VariableName,
+        T_STRING_VARNAME    => TokenKind::StringVarname,
         T_COMMENT           => TokenKind::CommentToken,
-        T_DOC_COMMENT       => TokenKind::DocCommentToken
+        T_DOC_COMMENT       => TokenKind::DocCommentToken,
+        T_NUM_STRING        => TokenKind::NumStringToken
     ];
 
     const PARSE_CONTEXT_TO_PREFIX = [
diff --git a/src/TokenKind.php b/src/TokenKind.php
@@ -214,6 +214,7 @@ class TokenKind {
     const IntegerLiteralToken = 416;
     const CommentToken = 417;
     const DocCommentToken = 418;
+    const NumStringToken = 419;
 
     // TODO type annotations - PHP7
 }
diff --git a/tests/cases/lexical/stringLiteral23.php b/tests/cases/lexical/stringLiteral23.php
@@ -0,0 +1,2 @@
+<?php
+"$x"
diff --git a/tests/cases/lexical/stringLiteral23.php.tokens b/tests/cases/lexical/stringLiteral23.php.tokens
@@ -0,0 +1,22 @@
+[
+    {
+        "kind": "ScriptSectionStartTag",
+        "textLength": 6
+    },
+    {
+        "kind": "DoubleQuoteToken",
+        "textLength": 1
+    },
+    {
+        "kind": "VariableName",
+        "textLength": 2
+    },
+    {
+        "kind": "DoubleQuoteToken",
+        "textLength": 1
+    },
+    {
+        "kind": "EndOfFileToken",
+        "textLength": 0
+    }
+]
diff --git a/tests/cases/lexical/stringLiteral24.php b/tests/cases/lexical/stringLiteral24.php
@@ -0,0 +1,2 @@
+<?php
+"${x}"
diff --git a/tests/cases/lexical/stringLiteral24.php.tokens b/tests/cases/lexical/stringLiteral24.php.tokens
@@ -0,0 +1,30 @@
+[
+    {
+        "kind": "ScriptSectionStartTag",
+        "textLength": 6
+    },
+    {
+        "kind": "DoubleQuoteToken",
+        "textLength": 1
+    },
+    {
+        "kind": "DollarOpenBraceToken",
+        "textLength": 2
+    },
+    {
+        "kind": "StringVarname",
+        "textLength": 1
+    },
+    {
+        "kind": "CloseBraceToken",
+        "textLength": 1
+    },
+    {
+        "kind": "DoubleQuoteToken",
+        "textLength": 1
+    },
+    {
+        "kind": "EndOfFileToken",
+        "textLength": 0
+    }
+]
diff --git a/tests/cases/lexical/stringLiteral25.php b/tests/cases/lexical/stringLiteral25.php
@@ -0,0 +1,2 @@
+<?php
+"${$x}"
diff --git a/tests/cases/lexical/stringLiteral25.php.tokens b/tests/cases/lexical/stringLiteral25.php.tokens
@@ -0,0 +1,30 @@
+[
+    {
+        "kind": "ScriptSectionStartTag",
+        "textLength": 6
+    },
+    {
+        "kind": "DoubleQuoteToken",
+        "textLength": 1
+    },
+    {
+        "kind": "DollarOpenBraceToken",
+        "textLength": 2
+    },
+    {
+        "kind": "VariableName",
+        "textLength": 2
+    },
+    {
+        "kind": "CloseBraceToken",
+        "textLength": 1
+    },
+    {
+        "kind": "DoubleQuoteToken",
+        "textLength": 1
+    },
+    {
+        "kind": "EndOfFileToken",
+        "textLength": 0
+    }
+]
diff --git a/tests/cases/lexical/stringLiteral26.php b/tests/cases/lexical/stringLiteral26.php
@@ -0,0 +1,2 @@
+<?php
+"{$x}"
diff --git a/tests/cases/lexical/stringLiteral26.php.tokens b/tests/cases/lexical/stringLiteral26.php.tokens
@@ -0,0 +1,30 @@
+[
+    {
+        "kind": "ScriptSectionStartTag",
+        "textLength": 6
+    },
+    {
+        "kind": "DoubleQuoteToken",
+        "textLength": 1
+    },
+    {
+        "kind": "OpenBraceDollarToken",
+        "textLength": 1
+    },
+    {
+        "kind": "VariableName",
+        "textLength": 2
+    },
+    {
+        "kind": "CloseBraceToken",
+        "textLength": 1
+    },
+    {
+        "kind": "DoubleQuoteToken",
+        "textLength": 1
+    },
+    {
+        "kind": "EndOfFileToken",
+        "textLength": 0
+    }
+]
diff --git a/tests/cases/lexical/stringLiteral27.php b/tests/cases/lexical/stringLiteral27.php
@@ -0,0 +1,2 @@
+<?php
+"$x[0]"
diff --git a/tests/cases/lexical/stringLiteral27.php.tokens b/tests/cases/lexical/stringLiteral27.php.tokens
@@ -0,0 +1,34 @@
+[
+    {
+        "kind": "ScriptSectionStartTag",
+        "textLength": 6
+    },
+    {
+        "kind": "DoubleQuoteToken",
+        "textLength": 1
+    },
+    {
+        "kind": "VariableName",
+        "textLength": 2
+    },
+    {
+        "kind": "OpenBracketToken",
+        "textLength": 1
+    },
+    {
+        "kind": "NumStringToken",
+        "textLength": 1
+    },
+    {
+        "kind": "CloseBracketToken",
+        "textLength": 1
+    },
+    {
+        "kind": "DoubleQuoteToken",
+        "textLength": 1
+    },
+    {
+        "kind": "EndOfFileToken",
+        "textLength": 0
+    }
+]
diff --git a/tests/cases/lexical/stringLiteral28.php b/tests/cases/lexical/stringLiteral28.php
@@ -0,0 +1,2 @@
+<?php
+"$x->y"
diff --git a/tests/cases/lexical/stringLiteral28.php.tokens b/tests/cases/lexical/stringLiteral28.php.tokens
@@ -0,0 +1,30 @@
+[
+    {
+        "kind": "ScriptSectionStartTag",
+        "textLength": 6
+    },
+    {
+        "kind": "DoubleQuoteToken",
+        "textLength": 1
+    },
+    {
+        "kind": "VariableName",
+        "textLength": 2
+    },
+    {
+        "kind": "ArrowToken",
+        "textLength": 2
+    },
+    {
+        "kind": "Name",
+        "textLength": 1
+    },
+    {
+        "kind": "DoubleQuoteToken",
+        "textLength": 1
+    },
+    {
+        "kind": "EndOfFileToken",
+        "textLength": 0
+    }
+]

Original file line number	Diff line number	Diff line change
`@@ -214,6 +214,7 @@ class TokenKind {`
`214`	`214`	`const IntegerLiteralToken = 416;`
`215`	`215`	`const CommentToken = 417;`
`216`	`216`	`const DocCommentToken = 418;`
	`217`	`+ const NumStringToken = 419;`
`217`	`218`
`218`	`219`	`// TODO type annotations - PHP7`
`219`	`220`	`}`