Tokenizer/PHP: bug fix in improved context sensitive keyword support [2]

While investigating 3607, I figured adding a test with a function declared to return by reference would also not be amiss and found that that situation was not accounted for.

This commit fixes that.
Includes unit test.

Author: jrfnl

src/Tokenizers/PHP.php

@@ -607,38 +607,57 @@ protected function tokenize($string)
 
             if ($tokenIsArray === true
                 && isset(Util\Tokens::$contextSensitiveKeywords[$token[0]]) === true
-                && isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
+                && (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
+                || $finalTokens[$lastNotEmptyToken]['content'] === '&')
             ) {
-                $preserveKeyword = false;
+                if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
+                    $preserveKeyword = false;
 
-                // `new class`, and `new static` should be preserved.
-                if ($finalTokens[$lastNotEmptyToken]['code'] === T_NEW
-                    && ($token[0] === T_CLASS
-                    || $token[0] === T_STATIC)
-                ) {
-                    $preserveKeyword = true;
-                }
+                    // `new class`, and `new static` should be preserved.
+                    if ($finalTokens[$lastNotEmptyToken]['code'] === T_NEW
+                        && ($token[0] === T_CLASS
+                        || $token[0] === T_STATIC)
+                    ) {
+                        $preserveKeyword = true;
+                    }
 
-                // `new class extends` `new class implements` should be preserved
-                if (($token[0] === T_EXTENDS || $token[0] === T_IMPLEMENTS)
-                    && $finalTokens[$lastNotEmptyToken]['code'] === T_CLASS
-                ) {
-                    $preserveKeyword = true;
-                }
+                    // `new class extends` `new class implements` should be preserved
+                    if (($token[0] === T_EXTENDS || $token[0] === T_IMPLEMENTS)
+                        && $finalTokens[$lastNotEmptyToken]['code'] === T_CLASS
+                    ) {
+                        $preserveKeyword = true;
+                    }
+
+                    // `namespace\` should be preserved
+                    if ($token[0] === T_NAMESPACE) {
+                        for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
+                            if (is_array($tokens[$i]) === false) {
+                                break;
+                            }
+
+                            if (isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true) {
+                                continue;
+                            }
+
+                            if ($tokens[$i][0] === T_NS_SEPARATOR) {
+                                $preserveKeyword = true;
+                            }
 
-                // `namespace\` should be preserved
-                if ($token[0] === T_NAMESPACE) {
-                    for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
-                        if (is_array($tokens[$i]) === false) {
                             break;
                         }
+                    }
+                }//end if
+
+                if ($finalTokens[$lastNotEmptyToken]['content'] === '&') {
+                    $preserveKeyword = true;
 
-                        if (isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true) {
+                    for ($i = ($lastNotEmptyToken - 1); $i >= 0; $i--) {
+                        if (isset(Util\Tokens::$emptyTokens[$finalTokens[$i]['code']]) === true) {
                             continue;
                         }
 
-                        if ($tokens[$i][0] === T_NS_SEPARATOR) {
-                            $preserveKeyword = true;
+                        if ($finalTokens[$i]['code'] === T_FUNCTION) {
+                            $preserveKeyword = false;
                         }
 
                         break;

tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc

@@ -217,3 +217,4 @@ class Foo extends /* testNamespaceInNameIsKeyword */ namespace\Exception
 {}
 
 function /* testKeywordAfterFunctionShouldBeString */ eval() {}
+function /* testKeywordAfterFunctionByRefShouldBeString */ &switch() {}

tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php

@@ -124,6 +124,7 @@ public function dataStrings()
             ['/* testNamespaceNameIsString3 */'],
 
             ['/* testKeywordAfterFunctionShouldBeString */'],
+            ['/* testKeywordAfterFunctionByRefShouldBeString */'],
         ];
 
     }//end dataStrings()