fix tokenizer to not use token_get_all()

Author: rlerdorf

src/PhpTokenizer.php

@@ -108,21 +108,24 @@ public static function getTokensArrayFromContent(
 
         // Convert tokens from token_get_all to Token instances,
         // skipping whitespace and (usually, when parseContext is null) comments.
-        $tokens = \array_values($tokens);
-        $tokenCount = \count($tokens);
+        $keys = \array_keys($tokens);
+        $tokenCount = \count($keys);
         for ($i = 0; $i < $tokenCount; $i++) {
-            $token = $tokens[$i];
+            $token = $tokens[$keys[$i]];
             if (\is_array($token)) {
                 $tokenKind = $token[0];
                 $strlen = \strlen($token[1]);
             } else {
-                if ($token === '|' && $i + 1 < $tokenCount && $tokens[$i + 1] === '>') {
-                    $pos += 1; // '|'
-                    $i++;
-                    $pos += 1; // '>'
-                    $arr[] = new Token(TokenKind::PipeToken, $fullStart, $start, $pos - $fullStart);
-                    $start = $fullStart = $pos;
-                    continue;
+                if ($token === '|' && $i + 1 < $tokenCount) {
+                    $nextToken = $tokens[$keys[$i + 1]];
+                    if (!\is_array($nextToken) && $nextToken === '>') {
+                        $pos += 1; // '|'
+                        $i++;
+                        $pos += 1; // '>'
+                        $arr[] = new Token(TokenKind::PipeToken, $fullStart, $start, $pos - $fullStart);
+                        $start = $fullStart = $pos;
+                        continue;
+                    }
                 }
                 $pos += \strlen($token);
                 $newTokenKind = self::TOKEN_MAP[$token] ?? TokenKind::Unknown;