Skip to content

Commit 9bb043e

Browse files
committed
Support php 8.0's "Treat namespaced names as single token"
Without this, tolerant-php-parser won't work in php 8.0. Fixes microsoft#327
1 parent b4f5f2c commit 9bb043e

File tree

1 file changed

+51
-0
lines changed

1 file changed

+51
-0
lines changed

src/PhpTokenizer.php

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,57 @@ public static function getTokensArrayFromContent(
130130
$arr[] = new Token(TokenKind::ScriptSectionStartTag, $fullStart, $start, $pos-$fullStart);
131131
$start = $fullStart = $pos;
132132
break;
133+
case \PHP_VERSION_ID >= 80000 ? \T_NAME_QUALIFIED : -1000:
134+
case \PHP_VERSION_ID >= 80000 ? \T_NAME_FULLY_QUALIFIED : -1001:
135+
// NOTE: This switch is called on every token of every file being parsed, so this traded performance for readability.
136+
//
137+
// PHP's Opcache is able to optimize switches that are exclusively known longs,
138+
// but not switches that mix strings and longs or have unknown longs.
139+
// Longs are only known if they're declared within the same *class* or an internal constant (tokenizer).
140+
//
141+
// For some reason, the SWITCH_LONG opcode was not generated when the expression was part of a class constant.
142+
// (seen with php -d opcache.opt_debug_level=0x20000)
143+
//
144+
// Use negative values because that's not expected to overlap with token kinds that token_get_all() will return.
145+
//
146+
// T_NAME_* was added in php 8.0 to forbid whitespace between parts of names.
147+
// Here, emulate the tokenization of php 7 by splitting it up into 1 or more tokens.
148+
foreach (\explode('\\', $token[1]) as $i => $name) {
149+
if ($i) {
150+
$arr[] = new Token(TokenKind::BackslashToken, $fullStart, $start, 1 + $start - $fullStart);
151+
$start++;
152+
$fullStart = $start;
153+
}
154+
if ($name === '') {
155+
continue;
156+
}
157+
// TODO: TokenStringMaps::RESERVED_WORDS[$name] ?? TokenKind::Name for compatibility?
158+
$len = \strlen($name);
159+
$arr[] = new Token(TokenKind::Name, $fullStart, $start, $len + $start - $fullStart);
160+
$start += $len;
161+
$fullStart = $start;
162+
}
163+
break;
164+
case \PHP_VERSION_ID >= 80000 ? \T_NAME_RELATIVE : -1002:
165+
// This is a namespace-relative name: namespace\...
166+
foreach (\explode('\\', $token[1]) as $i => $name) {
167+
$len = \strlen($name);
168+
if (!$i) {
169+
$arr[] = new Token(TokenKind::NamespaceKeyword, $fullStart, $start, $len + $start - $fullStart);
170+
$start += $len;
171+
$fullStart = $start;
172+
continue;
173+
}
174+
$arr[] = new Token(TokenKind::BackslashToken, $fullStart, $start, 1);
175+
$start++;
176+
177+
// TODO: TokenStringMaps::RESERVED_WORDS[$name] ?? TokenKind::Name for compatibility?
178+
$arr[] = new Token(TokenKind::Name, $start, $start, $len);
179+
180+
$start += $len;
181+
$fullStart = $start;
182+
}
183+
break;
133184
case \T_COMMENT:
134185
case \T_DOC_COMMENT:
135186
if ($treatCommentsAsTrivia) {

0 commit comments

Comments
 (0)