Skip to content

Commit be9cfce

Browse files
roblourensroblourens
authored
Merge pull request microsoft#331 from TysonAndre/relative-token
Support php 8.0's "Treat namespaced names as single token" and "?->"
2 parents b4f5f2c + 2c63e8a commit be9cfce

12 files changed

+284
-5
lines changed

‎src/Parser.php‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1199,7 +1199,7 @@ private function parseTemplateStringExpression($parentNode) {
11991199
$token = $this->getCurrentToken();
12001200
if ($token->kind === TokenKind::OpenBracketToken) {
12011201
return $this->parseTemplateStringSubscriptExpression($var);
1202-
} else if ($token->kind === TokenKind::ArrowToken) {
1202+
} else if ($token->kind === TokenKind::ArrowToken || $token->kind === TokenKind::QuestionArrowToken) {
12031203
return $this->parseTemplateStringMemberAccessExpression($var);
12041204
} else {
12051205
return $var;
@@ -1248,7 +1248,7 @@ private function parseTemplateStringMemberAccessExpression($expression) : Member
12481248
$expression->parent = $memberAccessExpression;
12491249

12501250
$memberAccessExpression->dereferencableExpression = $expression;
1251-
$memberAccessExpression->arrowToken = $this->eat1(TokenKind::ArrowToken);
1251+
$memberAccessExpression->arrowToken = $this->eat(TokenKind::ArrowToken, TokenKind::QuestionArrowToken);
12521252
$memberAccessExpression->memberName = $this->eat1(TokenKind::Name);
12531253

12541254
return $memberAccessExpression;
@@ -2711,7 +2711,7 @@ private function parsePostfixExpressionRest($expression, $allowUpdateExpression
27112711
return $expression;
27122712
}
27132713

2714-
if ($tokenKind === TokenKind::ArrowToken) {
2714+
if ($tokenKind === TokenKind::ArrowToken || $tokenKind === TokenKind::QuestionArrowToken) {
27152715
$expression = $this->parseMemberAccessExpression($expression);
27162716
return $this->parsePostfixExpressionRest($expression);
27172717
}
@@ -2836,7 +2836,7 @@ private function parseMemberAccessExpression($expression):MemberAccessExpression
28362836
$expression->parent = $memberAccessExpression;
28372837

28382838
$memberAccessExpression->dereferencableExpression = $expression;
2839-
$memberAccessExpression->arrowToken = $this->eat1(TokenKind::ArrowToken);
2839+
$memberAccessExpression->arrowToken = $this->eat(TokenKind::ArrowToken, TokenKind::QuestionArrowToken);
28402840
$memberAccessExpression->memberName = $this->parseMemberName($memberAccessExpression);
28412841

28422842
return $memberAccessExpression;

‎src/PhpTokenizer.php‎

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,9 @@
1010
// The replacement value is arbitrary - it just has to be different from other values of token constants.
1111
define(__NAMESPACE__ . '\T_COALESCE_EQUAL', defined('T_COALESCE_EQUAL') ? constant('T_COALESCE_EQUAL') : 'T_COALESCE_EQUAL');
1212
define(__NAMESPACE__ . '\T_FN', defined('T_FN') ? constant('T_FN') : 'T_FN');
13-
// If this predaates PHP 8.0, T_MATCH is unavailable. The replacement value is arbitrary - it just has to be different from other values of token constants.
13+
// If this predates PHP 8.0, T_MATCH is unavailable. The replacement value is arbitrary - it just has to be different from other values of token constants.
1414
define(__NAMESPACE__ . '\T_MATCH', defined('T_MATCH') ? constant('T_MATCH') : 'T_MATCH');
15+
define(__NAMESPACE__ . '\T_NULLSAFE_OBJECT_OPERATOR', defined('T_NULLSAFE_OBJECT_OPERATOR') ? constant('T_NULLSAFE_OBJECT_OPERATOR') : 'T_MATCH');
1516

1617
/**
1718
* Tokenizes content using PHP's built-in `token_get_all`, and converts to "lightweight" Token representation.
@@ -130,6 +131,57 @@ public static function getTokensArrayFromContent(
130131
$arr[] = new Token(TokenKind::ScriptSectionStartTag, $fullStart, $start, $pos-$fullStart);
131132
$start = $fullStart = $pos;
132133
break;
134+
case \PHP_VERSION_ID >= 80000 ? \T_NAME_QUALIFIED : -1000:
135+
case \PHP_VERSION_ID >= 80000 ? \T_NAME_FULLY_QUALIFIED : -1001:
136+
// NOTE: This switch is called on every token of every file being parsed, so this traded performance for readability.
137+
//
138+
// PHP's Opcache is able to optimize switches that are exclusively known longs,
139+
// but not switches that mix strings and longs or have unknown longs.
140+
// Longs are only known if they're declared within the same *class* or an internal constant (tokenizer).
141+
//
142+
// For some reason, the SWITCH_LONG opcode was not generated when the expression was part of a class constant.
143+
// (seen with php -d opcache.opt_debug_level=0x20000)
144+
//
145+
// Use negative values because that's not expected to overlap with token kinds that token_get_all() will return.
146+
//
147+
// T_NAME_* was added in php 8.0 to forbid whitespace between parts of names.
148+
// Here, emulate the tokenization of php 7 by splitting it up into 1 or more tokens.
149+
foreach (\explode('\\', $token[1]) as $i => $name) {
150+
if ($i) {
151+
$arr[] = new Token(TokenKind::BackslashToken, $fullStart, $start, 1 + $start - $fullStart);
152+
$start++;
153+
$fullStart = $start;
154+
}
155+
if ($name === '') {
156+
continue;
157+
}
158+
// TODO: TokenStringMaps::RESERVED_WORDS[$name] ?? TokenKind::Name for compatibility?
159+
$len = \strlen($name);
160+
$arr[] = new Token(TokenKind::Name, $fullStart, $start, $len + $start - $fullStart);
161+
$start += $len;
162+
$fullStart = $start;
163+
}
164+
break;
165+
case \PHP_VERSION_ID >= 80000 ? \T_NAME_RELATIVE : -1002:
166+
// This is a namespace-relative name: namespace\...
167+
foreach (\explode('\\', $token[1]) as $i => $name) {
168+
$len = \strlen($name);
169+
if (!$i) {
170+
$arr[] = new Token(TokenKind::NamespaceKeyword, $fullStart, $start, $len + $start - $fullStart);
171+
$start += $len;
172+
$fullStart = $start;
173+
continue;
174+
}
175+
$arr[] = new Token(TokenKind::BackslashToken, $fullStart, $start, 1);
176+
$start++;
177+
178+
// TODO: TokenStringMaps::RESERVED_WORDS[$name] ?? TokenKind::Name for compatibility?
179+
$arr[] = new Token(TokenKind::Name, $start, $start, $len);
180+
181+
$start += $len;
182+
$fullStart = $start;
183+
}
184+
break;
133185
case \T_COMMENT:
134186
case \T_DOC_COMMENT:
135187
if ($treatCommentsAsTrivia) {
@@ -256,6 +308,7 @@ protected static function tokenGetAll(string $content, $parseContext): array
256308
"}" => TokenKind::CloseBraceToken,
257309
"." => TokenKind::DotToken,
258310
T_OBJECT_OPERATOR => TokenKind::ArrowToken,
311+
T_NULLSAFE_OBJECT_OPERATOR => TokenKind::QuestionArrowToken,
259312
T_INC => TokenKind::PlusPlusToken,
260313
T_DEC => TokenKind::MinusMinusToken,
261314
T_POW => TokenKind::AsteriskAsteriskToken,

‎src/TokenKind.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ class TokenKind {
149149
const BacktickToken = 260;
150150
const QuestionToken = 261;
151151
const QuestionQuestionEqualsToken = 262;
152+
const QuestionArrowToken = 263;
152153

153154
const DecimalLiteralToken = 301;
154155
const OctalLiteralToken = 302;

‎src/TokenStringMaps.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ class TokenStringMaps {
160160
"^=" => TokenKind::CaretEqualsToken,
161161
"|=" => TokenKind::BarEqualsToken,
162162
"," => TokenKind::CommaToken,
163+
"?->" => TokenKind::QuestionArrowToken,
163164
"??" => TokenKind::QuestionQuestionToken,
164165
"??=" => TokenKind::QuestionQuestionEqualsToken,
165166
"<=>" => TokenKind::LessThanEqualsGreaterThanToken,

‎tests/cases/parser80/nullsafe_operator1.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
<?php
2+
$x = $a?->b;

‎tests/cases/parser80/nullsafe_operator1.php.diag‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[]

‎tests/cases/parser80/nullsafe_operator1.php.tree‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{
2+
"SourceFileNode": {
3+
"statementList": [
4+
{
5+
"InlineHtml": {
6+
"scriptSectionEndTag": null,
7+
"text": null,
8+
"scriptSectionStartTag": {
9+
"kind": "ScriptSectionStartTag",
10+
"textLength": 6
11+
}
12+
}
13+
},
14+
{
15+
"ExpressionStatement": {
16+
"expression": {
17+
"AssignmentExpression": {
18+
"leftOperand": {
19+
"Variable": {
20+
"dollar": null,
21+
"name": {
22+
"kind": "VariableName",
23+
"textLength": 2
24+
}
25+
}
26+
},
27+
"operator": {
28+
"kind": "EqualsToken",
29+
"textLength": 1
30+
},
31+
"byRef": null,
32+
"rightOperand": {
33+
"MemberAccessExpression": {
34+
"dereferencableExpression": {
35+
"Variable": {
36+
"dollar": null,
37+
"name": {
38+
"kind": "VariableName",
39+
"textLength": 2
40+
}
41+
}
42+
},
43+
"arrowToken": {
44+
"kind": "QuestionArrowToken",
45+
"textLength": 3
46+
},
47+
"memberName": {
48+
"kind": "Name",
49+
"textLength": 1
50+
}
51+
}
52+
}
53+
}
54+
},
55+
"semicolon": {
56+
"kind": "SemicolonToken",
57+
"textLength": 1
58+
}
59+
}
60+
}
61+
],
62+
"endOfFileToken": {
63+
"kind": "EndOfFileToken",
64+
"textLength": 0
65+
}
66+
}
67+
}

‎tests/cases/parser80/nullsafe_operator2.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
<?php
2+
$a?->foo(1);

‎tests/cases/parser80/nullsafe_operator2.php.diag‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[]

‎tests/cases/parser80/nullsafe_operator2.php.tree‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
{
2+
"SourceFileNode": {
3+
"statementList": [
4+
{
5+
"InlineHtml": {
6+
"scriptSectionEndTag": null,
7+
"text": null,
8+
"scriptSectionStartTag": {
9+
"kind": "ScriptSectionStartTag",
10+
"textLength": 6
11+
}
12+
}
13+
},
14+
{
15+
"ExpressionStatement": {
16+
"expression": {
17+
"CallExpression": {
18+
"callableExpression": {
19+
"MemberAccessExpression": {
20+
"dereferencableExpression": {
21+
"Variable": {
22+
"dollar": null,
23+
"name": {
24+
"kind": "VariableName",
25+
"textLength": 2
26+
}
27+
}
28+
},
29+
"arrowToken": {
30+
"kind": "QuestionArrowToken",
31+
"textLength": 3
32+
},
33+
"memberName": {
34+
"kind": "Name",
35+
"textLength": 3
36+
}
37+
}
38+
},
39+
"openParen": {
40+
"kind": "OpenParenToken",
41+
"textLength": 1
42+
},
43+
"argumentExpressionList": {
44+
"ArgumentExpressionList": {
45+
"children": [
46+
{
47+
"ArgumentExpression": {
48+
"byRefToken": null,
49+
"dotDotDotToken": null,
50+
"expression": {
51+
"NumericLiteral": {
52+
"children": {
53+
"kind": "IntegerLiteralToken",
54+
"textLength": 1
55+
}
56+
}
57+
}
58+
}
59+
}
60+
]
61+
}
62+
},
63+
"closeParen": {
64+
"kind": "CloseParenToken",
65+
"textLength": 1
66+
}
67+
}
68+
},
69+
"semicolon": {
70+
"kind": "SemicolonToken",
71+
"textLength": 1
72+
}
73+
}
74+
}
75+
],
76+
"endOfFileToken": {
77+
"kind": "EndOfFileToken",
78+
"textLength": 0
79+
}
80+
}
81+
}

0 commit comments

Comments
 (0)