Skip to content

Commit 7f5c55d

Browse files
gsherwoodgsherwood
committed
Merge branch 'php-8.1/tokenizer-php-bugfix-octal-explicit-notation' of https://github.com/jrfnl/PHP_CodeSniffer 
2 parents e9f6c43 + 915b12a commit 7f5c55d

File tree

3 files changed

+85
-34
lines changed

3 files changed

+85
-34
lines changed

‎src/Tokenizers/PHP.php‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -752,16 +752,32 @@ protected function tokenize($string)
752752
&& $tokens[($stackPtr + 1)][0] === T_STRING
753753
&& strtolower($tokens[($stackPtr + 1)][1][0]) === 'o'
754754
&& $tokens[($stackPtr + 1)][1][1] !== '_')
755+
&& preg_match('`^(o[0-7]+(?:_[0-7]+)?)([0-9_]*)$`i', $tokens[($stackPtr + 1)][1], $matches) === 1
755756
) {
756757
$finalTokens[$newStackPtr] = [
757758
'code' => T_LNUMBER,
758759
'type' => 'T_LNUMBER',
759-
'content' => $token[1] .= $tokens[($stackPtr + 1)][1],
760+
'content' => $token[1] .= $matches[1],
760761
];
761-
$stackPtr++;
762762
$newStackPtr++;
763+
764+
if (isset($matches[2]) === true && $matches[2] !== '') {
765+
$type = 'T_LNUMBER';
766+
if ($matches[2][0] === '_') {
767+
$type = 'T_STRING';
768+
}
769+
770+
$finalTokens[$newStackPtr] = [
771+
'code' => constant($type),
772+
'type' => $type,
773+
'content' => $matches[2],
774+
];
775+
$newStackPtr++;
776+
}
777+
778+
$stackPtr++;
763779
continue;
764-
}
780+
}//end if
765781

766782
/*
767783
PHP 8.1 introduced two dedicated tokens for the & character.

‎tests/Core/Tokenizer/BackfillExplicitOctalNotationTest.inc‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,15 @@ $foo = 0o_137;
1414

1515
/* testInvalid2 */
1616
$foo = 0O_41;
17+
18+
/* testInvalid3 */
19+
$foo = 0o91;
20+
21+
/* testInvalid4 */
22+
$foo = 0O282;
23+
24+
/* testInvalid5 */
25+
$foo = 0o28_2;
26+
27+
/* testInvalid6 */
28+
$foo = 0o2_82;

‎tests/Core/Tokenizer/BackfillExplicitOctalNotationTest.php‎

Lines changed: 54 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,26 @@ class BackfillExplicitOctalNotationTest extends AbstractMethodUnitTest
1818
/**
1919
* Test that explicitly-defined octal values are tokenized as a single number and not as a number and a string.
2020
*
21-
* @param array $testData The data required for the specific test case.
21+
* @param string $marker The comment which prefaces the target token in the test file.
22+
* @param string $value The expected content of the token
23+
* @param int|string $nextToken The expected next token.
24+
* @param string $nextContent The expected content of the next token.
2225
*
2326
* @dataProvider dataExplicitOctalNotation
2427
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
2528
*
2629
* @return void
2730
*/
28-
public function testExplicitOctalNotation($testData)
31+
public function testExplicitOctalNotation($marker, $value, $nextToken, $nextContent)
2932
{
3033
$tokens = self::$phpcsFile->getTokens();
3134

32-
$number = $this->getTargetToken($testData['marker'], [T_LNUMBER]);
35+
$number = $this->getTargetToken($marker, [T_LNUMBER]);
3336

34-
$this->assertSame(constant($testData['type']), $tokens[$number]['code']);
35-
$this->assertSame($testData['type'], $tokens[$number]['type']);
36-
$this->assertSame($testData['value'], $tokens[$number]['content']);
37+
$this->assertSame($value, $tokens[$number]['content'], 'Content of integer token does not match expectation');
38+
39+
$this->assertSame($nextToken, $tokens[($number + 1)]['code'], 'Next token is not the expected type, but '.$tokens[($number + 1)]['type']);
40+
$this->assertSame($nextContent, $tokens[($number + 1)]['content'], 'Next token did not have the expected contents');
3741

3842
}//end testExplicitOctalNotation()
3943

@@ -49,39 +53,58 @@ public function dataExplicitOctalNotation()
4953
{
5054
return [
5155
[
52-
[
53-
'marker' => '/* testExplicitOctal */',
54-
'type' => 'T_LNUMBER',
55-
'value' => '0o137041',
56-
],
56+
'marker' => '/* testExplicitOctal */',
57+
'value' => '0o137041',
58+
'nextToken' => T_SEMICOLON,
59+
'nextContent' => ';',
60+
],
61+
[
62+
'marker' => '/* testExplicitOctalCapitalised */',
63+
'value' => '0O137041',
64+
'nextToken' => T_SEMICOLON,
65+
'nextContent' => ';',
66+
],
67+
[
68+
'marker' => '/* testExplicitOctalWithNumericSeparator */',
69+
'value' => '0o137_041',
70+
'nextToken' => T_SEMICOLON,
71+
'nextContent' => ';',
72+
],
73+
[
74+
'marker' => '/* testInvalid1 */',
75+
'value' => '0',
76+
'nextToken' => T_STRING,
77+
'nextContent' => 'o_137',
78+
],
79+
[
80+
'marker' => '/* testInvalid2 */',
81+
'value' => '0',
82+
'nextToken' => T_STRING,
83+
'nextContent' => 'O_41',
5784
],
5885
[
59-
[
60-
'marker' => '/* testExplicitOctalCapitalised */',
61-
'type' => 'T_LNUMBER',
62-
'value' => '0O137041',
63-
],
86+
'marker' => '/* testInvalid3 */',
87+
'value' => '0',
88+
'nextToken' => T_STRING,
89+
'nextContent' => 'o91',
6490
],
6591
[
66-
[
67-
'marker' => '/* testExplicitOctalWithNumericSeparator */',
68-
'type' => 'T_LNUMBER',
69-
'value' => '0o137_041',
70-
],
92+
'marker' => '/* testInvalid4 */',
93+
'value' => '0O2',
94+
'nextToken' => T_LNUMBER,
95+
'nextContent' => '82',
7196
],
7297
[
73-
[
74-
'marker' => '/* testInvalid1 */',
75-
'type' => 'T_LNUMBER',
76-
'value' => '0',
77-
],
98+
'marker' => '/* testInvalid5 */',
99+
'value' => '0o2',
100+
'nextToken' => T_LNUMBER,
101+
'nextContent' => '8_2',
78102
],
79103
[
80-
[
81-
'marker' => '/* testInvalid2 */',
82-
'type' => 'T_LNUMBER',
83-
'value' => '0',
84-
],
104+
'marker' => '/* testInvalid6 */',
105+
'value' => '0o2',
106+
'nextToken' => T_STRING,
107+
'nextContent' => '_82',
85108
],
86109
];
87110

0 commit comments

Comments
 (0)