Skip to content

Support multi-byte characters (Unicode) #69

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Support multi-byte characters (Unicode) #69

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
220720d
Support multi-byte characters (Unicode)
bluedasher13 May 28, 2020
ab0aad8
Support multi-byte characters (Unicode)
bluedasher13 May 28, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions src/Seld/JsonLint/JsonParser.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,9 +198,9 @@ public function parse($input, $flags = 0)
}

$message = null;
if (\in_array("'STRING'", $expected) && \in_array(substr($this->lexer->match, 0, 1), array('"', "'"))) {
if (\in_array("'STRING'", $expected) && \in_array(mb_substr($this->lexer->match, 0, 1), array('"', "'"))) {
$message = "Invalid string";
if ("'" === substr($this->lexer->match, 0, 1)) {
if ("'" === mb_substr($this->lexer->match, 0, 1)) {
$message .= ", it appears you used single quotes instead of double quotes";
} elseif (preg_match('{".+?(\\\\[^"bfnrt/\\\\u](...)?)}', $this->lexer->getFullUpcomingInput(), $match)) {
$message .= ", it appears you have an unescaped backslash at: ".$match[1];
Expand All @@ -218,7 +218,7 @@ public function parse($input, $flags = 0)
$errStr .= implode(', ', $expected);
}

if (',' === substr(trim($this->lexer->getPastInput()), -1)) {
if (',' === mb_substr(trim($this->lexer->getPastInput()), -1)) {
$errStr .= " - It appears you have an extra trailing comma";
}

Expand Down Expand Up @@ -347,10 +347,10 @@ private function performAction(stdClass $yyval, $yytext, $yyleng, $yylineno, $yy
$yyval->token = $yytext;
break;
case 2:
if (strpos($yytext, 'e') !== false || strpos($yytext, 'E') !== false) {
if (mb_strpos($yytext, 'e') !== false || mb_strpos($yytext, 'E') !== false) {
$yyval->token = \floatval($yytext);
} else {
$yyval->token = strpos($yytext, '.') === false ? \intval($yytext) : \floatval($yytext);
$yyval->token = mb_strpos($yytext, '.') === false ? \intval($yytext) : \floatval($yytext);
}
break;
case 3:
Expand Down Expand Up @@ -468,7 +468,7 @@ private function stringInterpolation($match)
case '\/':
return "/";
default:
return html_entity_decode('&#x'.ltrim(substr($match[0], 2), '0').';', ENT_QUOTES, 'UTF-8');
return html_entity_decode('&#x'.ltrim(mb_substr($match[0], 2), '0').';', ENT_QUOTES, 'UTF-8');
}
}

Expand All @@ -495,7 +495,7 @@ private function failOnBOM($input)
// UTF-8 ByteOrderMark sequence
$bom = "\xEF\xBB\xBF";

if (substr($input, 0, 3) === $bom) {
if (mb_substr($input, 0, 3) === $bom) {
$this->parseError("BOM detected, make sure your input does not include a Unicode Byte-Order-Mark", array());
}
}
Expand Down
68 changes: 34 additions & 34 deletions src/Seld/JsonLint/Lexer.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,20 +20,20 @@ class Lexer
{
private $EOF = 1;
private $rules = array(
0 => '/\G\s+/',
1 => '/\G-?([0-9]|[1-9][0-9]+)(\.[0-9]+)?([eE][+-]?[0-9]+)?\b/',
2 => '{\G"(?>\\\\["bfnrt/\\\\]|\\\\u[a-fA-F0-9]{4}|[^\0-\x1f\\\\"]++)*+"}',
3 => '/\G\{/',
4 => '/\G\}/',
5 => '/\G\[/',
6 => '/\G\]/',
7 => '/\G,/',
8 => '/\G:/',
9 => '/\Gtrue\b/',
10 => '/\Gfalse\b/',
11 => '/\Gnull\b/',
12 => '/\G$/',
13 => '/\G./',
0 => '/\G\s+/u',
1 => '/\G-?([0-9]|[1-9][0-9]+)(\.[0-9]+)?([eE][+-]?[0-9]+)?\b/u',
2 => '{\G"(?>\\\\["bfnrt/\\\\]|\\\\u[a-fA-F0-9]{4}|[^\0-\x1f\\\\"]++)*+"}u',
3 => '/\G\{/u',
4 => '/\G\}/u',
5 => '/\G\[/u',
6 => '/\G\]/u',
7 => '/\G,/u',
8 => '/\G:/u',
9 => '/\Gtrue\b/u',
10 => '/\Gfalse\b/u',
11 => '/\Gnull\b/u',
12 => '/\G$/u',
13 => '/\G./u',
);

private $conditions = array(
Expand Down Expand Up @@ -82,37 +82,37 @@ public function setInput($input)
public function showPosition()
{
$pre = str_replace("\n", '', $this->getPastInput());
$c = str_repeat('-', max(0, \strlen($pre) - 1)); // new Array(pre.length + 1).join("-");
$c = str_repeat('-', max(0, \mb_strlen($pre) - 1)); // new Array(pre.length + 1).join("-");

return $pre . str_replace("\n", '', $this->getUpcomingInput()) . "\n" . $c . "^";
}

public function getPastInput()
{
$pastLength = $this->offset - \strlen($this->match);
$pastLength = $this->offset - \mb_strlen($this->match);

return ($pastLength > 20 ? '...' : '') . substr($this->input, max(0, $pastLength - 20), min(20, $pastLength));
return ($pastLength > 20 ? '...' : '') . mb_substr($this->input, max(0, $pastLength - 20), min(20, $pastLength));
}

public function getUpcomingInput()
{
$next = $this->match;
if (\strlen($next) < 20) {
$next .= substr($this->input, $this->offset, 20 - \strlen($next));
if (\mb_strlen($next) < 20) {
$next .= mb_substr($this->input, $this->offset, 20 - \mb_strlen($next));
}

return substr($next, 0, 20) . (\strlen($next) > 20 ? '...' : '');
return mb_substr($next, 0, 20) . (\mb_strlen($next) > 20 ? '...' : '');
}

public function getFullUpcomingInput()
{
$next = $this->match;
if (substr($next, 0, 1) === '"' && substr_count($next, '"') === 1) {
$len = \strlen($this->input);
$strEnd = min(strpos($this->input, '"', $this->offset + 1) ?: $len, strpos($this->input, "\n", $this->offset + 1) ?: $len);
$next .= substr($this->input, $this->offset, $strEnd - $this->offset);
} elseif (\strlen($next) < 20) {
$next .= substr($this->input, $this->offset, 20 - \strlen($next));
if (mb_substr($next, 0, 1) === '"' && mb_substr_count($next, '"') === 1) {
$len = \mb_strlen($this->input);
$strEnd = min(mb_strpos($this->input, '"', $this->offset + 1) ?: $len, mb_strpos($this->input, "\n", $this->offset + 1) ?: $len);
$next .= mb_substr($this->input, $this->offset, $strEnd - $this->offset);
} elseif (\mb_strlen($next) < 20) {
$next .= mb_substr($this->input, $this->offset, 20 - \mb_strlen($next));
}

return $next;
Expand All @@ -128,7 +128,7 @@ private function next()
if ($this->done) {
return $this->EOF;
}
if ($this->offset === \strlen($this->input)) {
if ($this->offset === \mb_strlen($this->input)) {
$this->done = true;
}

Expand All @@ -146,8 +146,8 @@ private function next()
$rulesLen = \count($rules);

for ($i=0; $i < $rulesLen; $i++) {
if (preg_match($this->rules[$rules[$i]], $this->input, $match, 0, $this->offset)) {
preg_match_all('/\n.*/', $match[0], $lines);
if (preg_match($this->rules[$rules[$i]], $this->input, $match, 0, strlen(mb_substr($this->input, 0, $this->offset)))) {
preg_match_all('/\n.*/u', $match[0], $lines);
$lines = $lines[0];
if ($lines) {
$this->yylineno += \count($lines);
Expand All @@ -157,13 +157,13 @@ private function next()
'first_line' => $this->yylloc['last_line'],
'last_line' => $this->yylineno+1,
'first_column' => $this->yylloc['last_column'],
'last_column' => $lines ? \strlen($lines[\count($lines) - 1]) - 1 : $this->yylloc['last_column'] + \strlen($match[0]),
'last_column' => $lines ? \mb_strlen($lines[\count($lines) - 1]) - 1 : $this->yylloc['last_column'] + \mb_strlen($match[0]),
);
$this->yytext .= $match[0];
$this->match .= $match[0];
$this->yyleng = \strlen($this->yytext);
$this->yyleng = \mb_strlen($this->yytext);
$this->more = false;
$this->offset += \strlen($match[0]);
$this->offset += \mb_strlen($match[0]);
$token = $this->performAction($rules[$i], $this->conditionStack[\count($this->conditionStack)-1]);
if ($token) {
return $token;
Expand All @@ -173,7 +173,7 @@ private function next()
}
}

if ($this->offset === \strlen($this->input)) {
if ($this->offset === \mb_strlen($this->input)) {
return $this->EOF;
}

Expand Down Expand Up @@ -201,7 +201,7 @@ private function performAction($avoiding_name_collisions, $YY_START)
return 6;
break;
case 2:
$this->yytext = substr($this->yytext, 1, $this->yyleng-2);
$this->yytext = mb_substr($this->yytext, 1, $this->yyleng-2);

return 4;
case 3:
Expand Down