Merge pull request #200 from Brille24/string_tokenization_part2

Adding the a test for combined tokens

Author: dbu

src/PHPCR/Util/QOM/Sql2Scanner.php

@@ -149,7 +149,9 @@ protected function scan($sql2)
         $stringStartCharacter = false;
         $isEscaped = false;
         $escapedQuotesCount = 0;
-        foreach (\str_split($sql2) as $index => $character) {
+        $splitString = \str_split($sql2);
+        for ($index = 0; $index < count($splitString); $index++) {
+            $character = $splitString[$index];
             if (!$stringStartCharacter && in_array($character, [' ', "\t", "\n"], true)) {
                 if ($currentToken !== '') {
                     $tokens[] = $currentToken;
@@ -165,12 +167,27 @@ protected function scan($sql2)
                 $currentToken = '';
                 continue;
             }
+
+            // Handling the squared brackets in queries
+            if (!$isEscaped && $character === '[') {
+                if ($currentToken !== '') {
+                    $tokens[] = $currentToken;
+                }
+                $stringSize = $this->parseBrackets($sql2, $index);
+                $bracketContent = substr($sql2, $index + 1, $stringSize - 2);
+                $tokens[] = '['.trim($bracketContent, '"').']';
+
+                // We need to subtract one here because the for loop will automatically increment the index
+                $index += $stringSize - 1;
+                continue;
+            }
+
             $currentToken .= $character;
 
             if (!$isEscaped && in_array($character, ['"', "'"], true)) {
                 // Checking if the previous or next value is a ' to handle the weird SQL strings
                 // This will not check if the amount of quotes is even
-                $nextCharacter = $this->getCharacterAtIndex($sql2, $index + 1);
+                $nextCharacter = $splitString[$index + 1] ?? '';
                 if ($character === "'" && $nextCharacter === "'") {
                     $isEscaped = true;
                     $escapedQuotesCount++;
@@ -188,6 +205,12 @@ protected function scan($sql2)
                 } elseif (!$stringStartCharacter) {
                     // If there is no start character already we have found the beginning of a new string
                     $stringStartCharacter = $character;
+
+                    // When tokenizing `AS"abc"` add the current token (AS) as token already
+                    if (strlen($currentToken) > 1) {
+                        $tokens[] = substr($currentToken, 0, strlen($currentToken) - 1);
+                        $currentToken = $character;
+                    }
                 }
             }
             $isEscaped = $character === '\\';
@@ -203,12 +226,10 @@ protected function scan($sql2)
         return $tokens;
     }
 
-    private function getCharacterAtIndex($string, $index)
+    private function parseBrackets(string $query, int $index): int
     {
-        if ($index < strlen($string)) {
-            return $string[$index];
-        }
+        $endPosition = strpos($query, ']', $index) + 1;
 
-        return '';
+        return $endPosition - $index;
     }
 }

tests/PHPCR/Tests/Util/QOM/Sql2ScannerTest.php

@@ -28,9 +28,9 @@ public function testToken()
     /**
      * @dataProvider dataTestStringTokenization
      */
-    public function testStringTokenization()
+    public function testStringTokenization(string $query)
     {
-        $scanner = new Sql2Scanner('SELECT page.* FROM [nt:unstructured] AS page WHERE name ="Hello world"');
+        $scanner = new Sql2Scanner($query);
         $expected = [
             'SELECT',
             'page',
@@ -49,7 +49,7 @@ public function testStringTokenization()
         $this->expectTokensFromScanner($scanner, $expected);
     }
 
-    public function dataTestStringTokenization()
+    public function dataTestStringTokenization(): array
     {
         $multilineQuery = <<<'SQL'
 SELECT page.* 
@@ -124,6 +124,57 @@ public function testSQLEscapedStrings2()
         $this->expectTokensFromScanner($scanner, $expected);
     }
 
+    public function testSquareBrackets()
+    {
+        $sql = 'WHERE ISSAMENODE(file, ["/home node"])';
+
+        $scanner = new Sql2Scanner($sql);
+        $expected = [
+            'WHERE',
+            'ISSAMENODE',
+            '(',
+            'file',
+            ',',
+            '[/home node]',
+            ')',
+        ];
+
+        $this->expectTokensFromScanner($scanner, $expected);
+    }
+
+    public function testSquareBracketsWithoutQuotes()
+    {
+        $sql = 'WHERE ISSAMENODE(file, [/home node])';
+
+        $scanner = new Sql2Scanner($sql);
+        $expected = [
+            'WHERE',
+            'ISSAMENODE',
+            '(',
+            'file',
+            ',',
+            '[/home node]',
+            ')',
+        ];
+
+        $this->expectTokensFromScanner($scanner, $expected);
+    }
+
+    public function testTokenizingWithMissingSpaces()
+    {
+        $sql = 'SELECT * AS"all"';
+
+        $scanner = new Sql2Scanner($sql);
+        $expected = [
+            'SELECT',
+            '*',
+            'AS',
+            '"all"',
+        ];
+
+        $this->expectTokensFromScanner($scanner, $expected);
+    }
+
     public function testThrowingErrorOnUnclosedString()
     {
         $this->expectException(InvalidQueryException::class);