Assert "function" keyword is not "reserved"

"reserved" keyword is tokenized sooner thus the same keyword defined as "function" is useless.

Author: mvorisek

src/Tokenizer.php

@@ -416,19 +416,15 @@ final class Tokenizer
         'CENTROID',
         'CHAR',
         'CHARACTER_LENGTH',
-        'CHARSET',
         'CHAR_LENGTH',
         'CHECKSUM_AGG',
         'COALESCE',
         'COERCIBILITY',
-        'COLLATION',
         'COMPRESS',
         'CONCAT',
         'CONCAT_WS',
         'CONNECTION_ID',
-        'CONTAINS',
         'CONV',
-        'CONVERT',
         'CONVERT_TZ',
         'CONVEXHULL',
         'COS',
@@ -441,23 +437,19 @@ final class Tokenizer
         'CURDATE',
         'CURRENT_DATE',
         'CURRENT_TIME',
-        'CURRENT_TIMESTAMP',
         'CURRENT_USER',
         'CURTIME',
-        'DATABASE',
         'DATE',
         'DATEDIFF',
         'DATE_ADD',
         'DATE_DIFF',
         'DATE_FORMAT',
         'DATE_SUB',
-        'DAY',
         'DAYNAME',
         'DAYOFMONTH',
         'DAYOFWEEK',
         'DAYOFYEAR',
         'DECODE',
-        'DEFAULT',
         'DEGREES',
         'DENSE_RANK',
         'DES_DECRYPT',
@@ -505,17 +497,12 @@ final class Tokenizer
         'GROUP_CONCAT',
         'GROUP_UNIQUE_USERS',
         'HEX',
-        'HOUR',
-        'IF',
-        'IFNULL',
         'INET_ATON',
         'INET_NTOA',
-        'INSERT',
         'INSTR',
         'INTERIORRINGN',
         'INTERSECTION',
         'INTERSECTS',
-        'INTERVAL',
         'ISCLOSED',
         'ISEMPTY',
         'ISNULL',
@@ -525,12 +512,10 @@ final class Tokenizer
         'IS_USED_LOCK',
         'LAG',
         'LAST_DAY',
-        'LAST_INSERT_ID',
         'LAST_VALUE',
         'LCASE',
         'LEAD',
         'LEAST',
-        'LEFT',
         'LENGTH',
         'LINEFROMTEXT',
         'LINEFROMWKB',
@@ -565,11 +550,9 @@ final class Tokenizer
         'MICROSECOND',
         'MID',
         'MIN',
-        'MINUTE',
         'MLINEFROMTEXT',
         'MLINEFROMWKB',
         'MOD',
-        'MONTH',
         'MONTHNAME',
         'MPOINTFROMTEXT',
         'MPOINTFROMWKB',
@@ -597,7 +580,6 @@ final class Tokenizer
         'OLD_PASSWORD',
         'ORD',
         'OVERLAPS',
-        'PASSWORD',
         'PERCENTILE_CONT',
         'PERCENTILE_DISC',
         'PERCENT_RANK',
@@ -625,16 +607,13 @@ final class Tokenizer
         'RELATED',
         'RELEASE_LOCK',
         'REPEAT',
-        'REPLACE',
         'REVERSE',
-        'RIGHT',
         'ROUND',
         'ROW_COUNT',
         'ROW_NUMBER',
         'RPAD',
         'RTRIM',
         'SCHEMA',
-        'SECOND',
         'SEC_TO_TIME',
         'SESSION_USER',
         'SHA',
@@ -676,7 +655,6 @@ final class Tokenizer
         'TOUCHES',
         'TO_DAYS',
         'TRIM',
-        'TRUNCATE',
         'UCASE',
         'UNCOMPRESS',
         'UNCOMPRESSED_LENGTH',

tests/TokenizerTest.php

@@ -12,6 +12,7 @@
 use PHPUnit\Framework\TestCase;
 use ReflectionClass;
 
+use function array_diff_key;
 use function array_filter;
 use function array_merge;
 use function array_unique;
@@ -68,6 +69,19 @@ public function testKeywordsAreSingleUpperWord(): void
         self::assertSame([], $kwsDiff);
     }
 
+    public function testKeywordsAreDisjunctive(): void
+    {
+        $tokenizerKeywords = array_merge(
+            $this->getTokenizerList('reserved'),
+            $this->getTokenizerList('functions'),
+        );
+
+        self::assertSame(
+            [],
+            array_diff_key($tokenizerKeywords, array_unique($tokenizerKeywords)),
+        );
+    }
+
     /** @param list<Token> $expectedTokens */
     public static function assertEqualsTokens(array $expectedTokens, Cursor $cursor): void
     {