make everyhing inside strings raw to make sure strings are correctly tokenized

Author: efekos

src/lexer.ts

@@ -91,21 +91,21 @@ export function tokenizeSyx(source: string): Token[] {
                 curPos++;
             }
         }
-        if (src[0] === '(') tokens.push({ type: TokenType.OpenParen, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === ')') tokens.push({ type: TokenType.CloseParen, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '{') tokens.push({ type: TokenType.OpenBrace, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '}') tokens.push({ type: TokenType.CloseBrace, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '[') tokens.push({ type: TokenType.OpenSquare, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === ']') tokens.push({ type: TokenType.CloseSquare, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === ',') tokens.push({ type: TokenType.Comma, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === ';') tokens.push({ type: TokenType.Semicolon, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '<') tokens.push({ type: TokenType.OpenDiamond, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '>') tokens.push({ type: TokenType.CloseDiamond, value: src.shift(), range: opr(curLine, curPos++) });
+        if (src[0] === '(') tokens.push({ type: inString?20:TokenType.OpenParen, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === ')') tokens.push({ type: inString?20:TokenType.CloseParen, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === '{') tokens.push({ type: inString?20:TokenType.OpenBrace, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === '}') tokens.push({ type: inString?20:TokenType.CloseBrace, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === '[') tokens.push({ type: inString?20:TokenType.OpenSquare, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === ']') tokens.push({ type: inString?20:TokenType.CloseSquare, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === ',') tokens.push({ type: inString?20:TokenType.Comma, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === ';') tokens.push({ type: inString?20:TokenType.Semicolon, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === '<') tokens.push({ type: inString?20:TokenType.OpenDiamond, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === '>') tokens.push({ type: inString?20:TokenType.CloseDiamond, value: src.shift(), range: opr(curLine, curPos++) });
         else if (src[0] === '\'') { tokens.push({ type: TokenType.SingleQuote, value: src.shift(), range: opr(curLine, curPos++) }); t('\''); }
         else if (src[0] === '"') { tokens.push({ type: TokenType.DoubleQuote, value: src.shift(), range: opr(curLine, curPos++) }); t('"'); }
-        else if (src[0] === '|') tokens.push({ type: TokenType.VarSeperator, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === '|') tokens.push({ type: inString?20:TokenType.VarSeperator, value: src.shift(), range: opr(curLine, curPos++) });
         else if (src[0] === '+' && chars.includes(src[1])) {
-            if (src[1] === 's') tokens.push({ type: TokenType.WhitespaceIdentifier, value: '+s', range: tpr(pos(curLine, curPos), pos(curLine, curPos + 2)) });
+            if (src[1] === 's') tokens.push({ type: inString?20:TokenType.WhitespaceIdentifier, value: '+s', range: tpr(pos(curLine, curPos), pos(curLine, curPos + 2)) });
             curPos += 2;
             src.shift(); src.shift();
         } else if (isInt(src[0])) {
@@ -116,7 +116,7 @@ export function tokenizeSyx(source: string): Token[] {
             }
 
             curPos += ident.length;
-            tokens.push({ type: TokenType.IntNumber, value: ident, range: tpr(pos(curLine, startPos), pos(curLine, curPos)) });
+            tokens.push({ type: inString?20:TokenType.IntNumber, value: ident, range: tpr(pos(curLine, startPos), pos(curLine, curPos)) });
         } else if (isAlphabetic(src[0])) {
             let ident = '';
             const startPos = curPos;
@@ -126,7 +126,7 @@ export function tokenizeSyx(source: string): Token[] {
             }
 
             const reserved = keywords[ident];
-            tokens.push({ type: reserved ?? TokenType.Identifier, value: ident, range: tpr(pos(curLine, startPos), pos(curLine, curPos)) });
+            tokens.push({ type: inString?20:reserved ?? TokenType.Identifier, value: ident, range: tpr(pos(curLine, startPos), pos(curLine, curPos)) });
         } else if (isSkippable(src[0]) && !inString) {
             src.shift();
             curPos++;
@@ -161,7 +161,7 @@ export function tokenizeSys(source: string): Token[] {
     let curLine = 1;
 
     while (src.length > 0 && `${src[0]}${src[1]}${src[2]}` !== ':::') {
-        if (src[0] === ';') tokens.push({ type: TokenType.Semicolon, value: src.shift(), range: opr(curLine, curPos++) });
+        if (src[0] === ';') tokens.push({ type: inString?20:TokenType.Semicolon, value: src.shift(), range: opr(curLine, curPos++) });
         else if (src[0] === '\'') { tokens.push({ type: TokenType.SingleQuote, value: src.shift(), range: opr(curLine, curPos++) }); t('\''); }
         else if (src[0] === '"') { tokens.push({ type: TokenType.DoubleQuote, value: src.shift(), range: opr(curLine, curPos++) }); t('"'); }
         else if (isAlphabetic(src[0])) {