contain every character and skip comment tokens in strings

Author: efekos

src/lexer.ts

@@ -69,35 +69,42 @@ function tpr(start: Position, end: Position): Range {
  * @param {string} source Source string.
  * @returns A list of tokens generated from source string.
  * @author efekos
- * @version 1.0.7
- * @since 0.0.1-alpha
- * @throws LexerError if an error occurs.
+ * @version 1.0.8
+ * @since 0.0.2-alpha
  */
 export function tokenizeSyx(source: string): Token[] {
     const tokens: Token[] = [];
     const src = source.split('');
+    let lastString = 'n';
+    let inString = false;
+    function t(s:string){
+        if(lastString==='\''&&s==='\'') {lastString='n';inString = !inString;}
+        if(lastString==='"'&&s==='"') {lastString='n';inString = !inString;}
+        if(lastString==='n') {lastString=s;inString=!inString;}
+    }
     let curPos = 1;
     let curLine = 1;
 
     while (src.length > 0) {
-        if (src[0] === '/' && src[1] === '/') {
+        if (src[0] === '/' && src[1] === '/'&&!inString) {
             while (src.length > 0 && src[0] as string !== '\n') {
                 src.shift();
+                curPos++;
             }
         }
-        if (src[0] === '(') tokens.push({ type: TokenType.OpenParen, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === ')') tokens.push({ type: TokenType.CloseParen, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '{') tokens.push({ type: TokenType.OpenBrace, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '}') tokens.push({ type: TokenType.CloseBrace, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '[') tokens.push({ type: TokenType.OpenSquare, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === ']') tokens.push({ type: TokenType.CloseSquare, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === ',') tokens.push({ type: TokenType.Comma, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === ';') tokens.push({ type: TokenType.Semicolon, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '<') tokens.push({ type: TokenType.OpenDiamond, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '>') tokens.push({ type: TokenType.CloseDiamond, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '\'') tokens.push({ type: TokenType.SingleQuote, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '"') tokens.push({ type: TokenType.DoubleQuote, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '|') tokens.push({ type: TokenType.VarSeperator, value: src.shift(), range: opr(curLine, curPos++) });
+        if (src[0] === '(') tokens.push({ type: inString?20:TokenType.OpenParen, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === ')') tokens.push({ type: inString?20:TokenType.CloseParen, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === '{') tokens.push({ type: inString?20:TokenType.OpenBrace, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === '}') tokens.push({ type: inString?20:TokenType.CloseBrace, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === '[') tokens.push({ type: inString?20:TokenType.OpenSquare, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === ']') tokens.push({ type: inString?20:TokenType.CloseSquare, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === ',') tokens.push({ type: inString?20:TokenType.Comma, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === ';') tokens.push({ type: inString?20:TokenType.Semicolon, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === '<') tokens.push({ type: inString?20:TokenType.OpenDiamond, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === '>') tokens.push({ type: inString?20:TokenType.CloseDiamond, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === '\'') {tokens.push({ type: TokenType.SingleQuote, value: src.shift(), range: opr(curLine, curPos++) });t('\'');}
+        else if (src[0] === '"') {tokens.push({ type: TokenType.DoubleQuote, value: src.shift(), range: opr(curLine, curPos++) });t('"');}
+        else if (src[0] === '|') tokens.push({ type: inString?20:TokenType.VarSeperator, value: src.shift(), range: opr(curLine, curPos++) });
         else if (src[0] === '+' && chars.includes(src[1])) {
             if (src[1] === 's') tokens.push({ type: TokenType.WhitespaceIdentifier, value: '+s', range: tpr(pos(curLine, curPos), pos(curLine, curPos + 2)) });
             curPos += 2;
@@ -121,7 +128,7 @@ export function tokenizeSyx(source: string): Token[] {
 
             const reserved = keywords[ident];
             tokens.push({ type: reserved ?? TokenType.Identifier, value: ident, range: tpr(pos(curLine, startPos), pos(curLine, curPos)) });
-        } else if (isSkippable(src[0])) {
+        } else if (isSkippable(src[0])&&!inString) {
             src.shift();
             curPos++;
             if (src[0] === '\n') { curLine++; curPos = 0; };
@@ -138,20 +145,27 @@ export function tokenizeSyx(source: string): Token[] {
  * @param {string} source Source string.
  * @returns A list of tokens generated from the source file.
  * @author efekos
- * @version 1.0.4
- * @since 0.0.1-alpha
+ * @version 1.0.5
+ * @since 0.0.2-alpha
  */
 export function tokenizeSys(source: string): Token[] {
     const src = source.split('');
     const tokens: Token[] = [];
+    let lastString = 'n';
+    let inString = false;
+    function t(s:string){
+        if(lastString==='\''&&s==='\'') {lastString='n';inString = !inString;}
+        if(lastString==='"'&&s==='"') {lastString='n';inString = !inString;}
+        if(lastString==='n') {lastString=s;inString=!inString;}
+    }
 
     let curPos = 0;
     let curLine = 1;
 
     while (src.length > 0 && `${src[0]}${src[1]}${src[2]}` !== ':::') {
-        if (src[0] === ';') tokens.push({ type: TokenType.Semicolon, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '\'') tokens.push({ type: TokenType.SingleQuote, value: src.shift(), range: opr(curLine, curPos++) });
-        else if (src[0] === '"') tokens.push({ type: TokenType.DoubleQuote, value: src.shift(), range: opr(curLine, curPos++) });
+        if (src[0] === ';') tokens.push({ type: inString?20:TokenType.Semicolon, value: src.shift(), range: opr(curLine, curPos++) });
+        else if (src[0] === '\'') {tokens.push({ type: TokenType.SingleQuote, value: src.shift(), range: opr(curLine, curPos++) });t('\'');}
+        else if (src[0] === '"') {tokens.push({ type: TokenType.DoubleQuote, value: src.shift(), range: opr(curLine, curPos++) });t('"');}
         else if (isAlphabetic(src[0])) {
             let ident = '';
             const startPos = curPos;
@@ -162,7 +176,7 @@ export function tokenizeSys(source: string): Token[] {
 
             const reserved = keywords[ident];
             tokens.push({ type: reserved ?? TokenType.Identifier, value: ident, range: tpr(pos(curLine, startPos), pos(curLine, curPos)) });
-        } else if (isSkippable(src[0])) {
+        } else if (isSkippable(src[0])&&!inString) {
             src.shift();
             curPos++;
             if (src[0] === '\n') curLine++;