From 69e04a430834b7f7b49b1f5a4546e10682615945 Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 14 May 2025 17:49:38 +0100 Subject: [PATCH 1/3] Remove commas when creating token --- src/parser.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 43b62708..796eea61 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -27,8 +27,9 @@ function createToken(buffer: string, type: TokenType) { if (type === TokenType.NUM) { // This can happen if users simply type ".", which get parsed as number. - if (isNaN(+buffer)) throw ExprError.invalidExpression(); - return new ExprNumber(+buffer); + const numberWithoutCommas = buffer.replace(/,/g, ''); + if (isNaN(+numberWithoutCommas)) throw ExprError.invalidExpression(); + return new ExprNumber(+numberWithoutCommas); } if (type === TokenType.VAR) { From eca495da39c43643b3c108372fb4ff9e8103eaa8 Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 14 May 2025 17:50:01 +0100 Subject: [PATCH 2/3] Special handling for commas inside functions --- src/parser.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/parser.ts b/src/parser.ts index 796eea61..164badbd 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -55,9 +55,16 @@ export function tokenize(str: string) { const tokens = []; let buffer = ''; let type = TokenType.UNKNOWN; + let parenDepth = 0; for (const s of str) { + if (['(', '[', '{'].includes(s)) { + parenDepth++; + } else if ([')', ']', '}'].includes(s)) { + parenDepth--; + } + // Handle Strings if (s === '"') { const newType: TokenType = (((type as TokenType) === TokenType.STR) ? TokenType.UNKNOWN : TokenType.STR); @@ -71,6 +78,22 @@ export function tokenize(str: string) { continue; } + // Special handling for commas + if (s === ',') { + // If we're in a number and not inside parenthesis include the comma + if (type === TokenType.NUM && parenDepth === 0) { + buffer += s; + continue; + } + // Otherwise treat it as an operator + const token = createToken(buffer, type); + if (token) tokens.push(token); + tokens.push(new ExprOperator(',')); + buffer = ''; + type = TokenType.UNKNOWN; + continue; + } + const sType = s.match(/[0-9.]/) ? TokenType.NUM : IDENTIFIER_SYMBOLS.includes(s) ? TokenType.VAR : OPERATOR_SYMBOLS.includes(s) ? TokenType.OP : From d2e1d54ffbc1145056d7c7a30a763160b228d81b Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 14 May 2025 17:50:12 +0100 Subject: [PATCH 3/3] Some tests for ignoring commas --- test/parsing-test.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/parsing-test.ts b/test/parsing-test.ts index a3447aff..1d1c3063 100644 --- a/test/parsing-test.ts +++ b/test/parsing-test.ts @@ -34,6 +34,19 @@ tape('functions', (test) => { test.end(); }); +tape('commas', (test) => { + test.equal(str('1,1'), '11'); + test.equal(str('1,1 + 2,2'), '11 + 22'); + test.equal(str('1,000 + 2,000'), '1000 + 2000'); + test.equal(str('1,'), '1'); // Trailing comma + // test.equal(str(',1'), '1'); // Leading comma TODO: failing + test.equal(str('1,2,3'), '123'); // Multiple commas + test.equal(str('1,234,567.89'), '1234567.89'); // Multiple commas with a decimal point + test.equal(str('1,,2'), '12'); // Multiple consecutive commas + test.equal(str('1,2,'), '12'); // Multiple commas with trailing + test.end(); +}); + tape('strings', (test) => { test.equal(str('"A" + "B"'), '"A" + "B"'); test.equal(str('"A"_"B"'), '"A"_"B"');