From 9d90f16d476183ef5a3c67bddfbe9ba88b04692b Mon Sep 17 00:00:00 2001 From: Diego Miguel Date: Mon, 20 Oct 2025 17:05:37 -0300 Subject: [PATCH 1/5] fix: Fixed brazilian accent parsing when table have latin1 charset --- lib/common.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/common.js b/lib/common.js index 6b4dbe2..ad472f6 100644 --- a/lib/common.js +++ b/lib/common.js @@ -443,6 +443,12 @@ exports.readMysqlValue = function ( } else if (defPrefix === 'varbin') { result = parser.parseBuffer(size); } else { + if (column.charset !== null) { + // Javascript UTF8 always allows up to 4 bytes per character + column.charset = column.charset === 'utf8mb4' ? 'utf8' : column.charset; + parser._encoding = column.charset; + } + result = parser.parseString(size); } break; From c52aca47029f423fe7b83cc4cd9c8000cff0eb78 Mon Sep 17 00:00:00 2001 From: Diego Miguel Date: Tue, 21 Oct 2025 10:40:41 -0300 Subject: [PATCH 2/5] : fix: Fixed verification of string charset --- lib/common.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/common.js b/lib/common.js index ad472f6..6ca1108 100644 --- a/lib/common.js +++ b/lib/common.js @@ -444,9 +444,14 @@ exports.readMysqlValue = function ( result = parser.parseBuffer(size); } else { if (column.charset !== null) { - // Javascript UTF8 always allows up to 4 bytes per character - column.charset = column.charset === 'utf8mb4' ? 'utf8' : column.charset; - parser._encoding = column.charset; + switch (column.charset) { + // Javascript UTF8 always allows up to 4 bytes per character + case 'utf8mb3': + case 'utf8mb4': + column.charset = 'utf8'; + parser._encoding = column.charset; + break; + } } result = parser.parseString(size); From 4874387a83394e812e24cf504f66473d64468062 Mon Sep 17 00:00:00 2001 From: Diego Miguel Date: Tue, 21 Oct 2025 10:50:45 -0300 Subject: [PATCH 3/5] fix: Fixed position to change parser encoding --- lib/common.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/common.js b/lib/common.js index 6ca1108..f8bf0de 100644 --- a/lib/common.js +++ b/lib/common.js @@ -449,9 +449,10 @@ exports.readMysqlValue = function ( case 'utf8mb3': case 'utf8mb4': column.charset = 'utf8'; - parser._encoding = column.charset; break; } + + parser._encoding = column.charset; } result = parser.parseString(size); From 12db0a51995f9c2c5814cf196ba78b04d891907b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 28 Oct 2025 16:23:25 +0100 Subject: [PATCH 4/5] Add tests --- test/types.js | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/test/types.js b/test/types.js index 8f18323..599d5d9 100644 --- a/test/types.js +++ b/test/types.js @@ -68,32 +68,24 @@ function defineTypeTest(name, fields, testRows, customTest) { } }; - expectEvents( - test, - eventLog, - [ - expectedWrite - ], - testRows.length, - () => { - test.equal(errorLog.length, 0); + expectEvents(test, eventLog, [expectedWrite], testRows.length, () => { + test.equal(errorLog.length, 0); - const binlogRows = eventLog.reduce((prev, curr) => { - if (curr.getTypeName() === 'WriteRows') { - prev = prev.concat(curr.rows); - } - return prev; - }, []); - - if (customTest) { - customTest.bind(selectResult)(test, { rows: binlogRows }); - } else { - assert.deepEqual(selectResult, binlogRows); + const binlogRows = eventLog.reduce((prev, curr) => { + if (curr.getTypeName() === 'WriteRows') { + prev = prev.concat(curr.rows); } + return prev; + }, []); - test.end(); + if (customTest) { + customTest.bind(selectResult)(test, { rows: binlogRows }); + } else { + assert.deepEqual(selectResult, binlogRows); } - ); + + test.end(); + }); }); }); }); @@ -293,3 +285,12 @@ defineTypeTest( [null, null, null, null] ] ); + +defineTypeTest( + 'charsets', + ['CHAR COLLATE latin1_general_ci', 'VARCHAR(15) COLLATE latin1_general_ci', 'TEXT COLLATE latin1_general_ci'], + [ + ["'ascii only'", "'ascii only'", "'ascii only'"], + ["'João'", "'João'", "'João'"] + ] +); From ec8ffb9a2976266d90cf6713221c49397ca1c373 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 28 Oct 2025 16:24:45 +0100 Subject: [PATCH 5/5] Add changeset entry --- .changeset/big-plants-joke.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/big-plants-joke.md diff --git a/.changeset/big-plants-joke.md b/.changeset/big-plants-joke.md new file mode 100644 index 0000000..b434e65 --- /dev/null +++ b/.changeset/big-plants-joke.md @@ -0,0 +1,5 @@ +--- +'@powersync/mysql-zongji': patch +--- + +Fix decoding varchar and string values with utf8mb3 and utf8mb4 charsets.