Skip to content

Commit ac5bdc4

Browse files
mattrobenoltayrton
mattrobenolt
and
ayrton
authored
Fix cast logic (#174)
* Fix cast logic There's a lot to unpack here, but the tl;dr is to refer to the charset ultimately to determine if the data is UTF8, if it is, we can decode it to a UTF8 string. This fixes behavior around CHAR/TEXT fields with a binary collation, being surfaces as BINARY/BLOB types by MySQL. For all intents and purposes, BLOB/BINARY/CHAR/TEXT are all effectively identical and interchangeable, the only differentiator is their charset. Either they are a UTF-8 charset, or a binary charset or some other charset. Fixes #169 --------- Co-authored-by: Ayrton <git@ayrton.be>
1 parent 3ec5d36 commit ac5bdc4

File tree

13 files changed

+1053
-99
lines changed

13 files changed

+1053
-99
lines changed

‎__tests__/cast.test.ts‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
import { cast } from '../src/cast'
2+
3+
describe('cast', () => {
4+
test('casts NULL values', () => {
5+
expect(
6+
cast(
7+
{
8+
name: 'email',
9+
type: 'VARCHAR'
10+
},
11+
null
12+
)
13+
).toEqual(null)
14+
})
15+
16+
test('casts INT64, UINT64 values', () => {
17+
expect(
18+
cast(
19+
{
20+
name: 'id',
21+
type: 'UINT64'
22+
},
23+
'1'
24+
)
25+
).toEqual('1')
26+
})
27+
28+
test('casts DATETIME, DATE, TIMESTAMP, TIME values', () => {
29+
expect(
30+
cast(
31+
{
32+
name: 'created_at',
33+
type: 'DATETIME'
34+
},
35+
'2024-01-01 00:00:00'
36+
)
37+
).toEqual('2024-01-01 00:00:00')
38+
})
39+
40+
test('casts DECIMAL values', () => {
41+
expect(
42+
cast(
43+
{
44+
name: 'decimal',
45+
type: 'DECIMAL'
46+
},
47+
'5.4'
48+
)
49+
).toEqual('5.4')
50+
})
51+
52+
test('casts JSON values', () => {
53+
expect(
54+
cast(
55+
{
56+
name: 'metadata',
57+
type: 'JSON'
58+
},
59+
'{ "color": "blue" }'
60+
)
61+
).toStrictEqual({ color: 'blue' })
62+
})
63+
64+
test('casts INT8, UINT8, INT16, UINT16, INT24, UINT24, INT32, UINT32, INT64, UINT64, YEAR values', () => {
65+
expect(
66+
cast(
67+
{
68+
name: 'verified',
69+
type: 'INT8'
70+
},
71+
'1'
72+
)
73+
).toEqual(1)
74+
expect(
75+
cast(
76+
{
77+
name: 'age',
78+
type: 'INT32'
79+
},
80+
'21'
81+
)
82+
).toEqual(21)
83+
})
84+
85+
test('casts FLOAT32, FLOAT64 values', () => {
86+
expect(
87+
cast(
88+
{
89+
name: 'float',
90+
type: 'FLOAT32'
91+
},
92+
'20.4'
93+
)
94+
).toEqual(20.4)
95+
expect(
96+
cast(
97+
{
98+
name: 'double',
99+
type: 'FLOAT64'
100+
},
101+
'101.4'
102+
)
103+
).toEqual(101.4)
104+
})
105+
106+
test('casts BLOB, BIT, GEOMETRY, BINARY, VARBINARY values', () => {
107+
/** See e2e tests in __tests__/golden.test.ts. */
108+
})
109+
110+
test('casts BINARY, VARBINARY string values', () => {
111+
/** See e2e tests in __tests__/golden.test.ts. */
112+
})
113+
114+
test('casts VARCHAR values', () => {
115+
expect(
116+
cast(
117+
{
118+
name: 'email',
119+
type: 'VARCHAR',
120+
charset: 255
121+
},
122+
'user@planetscale.com'
123+
)
124+
).toEqual('user@planetscale.com')
125+
})
126+
})

‎__tests__/golden.test.ts‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
import { connect } from '../dist/index'
2+
import { fetch, MockAgent, setGlobalDispatcher } from 'undici'
3+
4+
import database from '../golden/database.json'
5+
import dual from '../golden/dual.json'
6+
7+
const mockHosts = ['http://localhost:8080', 'https://example.com']
8+
const EXECUTE_PATH = '/psdb.v1alpha1.Database/Execute'
9+
10+
const mockAgent = new MockAgent()
11+
mockAgent.disableNetConnect()
12+
13+
setGlobalDispatcher(mockAgent)
14+
15+
const config = {
16+
username: 'someuser',
17+
password: 'password',
18+
host: 'example.com',
19+
fetch
20+
}
21+
22+
// Provide the base url to the request
23+
const mockPool = mockAgent.get((value) => mockHosts.includes(value))
24+
const mockSession = 42
25+
26+
describe('golden', () => {
27+
test('runs e2e database tests', async () => {
28+
const mockResponse = {
29+
session: mockSession,
30+
result: database,
31+
timing: 1
32+
}
33+
34+
const want = {
35+
id: '1',
36+
a: 1,
37+
b: 1,
38+
c: 1,
39+
d: 1,
40+
e: '1',
41+
f: '1.1',
42+
g: '1.1',
43+
h: 1.1,
44+
i: 1.1,
45+
j: uint8ArrayFromHex('0x07'),
46+
k: '1000-01-01',
47+
l: '1000-01-01 01:01:01',
48+
m: '1970-01-01 00:01:01',
49+
n: '01:01:01',
50+
o: 2006,
51+
p: 'p',
52+
q: 'q',
53+
r: uint8ArrayFromHex('0x72000000'),
54+
s: uint8ArrayFromHex('0x73'),
55+
t: uint8ArrayFromHex('0x74'),
56+
u: uint8ArrayFromHex('0x75'),
57+
v: uint8ArrayFromHex('0x76'),
58+
w: uint8ArrayFromHex('0x77'),
59+
x: 'x',
60+
y: 'y',
61+
z: 'z',
62+
aa: 'aa',
63+
ab: 'foo',
64+
ac: 'foo,bar',
65+
ad: { ad: null },
66+
ae: uint8ArrayFromHex(
67+
'0x0000000001020000000300000000000000000000000000000000000000000000000000F03F000000000000F03F00000000000000400000000000000000'
68+
),
69+
af: uint8ArrayFromHex('0x000000000101000000000000000000F03F000000000000F03F'),
70+
ag: uint8ArrayFromHex(
71+
'0x0000000001020000000300000000000000000000000000000000000000000000000000F03F000000000000F03F00000000000000400000000000000000'
72+
),
73+
ah: uint8ArrayFromHex(
74+
'0x00000000010300000002000000040000000000000000000000000000000000000000000000000000000000000000000840000000000000084000000000000000000000000000000000000000000000000004000000000000000000F03F000000000000F03F000000000000F03F00000000000000400000000000000040000000000000F03F000000000000F03F000000000000F03F'
75+
),
76+
ai: 1,
77+
aj: 1,
78+
ak: 1,
79+
al: '1',
80+
xa: 'xa',
81+
xb: 'xb',
82+
xc: uint8ArrayFromHex('0x78630000'),
83+
xd: 'xd',
84+
NULL: null
85+
}
86+
87+
mockPool.intercept({ path: EXECUTE_PATH, method: 'POST' }).reply(200, (opts: any) => {
88+
expect(opts.headers['Authorization']).toMatch(/Basic /)
89+
const bodyObj = JSON.parse(opts.body.toString())
90+
expect(bodyObj.session).toEqual(null)
91+
return mockResponse
92+
})
93+
94+
const connection = connect(config)
95+
const got = await connection.execute('xxx')
96+
97+
expect(got.rows[0]).toEqual(want)
98+
})
99+
100+
test('runs e2e dual tests', async () => {
101+
const mockResponse = {
102+
session: mockSession,
103+
result: dual,
104+
timing: 1
105+
}
106+
107+
const want = {
108+
a: 'ÿ'
109+
}
110+
111+
mockPool.intercept({ path: EXECUTE_PATH, method: 'POST' }).reply(200, (opts: any) => {
112+
expect(opts.headers['Authorization']).toMatch(/Basic /)
113+
const bodyObj = JSON.parse(opts.body.toString())
114+
expect(bodyObj.session).toEqual(null)
115+
return mockResponse
116+
})
117+
118+
const connection = connect(config)
119+
const got = await connection.execute('xxx')
120+
121+
expect(got.rows[0]).toEqual(want)
122+
})
123+
})
124+
125+
function uint8ArrayFromHex(text: string) {
126+
if (text.startsWith('0x')) {
127+
text = text.slice(2)
128+
}
129+
return Uint8Array.from((text.match(/.{1,2}/g) ?? []).map((byte) => parseInt(byte, 16)))
130+
}

‎__tests__/index.test.ts‎

Lines changed: 7 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,24 @@
11
import SqlString from 'sqlstring'
2-
import { cast, connect, format, hex, DatabaseError, type Cast } from '../dist/index'
2+
import { connect, format, hex, DatabaseError, type Cast } from '../dist/index'
33
import { fetch, MockAgent, setGlobalDispatcher } from 'undici'
44
import packageJSON from '../package.json'
55

66
const mockHosts = ['http://localhost:8080', 'https://example.com']
77
const CREATE_SESSION_PATH = '/psdb.v1alpha1.Database/CreateSession'
88
const EXECUTE_PATH = '/psdb.v1alpha1.Database/Execute'
9+
10+
const mockAgent = new MockAgent()
11+
mockAgent.disableNetConnect()
12+
13+
setGlobalDispatcher(mockAgent)
14+
915
const config = {
1016
username: 'someuser',
1117
password: 'password',
1218
host: 'example.com',
1319
fetch
1420
}
1521

16-
const mockAgent = new MockAgent()
17-
mockAgent.disableNetConnect()
18-
19-
setGlobalDispatcher(mockAgent)
20-
2122
// Provide the base url to the request
2223
const mockPool = mockAgent.get((value) => mockHosts.includes(value))
2324
const mockSession = 42
@@ -615,28 +616,3 @@ describe('hex', () => {
615616
expect(hex('\0')).toEqual('0x00')
616617
})
617618
})
618-
619-
describe('cast', () => {
620-
test('casts int to number', () => {
621-
expect(cast({ name: 'test', type: 'INT8' }, '12')).toEqual(12)
622-
})
623-
624-
test('casts float to number', () => {
625-
expect(cast({ name: 'test', type: 'FLOAT32' }, '2.32')).toEqual(2.32)
626-
expect(cast({ name: 'test', type: 'FLOAT64' }, '2.32')).toEqual(2.32)
627-
})
628-
629-
test('casts binary data to array of 8-bit unsigned integers', () => {
630-
expect(cast({ name: 'test', type: 'BLOB' }, '')).toEqual(new Uint8Array([]))
631-
expect(cast({ name: 'test', type: 'BLOB' }, 'Å')).toEqual(new Uint8Array([197]))
632-
expect(cast({ name: 'test', type: 'VARBINARY' }, 'Å')).toEqual(new Uint8Array([197]))
633-
})
634-
635-
test('casts binary text data to text', () => {
636-
expect(cast({ name: 'test', type: 'VARBINARY', flags: 4225 }, 'table')).toEqual('table')
637-
})
638-
639-
test('casts JSON string to JSON object', () => {
640-
expect(cast({ name: 'test', type: 'JSON' }, '{ "foo": "bar" }')).toStrictEqual({ foo: 'bar' })
641-
})
642-
})

‎__tests__/text.test.ts‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,25 @@
1-
import { decode, hex, uint8Array, uint8ArrayToHex } from '../src/text'
1+
import { decodeUtf8, hex, uint8Array, uint8ArrayToHex } from '../src/text'
22

33
describe('text', () => {
4-
describe('decode', () => {
4+
describe('decodeUtf8', () => {
55
test('decodes ascii bytes', () => {
6-
expect(decode('a')).toEqual('a')
6+
expect(decodeUtf8('a')).toEqual('a')
77
})
88

99
test('decodes empty string', () => {
10-
expect(decode('')).toEqual('')
10+
expect(decodeUtf8('')).toEqual('')
1111
})
1212

1313
test('decodes null value', () => {
14-
expect(decode(null)).toEqual('')
14+
expect(decodeUtf8(null)).toEqual('')
1515
})
1616

1717
test('decodes undefined value', () => {
18-
expect(decode(undefined)).toEqual('')
18+
expect(decodeUtf8(undefined)).toEqual('')
1919
})
2020

2121
test('decodes multi-byte characters', () => {
22-
expect(decode('\xF0\x9F\xA4\x94')).toEqual('🤔')
22+
expect(decodeUtf8('\xF0\x9F\xA4\x94')).toEqual('🤔')
2323
})
2424
})
2525

‎golden/README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Golden tests
2+
3+
This generates a "golden" test result that feeds the "parse e2e" test suite.
4+
5+
The intent is a full round trip with a known table that exercises every column type with known correct data.
6+
7+
This excercises different collations, charsets, every integer type.
8+
9+
`test.sql` acts as the seed data against a PlanetScale branch, then we fetch the data back with `curl`.
10+
11+
We can run different queries inside the `generate.sh` script either against the current data, or tests that run against `dual`.
12+
13+
The results are stored back in `$case.json` and we directly import these JSON files into the test suite.
14+
15+
Along with this is a `cli.txt` which is the result of running `select * from test` in a mysql CLI dumping the full human readable table. This table is a good reference for what is expected to be human readable or not. Raw binary data is represented as hexadecimal, vs UTF8 strings are readable.

‎golden/cli.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
+----+------+------+------+------+------+------+------+------+------+------------+------------+---------------------+---------------------+----------+------+------+------+------------+------------+------------+------------+------------+------------+------+------+------+------+------+---------+--------------+------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------+------+------+------+------+------+------------+------+
2+
| id | a | b | c | d | e | f | g | h | i | j | k | l | m | n | o | p | q | r | s | t | u | v | w | x | y | z | aa | ab | ac | ad | ae | af | ag | ah | ai | aj | ak | al | xa | xb | xc | xd |
3+
+----+------+------+------+------+------+------+------+------+------+------------+------------+---------------------+---------------------+----------+------+------+------+------------+------------+------------+------------+------------+------------+------+------+------+------+------+---------+--------------+------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------+------+------+------+------+------+------------+------+
4+
| 1 | 1 | 1 | 1 | 1 | 1 | 1.1 | 1.1 | 1.1 | 1.1 | 0x07 | 1000-01-01 | 1000-01-01 01:01:01 | 1970-01-01 00:01:01 | 01:01:01 | 2006 | p | q | 0x72000000 | 0x73 | 0x74 | 0x75 | 0x76 | 0x77 | x | y | z | aa | foo | foo,bar | {"ad": null} | 0x0000000001020000000300000000000000000000000000000000000000000000000000F03F000000000000F03F00000000000000400000000000000000 | 0x000000000101000000000000000000F03F000000000000F03F | 0x0000000001020000000300000000000000000000000000000000000000000000000000F03F000000000000F03F00000000000000400000000000000000 | 0x00000000010300000002000000040000000000000000000000000000000000000000000000000000000000000000000840000000000000084000000000000000000000000000000000000000000000000004000000000000000000F03F000000000000F03F000000000000F03F00000000000000400000000000000040000000000000F03F000000000000F03F000000000000F03F | 1 | 1 | 1 | 1 | xa | xb | 0x78630000 | xd |
5+
+----+------+------+------+------+------+------+------+------+------+------------+------------+---------------------+---------------------+----------+------+------+------+------------+------------+------------+------------+------------+------------+------+------+------+------+------+---------+--------------+------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------+------+------+------+------+------+------------+------+

0 commit comments

Comments
 (0)