Skip to content

Commit

Permalink
Fix cast logic
Browse files Browse the repository at this point in the history 
There's a lot to unpack here, but the tl;dr is to refer to the charset
ultimately to determine if the data is UTF8, if it is, we can decode it
to a UTF8 string.

This fixes behavior around CHAR/TEXT fields with a binary collation,
being surfaces as BINARY/BLOB types by MySQL.

For all intents and purposes, BLOB/BINARY/CHAR/TEXT are all effectively
identical and interchangeable, the only differentiator is their charset.
Either they are a UTF-8 charset, or a binary charset or some other
charset.

Fixes #169
  • Loading branch information
@mattrobenolt
mattrobenolt committed Apr 25, 2024
1 parent 3ec5d36 commit baa6bad
Show file tree
Hide file tree
Showing 8 changed files with 645 additions and 30 deletions.
85 changes: 84 additions & 1 deletion __tests__/index.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,13 @@ const config = {
fetch
}

function uint8ArrayFromHex(text: string): Uint8Array {
if (text.startsWith('0x')) {
text = text.slice(2)
}
return Uint8Array.from((text.match(/.{1,2}/g) ?? []).map((byte) => parseInt(byte, 16)))
}

const mockAgent = new MockAgent()
mockAgent.disableNetConnect()

Expand Down Expand Up @@ -633,10 +640,86 @@ describe('cast', () => {
})

test('casts binary text data to text', () => {
expect(cast({ name: 'test', type: 'VARBINARY', flags: 4225 }, 'table')).toEqual('table')
expect(cast({ name: 'test', type: 'VARBINARY', charset: 255 }, 'table')).toEqual('table')
})

test('casts JSON string to JSON object', () => {
expect(cast({ name: 'test', type: 'JSON' }, '{ "foo": "bar" }')).toStrictEqual({ foo: 'bar' })
})
})

describe('parse e2e', () => {
test('golden test', async () => {
const mockResponse = {
session: mockSession,
result: JSON.parse(
'{"fields":[{"name":"id","type":"INT64","table":"test","orgTable":"test","database":"mattdb","orgName":"id","columnLength":20,"charset":63,"flags":49667},{"name":"a","type":"INT8","table":"test","orgTable":"test","database":"mattdb","orgName":"a","columnLength":4,"charset":63,"flags":32768},{"name":"b","type":"INT16","table":"test","orgTable":"test","database":"mattdb","orgName":"b","columnLength":6,"charset":63,"flags":32768},{"name":"c","type":"INT24","table":"test","orgTable":"test","database":"mattdb","orgName":"c","columnLength":9,"charset":63,"flags":32768},{"name":"d","type":"INT32","table":"test","orgTable":"test","database":"mattdb","orgName":"d","columnLength":11,"charset":63,"flags":32768},{"name":"e","type":"INT64","table":"test","orgTable":"test","database":"mattdb","orgName":"e","columnLength":20,"charset":63,"flags":32768},{"name":"f","type":"DECIMAL","table":"test","orgTable":"test","database":"mattdb","orgName":"f","columnLength":4,"charset":63,"decimals":1,"flags":32768},{"name":"g","type":"DECIMAL","table":"test","orgTable":"test","database":"mattdb","orgName":"g","columnLength":4,"charset":63,"decimals":1,"flags":32768},{"name":"h","type":"FLOAT32","table":"test","orgTable":"test","database":"mattdb","orgName":"h","columnLength":12,"charset":63,"decimals":31,"flags":32768},{"name":"i","type":"FLOAT64","table":"test","orgTable":"test","database":"mattdb","orgName":"i","columnLength":22,"charset":63,"decimals":31,"flags":32768},{"name":"j","type":"BIT","table":"test","orgTable":"test","database":"mattdb","orgName":"j","columnLength":3,"charset":63,"flags":32},{"name":"k","type":"DATE","table":"test","orgTable":"test","database":"mattdb","orgName":"k","columnLength":10,"charset":63,"flags":128},{"name":"l","type":"DATETIME","table":"test","orgTable":"test","database":"mattdb","orgName":"l","columnLength":19,"charset":63,"flags":128},{"name":"m","type":"TIMESTAMP","table":"test","orgTable":"test","database":"mattdb","orgName":"m","columnLength":19,"charset":63,"flags":128},{"name":"n","type":"TIME","table":"test","orgTable":"test","database":"mattdb","orgName":"n","columnLength":10,"charset":63,"flags":128},{"name":"o","type":"YEAR","table":"test","orgTable":"test","database":"mattdb","orgName":"o","columnLength":4,"charset":63,"flags":32864},{"name":"p","type":"CHAR","table":"test","orgTable":"test","database":"mattdb","orgName":"p","columnLength":16,"charset":255},{"name":"q","type":"VARCHAR","table":"test","orgTable":"test","database":"mattdb","orgName":"q","columnLength":16,"charset":255},{"name":"r","type":"BINARY","table":"test","orgTable":"test","database":"mattdb","orgName":"r","columnLength":4,"charset":63,"flags":128},{"name":"s","type":"VARBINARY","table":"test","orgTable":"test","database":"mattdb","orgName":"s","columnLength":4,"charset":63,"flags":128},{"name":"t","type":"BLOB","table":"test","orgTable":"test","database":"mattdb","orgName":"t","columnLength":255,"charset":63,"flags":144},{"name":"u","type":"BLOB","table":"test","orgTable":"test","database":"mattdb","orgName":"u","columnLength":65535,"charset":63,"flags":144},{"name":"v","type":"BLOB","table":"test","orgTable":"test","database":"mattdb","orgName":"v","columnLength":16777215,"charset":63,"flags":144},{"name":"w","type":"BLOB","table":"test","orgTable":"test","database":"mattdb","orgName":"w","columnLength":4294967295,"charset":63,"flags":144},{"name":"x","type":"TEXT","table":"test","orgTable":"test","database":"mattdb","orgName":"x","columnLength":1020,"charset":255,"flags":16},{"name":"y","type":"TEXT","table":"test","orgTable":"test","database":"mattdb","orgName":"y","columnLength":262140,"charset":255,"flags":16},{"name":"z","type":"TEXT","table":"test","orgTable":"test","database":"mattdb","orgName":"z","columnLength":67108860,"charset":255,"flags":16},{"name":"aa","type":"TEXT","table":"test","orgTable":"test","database":"mattdb","orgName":"aa","columnLength":4294967295,"charset":255,"flags":16},{"name":"ab","type":"ENUM","table":"test","orgTable":"test","database":"mattdb","orgName":"ab","columnLength":12,"charset":255,"flags":256},{"name":"ac","type":"SET","table":"test","orgTable":"test","database":"mattdb","orgName":"ac","columnLength":28,"charset":255,"flags":2048},{"name":"ad","type":"JSON","table":"test","orgTable":"test","database":"mattdb","orgName":"ad","columnLength":4294967295,"charset":63,"flags":144},{"name":"ae","type":"GEOMETRY","table":"test","orgTable":"test","database":"mattdb","orgName":"ae","columnLength":4294967295,"charset":63,"flags":144},{"name":"af","type":"GEOMETRY","table":"test","orgTable":"test","database":"mattdb","orgName":"af","columnLength":4294967295,"charset":63,"flags":144},{"name":"ag","type":"GEOMETRY","table":"test","orgTable":"test","database":"mattdb","orgName":"ag","columnLength":4294967295,"charset":63,"flags":144},{"name":"ah","type":"GEOMETRY","table":"test","orgTable":"test","database":"mattdb","orgName":"ah","columnLength":4294967295,"charset":63,"flags":144},{"name":"ai","type":"UINT8","table":"test","orgTable":"test","database":"mattdb","orgName":"ai","columnLength":3,"charset":63,"flags":32800},{"name":"aj","type":"UINT24","table":"test","orgTable":"test","database":"mattdb","orgName":"aj","columnLength":8,"charset":63,"flags":32800},{"name":"ak","type":"UINT32","table":"test","orgTable":"test","database":"mattdb","orgName":"ak","columnLength":10,"charset":63,"flags":32800},{"name":"al","type":"UINT64","table":"test","orgTable":"test","database":"mattdb","orgName":"al","columnLength":20,"charset":63,"flags":32800},{"name":"xa","type":"BINARY","table":"test","orgTable":"test","database":"mattdb","orgName":"xa","columnLength":16,"charset":255,"flags":128},{"name":"xb","type":"BINARY","table":"test","orgTable":"test","database":"mattdb","orgName":"xb","columnLength":16,"charset":255,"flags":128},{"name":"xc","type":"BINARY","table":"test","orgTable":"test","database":"mattdb","orgName":"xc","columnLength":4,"charset":63,"flags":128},{"name":"xd","type":"BLOB","table":"test","orgTable":"test","database":"mattdb","orgName":"xd","columnLength":262140,"charset":255,"flags":144}],"rows":[{"lengths":["1","1","1","1","1","1","3","3","3","3","1","10","19","19","8","4","1","1","4","1","1","1","1","1","1","1","1","2","3","7","12","61","25","61","149","1","1","1","1","2","2","4","2"],"values":"MTExMTExMS4xMS4xMS4xMS4xBzEwMDAtMDEtMDExMDAwLTAxLTAxIDAxOjAxOjAxMTk3MC0wMS0wMSAwMDowMTowMTAxOjAxOjAxMjAwNnBxcgAAAHN0dXZ3eHl6YWFmb29mb28sYmFyeyJhZCI6IG51bGx9AAAAAAECAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAAAAAAAABAQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAQIAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAAAAAAAAAAEDAAAAAgAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIQAAAAAAAAAhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8xMTExeGF4YnhjAAB4ZA=="}]}'
),
timing: 1
}

const want = {
id: '1',
a: 1,
b: 1,
c: 1,
d: 1,
e: '1',
f: '1.1',
g: '1.1',
h: 1.1,
i: 1.1,
j: uint8ArrayFromHex('0x07'),
k: '1000-01-01',
l: '1000-01-01 01:01:01',
m: '1970-01-01 00:01:01',
n: '01:01:01',
o: 2006,
p: 'p',
q: 'q',
r: uint8ArrayFromHex('0x72000000'),
s: uint8ArrayFromHex('0x73'),
t: uint8ArrayFromHex('0x74'),
u: uint8ArrayFromHex('0x75'),
v: uint8ArrayFromHex('0x76'),
w: uint8ArrayFromHex('0x77'),
x: 'x',
y: 'y',
z: 'z',
aa: 'aa',
ab: 'foo',
ac: 'foo,bar',
ad: { ad: null },
ae: uint8ArrayFromHex(
'0x0000000001020000000300000000000000000000000000000000000000000000000000F03F000000000000F03F00000000000000400000000000000000'
),
af: uint8ArrayFromHex('0x000000000101000000000000000000F03F000000000000F03F'),
ag: uint8ArrayFromHex(
'0x0000000001020000000300000000000000000000000000000000000000000000000000F03F000000000000F03F00000000000000400000000000000000'
),
ah: uint8ArrayFromHex(
'0x00000000010300000002000000040000000000000000000000000000000000000000000000000000000000000000000840000000000000084000000000000000000000000000000000000000000000000004000000000000000000F03F000000000000F03F000000000000F03F00000000000000400000000000000040000000000000F03F000000000000F03F000000000000F03F'
),
ai: 1,
aj: 1,
ak: 1,
al: '1',
xa: 'xa',
xb: 'xb',
xc: uint8ArrayFromHex('0x78630000'),
xd: 'xd'
}

mockPool.intercept({ path: EXECUTE_PATH, method: 'POST' }).reply(200, (opts: any) => {
expect(opts.headers['Authorization']).toMatch(/Basic /)
const bodyObj = JSON.parse(opts.body.toString())
expect(bodyObj.session).toEqual(null)
return mockResponse
})

const connection = connect(config)
const got = await connection.execute('SELECT * from `test`')

expect(got.rows[0]).toEqual(want)
})
})
12 changes: 6 additions & 6 deletions __tests__/text.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,25 @@
import { decode, hex, uint8Array, uint8ArrayToHex } from '../src/text'
import { decodeUTF8, hex, uint8Array, uint8ArrayToHex } from '../src/text'

describe('text', () => {
describe('decode', () => {
test('decodes ascii bytes', () => {
expect(decode('a')).toEqual('a')
expect(decodeUTF8('a')).toEqual('a')
})

test('decodes empty string', () => {
expect(decode('')).toEqual('')
expect(decodeUTF8('')).toEqual('')
})

test('decodes null value', () => {
expect(decode(null)).toEqual('')
expect(decodeUTF8(null)).toEqual('')
})

test('decodes undefined value', () => {
expect(decode(undefined)).toEqual('')
expect(decodeUTF8(undefined)).toEqual('')
})

test('decodes multi-byte characters', () => {
expect(decode('\xF0\x9F\xA4\x94')).toEqual('🤔')
expect(decodeUTF8('\xF0\x9F\xA4\x94')).toEqual('🤔')
})
})

Expand Down
13 changes: 13 additions & 0 deletions golden/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Golden tests

This generates a "golden" test result that feeds the "parse e2e golden test".

The intent is a full round trip with a known table that exercises every column type with known correct data.

This excercises different collations, charsets, every integer type.

`test.sql` acts as the seed data against a PlanetScale branch, then we fetch the data back with `curl`.

The result is stored in `results.json` while a compact version is stored in `results-compact.json`. This compact version is what is shoved into the mock test result for convenience.

Along with this is a `cli.txt` which is the result of running `select * from test` in a mysql CLI dumping the full human readable table. This table is a good reference for what is expected to be human readable or not. Raw binary data is represented as hexadecimal, vs UTF8 strings are readable.
5 changes: 5 additions & 0 deletions golden/generate.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
+----+------+------+------+------+------+------+------+------+------+------------+------------+---------------------+---------------------+----------+------+------+------+------------+------------+------------+------------+------------+------------+------+------+------+------+------+---------+--------------+------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------+------+------------+------+
| id | a | b | c | d | e | f | g | h | i | j | k | l | m | n | o | p | q | r | s | t | u | v | w | x | y | z | aa | ab | ac | ad | ae | af | ag | ah | xa | xb | xc | xd |
+----+------+------+------+------+------+------+------+------+------+------------+------------+---------------------+---------------------+----------+------+------+------+------------+------------+------------+------------+------------+------------+------+------+------+------+------+---------+--------------+------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------+------+------------+------+
| 1 | 1 | 1 | 1 | 1 | 1 | 1.1 | 1.1 | 1.1 | 1.1 | 0x07 | 1000-01-01 | 1000-01-01 01:01:01 | 1970-01-01 00:01:01 | 01:01:01 | 2006 | p | q | 0x72000000 | 0x73 | 0x74 | 0x75 | 0x76 | 0x77 | x | y | z | aa | foo | foo,bar | {"ad": null} | 0x0000000001020000000300000000000000000000000000000000000000000000000000F03F000000000000F03F00000000000000400000000000000000 | 0x000000000101000000000000000000F03F000000000000F03F | 0x0000000001020000000300000000000000000000000000000000000000000000000000F03F000000000000F03F00000000000000400000000000000000 | 0x00000000010300000002000000040000000000000000000000000000000000000000000000000000000000000000000840000000000000084000000000000000000000000000000000000000000000000004000000000000000000F03F000000000000F03F000000000000F03F00000000000000400000000000000040000000000000F03F000000000000F03F000000000000F03F | xa | xb | 0x78630000 | xd |
+----+------+------+------+------+------+------+------+------+------+------------+------------+---------------------+---------------------+----------+------+------+------+------------+------------+------------+------------+------------+------------+------+------+------+------+------+---------+--------------+------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------+------+------------+------+
Loading

0 comments on commit baa6bad

Please sign in to comment.