Fix cast logic

There's a lot to unpack here, but the tl;dr is to refer to the charset
ultimately to determine if the data is UTF8, if it is, we can decode it
to a UTF8 string.

This fixes behavior around CHAR/TEXT fields with a binary collation,
being surfaces as BINARY/BLOB types by MySQL.

For all intents and purposes, BLOB/BINARY/CHAR/TEXT are all effectively
identical and interchangeable, the only differentiator is their charset.
Either they are a UTF-8 charset, or a binary charset or some other
charset.

Fixes #169

__tests__/index.test.ts

@@ -13,6 +13,13 @@ const config = {
   fetch
 }
 
+function uint8ArrayFromHex(text: string): Uint8Array {
+  if (text.startsWith('0x')) {
+    text = text.slice(2)
+  }
+  return Uint8Array.from((text.match(/.{1,2}/g) ?? []).map((byte) => parseInt(byte, 16)))
+}
+
 const mockAgent = new MockAgent()
 mockAgent.disableNetConnect()
 
@@ -627,16 +634,119 @@ describe('cast', () => {
   })
 
   test('casts binary data to array of 8-bit unsigned integers', () => {
-    expect(cast({ name: 'test', type: 'BLOB' }, '')).toEqual(new Uint8Array([]))
-    expect(cast({ name: 'test', type: 'BLOB' }, 'Å')).toEqual(new Uint8Array([197]))
-    expect(cast({ name: 'test', type: 'VARBINARY' }, 'Å')).toEqual(new Uint8Array([197]))
+    expect(cast({ name: 'test', type: 'BLOB', charset: 63 }, '')).toEqual(new Uint8Array([]))
+    expect(cast({ name: 'test', type: 'BLOB', charset: 63 }, 'Å')).toEqual(new Uint8Array([197]))
+    expect(cast({ name: 'test', type: 'VARBINARY', charset: 63 }, 'Å')).toEqual(new Uint8Array([197]))
   })
 
   test('casts binary text data to text', () => {
-    expect(cast({ name: 'test', type: 'VARBINARY', flags: 4225 }, 'table')).toEqual('table')
+    expect(cast({ name: 'test', type: 'VARBINARY', charset: 255 }, 'table')).toEqual('table')
   })
 
   test('casts JSON string to JSON object', () => {
     expect(cast({ name: 'test', type: 'JSON' }, '{ "foo": "bar" }')).toStrictEqual({ foo: 'bar' })
   })
 })
+
+describe('parse e2e', () => {
+  test('golden (testdb.json)', async () => {
+    const mockResponse = {
+      session: mockSession,
+      result: JSON.parse(
+        '{"fields":[{"name":"id","type":"INT64","table":"test","orgTable":"test","database":"mattdb","orgName":"id","columnLength":20,"charset":63,"flags":49667},{"name":"a","type":"INT8","table":"test","orgTable":"test","database":"mattdb","orgName":"a","columnLength":4,"charset":63,"flags":32768},{"name":"b","type":"INT16","table":"test","orgTable":"test","database":"mattdb","orgName":"b","columnLength":6,"charset":63,"flags":32768},{"name":"c","type":"INT24","table":"test","orgTable":"test","database":"mattdb","orgName":"c","columnLength":9,"charset":63,"flags":32768},{"name":"d","type":"INT32","table":"test","orgTable":"test","database":"mattdb","orgName":"d","columnLength":11,"charset":63,"flags":32768},{"name":"e","type":"INT64","table":"test","orgTable":"test","database":"mattdb","orgName":"e","columnLength":20,"charset":63,"flags":32768},{"name":"f","type":"DECIMAL","table":"test","orgTable":"test","database":"mattdb","orgName":"f","columnLength":4,"charset":63,"decimals":1,"flags":32768},{"name":"g","type":"DECIMAL","table":"test","orgTable":"test","database":"mattdb","orgName":"g","columnLength":4,"charset":63,"decimals":1,"flags":32768},{"name":"h","type":"FLOAT32","table":"test","orgTable":"test","database":"mattdb","orgName":"h","columnLength":12,"charset":63,"decimals":31,"flags":32768},{"name":"i","type":"FLOAT64","table":"test","orgTable":"test","database":"mattdb","orgName":"i","columnLength":22,"charset":63,"decimals":31,"flags":32768},{"name":"j","type":"BIT","table":"test","orgTable":"test","database":"mattdb","orgName":"j","columnLength":3,"charset":63,"flags":32},{"name":"k","type":"DATE","table":"test","orgTable":"test","database":"mattdb","orgName":"k","columnLength":10,"charset":63,"flags":128},{"name":"l","type":"DATETIME","table":"test","orgTable":"test","database":"mattdb","orgName":"l","columnLength":19,"charset":63,"flags":128},{"name":"m","type":"TIMESTAMP","table":"test","orgTable":"test","database":"mattdb","orgName":"m","columnLength":19,"charset":63,"flags":128},{"name":"n","type":"TIME","table":"test","orgTable":"test","database":"mattdb","orgName":"n","columnLength":10,"charset":63,"flags":128},{"name":"o","type":"YEAR","table":"test","orgTable":"test","database":"mattdb","orgName":"o","columnLength":4,"charset":63,"flags":32864},{"name":"p","type":"CHAR","table":"test","orgTable":"test","database":"mattdb","orgName":"p","columnLength":16,"charset":255},{"name":"q","type":"VARCHAR","table":"test","orgTable":"test","database":"mattdb","orgName":"q","columnLength":16,"charset":255},{"name":"r","type":"BINARY","table":"test","orgTable":"test","database":"mattdb","orgName":"r","columnLength":4,"charset":63,"flags":128},{"name":"s","type":"VARBINARY","table":"test","orgTable":"test","database":"mattdb","orgName":"s","columnLength":4,"charset":63,"flags":128},{"name":"t","type":"BLOB","table":"test","orgTable":"test","database":"mattdb","orgName":"t","columnLength":255,"charset":63,"flags":144},{"name":"u","type":"BLOB","table":"test","orgTable":"test","database":"mattdb","orgName":"u","columnLength":65535,"charset":63,"flags":144},{"name":"v","type":"BLOB","table":"test","orgTable":"test","database":"mattdb","orgName":"v","columnLength":16777215,"charset":63,"flags":144},{"name":"w","type":"BLOB","table":"test","orgTable":"test","database":"mattdb","orgName":"w","columnLength":4294967295,"charset":63,"flags":144},{"name":"x","type":"TEXT","table":"test","orgTable":"test","database":"mattdb","orgName":"x","columnLength":1020,"charset":255,"flags":16},{"name":"y","type":"TEXT","table":"test","orgTable":"test","database":"mattdb","orgName":"y","columnLength":262140,"charset":255,"flags":16},{"name":"z","type":"TEXT","table":"test","orgTable":"test","database":"mattdb","orgName":"z","columnLength":67108860,"charset":255,"flags":16},{"name":"aa","type":"TEXT","table":"test","orgTable":"test","database":"mattdb","orgName":"aa","columnLength":4294967295,"charset":255,"flags":16},{"name":"ab","type":"ENUM","table":"test","orgTable":"test","database":"mattdb","orgName":"ab","columnLength":12,"charset":255,"flags":256},{"name":"ac","type":"SET","table":"test","orgTable":"test","database":"mattdb","orgName":"ac","columnLength":28,"charset":255,"flags":2048},{"name":"ad","type":"JSON","table":"test","orgTable":"test","database":"mattdb","orgName":"ad","columnLength":4294967295,"charset":63,"flags":144},{"name":"ae","type":"GEOMETRY","table":"test","orgTable":"test","database":"mattdb","orgName":"ae","columnLength":4294967295,"charset":63,"flags":144},{"name":"af","type":"GEOMETRY","table":"test","orgTable":"test","database":"mattdb","orgName":"af","columnLength":4294967295,"charset":63,"flags":144},{"name":"ag","type":"GEOMETRY","table":"test","orgTable":"test","database":"mattdb","orgName":"ag","columnLength":4294967295,"charset":63,"flags":144},{"name":"ah","type":"GEOMETRY","table":"test","orgTable":"test","database":"mattdb","orgName":"ah","columnLength":4294967295,"charset":63,"flags":144},{"name":"ai","type":"UINT8","table":"test","orgTable":"test","database":"mattdb","orgName":"ai","columnLength":3,"charset":63,"flags":32800},{"name":"aj","type":"UINT24","table":"test","orgTable":"test","database":"mattdb","orgName":"aj","columnLength":8,"charset":63,"flags":32800},{"name":"ak","type":"UINT32","table":"test","orgTable":"test","database":"mattdb","orgName":"ak","columnLength":10,"charset":63,"flags":32800},{"name":"al","type":"UINT64","table":"test","orgTable":"test","database":"mattdb","orgName":"al","columnLength":20,"charset":63,"flags":32800},{"name":"xa","type":"BINARY","table":"test","orgTable":"test","database":"mattdb","orgName":"xa","columnLength":16,"charset":255,"flags":128},{"name":"xb","type":"BINARY","table":"test","orgTable":"test","database":"mattdb","orgName":"xb","columnLength":16,"charset":255,"flags":128},{"name":"xc","type":"BINARY","table":"test","orgTable":"test","database":"mattdb","orgName":"xc","columnLength":4,"charset":63,"flags":128},{"name":"xd","type":"BLOB","table":"test","orgTable":"test","database":"mattdb","orgName":"xd","columnLength":262140,"charset":255,"flags":144},{"name":"NULL","charset":63,"flags":32896}],"rows":[{"lengths":["1","1","1","1","1","1","3","3","3","3","1","10","19","19","8","4","1","1","4","1","1","1","1","1","1","1","1","2","3","7","12","61","25","61","149","1","1","1","1","2","2","4","2","-1"],"values":"MTExMTExMS4xMS4xMS4xMS4xBzEwMDAtMDEtMDExMDAwLTAxLTAxIDAxOjAxOjAxMTk3MC0wMS0wMSAwMDowMTowMTAxOjAxOjAxMjAwNnBxcgAAAHN0dXZ3eHl6YWFmb29mb28sYmFyeyJhZCI6IG51bGx9AAAAAAECAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAAAEAAAAAAAAAAAAAAAAABAQAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAQIAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAAAAAAAAAAEDAAAAAgAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIQAAAAAAAAAhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAABAAAAAAAAAAEAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8xMTExeGF4YnhjAAB4ZA=="}]}',
+      ),
+      timing: 1
+    }
+
+    const want = {
+      id: '1',
+      a: 1,
+      b: 1,
+      c: 1,
+      d: 1,
+      e: '1',
+      f: '1.1',
+      g: '1.1',
+      h: 1.1,
+      i: 1.1,
+      j: uint8ArrayFromHex('0x07'),
+      k: '1000-01-01',
+      l: '1000-01-01 01:01:01',
+      m: '1970-01-01 00:01:01',
+      n: '01:01:01',
+      o: 2006,
+      p: 'p',
+      q: 'q',
+      r: uint8ArrayFromHex('0x72000000'),
+      s: uint8ArrayFromHex('0x73'),
+      t: uint8ArrayFromHex('0x74'),
+      u: uint8ArrayFromHex('0x75'),
+      v: uint8ArrayFromHex('0x76'),
+      w: uint8ArrayFromHex('0x77'),
+      x: 'x',
+      y: 'y',
+      z: 'z',
+      aa: 'aa',
+      ab: 'foo',
+      ac: 'foo,bar',
+      ad: { ad: null },
+      ae: uint8ArrayFromHex(
+        '0x0000000001020000000300000000000000000000000000000000000000000000000000F03F000000000000F03F00000000000000400000000000000000'
+      ),
+      af: uint8ArrayFromHex('0x000000000101000000000000000000F03F000000000000F03F'),
+      ag: uint8ArrayFromHex(
+        '0x0000000001020000000300000000000000000000000000000000000000000000000000F03F000000000000F03F00000000000000400000000000000000'
+      ),
+      ah: uint8ArrayFromHex(
+        '0x00000000010300000002000000040000000000000000000000000000000000000000000000000000000000000000000840000000000000084000000000000000000000000000000000000000000000000004000000000000000000F03F000000000000F03F000000000000F03F00000000000000400000000000000040000000000000F03F000000000000F03F000000000000F03F'
+      ),
+      ai: 1,
+      aj: 1,
+      ak: 1,
+      al: '1',
+      xa: 'xa',
+      xb: 'xb',
+      xc: uint8ArrayFromHex('0x78630000'),
+      xd: 'xd',
+      NULL: null
+    }
+
+    mockPool.intercept({ path: EXECUTE_PATH, method: 'POST' }).reply(200, (opts: any) => {
+      expect(opts.headers['Authorization']).toMatch(/Basic /)
+      const bodyObj = JSON.parse(opts.body.toString())
+      expect(bodyObj.session).toEqual(null)
+      return mockResponse
+    })
+
+    const connection = connect(config)
+    const got = await connection.execute('xxx')
+
+    expect(got.rows[0]).toEqual(want)
+  })
+
+  test('golden (dual.json)', async () => {
+    const mockResponse = {
+      session: mockSession,
+      result: JSON.parse(
+        '{"fields":[{"name":"a","type":"VARCHAR","charset":8,"flags":1}],"rows":[{"lengths":["2"],"values":"w78="}]}',
+      ),
+      timing: 1
+    }
+
+    const want = {
+      a: 'ÿ'
+    }
+
+    mockPool.intercept({ path: EXECUTE_PATH, method: 'POST' }).reply(200, (opts: any) => {
+      expect(opts.headers['Authorization']).toMatch(/Basic /)
+      const bodyObj = JSON.parse(opts.body.toString())
+      expect(bodyObj.session).toEqual(null)
+      return mockResponse
+    })
+
+    const connection = connect(config)
+    const got = await connection.execute('xxx')
+
+    expect(got.rows[0]).toEqual(want)
+  })
+})

__tests__/text.test.ts

@@ -1,25 +1,25 @@
-import { decode, hex, uint8Array, uint8ArrayToHex } from '../src/text'
+import { decodeUTF8, hex, uint8Array, uint8ArrayToHex } from '../src/text'
 
 describe('text', () => {
   describe('decode', () => {
     test('decodes ascii bytes', () => {
-      expect(decode('a')).toEqual('a')
+      expect(decodeUTF8('a')).toEqual('a')
     })
 
     test('decodes empty string', () => {
-      expect(decode('')).toEqual('')
+      expect(decodeUTF8('')).toEqual('')
     })
 
     test('decodes null value', () => {
-      expect(decode(null)).toEqual('')
+      expect(decodeUTF8(null)).toEqual('')
     })
 
     test('decodes undefined value', () => {
-      expect(decode(undefined)).toEqual('')
+      expect(decodeUTF8(undefined)).toEqual('')
     })
 
     test('decodes multi-byte characters', () => {
-      expect(decode('\xF0\x9F\xA4\x94')).toEqual('🤔')
+      expect(decodeUTF8('\xF0\x9F\xA4\x94')).toEqual('🤔')
     })
   })
 

golden/README.md

@@ -0,0 +1,15 @@
+# Golden tests
+
+This generates a "golden" test result that feeds the "parse e2e" test suite.
+
+The intent is a full round trip with a known table that exercises every column type with known correct data.
+
+This excercises different collations, charsets, every integer type.
+
+`test.sql` acts as the seed data against a PlanetScale branch, then we fetch the data back with `curl`.
+
+We can run different queries inside the `generate.sh` script either against the current data, or tests that run against `dual`.
+
+The results are stored back in `$case.json` and a compact version stored in `$case-compact.json`. This compact version is what is shoved into the mock test result for convenience.
+
+Along with this is a `cli.txt` which is the result of running `select * from test` in a mysql CLI dumping the full human readable table. This table is a good reference for what is expected to be human readable or not. Raw binary data is represented as hexadecimal, vs UTF8 strings are readable.

golden/cli.txt

@@ -0,0 +1,5 @@
++----+------+------+------+------+------+------+------+------+------+------------+------------+---------------------+---------------------+----------+------+------+------+------------+------------+------------+------------+------------+------------+------+------+------+------+------+---------+--------------+------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------+------+------+------+------+------+------------+------+
+| id | a    | b    | c    | d    | e    | f    | g    | h    | i    | j          | k          | l                   | m                   | n        | o    | p    | q    | r          | s          | t          | u          | v          | w          | x    | y    | z    | aa   | ab   | ac      | ad           | ae                                                                                                                           | af                                                   | ag                                                                                                                           | ah                                                                                                                                                                                                                                                                                                           | ai   | aj   | ak   | al   | xa   | xb   | xc         | xd   |
++----+------+------+------+------+------+------+------+------+------+------------+------------+---------------------+---------------------+----------+------+------+------+------------+------------+------------+------------+------------+------------+------+------+------+------+------+---------+--------------+------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------+------+------+------+------+------+------------+------+
+|  1 |    1 |    1 |    1 |    1 |    1 |  1.1 |  1.1 |  1.1 |  1.1 | 0x07       | 1000-01-01 | 1000-01-01 01:01:01 | 1970-01-01 00:01:01 | 01:01:01 | 2006 | p    | q    | 0x72000000 | 0x73       | 0x74       | 0x75       | 0x76       | 0x77       | x    | y    | z    | aa   | foo  | foo,bar | {"ad": null} | 0x0000000001020000000300000000000000000000000000000000000000000000000000F03F000000000000F03F00000000000000400000000000000000 | 0x000000000101000000000000000000F03F000000000000F03F | 0x0000000001020000000300000000000000000000000000000000000000000000000000F03F000000000000F03F00000000000000400000000000000000 | 0x00000000010300000002000000040000000000000000000000000000000000000000000000000000000000000000000840000000000000084000000000000000000000000000000000000000000000000004000000000000000000F03F000000000000F03F000000000000F03F00000000000000400000000000000040000000000000F03F000000000000F03F000000000000F03F |    1 |    1 |    1 |    1 | xa   | xb   | 0x78630000 | xd   |
++----+------+------+------+------+------+------+------+------+------+------------+------------+---------------------+---------------------+----------+------+------+------+------------+------------+------------+------------+------------+------------+------+------+------+------+------+---------+--------------+------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------+------+------+------+------+------+------------+------+

golden/dual-compact.json

@@ -0,0 +1 @@
+{"fields":[{"name":"a","type":"VARCHAR","charset":8,"flags":1}],"rows":[{"lengths":["2"],"values":"w78="}]}

golden/dual.json

@@ -0,0 +1,18 @@
+{
+  "fields": [
+    {
+      "name": "a",
+      "type": "VARCHAR",
+      "charset": 8,
+      "flags": 1
+    }
+  ],
+  "rows": [
+    {
+      "lengths": [
+        "2"
+      ],
+      "values": "w78="
+    }
+  ]
+}

golden/generate.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -e
+
+run_sql() {
+  name=$1
+  query=$2
+  echo "{\"query\":\"$query\"}" |
+  curl -s -u $MYSQL_USER:$MYSQL_PWD \
+  -d@- -H'content-type: application/json' \
+  https://$MYSQL_HOST/psdb.v1alpha1.Database/Execute | jq .result > $name.json
+  jq -c . $name.json > $name-compact.json
+  jq . $name.json
+}
+
+cat test.sql | mysql -h $MYSQL_HOST -u $MYSQL_USER -p$MYSQL_PWD
+
+run_sql 'testdb' 'select *, NULL from `test`'
+run_sql 'dual' 'select _latin1 0xff as a'