Skip to content
This repository has been archived by the owner on Jan 13, 2025. It is now read-only.

Add parseJsonWithBigInts helper #3193

Merged
merged 1 commit into from
Sep 11, 2024
Merged

Add parseJsonWithBigInts helper #3193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 191 additions & 0 deletions packages/rpc-transport-http/src/__tests__/large-json-file.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
[
{
"_id": "66c71d2bd28af9a3c7f1a766",
"index": 0,
"guid": "b6465798-e542-4eb0-a5c1-59e3933290ee",
"isActive": false,
"balance": "$1,274.63",
"lamports": 142302234983644260,
"picture": "http://placehold.it/32x32",
"age": 29,
"eyeColor": "blue",
"name": "Collier Carson",
"gender": "male",
"company": "NAMEBOX",
"email": "colliercarson@namebox.com",
"phone": "+1 (888) 428-2044",
"address": "826 Irving Place, Colton, Illinois, 4908",
"about": "Consectetur reprehenderit aliqua eu esse voluptate cupidatat sint anim ex ipsum pariatur eu laborum. Mollit exercitation excepteur consectetur exercitation. Duis non incididunt pariatur consectetur tempor esse nulla aute ad. Ut exercitation tempor duis mollit. Mollit labore sint non est cillum. In esse ullamco fugiat velit est amet ad laboris occaecat officia nisi.",
"registered": "2024-01-12T05:09:11 -00:00",
"latitude": 49.638368,
"longitude": -113.827497,
"tags": ["qui", "excepteur", "aliquip", "amet", "ipsum", "labore", "officia"],
"friends": [
{ "id": 0, "name": "Jeannette Galloway" },
{ "id": 1, "name": "Gordon Murray" },
{ "id": 2, "name": "Randolph Sullivan" }
]
},
{
"_id": "66c71d2b0a5ca9a164428ff8",
"index": 1,
"guid": "dafaa74b-5fa0-4181-9ba7-371aea1188a2",
"isActive": false,
"balance": "$2,863.05",
"lamports": 142302234983644260,
"picture": "http://placehold.it/32x32",
"age": 38,
"eyeColor": "green",
"name": "Shields Fischer",
"gender": "male",
"company": "RONELON",
"email": "shieldsfischer@ronelon.com",
"phone": "+1 (988) 401-2468",
"address": "431 Norman Avenue, Innsbrook, New Mexico, 5118",
"about": "Magna velit tempor est sint elit commodo mollit mollit exercitation reprehenderit id in ullamco quis. Sint reprehenderit minim voluptate culpa fugiat et aliqua enim ipsum. Nostrud magna dolore excepteur occaecat non ex sint eiusmod. Occaecat deserunt labore nostrud veniam adipisicing ullamco esse. Aliqua sunt nostrud proident veniam cupidatat voluptate enim ipsum. Dolore consequat anim eiusmod laboris sunt pariatur anim fugiat sunt velit quis officia. Enim dolore laborum commodo eiusmod nisi nisi.",
"registered": "2019-11-17T03:40:22 -00:00",
"latitude": 11.324387,
"longitude": -80.832796,
"tags": ["eu", "voluptate", "id", "id", "dolor", "nostrud", "nulla"],
"friends": [
{ "id": 0, "name": "Vilma Fox" },
{ "id": 1, "name": "Concetta Ross" },
{ "id": 2, "name": "Wilkins Howe" }
]
},
{
"_id": "66c71d2bcdba189600f87199",
"index": 2,
"guid": "06835d94-0a4f-4e89-b889-798c59918fad",
"isActive": true,
"balance": "$2,307.46",
"lamports": 142302234983644260,
"picture": "http://placehold.it/32x32",
"age": 33,
"eyeColor": "green",
"name": "Penelope Cabrera",
"gender": "female",
"company": "GLUKGLUK",
"email": "penelopecabrera@glukgluk.com",
"phone": "+1 (884) 419-2242",
"address": "743 Dooley Street, Jessie, Palau, 8602",
"about": "Culpa ut veniam reprehenderit do. Sunt ut excepteur cillum laboris esse occaecat officia amet et duis nostrud excepteur. Incididunt reprehenderit ullamco incididunt velit incididunt esse officia eu dolor irure nostrud eiusmod ut quis. Irure mollit consequat sunt commodo non exercitation. Aliquip labore aute sunt deserunt ullamco proident est esse. Minim velit voluptate consequat voluptate dolore. Eu commodo Lorem sint aliquip amet do do ad culpa Lorem nulla anim quis.",
"registered": "2019-10-10T10:38:26 -01:00",
"latitude": 34.456814,
"longitude": 60.745321,
"tags": ["est", "qui", "cupidatat", "excepteur", "ut", "ad", "veniam"],
"friends": [
{ "id": 0, "name": "Elizabeth Huber" },
{ "id": 1, "name": "Sally Robbins" },
{ "id": 2, "name": "Daisy Hunt" }
]
},
{
"_id": "66c71d2bf99c62818eb44055",
"index": 3,
"guid": "c496d7cc-eaf8-4b86-ab12-bb9fd9946d21",
"isActive": false,
"balance": "$1,885.35",
"lamports": 142302234983644260,
"picture": "http://placehold.it/32x32",
"age": 30,
"eyeColor": "blue",
"name": "Mae Fulton",
"gender": "female",
"company": "COREPAN",
"email": "maefulton@corepan.com",
"phone": "+1 (898) 433-2640",
"address": "494 Greene Avenue, Dyckesville, South Carolina, 4544",
"about": "Id commodo quis sint sint voluptate culpa deserunt ad anim sint reprehenderit. Proident occaecat non et dolore magna aliquip ut adipisicing enim officia aliquip consequat. Ullamco reprehenderit cupidatat in est proident aliquip minim sint elit eu magna irure velit.",
"registered": "2022-12-24T04:14:42 -00:00",
"latitude": -78.635799,
"longitude": 53.578193,
"tags": ["minim", "eu", "sint", "culpa", "consequat", "ullamco", "exercitation"],
"friends": [
{ "id": 0, "name": "Patrick Middleton" },
{ "id": 1, "name": "Buchanan Patton" },
{ "id": 2, "name": "Marilyn Cash" }
]
},
{
"_id": "66c71d2b17723c12ff35509a",
"index": 4,
"guid": "892f741f-39a6-4157-8deb-3fa5cb7c89ac",
"isActive": false,
"balance": "$1,720.04",
"lamports": 142302234983644260,
"picture": "http://placehold.it/32x32",
"age": 29,
"eyeColor": "blue",
"name": "Nell Newton",
"gender": "female",
"company": "ASIMILINE",
"email": "nellnewton@asimiline.com",
"phone": "+1 (973) 445-2478",
"address": "494 Elton Street, Alafaya, Arizona, 1347",
"about": "Aliquip consectetur dolor pariatur qui. Enim consequat adipisicing aliqua Lorem amet consectetur irure est laborum ipsum. Magna exercitation excepteur incididunt cupidatat aliquip do tempor non. Et mollit quis quis tempor enim cillum id fugiat. Nisi eiusmod cillum reprehenderit in eiusmod labore est fugiat et officia.",
"registered": "2014-06-19T08:01:53 -01:00",
"latitude": 59.571221,
"longitude": -78.053459,
"tags": ["fugiat", "deserunt", "laborum", "eu", "sit", "ex", "dolor"],
"friends": [
{ "id": 0, "name": "Cheryl Harding" },
{ "id": 1, "name": "Laurie Logan" },
{ "id": 2, "name": "Johns Moody" }
]
},
{
"_id": "66c71d2b5ee698e22b8e9925",
"index": 5,
"guid": "ba974954-e9e1-4d8c-8847-3f28e3a6175b",
"isActive": false,
"balance": "$3,203.47",
"lamports": 142302234983644260,
"picture": "http://placehold.it/32x32",
"age": 22,
"eyeColor": "blue",
"name": "Albert Harmon",
"gender": "male",
"company": "EXOBLUE",
"email": "albertharmon@exoblue.com",
"phone": "+1 (954) 439-3813",
"address": "313 Fountain Avenue, Beason, Rhode Island, 9738",
"about": "Dolor excepteur nisi ut aliqua occaecat non ipsum irure id eu proident duis nulla. Ex dolor do excepteur aliqua officia consectetur ea reprehenderit occaecat sit qui aliquip aute. Occaecat proident cillum in ullamco nostrud laboris nisi excepteur. Incididunt velit cupidatat veniam fugiat.",
"registered": "2016-11-18T08:07:12 -00:00",
"latitude": 42.687475,
"longitude": 39.897468,
"tags": ["minim", "in", "adipisicing", "officia", "culpa", "in", "do"],
"friends": [
{ "id": 0, "name": "Macdonald Mckinney" },
{ "id": 1, "name": "Virginia Crawford" },
{ "id": 2, "name": "Jimmie Lawrence" }
]
},
{
"_id": "66c71d2b06572e5de0348e08",
"index": 6,
"guid": "59bd67a0-44e0-43dd-aebf-2b7422f273b5",
"isActive": true,
"balance": "$2,639.98",
"lamports": 142302234983644260,
"picture": "http://placehold.it/32x32",
"age": 38,
"eyeColor": "blue",
"name": "Jannie Case",
"gender": "female",
"company": "VALREDA",
"email": "janniecase@valreda.com",
"phone": "+1 (851) 464-3024",
"address": "201 Pierrepont Place, Whitehaven, Marshall Islands, 7595",
"about": "Dolore laboris anim non sint cillum cupidatat enim veniam aliqua amet ipsum anim sit ipsum. Id labore labore nostrud incididunt. Lorem culpa quis ad dolor cupidatat elit aliqua officia laborum. Reprehenderit irure dolore do sunt do dolor laborum officia Lorem anim cupidatat consequat. Sunt aute est ad laborum ea magna elit esse sit in nostrud laborum sint. Proident velit eu id laboris commodo aliqua nisi elit consectetur ut exercitation commodo ut ad. Id aute tempor laboris officia fugiat non consectetur nisi ipsum cillum sint anim anim consequat.",
"registered": "2014-10-10T02:16:44 -01:00",
"latitude": 11.356616,
"longitude": -64.69489,
"tags": ["culpa", "qui", "exercitation", "elit", "veniam", "aliquip", "Lorem"],
"friends": [
{ "id": 0, "name": "Bowers Britt" },
{ "id": 1, "name": "Tommie Morris" },
{ "id": 2, "name": "Olsen Pollard" }
]
}
]
90 changes: 90 additions & 0 deletions packages/rpc-transport-http/src/__tests__/parse-json-with-bigints-test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import fs from 'fs';
import path from 'path';

import { parseJsonWithBigInts } from '../parse-json-with-bigints';

const MAX_SAFE_INTEGER = BigInt(Number.MAX_SAFE_INTEGER);
const MAX_SAFE_INTEGER_PLUS_ONE = BigInt(Number.MAX_SAFE_INTEGER) + 1n;

describe('parseJsonWithBigInts', () => {
it.each`
input | expectedBigInt
${'0'} | ${0n}
${'-0'} | ${-0n}
${'1'} | ${1n}
${'-1'} | ${-1n}
${'42'} | ${42n}
${'-42'} | ${-42n}
${'1e5'} | ${100000n}
${'-1e5'} | ${-100000n}
${'1E5'} | ${100000n}
${'-1E5'} | ${-100000n}
${'123e+32'} | ${123n * 10n ** 32n}
${'-123e+32'} | ${-123n * 10n ** 32n}
${'123E+32'} | ${123n * 10n ** 32n}
${'-123E+32'} | ${-123n * 10n ** 32n}
${MAX_SAFE_INTEGER.toString()} | ${MAX_SAFE_INTEGER}
${MAX_SAFE_INTEGER_PLUS_ONE.toString()} | ${MAX_SAFE_INTEGER_PLUS_ONE}
`('parses $input as a bigint', ({ expectedBigInt, input }) => {
expect(parseJsonWithBigInts(input)).toBe(expectedBigInt);
});
it('parses BigInts within nested structures', () => {
const input = '{ "alice": 42, "bob": [3.14, 3e+8, { "baz": 1234567890123456789012345678901234567890 }] }';
expect(parseJsonWithBigInts(input)).toStrictEqual({
alice: 42n,
bob: [3.14, BigInt(3e8), { baz: 1234567890123456789012345678901234567890n }],
});
});
it.each`
input | expectedNumber
${'0.5'} | ${0.5}
${'-0.5'} | ${-0.5}
${'3.14159265'} | ${3.14159265}
${'-3.14159265'} | ${-3.14159265}
${'1e-5'} | ${1e-5}
${'-1e-5'} | ${-1e-5}
${'1E-5'} | ${1e-5}
${'-1E-5'} | ${-1e-5}
${'1e-32'} | ${1e-32}
${'-1189e-32'} | ${-1189e-32}
`('parses $input as a number', ({ expectedNumber, input }) => {
expect(parseJsonWithBigInts(input)).toBe(expectedNumber);
});
it.each([
'null',
'false',
'true',
'[]',
'[null, true, false]',
'{}',
'{ "foo": "bar" }',
'""',
'"Hello World"',
'"42 apples"',
'"base64"',
'"\\base64"',
'"\\"base64"',
'"\\\\base64"',
'"\\"\\"base64"',
'"\\\\\\"base64"',
'"\\\\\\"\\"base64"',
'"He said: \\"I will eat 3 bananas\\""',
'{ "message_100": "Hello to the 1st World" }',
'{ "message_200": "Hello to the \\"2nd World\\"" }',
'{"data":["","base64"]}',
])('does not alter the value of %s', input => {
expect(parseJsonWithBigInts(input)).toStrictEqual(JSON.parse(input));
});
it('can parse complex JSON files', () => {
const largeJsonPath = path.join(__dirname, 'large-json-file.json');
const largeJsonString = fs.readFileSync(largeJsonPath, 'utf8');
const expectedResult = JSON.parse(largeJsonString, (key, value) => {
// eslint-disable-next-line jest/no-conditional-in-test
if (key === 'lamports') return 142302234983644260n;
// eslint-disable-next-line jest/no-conditional-in-test
if (typeof value === 'number' && Number.isInteger(value)) return BigInt(value);
return value;
});
expect(parseJsonWithBigInts(largeJsonString)).toStrictEqual(expectedResult);
});
});
81 changes: 81 additions & 0 deletions packages/rpc-transport-http/src/parse-json-with-bigints.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/**
* This function is a replacement for `JSON.parse` that can handle large
* unsafe integers by parsing them as BigInts. It transforms every
* numerical value into a BigInt without loss of precision.
*/
export function parseJsonWithBigInts(json: string): unknown {
return JSON.parse(wrapIntegersInBigIntValueObject(json), (_, value) => {
return isBigIntValueObject(value) ? unwrapBigIntValueObject(value) : value;
});
}

function wrapIntegersInBigIntValueObject(json: string): string {
const out = [];
let inQuote = false;
for (let ii = 0; ii < json.length; ii++) {
let isEscaped = false;
if (json[ii] === '\\') {
out.push(json[ii++]);
isEscaped = !isEscaped;
}
if (json[ii] === '"') {
out.push(json[ii]);
if (!isEscaped) {
inQuote = !inQuote;
}
continue;
}
if (!inQuote) {
const consumedNumber = consumeNumber(json, ii);
if (consumedNumber?.length) {
ii += consumedNumber.length - 1;
// Don't wrap numbers that contain a decimal point or a negative exponent.
if (consumedNumber.match(/\.|[eE]-/)) {
out.push(consumedNumber);
} else {
out.push(wrapBigIntValueObject(consumedNumber));
}
continue;
}
}
out.push(json[ii]);
}

return out.join('');
}

function consumeNumber(json: string, ii: number): string | null {
/** @see https://stackoverflow.com/a/13340826/11440277 */
const JSON_NUMBER_REGEX = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/;

// Stop early if the first character isn't a digit or a minus sign.
if (!json[ii]?.match(/[-\d]/)) {
return null;
}

// Otherwise, check if the next characters form a valid JSON number.
const numberMatch = json.slice(ii).match(JSON_NUMBER_REGEX);
return numberMatch ? numberMatch[0] : null;
}

type BigIntValueObject = {
// `$` implies 'this is a value object'.
// `n` implies 'interpret the value as a bigint'.
$n: string;
};

function wrapBigIntValueObject(value: string): string {
return `{"$n":"${value}"}`;
}

function unwrapBigIntValueObject({ $n }: BigIntValueObject): bigint {
if ($n.match(/[eE]/)) {
const [units, exponent] = $n.split(/[eE]/);
return BigInt(units) * BigInt(10) ** BigInt(exponent);
}
return BigInt($n);
}

function isBigIntValueObject(value: unknown): value is BigIntValueObject {
return !!value && typeof value === 'object' && '$n' in value && typeof value.$n === 'string';
}