From 0330a51277d8afcfaac0e8473d616ecbdd1e8a52 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Thu, 30 Mar 2023 15:10:07 +0200 Subject: [PATCH 01/11] feat: initial pass at jsonpath expression constraints --- package-lock.json | 30 ++++++++ package.json | 1 + src/api/query-helpers.ts | 144 ++++++++++++++++++++++++++++++++++++ src/tests/jsonpath-tests.ts | 75 +++++++++++++++++++ 4 files changed, 250 insertions(+) create mode 100644 src/tests/jsonpath-tests.ts diff --git a/package-lock.json b/package-lock.json index d56b073d3d..7cfb39c401 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,6 +42,7 @@ "getopts": "2.3.0", "http-proxy-middleware": "2.0.1", "jsonc-parser": "3.0.0", + "jsonpathly": "1.7.5", "jsonrpc-lite": "2.2.0", "lru-cache": "6.0.0", "micro-base58": "0.5.1", @@ -3381,6 +3382,14 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/antlr4": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/antlr4/-/antlr4-4.12.0.tgz", + "integrity": "sha512-23iB5IzXJZRZeK9TigzUyrNc9pSmNqAerJRBcNq1ETrmttMWRgaYZzC561IgEO3ygKsDJTYDTozABXa4b/fTQQ==", + "engines": { + "node": ">=16" + } + }, "node_modules/anymatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", @@ -8072,6 +8081,14 @@ "node >= 0.2.0" ] }, + "node_modules/jsonpathly": { + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/jsonpathly/-/jsonpathly-1.7.5.tgz", + "integrity": "sha512-/gyDvxnTrwMPRDNsKKG5wNgRri8SlBCw84w210I2yl1AiPCmA7vyhWVe+dhobDatdc5hA6MUh1F5QhzazDttHg==", + "dependencies": { + "antlr4": "^4.10.1" + } + }, "node_modules/jsonrpc-lite": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/jsonrpc-lite/-/jsonrpc-lite-2.2.0.tgz", @@ -14654,6 +14671,11 @@ "color-convert": "^2.0.1" } }, + "antlr4": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/antlr4/-/antlr4-4.12.0.tgz", + "integrity": "sha512-23iB5IzXJZRZeK9TigzUyrNc9pSmNqAerJRBcNq1ETrmttMWRgaYZzC561IgEO3ygKsDJTYDTozABXa4b/fTQQ==" + }, "anymatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", @@ -18241,6 +18263,14 @@ "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=", "dev": true }, + "jsonpathly": { + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/jsonpathly/-/jsonpathly-1.7.5.tgz", + "integrity": "sha512-/gyDvxnTrwMPRDNsKKG5wNgRri8SlBCw84w210I2yl1AiPCmA7vyhWVe+dhobDatdc5hA6MUh1F5QhzazDttHg==", + "requires": { + "antlr4": "^4.10.1" + } + }, "jsonrpc-lite": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/jsonrpc-lite/-/jsonrpc-lite-2.2.0.tgz", diff --git a/package.json b/package.json index 8a1720d8e0..b59ee7cc5e 100644 --- a/package.json +++ b/package.json @@ -150,6 +150,7 @@ "getopts": "2.3.0", "http-proxy-middleware": "2.0.1", "jsonc-parser": "3.0.0", + "jsonpathly": "1.7.5", "jsonrpc-lite": "2.2.0", "lru-cache": "6.0.0", "micro-base58": "0.5.1", diff --git a/src/api/query-helpers.ts b/src/api/query-helpers.ts index 56f938f3dd..eb8f1068c0 100644 --- a/src/api/query-helpers.ts +++ b/src/api/query-helpers.ts @@ -3,6 +3,8 @@ import { NextFunction, Request, Response } from 'express'; import { has0xPrefix, hexToBuffer, parseEventTypeStrings, isValidPrincipal } from './../helpers'; import { InvalidRequestError, InvalidRequestErrorType } from '../errors'; import { DbEventTypeId } from './../datastore/common'; +import * as jp from 'jsonpathly'; +import { JsonPathElement } from 'jsonpathly/dist/types/parser/types'; function handleBadRequest(res: Response, next: NextFunction, errorMessage: string): never { const error = new InvalidRequestError(errorMessage, InvalidRequestErrorType.bad_request); @@ -11,6 +13,58 @@ function handleBadRequest(res: Response, next: NextFunction, errorMessage: strin throw error; } +export function validateJsonPathQuery( + req: Request, + res: Response, + next: NextFunction, + paramName: string, + paramRequired: TRequired +): TRequired extends true ? string | never : string | null { + if (!(paramName in req.query)) { + if (paramRequired) { + handleBadRequest(res, next, `Request is missing required "${paramName}" query parameter`); + } else { + return null as TRequired extends true ? string | never : string | null; + } + } + const jsonPathInput = req.query[paramName]; + if (typeof jsonPathInput !== 'string') { + handleBadRequest( + res, + next, + `Unexpected type for '${paramName}' parameter: ${JSON.stringify(jsonPathInput)}` + ); + } + + const maxCharLength = 200; + const maxOperations = 6; + + if (jsonPathInput.length > maxCharLength) { + handleBadRequest( + res, + next, + `JsonPath parameter '${paramName}' is invalid: char length exceeded, max=${maxCharLength}, received=${jsonPathInput.length}` + ); + } + + const jsonPathComplexity = getJsonPathComplexity(jsonPathInput); + if ('error' in jsonPathComplexity) { + handleBadRequest( + res, + next, + `JsonPath parameter '${paramName}' is invalid: ${jsonPathComplexity.error.message}` + ); + } + if (jsonPathComplexity.operations > maxOperations) { + handleBadRequest( + res, + next, + `JsonPath parameter '${paramName}' is invalid: operations exceeded, max=${maxOperations}, received=${jsonPathComplexity.operations}` + ); + } + return jsonPathInput; +} + export function booleanValueForParam( req: Request, res: Response, @@ -45,6 +99,96 @@ export function booleanValueForParam( ); } +/** + * Determine how complex a jsonpath expression is. This uses the jsonpathly library to parse the expression into a tree which + * is then evaluated to determine the number of operations in the expression. + * Returns an error if the expression is invalid or can't be parsed. + */ +export function getJsonPathComplexity(jsonpath: string): { error: Error } | { operations: number } { + if (jsonpath.trim().length === 0) { + return { error: new Error(`Invalid jsonpath, empty expression string`) }; + } + let pathTree: JsonPathElement | null; + try { + pathTree = jp.parse(jsonpath); + } catch (error: any) { + return { error: error instanceof Error ? error : new Error(error.toString()) }; + } + + if (!pathTree) { + return { error: new Error(`Invalid jsonpath, evaluated as null`) }; + } + + let operations: number; + try { + operations = countJsonPathOperations(pathTree); + } catch (error: any) { + return { error: error instanceof Error ? error : new Error(error.toString()) }; + } + + return { operations }; +} + +export function countJsonPathOperations(element: JsonPathElement): number { + let count = 0; + const stack: { + element: JsonPathElement; + operationToAdd: number; + }[] = [{ element, operationToAdd: 0 }]; + + while (stack.length > 0) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const { element, operationToAdd } = stack.pop()!; + count += operationToAdd; + + switch (element.type) { + case 'operation': + case 'logicalExpression': + case 'comparator': + stack.push({ element: element.left, operationToAdd: 1 }); + if (element.right) { + stack.push({ element: element.right, operationToAdd: 0 }); + } + break; + case 'groupOperation': + case 'groupExpression': + case 'notExpression': + case 'filterExpression': + case 'bracketExpression': + stack.push({ element: element.value, operationToAdd: 1 }); + break; + case 'subscript': + stack.push({ element: element.value, operationToAdd: 0 }); + case 'root': + case 'current': + if (element.next) { + stack.push({ element: element.next, operationToAdd: 0 }); + } + break; + case 'unions': + case 'indexes': + for (const value of element.values) { + stack.push({ element: value, operationToAdd: 1 }); + } + break; + case 'slices': + case 'dotdot': + case 'dot': + case 'bracketMember': + case 'wildcard': + case 'numericLiteral': + case 'stringLiteral': + case 'identifier': + case 'value': + // These element types don't have nested elements to process + count++; + break; + } + } + + return count; +} + /** * Determines if the query parameters of a request are intended to include unanchored tx data. * If an error is encountered while parsing the query param then a 400 response with an error message diff --git a/src/tests/jsonpath-tests.ts b/src/tests/jsonpath-tests.ts new file mode 100644 index 0000000000..0553d8510b --- /dev/null +++ b/src/tests/jsonpath-tests.ts @@ -0,0 +1,75 @@ +/* eslint-disable prettier/prettier */ +import { getJsonPathComplexity } from '../api/query-helpers'; + +describe('jsonpath tests', () => { + + const jsonPathVectors: [string, number][] = [ + [ '$.key', 1 ], + [ '$.key.subkey', 2 ], + [ '$.array[*]', 3 ], + [ '$.array[*].subkey', 4 ], + [ '$.key1.key2.subkey', 3 ], + [ '$.key1.key2.array[*].subkey', 6 ], + [ '$.key.subkey1.subkey2.subkey3.subkey4', 5 ], + [ '$.array[*].subkey1.subkey2.subkey3.subkey4', 7 ], + [ '$.key1.key2.subkey1.subkey2.subkey3.subkey4', 6 ], + [ '$.key1.key2.array[*].subkey1.subkey2.subkey3.subkey4', 9 ], + [ '$.store.book[*].author', 5 ], + [ '$.store.book[*].title', 5 ], + [ '$.store.book[0,1].title', 8 ], + [ '$.store.book[0:2].title', 5 ], + [ '$.store.book[-1:].title', 5 ], + [ '$.store.book[-2].author', 4 ], + [ '$.store..price', 2 ], + [ '$..book[2]', 2 ], + [ '$..book[-2]', 2 ], + [ '$..book[0,1]', 6 ], + [ '$..book[:2]', 3 ], + [ '$..book[1:2]', 3 ], + [ '$..book[-2:]', 3 ], + [ '$..book[2:]', 3 ], + [ '$.store.book[?(@.isbn)]', 5 ], + [ '$..book[?(@.price<10)]', 6 ], + [ '$..book[?(@.price==8.95)]', 6 ], + [ '$..book[?(@.price!=8.95)]', 6 ], + [ '$..book[?(@.price<30 && @.category=="fiction")]', 10 ], + [ '$..book[?(@.price<30 || @.category=="fiction")]', 10 ], + [ '$.store.book[0:2].author[0:3].name[0:3].title', 11 ], + [ '$.store..author[0:2].books[0:2].title', 8 ], + [ '$.store.book[?(@.isbn)].publisher[0:2].name', 9 ], + [ '$.store.book[?(@.price<10)].author[0:2].books[0:2].title', 14 ], + [ '$..book[0:2].author[0:3].name[0:3].title', 10 ], + [ '$..book[-2:].author[0:3].name[0:3].title', 10 ], + [ '$..book[2:].author[0:3].name[0:3].title', 10 ], + [ '$..book[?(@.price<30 && @.category=="fiction")].author[0:2].books[0:2].title', 17 ], + [ '$..book[?(@.price<30 || @.category=="fiction")].author[0:2].books[0:2].title', 17 ], + [ '$.store.book[0:2].title[0:3].author[0:3].name', 11 ], + [ '$.store.book[?(@.isbn)].publisher[0:2].name[0:3].address', 12 ], + [ '$..book[0:2].title[0:3].author[0:3].name[0:3].address', 13 ], + [ '$..book[-2:].title[0:3].author[0:3].name[0:3].address', 13 ], + [ '$..book[2:].title[0:3].author[0:3].name[0:3].address', 13 ], + [ '$..book[?(@.price<30 && @.category=="fiction")].title[0:3].author[0:2].name[0:3].address', 20 ], + [ '$..book[?(@.price<30 || @.category=="fiction")].title[0:3].author[0:2].name[0:3].address', 20 ], + [ '$.store..author[0:2].books[0:2].title[0:3].publisher[0:2].name', 14 ], + [ '$.store.book[?(@.isbn)].publisher[0:2].name[0:3].address[0:2].city', 15 ], + ]; + + test.each(jsonPathVectors)('test jsonpath operation complexity: %p', (input, operations) => { + const complexity = getJsonPathComplexity(input); + expect(complexity).toEqual({ operations }); + }); + + /* + test.skip('generate vector data', () => { + const result = postgresExampleVectors.map(([input]) => { + const complexity = getJsonPathComplexity(input); + if ('error' in complexity) { + return [input, complexity.error.message]; + } + return [input, complexity.operations]; + }); + console.log(result); + }); + */ + +}); From 81e8490228615303b2e86d89221029f914312794 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Thu, 30 Mar 2023 21:02:16 +0200 Subject: [PATCH 02/11] feat: add clarity value jsonb column to contract_logs table --- migrations/1680181889941_contract_log_json.js | 14 ++ src/datastore/migrations.ts | 125 ++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 migrations/1680181889941_contract_log_json.js diff --git a/migrations/1680181889941_contract_log_json.js b/migrations/1680181889941_contract_log_json.js new file mode 100644 index 0000000000..2fbf562b49 --- /dev/null +++ b/migrations/1680181889941_contract_log_json.js @@ -0,0 +1,14 @@ +/** @param { import("node-pg-migrate").MigrationBuilder } pgm */ +exports.up = async pgm => { + pgm.addColumn('contract_logs', { + value_json: { + type: 'jsonb', + }, + }); +} + +/** @param { import("node-pg-migrate").MigrationBuilder } pgm */ +exports.down = pgm => { + pgm.dropIndex('contract_logs', 'value_json_path_ops_idx'); + pgm.dropColumn('contract_logs', 'value_json'); +} diff --git a/src/datastore/migrations.ts b/src/datastore/migrations.ts index 931cc49f67..205efbb39f 100644 --- a/src/datastore/migrations.ts +++ b/src/datastore/migrations.ts @@ -1,6 +1,8 @@ import * as path from 'path'; import PgMigrate, { RunnerOption } from 'node-pg-migrate'; import { Client } from 'pg'; +import * as PgCursor from 'pg-cursor'; +import { ClarityTypeID, ClarityValue, decodeClarityValue } from 'stacks-encoding-native-js'; import { APP_DIR, isDevEnv, isTestEnv, logError, logger, REPO_DIR } from '../helpers'; import { getPgClientConfig, PgClientConfig } from './connection-legacy'; import { connectPostgres, PgServer } from './connection'; @@ -43,6 +45,7 @@ export async function runMigrations( runnerOpts.schema = clientConfig.schema; } await PgMigrate(runnerOpts); + await completeSqlMigrations(client, clientConfig); } catch (error) { logError(`Error running pg-migrate`, error); throw error; @@ -99,3 +102,125 @@ export async function dangerousDropAllTables(opts?: { await sql.end(); } } + +function clarityValueToJson(clarityValue: ClarityValue): any { + switch (clarityValue.type_id) { + case ClarityTypeID.Int: + case ClarityTypeID.UInt: + case ClarityTypeID.BoolTrue: + case ClarityTypeID.BoolFalse: + return clarityValue.value; + case ClarityTypeID.StringAscii: + case ClarityTypeID.StringUtf8: + return clarityValue.data; + case ClarityTypeID.ResponseOk: + case ClarityTypeID.OptionalSome: + return clarityValueToJson(clarityValue.value); + case ClarityTypeID.PrincipalStandard: + return clarityValue.address; + case ClarityTypeID.PrincipalContract: + return clarityValue.address + '.' + clarityValue.contract_name; + case ClarityTypeID.ResponseError: + return { _error: clarityValueToJson(clarityValue.value) }; + case ClarityTypeID.OptionalNone: + return null; + case ClarityTypeID.List: + return clarityValue.list.map(clarityValueToJson) as any; + case ClarityTypeID.Tuple: + return Object.fromEntries( + Object.entries(clarityValue.data).map(([key, value]) => [key, clarityValueToJson(value)]) + ); + case ClarityTypeID.Buffer: + return clarityValue.hex; + } + // @ts-expect-error - all ClarityTypeID cases are handled above + throw new Error(`Unexpected Clarity type ID: ${clarityValue.type_id}`); +} + +// Function to finish running sql migrations that are too complex for the node-pg-migrate library. +async function completeSqlMigrations(client: Client, clientConfig: PgClientConfig) { + try { + await client.query('BEGIN'); + await complete_1680181889941_contract_log_json(client, clientConfig); + await client.query('COMMIT'); + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } +} + +async function complete_1680181889941_contract_log_json( + client: Client, + clientConfig: PgClientConfig +) { + // Determine if this migration has already been run by checking if the bew column is nullable. + const result = await client.query(` + SELECT is_nullable + FROM information_schema.columns + WHERE table_name = 'contract_logs' AND column_name = 'value_json' + `); + const migrationNeeded = result.rows[0].is_nullable === 'YES'; + if (!migrationNeeded) { + return; + } + logger.info(`Running migration 1680181889941_contract_log_json..`); + + const getContractLogs = async function* () { + const cursorBatchSize = 1000; + const cursorClient = new Client(clientConfig); + try { + await cursorClient.connect(); + type CursorRow = { id: string; value: string }; + const cursor = new PgCursor('SELECT id, value FROM contract_logs'); + const cursorQuery = cursorClient.query(cursor); + let rows: CursorRow[] = []; + do { + rows = await new Promise((resolve, reject) => { + cursorQuery.read(cursorBatchSize, (error, rows) => + error ? reject(error) : resolve(rows) + ); + }); + for (const row of rows) { + yield row; + } + } while (rows.length > 0); + } finally { + await cursorClient.end(); + } + }; + + const rowCountQuery = await client.query<{ count: number }>( + 'SELECT COUNT(*)::integer FROM contract_logs' + ); + const totalRowCount = rowCountQuery.rows[0].count; + let rowsProcessed = 0; + let lastPercentComplete = 0; + const percentLogInterval = 3; + + for await (const row of getContractLogs()) { + const decoded = decodeClarityValue(row.value); + const clarityValueJson = clarityValueToJson(decoded); + const json = JSON.stringify(clarityValueJson); + await client.query({ + name: 'update_contract_log_json', + text: 'UPDATE contract_logs SET value_json = $1 WHERE id = $2', + values: [json, row.id], + }); + rowsProcessed++; + const percentComplete = Math.round((rowsProcessed / totalRowCount) * 100); + if (percentComplete > lastPercentComplete + percentLogInterval) { + lastPercentComplete = percentComplete; + logger.info(`Running migration 1680181889941_contract_log_json.. ${percentComplete}%`); + } + } + + logger.info(`Running migration 1680181889941_contract_log_json.. set NOT NULL`); + await client.query(`ALTER TABLE contract_logs ALTER COLUMN value_json SET NOT NULL`); + + logger.info('Running migration 1680181889941_contract_log_json.. creating index'); + await client.query( + `CREATE INDEX contract_logs_jsonpathops_idx ON contract_logs USING GIN (value_json jsonb_path_ops)` + ); + + logger.info(`Running migration 1680181889941_contract_log_json.. 100%`); +} From 838fa4e6f50d211673af467b9cc0cb7d0557e351 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Fri, 31 Mar 2023 12:19:15 +0200 Subject: [PATCH 03/11] chore: simplify pg cursor --- src/datastore/migrations.ts | 58 ++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/src/datastore/migrations.ts b/src/datastore/migrations.ts index 205efbb39f..1ea5d27f1f 100644 --- a/src/datastore/migrations.ts +++ b/src/datastore/migrations.ts @@ -1,6 +1,6 @@ import * as path from 'path'; import PgMigrate, { RunnerOption } from 'node-pg-migrate'; -import { Client } from 'pg'; +import { Client, QueryResultRow } from 'pg'; import * as PgCursor from 'pg-cursor'; import { ClarityTypeID, ClarityValue, decodeClarityValue } from 'stacks-encoding-native-js'; import { APP_DIR, isDevEnv, isTestEnv, logError, logger, REPO_DIR } from '../helpers'; @@ -149,14 +149,36 @@ async function completeSqlMigrations(client: Client, clientConfig: PgClientConfi } } +async function* pgCursorQuery(args: { + clientConfig: PgClientConfig; + queryText: string; + queryValues?: any[]; + batchSize: number; +}) { + const cursorClient = new Client(args.clientConfig); + try { + await cursorClient.connect(); + const cursor = new PgCursor(args.queryText, args.queryValues); + const cursorQuery = cursorClient.query(cursor); + let rows: R[] = []; + do { + rows = await new Promise((resolve, reject) => { + cursorQuery.read(args.batchSize, (error, rows) => (error ? reject(error) : resolve(rows))); + }); + yield* rows; + } while (rows.length > 0); + } finally { + await cursorClient.end(); + } +} + async function complete_1680181889941_contract_log_json( client: Client, clientConfig: PgClientConfig ) { // Determine if this migration has already been run by checking if the bew column is nullable. const result = await client.query(` - SELECT is_nullable - FROM information_schema.columns + SELECT is_nullable FROM information_schema.columns WHERE table_name = 'contract_logs' AND column_name = 'value_json' `); const migrationNeeded = result.rows[0].is_nullable === 'YES'; @@ -165,29 +187,11 @@ async function complete_1680181889941_contract_log_json( } logger.info(`Running migration 1680181889941_contract_log_json..`); - const getContractLogs = async function* () { - const cursorBatchSize = 1000; - const cursorClient = new Client(clientConfig); - try { - await cursorClient.connect(); - type CursorRow = { id: string; value: string }; - const cursor = new PgCursor('SELECT id, value FROM contract_logs'); - const cursorQuery = cursorClient.query(cursor); - let rows: CursorRow[] = []; - do { - rows = await new Promise((resolve, reject) => { - cursorQuery.read(cursorBatchSize, (error, rows) => - error ? reject(error) : resolve(rows) - ); - }); - for (const row of rows) { - yield row; - } - } while (rows.length > 0); - } finally { - await cursorClient.end(); - } - }; + const contractLogsCursor = pgCursorQuery<{ id: string; value: string }>({ + clientConfig, + queryText: 'SELECT id, value FROM contract_logs', + batchSize: 1000, + }); const rowCountQuery = await client.query<{ count: number }>( 'SELECT COUNT(*)::integer FROM contract_logs' @@ -197,7 +201,7 @@ async function complete_1680181889941_contract_log_json( let lastPercentComplete = 0; const percentLogInterval = 3; - for await (const row of getContractLogs()) { + for await (const row of contractLogsCursor) { const decoded = decodeClarityValue(row.value); const clarityValueJson = clarityValueToJson(decoded); const json = JSON.stringify(clarityValueJson); From ab31da6731eef681efc4b69bb572d4f19855ec11 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Fri, 31 Mar 2023 16:38:45 +0200 Subject: [PATCH 04/11] feat: encode contract_log clarity values to json and insert in pg --- src/datastore/common.ts | 1 + src/datastore/migrations.ts | 58 ++++---------- src/datastore/pg-write-store.ts | 12 ++- src/helpers.ts | 70 +++++++++++++++- src/tests/cv-compact-json-encoding-tests.ts | 89 +++++++++++++++++++++ 5 files changed, 187 insertions(+), 43 deletions(-) create mode 100644 src/tests/cv-compact-json-encoding-tests.ts diff --git a/src/datastore/common.ts b/src/datastore/common.ts index 3ff9b90a2c..fdff95becd 100644 --- a/src/datastore/common.ts +++ b/src/datastore/common.ts @@ -1379,6 +1379,7 @@ export interface SmartContractEventInsertValues { contract_identifier: string; topic: string; value: PgBytea; + value_json: string; } export interface BurnchainRewardInsertValues { diff --git a/src/datastore/migrations.ts b/src/datastore/migrations.ts index 1ea5d27f1f..2a335c220c 100644 --- a/src/datastore/migrations.ts +++ b/src/datastore/migrations.ts @@ -2,8 +2,15 @@ import * as path from 'path'; import PgMigrate, { RunnerOption } from 'node-pg-migrate'; import { Client, QueryResultRow } from 'pg'; import * as PgCursor from 'pg-cursor'; -import { ClarityTypeID, ClarityValue, decodeClarityValue } from 'stacks-encoding-native-js'; -import { APP_DIR, isDevEnv, isTestEnv, logError, logger, REPO_DIR } from '../helpers'; +import { + APP_DIR, + clarityValueToCompactJson, + isDevEnv, + isTestEnv, + logError, + logger, + REPO_DIR, +} from '../helpers'; import { getPgClientConfig, PgClientConfig } from './connection-legacy'; import { connectPostgres, PgServer } from './connection'; import { databaseHasData } from './event-requests'; @@ -103,40 +110,6 @@ export async function dangerousDropAllTables(opts?: { } } -function clarityValueToJson(clarityValue: ClarityValue): any { - switch (clarityValue.type_id) { - case ClarityTypeID.Int: - case ClarityTypeID.UInt: - case ClarityTypeID.BoolTrue: - case ClarityTypeID.BoolFalse: - return clarityValue.value; - case ClarityTypeID.StringAscii: - case ClarityTypeID.StringUtf8: - return clarityValue.data; - case ClarityTypeID.ResponseOk: - case ClarityTypeID.OptionalSome: - return clarityValueToJson(clarityValue.value); - case ClarityTypeID.PrincipalStandard: - return clarityValue.address; - case ClarityTypeID.PrincipalContract: - return clarityValue.address + '.' + clarityValue.contract_name; - case ClarityTypeID.ResponseError: - return { _error: clarityValueToJson(clarityValue.value) }; - case ClarityTypeID.OptionalNone: - return null; - case ClarityTypeID.List: - return clarityValue.list.map(clarityValueToJson) as any; - case ClarityTypeID.Tuple: - return Object.fromEntries( - Object.entries(clarityValue.data).map(([key, value]) => [key, clarityValueToJson(value)]) - ); - case ClarityTypeID.Buffer: - return clarityValue.hex; - } - // @ts-expect-error - all ClarityTypeID cases are handled above - throw new Error(`Unexpected Clarity type ID: ${clarityValue.type_id}`); -} - // Function to finish running sql migrations that are too complex for the node-pg-migrate library. async function completeSqlMigrations(client: Client, clientConfig: PgClientConfig) { try { @@ -202,13 +175,11 @@ async function complete_1680181889941_contract_log_json( const percentLogInterval = 3; for await (const row of contractLogsCursor) { - const decoded = decodeClarityValue(row.value); - const clarityValueJson = clarityValueToJson(decoded); - const json = JSON.stringify(clarityValueJson); + const clarityValJson = JSON.stringify(clarityValueToCompactJson(row.value)); await client.query({ name: 'update_contract_log_json', text: 'UPDATE contract_logs SET value_json = $1 WHERE id = $2', - values: [json, row.id], + values: [clarityValJson, row.id], }); rowsProcessed++; const percentComplete = Math.round((rowsProcessed / totalRowCount) * 100); @@ -221,10 +192,15 @@ async function complete_1680181889941_contract_log_json( logger.info(`Running migration 1680181889941_contract_log_json.. set NOT NULL`); await client.query(`ALTER TABLE contract_logs ALTER COLUMN value_json SET NOT NULL`); - logger.info('Running migration 1680181889941_contract_log_json.. creating index'); + logger.info('Running migration 1680181889941_contract_log_json.. creating jsonb_path_ops index'); await client.query( `CREATE INDEX contract_logs_jsonpathops_idx ON contract_logs USING GIN (value_json jsonb_path_ops)` ); + // logger.info('Running migration 1680181889941_contract_log_json.. creating jsonb_ops index'); + // await client.query( + // `CREATE INDEX contract_logs_jsonops_idx ON contract_logs USING GIN (value_json jsonb_ops)` + // ); + logger.info(`Running migration 1680181889941_contract_log_json.. 100%`); } diff --git a/src/datastore/pg-write-store.ts b/src/datastore/pg-write-store.ts index b0d9fe9ed5..c504f8fa1c 100644 --- a/src/datastore/pg-write-store.ts +++ b/src/datastore/pg-write-store.ts @@ -1,4 +1,12 @@ -import { logger, logError, getOrAdd, batchIterate, isProdEnv, I32_MAX } from '../helpers'; +import { + logger, + logError, + getOrAdd, + batchIterate, + isProdEnv, + I32_MAX, + clarityValueToCompactJson, +} from '../helpers'; import { DbBlock, DbTx, @@ -1228,6 +1236,7 @@ export class PgWriteStore extends PgStore { contract_identifier: event.contract_identifier, topic: event.topic, value: event.value, + value_json: JSON.stringify(clarityValueToCompactJson(event.value)), })); const res = await sql` INSERT INTO contract_logs ${sql(values)} @@ -1253,6 +1262,7 @@ export class PgWriteStore extends PgStore { contract_identifier: event.contract_identifier, topic: event.topic, value: event.value, + value_json: JSON.stringify(clarityValueToCompactJson(event.value)), }; await sql` INSERT INTO contract_logs ${sql(values)} diff --git a/src/helpers.ts b/src/helpers.ts index 77316307fa..4d1adaa575 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -14,7 +14,13 @@ import * as dotenv from 'dotenv-flow'; import * as http from 'http'; import { isArrayBufferView } from 'node:util/types'; import * as path from 'path'; -import { isValidStacksAddress, stacksToBitcoinAddress } from 'stacks-encoding-native-js'; +import { + ClarityTypeID, + ClarityValue, + decodeClarityValue, + isValidStacksAddress, + stacksToBitcoinAddress, +} from 'stacks-encoding-native-js'; import * as stream from 'stream'; import * as ecc from 'tiny-secp256k1'; import * as util from 'util'; @@ -327,6 +333,68 @@ export function isValidPrincipal( return false; } +/** + * Encodes a Clarity value into a JSON object. This encoding does _not_ preserve exact Clarity type information. + * Instead, values are mapped to a JSON object that is optimized for readability and interoperability with + * JSON-based workflows (e.g. Postgres `jsonpath` support). + * * OptionalSome and ResponseOk are unwrapped (i.e. the value is encoded directly without any nesting). + * * OptionalNone is encoded as json `null`. + * * ResponseError is encoded as an object with a single key `_error`. + * * Buffers are encoded as an object containing the key `hex` with the hex-encoded string as the value, + * and the key `utf8` with the utf8-encoded string as the value. When decoding a Buffer into a string that + * does not exclusively contain valid UTF-8 data, the Unicode replacement character U+FFFD `�` will be used + * to represent those errors. + * * Ints and UInts are encoded as string-quoted integers. + * * Booleans are encoded as booleans. + * * Principals are encoded as strings, e.g. `
` or `
.`. + * * StringAscii and StringUtf8 are both encoded as regular json strings. + * * Lists are encoded as json arrays. + * * Tuples are encoded as json objects. + * @param cv - the Clarity value to encode, or a hex-encoded string representation of a Clarity value. + */ +export function clarityValueToCompactJson(clarityValue: ClarityValue | string): any { + let cv: ClarityValue; + if (typeof clarityValue === 'string') { + cv = decodeClarityValue(clarityValue); + } else { + cv = clarityValue; + } + switch (cv.type_id) { + case ClarityTypeID.Int: + case ClarityTypeID.UInt: + case ClarityTypeID.BoolTrue: + case ClarityTypeID.BoolFalse: + return cv.value; + case ClarityTypeID.StringAscii: + case ClarityTypeID.StringUtf8: + return cv.data; + case ClarityTypeID.ResponseOk: + case ClarityTypeID.OptionalSome: + return clarityValueToCompactJson(cv.value); + case ClarityTypeID.PrincipalStandard: + return cv.address; + case ClarityTypeID.PrincipalContract: + return cv.address + '.' + cv.contract_name; + case ClarityTypeID.ResponseError: + return { _error: clarityValueToCompactJson(cv.value) }; + case ClarityTypeID.OptionalNone: + return null; + case ClarityTypeID.List: + return cv.list.map(clarityValueToCompactJson) as any; + case ClarityTypeID.Tuple: + return Object.fromEntries( + Object.entries(cv.data).map(([key, value]) => [key, clarityValueToCompactJson(value)]) + ); + case ClarityTypeID.Buffer: + return { + hex: cv.buffer, + utf8: Buffer.from(cv.buffer.substring(2), 'hex').toString('utf8'), + }; + } + // @ts-expect-error - all ClarityTypeID cases are handled above + throw new Error(`Unexpected Clarity type ID: ${cv.type_id}`); +} + export type HttpClientResponse = http.IncomingMessage & { statusCode: number; statusMessage: string; diff --git a/src/tests/cv-compact-json-encoding-tests.ts b/src/tests/cv-compact-json-encoding-tests.ts new file mode 100644 index 0000000000..8746a27f46 --- /dev/null +++ b/src/tests/cv-compact-json-encoding-tests.ts @@ -0,0 +1,89 @@ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable prettier/prettier */ +import { clarityValueToCompactJson } from '../helpers'; +import { decodeClarityValueToRepr } from 'stacks-encoding-native-js'; + +describe('clarity value compact json encoding', () => { + + const clarityCompactJsonVectors: [repr: string, hexEncodedCV: string, compactJson: string][] = [ + [ + `(tuple (action "voted") (data (tuple (contract 'SP3MBWGMCVC9KZ5DTAYFMG1D0AEJCR7NENTM3FTK5.proposal-farm-xusd-usda) (end-block-height u98088) (is-ended false) (no-votes u0) (proposer 'SP3MBWGMCVC9KZ5DTAYFMG1D0AEJCR7NENTM3FTK5) (start-block-height u97080) (title u"xUSD/USDA Farming at Arkadiko") (url u"https://discord.com/channels/923585641554518026/934026731256422450") (yes-votes u14965111242))) (type "proposal"))`, + '0x0c0000000306616374696f6e0d00000005766f74656404646174610c0000000908636f6e74726163740616e8be428cdb133f95ba579f4805a053a4cc1eaeae1770726f706f73616c2d6661726d2d787573642d7573646110656e642d626c6f636b2d6865696768740100000000000000000000000000017f280869732d656e64656404086e6f2d766f74657301000000000000000000000000000000000870726f706f7365720516e8be428cdb133f95ba579f4805a053a4cc1eaeae1273746172742d626c6f636b2d6865696768740100000000000000000000000000017b38057469746c650e0000001d785553442f55534441204661726d696e672061742041726b6164696b6f0375726c0e0000004268747470733a2f2f646973636f72642e636f6d2f6368616e6e656c732f3932333538353634313535343531383032362f393334303236373331323536343232343530097965732d766f746573010000000000000000000000037bfd79ca04747970650d0000000870726f706f73616c', + '{"action":"voted","data":{"contract":"SP3MBWGMCVC9KZ5DTAYFMG1D0AEJCR7NENTM3FTK5.proposal-farm-xusd-usda","end-block-height":"98088","is-ended":false,"no-votes":"0","proposer":"SP3MBWGMCVC9KZ5DTAYFMG1D0AEJCR7NENTM3FTK5","start-block-height":"97080","title":"xUSD/USDA Farming at Arkadiko","url":"https://discord.com/channels/923585641554518026/934026731256422450","yes-votes":"14965111242"},"type":"proposal"}' + ], + [ + `(tuple (action "accept-bid") (payload (tuple (action_event_index u1) (bid_amount u22000000) (bid_id u4572) (bidder_address 'SP1JSDAGAYXCTGVMNTD3H2RHP8XYTFR9Q813TPZK8) (collection_id 'SPJW1XE278YMCEYMXB8ZFGJMH8ZVAAEDP2S2PJYG.happy-welsh) (expiration_block u97322) (royalty (tuple (percent u250) (recipient_address 'SP28W4CM7T6RYGXHDA0BQPBYNKYSTSFCP0M6FJB0T))) (seller_address 'SP1NGMS9Z48PRXFAG2MKBSP0PWERF07C0KV9SPJ66) (token_id u907))))`, + '0x0c0000000206616374696f6e0d0000000a6163636570742d626964077061796c6f61640c0000000912616374696f6e5f6576656e745f696e64657801000000000000000000000000000000010a6269645f616d6f756e7401000000000000000000000000014fb180066269645f696401000000000000000000000000000011dc0e6269646465725f6164647265737305166596aa0af759a86e95d347116236477da7e137400d636f6c6c656374696f6e5f6964061625c0f5c23a3d463bd4ead1f7c2548a3fb529cdb00b68617070792d77656c73681065787069726174696f6e5f626c6f636b0100000000000000000000000000017c2a07726f79616c74790c000000020770657263656e7401000000000000000000000000000000fa11726563697069656e745f61646472657373051691c23287d1b1e8762d50177b2fd59fb3acbd96050e73656c6c65725f6164647265737305166b0a653f222d8ebd501526bcd816e3b0f01d809e08746f6b656e5f6964010000000000000000000000000000038b', + '{"action":"accept-bid","payload":{"action_event_index":"1","bid_amount":"22000000","bid_id":"4572","bidder_address":"SP1JSDAGAYXCTGVMNTD3H2RHP8XYTFR9Q813TPZK8","collection_id":"SPJW1XE278YMCEYMXB8ZFGJMH8ZVAAEDP2S2PJYG.happy-welsh","expiration_block":"97322","royalty":{"percent":"250","recipient_address":"SP28W4CM7T6RYGXHDA0BQPBYNKYSTSFCP0M6FJB0T"},"seller_address":"SP1NGMS9Z48PRXFAG2MKBSP0PWERF07C0KV9SPJ66","token_id":"907"}}' + ], + [ + `(tuple (action "swap-x-for-y") (data (tuple (balance-x u315175871745570) (balance-y u11937495544) (fee-rate-x u300000) (fee-rate-y u300000) (fee-rebate u50000000) (fee-to-address 'SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.multisig-fwp-wstx-wbtc-50-50-v1-01) (oracle-average u95000000) (oracle-enabled true) (oracle-resilient u3871) (pool-token 'SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.fwp-wstx-wbtc-50-50-v1-01) (total-supply u3290909475111))) (object "pool"))`, + '0x0c0000000306616374696f6e0d0000000c737761702d782d666f722d7904646174610c0000000b0962616c616e63652d7801000000000000000000011ea699e08e220962616c616e63652d7901000000000000000000000002c787b9f80a6665652d726174652d7801000000000000000000000000000493e00a6665652d726174652d7901000000000000000000000000000493e00a6665652d7265626174650100000000000000000000000002faf0800e6665652d746f2d616464726573730616e685b016b3b6cd9ebf35f38e5ae29392e2acd51d226d756c74697369672d6677702d777374782d776274632d35302d35302d76312d30310e6f7261636c652d617665726167650100000000000000000000000005a995c00e6f7261636c652d656e61626c656403106f7261636c652d726573696c69656e740100000000000000000000000000000f1f0a706f6f6c2d746f6b656e0616e685b016b3b6cd9ebf35f38e5ae29392e2acd51d196677702d777374782d776274632d35302d35302d76312d30310c746f74616c2d737570706c79010000000000000000000002fe397d8127066f626a6563740d00000004706f6f6c', + '{"action":"swap-x-for-y","data":{"balance-x":"315175871745570","balance-y":"11937495544","fee-rate-x":"300000","fee-rate-y":"300000","fee-rebate":"50000000","fee-to-address":"SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.multisig-fwp-wstx-wbtc-50-50-v1-01","oracle-average":"95000000","oracle-enabled":true,"oracle-resilient":"3871","pool-token":"SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.fwp-wstx-wbtc-50-50-v1-01","total-supply":"3290909475111"},"object":"pool"}' + ], + [ + `(tuple (action "swap-x-for-y") (data (tuple (balance-x u39248987199) (balance-y u294901858687020) (enabled true) (fee-balance-x u209257577) (fee-balance-y u3188281977630) (fee-to-address (some 'SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR)) (name "wSTX-WELSH") (shares-total u3135066566725) (swap-token 'SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR.arkadiko-swap-token-wstx-welsh))) (object "pair"))`, + '0x0c0000000306616374696f6e0d0000000c737761702d782d666f722d7904646174610c000000090962616c616e63652d7801000000000000000000000009236c043f0962616c616e63652d7901000000000000000000010c363087d82c07656e61626c6564030d6665652d62616c616e63652d78010000000000000000000000000c7904690d6665652d62616c616e63652d79010000000000000000000002e6546a2b1e0e6665652d746f2d616464726573730a0516982f3ec112a5f5928a5c96a914bd733793b896a5046e616d650d0000000a775354582d57454c53480c7368617265732d746f74616c010000000000000000000002d9f08770450a737761702d746f6b656e0616982f3ec112a5f5928a5c96a914bd733793b896a51e61726b6164696b6f2d737761702d746f6b656e2d777374782d77656c7368066f626a6563740d0000000470616972', + '{"action":"swap-x-for-y","data":{"balance-x":"39248987199","balance-y":"294901858687020","enabled":true,"fee-balance-x":"209257577","fee-balance-y":"3188281977630","fee-to-address":"SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR","name":"wSTX-WELSH","shares-total":"3135066566725","swap-token":"SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR.arkadiko-swap-token-wstx-welsh"},"object":"pair"}' + ], + [ + `(tuple (attachment (tuple (attachment-index u360918) (hash 0xffcbf70fa7fa89adcbd4d80530d2cc93a3f1d236) (metadata (tuple (name 0x64657369676e6d616e6167656d656e74636f6e73756c74696e67) (namespace 0x627463) (op "name-register") (tx-sender 'SP3JH2C1BW3TY23AW5X12TPBFMSZHTV0TBZ1KH5S9))))))`, + '0x0c000000010a6174746163686d656e740c00000003106174746163686d656e742d696e64657801000000000000000000000000000581d604686173680200000014ffcbf70fa7fa89adcbd4d80530d2cc93a3f1d236086d657461646174610c00000004046e616d65020000001a64657369676e6d616e6167656d656e74636f6e73756c74696e67096e616d6573706163650200000003627463026f700d0000000d6e616d652d72656769737465720974782d73656e6465720516e511302be0f5e10d5c2f422d596fa67f1d6c1a5f', + '{"attachment":{"attachment-index":"360918","hash":{"hex":"0xffcbf70fa7fa89adcbd4d80530d2cc93a3f1d236","utf8":"���\\u000f�������\\u00050�̓���6"},"metadata":{"name":{"hex":"0x64657369676e6d616e6167656d656e74636f6e73756c74696e67","utf8":"designmanagementconsulting"},"namespace":{"hex":"0x627463","utf8":"btc"},"op":"name-register","tx-sender":"SP3JH2C1BW3TY23AW5X12TPBFMSZHTV0TBZ1KH5S9"}}}' + ], + [ + `(tuple (amountStacked u7420000000000) (cityId u1) (cityName "mia") (cityTreasury 'SP8A9HZ3PKST0S42VM9523Z9NV42SZ026V4K39WH.ccd002-treasury-mia-stacking) (event "stacking") (firstCycle u54) (lastCycle u59) (lockPeriod u6) (userId u136))`, + '0x0c000000090d616d6f756e74537461636b6564010000000000000000000006bf9a76d80006636974794964010000000000000000000000000000000108636974794e616d650d000000036d69610c636974795472656173757279061610a4c7e3b4f3a06482dd12510fe9aec82cfc02361c6363643030322d74726561737572792d6d69612d737461636b696e67056576656e740d00000008737461636b696e670a66697273744379636c650100000000000000000000000000000036096c6173744379636c65010000000000000000000000000000003b0a6c6f636b506572696f640100000000000000000000000000000006067573657249640100000000000000000000000000000088', + '{"amountStacked":"7420000000000","cityId":"1","cityName":"mia","cityTreasury":"SP8A9HZ3PKST0S42VM9523Z9NV42SZ026V4K39WH.ccd002-treasury-mia-stacking","event":"stacking","firstCycle":"54","lastCycle":"59","lockPeriod":"6","userId":"136"}' + ], + [ + '0x3231333431363630323537', + '0x020000000b3231333431363630323537', + '{"hex":"0x3231333431363630323537","utf8":"21341660257"}' + ], + [ + 'none', + '0x09', + 'null' + ], + [ + '"``... to be a completely separate network and separate block chain, yet share CPU power with Bitcoin`` - Satoshi Nakamoto"', + '0x0d0000007960602e2e2e20746f206265206120636f6d706c6574656c79207365706172617465206e6574776f726b20616e6420736570617261746520626c6f636b20636861696e2c207965742073686172652043505520706f776572207769746820426974636f696e6060202d205361746f736869204e616b616d6f746f', + '"``... to be a completely separate network and separate block chain, yet share CPU power with Bitcoin`` - Satoshi Nakamoto"' + ], + [ + "'SP1K1A1PMGW2ZJCNF46NWZWHG8TS1D23EGH1KNK60", + '0x0516661506d48705f932af21abcff23046b216886e84', + '"SP1K1A1PMGW2ZJCNF46NWZWHG8TS1D23EGH1KNK60"' + ], + [ + '"Is this thing on?"', + '0x0d0000001149732074686973207468696e67206f6e3f', + '"Is this thing on?"' + ], + [ + '0x324d775773533233676638346554', + '0x020000000e324d775773533233676638346554', + '{"hex":"0x324d775773533233676638346554","utf8":"2MwWsS23gf84eT"}' + ] + ]; + + test.each(clarityCompactJsonVectors)('test Clarity value compact json encoding: %p', (reprString, hexEncodedCV, compactJson) => { + const encodedJson = clarityValueToCompactJson(hexEncodedCV); + const reprResult = decodeClarityValueToRepr(hexEncodedCV); + expect(encodedJson).toEqual(JSON.parse(compactJson)); + expect(reprResult).toBe(reprString); + }); + + /* + test.skip('generate', () => { + const result = clarityCompactJsonVectors.map(([, hexEncodedCV, compactJson]) => { + const encodedJson = clarityValueToCompactJson(hexEncodedCV); + const reprResult = decodeClarityValueToRepr(hexEncodedCV); + return [reprResult, hexEncodedCV, JSON.stringify(encodedJson)]; + }); + console.log(result); + }); + */ + +}); From 565a26f0fdacda4e252ffb413a433caa9c5ec377 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Fri, 31 Mar 2023 18:39:44 +0200 Subject: [PATCH 05/11] feat: support `filter_path` and `contains` json operators in `/contract//events` endpoint --- src/api/controllers/db-controller.ts | 6 +++ src/api/query-helpers.ts | 8 ++-- src/api/routes/contract.ts | 46 +++++++++++++++++++++- src/datastore/pg-store.ts | 57 ++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 6 deletions(-) diff --git a/src/api/controllers/db-controller.ts b/src/api/controllers/db-controller.ts index 8a63c4da0d..f0effa1c1e 100644 --- a/src/api/controllers/db-controller.ts +++ b/src/api/controllers/db-controller.ts @@ -346,6 +346,12 @@ export function parseDbEvent(dbEvent: DbEvent): TransactionEvent { }, }, }; + if ('value_json' in dbEvent) { + Object.defineProperty(event.contract_log.value, 'json', { + value: dbEvent.value_json, + enumerable: true, + }); + } return event; } case DbEventTypeId.StxLock: { diff --git a/src/api/query-helpers.ts b/src/api/query-helpers.ts index eb8f1068c0..186bf1e252 100644 --- a/src/api/query-helpers.ts +++ b/src/api/query-helpers.ts @@ -18,10 +18,10 @@ export function validateJsonPathQuery( res: Response, next: NextFunction, paramName: string, - paramRequired: TRequired + args: { paramRequired: TRequired; maxCharLength: number; maxOperations: number } ): TRequired extends true ? string | never : string | null { if (!(paramName in req.query)) { - if (paramRequired) { + if (args.paramRequired) { handleBadRequest(res, next, `Request is missing required "${paramName}" query parameter`); } else { return null as TRequired extends true ? string | never : string | null; @@ -36,8 +36,8 @@ export function validateJsonPathQuery( ); } - const maxCharLength = 200; - const maxOperations = 6; + const maxCharLength = args.maxCharLength; + const maxOperations = args.maxOperations; if (jsonPathInput.length > maxCharLength) { handleBadRequest( diff --git a/src/api/routes/contract.ts b/src/api/routes/contract.ts index 531511ec0f..8643b39f61 100644 --- a/src/api/routes/contract.ts +++ b/src/api/routes/contract.ts @@ -2,7 +2,7 @@ import * as express from 'express'; import { asyncHandler } from '../async-handler'; import { getPagingQueryLimit, parsePagingQueryInput, ResourceType } from '../pagination'; import { parseDbEvent } from '../controllers/db-controller'; -import { parseTraitAbi } from '../query-helpers'; +import { parseTraitAbi, validateJsonPathQuery } from '../query-helpers'; import { PgStore } from '../../datastore/pg-store'; export function createContractRouter(db: PgStore): express.Router { @@ -50,14 +50,56 @@ export function createContractRouter(db: PgStore): express.Router { router.get( '/:contract_id/events', - asyncHandler(async (req, res) => { + asyncHandler(async (req, res, next) => { const { contract_id } = req.params; const limit = getPagingQueryLimit(ResourceType.Contract, req.query.limit); const offset = parsePagingQueryInput(req.query.offset ?? 0); + + /* + const filterPath = validateJsonPathQuery(req, res, next, 'filter_path', { + paramRequired: false, + maxCharLength: 200, + maxOperations: 6, + }); + */ + + const filterPath = (req.query['filter_path'] ?? null) as string | null; + const maxFilterPathCharLength = 200; + if (filterPath && filterPath.length > maxFilterPathCharLength) { + res.status(400).json({ + error: `'filter_path' query param value exceeds ${maxFilterPathCharLength} character limit`, + }); + return; + } + + const containsJsonQuery = req.query['contains']; + if (containsJsonQuery && typeof containsJsonQuery !== 'string') { + res.status(400).json({ error: `'contains' query param must be a string` }); + return; + } + let containsJson: any | undefined; + const maxContainsJsonCharLength = 200; + if (containsJsonQuery) { + if (containsJsonQuery.length > maxContainsJsonCharLength) { + res.status(400).json({ + error: `'contains' query param value exceeds ${maxContainsJsonCharLength} character limit`, + }); + return; + } + try { + containsJson = JSON.parse(containsJsonQuery); + } catch (error) { + res.status(400).json({ error: `'contains' query param value must be valid JSON` }); + return; + } + } + const eventsQuery = await db.getSmartContractEvents({ contractId: contract_id, limit, offset, + filterPath, + containsJson, }); if (!eventsQuery.found) { res.status(404).json({ error: `cannot find events for contract by ID: ${contract_id}` }); diff --git a/src/datastore/pg-store.ts b/src/datastore/pg-store.ts index 1afc8539de..8a7400540a 100644 --- a/src/datastore/pg-store.ts +++ b/src/datastore/pg-store.ts @@ -2094,6 +2094,63 @@ export class PgStore { contractId, limit, offset, + filterPath, + containsJson, + }: { + contractId: string; + limit: number; + offset: number; + filterPath: string | null; + containsJson: any | undefined; + }): Promise> { + const hasFilterPath = filterPath !== null; + const hasJsonContains = containsJson !== undefined; + + const logResults = await this.sql< + { + event_index: number; + tx_id: string; + tx_index: number; + block_height: number; + contract_identifier: string; + topic: string; + value: string; + value_json: any; + }[] + >` + SELECT + event_index, tx_id, tx_index, block_height, contract_identifier, topic, value, value_json + FROM contract_logs + WHERE canonical = true AND microblock_canonical = true AND contract_identifier = ${contractId} + ${hasFilterPath ? this.sql`AND value_json @? ${filterPath}::jsonpath` : this.sql``} + ${hasJsonContains ? this.sql`AND value_json @> ${containsJson}::jsonb` : this.sql``} + ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC + LIMIT ${limit} + OFFSET ${offset} + `; + + const result = logResults.map(result => { + const event: DbSmartContractEvent & { value_json: any } = { + event_index: result.event_index, + tx_id: result.tx_id, + tx_index: result.tx_index, + block_height: result.block_height, + canonical: true, + event_type: DbEventTypeId.SmartContractLog, + contract_identifier: result.contract_identifier, + topic: result.topic, + value: result.value, + value_json: result.value_json, + }; + return event; + }); + return { found: true, result }; + } + + async getSmartContractEventsFilteredByJsonPath({ + contractId, + limit, + offset, }: { contractId: string; limit: number; From 65cba05f0523c778aa9bd0b6e4c5e8d13f551969 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Fri, 31 Mar 2023 18:57:20 +0200 Subject: [PATCH 06/11] docs: update contract events endpoint docs --- docs/openapi.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 50241a31ff..7d9ff093db 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1006,6 +1006,20 @@ paths: schema: type: string example: "SP6P4EJF0VG8V0RB3TQQKJBHDQKEF6NVRD1KZE3C.satoshibles" + - name: contains + in: query + description: Optional stringified JSON to select only results that contain the given JSON + required: false + schema: + type: string + example: '{"attachment":{"metadata":{"op":"name-register"}}}' + - name: filter_path + in: query + description: Optional [`jsonpath` expression](https://www.postgresql.org/docs/14/functions-json.html#FUNCTIONS-SQLJSON-PATH) to select only results that contain items matching the expression + required: false + schema: + type: string + example: '$.attachment.metadata?(@.op=="name-register")' - name: limit in: query description: max number of contract events to fetch From 3829a1ee92df34400ad8ca80b69edb7600c78962 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Fri, 31 Mar 2023 20:35:00 +0200 Subject: [PATCH 07/11] chore: cleanup --- src/api/controllers/db-controller.ts | 6 ------ src/datastore/pg-store.ts | 8 +++----- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/api/controllers/db-controller.ts b/src/api/controllers/db-controller.ts index f0effa1c1e..8a63c4da0d 100644 --- a/src/api/controllers/db-controller.ts +++ b/src/api/controllers/db-controller.ts @@ -346,12 +346,6 @@ export function parseDbEvent(dbEvent: DbEvent): TransactionEvent { }, }, }; - if ('value_json' in dbEvent) { - Object.defineProperty(event.contract_log.value, 'json', { - value: dbEvent.value_json, - enumerable: true, - }); - } return event; } case DbEventTypeId.StxLock: { diff --git a/src/datastore/pg-store.ts b/src/datastore/pg-store.ts index 8a7400540a..7c140b893d 100644 --- a/src/datastore/pg-store.ts +++ b/src/datastore/pg-store.ts @@ -2102,7 +2102,7 @@ export class PgStore { offset: number; filterPath: string | null; containsJson: any | undefined; - }): Promise> { + }): Promise> { const hasFilterPath = filterPath !== null; const hasJsonContains = containsJson !== undefined; @@ -2119,7 +2119,7 @@ export class PgStore { }[] >` SELECT - event_index, tx_id, tx_index, block_height, contract_identifier, topic, value, value_json + event_index, tx_id, tx_index, block_height, contract_identifier, topic, value FROM contract_logs WHERE canonical = true AND microblock_canonical = true AND contract_identifier = ${contractId} ${hasFilterPath ? this.sql`AND value_json @? ${filterPath}::jsonpath` : this.sql``} @@ -2128,9 +2128,8 @@ export class PgStore { LIMIT ${limit} OFFSET ${offset} `; - const result = logResults.map(result => { - const event: DbSmartContractEvent & { value_json: any } = { + const event: DbSmartContractEvent = { event_index: result.event_index, tx_id: result.tx_id, tx_index: result.tx_index, @@ -2140,7 +2139,6 @@ export class PgStore { contract_identifier: result.contract_identifier, topic: result.topic, value: result.value, - value_json: result.value_json, }; return event; }); From 1b6ae0066fd79f077a8d3e1ed64e15da6d6947bf Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Fri, 31 Mar 2023 20:42:52 +0200 Subject: [PATCH 08/11] chore: cleanup 2 --- src/datastore/pg-store.ts | 45 --------------------------------------- 1 file changed, 45 deletions(-) diff --git a/src/datastore/pg-store.ts b/src/datastore/pg-store.ts index 7c140b893d..17848c27df 100644 --- a/src/datastore/pg-store.ts +++ b/src/datastore/pg-store.ts @@ -2145,51 +2145,6 @@ export class PgStore { return { found: true, result }; } - async getSmartContractEventsFilteredByJsonPath({ - contractId, - limit, - offset, - }: { - contractId: string; - limit: number; - offset: number; - }): Promise> { - const logResults = await this.sql< - { - event_index: number; - tx_id: string; - tx_index: number; - block_height: number; - contract_identifier: string; - topic: string; - value: string; - }[] - >` - SELECT - event_index, tx_id, tx_index, block_height, contract_identifier, topic, value - FROM contract_logs - WHERE canonical = true AND microblock_canonical = true AND contract_identifier = ${contractId} - ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC - LIMIT ${limit} - OFFSET ${offset} - `; - const result = logResults.map(result => { - const event: DbSmartContractEvent = { - event_index: result.event_index, - tx_id: result.tx_id, - tx_index: result.tx_index, - block_height: result.block_height, - canonical: true, - event_type: DbEventTypeId.SmartContractLog, - contract_identifier: result.contract_identifier, - topic: result.topic, - value: result.value, - }; - return event; - }); - return { found: true, result }; - } - async getSmartContractByTrait(args: { trait: ClarityAbi; limit: number; From eba8472525f02d55c061dc8f7d6da095a662827b Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Mon, 3 Apr 2023 15:04:47 +0200 Subject: [PATCH 09/11] feat: encode clarity ints as json numbers where possible --- src/datastore/migrations.ts | 8 ++++---- src/helpers.ts | 5 ++++- src/tests/cv-compact-json-encoding-tests.ts | 12 ++++++------ 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/datastore/migrations.ts b/src/datastore/migrations.ts index 2a335c220c..9b3f6a62c5 100644 --- a/src/datastore/migrations.ts +++ b/src/datastore/migrations.ts @@ -197,10 +197,10 @@ async function complete_1680181889941_contract_log_json( `CREATE INDEX contract_logs_jsonpathops_idx ON contract_logs USING GIN (value_json jsonb_path_ops)` ); - // logger.info('Running migration 1680181889941_contract_log_json.. creating jsonb_ops index'); - // await client.query( - // `CREATE INDEX contract_logs_jsonops_idx ON contract_logs USING GIN (value_json jsonb_ops)` - // ); + logger.info('Running migration 1680181889941_contract_log_json.. creating jsonb_ops index'); + await client.query( + `CREATE INDEX contract_logs_jsonops_idx ON contract_logs USING GIN (value_json jsonb_ops)` + ); logger.info(`Running migration 1680181889941_contract_log_json.. 100%`); } diff --git a/src/helpers.ts b/src/helpers.ts index 4d1adaa575..62d38ffe29 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -344,7 +344,8 @@ export function isValidPrincipal( * and the key `utf8` with the utf8-encoded string as the value. When decoding a Buffer into a string that * does not exclusively contain valid UTF-8 data, the Unicode replacement character U+FFFD `�` will be used * to represent those errors. - * * Ints and UInts are encoded as string-quoted integers. + * * Ints and UInts that are in the range of a safe js integers are encoded as numbers, otherwise they + * are encoded as string-quoted integers. * * Booleans are encoded as booleans. * * Principals are encoded as strings, e.g. `
` or `
.`. * * StringAscii and StringUtf8 are both encoded as regular json strings. @@ -362,6 +363,8 @@ export function clarityValueToCompactJson(clarityValue: ClarityValue | string): switch (cv.type_id) { case ClarityTypeID.Int: case ClarityTypeID.UInt: + const intVal = parseInt(cv.value); + return Number.isSafeInteger(intVal) ? intVal : cv.value; case ClarityTypeID.BoolTrue: case ClarityTypeID.BoolFalse: return cv.value; diff --git a/src/tests/cv-compact-json-encoding-tests.ts b/src/tests/cv-compact-json-encoding-tests.ts index 8746a27f46..0ae6a13473 100644 --- a/src/tests/cv-compact-json-encoding-tests.ts +++ b/src/tests/cv-compact-json-encoding-tests.ts @@ -9,32 +9,32 @@ describe('clarity value compact json encoding', () => { [ `(tuple (action "voted") (data (tuple (contract 'SP3MBWGMCVC9KZ5DTAYFMG1D0AEJCR7NENTM3FTK5.proposal-farm-xusd-usda) (end-block-height u98088) (is-ended false) (no-votes u0) (proposer 'SP3MBWGMCVC9KZ5DTAYFMG1D0AEJCR7NENTM3FTK5) (start-block-height u97080) (title u"xUSD/USDA Farming at Arkadiko") (url u"https://discord.com/channels/923585641554518026/934026731256422450") (yes-votes u14965111242))) (type "proposal"))`, '0x0c0000000306616374696f6e0d00000005766f74656404646174610c0000000908636f6e74726163740616e8be428cdb133f95ba579f4805a053a4cc1eaeae1770726f706f73616c2d6661726d2d787573642d7573646110656e642d626c6f636b2d6865696768740100000000000000000000000000017f280869732d656e64656404086e6f2d766f74657301000000000000000000000000000000000870726f706f7365720516e8be428cdb133f95ba579f4805a053a4cc1eaeae1273746172742d626c6f636b2d6865696768740100000000000000000000000000017b38057469746c650e0000001d785553442f55534441204661726d696e672061742041726b6164696b6f0375726c0e0000004268747470733a2f2f646973636f72642e636f6d2f6368616e6e656c732f3932333538353634313535343531383032362f393334303236373331323536343232343530097965732d766f746573010000000000000000000000037bfd79ca04747970650d0000000870726f706f73616c', - '{"action":"voted","data":{"contract":"SP3MBWGMCVC9KZ5DTAYFMG1D0AEJCR7NENTM3FTK5.proposal-farm-xusd-usda","end-block-height":"98088","is-ended":false,"no-votes":"0","proposer":"SP3MBWGMCVC9KZ5DTAYFMG1D0AEJCR7NENTM3FTK5","start-block-height":"97080","title":"xUSD/USDA Farming at Arkadiko","url":"https://discord.com/channels/923585641554518026/934026731256422450","yes-votes":"14965111242"},"type":"proposal"}' + '{"action":"voted","data":{"contract":"SP3MBWGMCVC9KZ5DTAYFMG1D0AEJCR7NENTM3FTK5.proposal-farm-xusd-usda","end-block-height":98088,"is-ended":false,"no-votes":0,"proposer":"SP3MBWGMCVC9KZ5DTAYFMG1D0AEJCR7NENTM3FTK5","start-block-height":97080,"title":"xUSD/USDA Farming at Arkadiko","url":"https://discord.com/channels/923585641554518026/934026731256422450","yes-votes":14965111242},"type":"proposal"}' ], [ `(tuple (action "accept-bid") (payload (tuple (action_event_index u1) (bid_amount u22000000) (bid_id u4572) (bidder_address 'SP1JSDAGAYXCTGVMNTD3H2RHP8XYTFR9Q813TPZK8) (collection_id 'SPJW1XE278YMCEYMXB8ZFGJMH8ZVAAEDP2S2PJYG.happy-welsh) (expiration_block u97322) (royalty (tuple (percent u250) (recipient_address 'SP28W4CM7T6RYGXHDA0BQPBYNKYSTSFCP0M6FJB0T))) (seller_address 'SP1NGMS9Z48PRXFAG2MKBSP0PWERF07C0KV9SPJ66) (token_id u907))))`, '0x0c0000000206616374696f6e0d0000000a6163636570742d626964077061796c6f61640c0000000912616374696f6e5f6576656e745f696e64657801000000000000000000000000000000010a6269645f616d6f756e7401000000000000000000000000014fb180066269645f696401000000000000000000000000000011dc0e6269646465725f6164647265737305166596aa0af759a86e95d347116236477da7e137400d636f6c6c656374696f6e5f6964061625c0f5c23a3d463bd4ead1f7c2548a3fb529cdb00b68617070792d77656c73681065787069726174696f6e5f626c6f636b0100000000000000000000000000017c2a07726f79616c74790c000000020770657263656e7401000000000000000000000000000000fa11726563697069656e745f61646472657373051691c23287d1b1e8762d50177b2fd59fb3acbd96050e73656c6c65725f6164647265737305166b0a653f222d8ebd501526bcd816e3b0f01d809e08746f6b656e5f6964010000000000000000000000000000038b', - '{"action":"accept-bid","payload":{"action_event_index":"1","bid_amount":"22000000","bid_id":"4572","bidder_address":"SP1JSDAGAYXCTGVMNTD3H2RHP8XYTFR9Q813TPZK8","collection_id":"SPJW1XE278YMCEYMXB8ZFGJMH8ZVAAEDP2S2PJYG.happy-welsh","expiration_block":"97322","royalty":{"percent":"250","recipient_address":"SP28W4CM7T6RYGXHDA0BQPBYNKYSTSFCP0M6FJB0T"},"seller_address":"SP1NGMS9Z48PRXFAG2MKBSP0PWERF07C0KV9SPJ66","token_id":"907"}}' + '{"action":"accept-bid","payload":{"action_event_index":1,"bid_amount":22000000,"bid_id":4572,"bidder_address":"SP1JSDAGAYXCTGVMNTD3H2RHP8XYTFR9Q813TPZK8","collection_id":"SPJW1XE278YMCEYMXB8ZFGJMH8ZVAAEDP2S2PJYG.happy-welsh","expiration_block":97322,"royalty":{"percent":250,"recipient_address":"SP28W4CM7T6RYGXHDA0BQPBYNKYSTSFCP0M6FJB0T"},"seller_address":"SP1NGMS9Z48PRXFAG2MKBSP0PWERF07C0KV9SPJ66","token_id":907}}' ], [ `(tuple (action "swap-x-for-y") (data (tuple (balance-x u315175871745570) (balance-y u11937495544) (fee-rate-x u300000) (fee-rate-y u300000) (fee-rebate u50000000) (fee-to-address 'SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.multisig-fwp-wstx-wbtc-50-50-v1-01) (oracle-average u95000000) (oracle-enabled true) (oracle-resilient u3871) (pool-token 'SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.fwp-wstx-wbtc-50-50-v1-01) (total-supply u3290909475111))) (object "pool"))`, '0x0c0000000306616374696f6e0d0000000c737761702d782d666f722d7904646174610c0000000b0962616c616e63652d7801000000000000000000011ea699e08e220962616c616e63652d7901000000000000000000000002c787b9f80a6665652d726174652d7801000000000000000000000000000493e00a6665652d726174652d7901000000000000000000000000000493e00a6665652d7265626174650100000000000000000000000002faf0800e6665652d746f2d616464726573730616e685b016b3b6cd9ebf35f38e5ae29392e2acd51d226d756c74697369672d6677702d777374782d776274632d35302d35302d76312d30310e6f7261636c652d617665726167650100000000000000000000000005a995c00e6f7261636c652d656e61626c656403106f7261636c652d726573696c69656e740100000000000000000000000000000f1f0a706f6f6c2d746f6b656e0616e685b016b3b6cd9ebf35f38e5ae29392e2acd51d196677702d777374782d776274632d35302d35302d76312d30310c746f74616c2d737570706c79010000000000000000000002fe397d8127066f626a6563740d00000004706f6f6c', - '{"action":"swap-x-for-y","data":{"balance-x":"315175871745570","balance-y":"11937495544","fee-rate-x":"300000","fee-rate-y":"300000","fee-rebate":"50000000","fee-to-address":"SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.multisig-fwp-wstx-wbtc-50-50-v1-01","oracle-average":"95000000","oracle-enabled":true,"oracle-resilient":"3871","pool-token":"SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.fwp-wstx-wbtc-50-50-v1-01","total-supply":"3290909475111"},"object":"pool"}' + '{"action":"swap-x-for-y","data":{"balance-x":315175871745570,"balance-y":11937495544,"fee-rate-x":300000,"fee-rate-y":300000,"fee-rebate":50000000,"fee-to-address":"SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.multisig-fwp-wstx-wbtc-50-50-v1-01","oracle-average":95000000,"oracle-enabled":true,"oracle-resilient":3871,"pool-token":"SP3K8BC0PPEVCV7NZ6QSRWPQ2JE9E5B6N3PA0KBR9.fwp-wstx-wbtc-50-50-v1-01","total-supply":3290909475111},"object":"pool"}' ], [ `(tuple (action "swap-x-for-y") (data (tuple (balance-x u39248987199) (balance-y u294901858687020) (enabled true) (fee-balance-x u209257577) (fee-balance-y u3188281977630) (fee-to-address (some 'SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR)) (name "wSTX-WELSH") (shares-total u3135066566725) (swap-token 'SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR.arkadiko-swap-token-wstx-welsh))) (object "pair"))`, '0x0c0000000306616374696f6e0d0000000c737761702d782d666f722d7904646174610c000000090962616c616e63652d7801000000000000000000000009236c043f0962616c616e63652d7901000000000000000000010c363087d82c07656e61626c6564030d6665652d62616c616e63652d78010000000000000000000000000c7904690d6665652d62616c616e63652d79010000000000000000000002e6546a2b1e0e6665652d746f2d616464726573730a0516982f3ec112a5f5928a5c96a914bd733793b896a5046e616d650d0000000a775354582d57454c53480c7368617265732d746f74616c010000000000000000000002d9f08770450a737761702d746f6b656e0616982f3ec112a5f5928a5c96a914bd733793b896a51e61726b6164696b6f2d737761702d746f6b656e2d777374782d77656c7368066f626a6563740d0000000470616972', - '{"action":"swap-x-for-y","data":{"balance-x":"39248987199","balance-y":"294901858687020","enabled":true,"fee-balance-x":"209257577","fee-balance-y":"3188281977630","fee-to-address":"SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR","name":"wSTX-WELSH","shares-total":"3135066566725","swap-token":"SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR.arkadiko-swap-token-wstx-welsh"},"object":"pair"}' + '{"action":"swap-x-for-y","data":{"balance-x":39248987199,"balance-y":294901858687020,"enabled":true,"fee-balance-x":209257577,"fee-balance-y":3188281977630,"fee-to-address":"SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR","name":"wSTX-WELSH","shares-total":3135066566725,"swap-token":"SP2C2YFP12AJZB4MABJBAJ55XECVS7E4PMMZ89YZR.arkadiko-swap-token-wstx-welsh"},"object":"pair"}' ], [ `(tuple (attachment (tuple (attachment-index u360918) (hash 0xffcbf70fa7fa89adcbd4d80530d2cc93a3f1d236) (metadata (tuple (name 0x64657369676e6d616e6167656d656e74636f6e73756c74696e67) (namespace 0x627463) (op "name-register") (tx-sender 'SP3JH2C1BW3TY23AW5X12TPBFMSZHTV0TBZ1KH5S9))))))`, '0x0c000000010a6174746163686d656e740c00000003106174746163686d656e742d696e64657801000000000000000000000000000581d604686173680200000014ffcbf70fa7fa89adcbd4d80530d2cc93a3f1d236086d657461646174610c00000004046e616d65020000001a64657369676e6d616e6167656d656e74636f6e73756c74696e67096e616d6573706163650200000003627463026f700d0000000d6e616d652d72656769737465720974782d73656e6465720516e511302be0f5e10d5c2f422d596fa67f1d6c1a5f', - '{"attachment":{"attachment-index":"360918","hash":{"hex":"0xffcbf70fa7fa89adcbd4d80530d2cc93a3f1d236","utf8":"���\\u000f�������\\u00050�̓���6"},"metadata":{"name":{"hex":"0x64657369676e6d616e6167656d656e74636f6e73756c74696e67","utf8":"designmanagementconsulting"},"namespace":{"hex":"0x627463","utf8":"btc"},"op":"name-register","tx-sender":"SP3JH2C1BW3TY23AW5X12TPBFMSZHTV0TBZ1KH5S9"}}}' + '{"attachment":{"attachment-index":360918,"hash":{"hex":"0xffcbf70fa7fa89adcbd4d80530d2cc93a3f1d236","utf8":"���\\u000f�������\\u00050�̓���6"},"metadata":{"name":{"hex":"0x64657369676e6d616e6167656d656e74636f6e73756c74696e67","utf8":"designmanagementconsulting"},"namespace":{"hex":"0x627463","utf8":"btc"},"op":"name-register","tx-sender":"SP3JH2C1BW3TY23AW5X12TPBFMSZHTV0TBZ1KH5S9"}}}' ], [ `(tuple (amountStacked u7420000000000) (cityId u1) (cityName "mia") (cityTreasury 'SP8A9HZ3PKST0S42VM9523Z9NV42SZ026V4K39WH.ccd002-treasury-mia-stacking) (event "stacking") (firstCycle u54) (lastCycle u59) (lockPeriod u6) (userId u136))`, '0x0c000000090d616d6f756e74537461636b6564010000000000000000000006bf9a76d80006636974794964010000000000000000000000000000000108636974794e616d650d000000036d69610c636974795472656173757279061610a4c7e3b4f3a06482dd12510fe9aec82cfc02361c6363643030322d74726561737572792d6d69612d737461636b696e67056576656e740d00000008737461636b696e670a66697273744379636c650100000000000000000000000000000036096c6173744379636c65010000000000000000000000000000003b0a6c6f636b506572696f640100000000000000000000000000000006067573657249640100000000000000000000000000000088', - '{"amountStacked":"7420000000000","cityId":"1","cityName":"mia","cityTreasury":"SP8A9HZ3PKST0S42VM9523Z9NV42SZ026V4K39WH.ccd002-treasury-mia-stacking","event":"stacking","firstCycle":"54","lastCycle":"59","lockPeriod":"6","userId":"136"}' + '{"amountStacked":7420000000000,"cityId":1,"cityName":"mia","cityTreasury":"SP8A9HZ3PKST0S42VM9523Z9NV42SZ026V4K39WH.ccd002-treasury-mia-stacking","event":"stacking","firstCycle":54,"lastCycle":59,"lockPeriod":6,"userId":136}' ], [ '0x3231333431363630323537', From 874ad8b41b1410dc9e3e6fc9f279fa93fccb5462 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Mon, 3 Apr 2023 16:20:48 +0200 Subject: [PATCH 10/11] fix: jsonpath disallowed operations --- package-lock.json | 30 -------- package.json | 1 - src/api/query-helpers.ts | 142 ++++++++++-------------------------- src/api/routes/contract.ts | 12 --- src/tests/jsonpath-tests.ts | 98 ++++++++++--------------- 5 files changed, 76 insertions(+), 207 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7cfb39c401..d56b073d3d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,7 +42,6 @@ "getopts": "2.3.0", "http-proxy-middleware": "2.0.1", "jsonc-parser": "3.0.0", - "jsonpathly": "1.7.5", "jsonrpc-lite": "2.2.0", "lru-cache": "6.0.0", "micro-base58": "0.5.1", @@ -3382,14 +3381,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/antlr4": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/antlr4/-/antlr4-4.12.0.tgz", - "integrity": "sha512-23iB5IzXJZRZeK9TigzUyrNc9pSmNqAerJRBcNq1ETrmttMWRgaYZzC561IgEO3ygKsDJTYDTozABXa4b/fTQQ==", - "engines": { - "node": ">=16" - } - }, "node_modules/anymatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", @@ -8081,14 +8072,6 @@ "node >= 0.2.0" ] }, - "node_modules/jsonpathly": { - "version": "1.7.5", - "resolved": "https://registry.npmjs.org/jsonpathly/-/jsonpathly-1.7.5.tgz", - "integrity": "sha512-/gyDvxnTrwMPRDNsKKG5wNgRri8SlBCw84w210I2yl1AiPCmA7vyhWVe+dhobDatdc5hA6MUh1F5QhzazDttHg==", - "dependencies": { - "antlr4": "^4.10.1" - } - }, "node_modules/jsonrpc-lite": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/jsonrpc-lite/-/jsonrpc-lite-2.2.0.tgz", @@ -14671,11 +14654,6 @@ "color-convert": "^2.0.1" } }, - "antlr4": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/antlr4/-/antlr4-4.12.0.tgz", - "integrity": "sha512-23iB5IzXJZRZeK9TigzUyrNc9pSmNqAerJRBcNq1ETrmttMWRgaYZzC561IgEO3ygKsDJTYDTozABXa4b/fTQQ==" - }, "anymatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", @@ -18263,14 +18241,6 @@ "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=", "dev": true }, - "jsonpathly": { - "version": "1.7.5", - "resolved": "https://registry.npmjs.org/jsonpathly/-/jsonpathly-1.7.5.tgz", - "integrity": "sha512-/gyDvxnTrwMPRDNsKKG5wNgRri8SlBCw84w210I2yl1AiPCmA7vyhWVe+dhobDatdc5hA6MUh1F5QhzazDttHg==", - "requires": { - "antlr4": "^4.10.1" - } - }, "jsonrpc-lite": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/jsonrpc-lite/-/jsonrpc-lite-2.2.0.tgz", diff --git a/package.json b/package.json index b59ee7cc5e..8a1720d8e0 100644 --- a/package.json +++ b/package.json @@ -150,7 +150,6 @@ "getopts": "2.3.0", "http-proxy-middleware": "2.0.1", "jsonc-parser": "3.0.0", - "jsonpathly": "1.7.5", "jsonrpc-lite": "2.2.0", "lru-cache": "6.0.0", "micro-base58": "0.5.1", diff --git a/src/api/query-helpers.ts b/src/api/query-helpers.ts index 186bf1e252..eb156858ff 100644 --- a/src/api/query-helpers.ts +++ b/src/api/query-helpers.ts @@ -3,8 +3,6 @@ import { NextFunction, Request, Response } from 'express'; import { has0xPrefix, hexToBuffer, parseEventTypeStrings, isValidPrincipal } from './../helpers'; import { InvalidRequestError, InvalidRequestErrorType } from '../errors'; import { DbEventTypeId } from './../datastore/common'; -import * as jp from 'jsonpathly'; -import { JsonPathElement } from 'jsonpathly/dist/types/parser/types'; function handleBadRequest(res: Response, next: NextFunction, errorMessage: string): never { const error = new InvalidRequestError(errorMessage, InvalidRequestErrorType.bad_request); @@ -18,7 +16,7 @@ export function validateJsonPathQuery( res: Response, next: NextFunction, paramName: string, - args: { paramRequired: TRequired; maxCharLength: number; maxOperations: number } + args: { paramRequired: TRequired; maxCharLength: number } ): TRequired extends true ? string | never : string | null { if (!(paramName in req.query)) { if (args.paramRequired) { @@ -37,7 +35,6 @@ export function validateJsonPathQuery( } const maxCharLength = args.maxCharLength; - const maxOperations = args.maxOperations; if (jsonPathInput.length > maxCharLength) { handleBadRequest( @@ -47,24 +44,51 @@ export function validateJsonPathQuery( ); } - const jsonPathComplexity = getJsonPathComplexity(jsonPathInput); - if ('error' in jsonPathComplexity) { + const disallowedOperation = containsDisallowedJsonPathOperation(jsonPathInput); + if (disallowedOperation) { handleBadRequest( res, next, - `JsonPath parameter '${paramName}' is invalid: ${jsonPathComplexity.error.message}` - ); - } - if (jsonPathComplexity.operations > maxOperations) { - handleBadRequest( - res, - next, - `JsonPath parameter '${paramName}' is invalid: operations exceeded, max=${maxOperations}, received=${jsonPathComplexity.operations}` + `JsonPath parameter '${paramName}' is invalid: contains disallowed operation '${disallowedOperation.operation}'` ); } + return jsonPathInput; } +/** + * Disallow operations that could be used to perform expensive queries. + * See https://www.postgresql.org/docs/14/functions-json.html + */ +export function containsDisallowedJsonPathOperation( + jsonPath: string +): false | { operation: string } { + const normalizedPath = jsonPath.replace(/\s+/g, '').toLowerCase(); + const hasDisallowedOperations: [() => boolean, string][] = [ + [() => normalizedPath.includes('.*'), '.* wildcard accessor'], + [() => normalizedPath.includes('[*]'), '[*] wildcard array accessor'], + [() => /\[\d+to(\d+|last)\]/.test(normalizedPath), '[n to m] array range accessor'], + [() => /\[[^\]]*\([^\)]*\)[^\]]*\]/.test(normalizedPath), '[()] array expression accessor'], + [() => normalizedPath.includes('.type('), '.type()'], + [() => normalizedPath.includes('.size('), '.size()'], + [() => normalizedPath.includes('.double('), '.double()'], + [() => normalizedPath.includes('.ceiling('), '.ceiling()'], + [() => normalizedPath.includes('.floor('), '.floor()'], + [() => normalizedPath.includes('.abs('), '.abs()'], + [() => normalizedPath.includes('.datetime('), '.datetime()'], + [() => normalizedPath.includes('.keyvalue('), '.keyvalue()'], + [() => normalizedPath.includes('isunknown'), 'is unknown'], + [() => normalizedPath.includes('like_regex'), 'like_regex'], + [() => normalizedPath.includes('startswith'), 'starts with'], + ]; + for (const [hasDisallowedOperation, disallowedOperationName] of hasDisallowedOperations) { + if (hasDisallowedOperation()) { + return { operation: disallowedOperationName }; + } + } + return false; +} + export function booleanValueForParam( req: Request, res: Response, @@ -99,96 +123,6 @@ export function booleanValueForParam( ); } -/** - * Determine how complex a jsonpath expression is. This uses the jsonpathly library to parse the expression into a tree which - * is then evaluated to determine the number of operations in the expression. - * Returns an error if the expression is invalid or can't be parsed. - */ -export function getJsonPathComplexity(jsonpath: string): { error: Error } | { operations: number } { - if (jsonpath.trim().length === 0) { - return { error: new Error(`Invalid jsonpath, empty expression string`) }; - } - let pathTree: JsonPathElement | null; - try { - pathTree = jp.parse(jsonpath); - } catch (error: any) { - return { error: error instanceof Error ? error : new Error(error.toString()) }; - } - - if (!pathTree) { - return { error: new Error(`Invalid jsonpath, evaluated as null`) }; - } - - let operations: number; - try { - operations = countJsonPathOperations(pathTree); - } catch (error: any) { - return { error: error instanceof Error ? error : new Error(error.toString()) }; - } - - return { operations }; -} - -export function countJsonPathOperations(element: JsonPathElement): number { - let count = 0; - const stack: { - element: JsonPathElement; - operationToAdd: number; - }[] = [{ element, operationToAdd: 0 }]; - - while (stack.length > 0) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const { element, operationToAdd } = stack.pop()!; - count += operationToAdd; - - switch (element.type) { - case 'operation': - case 'logicalExpression': - case 'comparator': - stack.push({ element: element.left, operationToAdd: 1 }); - if (element.right) { - stack.push({ element: element.right, operationToAdd: 0 }); - } - break; - case 'groupOperation': - case 'groupExpression': - case 'notExpression': - case 'filterExpression': - case 'bracketExpression': - stack.push({ element: element.value, operationToAdd: 1 }); - break; - case 'subscript': - stack.push({ element: element.value, operationToAdd: 0 }); - case 'root': - case 'current': - if (element.next) { - stack.push({ element: element.next, operationToAdd: 0 }); - } - break; - case 'unions': - case 'indexes': - for (const value of element.values) { - stack.push({ element: value, operationToAdd: 1 }); - } - break; - case 'slices': - case 'dotdot': - case 'dot': - case 'bracketMember': - case 'wildcard': - case 'numericLiteral': - case 'stringLiteral': - case 'identifier': - case 'value': - // These element types don't have nested elements to process - count++; - break; - } - } - - return count; -} - /** * Determines if the query parameters of a request are intended to include unanchored tx data. * If an error is encountered while parsing the query param then a 400 response with an error message diff --git a/src/api/routes/contract.ts b/src/api/routes/contract.ts index 8643b39f61..9be17cc76a 100644 --- a/src/api/routes/contract.ts +++ b/src/api/routes/contract.ts @@ -55,22 +55,10 @@ export function createContractRouter(db: PgStore): express.Router { const limit = getPagingQueryLimit(ResourceType.Contract, req.query.limit); const offset = parsePagingQueryInput(req.query.offset ?? 0); - /* const filterPath = validateJsonPathQuery(req, res, next, 'filter_path', { paramRequired: false, maxCharLength: 200, - maxOperations: 6, }); - */ - - const filterPath = (req.query['filter_path'] ?? null) as string | null; - const maxFilterPathCharLength = 200; - if (filterPath && filterPath.length > maxFilterPathCharLength) { - res.status(400).json({ - error: `'filter_path' query param value exceeds ${maxFilterPathCharLength} character limit`, - }); - return; - } const containsJsonQuery = req.query['contains']; if (containsJsonQuery && typeof containsJsonQuery !== 'string') { diff --git a/src/tests/jsonpath-tests.ts b/src/tests/jsonpath-tests.ts index 0553d8510b..21d27fcfa5 100644 --- a/src/tests/jsonpath-tests.ts +++ b/src/tests/jsonpath-tests.ts @@ -1,72 +1,50 @@ +/* eslint-disable @typescript-eslint/no-unsafe-return */ /* eslint-disable prettier/prettier */ -import { getJsonPathComplexity } from '../api/query-helpers'; +import { containsDisallowedJsonPathOperation } from '../api/query-helpers'; describe('jsonpath tests', () => { - const jsonPathVectors: [string, number][] = [ - [ '$.key', 1 ], - [ '$.key.subkey', 2 ], - [ '$.array[*]', 3 ], - [ '$.array[*].subkey', 4 ], - [ '$.key1.key2.subkey', 3 ], - [ '$.key1.key2.array[*].subkey', 6 ], - [ '$.key.subkey1.subkey2.subkey3.subkey4', 5 ], - [ '$.array[*].subkey1.subkey2.subkey3.subkey4', 7 ], - [ '$.key1.key2.subkey1.subkey2.subkey3.subkey4', 6 ], - [ '$.key1.key2.array[*].subkey1.subkey2.subkey3.subkey4', 9 ], - [ '$.store.book[*].author', 5 ], - [ '$.store.book[*].title', 5 ], - [ '$.store.book[0,1].title', 8 ], - [ '$.store.book[0:2].title', 5 ], - [ '$.store.book[-1:].title', 5 ], - [ '$.store.book[-2].author', 4 ], - [ '$.store..price', 2 ], - [ '$..book[2]', 2 ], - [ '$..book[-2]', 2 ], - [ '$..book[0,1]', 6 ], - [ '$..book[:2]', 3 ], - [ '$..book[1:2]', 3 ], - [ '$..book[-2:]', 3 ], - [ '$..book[2:]', 3 ], - [ '$.store.book[?(@.isbn)]', 5 ], - [ '$..book[?(@.price<10)]', 6 ], - [ '$..book[?(@.price==8.95)]', 6 ], - [ '$..book[?(@.price!=8.95)]', 6 ], - [ '$..book[?(@.price<30 && @.category=="fiction")]', 10 ], - [ '$..book[?(@.price<30 || @.category=="fiction")]', 10 ], - [ '$.store.book[0:2].author[0:3].name[0:3].title', 11 ], - [ '$.store..author[0:2].books[0:2].title', 8 ], - [ '$.store.book[?(@.isbn)].publisher[0:2].name', 9 ], - [ '$.store.book[?(@.price<10)].author[0:2].books[0:2].title', 14 ], - [ '$..book[0:2].author[0:3].name[0:3].title', 10 ], - [ '$..book[-2:].author[0:3].name[0:3].title', 10 ], - [ '$..book[2:].author[0:3].name[0:3].title', 10 ], - [ '$..book[?(@.price<30 && @.category=="fiction")].author[0:2].books[0:2].title', 17 ], - [ '$..book[?(@.price<30 || @.category=="fiction")].author[0:2].books[0:2].title', 17 ], - [ '$.store.book[0:2].title[0:3].author[0:3].name', 11 ], - [ '$.store.book[?(@.isbn)].publisher[0:2].name[0:3].address', 12 ], - [ '$..book[0:2].title[0:3].author[0:3].name[0:3].address', 13 ], - [ '$..book[-2:].title[0:3].author[0:3].name[0:3].address', 13 ], - [ '$..book[2:].title[0:3].author[0:3].name[0:3].address', 13 ], - [ '$..book[?(@.price<30 && @.category=="fiction")].title[0:3].author[0:2].name[0:3].address', 20 ], - [ '$..book[?(@.price<30 || @.category=="fiction")].title[0:3].author[0:2].name[0:3].address', 20 ], - [ '$.store..author[0:2].books[0:2].title[0:3].publisher[0:2].name', 14 ], - [ '$.store.book[?(@.isbn)].publisher[0:2].name[0:3].address[0:2].city', 15 ], + const jsonPathVectors: [string, string | null][] = [ + [ '$.track.segments.location', null ], + [ '$[0] + 3', null ], + [ '+ $.x', null ], + [ '7 - $[0]', null ], + [ '- $.x', null ], + [ '2 * $[0]', null ], + [ '$[0] / 2', null ], + [ '$[0] % 10', null ], + [ '$.**.HR', '.* wildcard accessor' ], + [ '$.* ? (@.v == "a")', '.* wildcard accessor' ], + [ '$.** ? (@.v == "a")', '.* wildcard accessor' ], + [ '$.track.segments[*].location', '[*] wildcard array accessor' ], + [ '$.[1 to 37634].a', '[n to m] array range accessor' ], + [ '$.[555 to last].a', '[n to m] array range accessor' ], + [ '$.[(3 + 4) to last].a', '[()] array expression accessor' ], + [ '$.[1 to (3 + 4)].a', '[()] array expression accessor' ], + [ '$.t.type()', '.type()' ], + [ '$.m.size()', '.size()' ], + [ '$.len.double() * 2', '.double()' ], + [ '$.h.ceiling()', '.ceiling()' ], + [ '$.h.floor()', '.floor()' ], + [ '$.z.abs()', '.abs()' ], + [ '$.a ? (@.datetime() < "2015-08-2".datetime())', '.datetime()' ], + [ '$.a.datetime("HH24:MI")', '.datetime()' ], + [ '$.keyvalue()', '.keyvalue()' ], + [ '$.a ? (@ like_regex "^abc")', 'like_regex' ], + [ '$.a ? (@ starts with "John")', 'starts with' ], + [ '$.a ? ((@ > 0) is unknown)', 'is unknown' ] ]; - test.each(jsonPathVectors)('test jsonpath operation complexity: %p', (input, operations) => { - const complexity = getJsonPathComplexity(input); - expect(complexity).toEqual({ operations }); + test.each(jsonPathVectors)('test jsonpath operation complexity: %p', (input, result) => { + const disallowed = containsDisallowedJsonPathOperation(input); + expect(disallowed).toEqual(result === null ? false : { operation: result }); }); /* - test.skip('generate vector data', () => { - const result = postgresExampleVectors.map(([input]) => { - const complexity = getJsonPathComplexity(input); - if ('error' in complexity) { - return [input, complexity.error.message]; - } - return [input, complexity.operations]; + test('generate vector data', () => { + const result = postgresExampleVectors.map((input) => { + const complexity = containsDisallowedJsonPathOperation(input); + return [input, complexity ? complexity.operation : null]; }); console.log(result); }); From 6d94c0e7e415cfd936c8547b7585ace026b48f28 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Fri, 14 Apr 2023 15:00:14 +0200 Subject: [PATCH 11/11] feat: use jsonpath-pg lib to accurately determine expression complexity and ban some operations --- package-lock.json | 11 +++ package.json | 1 + src/api/query-helpers.ts | 132 ++++++++++++++++++++++++++++-------- src/tests/jsonpath-tests.ts | 61 +++++++++-------- 4 files changed, 150 insertions(+), 55 deletions(-) diff --git a/package-lock.json b/package-lock.json index d56b073d3d..243216bf75 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,6 +42,7 @@ "getopts": "2.3.0", "http-proxy-middleware": "2.0.1", "jsonc-parser": "3.0.0", + "jsonpath-pg": "1.0.1", "jsonrpc-lite": "2.2.0", "lru-cache": "6.0.0", "micro-base58": "0.5.1", @@ -8072,6 +8073,11 @@ "node >= 0.2.0" ] }, + "node_modules/jsonpath-pg": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/jsonpath-pg/-/jsonpath-pg-1.0.1.tgz", + "integrity": "sha512-fx0cqOxczvh2HdyBGSRrTfzfRTmvS4gMFlOb13Q96c0TOv7f9uzEmdUE1TWhBJhanbOtxA3Oy11LNd87L6Nnrg==" + }, "node_modules/jsonrpc-lite": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/jsonrpc-lite/-/jsonrpc-lite-2.2.0.tgz", @@ -18241,6 +18247,11 @@ "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=", "dev": true }, + "jsonpath-pg": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/jsonpath-pg/-/jsonpath-pg-1.0.1.tgz", + "integrity": "sha512-fx0cqOxczvh2HdyBGSRrTfzfRTmvS4gMFlOb13Q96c0TOv7f9uzEmdUE1TWhBJhanbOtxA3Oy11LNd87L6Nnrg==" + }, "jsonrpc-lite": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/jsonrpc-lite/-/jsonrpc-lite-2.2.0.tgz", diff --git a/package.json b/package.json index 8a1720d8e0..890a4b83c2 100644 --- a/package.json +++ b/package.json @@ -150,6 +150,7 @@ "getopts": "2.3.0", "http-proxy-middleware": "2.0.1", "jsonc-parser": "3.0.0", + "jsonpath-pg": "1.0.1", "jsonrpc-lite": "2.2.0", "lru-cache": "6.0.0", "micro-base58": "0.5.1", diff --git a/src/api/query-helpers.ts b/src/api/query-helpers.ts index eb156858ff..c42c50f7c7 100644 --- a/src/api/query-helpers.ts +++ b/src/api/query-helpers.ts @@ -3,6 +3,7 @@ import { NextFunction, Request, Response } from 'express'; import { has0xPrefix, hexToBuffer, parseEventTypeStrings, isValidPrincipal } from './../helpers'; import { InvalidRequestError, InvalidRequestErrorType } from '../errors'; import { DbEventTypeId } from './../datastore/common'; +import { jsonpathToAst, JsonpathAst, JsonpathItem } from 'jsonpath-pg'; function handleBadRequest(res: Response, next: NextFunction, errorMessage: string): never { const error = new InvalidRequestError(errorMessage, InvalidRequestErrorType.bad_request); @@ -44,12 +45,18 @@ export function validateJsonPathQuery( ); } - const disallowedOperation = containsDisallowedJsonPathOperation(jsonPathInput); - if (disallowedOperation) { + let ast: JsonpathAst; + try { + ast = jsonpathToAst(jsonPathInput); + } catch (error) { + handleBadRequest(res, next, `JsonPath parameter '${paramName}' is invalid: ${error}`); + } + const astComplexity = calculateJsonpathComplexity(ast); + if (typeof astComplexity !== 'number') { handleBadRequest( res, next, - `JsonPath parameter '${paramName}' is invalid: contains disallowed operation '${disallowedOperation.operation}'` + `JsonPath parameter '${paramName}' is invalid: contains disallowed operation '${astComplexity.disallowedOperation}'` ); } @@ -57,36 +64,105 @@ export function validateJsonPathQuery( } /** + * Scan the a jsonpath expression to determine complexity. * Disallow operations that could be used to perform expensive queries. * See https://www.postgresql.org/docs/14/functions-json.html */ -export function containsDisallowedJsonPathOperation( - jsonPath: string -): false | { operation: string } { - const normalizedPath = jsonPath.replace(/\s+/g, '').toLowerCase(); - const hasDisallowedOperations: [() => boolean, string][] = [ - [() => normalizedPath.includes('.*'), '.* wildcard accessor'], - [() => normalizedPath.includes('[*]'), '[*] wildcard array accessor'], - [() => /\[\d+to(\d+|last)\]/.test(normalizedPath), '[n to m] array range accessor'], - [() => /\[[^\]]*\([^\)]*\)[^\]]*\]/.test(normalizedPath), '[()] array expression accessor'], - [() => normalizedPath.includes('.type('), '.type()'], - [() => normalizedPath.includes('.size('), '.size()'], - [() => normalizedPath.includes('.double('), '.double()'], - [() => normalizedPath.includes('.ceiling('), '.ceiling()'], - [() => normalizedPath.includes('.floor('), '.floor()'], - [() => normalizedPath.includes('.abs('), '.abs()'], - [() => normalizedPath.includes('.datetime('), '.datetime()'], - [() => normalizedPath.includes('.keyvalue('), '.keyvalue()'], - [() => normalizedPath.includes('isunknown'), 'is unknown'], - [() => normalizedPath.includes('like_regex'), 'like_regex'], - [() => normalizedPath.includes('startswith'), 'starts with'], - ]; - for (const [hasDisallowedOperation, disallowedOperationName] of hasDisallowedOperations) { - if (hasDisallowedOperation()) { - return { operation: disallowedOperationName }; +export function calculateJsonpathComplexity( + ast: JsonpathAst +): number | { disallowedOperation: string } { + let totalComplexity = 0; + const stack: JsonpathItem[] = [...ast.expr]; + + while (stack.length > 0) { + const item = stack.pop() as JsonpathItem; + + switch (item.type) { + // Recursive lookup operations not allowed + case '[*]': + case '.*': + case '.**': + // string "starts with" operation not allowed + case 'starts with': + // string regex operations not allowed + case 'like_regex': + // Index range operations not allowed + case 'last': + // Type coercion not allowed + case 'is_unknown': + // Item method operations not allowed + case 'type': + case 'size': + case 'double': + case 'ceiling': + case 'floor': + case 'abs': + case 'datetime': + case 'keyvalue': + return { disallowedOperation: item.type }; + + // Array index accessor + case '[subscript]': + if (item.elems.some(elem => elem.to.length > 0)) { + // Range operations not allowed + return { disallowedOperation: '[n to m] array range accessor' }; + } else { + totalComplexity += 1; + stack.push(...item.elems.flatMap(elem => elem.from)); + } + break; + + // Simple path navigation operations + case '$': + case '@': + break; + + // Path literals + case '$variable': + case '.key': + case 'null': + case 'string': + case 'numeric': + case 'bool': + totalComplexity += 1; + break; + + // Binary operations + case '&&': + case '||': + case '==': + case '!=': + case '<': + case '>': + case '<=': + case '>=': + case '+': + case '-': + case '*': + case '/': + case '%': + totalComplexity += 3; + stack.push(...item.left, ...item.right); + break; + + // Unary operations + case '?': + case '!': + case '+unary': + case '-unary': + case 'exists': + totalComplexity += 2; + stack.push(...item.arg); + break; + + default: + // @ts-expect-error - exhaustive switch + const unexpectedTypeID = item.type; + throw new Error(`Unexpected jsonpath expression type ID: ${unexpectedTypeID}`); } } - return false; + + return totalComplexity; } export function booleanValueForParam( diff --git a/src/tests/jsonpath-tests.ts b/src/tests/jsonpath-tests.ts index 21d27fcfa5..2d3c6a667c 100644 --- a/src/tests/jsonpath-tests.ts +++ b/src/tests/jsonpath-tests.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-return */ /* eslint-disable prettier/prettier */ -import { containsDisallowedJsonPathOperation } from '../api/query-helpers'; +import jsonpathToAst from 'jsonpath-pg'; +import { calculateJsonpathComplexity } from '../api/query-helpers'; describe('jsonpath tests', () => { @@ -13,41 +14,47 @@ describe('jsonpath tests', () => { [ '2 * $[0]', null ], [ '$[0] / 2', null ], [ '$[0] % 10', null ], - [ '$.**.HR', '.* wildcard accessor' ], - [ '$.* ? (@.v == "a")', '.* wildcard accessor' ], - [ '$.** ? (@.v == "a")', '.* wildcard accessor' ], - [ '$.track.segments[*].location', '[*] wildcard array accessor' ], - [ '$.[1 to 37634].a', '[n to m] array range accessor' ], - [ '$.[555 to last].a', '[n to m] array range accessor' ], - [ '$.[(3 + 4) to last].a', '[()] array expression accessor' ], - [ '$.[1 to (3 + 4)].a', '[()] array expression accessor' ], - [ '$.t.type()', '.type()' ], - [ '$.m.size()', '.size()' ], - [ '$.len.double() * 2', '.double()' ], - [ '$.h.ceiling()', '.ceiling()' ], - [ '$.h.floor()', '.floor()' ], - [ '$.z.abs()', '.abs()' ], - [ '$.a ? (@.datetime() < "2015-08-2".datetime())', '.datetime()' ], - [ '$.a.datetime("HH24:MI")', '.datetime()' ], - [ '$.keyvalue()', '.keyvalue()' ], + [ '$[1,3,4,6]', null ], + [ '$[555 to LAST]', '[n to m] array range accessor' ], + [ '$[1 to 37634].a', '[n to m] array range accessor' ], + [ '$[3,4 to last].a', '[n to m] array range accessor' ], + [ '$[(3 + 4) to last].a', '[n to m] array range accessor' ], + [ '$[1 to (3 + 4)].a', '[n to m] array range accessor' ], + [ '$.**.HR', '.**' ], + [ '$.* ? (@.v == "a")', '.*' ], + [ '$.** ? (@.v == "a")', '.**' ], + [ '$.track.segments[*].location', '[*]' ], + [ '$.t.type()', 'type' ], + [ '$.m.size()', 'size' ], + [ '$.len.double() * 2', 'double' ], + [ '$.h.ceiling()', 'ceiling' ], + [ '$.h.floor()', 'floor' ], + [ '$.z.abs()', 'abs' ], + [ '$.a ? (@.datetime() < "2015-08-2".datetime())', 'datetime' ], + [ '$.a.datetime("HH24:MI")', 'datetime' ], + [ '$.keyvalue()', 'keyvalue' ], [ '$.a ? (@ like_regex "^abc")', 'like_regex' ], [ '$.a ? (@ starts with "John")', 'starts with' ], - [ '$.a ? ((@ > 0) is unknown)', 'is unknown' ] + [ '$.a ? ((@ > 0) is unknown)', 'is_unknown' ] ]; test.each(jsonPathVectors)('test jsonpath operation complexity: %p', (input, result) => { - const disallowed = containsDisallowedJsonPathOperation(input); - expect(disallowed).toEqual(result === null ? false : { operation: result }); + const ast = jsonpathToAst(input); + const disallowed = calculateJsonpathComplexity(ast); + if (typeof disallowed === 'number') { + expect(result).toBe(null); + } else { + expect(disallowed.disallowedOperation).toBe(result); + } }); - /* - test('generate vector data', () => { - const result = postgresExampleVectors.map((input) => { - const complexity = containsDisallowedJsonPathOperation(input); - return [input, complexity ? complexity.operation : null]; + test.skip('generate vector data', () => { + const result = jsonPathVectors.map(([input]) => { + const ast = jsonpathToAst(input); + const complexity = calculateJsonpathComplexity(ast); + return [input, typeof complexity !== 'number' ? complexity.disallowedOperation : null]; }); console.log(result); }); - */ });