diff --git a/CHANGES.md b/CHANGES.md index 497fedd..d5382a6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,7 @@ ## Unreleased - Export `Statement` in both Python and Javascript target - Fixed query parsing when expression includes special characters like `\n`, `\r`, or `\t` +- Fixed sqlparse crash on missing error context ## 2024/09/18 v0.0.7 - Improve error matching on single statement diff --git a/cratedb_sqlparse_js/cratedb_sqlparse/parser.js b/cratedb_sqlparse_js/cratedb_sqlparse/parser.js index 0aef943..2099f7a 100644 --- a/cratedb_sqlparse_js/cratedb_sqlparse/parser.js +++ b/cratedb_sqlparse_js/cratedb_sqlparse/parser.js @@ -124,11 +124,23 @@ class ExceptionCollectorListener extends ErrorListener { syntaxError(recognizer, offendingSymbol, line, column, msg, e) { super.syntaxError(recognizer, offendingSymbol, line, column, msg, e); - const error = new ParseError( - e.ctx.parser.getTokenStream().getText(new Interval( + let query; + + if (e !== null) { + query = e.ctx.parser.getTokenStream().getText(new Interval( e.ctx.start, e.offendingToken.tokenIndex) - ), + ) + + } else { + + let min_to_check = Math.max(1, offendingSymbol.tokenIndex - 2) + let tokens = recognizer.getTokenStream().tokens.slice(min_to_check, offendingSymbol.tokenIndex + 1) + query = tokens.map((el) => el.text).join("") + } + + const error = new ParseError( + query, msg, offendingSymbol, e @@ -221,6 +233,7 @@ export function sqlparse(query, raise_exception = false) { const statementsContext = tree.children.filter((children) => children instanceof SqlBaseParser.StatementContext) let statements = [] + for (const statementContext of statementsContext) { let stmt = new Statement(statementContext) diff --git a/cratedb_sqlparse_js/package-lock.json b/cratedb_sqlparse_js/package-lock.json index 2b3838b..3e5da37 100644 --- a/cratedb_sqlparse_js/package-lock.json +++ b/cratedb_sqlparse_js/package-lock.json @@ -15,7 +15,7 @@ "jsdoc": "^4.0.3", "terser": "^5.34.1", "vite": "^5.4.8", - "vitest": "^2.1.1" + "vitest": "^2.1.2" } }, "node_modules/@babel/parser": { @@ -716,13 +716,13 @@ } }, "node_modules/@vitest/expect": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.1.tgz", - "integrity": "sha512-YeueunS0HiHiQxk+KEOnq/QMzlUuOzbU1Go+PgAsHvvv3tUkJPm9xWt+6ITNTlzsMXUjmgm5T+U7KBPK2qQV6w==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.2.tgz", + "integrity": "sha512-FEgtlN8mIUSEAAnlvn7mP8vzaWhEaAEvhSXCqrsijM7K6QqjB11qoRZYEd4AKSCDz8p0/+yH5LzhZ47qt+EyPg==", "dev": true, "dependencies": { - "@vitest/spy": "2.1.1", - "@vitest/utils": "2.1.1", + "@vitest/spy": "2.1.2", + "@vitest/utils": "2.1.2", "chai": "^5.1.1", "tinyrainbow": "^1.2.0" }, @@ -731,9 +731,9 @@ } }, "node_modules/@vitest/mocker": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-2.1.1.tgz", - "integrity": "sha512-LNN5VwOEdJqCmJ/2XJBywB11DLlkbY0ooDJW3uRX5cZyYCrc4PI/ePX0iQhE3BiEGiQmK4GE7Q/PqCkkaiPnrA==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-2.1.2.tgz", + "integrity": "sha512-ExElkCGMS13JAJy+812fw1aCv2QO/LBK6CyO4WOPAzLTmve50gydOlWhgdBJPx2ztbADUq3JVI0C5U+bShaeEA==", "dev": true, "dependencies": { "@vitest/spy": "^2.1.0-beta.1", @@ -744,7 +744,7 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "@vitest/spy": "2.1.1", + "@vitest/spy": "2.1.2", "msw": "^2.3.5", "vite": "^5.0.0" }, @@ -758,9 +758,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.1.1.tgz", - "integrity": "sha512-SjxPFOtuINDUW8/UkElJYQSFtnWX7tMksSGW0vfjxMneFqxVr8YJ979QpMbDW7g+BIiq88RAGDjf7en6rvLPPQ==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.1.2.tgz", + "integrity": "sha512-FIoglbHrSUlOJPDGIrh2bjX1sNars5HbxlcsFKCtKzu4+5lpsRhOCVcuzp0fEhAGHkPZRIXVNzPcpSlkoZ3LuA==", "dev": true, "dependencies": { "tinyrainbow": "^1.2.0" @@ -770,12 +770,12 @@ } }, "node_modules/@vitest/runner": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.1.1.tgz", - "integrity": "sha512-uTPuY6PWOYitIkLPidaY5L3t0JJITdGTSwBtwMjKzo5O6RCOEncz9PUN+0pDidX8kTHYjO0EwUIvhlGpnGpxmA==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.1.2.tgz", + "integrity": "sha512-UCsPtvluHO3u7jdoONGjOSil+uON5SSvU9buQh3lP7GgUXHp78guN1wRmZDX4wGK6J10f9NUtP6pO+SFquoMlw==", "dev": true, "dependencies": { - "@vitest/utils": "2.1.1", + "@vitest/utils": "2.1.2", "pathe": "^1.1.2" }, "funding": { @@ -783,12 +783,12 @@ } }, "node_modules/@vitest/snapshot": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.1.1.tgz", - "integrity": "sha512-BnSku1WFy7r4mm96ha2FzN99AZJgpZOWrAhtQfoxjUU5YMRpq1zmHRq7a5K9/NjqonebO7iVDla+VvZS8BOWMw==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.1.2.tgz", + "integrity": "sha512-xtAeNsZ++aRIYIUsek7VHzry/9AcxeULlegBvsdLncLmNCR6tR8SRjn8BbDP4naxtccvzTqZ+L1ltZlRCfBZFA==", "dev": true, "dependencies": { - "@vitest/pretty-format": "2.1.1", + "@vitest/pretty-format": "2.1.2", "magic-string": "^0.30.11", "pathe": "^1.1.2" }, @@ -797,9 +797,9 @@ } }, "node_modules/@vitest/spy": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.1.1.tgz", - "integrity": "sha512-ZM39BnZ9t/xZ/nF4UwRH5il0Sw93QnZXd9NAZGRpIgj0yvVwPpLd702s/Cx955rGaMlyBQkZJ2Ir7qyY48VZ+g==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.1.2.tgz", + "integrity": "sha512-GSUi5zoy+abNRJwmFhBDC0yRuVUn8WMlQscvnbbXdKLXX9dE59YbfwXxuJ/mth6eeqIzofU8BB5XDo/Ns/qK2A==", "dev": true, "dependencies": { "tinyspy": "^3.0.0" @@ -809,12 +809,12 @@ } }, "node_modules/@vitest/utils": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.1.1.tgz", - "integrity": "sha512-Y6Q9TsI+qJ2CC0ZKj6VBb+T8UPz593N113nnUykqwANqhgf3QkZeHFlusgKLTqrnVHbj/XDKZcDHol+dxVT+rQ==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.1.2.tgz", + "integrity": "sha512-zMO2KdYy6mx56btx9JvAqAZ6EyS3g49krMPPrgOp1yxGZiA93HumGk+bZ5jIZtOg5/VBYl5eBmGRQHqq4FG6uQ==", "dev": true, "dependencies": { - "@vitest/pretty-format": "2.1.1", + "@vitest/pretty-format": "2.1.2", "loupe": "^3.1.1", "tinyrainbow": "^1.2.0" }, @@ -1029,15 +1029,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/get-func-name": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", - "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==", - "dev": true, - "engines": { - "node": "*" - } - }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -1107,13 +1098,10 @@ "dev": true }, "node_modules/loupe": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.1.1.tgz", - "integrity": "sha512-edNu/8D5MKVfGVFRhFf8aAxiTM6Wumfz5XsaatSxlD3w4R1d/WEKUTydCdPGbl9K7QG/Ca3GnDV2sIKIpXRQcw==", - "dev": true, - "dependencies": { - "get-func-name": "^2.0.1" - } + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.1.2.tgz", + "integrity": "sha512-23I4pFZHmAemUnz8WZXbYRSKYj801VDaNv9ETuMh7IrMc7VuVVSo+Z9iLE3ni30+U48iDWfi30d3twAXBYmnCg==", + "dev": true }, "node_modules/magic-string": { "version": "0.30.11", @@ -1502,9 +1490,9 @@ } }, "node_modules/vite-node": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.1.1.tgz", - "integrity": "sha512-N/mGckI1suG/5wQI35XeR9rsMsPqKXzq1CdUndzVstBj/HvyxxGctwnK6WX43NGt5L3Z5tcRf83g4TITKJhPrA==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.1.2.tgz", + "integrity": "sha512-HPcGNN5g/7I2OtPjLqgOtCRu/qhVvBxTUD3qzitmL0SrG1cWFzxzhMDWussxSbrRYWqnKf8P2jiNhPMSN+ymsQ==", "dev": true, "dependencies": { "cac": "^6.7.14", @@ -1523,18 +1511,18 @@ } }, "node_modules/vitest": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.1.1.tgz", - "integrity": "sha512-97We7/VC0e9X5zBVkvt7SGQMGrRtn3KtySFQG5fpaMlS+l62eeXRQO633AYhSTC3z7IMebnPPNjGXVGNRFlxBA==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.1.2.tgz", + "integrity": "sha512-veNjLizOMkRrJ6xxb+pvxN6/QAWg95mzcRjtmkepXdN87FNfxAss9RKe2far/G9cQpipfgP2taqg0KiWsquj8A==", "dev": true, "dependencies": { - "@vitest/expect": "2.1.1", - "@vitest/mocker": "2.1.1", - "@vitest/pretty-format": "^2.1.1", - "@vitest/runner": "2.1.1", - "@vitest/snapshot": "2.1.1", - "@vitest/spy": "2.1.1", - "@vitest/utils": "2.1.1", + "@vitest/expect": "2.1.2", + "@vitest/mocker": "2.1.2", + "@vitest/pretty-format": "^2.1.2", + "@vitest/runner": "2.1.2", + "@vitest/snapshot": "2.1.2", + "@vitest/spy": "2.1.2", + "@vitest/utils": "2.1.2", "chai": "^5.1.1", "debug": "^4.3.6", "magic-string": "^0.30.11", @@ -1545,7 +1533,7 @@ "tinypool": "^1.0.0", "tinyrainbow": "^1.2.0", "vite": "^5.0.0", - "vite-node": "2.1.1", + "vite-node": "2.1.2", "why-is-node-running": "^2.3.0" }, "bin": { @@ -1560,8 +1548,8 @@ "peerDependencies": { "@edge-runtime/vm": "*", "@types/node": "^18.0.0 || >=20.0.0", - "@vitest/browser": "2.1.1", - "@vitest/ui": "2.1.1", + "@vitest/browser": "2.1.2", + "@vitest/ui": "2.1.2", "happy-dom": "*", "jsdom": "*" }, diff --git a/cratedb_sqlparse_js/package.json b/cratedb_sqlparse_js/package.json index f06d67f..4e83c77 100644 --- a/cratedb_sqlparse_js/package.json +++ b/cratedb_sqlparse_js/package.json @@ -47,7 +47,7 @@ "jsdoc": "^4.0.3", "terser": "^5.34.1", "vite": "^5.4.8", - "vitest": "^2.1.1" + "vitest": "^2.1.2" }, "scripts": { "test": "vitest run", diff --git a/cratedb_sqlparse_js/tests/exceptions.test.js b/cratedb_sqlparse_js/tests/exceptions.test.js index 0e5f3f7..e982d45 100644 --- a/cratedb_sqlparse_js/tests/exceptions.test.js +++ b/cratedb_sqlparse_js/tests/exceptions.test.js @@ -15,7 +15,13 @@ test('Error should be collected and not thrown by default', () => { expect(() => stmts).not.toThrowError() }) -test('Several Errors should be collected and not thrown by default', () => { +test('Single error should be collected', () => { + const stmt = sqlparse("SELECT A,B,C,D FROM tbl1 WHERE A ? '%A'") + expect(stmt[0].exception).toBeDefined() + expect(stmt[0].exception.msg).toBe("mismatched input '?' expecting {, ';'}") + expect(stmt[0].exception.query).toBe("SELECT A,B,C,D FROM tbl1 WHERE A ?") +}) +test('Several errors should be collected and not thrown by default', () => { const stmts = sqlparse(` SELECT A FROM tbl1 where; SELECT 1; @@ -73,6 +79,31 @@ test('Exception message is correct', () => { }) +test('White or special characters should not avoid exception catching', () => { + // https://github.com/crate/cratedb-sqlparse/issues/67 + const stmts = [ + `SELECT 1\n limit `, + `SELECT 1\r limit `, + `SELECT 1\t limit `, + `SELECT 1 limit ` + ] + for (const stmt in stmts) { + let r = sqlparse(stmt) + expect(r[0].exception).toBeDefined(); + } +}) + +test('Missing token error should not panic', ()=> { + // See https://github.com/crate/cratedb-sqlparse/issues/66 + sqlparse(` + CREATE TABLE t01 ( + "x" OBJECT (DYNAMIC), + "y" OBJECT (DYNAMIC) AS ("z" ARRAY(OBJECT (DYNAMIC)) + ); +`) +}) + + test('Whitetest or special characters should not avoid exception catching', () => { // https://github.com/crate/cratedb-sqlparse/issues/67 const stmts = [ diff --git a/cratedb_sqlparse_py/cratedb_sqlparse/parser.py b/cratedb_sqlparse_py/cratedb_sqlparse/parser.py index e42dcab..e87d9b6 100644 --- a/cratedb_sqlparse_py/cratedb_sqlparse/parser.py +++ b/cratedb_sqlparse_py/cratedb_sqlparse/parser.py @@ -114,13 +114,31 @@ def __init__(self): self.errors = [] def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e): + if e: + query = recognizer.getTokenStream().getText(e.ctx.start, offendingSymbol.tokenIndex) + + else: + # If antlr4 doesn't give us an error object, we heuristically create a query, or a piece of it + # so we increase the chances of it being correctly assigned. + # It means that theoretically if you input two wrong queries that antlr4 manages + # to distinguish as two different statements (which is hard already), and both are similar + # the errors could be matched wrongly. Still pretty rare, and it is very hard to come up with + # an actual query that does it. + + # The newly generated query will be either the offendingToken + one token to the left + # or offendingToken + two tokens to the left, if the second is possible it takes precedence. + min_token_to_check = max(1, offendingSymbol.tokenIndex - 2) + + tokens = recognizer.getTokenStream().tokens[min_token_to_check : offendingSymbol.tokenIndex + 1] + + query = "".join(token.text for token in tokens) + error = ParsingException( msg=msg, offending_token=offendingSymbol, - e=e, - query=e.ctx.parser.getTokenStream().getText(e.ctx.start, e.offendingToken.tokenIndex), + e=e if e else type("NotViableInput", (Exception,), {})(), + query=query, ) - self.errors.append(error) diff --git a/cratedb_sqlparse_py/tests/test_exceptions.py b/cratedb_sqlparse_py/tests/test_exceptions.py index e095369..4717bf0 100644 --- a/cratedb_sqlparse_py/tests/test_exceptions.py +++ b/cratedb_sqlparse_py/tests/test_exceptions.py @@ -98,3 +98,16 @@ def test_sqlparse_catches_exception(): """ for stmt in stmts: assert sqlparse(stmt)[0].exception + + +def test_sqlparse_should_not_panic(): + from cratedb_sqlparse import sqlparse + + sqlparse(""" + CREATE TABLE t01 ( + "x" OBJECT (DYNAMIC), + "y" OBJECT (DYNAMIC) AS ("z" ARRAY(OBJECT (DYNAMIC)) + ); + """)[0] + + # That's it, it shouldn't raise a runtime Exception.