From 56515aba6ca893f3977c1c4ea9f2de5d2314f179 Mon Sep 17 00:00:00 2001 From: Andreas Reichel Date: Sat, 23 Dec 2023 18:07:48 +0700 Subject: [PATCH] fix: refactor `JsonExpression`, avoiding expensive semantic lookahead and improving performance --- .../jsqlparser/expression/JsonExpression.java | 57 ++++++-- .../net/sf/jsqlparser/parser/JSqlParserCC.jjt | 128 ++++++++---------- .../select/NestedBracketsPerformanceTest.java | 127 +++++++++++++++++ .../statement/select/PostgresTest.java | 4 +- 4 files changed, 234 insertions(+), 82 deletions(-) diff --git a/src/main/java/net/sf/jsqlparser/expression/JsonExpression.java b/src/main/java/net/sf/jsqlparser/expression/JsonExpression.java index 5d6aa794c..9fe2811cf 100644 --- a/src/main/java/net/sf/jsqlparser/expression/JsonExpression.java +++ b/src/main/java/net/sf/jsqlparser/expression/JsonExpression.java @@ -9,16 +9,31 @@ */ package net.sf.jsqlparser.expression; +import net.sf.jsqlparser.parser.ASTNodeAccessImpl; + +import java.util.AbstractMap; import java.util.ArrayList; +import java.util.Collection; import java.util.List; - -import net.sf.jsqlparser.parser.ASTNodeAccessImpl; +import java.util.Map; public class JsonExpression extends ASTNodeAccessImpl implements Expression { private Expression expr; - private List idents = new ArrayList(); - private List operators = new ArrayList(); + private final List> idents = new ArrayList<>(); + + public JsonExpression() { + + } + + public JsonExpression(Expression expr) { + this.expr = expr; + } + + public JsonExpression(Expression expr, List> idents) { + this.expr = expr; + this.idents.addAll(idents); + } @Override public void accept(ExpressionVisitor expressionVisitor) { @@ -34,24 +49,46 @@ public void setExpression(Expression expr) { } public void addIdent(String ident, String operator) { - idents.add(ident); - operators.add(operator); + idents.add(new AbstractMap.SimpleEntry<>(ident, operator)); } - public List getIdents() { + public void addAllIdents(Collection> idents) { + this.idents.addAll(idents); + } + + public List> getIdentList() { return idents; } + public Map.Entry getIdent(int index) { + return idents.get(index); + } + + @Deprecated + public List getIdents() { + ArrayList l = new ArrayList<>(); + for (Map.Entry ident : idents) { + l.add(ident.getKey()); + } + + return l; + } + + @Deprecated public List getOperators() { - return operators; + ArrayList l = new ArrayList<>(); + for (Map.Entry ident : idents) { + l.add(ident.getValue()); + } + return l; } @Override public String toString() { StringBuilder b = new StringBuilder(); b.append(expr.toString()); - for (int i = 0; i < idents.size(); i++) { - b.append(operators.get(i)).append(idents.get(i)); + for (Map.Entry ident : idents) { + b.append(ident.getValue()).append(ident.getKey()); } return b.toString(); } diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index fc8d2fe47..37b7a329f 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -3638,7 +3638,27 @@ Expression Between(Expression leftExpression) : } { [ { result.setNot(true); }] - betweenExpressionStart=SimpleExpression() betweenExpressionEnd=SimpleExpression() + + ( + LOOKAHEAD( 3 ) betweenExpressionStart = ParenthesedSelect() + | + LOOKAHEAD( SimpleFunction() ) betweenExpressionStart = SimpleFunction() + | + LOOKAHEAD( RegularCondition() ) betweenExpressionStart = RegularCondition() + | + betweenExpressionStart = SimpleExpression() + ) + + + ( + LOOKAHEAD( 3 ) betweenExpressionEnd = ParenthesedSelect() + | + LOOKAHEAD( SimpleFunction() ) betweenExpressionEnd = SimpleFunction() + | + LOOKAHEAD( RegularCondition() ) betweenExpressionEnd = RegularCondition() + | + betweenExpressionEnd = SimpleExpression() + ) { result.setLeftExpression(leftExpression); @@ -3985,7 +4005,10 @@ Expression SimpleExpression(): user = UserVariable() ( operation = "=" | operation = ":=" ) ] - retval=ConcatExpression() + + ( + retval=ConcatExpression() + ) ) { if (user != null) { @@ -4172,6 +4195,8 @@ Expression PrimaryExpression() #PrimaryExpression: boolean exclamationMarkNot = false; boolean dateExpressionAllowed = true; ExpressionList list; + + final List> jsonIdents = new ArrayList>(); } { [ { not=true; } | "!" { not=true; exclamationMarkNot=true; } ] @@ -4195,15 +4220,13 @@ Expression PrimaryExpression() #PrimaryExpression: | retval=XMLSerializeExpr() - | LOOKAHEAD(JsonExpression(), {!interrupted}) retval=JsonExpression() - - | LOOKAHEAD(JsonFunction(), {!interrupted}) retval = JsonFunction() + | LOOKAHEAD(3, { !interrupted}) retval = JsonFunction() - | LOOKAHEAD(JsonAggregateFunction(), {!interrupted}) retval = JsonAggregateFunction() + | LOOKAHEAD(3, { !interrupted}) retval = JsonAggregateFunction() - | LOOKAHEAD(FullTextSearch(), {!interrupted}) retval = FullTextSearch() + | LOOKAHEAD(3, { !interrupted}) retval = FullTextSearch() - | LOOKAHEAD(Function(), {!interrupted}) retval=Function() [ LOOKAHEAD(2) retval = AnalyticExpression( (Function) retval ) ] + | LOOKAHEAD(Function(), { !interrupted}) retval=Function() [ LOOKAHEAD(2) retval = AnalyticExpression( (Function) retval ) ] | LOOKAHEAD(2, {!interrupted}) retval = IntervalExpression() { dateExpressionAllowed = false; } @@ -4267,12 +4290,28 @@ Expression PrimaryExpression() #PrimaryExpression: [ retval = ArrayExpression(retval) ] ( "::" type=ColDataType() { - castExpr = new CastExpression(); - castExpr.setUseCastKeyword(false); - castExpr.setLeftExpression(retval); - castExpr.setColDataType(type); - retval=castExpr; - } )* + castExpr = new CastExpression(); + castExpr.setUseCastKeyword(false); + castExpr.setLeftExpression(retval); + castExpr.setColDataType(type); + retval=castExpr; + } + )* + + // Check for JSON operands + [ + LOOKAHEAD(2) ( + "->" (token= | token=) { jsonIdents.add(new AbstractMap.SimpleEntry(token.image,"->")); } + | + "->>" (token= | token=) { jsonIdents.add(new AbstractMap.SimpleEntry(token.image,"->>")); } + | + "#>" token= { jsonIdents.add(new AbstractMap.SimpleEntry(token.image,"#>")); } + | + "#>>" token= { jsonIdents.add(new AbstractMap.SimpleEntry(token.image,"#>>")); } + )+ + retval = JsonExpression(retval, jsonIdents) + ] + ( LOOKAHEAD(2) timezoneRightExpr=PrimaryExpression() { if (timezoneExpr == null) @@ -4428,65 +4467,13 @@ Expression ParenthesedExpression(): } } -JsonExpression JsonExpression() : { - JsonExpression result = new JsonExpression(); - Expression expr; +JsonExpression JsonExpression(Expression expr, List> idents) : { + JsonExpression result = new JsonExpression(expr, idents); Token token; ColDataType type = null; CastExpression castExpr = null; } { - ( - LOOKAHEAD(3, {!interrupted}) expr=CaseWhenExpression() - | - expr = SimpleJdbcParameter() - | - LOOKAHEAD(2, {!interrupted}) expr=JdbcNamedParameter() - | - expr=UserVariable() - | - LOOKAHEAD(JsonFunction(), {getAsBoolean(Feature.allowComplexParsing) && !interrupted}) expr = JsonFunction() - | - LOOKAHEAD(JsonAggregateFunction(), {getAsBoolean(Feature.allowComplexParsing) && !interrupted}) expr = JsonAggregateFunction() - | - LOOKAHEAD(FullTextSearch(), {getAsBoolean(Feature.allowComplexParsing) && !interrupted}) expr = FullTextSearch() - | - LOOKAHEAD( Function() , {getAsBoolean(Feature.allowComplexParsing) && !interrupted} ) expr=Function() - | - LOOKAHEAD( 2, {!interrupted} ) expr=Column() - | - token= { expr = new StringValue(token.image); } - | - LOOKAHEAD(ParenthesedExpression(), {getAsBoolean(Feature.allowComplexParsing)} ) expr = ParenthesedExpression() - | - LOOKAHEAD( 3, {getAsBoolean(Feature.allowComplexParsing) && !interrupted}) expr=ParenthesedSelect() - ) - - ( - "::" type=ColDataType() - { - castExpr = new CastExpression(); - castExpr.setUseCastKeyword(false); - castExpr.setLeftExpression(expr); - castExpr.setColDataType(type); - expr=castExpr; - } - )* - { - result.setExpression(expr); - } - - ( - LOOKAHEAD(2) ( - "->" (token= | token=) {result.addIdent(token.image,"->");} - | - "->>" (token= | token=) {result.addIdent(token.image,"->>");} - | - "#>" token= {result.addIdent(token.image,"#>");} - | - "#>>" token= {result.addIdent(token.image,"#>>");} - ) - )+ // chaining JSON Expressions, e.g. // '{"obj":{"field": "value"}}'::JSON -> 'obj'::TEXT ->> 'field'::TEXT @@ -4504,8 +4491,7 @@ JsonExpression JsonExpression() : { ) )+ { - result = new JsonExpression(); - result.setExpression(expr); + result = new JsonExpression(expr); } ( @@ -5162,6 +5148,8 @@ Function SimpleFunction(): | LOOKAHEAD( AllTableColumns() ) expr=AllTableColumns() | + LOOKAHEAD( 3 ) expr = ParenthesedSelect() + | LOOKAHEAD( SimpleFunction() ) expr = SimpleFunction() | LOOKAHEAD( RegularCondition() ) expr = RegularCondition() diff --git a/src/test/java/net/sf/jsqlparser/statement/select/NestedBracketsPerformanceTest.java b/src/test/java/net/sf/jsqlparser/statement/select/NestedBracketsPerformanceTest.java index 6a85d65d2..569b88873 100644 --- a/src/test/java/net/sf/jsqlparser/statement/select/NestedBracketsPerformanceTest.java +++ b/src/test/java/net/sf/jsqlparser/statement/select/NestedBracketsPerformanceTest.java @@ -228,4 +228,131 @@ public void testIssue1103() throws JSQLParserException { + ",0),0),0),0),0),0),0),0)\n" + ",0),0),0),0),0),0),0),0)", true); } + + @Test + public void testDeepFunctionParameters() throws JSQLParserException { + String sqlStr = "SELECT a.*\n" + + " , To_Char( a.eingangsdat, 'MM.YY' ) AS eingmonat\n" + + " , ( SELECT Trim( b.atext )\n" + + " FROM masseinheiten x\n" + + " , a_lmt b\n" + + " WHERE x.a_text_id = b.a_text_id\n" + + " AND b.sprach_kz = sprache\n" + + " AND x.masseinh_id = a.masseinh_id ) AS reklamengesonst_bez\n" + + " , ( SELECT Trim( name ) || ' ' || Trim( vorname ) AS eingangerfasser_name\n" + + " FROM personal\n" + + " WHERE mandanten_id = m_personal\n" + + " AND personal_id = eingangerfasser ) AS eingangerfasser_name\n" + + " , Nvl( ( SELECT Max( change_date )\n" + + " FROM besch_statusaenderung\n" + + " WHERE beschwerden_id = a.beschwerden_id\n" + + " AND beschstatus_id = 9\n" + + " AND Nvl( inaktiv, 'F' ) != 'T' ), sysdate ) AS abschlussdatum\n" + + " , a.sachstand\n" + + " , a.bewertung\n" + + " , a.massnahmen\n" + + " , ( Decode( Nvl( ( SELECT Max( Trunc( change_date ) ) - Trunc( a.adate )\n" + + " FROM besch_statusaenderung\n" + + " WHERE beschwerden_id = a.beschwerden_id\n" + + " AND beschstatus_id = 9\n" + + " AND Nvl( inaktiv, 'F' ) != 'T' ), - 1 )\n" + + " , - 1, Trunc( sysdate ) - Trunc( a.adate ) - ( SELECT Count()\n" + + " FROM firmenkalender\n" + + " WHERE firma_id = firmen_id\n" + + " AND Nvl( b_verkauf, 'F' ) = 'T'\n" + + " AND kal_datum BETWEEN Trunc( a.adate )\n" + + " AND Trunc( sysdate ) )\n" + + " , Nvl( ( SELECT Max( Trunc( change_date ) ) - Trunc( a.adate )\n" + + " FROM besch_statusaenderung\n" + + " WHERE beschwerden_id = a.beschwerden_id\n" + + " AND beschstatus_id = 9\n" + + " AND Nvl( inaktiv, 'F' ) != 'T' ), - 1 )\n" + + " - ( SELECT Count()\n" + + " FROM firmenkalender\n" + + " WHERE firma_id = firmen_id\n" + + " AND Nvl( b_verkauf, 'F' ) = 'T'\n" + + " AND kal_datum BETWEEN Trunc( a.adate )\n" + + " AND ( SELECT Max( Trunc( change_date ) )\n" + + " FROM besch_statusaenderung\n" + + " WHERE beschwerden_id = a.beschwerden_id\n" + + " AND beschstatus_id = 9\n" + + " AND Nvl( inaktiv, 'F' ) != 'T' ) ) ) + 1 ) AS laufzeit\n" + + " , Nvl( ( SELECT grenzwert\n" + + " FROM beschfehler\n" + + " WHERE beschfehler_id = a.beschwkat_id ), 0 ) AS grenzwert\n" + + " , Nvl( ( SELECT warnwert\n" + + " FROM beschfehler\n" + + " WHERE beschfehler_id = a.beschwkat_id ), 0 ) AS warnwert\n" + + " , a.beschstatus_id AS pruef_status\n" + + " , ( CASE\n" + + " WHEN ( ( Decode( Nvl( ( SELECT Max( Trunc( change_date ) ) - Trunc( a.adate )\n" + + " FROM besch_statusaenderung\n" + + " WHERE beschwerden_id = a.beschwerden_id\n" + + " AND beschstatus_id = 9\n" + + " AND Nvl( inaktiv, 'F' ) != 'T' ), - 1 )\n" + + " , - 1, Trunc( sysdate ) - Trunc( a.adate ) - ( SELECT Count()\n" + + " FROM firmenkalender\n" + + " WHERE firma_id = firmen_id\n" + + " AND Nvl( b_verkauf, 'F' ) = 'T'\n" + + " AND kal_datum BETWEEN Trunc( a.adate )\n" + + " AND Trunc( sysdate ) )\n" + + " , Nvl( ( SELECT Max( Trunc( change_date ) ) - Trunc( a.adate )\n" + + " FROM besch_statusaenderung\n" + + " WHERE beschwerden_id = a.beschwerden_id\n" + + " AND beschstatus_id = 9\n" + + " AND Nvl( inaktiv, 'F' ) != 'T' ), - 1 )\n" + + " - ( SELECT Count()\n" + + " FROM firmenkalender\n" + + " WHERE firma_id = firmen_id\n" + + " AND Nvl( b_verkauf, 'F' ) = 'T'\n" + + " AND kal_datum BETWEEN Trunc( a.adate )\n" + + " AND ( SELECT Max( Trunc( change_date ) )\n" + + " FROM besch_statusaenderung\n" + + " WHERE beschwerden_id = a.beschwerden_id\n" + + " AND beschstatus_id = 9\n" + + " AND Nvl( inaktiv, 'F' ) != 'T' ) ) ) + 1 ) - Nvl( ( SELECT grenzwert\n" + + " FROM beschfehler\n" + + " WHERE beschfehler_id = a.beschwkat_id ), 0 ) ) < 0\n" + + " THEN 0\n" + + " ELSE ( ( Decode( Nvl( ( SELECT Max( Trunc( change_date ) ) - Trunc( a.adate )\n" + + " FROM besch_statusaenderung\n" + + " WHERE beschwerden_id = a.beschwerden_id\n" + + " AND beschstatus_id = 9\n" + + " AND Nvl( inaktiv, 'F' ) != 'T' ), - 1 )\n" + + " , - 1, Trunc( sysdate ) - Trunc( a.adate ) - ( SELECT Count()\n" + + " FROM firmenkalender\n" + + " WHERE firma_id = firmen_id\n" + + " AND Nvl( b_verkauf, 'F' ) = 'T'\n" + + " AND kal_datum BETWEEN Trunc( a.adate )\n" + + " AND Trunc( sysdate ) )\n" + + " , Nvl( ( SELECT Max( Trunc( change_date ) ) - Trunc( a.adate )\n" + + " FROM besch_statusaenderung\n" + + " WHERE beschwerden_id = a.beschwerden_id\n" + + " AND beschstatus_id = 9\n" + + " AND Nvl( inaktiv, 'F' ) != 'T' ), - 1 )\n" + + " - ( SELECT Count( * )\n" + + " FROM firmenkalender\n" + + " WHERE firma_id = firmen_id\n" + + " AND Nvl( b_verkauf, 'F' ) = 'T'\n" + + " AND kal_datum BETWEEN Trunc( a.adate )\n" + + " AND ( SELECT Max( Trunc( change_date ) )\n" + + " FROM besch_statusaenderung\n" + + " WHERE beschwerden_id = a.beschwerden_id\n" + + " AND beschstatus_id = 9\n" + + " AND Nvl( inaktiv, 'F' ) != 'T' ) ) ) + 1 ) - Nvl( ( SELECT grenzwert\n" + + " FROM beschfehler\n" + + " WHERE beschfehler_id = a.beschwkat_id ), 0 ) )\n" + + " END ) AS grenz_ueber\n" + + "FROM beschwerden a\n" + + "WHERE a.mandanten_id = m_beschwerde\n" + + " AND a.rec_status <> '9'\n" + + " AND EXISTS ( SELECT 1\n" + + " FROM besch_statusaenderung\n" + + " WHERE beschwerden_id = a.beschwerden_id )\n" + + " AND Nvl( ( SELECT grenzwert\n" + + " FROM beschfehler\n" + + " WHERE beschfehler_id = a.beschwkat_id ), 0 ) > 0\n"; + + assertSqlCanBeParsedAndDeparsed(sqlStr, true); + } } diff --git a/src/test/java/net/sf/jsqlparser/statement/select/PostgresTest.java b/src/test/java/net/sf/jsqlparser/statement/select/PostgresTest.java index ffa2c8012..56d36d854 100644 --- a/src/test/java/net/sf/jsqlparser/statement/select/PostgresTest.java +++ b/src/test/java/net/sf/jsqlparser/statement/select/PostgresTest.java @@ -57,9 +57,9 @@ public void testJSonExpressionIssue1696() throws JSQLParserException { String sqlStr = "SELECT '{\"key\": \"value\"}'::json -> 'key' AS X"; PlainSelect plainSelect = (PlainSelect) assertSqlCanBeParsedAndDeparsed(sqlStr, true); SelectItem selectExpressionItem = - (SelectItem) plainSelect.getSelectItems().get(0); + plainSelect.getSelectItems().get(0); Assertions.assertEquals("'key'", - selectExpressionItem.getExpression(JsonExpression.class).getIdents().get(0)); + selectExpressionItem.getExpression(JsonExpression.class).getIdent(0).getKey()); } @Test