diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index 52c1087ae1..d420467a9a 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -26,6 +26,7 @@ import org.opensearch.sql.ast.expression.EqualTo; import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Function; +import org.opensearch.sql.ast.expression.In; import org.opensearch.sql.ast.expression.Interval; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.Not; @@ -177,6 +178,24 @@ public Expression visitWindowFunction(WindowFunction node, AnalysisContext conte return expr; } + @Override + public Expression visitIn(In node, AnalysisContext context) { + return visitIn(node.getField(), node.getValueList(), context); + } + + private Expression visitIn( + UnresolvedExpression field, List valueList, AnalysisContext context) { + if (valueList.size() == 1) { + return visitCompare(new Compare("=", field, valueList.get(0)), context); + } else if (valueList.size() > 1) { + return dsl.or( + visitCompare(new Compare("=", field, valueList.get(0)), context), + visitIn(field, valueList.subList(1, valueList.size()), context)); + } else { + throw new SemanticCheckException("Values in In clause should not be empty"); + } + } + @Override public Expression visitCompare(Compare node, AnalysisContext context) { FunctionName functionName = FunctionName.of(node.getOperator()); diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index 1266eae73f..7457868fc6 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -269,6 +269,11 @@ public static UnresolvedExpression in( return new In(field, Arrays.asList(valueList)); } + public static UnresolvedExpression in( + UnresolvedExpression field, List valueList) { + return new In(field, valueList); + } + public static UnresolvedExpression compare( String operator, UnresolvedExpression left, UnresolvedExpression right) { return new Compare(operator, left, right); diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java index e2cb84c5a6..ea1c1008a2 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java @@ -18,9 +18,9 @@ import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; +import java.util.Collections; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.opensearch.sql.analysis.symbol.Namespace; @@ -28,7 +28,6 @@ import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.DataType; -import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.common.antlr.SyntaxCheckException; @@ -36,7 +35,6 @@ import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.window.aggregation.AggregateWindowFunction; import org.springframework.context.annotation.Configuration; @@ -319,6 +317,21 @@ void visit_span() { ); } + @Test + void visit_in() { + assertAnalyzeEqual( + dsl.or( + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(1)), + dsl.or( + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(2)), + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(3)))), + AstDSL.in(field("integer_value"), intLiteral(1), intLiteral(2), intLiteral(3))); + + assertThrows( + SemanticCheckException.class, + () -> analyze(AstDSL.in(field("integer_value"), Collections.emptyList()))); + } + protected Expression analyze(UnresolvedExpression unresolvedExpression) { return expressionAnalyzer.analyze(unresolvedExpression, analysisContext); } diff --git a/docs/category.json b/docs/category.json index 93b898f73d..b23fbdd8b2 100644 --- a/docs/category.json +++ b/docs/category.json @@ -21,7 +21,8 @@ "user/ppl/functions/datetime.rst", "user/ppl/functions/string.rst", "user/ppl/functions/condition.rst", - "user/ppl/functions/relevance.rst" + "user/ppl/functions/relevance.rst", + "user/ppl/functions/expressions.rst" ], "sql_cli": [ "user/dql/expressions.rst", @@ -36,4 +37,4 @@ "user/dql/aggregations.rst", "user/dql/complex.rst" ] -} \ No newline at end of file +} diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index fa3b737c7c..a167ce29b5 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -128,7 +128,10 @@ Operators +----------------+----------------------------------------+ | REGEXP | String matches regular expression test | +----------------+----------------------------------------+ - +| IN | IN value list test | ++----------------+----------------------------------------+ +| NOT IN | NOT IN value list test | ++----------------+----------------------------------------+ Basic Comparison Operator ------------------------- @@ -183,6 +186,19 @@ expr REGEXP pattern. The expr is string value, pattern is supports regular expre | 1 | 0 | +------------------------+------------------+ +IN value list test +------------------ + +Here is an example for IN value test:: + + os> SELECT 1 in (1, 2), 3 not in (1, 2); + fetched rows / total rows = 1/1 + +---------------+-------------------+ + | 1 in (1, 2) | 3 not in (1, 2) | + |---------------+-------------------| + | True | True | + +---------------+-------------------+ + Function Call ============= diff --git a/docs/user/ppl/functions/expressions.rst b/docs/user/ppl/functions/expressions.rst new file mode 100644 index 0000000000..c69252bc94 --- /dev/null +++ b/docs/user/ppl/functions/expressions.rst @@ -0,0 +1,158 @@ +=========== +Expressions +=========== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 3 + + +Introduction +============ + +Expressions, particularly value expressions, are those which return a scalar value. Expressions have different types and forms. For example, there are literal values as atom expression and arithmetic, predicate and function expression built on top of them. And also expressions can be used in different clauses, such as using arithmetic expression in ``Filter``, ``Stats`` command. + +Arithmetic Operators +==================== + +Description +----------- + +Operators +````````` + +Arithmetic expression is an expression formed by numeric literals and binary arithmetic operators as follows: + +1. ``+``: Add. +2. ``-``: Subtract. +3. ``*``: Multiply. +4. ``/``: Divide. For integers, the result is an integer with fractional part discarded. +5. ``%``: Modulo. This can be used with integers only with remainder of the division as result. + +Precedence +`````````` + +Parentheses can be used to control the precedence of arithmetic operators. Otherwise, operators of higher precedence is performed first. + +Type Conversion +``````````````` + +Implicit type conversion is performed when looking up operator signature. For example, an integer ``+`` a real number matches signature ``+(double,double)`` which results in a real number. This rule also applies to function call discussed below. + +Examples +-------- + +Here is an example for different type of arithmetic expressions:: + + os> source=accounts | where age > (25 + 5) | fields age ; + fetched rows / total rows = 3/3 + +-------+ + | age | + |-------| + | 32 | + | 36 | + | 33 | + +-------+ + +Predicate Operators +=================== + +Description +----------- + +Predicate operator is an expression that evaluated to be ture. The MISSING and NULL value comparison has following the rule. MISSING value only equal to MISSING value and less than all the other values. NULL value equals to NULL value, large than MISSING value, but less than all the other values. + +Operators +````````` + ++----------------+----------------------------------------+ +| name | description | ++----------------+----------------------------------------+ +| > | Greater than operator | ++----------------+----------------------------------------+ +| >= | Greater than or equal operator | ++----------------+----------------------------------------+ +| < | Less than operator | ++----------------+----------------------------------------+ +| != | Not equal operator | ++----------------+----------------------------------------+ +| <= | Less than or equal operator | ++----------------+----------------------------------------+ +| = | Equal operator | ++----------------+----------------------------------------+ +| LIKE | Simple Pattern matching | ++----------------+----------------------------------------+ +| IN | NULL value test | ++----------------+----------------------------------------+ +| AND | AND operator | ++----------------+----------------------------------------+ +| OR | OR operator | ++----------------+----------------------------------------+ +| XOR | XOR operator | ++----------------+----------------------------------------+ +| NOT | NOT NULL value test | ++----------------+----------------------------------------+ + +Examples +-------- + +Basic Predicate Operator +```````````````````````` + +Here is an example for comparison operators:: + + os> source=accounts | where age > 33 | fields age ; + fetched rows / total rows = 1/1 + +-------+ + | age | + |-------| + | 36 | + +-------+ + + +IN +`` + +IN operator test field in value lists:: + + os> source=accounts | where age in (32, 33) | fields age ; + fetched rows / total rows = 2/2 + +-------+ + | age | + |-------| + | 32 | + | 33 | + +-------+ + + +OR +`` + +OR operator :: + + os> source=accounts | where age = 32 OR age = 33 | fields age ; + fetched rows / total rows = 2/2 + +-------+ + | age | + |-------| + | 32 | + | 33 | + +-------+ + + +NOT +``` + +NOT operator :: + + os> source=accounts | where not age in (32, 33) | fields age ; + fetched rows / total rows = 2/2 + +-------+ + | age | + |-------| + | 36 | + | 28 | + +-------+ + diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index e2fb425ea6..29e3e56afc 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -60,6 +60,8 @@ The query start with search command and then flowing a set of command delimited * **Functions** + - `Expressions `_ + - `Math Functions `_ - `Date and Time Functions `_ @@ -82,4 +84,4 @@ The query start with search command and then flowing a set of command delimited * **Limitations** - - `Limitations `_ \ No newline at end of file + - `Limitations `_ diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java index d177062e01..264e601f20 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java @@ -384,10 +384,10 @@ public void aggAfterTwoTermsGroupBy() throws Exception { List lines = csvResult.getLines(); Assert.assertEquals(4, lines.size()); assertThat(lines, containsInAnyOrder( - equalTo("31.0"), - equalTo("28.0"), - equalTo("21.0"), - equalTo("24.0"))); + equalTo("31"), + equalTo("28"), + equalTo("21"), + equalTo("24"))); } @Test @@ -398,15 +398,15 @@ public void multipleAggAfterTwoTermsGroupBy() throws Exception { CSVResult csvResult = executeCsvRequest(query, false); List headers = csvResult.getHeaders(); Assert.assertEquals(2, headers.size()); - assertThat(headers, contains(equalTo("COUNT(*)"), equalTo("SUM(balance)"))); + assertThat(headers, contains(equalTo("COUNT(*)"), equalTo("sum(balance)"))); List lines = csvResult.getLines(); Assert.assertEquals(4, lines.size()); assertThat(lines, containsInAnyOrder( - equalTo("31.0,647425.0"), - equalTo("28.0,678337.0"), - equalTo("21.0,505660.0"), - equalTo("24.0,472771.0"))); + equalTo("31,647425"), + equalTo("28,678337"), + equalTo("21,505660"), + equalTo("24,472771"))); } @Test diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index b8ca4a1bf3..10fe84c503 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -48,7 +48,6 @@ import java.util.stream.Collectors; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.RuleContext; -import org.antlr.v4.runtime.Token; import org.opensearch.sql.ast.expression.AggregateFunction; import org.opensearch.sql.ast.expression.Alias; import org.opensearch.sql.ast.expression.AllFields; @@ -72,7 +71,6 @@ import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.ast.expression.Xor; import org.opensearch.sql.common.utils.StringUtils; -import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; import org.opensearch.sql.ppl.utils.ArgumentFactory; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index a59ebfddf6..d04bef8091 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -10,7 +10,6 @@ import static org.opensearch.sql.ast.dsl.AstDSL.agg; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.alias; -import static org.opensearch.sql.ast.dsl.AstDSL.allFields; import static org.opensearch.sql.ast.dsl.AstDSL.and; import static org.opensearch.sql.ast.dsl.AstDSL.argument; import static org.opensearch.sql.ast.dsl.AstDSL.booleanLiteral; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 4d82545a12..57ba12eae9 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -7,8 +7,13 @@ package org.opensearch.sql.ppl.utils; import static org.junit.Assert.assertEquals; +import static org.opensearch.sql.ast.dsl.AstDSL.field; +import static org.opensearch.sql.ast.dsl.AstDSL.projectWithArg; +import static org.opensearch.sql.ast.dsl.AstDSL.relation; +import java.util.Collections; import org.junit.Test; +import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; import org.opensearch.sql.ppl.parser.AstBuilder; import org.opensearch.sql.ppl.parser.AstExpressionBuilder; @@ -146,9 +151,20 @@ public void testDateFunction() { ); } + @Test + public void anonymizeFieldsNoArg() { + assertEquals("source=t | fields + f", + anonymize(projectWithArg(relation("t"), Collections.emptyList(), field("f"))) + ); + } + private String anonymize(String query) { AstBuilder astBuilder = new AstBuilder(new AstExpressionBuilder(), query); - final PPLQueryDataAnonymizer anonymizer = new PPLQueryDataAnonymizer(); - return anonymizer.anonymizeData(astBuilder.visit(parser.analyzeSyntax(query))); + return anonymize(astBuilder.visit(parser.analyzeSyntax(query))); + } + + private String anonymize(UnresolvedPlan plan) { + final PPLQueryDataAnonymizer anonymize = new PPLQueryDataAnonymizer(); + return anonymize.anonymizeData(plan); } } diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index dfdf792132..0b8f3c5250 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -249,7 +249,7 @@ intervalUnit | DAY_SECOND | DAY_MINUTE | DAY_HOUR | YEAR_MONTH ; -// Expressions, predicates +// predicates // Simplified approach for expression expression @@ -265,6 +265,11 @@ predicate | predicate IS nullNotnull #isNullPredicate | left=predicate NOT? LIKE right=predicate #likePredicate | left=predicate REGEXP right=predicate #regexpPredicate + | predicate NOT? IN '(' expressions ')' #inPredicate + ; + +expressions + : expression (',' expression)* ; expressionAtom diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index 21db2cb06f..12105a7620 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -237,6 +237,19 @@ public UnresolvedExpression visitRegexpPredicate(RegexpPredicateContext ctx) { Arrays.asList(visit(ctx.left), visit(ctx.right))); } + @Override + public UnresolvedExpression visitInPredicate(OpenSearchSQLParser.InPredicateContext ctx) { + UnresolvedExpression field = visit(ctx.predicate()); + List inLists = ctx + .expressions() + .expression() + .stream() + .map(this::visit) + .collect(Collectors.toList()); + UnresolvedExpression in = AstDSL.in(field, inLists); + return ctx.NOT() != null ? AstDSL.not(in) : in; + } + @Override public UnresolvedExpression visitAndExpression(AndExpressionContext ctx) { return new And(visit(ctx.left), visit(ctx.right)); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index 9683a7d94a..5112618264 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -444,6 +444,23 @@ public void relevanceMatch() { buildExprAst("match(message, 'search query', analyzer='keyword', operator='AND')")); } + @Test + public void canBuildInClause() { + assertEquals( + AstDSL.in(qualifiedName("age"), AstDSL.intLiteral(20), AstDSL.intLiteral(30)), + buildExprAst("age in (20, 30)")); + + assertEquals( + AstDSL.not(AstDSL.in(qualifiedName("age"), AstDSL.intLiteral(20), AstDSL.intLiteral(30))), + buildExprAst("age not in (20, 30)")); + + assertEquals( + AstDSL.in(qualifiedName("age"), + AstDSL.function("abs", AstDSL.intLiteral(20)), + AstDSL.function("abs", AstDSL.intLiteral(30))), + buildExprAst("age in (abs(20), abs(30))")); + } + private Node buildExprAst(String expr) { OpenSearchSQLLexer lexer = new OpenSearchSQLLexer(new CaseInsensitiveCharStream(expr)); OpenSearchSQLParser parser = new OpenSearchSQLParser(new CommonTokenStream(lexer));