diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index dc12bdab73..4202fa5fa4 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -10,6 +10,7 @@ import static org.opensearch.sql.ast.tree.Sort.NullOrder.NULL_LAST; import static org.opensearch.sql.ast.tree.Sort.SortOrder.ASC; import static org.opensearch.sql.ast.tree.Sort.SortOrder.DESC; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALOUS; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALY_GRADE; @@ -31,8 +32,10 @@ import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.Alias; import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.Field; +import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.Let; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.Map; @@ -43,6 +46,7 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; +import org.opensearch.sql.ast.tree.Highlight; import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.Limit; import org.opensearch.sql.ast.tree.Parse; @@ -55,6 +59,7 @@ import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; +import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.model.ExprMissingValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.exception.SemanticCheckException; @@ -71,6 +76,7 @@ import org.opensearch.sql.planner.logical.LogicalDedupe; import org.opensearch.sql.planner.logical.LogicalEval; import org.opensearch.sql.planner.logical.LogicalFilter; +import org.opensearch.sql.planner.logical.LogicalHighlight; import org.opensearch.sql.planner.logical.LogicalLimit; import org.opensearch.sql.planner.logical.LogicalMLCommons; import org.opensearch.sql.planner.logical.LogicalPlan; @@ -329,6 +335,23 @@ public LogicalPlan visitEval(Eval node, AnalysisContext context) { return new LogicalEval(child, expressionsBuilder.build()); } + /** + * Build {@link LogicalHighlight}. + */ + @Override + public LogicalPlan visitHighlight(Highlight node, AnalysisContext context) { + LogicalPlan child = node.getChild().get(0).accept(this, context); + + TypeEnvironment env = context.peek(); + env.define(new Symbol(Namespace.FIELD_NAME, + (((Alias) node.getExpression()).getName())), STRING); + + HighlightFunction unresolved = (HighlightFunction) ((Alias)node.getExpression()).getDelegated(); + Expression field = expressionAnalyzer.analyze(unresolved.getHighlightField(), context); + return new LogicalHighlight(child, field); + } + + /** * Build {@link ParseExpression} to context and skip to child nodes. */ diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index 670da5c85c..b4d95774d7 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -43,12 +43,13 @@ import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.expression.Xor; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.HighlightExpression; import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.NamedArgumentExpression; import org.opensearch.sql.expression.NamedExpression; @@ -191,9 +192,10 @@ public Expression visitWindowFunction(WindowFunction node, AnalysisContext conte } @Override - public Expression visitHighlight(HighlightFunction node, AnalysisContext context) { + public Expression visitHighlightFunction(HighlightFunction node, AnalysisContext context) { Expression expr = node.getHighlightField().accept(this, context); - return new HighlightExpression(expr); + String highlightStr = "highlight(" + StringUtils.unquoteText(expr.toString()) + ")"; + return new ReferenceExpression(highlightStr, ExprCoreType.STRING); } @Override diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index 17321bc473..0d151d5e8b 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -40,6 +40,7 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; +import org.opensearch.sql.ast.tree.Highlight; import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.Limit; import org.opensearch.sql.ast.tree.Parse; @@ -256,7 +257,11 @@ public T visitAD(AD node, C context) { return visitChildren(node, context); } - public T visitHighlight(HighlightFunction node, C context) { + public T visitHighlightFunction(HighlightFunction node, C context) { + return visitChildren(node, context); + } + + public T visitHighlight(Highlight node, C context) { return visitChildren(node, context); } } diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/HighlightFunction.java b/core/src/main/java/org/opensearch/sql/ast/expression/HighlightFunction.java index 5f1bb652d9..97ccb4cf5e 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/HighlightFunction.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/HighlightFunction.java @@ -24,7 +24,7 @@ public class HighlightFunction extends UnresolvedExpression { @Override public T accept(AbstractNodeVisitor nodeVisitor, C context) { - return nodeVisitor.visitHighlight(this, context); + return nodeVisitor.visitHighlightFunction(this, context); } @Override diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Highlight.java b/core/src/main/java/org/opensearch/sql/ast/tree/Highlight.java new file mode 100644 index 0000000000..93900b49a8 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Highlight.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** + * AST node represent Highlight operation. + */ +@Getter +@Setter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Highlight extends UnresolvedPlan { + private final UnresolvedExpression expression; + private UnresolvedPlan child; + + @Override + public Highlight attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitHighlight(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java b/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java index d53371dd58..b05b0924a8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java @@ -55,10 +55,6 @@ public T visitNamed(NamedExpression node, C context) { return node.getDelegated().accept(this, context); } - public T visitHighlight(HighlightExpression node, C context) { - return visitNode(node, context); - } - public T visitReference(ReferenceExpression node, C context) { return visitNode(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java b/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java deleted file mode 100644 index 9745696111..0000000000 --- a/core/src/main/java/org/opensearch/sql/expression/HighlightExpression.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression; - -import java.util.List; -import lombok.Getter; -import org.opensearch.sql.common.utils.StringUtils; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.expression.env.Environment; -import org.opensearch.sql.expression.function.BuiltinFunctionName; - -/** - * Highlight Expression. - */ -@Getter -public class HighlightExpression extends FunctionExpression { - private final Expression highlightField; - - /** - * HighlightExpression Constructor. - * @param highlightField : Highlight field for expression. - */ - public HighlightExpression(Expression highlightField) { - super(BuiltinFunctionName.HIGHLIGHT.getName(), List.of(highlightField)); - this.highlightField = highlightField; - } - - /** - * Return collection value matching highlight field. - * @param valueEnv : Dataset to parse value from. - * @return : collection value of highlight fields. - */ - @Override - public ExprValue valueOf(Environment valueEnv) { - String refName = "_highlight" + "." + StringUtils.unquoteText(getHighlightField().toString()); - return valueEnv.resolve(DSL.ref(refName, ExprCoreType.STRING)); - } - - /** - * Get type for HighlightExpression. - * @return : String type. - */ - @Override - public ExprType type() { - return ExprCoreType.ARRAY; - } - - @Override - public T accept(ExpressionNodeVisitor visitor, C context) { - return visitor.visitHighlight(this, context); - } -} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java index c3e5cc5594..86f11407b6 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java @@ -15,13 +15,11 @@ import java.util.Map; import java.util.stream.Collectors; import lombok.experimental.UtilityClass; -import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.HighlightExpression; import org.opensearch.sql.expression.NamedArgumentExpression; import org.opensearch.sql.expression.env.Environment; @@ -51,14 +49,6 @@ public void register(BuiltinFunctionRepository repository) { repository.register(match_phrase(BuiltinFunctionName.MATCH_PHRASE)); repository.register(match_phrase(BuiltinFunctionName.MATCHPHRASE)); repository.register(match_phrase_prefix()); - repository.register(highlight()); - } - - private static FunctionResolver highlight() { - FunctionName functionName = BuiltinFunctionName.HIGHLIGHT.getName(); - FunctionSignature functionSignature = new FunctionSignature(functionName, List.of(STRING)); - FunctionBuilder functionBuilder = arguments -> new HighlightExpression(arguments.get(0)); - return new FunctionResolver(functionName, ImmutableMap.of(functionSignature, functionBuilder)); } private static FunctionResolver match_bool_prefix() { diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/HighlightOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/HighlightOperator.java new file mode 100644 index 0000000000..f6cb73b704 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/physical/HighlightOperator.java @@ -0,0 +1,121 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.physical; + +import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; +import static org.opensearch.sql.expression.env.Environment.extendEnv; + +import com.google.common.collect.ImmutableMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.expression.env.Environment; + +/** + * HighlightOperator evaluates the {@link HighlightOperator#highlight} to put result + * into the output. Highlight fields in input are matched to the appropriate output. + * Direct mapping between input and output, as well as partial mapping is made + * dependent on highlight expression. + * + */ +@EqualsAndHashCode +public class HighlightOperator extends PhysicalPlan { + @Getter + private final PhysicalPlan input; + @Getter + private final Expression highlight; + + public HighlightOperator(PhysicalPlan input, Expression highlight) { + this.input = input; + this.highlight = highlight; + } + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitHighlight(this, context); + } + + @Override + public boolean hasNext() { + return input.hasNext(); + } + + @Override + public ExprValue next() { + ExprValue inputValue = input.next(); + Pair evalMap = mapHighlight(inputValue.bindingTuples()); + + if (STRUCT == inputValue.type()) { + ImmutableMap.Builder resultBuilder = new ImmutableMap.Builder<>(); + Map tupleValue = ExprValueUtils.getTupleValue(inputValue); + for (Map.Entry valueEntry : tupleValue.entrySet()) { + resultBuilder.put(valueEntry); + } + resultBuilder.put(evalMap); + return ExprTupleValue.fromExprValueMap(resultBuilder.build()); + } else { + return inputValue; + } + } + + /** + * Evaluate the expression in the {@link HighlightOperator#highlight} with {@link Environment}. + * @param env {@link Environment} + * @return The mapping of reference and {@link ExprValue} for expression. + */ + private Pair mapHighlight(Environment env) { + String osHighlightKey = "_highlight"; + if (!highlight.toString().contains("*")) { + osHighlightKey += "." + StringUtils.unquoteText(highlight.toString()); + } + + ReferenceExpression osOutputVar = DSL.ref(osHighlightKey, STRING); + ExprValue value = osOutputVar.valueOf(env); + + // In the event of multiple returned highlights and wildcard being + // used in conjunction with other highlight calls, we need to ensure + // only wildcard regex matching is mapped to wildcard call. + if (StringUtils.unquoteText(highlight.toString()).matches("(.+\\*)|(\\*.+)") + && value.type() == STRUCT) { + value = new ExprTupleValue( + new LinkedHashMap(value.tupleValue() + .entrySet() + .stream() + .filter(s -> s.getKey().matches( + StringUtils.unquoteText( + highlight.toString().replace("*", "(.*)")))) + .collect(Collectors.toMap( + e -> e.getKey(), + e -> e.getValue())))); + } + + String sqlHighlightKey = "highlight(" + StringUtils.unquoteText(highlight.toString()) + ")"; + ReferenceExpression sqlOutputVar = DSL.ref(sqlHighlightKey, STRING); + + // Add mapping for sql output and opensearch returned highlight fields + extendEnv(env, sqlOutputVar, value); + + return new ImmutablePair<>(sqlOutputVar.toString(), value); + } + + @Override + public List getChild() { + return List.of(this.input); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java index 646aae8220..40046879b1 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java @@ -79,4 +79,8 @@ public R visitMLCommons(PhysicalPlan node, C context) { public R visitAD(PhysicalPlan node, C context) { return visitNode(node, context); } + + public R visitHighlight(PhysicalPlan node, C context) { + return visitNode(node, context); + } } diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index d4d72dd1d7..7c23333023 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -22,7 +22,6 @@ import static org.opensearch.sql.ast.dsl.AstDSL.qualifiedName; import static org.opensearch.sql.ast.dsl.AstDSL.relation; import static org.opensearch.sql.ast.dsl.AstDSL.span; -import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; import static org.opensearch.sql.ast.tree.Sort.NullOrder; import static org.opensearch.sql.ast.tree.Sort.SortOption; import static org.opensearch.sql.ast.tree.Sort.SortOption.DEFAULT_ASC; @@ -54,7 +53,6 @@ import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; -import org.opensearch.sql.expression.HighlightExpression; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.window.WindowDefinition; import org.opensearch.sql.planner.logical.LogicalAD; @@ -195,6 +193,22 @@ public void top_source() { ); } + @Test + public void project_highlight() { + assertAnalyzeEqual( + LogicalPlanDSL.project( + LogicalPlanDSL.highlight(LogicalPlanDSL.relation("schema"), + DSL.literal("fieldA")), + DSL.named("highlight(fieldA)", DSL.ref("highlight(fieldA)", STRING)) + ), + AstDSL.projectWithArg( + AstDSL.relation("schema"), + AstDSL.defaultFieldsArgs(), + AstDSL.alias("highlight(fieldA)", new HighlightFunction(AstDSL.stringLiteral("fieldA"))) + ) + ); + } + @Test public void rename_to_invalid_expression() { SemanticCheckException exception = @@ -234,22 +248,6 @@ public void project_source() { AstDSL.alias("double_value", AstDSL.field("double_value")))); } - @Test - public void project_highlight() { - assertAnalyzeEqual( - LogicalPlanDSL.project( - LogicalPlanDSL.highlight(LogicalPlanDSL.relation("schema"), - DSL.literal("fieldA")), - DSL.named("highlight(fieldA)", new HighlightExpression(DSL.literal("fieldA"))) - ), - AstDSL.projectWithArg( - AstDSL.relation("schema"), - AstDSL.defaultFieldsArgs(), - AstDSL.alias("highlight(fieldA)", new HighlightFunction(AstDSL.stringLiteral("fieldA"))) - ) - ); - } - @Test public void remove_source() { assertAnalyzeEqual( diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java index 72db402552..157c743b1f 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java @@ -34,7 +34,6 @@ import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.DataType; -import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.RelevanceFieldList; import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.expression.UnresolvedExpression; @@ -45,7 +44,6 @@ import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.HighlightExpression; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.window.aggregation.AggregateWindowFunction; import org.springframework.context.annotation.Configuration; @@ -537,12 +535,6 @@ public void match_phrase_prefix_all_params() { ); } - @Test - void highlight() { - assertAnalyzeEqual(new HighlightExpression(DSL.literal("fieldA")), - new HighlightFunction(stringLiteral("fieldA"))); - } - protected Expression analyze(UnresolvedExpression unresolvedExpression) { return expressionAnalyzer.analyze(unresolvedExpression, analysisContext); } diff --git a/core/src/test/java/org/opensearch/sql/expression/ExpressionNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/expression/ExpressionNodeVisitorTest.java index b0b2bc5b2b..caf11064ae 100644 --- a/core/src/test/java/org/opensearch/sql/expression/ExpressionNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/ExpressionNodeVisitorTest.java @@ -34,7 +34,6 @@ class ExpressionNodeVisitorTest { @Test void should_return_null_by_default() { ExpressionNodeVisitor visitor = new ExpressionNodeVisitor(){}; - assertNull(new HighlightExpression(DSL.literal("Title")).accept(visitor, null)); assertNull(literal(10).accept(visitor, null)); assertNull(ref("name", STRING).accept(visitor, null)); assertNull(named("bool", literal(true)).accept(visitor, null)); diff --git a/core/src/test/java/org/opensearch/sql/expression/HighlightExpressionTest.java b/core/src/test/java/org/opensearch/sql/expression/HighlightExpressionTest.java deleted file mode 100644 index c6e2dccf69..0000000000 --- a/core/src/test/java/org/opensearch/sql/expression/HighlightExpressionTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; -import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; - -import com.google.common.collect.ImmutableMap; -import com.google.errorprone.annotations.DoNotCall; -import org.junit.jupiter.api.Test; -import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.expression.env.Environment; - -public class HighlightExpressionTest extends ExpressionTestBase { - - @Test - public void single_highlight_test() { - Environment hlTuple = ExprValueUtils.tupleValue( - ImmutableMap.of("_highlight.Title", "result value")).bindingTuples(); - HighlightExpression expr = new HighlightExpression(DSL.literal("Title")); - ExprValue resultVal = expr.valueOf(hlTuple); - - assertEquals(expr.type(), ARRAY); - assertEquals("result value", resultVal.stringValue()); - } - - @Test - public void missing_highlight_test() { - Environment hlTuple = ExprValueUtils.tupleValue( - ImmutableMap.of("_highlight.Title", "result value")).bindingTuples(); - HighlightExpression expr = new HighlightExpression(DSL.literal("invalid")); - ExprValue resultVal = expr.valueOf(hlTuple); - - assertTrue(resultVal.isMissing()); - } - - /** - * Enable me when '*' is supported in highlight. - */ - @DoNotCall - public void highlight_all_test() { - ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); - var hlBuilder = ImmutableMap.builder(); - hlBuilder.put("Title", ExprValueUtils.stringValue("correct result value")); - hlBuilder.put("Body", ExprValueUtils.stringValue("incorrect result value")); - builder.put("_highlight", ExprTupleValue.fromExprValueMap(hlBuilder.build())); - - HighlightExpression hlExpr = new HighlightExpression(DSL.literal("*")); - ExprValue resultVal = hlExpr.valueOf( - ExprTupleValue.fromExprValueMap(builder.build()).bindingTuples()); - assertEquals(ARRAY, resultVal.type()); - for (var field : resultVal.tupleValue().entrySet()) { - assertTrue(field.toString().contains(hlExpr.getHighlightField().toString())); - } - assertTrue(resultVal.tupleValue().containsValue( - ExprValueUtils.stringValue("\"correct result value\""))); - assertTrue(resultVal.tupleValue().containsValue( - ExprValueUtils.stringValue("\"correct result value\""))); - } -} diff --git a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalHighlightTest.java b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalHighlightTest.java new file mode 100644 index 0000000000..9cb0258da6 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalHighlightTest.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.logical; + +import static org.opensearch.sql.ast.dsl.AstDSL.alias; +import static org.opensearch.sql.ast.dsl.AstDSL.highlight; +import static org.opensearch.sql.ast.dsl.AstDSL.relation; +import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.analysis.AnalyzerTestBase; +import org.opensearch.sql.ast.tree.Highlight; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.config.ExpressionConfig; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +@Configuration +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {ExpressionConfig.class, AnalyzerTestBase.class}) +@ExtendWith(MockitoExtension.class) +public class LogicalHighlightTest extends AnalyzerTestBase { + @Test + public void analyze_highlight_with_one_field() { + assertAnalyzeEqual( + LogicalPlanDSL.highlight( + LogicalPlanDSL.relation("schema"), + DSL.literal("field")), + new Highlight( + alias("highlight('field')", + highlight(stringLiteral("field")))) + .attach(relation("schema"))); + } +} diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/HighlightOperatorTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/HighlightOperatorTest.java new file mode 100644 index 0000000000..520e99aeec --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/planner/physical/HighlightOperatorTest.java @@ -0,0 +1,109 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.physical; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.iterableWithSize; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.tupleValue; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.ReferenceExpression; + +@ExtendWith(MockitoExtension.class) +class HighlightOperatorTest extends PhysicalPlanTestBase { + @Mock + private PhysicalPlan inputPlan; + + @Test + public void do_nothing_with_none_tuple_value() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()).thenReturn(ExprValueUtils.integerValue(1)); + ReferenceExpression highlightReferenceExp = DSL.ref("reference", STRING); + PhysicalPlan plan = new HighlightOperator(inputPlan, highlightReferenceExp); + List result = execute(plan); + + assertTrue(((HighlightOperator)plan).getInput().equals(inputPlan)); + assertTrue(((HighlightOperator)plan).getHighlight().equals(highlightReferenceExp)); + assertThat(result, allOf(iterableWithSize(1), hasItems(ExprValueUtils.integerValue(1)))); + } + + @Test + public void highlight_one_field() { + when(inputPlan.hasNext()).thenReturn(true, true, true, false); + when(inputPlan.next()) + .thenReturn( + tupleValue(ImmutableMap.of( + "_highlight.region", "us-east-1", "action", "GET", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of( + "_highlight.region", "us-east-1", "action", "POST", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of( + "_highlight.region", "us-east-1", "action", "PUT", "response", 200))); + + assertThat( + execute(new HighlightOperator(inputPlan, DSL.ref("region", STRING))), + contains( + tupleValue(ImmutableMap.of( + "_highlight.region", "us-east-1", "action", "GET", + "response", 200, "highlight(region)", "us-east-1")), + tupleValue(ImmutableMap.of( + "_highlight.region", "us-east-1", "action", "POST", + "response", 200, "highlight(region)", "us-east-1")), + tupleValue(ImmutableMap.of( + "_highlight.region", "us-east-1", "action", "PUT", + "response", 200, "highlight(region)", "us-east-1")) + )); + } + + @Test + public void highlight_wildcard() { + when(inputPlan.hasNext()).thenReturn(true, true, false); + when(inputPlan.next()) + .thenReturn( + tupleValue(ImmutableMap.of( + "_highlight", ExprNullValue.of(), + "action", "GET", "response", 200))) + .thenReturn( + tupleValue(ImmutableMap.of( + "_highlight", tupleValue( + ImmutableMap.of("region", "us-east-1", "country", "us")), + "action", "GET", "response", 200))); + + assertThat( + execute(new HighlightOperator(inputPlan, DSL.ref("r*", STRING))), + contains( + tupleValue(ImmutableMap.of( + "_highlight", ExprNullValue.of(), + "action", "GET", + "response", 200, "highlight(r*)", ExprNullValue.of()) + ), + tupleValue(ImmutableMap.of( + "_highlight", tupleValue( + ImmutableMap.of("region", "us-east-1", "country", "us")), + "action", "GET", + "response", 200, "highlight(r*)", tupleValue( + ImmutableMap.of("region", "us-east-1"))) + ) + ) + ); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java index cd561f3c09..6aa6630749 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java @@ -11,6 +11,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.DSL.named; import com.google.common.base.Strings; @@ -132,6 +133,10 @@ public void test_PhysicalPlanVisitor_should_return_null() { PhysicalPlan limit = PhysicalPlanDSL.limit(plan, 1, 1); assertNull(limit.accept(new PhysicalPlanNodeVisitor() { }, null)); + + PhysicalPlan highlight = new HighlightOperator(plan, DSL.ref("reference", STRING)); + assertNull(highlight.accept(new PhysicalPlanNodeVisitor() { + }, null)); } @Test @@ -150,6 +155,14 @@ public void test_visitAD() { assertNull(physicalPlanNodeVisitor.visitAD(plan, null)); } + @Test + public void test_visitHighlight() { + PhysicalPlanNodeVisitor physicalPlanNodeVisitor = + new PhysicalPlanNodeVisitor() {}; + + assertNull(physicalPlanNodeVisitor.visitHighlight(plan, null)); + } + public static class PhysicalPlanPrinter extends PhysicalPlanNodeVisitor { public String print(PhysicalPlan node) { diff --git a/docs/user/ppl/functions/relevance.rst b/docs/user/ppl/functions/relevance.rst index 7f7cf50964..6cdb3e10f7 100644 --- a/docs/user/ppl/functions/relevance.rst +++ b/docs/user/ppl/functions/relevance.rst @@ -352,6 +352,33 @@ Another example to show how to set custom values for the optional parameters:: | 1 | The House at Pooh Corner | Alan Alexander Milne | +------+--------------------------+----------------------+ + +HIGHLIGHT +------------ + +Description +>>>>>>>>>>> + +``highlight(field_expression)`` + +The highlight function maps to the highlight function used in search engine to return highlight fields for the given search. +The syntax allows to specify the field in double quotes or single quotes or without any wrap. +Please refer to examples below: + +| ``highlight(title)`` + +Example searching for field Tags:: + + os> source=books | where query_string(['title'], 'Pooh House') | highlight(title); + fetched rows / total rows = 2/2 + +------+--------------------------+----------------------+----------------------------------------------+ + | id | title | author | highlight(title) | + |------+--------------------------+----------------------+----------------------------------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | [The House at Pooh Corner] | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | [Winnie-the-Pooh] | + +------+--------------------------+----------------------+----------------------------------------------+ + + Limitations >>>>>>>>>>> diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/HighlightFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/HighlightFunctionIT.java new file mode 100644 index 0000000000..01cf1ae957 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/HighlightFunctionIT.java @@ -0,0 +1,125 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.legacy.TestsConstants; + +public class HighlightFunctionIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.BEER); + } + + @Test + public void single_highlight_test() throws IOException { + JSONObject result = + executeQuery( + String.format( + "SOURCE=%s | WHERE match(Title, 'Cicerone') | highlight(Title)", TestsConstants.TEST_INDEX_BEER)); + + assertEquals(1, result.getInt("total")); + + assertTrue( + result.getJSONArray("datarows") + .getJSONArray(0) + .getJSONArray(19) + .getString(0) + .equals("What exactly is a Cicerone? What do they do?")); + + assertTrue( + result.getJSONArray("schema") + .getJSONObject(19) + .getString("name") + .equals("highlight(Title)")); + + assertTrue( + result.getJSONArray("schema") + .getJSONObject(19) + .getString("type") + .equals("string")); + } + + @Test + public void quoted_highlight_test() throws IOException { + JSONObject result = + executeQuery( + String.format( + "SOURCE=%s | WHERE match(Title, 'Cicerone') | highlight('Title')", TestsConstants.TEST_INDEX_BEER)); + assertEquals(1, result.getInt("total")); + } + + @Test + public void multiple_highlights_test() throws IOException { + JSONObject result = + executeQuery( + String.format( + "SOURCE=%s | WHERE multi_match([Title, Body], 'hops') | highlight('Title') | highlight(Body)", + TestsConstants.TEST_INDEX_BEER)); + assertEquals(2, result.getInt("total")); + } + + @Test + public void highlight_wildcard_test() throws IOException { + JSONObject result = + executeQuery( + String.format( + "SOURCE=%s | WHERE multi_match([Title, Body], 'Cicerone') | highlight('T*')", + TestsConstants.TEST_INDEX_BEER)); + + assertEquals(1, result.getInt("total")); + + assertTrue( + result.getJSONArray("datarows") + .getJSONArray(0) + .getJSONObject(19) + .getJSONArray("Title") + .get(0) + .equals("What exactly is a Cicerone? What do they do?")); + + assertTrue( + result.getJSONArray("schema") + .getJSONObject(19) + .getString("name") + .equals("highlight('T*')")); + } + + @Test + public void highlight_all_test() throws IOException { + JSONObject result = + executeQuery( + String.format( + "SOURCE=%s | WHERE multi_match([Title, Body], 'Cicerone') | highlight('*')", + TestsConstants.TEST_INDEX_BEER)); + + assertEquals(1, result.getInt("total")); + + assertTrue( + result.getJSONArray("datarows") + .getJSONArray(0) + .getJSONObject(19) + .getJSONArray("Title") + .get(0) + .equals("What exactly is a Cicerone? What do they do?")); + + assertTrue( + result.getJSONArray("datarows") + .getJSONArray(0) + .getJSONObject(19) + .getJSONArray("Body") + .get(0) + .equals("

Recently I've started seeing references to the term 'Cicerone' pop up around the internet; generally")); + + assertTrue( + result.getJSONArray("schema") + .getJSONObject(19) + .getString("name") + .equals("highlight('*')")); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java index 422f71968f..756d533b62 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java @@ -26,7 +26,7 @@ public void single_highlight_test() { String query = "SELECT Tags, highlight('Tags') FROM %s WHERE match(Tags, 'yeast') LIMIT 1"; JSONObject response = executeJdbcRequest(String.format(query, TestsConstants.TEST_INDEX_BEER)); verifySchema(response, schema("Tags", null, "text"), - schema("highlight('Tags')", null, "nested")); + schema("highlight('Tags')", null, "keyword")); assertEquals(1, response.getInt("total")); } @@ -35,7 +35,7 @@ public void accepts_unquoted_test() { String query = "SELECT Tags, highlight(Tags) FROM %s WHERE match(Tags, 'yeast') LIMIT 1"; JSONObject response = executeJdbcRequest(String.format(query, TestsConstants.TEST_INDEX_BEER)); verifySchema(response, schema("Tags", null, "text"), - schema("highlight(Tags)", null, "nested")); + schema("highlight(Tags)", null, "keyword")); assertEquals(1, response.getInt("total")); } @@ -43,38 +43,35 @@ public void accepts_unquoted_test() { public void multiple_highlight_test() { String query = "SELECT highlight(Title), highlight(Body) FROM %s WHERE MULTI_MATCH([Title, Body], 'hops') LIMIT 1"; JSONObject response = executeJdbcRequest(String.format(query, TestsConstants.TEST_INDEX_BEER)); - verifySchema(response, schema("highlight(Title)", null, "nested"), - schema("highlight(Body)", null, "nested")); + verifySchema(response, schema("highlight(Title)", null, "keyword"), + schema("highlight(Body)", null, "keyword")); assertEquals(1, response.getInt("total")); } - // Enable me when * is supported - @DoNotCall + @Test public void wildcard_highlight_test() { String query = "SELECT highlight('*itle') FROM %s WHERE MULTI_MATCH([Title, Body], 'hops') LIMIT 1"; JSONObject response = executeJdbcRequest(String.format(query, TestsConstants.TEST_INDEX_BEER)); - verifySchema(response, schema("highlight('*itle')", null, "nested")); + verifySchema(response, schema("highlight('*itle')", null, "keyword")); assertEquals(1, response.getInt("total")); } - // Enable me when * is supported - @DoNotCall + @Test public void wildcard_multi_field_highlight_test() { String query = "SELECT highlight('T*') FROM %s WHERE MULTI_MATCH([Title, Tags], 'hops') LIMIT 1"; JSONObject response = executeJdbcRequest(String.format(query, TestsConstants.TEST_INDEX_BEER)); - verifySchema(response, schema("highlight('T*')", null, "nested")); + verifySchema(response, schema("highlight('T*')", null, "keyword")); var resultMap = response.getJSONArray("datarows").getJSONArray(0).getJSONObject(0); assertEquals(1, response.getInt("total")); - assertTrue(resultMap.has("highlight(\"T*\").Title")); - assertTrue(resultMap.has("highlight(\"T*\").Tags")); + assertTrue(resultMap.has("Title")); + assertTrue(resultMap.has("Tags")); } - // Enable me when * is supported - @DoNotCall + @Test public void highlight_all_test() { String query = "SELECT highlight('*') FROM %s WHERE MULTI_MATCH([Title, Body], 'hops') LIMIT 1"; JSONObject response = executeJdbcRequest(String.format(query, TestsConstants.TEST_INDEX_BEER)); - verifySchema(response, schema("highlight('*')", null, "nested")); + verifySchema(response, schema("highlight('*')", null, "keyword")); assertEquals(1, response.getInt("total")); } @@ -82,7 +79,7 @@ public void highlight_all_test() { public void highlight_no_limit_test() { String query = "SELECT highlight(Body) FROM %s WHERE MATCH(Body, 'hops')"; JSONObject response = executeJdbcRequest(String.format(query, TestsConstants.TEST_INDEX_BEER)); - verifySchema(response, schema("highlight(Body)", null, "nested")); + verifySchema(response, schema("highlight(Body)", null, "keyword")); assertEquals(2, response.getInt("total")); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java index 45d2b12620..78918ca552 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java @@ -14,6 +14,7 @@ import org.opensearch.sql.planner.physical.DedupeOperator; import org.opensearch.sql.planner.physical.EvalOperator; import org.opensearch.sql.planner.physical.FilterOperator; +import org.opensearch.sql.planner.physical.HighlightOperator; import org.opensearch.sql.planner.physical.LimitOperator; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.planner.physical.ProjectOperator; @@ -150,6 +151,15 @@ public PhysicalPlan visitAD(PhysicalPlan node, Object context) { ); } + @Override + public PhysicalPlan visitHighlight(PhysicalPlan node, Object context) { + HighlightOperator hlOperator = (HighlightOperator) node; + return doProtect( + new HighlightOperator(visitInput(hlOperator.getInput(), context), + ((HighlightOperator) node).getHighlight()) + ); + } + PhysicalPlan visitInput(PhysicalPlan node, Object context) { if (null == node) { return node; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index ef6159020f..629708c054 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -37,6 +37,7 @@ import org.opensearch.sql.planner.logical.LogicalMLCommons; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.HighlightOperator; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.Table; @@ -207,8 +208,9 @@ public PhysicalPlan visitAD(LogicalAD node, OpenSearchIndexScan context) { @Override public PhysicalPlan visitHighlight(LogicalHighlight node, OpenSearchIndexScan context) { - context.getRequestBuilder().pushDownHighlight(node.getHighlightField().toString()); - return visitChild(node, context); + context.getRequestBuilder().pushDownHighlight( + StringUtils.unquoteText(node.getHighlightField().toString())); + return new HighlightOperator(visitChild(node, context), node.getHighlightField()); } } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java index fded7848b6..6bb54dbcbd 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java @@ -58,6 +58,7 @@ import org.opensearch.sql.opensearch.planner.physical.MLCommonsOperator; import org.opensearch.sql.opensearch.setting.OpenSearchSettings; import org.opensearch.sql.opensearch.storage.OpenSearchIndexScan; +import org.opensearch.sql.planner.physical.HighlightOperator; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlanDSL; @@ -293,6 +294,20 @@ public void testVisitAD() { executionProtector.visitAD(adOperator, null)); } + @Test + public void testVisitHighlight() { + HighlightOperator hlOperator = + new HighlightOperator( + values(emptyList()), + DSL.ref("reference", STRING)); + + assertEquals(executionProtector.doProtect(hlOperator), + executionProtector.visitHighlight(hlOperator, null)); + + assertEquals(executionProtector.doProtect(hlOperator), + executionProtector.visitInput(hlOperator, null)); + } + PhysicalPlan resourceMonitor(PhysicalPlan input) { return new ResourceMonitorPlan(input, resourceMonitor); } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 93df64d0b3..2c65483868 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -311,6 +311,7 @@ SLOP: 'SLOP'; TIE_BREAKER: 'TIE_BREAKER'; TYPE: 'TYPE'; ZERO_TERMS_QUERY: 'ZERO_TERMS_QUERY'; +HIGHLIGHT: 'HIGHLIGHT'; // SPAN KEYWORDS SPAN: 'SPAN'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index c83297459d..d54c127737 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -25,7 +25,7 @@ pplCommands commands : whereCommand | fieldsCommand | renameCommand | statsCommand | dedupCommand | sortCommand | evalCommand | headCommand - | topCommand | rareCommand | parseCommand | kmeansCommand | adCommand; + | topCommand | rareCommand | parseCommand | kmeansCommand | adCommand | highlightCommand ; searchCommand : (SEARCH)? fromClause #searchFrom @@ -37,6 +37,10 @@ describeCommand : DESCRIBE tableSourceClause ; +highlightCommand + : highlightFunction + ; + whereCommand : WHERE logicalExpression ; @@ -285,6 +289,10 @@ evalFunctionCall : evalFunctionName LT_PRTHS functionArgs RT_PRTHS ; +highlightFunction + : HIGHLIGHT LT_PRTHS field=relevanceField RT_PRTHS #highlightFunctionCall + ; + /** cast function */ dataTypeFunctionCall : CAST LT_PRTHS expression AS convertedDataType RT_PRTHS diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index d7f97e3d35..e733265dec 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -48,6 +48,7 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; +import org.opensearch.sql.ast.tree.Highlight; import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Project; @@ -127,6 +128,15 @@ public UnresolvedPlan visitWhereCommand(WhereCommandContext ctx) { return new Filter(internalVisitExpression(ctx.logicalExpression())); } + /** + * Highlight command. + */ + @Override + public UnresolvedPlan visitHighlightCommand(OpenSearchPPLParser.HighlightCommandContext ctx) { + return new Highlight(new Alias(StringUtils.unquoteText(getTextInQuery(ctx)), + internalVisitExpression(ctx.highlightFunction().getRuleContext()))); + } + /** * Fields command. */ diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 6e5893d6a3..c6fda554b0 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -62,6 +62,7 @@ import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Function; +import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.In; import org.opensearch.sql.ast.expression.Interval; import org.opensearch.sql.ast.expression.IntervalUnit; @@ -204,6 +205,12 @@ public UnresolvedExpression visitDistinctCountFunctionCall(DistinctCountFunction return new AggregateFunction("count", visit(ctx.valueExpression()), true); } + @Override + public UnresolvedExpression visitHighlightFunctionCall( + OpenSearchPPLParser.HighlightFunctionCallContext ctx) { + return new HighlightFunction(AstDSL.stringLiteral(ctx.relevanceField().getText())); + } + @Override public UnresolvedExpression visitPercentileAggFunction(PercentileAggFunctionContext ctx) { return new AggregateFunction(ctx.PERCENTILE().getText(), visit(ctx.aggField), diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java index dcf961dc24..ca77de87d6 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -80,6 +80,12 @@ public void testTopCommandWithoutNAndGroupByShouldPass() { assertNotEquals(null, tree); } + @Test + public void testHighlightShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=shakespeare | highlight(text_entry)"); + assertNotEquals(null, tree); + } + @Test public void can_parse_multi_match_relevance_function() { assertNotEquals(null, new PPLSyntaxParser().parse( diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index ce5f8f9ec5..e5506e6c12 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -25,6 +25,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.filter; import static org.opensearch.sql.ast.dsl.AstDSL.function; import static org.opensearch.sql.ast.dsl.AstDSL.head; +import static org.opensearch.sql.ast.dsl.AstDSL.highlight; import static org.opensearch.sql.ast.dsl.AstDSL.intLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.let; import static org.opensearch.sql.ast.dsl.AstDSL.map; @@ -52,6 +53,7 @@ import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.tree.AD; +import org.opensearch.sql.ast.tree.Highlight; import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; @@ -607,6 +609,26 @@ public void testParseCommand() { )); } + @Test + public void testQuotedHighlightCommand() { + assertEqual("source=t | highlight('FieldA')", + new Highlight( + alias("highlight('FieldA')", + highlight(stringLiteral("'FieldA'")))) + .attach(relation("t")) + ); + } + + @Test + public void testUnquotedHighlightCommand() { + assertEqual("source=t | highlight(FieldA)", + new Highlight( + alias("highlight(FieldA)", + highlight(stringLiteral("FieldA")))) + .attach(relation("t")) + ); + } + @Test public void testKmeansCommand() { assertEqual("source=t | kmeans centroids=3 iterations=2 distance_type='l1'", diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index f2aff5a7e7..7654222082 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -25,7 +25,6 @@ import static org.opensearch.sql.ast.dsl.AstDSL.exprList; import static org.opensearch.sql.ast.dsl.AstDSL.field; import static org.opensearch.sql.ast.dsl.AstDSL.filter; -import static org.opensearch.sql.ast.dsl.AstDSL.floatLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.function; import static org.opensearch.sql.ast.dsl.AstDSL.in; import static org.opensearch.sql.ast.dsl.AstDSL.intLiteral;