Add support for wildcard_query function to the new engine (#156)

common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java

@@ -108,6 +108,48 @@ public static String format(final String format, Object... args) {
     return String.format(Locale.ROOT, format, args);
   }
 
+  /**
+   * Converts sql wildcard character % and _ to * and ?.
+   * @param text string to be converted
+   * @return converted string
+   */
+  public static String convertSqlWildcardToLucene(String text) {
+    final char DEFAULT_ESCAPE = '\\';
+    StringBuilder convertedString = new StringBuilder(text.length());
+    boolean escaped = false;
+
+    for (char currentChar : text.toCharArray()) {
+      switch (currentChar) {
+        case DEFAULT_ESCAPE:
+          escaped = true;
+          convertedString.append(currentChar);
+          break;
+        case '%':
+          if (escaped) {
+            convertedString.deleteCharAt(convertedString.length() - 1);
+            convertedString.append("%");
+          } else {
+            convertedString.append("*");
+          }
+          escaped = false;
+          break;
+        case '_':
+          if (escaped) {
+            convertedString.deleteCharAt(convertedString.length() - 1);
+            convertedString.append("_");
+          } else {
+            convertedString.append('?');
+          }
+          escaped = false;
+          break;
+        default:
+          convertedString.append(currentChar);
+          escaped = false;
+      }
+    }
+    return convertedString.toString();
+  }
+
   private static boolean isQuoted(String text, String mark) {
     return !Strings.isNullOrEmpty(text) && text.startsWith(mark) && text.endsWith(mark);
   }

common/src/test/java/org/opensearch/sql/common/utils/ConvertSQLWildcardTest.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.common.utils;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+public class ConvertSQLWildcardTest {
+  @Test
+  public void test_escaping_sql_wildcards() {
+    assertEquals("%", StringUtils.convertSqlWildcardToLucene("\\%"));
+    assertEquals("\\*", StringUtils.convertSqlWildcardToLucene("\\*"));
+    assertEquals("_", StringUtils.convertSqlWildcardToLucene("\\_"));
+    assertEquals("\\?", StringUtils.convertSqlWildcardToLucene("\\?"));
+    assertEquals("%*", StringUtils.convertSqlWildcardToLucene("\\%%"));
+    assertEquals("*%", StringUtils.convertSqlWildcardToLucene("%\\%"));
+    assertEquals("%*%", StringUtils.convertSqlWildcardToLucene("\\%%\\%"));
+    assertEquals("*%*", StringUtils.convertSqlWildcardToLucene("%\\%%"));
+    assertEquals("_?", StringUtils.convertSqlWildcardToLucene("\\__"));
+    assertEquals("?_", StringUtils.convertSqlWildcardToLucene("_\\_"));
+    assertEquals("_?_", StringUtils.convertSqlWildcardToLucene("\\__\\_"));
+    assertEquals("?_?", StringUtils.convertSqlWildcardToLucene("_\\__"));
+    assertEquals("%\\*_\\?", StringUtils.convertSqlWildcardToLucene("\\%\\*\\_\\?"));
+  }
+}

core/src/main/java/org/opensearch/sql/expression/DSL.java

@@ -698,6 +698,10 @@ public static FunctionExpression match_bool_prefix(Expression... args) {
     return compile(BuiltinFunctionName.MATCH_BOOL_PREFIX, args);
   }
 
+  public static FunctionExpression wildcard_query(Expression... args) {
+    return compile(BuiltinFunctionName.WILDCARD_QUERY, args);
+  }
+
   public static FunctionExpression now(Expression... args) {
     return compile(BuiltinFunctionName.NOW, args);
   }

core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

@@ -221,7 +221,9 @@ public enum BuiltinFunctionName {
   QUERY(FunctionName.of("query")),
   MATCH_QUERY(FunctionName.of("match_query")),
   MATCHQUERY(FunctionName.of("matchquery")),
-  MULTI_MATCH(FunctionName.of("multi_match"));
+  MULTI_MATCH(FunctionName.of("multi_match")),
+  WILDCARDQUERY(FunctionName.of("wildcardquery")),
+  WILDCARD_QUERY(FunctionName.of("wildcard_query"));
 
   private final FunctionName name;
 

core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java

@@ -37,6 +37,8 @@ public void register(BuiltinFunctionRepository repository) {
     repository.register(match_phrase(BuiltinFunctionName.MATCH_PHRASE));
     repository.register(match_phrase(BuiltinFunctionName.MATCHPHRASE));
     repository.register(match_phrase_prefix());
+    repository.register(wildcard_query(BuiltinFunctionName.WILDCARD_QUERY));
+    repository.register(wildcard_query(BuiltinFunctionName.WILDCARDQUERY));
   }
 
   private static FunctionResolver match_bool_prefix() {
@@ -79,6 +81,11 @@ private static FunctionResolver query_string() {
     return new RelevanceFunctionResolver(funcName, STRUCT);
   }
 
+  private static FunctionResolver wildcard_query(BuiltinFunctionName wildcardQuery) {
+    FunctionName funcName = wildcardQuery.getName();
+    return new RelevanceFunctionResolver(funcName, STRING);
+  }
+
   public static class OpenSearchFunction extends FunctionExpression {
     private final FunctionName functionName;
     private final List<Expression> arguments;

core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java

@@ -540,6 +540,34 @@ void query_string_expression_two_fields() {
             AstDSL.unresolvedArg("query", stringLiteral("query_value"))));
   }
 
+  @Test
+  void wildcard_query_expression() {
+    assertAnalyzeEqual(
+        DSL.wildcard_query(
+            DSL.namedArgument("field", DSL.literal("test")),
+            DSL.namedArgument("query", DSL.literal("query_value*"))),
+        AstDSL.function("wildcard_query",
+            unresolvedArg("field", stringLiteral("test")),
+            unresolvedArg("query", stringLiteral("query_value*"))));
+  }
+
+  @Test
+  void wildcard_query_expression_all_params() {
+    assertAnalyzeEqual(
+        DSL.wildcard_query(
+            DSL.namedArgument("field", DSL.literal("test")),
+            DSL.namedArgument("query", DSL.literal("query_value*")),
+            DSL.namedArgument("boost", DSL.literal("1.5")),
+            DSL.namedArgument("case_insensitive", DSL.literal("true")),
+            DSL.namedArgument("rewrite", DSL.literal("scoring_boolean"))),
+        AstDSL.function("wildcard_query",
+            unresolvedArg("field", stringLiteral("test")),
+            unresolvedArg("query", stringLiteral("query_value*")),
+            unresolvedArg("boost", stringLiteral("1.5")),
+            unresolvedArg("case_insensitive", stringLiteral("true")),
+            unresolvedArg("rewrite", stringLiteral("scoring_boolean"))));
+  }
+
   @Test
   public void match_phrase_prefix_all_params() {
     assertAnalyzeEqual(

core/src/test/java/org/opensearch/sql/expression/function/OpenSearchFunctionsTest.java

@@ -197,4 +197,12 @@ void query_string() {
             fields.getValue(), query.getValue()),
         expr.toString());
   }
+
+  @Test
+  void wildcard_query() {
+    FunctionExpression expr = DSL.wildcard_query(field, query);
+    assertEquals(String.format("wildcard_query(field=%s, query=%s)",
+            field.getValue(), query.getValue()),
+        expr.toString());
+  }
 }

docs/user/dql/functions.rst

@@ -3118,6 +3118,58 @@ Example searching for field Tags::
     | [Winnie-the-<em>Pooh</em>]                   |
     +----------------------------------------------+
 
+WILDCARD_QUERY
+------------
+
+Description
+>>>>>>>>>>>
+
+``wildcard_query(field_expression, query_expression[, option=<option_value>]*)``
+
+The ``wildcard_query`` function maps to the ``wildcard_query`` query used in search engine. It returns documents that match provided text in the specified field.
+OpenSearch supports wildcard characters ``*`` and ``?``.  See the full description here: https://opensearch.org/docs/latest/opensearch/query-dsl/term/#wildcards.
+You may include a backslash ``\`` to escape SQL wildcard characters ``\%`` and ``\_``.
+
+Available parameters include:
+
+- boost
+- case_insensitive
+- rewrite
+
+For backward compatibility, ``wildcardquery`` is also supported and mapped to ``wildcard_query`` query as well.
+
+Example with only ``field`` and ``query`` expressions, and all other parameters are set default values::
+
+    os> select Body from wildcard where wildcard_query(Body, 'test wildcard*');
+    fetched rows / total rows = 7/7
+    +-------------------------------------------+
+    | Body                                      |
+    |-------------------------------------------|
+    | test wildcard                             |
+    | test wildcard in the end of the text%     |
+    | test wildcard in % the middle of the text |
+    | test wildcard %% beside each other        |
+    | test wildcard in the end of the text_     |
+    | test wildcard in _ the middle of the text |
+    | test wildcard __ beside each other        |
+    +-------------------------------------------+
+
+Another example to show how to set custom values for the optional parameters::
+
+    os> select Body from wildcard where wildcard_query(Body, 'test wildcard*', boost=0.7, case_insensitive=true, rewrite='constant_score');
+    fetched rows / total rows = 7/7
+    +-------------------------------------------+
+    | Body                                      |
+    |-------------------------------------------|
+    | test wildcard                             |
+    | test wildcard in the end of the text%     |
+    | test wildcard in % the middle of the text |
+    | test wildcard %% beside each other        |
+    | test wildcard in the end of the text_     |
+    | test wildcard in _ the middle of the text |
+    | test wildcard __ beside each other        |
+    +-------------------------------------------+
+
 System Functions
 ================
 
@@ -3142,3 +3194,5 @@ Example::
     |----------------+---------------+-----------------+------------------|
     | DATE           | INTEGER       | DATETIME        | STRUCT           |
     +----------------+---------------+-----------------+------------------+
+
+

doctest/test_data/wildcard.json

@@ -0,0 +1,20 @@
+{"index":{"_id":"0"}}
+{"Body":"test wildcard"}
+{"index":{"_id":"1"}}
+{"Body":"test wildcard in the end of the text%"}
+{"index":{"_id":"2"}}
+{"Body":"%test wildcard in the beginning of the text"}
+{"index":{"_id":"3"}}
+{"Body":"test wildcard in % the middle of the text"}
+{"index":{"_id":"4"}}
+{"Body":"test wildcard %% beside each other"}
+{"index":{"_id":"5"}}
+{"Body":"test wildcard in the end of the text_"}
+{"index":{"_id":"6"}}
+{"Body":"_test wildcard in the beginning of the text"}
+{"index":{"_id":"7"}}
+{"Body":"test wildcard in _ the middle of the text"}
+{"index":{"_id":"8"}}
+{"Body":"test wildcard __ beside each other"}
+{"index":{"_id":"9"}}
+{"Body":"test backslash wildcard \\_"}

doctest/test_docs.py

@@ -26,6 +26,7 @@
 NYC_TAXI = "nyc_taxi"
 BOOKS = "books"
 APACHE = "apache"
+WILDCARD = "wildcard"
 
 
 class DocTestConnection(OpenSearchConnection):
@@ -92,6 +93,7 @@ def set_up_test_indices(test):
     load_file("nyc_taxi.json", index_name=NYC_TAXI)
     load_file("books.json", index_name=BOOKS)
     load_file("apache.json", index_name=APACHE)
+    load_file("wildcard.json", index_name=WILDCARD)
 
 
 def load_file(filename, index_name):
@@ -120,7 +122,7 @@ def set_up(test):
 
 def tear_down(test):
     # drop leftover tables after each test
-    test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS, APACHE], ignore_unavailable=True)
+    test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS, APACHE, WILDCARD], ignore_unavailable=True)
 
 
 docsuite = partial(doctest.DocFileSuite,

doctest/test_mapping/wildcard.json

@@ -0,0 +1,9 @@
+{
+  "mappings" : {
+    "properties" : {
+      "Body" : {
+        "type" : "keyword"
+      }
+    }
+  }
+}

integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java

@@ -584,7 +584,11 @@ public enum Index {
     CALCS(TestsConstants.TEST_INDEX_CALCS,
         "calcs",
         getMappingFile("calcs_index_mappings.json"),
-        "src/test/resources/calcs.json"),;
+        "src/test/resources/calcs.json"),
+    WILDCARD(TestsConstants.TEST_INDEX_WILDCARD,
+        "wildcard",
+        getMappingFile("wildcard_index_mappings.json"),
+        "src/test/resources/wildcard.json"),;
 
     private final String name;
     private final String type;

integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java

@@ -53,6 +53,7 @@ public class TestsConstants {
   public final static String TEST_INDEX_BEER = TEST_INDEX + "_beer";
   public final static String TEST_INDEX_NULL_MISSING = TEST_INDEX + "_null_missing";
   public final static String TEST_INDEX_CALCS = TEST_INDEX + "_calcs";
+  public final static String TEST_INDEX_WILDCARD = TEST_INDEX + "_wildcard";
 
   public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
   public final static String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS";