Update tests and test data for relevancy search functions

integ-test/src/test/java/org/opensearch/sql/ppl/MatchIT.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+package org.opensearch.sql.ppl;
+
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
+import static org.opensearch.sql.util.MatcherUtils.rows;
+import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
+
+import java.io.IOException;
+import org.json.JSONObject;
+import org.junit.jupiter.api.Test;
+
+public class MatchIT extends PPLIntegTestCase {
+
+  @Override
+  public void init() throws IOException {
+    loadIndex(Index.BANK);
+  }
+
+  @Test
+  public void test_match_function() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | where match(firstname, 'Hattie') | fields firstname",
+                TEST_INDEX_BANK));
+    verifyDataRows(result, rows("Hattie"));
+  }
+}

integ-test/src/test/java/org/opensearch/sql/ppl/MatchPhraseIT.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+package org.opensearch.sql.ppl;
+
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_PHRASE;
+import static org.opensearch.sql.util.MatcherUtils.rows;
+import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
+
+import java.io.IOException;
+import org.json.JSONObject;
+import org.junit.Ignore;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+public class MatchPhraseIT extends PPLIntegTestCase {
+
+  @Override
+  public void init() throws IOException {
+    loadIndex(Index.PHRASE);
+  }
+
+  @Test
+  public void test_match_phrase_function() throws IOException {
+    JSONObject result =
+            executeQuery(
+                    String.format(
+                            "source=%s | where match_phrase(phrase, 'quick fox') | fields phrase", TEST_INDEX_PHRASE));
+    verifyDataRows(result, rows("quick fox"), rows("quick fox here"));
+  }
+
+  @Test
+  @Ignore("Not supported actually in PPL")
+  public void test_matchphrase_legacy_function() throws IOException {
+    JSONObject result =
+            executeQuery(
+                    String.format(
+                            "source=%s | where matchphrase(phrase, 'quick fox') | fields phrase", TEST_INDEX_PHRASE));
+    verifyDataRows(result, rows("quick fox"), rows("quick fox here"));
+  }
+
+  @Test
+  public void test_match_phrase_with_slop() throws IOException {
+    JSONObject result =
+            executeQuery(
+                    String.format(
+                            "source=%s | where match_phrase(phrase, 'brown fox', slop = 2) | fields phrase", TEST_INDEX_PHRASE));
+    verifyDataRows(result, rows("brown fox"), rows("fox brown"));
+  }
+}

integ-test/src/test/java/org/opensearch/sql/ppl/MatchPhrasePrefixWhereCommandIT.java → integ-test/src/test/java/org/opensearch/sql/ppl/MatchPhrasePrefixIT.java

@@ -13,7 +13,7 @@
 import org.json.JSONObject;
 import org.junit.Test;
 
-public class MatchPhrasePrefixWhereCommandIT extends PPLIntegTestCase {
+public class MatchPhrasePrefixIT extends PPLIntegTestCase {
 
   @Override
   public void init() throws IOException {
@@ -93,7 +93,7 @@ public void zero_term_query_all() throws IOException {
 
   @Test
   public void slop_is_2() throws IOException {
-    // When slop is 0, the terms are matched exactly in the order specified.
+    // When slop is 2, the terms are matched exactly in the order specified.
     // 'open' is used to match prefix of the next term.
     String query = "source = %s" +
         "| where match_phrase_prefix(Tags, 'gas ta', slop=2) " +
@@ -104,7 +104,7 @@ public void slop_is_2() throws IOException {
 
   @Test
   public void slop_is_3() throws IOException {
-    // When slop is 2, results will include phrases where the query terms are transposed.
+    // When slop is 3, results will include phrases where the query terms are transposed.
     String query = "source = %s" +
         "| where match_phrase_prefix(Tags, 'gas ta', slop=3)" +
         "| fields Tags";

integ-test/src/test/java/org/opensearch/sql/ppl/MultiMatchIT.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+package org.opensearch.sql.ppl;
+
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER;
+
+import java.io.IOException;
+
+import org.json.JSONObject;
+import org.junit.Test;
+
+public class MultiMatchIT extends PPLIntegTestCase {
+
+  @Override
+  public void init() throws IOException {
+    loadIndex(Index.BEER);
+  }
+
+  @Test
+  public void test_multi_match() throws IOException {
+    String query = "SOURCE=" + TEST_INDEX_BEER
+        + " | WHERE multi_match([\\\"Tags\\\" ^ 1.5, Title, `Body` 4.2], 'taste') | fields Id";
+    var result = executeQuery(query);
+    assertEquals(16, result.getInt("total"));
+  }
+
+  @Test
+  public void test_multi_match_all_params() throws IOException {
+    String query = "SOURCE=" + TEST_INDEX_BEER
+        + " | WHERE multi_match(['Body', Tags], 'taste beer', operator='and', analyzer=english,"
+        + "auto_generate_synonyms_phrase_query=true, boost = 0.77, cutoff_frequency=0.33,"
+        + "fuzziness = 'AUTO:1,5', fuzzy_transpositions = false, lenient = true, max_expansions = 25,"
+        + "minimum_should_match = '2<-25% 9<-3', prefix_length = 7, tie_breaker = 0.3,"
+        + "type = most_fields, slop = 2, zero_terms_query = 'ALL') | fields Id";
+    var result = executeQuery(query);
+    assertEquals(10, result.getInt("total"));
+  }
+
+  @Test
+  public void test_wildcard_multi_match() throws IOException {
+    String query1 = "SOURCE=" + TEST_INDEX_BEER
+        + " | WHERE multi_match(['Tags'], 'taste') | fields Id";
+    var result1 = executeQuery(query1);
+    String query2 = "SOURCE=" + TEST_INDEX_BEER
+        + " | WHERE multi_match(['T*'], 'taste') | fields Id";
+    var result2 = executeQuery(query2);
+    assertNotEquals(result2.getInt("total"), result1.getInt("total"));
+
+    String query3 = "source=" + TEST_INDEX_BEER
+        + " | where simple_query_string(['*Date'], '2014-01-22')";
+    JSONObject result3 = executeQuery(query3);
+    assertEquals(10, result3.getInt("total"));
+  }
+}

integ-test/src/test/java/org/opensearch/sql/ppl/QueryStringIT.java

@@ -22,14 +22,14 @@ public void init() throws IOException {
   public void all_fields_test() throws IOException {
     String query = "source=" + TEST_INDEX_BEER + " | where query_string([`*`], 'taste')";
     JSONObject result = executeQuery(query);
-    assertEquals(713, result.getInt("total"));
+    assertEquals(16, result.getInt("total"));
   }
 
   @Test
   public void mandatory_params_test() throws IOException {
     String query = "source=" + TEST_INDEX_BEER + " | where query_string([\\\"Tags\\\" ^ 1.5, Title, `Body` 4.2], 'taste')";
     JSONObject result = executeQuery(query);
-    assertEquals(713, result.getInt("total"));
+    assertEquals(16, result.getInt("total"));
   }
 
   @Test
@@ -44,7 +44,7 @@ public void all_params_test() throws IOException {
         + "fuzzy_transpositions = false, lenient = true, fuzzy_max_expansions = 25,"
         + "minimum_should_match = '2<-25% 9<-3', fuzzy_prefix_length = 7)";
     JSONObject result = executeQuery(query);
-    assertEquals(1990, result.getInt("total"));
+    assertEquals(49, result.getInt("total"));
   }
 
   @Test
@@ -57,8 +57,8 @@ public void wildcard_test() throws IOException {
     JSONObject result2 = executeQuery(query2);
     assertNotEquals(result1.getInt("total"), result2.getInt("total"));
 
-    String query3 = "source=" + TEST_INDEX_BEER + " | where query_string(['*Date'], '2015-01-29')";
+    String query3 = "source=" + TEST_INDEX_BEER + " | where query_string(['*Date'], '2014-01-22')";
     JSONObject result3 = executeQuery(query3);
-    assertEquals(5, result3.getInt("total"));
+    assertEquals(10, result3.getInt("total"));
   }
 }

integ-test/src/test/java/org/opensearch/sql/ppl/RelevanceFunctionIT.java

@@ -13,6 +13,8 @@
 import static org.opensearch.sql.util.MatcherUtils.verifySome;
 
 import java.io.IOException;
+
+import org.json.JSONObject;
 import org.junit.Test;
 
 public class RelevanceFunctionIT extends PPLIntegTestCase {
@@ -22,64 +24,102 @@ public void init() throws IOException {
   }
 
   @Test
-  public void test_multi_match() throws IOException {
-    String query = "SOURCE=" + TEST_INDEX_BEER
-        + " | WHERE multi_match([\\\"Tags\\\" ^ 1.5, Title, `Body` 4.2], 'taste') | fields Id";
-    var result = executeQuery(query);
-    assertEquals(713, result.getInt("total"));
+  public void test_wildcard_simple_query_string() throws IOException {
+    String query1 = "SOURCE=" + TEST_INDEX_BEER
+        + " | WHERE simple_query_string(['Tags'], 'taste') | fields Id";
+    var result1 = executeQuery(query1);
+    String query2 = "SOURCE=" + TEST_INDEX_BEER
+        + " | WHERE simple_query_string(['T*'], 'taste') | fields Id";
+    var result2 = executeQuery(query2);
+    assertNotEquals(result2.getInt("total"), result1.getInt("total"));
   }
 
+  /*
+  Dash/minus ('-') character is interpreted as NOT flag if it is activated by NOT or ALL `flags` value
+  `query1` searches for entries with '-free' in `Body`, `query2` - for entries without 'free' in `Body`
+   */
   @Test
-  public void test_simple_query_string() throws IOException {
-    String query = "SOURCE=" + TEST_INDEX_BEER
-        + " | WHERE simple_query_string([\\\"Tags\\\" ^ 1.5, Title, `Body` 4.2], 'taste') | fields Id";
+  public void verify_flags_in_simple_query_string() throws IOException {
+    String query1 = "SOURCE="
+        + TEST_INDEX_BEER + " | WHERE simple_query_string(['Body'], '-free', flags='NONE|PREFIX|ESCAPE')";
+    var result1 = executeQuery(query1);
+    String query2 = "SOURCE="
+        + TEST_INDEX_BEER + " | WHERE simple_query_string([Body], '-free', flags='NOT|AND|OR')";
+    var result2 = executeQuery(query2);
+    assertNotEquals(result2.getInt("total"), result1.getInt("total"));
+
+    String query = "SOURCE=" + TEST_INDEX_BEER;
     var result = executeQuery(query);
-    assertEquals(713, result.getInt("total"));
+    assertEquals(result2.getInt("total") + result1.getInt("total"), result.getInt("total"));
   }
 
+  /*
+  `escape` parameter switches regex-specific character escaping.
+  `query1` searches for entries with "\\?" in `Title`, `query2` - for "?"
+  Ref: QueryParserBase::escape in lucene code.
+   */
   @Test
-  public void test_multi_match_all_params() throws IOException {
-    String query = "SOURCE=" + TEST_INDEX_BEER
-        + " | WHERE multi_match(['Body', Tags], 'taste beer', operator='and', analyzer=english,"
-        + "auto_generate_synonyms_phrase_query=true, boost = 0.77, cutoff_frequency=0.33,"
-        + "fuzziness = 'AUTO:1,5', fuzzy_transpositions = false, lenient = true, max_expansions = 25,"
-        + "minimum_should_match = '2<-25% 9<-3', prefix_length = 7, tie_breaker = 0.3,"
-        + "type = most_fields, slop = 2, zero_terms_query = 'ALL') | fields Id";
-    var result = executeQuery(query);
-    assertEquals(424, result.getInt("total"));
+  public void verify_escape_in_query_string() throws IOException {
+    String query1 = "SOURCE="
+        + TEST_INDEX_BEER + " | WHERE query_string([Title], '?', escape=true);";
+    var result1 = executeQuery(query1);
+    String query2 = "SOURCE="
+        + TEST_INDEX_BEER + " | WHERE query_string([Title], '?', escape=false);";
+    var result2 = executeQuery(query2);
+    assertEquals(0, result1.getInt("total"));
+    assertEquals(8, result2.getInt("total"));
   }
 
+  /*
+  `default_operator`/`operator` in relevance search functions defines whether to search for all or for any words given.
+  `query1` returns matches with 'beer' and matches with 'taste',
+  `query2` returns matches with 'beer' and with 'taste' together.
+   */
   @Test
-  public void test_simple_query_string_all_params() throws IOException {
-    String query = "SOURCE=" + TEST_INDEX_BEER
-        + " | WHERE simple_query_string(['Body', Tags, Title], 'taste beer', default_operator='or',"
-        + "analyzer=english, analyze_wildcard = false, quote_field_suffix = '.exact',"
-        + "auto_generate_synonyms_phrase_query=true, boost = 0.77, flags='PREFIX',"
-        + "fuzzy_transpositions = false, lenient = true, fuzzy_max_expansions = 25,"
-        + "minimum_should_match = '2<-25% 9<-3', fuzzy_prefix_length = 7) | fields Id";
-    var result = executeQuery(query);
-    assertEquals(1990, result.getInt("total"));
+  public void verify_default_operator_in_query_string() throws IOException {
+    String query1 = "SOURCE="
+        + TEST_INDEX_BEER + " | WHERE query_string([Title], 'beer taste', default_operator='OR')";
+    var result1 = executeQuery(query1);
+    String query2 = "SOURCE="
+        + TEST_INDEX_BEER + " | WHERE query_string([Title], 'beer taste', default_operator='AND')";
+    var result2 = executeQuery(query2);
+    assertEquals(16, result1.getInt("total"));
+    assertEquals(4, result2.getInt("total"));
   }
 
   @Test
-  public void test_wildcard_multi_match() throws IOException {
-    String query1 = "SOURCE=" + TEST_INDEX_BEER
-        + " | WHERE multi_match(['Tags'], 'taste') | fields Id";
+  public void verify_default_operator_in_simple_query_string() throws IOException {
+    String query1 = "SOURCE="
+        + TEST_INDEX_BEER + " | WHERE simple_query_string([Title], 'beer taste', default_operator='OR')";
     var result1 = executeQuery(query1);
-    String query2 = "SOURCE=" + TEST_INDEX_BEER
-        + " | WHERE multi_match(['T*'], 'taste') | fields Id";
+    String query2 = "SOURCE="
+        + TEST_INDEX_BEER + " | WHERE simple_query_string([Title], 'beer taste', default_operator='AND')";
     var result2 = executeQuery(query2);
-    assertNotEquals(result2.getInt("total"), result1.getInt("total"));
+    assertEquals(16, result1.getInt("total"));
+    assertEquals(4, result2.getInt("total"));
   }
 
   @Test
-  public void test_wildcard_simple_query_string() throws IOException {
-    String query1 = "SOURCE=" + TEST_INDEX_BEER
-        + " | WHERE simple_query_string(['Tags'], 'taste') | fields Id";
+  public void verify_default_operator_in_multi_match() throws IOException {
+    String query1 = "SOURCE="
+        + TEST_INDEX_BEER + " | WHERE multi_match([Title], 'beer taste', operator='OR')";
     var result1 = executeQuery(query1);
-    String query2 = "SOURCE=" + TEST_INDEX_BEER
-        + " | WHERE simple_query_string(['T*'], 'taste') | fields Id";
+    String query2 = "SOURCE="
+        + TEST_INDEX_BEER + " | WHERE multi_match([Title], 'beer taste', operator='AND')";
     var result2 = executeQuery(query2);
-    assertNotEquals(result2.getInt("total"), result1.getInt("total"));
+    assertEquals(16, result1.getInt("total"));
+    assertEquals(4, result2.getInt("total"));
+  }
+
+  @Test
+  public void verify_operator_in_match() throws IOException {
+    String query1 = "SOURCE="
+        + TEST_INDEX_BEER + " | WHERE match(Title, 'beer taste', operator='OR')";
+    var result1 = executeQuery(query1);
+    String query2 = "SOURCE="
+        + TEST_INDEX_BEER + " | WHERE match(Title, 'beer taste', operator='AND')";
+    var result2 = executeQuery(query2);
+    assertEquals(16, result1.getInt("total"));
+    assertEquals(4, result2.getInt("total"));
   }
 }