Keep mismatch results when error occurs in comparison test

docs/dev/Testing.md

@@ -142,6 +142,12 @@ The workflow of generating test result is:
 
 ![The Workflow of Comparison Test](img/the-workflow-of-comparison-test.png)
 
+ 1. For the schema, name, type as well as their order requires to be the same.
+ 2. For the data rows, only data in each row matters with row order ignored.
+ 3. Report success if any other database result can match.
+ 4. Report error if all other databases throw exception.
+ 5. Report failure otherwise if mismatch and exception mixed.
+
 ### 3.6 Visualization
 
 TODO

integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/report/FailedTestCase.java

@@ -43,10 +43,17 @@ public class FailedTestCase extends TestCaseReport {
    */
   private final String explain;
 
-  public FailedTestCase(int id, String sql, List<DBResult> resultSets) {
+  /**
+   * Errors occurred for partial other databases.
+   */
+  private final String errors;
+
+
+  public FailedTestCase(int id, String sql, List<DBResult> resultSets, String errors) {
     super(id, sql, FAILURE);
     this.resultSets = resultSets;
     this.resultSets.sort(Comparator.comparing(DBResult::getDatabaseName));
+    this.errors = errors;
 
     // Generate explanation by diff the first result with remaining
     this.explain = resultSets.subList(1, resultSets.size())

integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java

@@ -140,18 +140,16 @@ private TestCaseReport compareWithOtherDb(String sql, DBResult esResult) {
 
         mismatchResults.add(otherDbResult);
 
-        // Cannot find any database result match
-        if (i == otherDbConnections.length - 1) {
-          return new FailedTestCase(nextId(), sql, mismatchResults);
-        }
       } catch (Exception e) {
         // Ignore and move on to next database
         reasons.append(extractRootCause(e)).append(";");
       }
     }
 
-    // Cannot find any database support this query
-    return new ErrorTestCase(nextId(), sql, "No other databases support this query: " + reasons);
+    if (mismatchResults.size() == 1) { // Only ES result on list. Cannot find other database support this query
+      return new ErrorTestCase(nextId(), sql, "No other databases support this query: " + reasons);
+    }
+    return new FailedTestCase(nextId(), sql, mismatchResults, reasons.toString());
   }
 
   private int nextId() {

integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/ComparisonTestTest.java

@@ -31,6 +31,7 @@
 import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Row;
 import com.amazon.opendistroforelasticsearch.sql.correctness.runner.resultset.Type;
 import com.amazon.opendistroforelasticsearch.sql.correctness.testset.TestQuerySet;
+import java.util.Collections;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -87,7 +88,7 @@ public void testFailureDueToInconsistency() {
 
     TestReport expected = new TestReport();
     expected.addTestCase(
-        new FailedTestCase(1, "SELECT * FROM accounts", asList(esResult, otherDbResult)));
+        new FailedTestCase(1, "SELECT * FROM accounts", asList(esResult, otherDbResult), ""));
     TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts"));
     assertEquals(expected, actual);
   }
@@ -137,7 +138,7 @@ public void testFailureDueToEventualInconsistency() {
 
     TestReport expected = new TestReport();
     expected.addTestCase(new FailedTestCase(1, "SELECT * FROM accounts",
-        asList(esResult, otherDbResult, anotherDbResult)));
+        asList(esResult, otherDbResult, anotherDbResult), ""));
     TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts"));
     assertEquals(expected, actual);
   }
@@ -192,6 +193,31 @@ public void testSuccessWhenOneDBSupportThisQuery() {
     assertEquals(expected, actual);
   }
 
+  @Test
+  public void testFailureDueToInconsistencyAndExceptionMixed() {
+    DBConnection otherDBConnection2 = mock(DBConnection.class);
+    when(otherDBConnection2.getDatabaseName()).thenReturn("ZZZ DB");
+    correctnessTest = new ComparisonTest(
+        esConnection, new DBConnection[] {otherDbConnection, otherDBConnection2}
+    );
+
+    DBResult esResult =
+        new DBResult("ES", asList(new Type("firstname", "text")), asList(new Row(asList("John"))));
+    DBResult otherResult =
+        new DBResult("Other", asList(new Type("firstname", "text")), Collections.emptyList());
+
+    when(esConnection.select(anyString())).thenReturn(esResult);
+    when(otherDbConnection.select(anyString())).thenReturn(otherResult);
+    when(otherDBConnection2.select(anyString()))
+        .thenThrow(new RuntimeException("Unsupported feature"));
+
+    TestReport expected = new TestReport();
+    expected.addTestCase(new FailedTestCase(1, "SELECT * FROM accounts",
+        asList(esResult, otherResult), "Unsupported feature;"));
+    TestReport actual = correctnessTest.verify(querySet("SELECT * FROM accounts"));
+    assertEquals(expected, actual);
+  }
+
   private TestQuerySet querySet(String query) {
     return new TestQuerySet(new String[] {query});
   }

integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestReportTest.java

@@ -68,8 +68,9 @@ public void testFailedReport() {
         new DBResult("Elasticsearch", singleton(new Type("firstName", "text")),
             singleton(new Row(asList("hello")))),
         new DBResult("H2", singleton(new Type("firstName", "text")),
-            singleton(new Row(asList("world"))))
-    )));
+            singleton(new Row(asList("world"))))),
+        "[SQLITE_ERROR] SQL error or missing database;"
+    ));
     JSONObject actual = new JSONObject(report);
     JSONObject expected = new JSONObject(
         "{" +
@@ -84,6 +85,7 @@ public void testFailedReport() {
             "      \"result\": 'Failed'," +
             "      \"sql\": \"SELECT * FROM accounts\"," +
             "      \"explain\": \"Data row at [0] is different: this=[Row(values=[hello])], other=[Row(values=[world])]\"," +
+            "      \"errors\": \"[SQLITE_ERROR] SQL error or missing database;\"," +
             "      \"resultSets\": [" +
             "        {" +
             "          \"database\": \"Elasticsearch\"," +