update

opendistro-for-elasticsearch · Jan 29, 2021 · 345bc2c · 345bc2c
1 parent 2740197
commit 345bc2c
Show file tree

Hide file tree

Showing 8 changed files with 63 additions and 78 deletions.
diff --git a/docs/experiment/ppl/cmd/search.rst b/docs/experiment/ppl/cmd/search.rst
@@ -32,14 +32,14 @@ PPL query::
 
     od> source=accounts;
     fetched rows / total rows = 4/4
-    +------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------+
-    | account_number   | firstname   | address              | gender   | city   | lastname   | balance   | employer   | state   | age   | email                 |
-    |------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------|
-    | 1                | Amber       | 880 Holmes Lane      | M        | Brogan | Duke       | 39225     | Pyrami     | IL      | 32    | amberduke@pyrami.com  |
-    | 6                | Hattie      | 671 Bristol Street   | M        | Dante  | Bond       | 5686      | Netagy     | TN      | 36    | hattiebond@netagy.com |
-    | 13               | Nanette     | 789 Madison Street   | F        | Nogal  | Bates      | 32838     | Quility    | VA      | 28    | null                  |
-    | 18               | Dale        | 467 Hutchinson Court | M        | Orick  | Adams      | 4180      | null       | MD      | 33    | daleadams@boink.com   |
-    +------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------+
+    +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+
+    | account_number   | firstname   | address              | balance   | gender   | city   | employer   | state   | age   | email                 | lastname   |
+    |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------|
+    | 1                | Amber       | 880 Holmes Lane      | 39225     | M        | Brogan | Pyrami     | IL      | 32    | amberduke@pyrami.com  | Duke       |
+    | 6                | Hattie      | 671 Bristol Street   | 5686      | M        | Dante  | Netagy     | TN      | 36    | hattiebond@netagy.com | Bond       |
+    | 13               | Nanette     | 789 Madison Street   | 32838     | F        | Nogal  | Quility    | VA      | 28    | null                  | Bates      |
+    | 18               | Dale        | 467 Hutchinson Court | 4180      | M        | Orick  | null       | MD      | 33    | daleadams@boink.com   | Adams      |
+    +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+
 
 Example 2: Fetch data with condition
 ====================================
@@ -50,10 +50,10 @@ PPL query::
 
     od> source=accounts account_number=1 or gender="F";
     fetched rows / total rows = 2/2
-    +------------------+-------------+--------------------+----------+--------+------------+-----------+------------+---------+-------+----------------------+
-    | account_number   | firstname   | address            | gender   | city   | lastname   | balance   | employer   | state   | age   | email                |
-    |------------------+-------------+--------------------+----------+--------+------------+-----------+------------+---------+-------+----------------------|
-    | 1                | Amber       | 880 Holmes Lane    | M        | Brogan | Duke       | 39225     | Pyrami     | IL      | 32    | amberduke@pyrami.com |
-    | 13               | Nanette     | 789 Madison Street | F        | Nogal  | Bates      | 32838     | Quility    | VA      | 28    | null                 |
-    +------------------+-------------+--------------------+----------+--------+------------+-----------+------------+---------+-------+----------------------+
+    +------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------+
+    | account_number   | firstname   | address            | balance   | gender   | city   | employer   | state   | age   | email                | lastname   |
+    |------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------|
+    | 1                | Amber       | 880 Holmes Lane    | 39225     | M        | Brogan | Pyrami     | IL      | 32    | amberduke@pyrami.com | Duke       |
+    | 13               | Nanette     | 789 Madison Street | 32838     | F        | Nogal  | Quility    | VA      | 28    | null                 | Bates      |
+    +------------------+-------------+--------------------+-----------+----------+--------+------------+---------+-------+----------------------+------------+
 
diff --git a/docs/user/general/identifiers.rst b/docs/user/general/identifiers.rst
@@ -40,14 +40,14 @@ Here are examples for using index pattern directly without quotes::
 
     od> SELECT * FROM *cc*nt*;
     fetched rows / total rows = 4/4
-    +------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------+
-    | account_number   | firstname   | address              | gender   | city   | lastname   | balance   | employer   | state   | age   | email                 |
-    |------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------|
-    | 1                | Amber       | 880 Holmes Lane      | M        | Brogan | Duke       | 39225     | Pyrami     | IL      | 32    | amberduke@pyrami.com  |
-    | 6                | Hattie      | 671 Bristol Street   | M        | Dante  | Bond       | 5686      | Netagy     | TN      | 36    | hattiebond@netagy.com |
-    | 13               | Nanette     | 789 Madison Street   | F        | Nogal  | Bates      | 32838     | Quility    | VA      | 28    | null                  |
-    | 18               | Dale        | 467 Hutchinson Court | M        | Orick  | Adams      | 4180      | null       | MD      | 33    | daleadams@boink.com   |
-    +------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------+
+    +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+
+    | account_number   | firstname   | address              | balance   | gender   | city   | employer   | state   | age   | email                 | lastname   |
+    |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------|
+    | 1                | Amber       | 880 Holmes Lane      | 39225     | M        | Brogan | Pyrami     | IL      | 32    | amberduke@pyrami.com  | Duke       |
+    | 6                | Hattie      | 671 Bristol Street   | 5686      | M        | Dante  | Netagy     | TN      | 36    | hattiebond@netagy.com | Bond       |
+    | 13               | Nanette     | 789 Madison Street   | 32838     | F        | Nogal  | Quility    | VA      | 28    | null                  | Bates      |
+    | 18               | Dale        | 467 Hutchinson Court | 4180      | M        | Orick  | null       | MD      | 33    | daleadams@boink.com   | Adams      |
+    +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+
 
 
 Delimited Identifiers
@@ -76,14 +76,14 @@ Here are examples for quoting an index name by back ticks::
 
     od> SELECT * FROM `accounts`;
     fetched rows / total rows = 4/4
-    +------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------+
-    | account_number   | firstname   | address              | gender   | city   | lastname   | balance   | employer   | state   | age   | email                 |
-    |------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------|
-    | 1                | Amber       | 880 Holmes Lane      | M        | Brogan | Duke       | 39225     | Pyrami     | IL      | 32    | amberduke@pyrami.com  |
-    | 6                | Hattie      | 671 Bristol Street   | M        | Dante  | Bond       | 5686      | Netagy     | TN      | 36    | hattiebond@netagy.com |
-    | 13               | Nanette     | 789 Madison Street   | F        | Nogal  | Bates      | 32838     | Quility    | VA      | 28    | null                  |
-    | 18               | Dale        | 467 Hutchinson Court | M        | Orick  | Adams      | 4180      | null       | MD      | 33    | daleadams@boink.com   |
-    +------------------+-------------+----------------------+----------+--------+------------+-----------+------------+---------+-------+-----------------------+
+    +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+
+    | account_number   | firstname   | address              | balance   | gender   | city   | employer   | state   | age   | email                 | lastname   |
+    |------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------|
+    | 1                | Amber       | 880 Holmes Lane      | 39225     | M        | Brogan | Pyrami     | IL      | 32    | amberduke@pyrami.com  | Duke       |
+    | 6                | Hattie      | 671 Bristol Street   | 5686      | M        | Dante  | Netagy     | TN      | 36    | hattiebond@netagy.com | Bond       |
+    | 13               | Nanette     | 789 Madison Street   | 32838     | F        | Nogal  | Quility    | VA      | 28    | null                  | Bates      |
+    | 18               | Dale        | 467 Hutchinson Court | 4180      | M        | Orick  | null       | MD      | 33    | daleadams@boink.com   | Adams      |
+    +------------------+-------------+----------------------+-----------+----------+--------+------------+---------+-------+-----------------------+------------+
 
 
 Case Sensitivity

diff --git a/...in/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/mapping/IndexMapping.java b/...in/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/mapping/IndexMapping.java
@@ -101,15 +101,6 @@ private void flatMappings(
             func.accept(fullFieldName, type);
           }
 
-          if (isMultiField(mapping)) {
-            ((Map<String, Map<String, Object>>) mapping.get("fields"))
-                .forEach(
-                    (innerFieldName, innerMapping) ->
-                        func.accept(
-                            fullFieldName + "." + innerFieldName,
-                            (String) innerMapping.getOrDefault("type", "object")));
-          }
-
           if (mapping.containsKey("properties")) { // Nested field
             flatMappings((Map<String, Object>) mapping.get("properties"), fullFieldName, func);
           }

diff --git a/...a/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java b/...a/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java
@@ -54,14 +54,22 @@ public class ElasticsearchIndex implements Table {
   /** Current Elasticsearch index name. */
   private final String indexName;
 
+  /**
+   * The cached mapping of field and type in index.
+   */
+  private Map<String, ExprType> cachedFieldTypes = null;
+
   /*
    * TODO: Assume indexName doesn't have wildcard.
    *  Need to either handle field name conflicts
    *   or lazy evaluate when query engine pulls field type.
    */
   @Override
   public Map<String, ExprType> getFieldTypes() {
-    return new ElasticsearchDescribeIndexRequest(client, indexName).getFieldTypes();
+    if (cachedFieldTypes == null) {
+      cachedFieldTypes = new ElasticsearchDescribeIndexRequest(client, indexName).getFieldTypes();
+    }
+    return cachedFieldTypes;
   }
 
   /**

diff --git a/...azon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClientTest.java b/...azon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchNodeClientTest.java
@@ -111,7 +111,7 @@ public void getIndexMappings() throws IOException {
     assertEquals(1, indexMappings.size());
 
     IndexMapping indexMapping = indexMappings.values().iterator().next();
-    assertEquals(20, indexMapping.size());
+    assertEquals(18, indexMapping.size());
     assertEquals("text", indexMapping.getFieldType("address"));
     assertEquals("integer", indexMapping.getFieldType("age"));
     assertEquals("double", indexMapping.getFieldType("balance"));
@@ -121,15 +121,13 @@ public void getIndexMappings() throws IOException {
     assertEquals("some_new_es_type_outside_type_system", indexMapping.getFieldType("new_field"));
     assertEquals("text", indexMapping.getFieldType("field with spaces"));
     assertEquals("text_keyword", indexMapping.getFieldType("employer"));
-    assertEquals("keyword", indexMapping.getFieldType("employer.raw"));
     assertEquals("nested", indexMapping.getFieldType("projects"));
     assertEquals("boolean", indexMapping.getFieldType("projects.active"));
     assertEquals("date", indexMapping.getFieldType("projects.release"));
     assertEquals("nested", indexMapping.getFieldType("projects.members"));
     assertEquals("text", indexMapping.getFieldType("projects.members.name"));
     assertEquals("object", indexMapping.getFieldType("manager"));
     assertEquals("text_keyword", indexMapping.getFieldType("manager.name"));
-    assertEquals("keyword", indexMapping.getFieldType("manager.name.keyword"));
     assertEquals("keyword", indexMapping.getFieldType("manager.address"));
     assertEquals("long", indexMapping.getFieldType("manager.salary"));
   }

diff --git a/...azon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClientTest.java b/...azon/opendistroforelasticsearch/sql/elasticsearch/client/ElasticsearchRestClientTest.java
@@ -110,7 +110,7 @@ void getIndexMappings() throws IOException {
     assertEquals(1, indexMappings.size());
 
     IndexMapping indexMapping = indexMappings.values().iterator().next();
-    assertEquals(20, indexMapping.size());
+    assertEquals(18, indexMapping.size());
     assertEquals("text", indexMapping.getFieldType("address"));
     assertEquals("integer", indexMapping.getFieldType("age"));
     assertEquals("double", indexMapping.getFieldType("balance"));
@@ -120,15 +120,13 @@ void getIndexMappings() throws IOException {
     assertEquals("some_new_es_type_outside_type_system", indexMapping.getFieldType("new_field"));
     assertEquals("text", indexMapping.getFieldType("field with spaces"));
     assertEquals("text_keyword", indexMapping.getFieldType("employer"));
-    assertEquals("keyword", indexMapping.getFieldType("employer.raw"));
     assertEquals("nested", indexMapping.getFieldType("projects"));
     assertEquals("boolean", indexMapping.getFieldType("projects.active"));
     assertEquals("date", indexMapping.getFieldType("projects.release"));
     assertEquals("nested", indexMapping.getFieldType("projects.members"));
     assertEquals("text", indexMapping.getFieldType("projects.members.name"));
     assertEquals("object", indexMapping.getFieldType("manager"));
     assertEquals("text_keyword", indexMapping.getFieldType("manager.name"));
-    assertEquals("keyword", indexMapping.getFieldType("manager.name.keyword"));
     assertEquals("keyword", indexMapping.getFieldType("manager.address"));
     assertEquals("long", indexMapping.getFieldType("manager.salary"));
   }

diff --git a/...rc/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java b/...rc/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java
@@ -136,26 +136,6 @@ public void selectWrongField() throws IOException {
     assertThat(getDataRows(response).length(), equalTo(RESPONSE_DEFAULT_MAX_SIZE));
   }
 
-  @Ignore("Issue 1019, Breaking Change, the keyword should not been allowed, semantic exception "
-      + "is expected")
-  @Test
-  public void selectKeyword() throws IOException {
-    JSONObject response = executeQuery(
-        String.format(Locale.ROOT, "SELECT firstname.keyword FROM %s",
-            TestsConstants.TEST_INDEX_ACCOUNT));
-
-    List<String> fields = Collections.singletonList("firstname.keyword");
-    assertContainsColumns(getSchema(response), fields);
-
-    /*
-     * firstname.keyword will appear in Schema but because there is no 'firstname.keyword' in SearchHits source
-     * the DataRows will output null.
-     *
-     * Looks like x-pack adds this keyword field to "docvalue_fields", this is likely how it ends up in SearchHits
-     */
-    // assertContainsData(getDataRows(protocol), fields);
-  }
-
   @Test
   public void selectScore() throws IOException {
     JSONObject response = executeQuery(
@@ -377,17 +357,6 @@ public void aggregationFunctionInSelectWithAlias() throws IOException {
     }
   }
 
-  @Test
-  public void aggregationFunctionInSelectGroupByMultipleFields() throws IOException {
-    JSONObject response = executeQuery(
-        String.format(Locale.ROOT, "SELECT SUM(age) FROM %s GROUP BY age, state.keyword",
-            TestsConstants.TEST_INDEX_ACCOUNT));
-
-    List<String> fields = Arrays.asList("SUM(age)");
-    assertContainsColumns(getSchema(response), fields);
-    assertContainsData(getDataRows(response), fields);
-  }
-
   @Test
   public void aggregationFunctionInSelectNoGroupBy() throws IOException {
     JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT SUM(age) FROM %s",

diff --git a/...g-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/QueryValidationIT.java b/...g-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/QueryValidationIT.java
@@ -65,10 +65,31 @@ public void testNonAggregatedSelectColumnPresentWithoutGroupByClause() throws IO
         .hasStatusCode(BAD_REQUEST)
         .hasErrorType("SemanticCheckException")
         .containsMessage("Explicit GROUP BY clause is required because expression [state] "
-                       + "contains non-aggregated column")
+            + "contains non-aggregated column")
         .whenExecute("SELECT state, AVG(age) FROM elasticsearch-sql_test_index_account");
   }
 
+  @Test
+  public void testQueryFieldWithKeyword() throws IOException {
+    expectResponseException()
+        .hasStatusCode(BAD_REQUEST)
+        .hasErrorType("SemanticCheckException")
+        .containsMessage(
+            "can't resolve Symbol(namespace=FIELD_NAME, name=firstname.keyword) in type env")
+        .whenExecute("SELECT firstname.keyword FROM elasticsearch-sql_test_index_account");
+  }
+
+  @Test
+  public void aggregationFunctionInSelectGroupByMultipleFields() throws IOException {
+    expectResponseException()
+        .hasStatusCode(BAD_REQUEST)
+        .hasErrorType("SemanticCheckException")
+        .containsMessage(
+            "can't resolve Symbol(namespace=FIELD_NAME, name=state.keyword) in type env")
+        .whenExecute(
+            "SELECT SUM(age) FROM elasticsearch-sql_test_index_account GROUP BY state.keyword");
+  }
+
   public ResponseExceptionAssertion expectResponseException() {
     return new ResponseExceptionAssertion(exceptionRule);
   }