ESQL: Add warning header when default LIMIT is applied (elastic#100894)…

… (elastic#101054) Fixes elastic#100555 ESQL [by default](elastic#99816) adds an implicit `LIMIT 500` to queries that do not define a limit. Since this can be confusing for the end user, with this PR we also add a warning to the response, making this default clear and explicit.
martijnvg · Oct 18, 2023 · c23a46b · c23a46b
1 parent 161cf53
commit c23a46b
Show file tree

Hide file tree

Showing 32 changed files with 328 additions and 18 deletions.
diff --git a/docs/reference/esql/esql-rest.asciidoc b/docs/reference/esql/esql-rest.asciidoc
@@ -107,7 +107,7 @@ s|Description
 
 |smile
 |application/smile
-|{wikipedia}/Smile_(data_interchange_format)[Smile] binary data format similar 
+|{wikipedia}/Smile_(data_interchange_format)[Smile] binary data format similar
 to CBOR
 
 |===
@@ -220,6 +220,7 @@ POST /_query
     | WHERE page_count > 300 AND author == "Frank Herbert"
     | STATS count = COUNT(*) by year
     | WHERE count > 0
+    | LIMIT 5
   """
 }
 ----
@@ -239,6 +240,7 @@ POST /_query
     | WHERE page_count > ? AND author == ?
     | STATS count = COUNT(*) by year
     | WHERE count > ?
+    | LIMIT 5
   """,
   "params": [300, "Frank Herbert", 0]
 }

diff --git a/docs/reference/esql/multivalued-fields.asciidoc b/docs/reference/esql/multivalued-fields.asciidoc
@@ -17,7 +17,7 @@ POST /mv/_bulk?refresh
 
 POST /_query
 {
-  "query": "FROM mv"
+  "query": "FROM mv | LIMIT 2"
 }
 ----
 
@@ -65,7 +65,7 @@ POST /mv/_bulk?refresh
 
 POST /_query
 {
-  "query": "FROM mv"
+  "query": "FROM mv | LIMIT 2"
 }
 ----
 
@@ -106,7 +106,7 @@ POST /mv/_bulk?refresh
 
 POST /_query
 {
-  "query": "FROM mv"
+  "query": "FROM mv | LIMIT 2"
 }
 ----
 
@@ -148,7 +148,7 @@ POST /mv/_bulk?refresh
 
 POST /_query
 {
-  "query": "FROM mv | EVAL b=TO_STRING(b)"
+  "query": "FROM mv | EVAL b=TO_STRING(b) | LIMIT 2"
 }
 ----
 
@@ -183,7 +183,7 @@ POST /mv/_bulk?refresh
 
 POST /_query
 {
-  "query": "FROM mv | EVAL b + 2, a + b"
+  "query": "FROM mv | EVAL b + 2, a + b | LIMIT 4"
 }
 ----
 
@@ -217,7 +217,7 @@ Work around this limitation by converting the field to single value with one of:
 ----
 POST /_query
 {
-  "query": "FROM mv | EVAL b=MV_MIN(b) | EVAL b + 2, a + b"
+  "query": "FROM mv | EVAL b=MV_MIN(b) | EVAL b + 2, a + b | LIMIT 4"
 }
 ----
 // TEST[continued]

diff --git a/...n/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java b/...n/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java
@@ -24,6 +24,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -248,6 +249,10 @@ private void removeEnrichPolicy() throws Exception {
     }
 
     private Response runESQLCommand(String user, String command) throws IOException {
+        if (command.toLowerCase(Locale.ROOT).contains("limit") == false) {
+            // add a (high) limit to avoid warnings on default limit
+            command += " | limit 10000000";
+        }
         Settings pragmas = Settings.EMPTY;
         if (Build.current().isSnapshot()) {
             Settings.Builder settings = Settings.builder();

diff --git a/...-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/HeapAttackIT.java b/...-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/HeapAttackIT.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.WarningsHandler;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -277,6 +278,7 @@ private Response query(String query, String filterPath) throws IOException {
         request.setOptions(
             RequestOptions.DEFAULT.toBuilder()
                 .setRequestConfig(RequestConfig.custom().setSocketTimeout(Math.toIntExact(TimeValue.timeValueMinutes(5).millis())).build())
+                .setWarningsHandler(WarningsHandler.PERMISSIVE)
         );
         return client().performRequest(request);
     }

diff --git a/.../esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml b/.../esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml
@@ -18,6 +18,7 @@ setup:
       warnings:
         - "Line 1:37: evaluation of [to_ip(coalesce(ip1.keyword, \"255.255.255.255\"))] failed, treating result as null. Only first 20 failures recorded."
         - "Line 1:37: java.lang.IllegalArgumentException: '127.0' is not an IP string literal."
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'FROM test | sort emp_no | eval ip = to_ip(coalesce(ip1.keyword, "255.255.255.255")) | keep emp_no, ip'
@@ -36,6 +37,7 @@ setup:
       warnings:
         - "Line 1:98: evaluation of [to_ip(x2)] failed, treating result as null. Only first 20 failures recorded."
         - "Line 1:98: java.lang.IllegalArgumentException: '127.00.1' is not an IP string literal."
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'FROM test | sort emp_no | eval x1 = concat(ip1, ip2), x2 = coalesce(x1, "255.255.255.255"), x3 = to_ip(x2) | keep emp_no, x*'

diff --git a/...gin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/...gin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml
@@ -1,5 +1,7 @@
 ---
 setup:
+  - skip:
+      features: allowed_warnings_regex
   - do:
       indices.create:
         index:  test
@@ -109,6 +111,8 @@ setup:
 ---
 "Test From":
   - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
       esql.query:
         body:
           query: 'from test'
@@ -266,6 +270,8 @@ setup:
 ---
 "Test Input Params":
   - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
       esql.query:
         body:
           query: 'row a = ? | eval b = ?, c = 1 + ?'
@@ -283,6 +289,8 @@ setup:
 
 
   - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
       esql.query:
         body:
           query: 'from test | where color == ? and count == ? and time == ? | keep data, count, color'

diff --git a/...ugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml b/...ugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml
@@ -1,5 +1,7 @@
 ---
 setup:
+  - skip:
+      features: warnings
   - do:
       indices.create:
         index:  test
@@ -109,6 +111,8 @@ setup:
 ---
 "Test From":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test'
@@ -130,6 +134,8 @@ setup:
 ---
 "Test simple grouping avg":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | where color == "red" | stats avg(data) by color'
@@ -144,6 +150,8 @@ setup:
 ---
 "Test From Stats Avg":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats avg(count)'
@@ -156,6 +164,8 @@ setup:
 ---
 "Test From Stats Avg With Alias":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats f1 = avg(count)'
@@ -168,6 +178,8 @@ setup:
 ---
 "Test From Stats Count":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats count(data)'
@@ -180,6 +192,8 @@ setup:
 ---
 "Test From Stats Count With Alias":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats dataCount = count(data)'
@@ -192,6 +206,8 @@ setup:
 ---
 "Test From Stats Min":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats min(count)'
@@ -204,6 +220,8 @@ setup:
 ---
 "Test From Stats Min With Alias":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats minCount=min(count)'
@@ -216,6 +234,8 @@ setup:
 ---
 "Test From Stats Max":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats max(count)'
@@ -228,6 +248,8 @@ setup:
 ---
 "Test From Stats Max With Alias":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats maxCount=max(count)'
@@ -255,6 +277,8 @@ setup:
 ---
 "Test Median On Long":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats med=median(count)'
@@ -267,6 +291,8 @@ setup:
 ---
 "Test Median On Double":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats med=median(count_d)'
@@ -279,6 +305,8 @@ setup:
 ---
 "Test Grouping Median On Long":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats med=median(count) by color | sort med'
@@ -294,6 +322,8 @@ setup:
 ---
 "Test Grouping Median On Double":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats med=median(count_d) by color | sort med'
@@ -309,6 +339,8 @@ setup:
 ---
 "Test Median Absolute Deviation On Long":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats med=median_absolute_deviation(count)'
@@ -321,6 +353,8 @@ setup:
 ---
 "Test Median Absolute Deviation On Double":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats med=median_absolute_deviation(count_d)'
@@ -333,6 +367,8 @@ setup:
 ---
 "Test Grouping Median Absolute Deviation On Long":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats med=median_absolute_deviation(count) by color | sort color'
@@ -348,6 +384,8 @@ setup:
 ---
 "Test Grouping Median Absolute Deviation On Double":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats med=median_absolute_deviation(count_d) by color | sort color'
@@ -363,6 +401,8 @@ setup:
 ---
 "Test From Stats Eval":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats avg_count = avg(count) | eval x = avg_count + 7'
@@ -374,6 +414,8 @@ setup:
 ---
 "Test Stats Where":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | stats x = avg(count) | where x > 100'
@@ -396,6 +438,8 @@ setup:
 ---
 "Test Eval Row With Null":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'row a = 1, b = 2, c = null | eval z = c + b + a'
@@ -419,6 +463,8 @@ setup:
 ---
 "Test Eval With Null And Count":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'from test | eval nullsum = count_d + null | stats count(nullsum)'
@@ -433,6 +479,8 @@ setup:
 ---
 "Test Eval With Multiple Expressions":
   - do:
+      warnings:
+        - "No limit defined, adding default limit of [500]"
       esql.query:
         body:
           query: 'row l=1, d=1.0, ln=1 + null, dn=1.0 + null | stats sum(l), sum(d), sum(ln), sum(dn)'