ESQL: Fix SearchStats#count(String) to count values not rows (elastic…

…#104891) SearchStats#count incorrectly counts the number of documents (or rows) in which a document appears instead of the actual number of values. This PR fixes this by looking at the term frequency instead of the doc count. Fix elastic#104795
mark-vieira · Jan 30, 2024 · 202a81f · 202a81f
1 parent 4567a84
commit 202a81f
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 2 deletions.
diff --git a/docs/changelog/104891.yaml b/docs/changelog/104891.yaml
@@ -0,0 +1,6 @@
+pr: 104891
+summary: "ESQL: Fix `SearchStats#count(String)` to count values not rows"
+area: ES|QL
+type: bug
+issues:
+ - 104795
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec
@@ -983,3 +983,25 @@ ROW a = 1, c = null
 COUNT(c):long | a:integer
             0 | 1
 ;
+
+countMultiValuesRow
+ROW keyword_field = ["foo", "bar"], int_field = [1, 2, 3] | STATS ck = COUNT(keyword_field), ci = COUNT(int_field), c = COUNT(*);
+
+ck:l | ci:l | c:l 
+2    | 3    | 1
+;
+
+
+countSource
+FROM employees | 
+STATS ck = COUNT(job_positions), 
+      cb = COUNT(is_rehired), 
+      cd = COUNT(salary_change), 
+      ci = COUNT(salary_change.int), 
+      c = COUNT(*), 
+      csv = COUNT(emp_no);
+
+ck:l | cb:l | cd:l | ci:l | c:l | csv:l 
+221  | 204  | 183  | 183  | 100 | 100
+;
+
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchStats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchStats.java
@@ -323,12 +323,12 @@ private static int countEntries(IndexReader indexReader, String field) {
                     if (fieldInfo.getPointIndexDimensionCount() > 0) {
                         PointValues points = reader.getPointValues(field);
                         if (points != null) {
-                            count += points.getDocCount();
+                            count += points.size();
                         }
                     } else if (fieldInfo.getIndexOptions() != IndexOptions.NONE) {
                         Terms terms = reader.terms(field);
                         if (terms != null) {
-                            count += terms.getDocCount();
+                            count += terms.getSumTotalTermFreq();
                         }
                     } else {
                         return -1; // no shortcut possible for fields that are not indexed