Adding aggregations support for the _ignored field

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -167,7 +167,7 @@ public void apply(Project project) {
             nonInputSystemProperties.systemProperty(TESTS_MAX_PARALLEL_FORKS_SYSPROP, () -> String.valueOf(task.getMaxParallelForks()));
 
             // Disable test failure reporting since this stuff is now captured in build scans
-            task.getExtensions().getExtraProperties().set(ElasticsearchTestBasePlugin.DUMP_OUTPUT_ON_FAILURE_PROP_NAME, false);
+            task.getExtensions().getExtraProperties().set(ElasticsearchTestBasePlugin.DUMP_OUTPUT_ON_FAILURE_PROP_NAME, true);
 
             // Disable the security manager and syscall filter since the test framework needs to fork processes
             task.systemProperty("tests.security.manager", "false");

docs/changelog/101373.yaml

@@ -0,0 +1,6 @@
+pr: 101373
+summary: Adding aggregations support for the `_ignored` field
+area: Search
+type: feature
+issues:
+ - 59946

docs/reference/mapping/fields/ignored-field.asciidoc

@@ -43,3 +43,20 @@ GET _search
   }
 }
 --------------------------------------------------
+
+Since 8.12.0, the `_ignored` field supports aggregations as well.
+For example, the below query finds all fields that got ignored:
+
+[source,console]
+--------------------------------------------------
+GET _search
+{
+  "aggs": {
+    "ignored_fields": {
+      "terms": {
+         "field": "_ignored"
+      }
+    }
+  }
+}
+--------------------------------------------------

plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java

@@ -281,8 +281,8 @@ public void testIgnoreAbove() throws IOException {
         fields = doc.rootDoc().getFields("field");
         assertThat(fields, empty());
         fields = doc.rootDoc().getFields("_ignored");
-        assertEquals(1, fields.size());
-        assertEquals("field", fields.get(0).stringValue());
+        assertEquals(2, fields.size());
+        assertTrue(fields.stream().anyMatch(field -> "field".equals(field.stringValue())));
     }
 
     public void testUpdateIgnoreAbove() throws IOException {

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml

@@ -42,9 +42,10 @@ fetch fields:
   - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 }
   - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 }
   - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] }
-  - length: { profile.shards.0.fetch.children: 2 }
+  - length: { profile.shards.0.fetch.children: 3 }
+  - match: { profile.shards.0.fetch.children.1.type: FetchIgnoredMetadataPhase }
   - match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase }
-  - match: { profile.shards.0.fetch.children.1.type: StoredFieldsPhase }
+  - match: { profile.shards.0.fetch.children.2.type: StoredFieldsPhase }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
@@ -71,14 +72,15 @@ fetch source:
   - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 }
   - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 }
   - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] }
-  - length: { profile.shards.0.fetch.children: 2 }
-  - match: { profile.shards.0.fetch.children.0.type: FetchSourcePhase }
-  - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
-  - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
-  - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
-  - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
-  - match: { profile.shards.0.fetch.children.0.debug.fast_path: 1 }
-  - match: { profile.shards.0.fetch.children.1.type: StoredFieldsPhase }
+  - length: { profile.shards.0.fetch.children: 3 }
+  - match: { profile.shards.0.fetch.children.1.type: FetchSourcePhase }
+  - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
+  - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
+  - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
+  - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
+  - match: { profile.shards.0.fetch.children.1.debug.fast_path: 1 }
+  - match: { profile.shards.0.fetch.children.2.type: StoredFieldsPhase }
+  - match: { profile.shards.0.fetch.children.0.type: FetchIgnoredMetadataPhase }
 
 ---
 fetch nested source:
@@ -135,18 +137,23 @@ fetch nested source:
   - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 }
   - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 }
   - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] }
-  - length: { profile.shards.0.fetch.children: 3 }
-  - match: { profile.shards.0.fetch.children.0.type: FetchSourcePhase }
+  - length: { profile.shards.0.fetch.children: 4 }
+  - match: { profile.shards.0.fetch.children.0.type: FetchIgnoredMetadataPhase }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
   - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
-  - match: { profile.shards.0.fetch.children.1.type: InnerHitsPhase }
+  - match: { profile.shards.0.fetch.children.1.type: FetchSourcePhase }
   - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
   - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
   - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
   - gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
-  - match: { profile.shards.0.fetch.children.2.type: StoredFieldsPhase }
+  - match: { profile.shards.0.fetch.children.2.type: InnerHitsPhase }
+  - gt: { profile.shards.0.fetch.children.2.breakdown.next_reader_count: 0 }
+  - gt: { profile.shards.0.fetch.children.2.breakdown.next_reader: 0 }
+  - gt: { profile.shards.0.fetch.children.2.breakdown.next_reader_count: 0 }
+  - gt: { profile.shards.0.fetch.children.2.breakdown.next_reader: 0 }
+  - match: { profile.shards.0.fetch.children.3.type: StoredFieldsPhase }
 
 ---
 disabling stored fields removes fetch sub phases:

server/src/internalClusterTest/java/org/elasticsearch/index/mapper/IgnoreFieldIT.java

@@ -0,0 +1,144 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.document.DocumentField;
+import org.elasticsearch.index.query.ExistsQueryBuilder;
+import org.elasticsearch.index.query.IdsQueryBuilder;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
+import org.elasticsearch.search.aggregations.metrics.InternalAvg;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.junit.Before;
+
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.hasSize;
+
+@SuppressWarnings("resource")
+public class IgnoreFieldIT extends ESIntegTestCase {
+
+    public static final String NUMERIC_FIELD_NAME = "numeric_field";
+    public static final String DATE_FIELD_NAME = "date_field";
+    public static final String TEST_INDEX = "test-index";
+    public static final String CORRECT_FIELD_TYPE_DOC_ID = "1";
+    public static final String WRONG_FIELD_TYPE_DOC_ID = "2";
+
+    @Before
+    public void createTestIndex() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("_doc")
+            .startObject("properties")
+            .startObject(NUMERIC_FIELD_NAME)
+            .field("type", "long")
+            .field("ignore_malformed", true)
+            .endObject()
+            .startObject(DATE_FIELD_NAME)
+            .field("type", "date")
+            .field("ignore_malformed", true)
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject();
+        assertAcked(indicesAdmin().prepareCreate(TEST_INDEX).setMapping(mapping));
+        indexTestDoc(NUMERIC_FIELD_NAME, CORRECT_FIELD_TYPE_DOC_ID, "42");
+        indexTestDoc(NUMERIC_FIELD_NAME, WRONG_FIELD_TYPE_DOC_ID, "forty-two");
+    }
+
+    public void testIgnoreFieldFetch() {
+        SearchResponse searchResponse = prepareSearch().setQuery(new IdsQueryBuilder().addIds(CORRECT_FIELD_TYPE_DOC_ID))
+            .addFetchField(NUMERIC_FIELD_NAME)
+            .get();
+        assertHitCount(searchResponse, 1);
+        SearchHit hit = searchResponse.getHits().getAt(0);
+        DocumentField numericField = hit.field(NUMERIC_FIELD_NAME);
+        assertNotNull(numericField);
+        assertEquals(42, (long) numericField.getValue());
+        DocumentField ignoredField = hit.field(IgnoredFieldMapper.NAME);
+        assertNull(ignoredField);
+
+        searchResponse = prepareSearch().setQuery(new IdsQueryBuilder().addIds(WRONG_FIELD_TYPE_DOC_ID))
+            .addFetchField(NUMERIC_FIELD_NAME)
+            .get();
+        assertHitCount(searchResponse, 1);
+        hit = searchResponse.getHits().getAt(0);
+        numericField = hit.field(NUMERIC_FIELD_NAME);
+        assertNotNull(numericField);
+        assertEquals("forty-two", numericField.getIgnoredValues().get(0));
+        ignoredField = hit.field(IgnoredFieldMapper.NAME);
+        assertNotNull(ignoredField);
+        assertEquals(NUMERIC_FIELD_NAME, ignoredField.getValue());
+    }
+
+    public void testExistsQuery() {
+        SearchResponse searchResponse = null;
+        try {
+            searchResponse = prepareSearch().setQuery(new ExistsQueryBuilder(IgnoredFieldMapper.NAME))
+                .addFetchField(NUMERIC_FIELD_NAME)
+                .get();
+            assertHitCount(searchResponse, 1);
+            SearchHit hit = searchResponse.getHits().getAt(0);
+            assertEquals(WRONG_FIELD_TYPE_DOC_ID, hit.getId());
+        } finally {
+            if (searchResponse != null) {
+                searchResponse.decRef();
+            }
+        }
+    }
+
+    public void testIgnoreFieldAggregation() {
+        SearchResponse avgSearch = null;
+        try {
+            indexTestDoc(NUMERIC_FIELD_NAME, "correct-44", "44");
+            avgSearch = prepareSearch(TEST_INDEX).setSize(0).addAggregation(avg("numeric-field-aggs").field(NUMERIC_FIELD_NAME)).get();
+            assertTrue(avgSearch.hasAggregations());
+            InternalAvg avg = avgSearch.getAggregations().get("numeric-field-aggs");
+            assertNotNull(avg);
+            assertEquals(43.0, avg.getValue(), 0.0);
+
+            indexTestDoc(NUMERIC_FIELD_NAME, "wrong-44", "forty-four");
+            indexTestDoc(DATE_FIELD_NAME, "wrong-date", "today");
+            SearchResponse termsSearch = prepareSearch(TEST_INDEX).setSize(0)
+                .addAggregation(terms("ignored-field-aggs").field(IgnoredFieldMapper.NAME))
+                .get();
+            assertTrue(termsSearch.hasAggregations());
+            StringTerms terms = termsSearch.getAggregations().get("ignored-field-aggs");
+            assertNotNull(terms);
+            assertThat(terms.getBuckets(), hasSize(2));
+            StringTerms.Bucket numericFieldBucket = terms.getBucketByKey(NUMERIC_FIELD_NAME);
+            assertEquals(NUMERIC_FIELD_NAME, numericFieldBucket.getKeyAsString());
+            assertEquals(2, numericFieldBucket.getDocCount());
+            StringTerms.Bucket dateFieldBucket = terms.getBucketByKey(DATE_FIELD_NAME);
+            assertEquals(DATE_FIELD_NAME, dateFieldBucket.getKeyAsString());
+            assertEquals(1, dateFieldBucket.getDocCount());
+        } finally {
+            if (avgSearch != null) {
+                avgSearch.decRef();
+            }
+        }
+    }
+
+    private static void indexTestDoc(String testField, String docId, String testValue) {
+        DocWriteResponse docWriteResponse = client().prepareIndex(TEST_INDEX)
+            .setId(docId)
+            .setSource(testField, testValue)
+            .setRefreshPolicy(IMMEDIATE)
+            .get();
+        assertEquals(RestStatus.CREATED, docWriteResponse.status());
+    }
+}