diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index ebb964bb648ca..6184b18048be9 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -28,11 +28,9 @@ import org.apache.lucene.search.Query; import org.elasticsearch.action.search.SearchShardTask; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; @@ -83,9 +81,7 @@ final class DefaultSearchContext extends SearchContext { private final SearchShardTarget shardTarget; private final LongSupplier relativeTimeSupplier; private SearchType searchType; - private final BigArrays bigArrays; private final IndexShard indexShard; - private final ClusterService clusterService; private final IndexService indexService; private final ContextIndexSearcher searcher; private final DfsSearchResult dfsResult; @@ -146,8 +142,6 @@ final class DefaultSearchContext extends SearchContext { DefaultSearchContext(ReaderContext readerContext, ShardSearchRequest request, SearchShardTarget shardTarget, - ClusterService clusterService, - BigArrays bigArrays, LongSupplier relativeTimeSupplier, TimeValue timeout, FetchPhase fetchPhase, @@ -157,14 +151,11 @@ final class DefaultSearchContext extends SearchContext { this.fetchPhase = fetchPhase; this.searchType = request.searchType(); this.shardTarget = shardTarget; - // SearchContexts use a BigArrays that can circuit break - this.bigArrays = bigArrays.withCircuitBreaking(); this.dfsResult = new DfsSearchResult(readerContext.id(), shardTarget, request); this.queryResult = new QuerySearchResult(readerContext.id(), shardTarget, request); this.fetchResult = new FetchSearchResult(readerContext.id(), shardTarget); this.indexService = readerContext.indexService(); this.indexShard = readerContext.indexShard(); - this.clusterService = clusterService; Engine.Searcher engineSearcher = readerContext.acquireSearcher("search"); this.searcher = new ContextIndexSearcher(engineSearcher.getIndexReader(), engineSearcher.getSimilarity(), @@ -457,11 +448,6 @@ public IndexShard indexShard() { return this.indexShard; } - @Override - public BigArrays bigArrays() { - return bigArrays; - } - @Override public BitsetFilterCache bitsetFilterCache() { return indexService.cache().bitsetFilterCache(); diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 57f0da88afd2d..8c416d9b0912f 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -768,8 +768,8 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear try { SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().getId(), reader.indexShard().shardId(), request.getClusterAlias(), OriginalIndices.NONE); - searchContext = new DefaultSearchContext(reader, request, shardTarget, clusterService, - bigArrays, threadPool::relativeTimeInMillis, timeout, fetchPhase, lowLevelCancellation); + searchContext = new DefaultSearchContext(reader, request, shardTarget, + threadPool::relativeTimeInMillis, timeout, fetchPhase, lowLevelCancellation); // we clone the query shard context here just for rewriting otherwise we // might end up with incorrect state since we are using now() or script services // during rewrite and normalized / evaluate templates etc. diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java index fe516114332c6..df13bb1d19c80 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -401,11 +400,6 @@ public String getType() { return NAME; } - @Override - protected AggregationBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException { - return super.doRewrite(queryShardContext); - } - @Override protected ValuesSourceRegistry.RegistryKey getRegistryKey() { return REGISTRY_KEY; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java index b5008f1e4adf1..cc6648113156a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java @@ -230,6 +230,12 @@ public final AggregationUsageService getUsageService() { */ public abstract Analyzer getIndexAnalyzer(Function unindexedFieldAnalyzer); + /** + * Is this request cacheable? Requests that have + * non-deterministic queries or scripts aren't cachable. + */ + public abstract boolean isCacheable(); + /** * Implementation of {@linkplain AggregationContext} for production usage * that wraps our ubiquitous {@link QueryShardContext} and anything else @@ -239,6 +245,7 @@ public final AggregationUsageService getUsageService() { */ public static class ProductionAggregationContext extends AggregationContext { private final QueryShardContext context; + private final BigArrays bigArrays; private final Query topLevelQuery; private final AggregationProfiler profiler; private final MultiBucketConsumer multiBucketConsumer; @@ -277,6 +284,7 @@ public ProductionAggregationContext( Supplier isCancelled ) { this.context = context; + this.bigArrays = context.bigArrays().withCircuitBreaking(); // We can break in searches. this.topLevelQuery = topLevelQuery; this.profiler = profiler; this.multiBucketConsumer = multiBucketConsumer; @@ -343,7 +351,7 @@ public ValuesSourceRegistry getValuesSourceRegistry() { @Override public BigArrays bigArrays() { - return context.bigArrays(); + return bigArrays; } @Override @@ -425,5 +433,10 @@ public CircuitBreaker breaker() { public Analyzer getIndexAnalyzer(Function unindexedFieldAnalyzer) { return context.getIndexAnalyzer(unindexedFieldAnalyzer); } + + @Override + public boolean isCacheable() { + return context.isCacheable(); + } } } diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java index f3c6bcebc9255..b4ae043bdb206 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java @@ -25,7 +25,6 @@ import org.elasticsearch.action.search.SearchShardTask; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryShardContext; @@ -200,11 +199,6 @@ public IndexShard indexShard() { return in.indexShard(); } - @Override - public BigArrays bigArrays() { - return in.bigArrays(); - } - @Override public BitsetFilterCache bitsetFilterCache() { return in.bitsetFilterCache(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 5a299ddfc2229..59e9aabb64810 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryShardContext; @@ -210,8 +209,6 @@ public final void assignRescoreDocIds(RescoreDocIds rescoreDocIds) { public abstract IndexShard indexShard(); - public abstract BigArrays bigArrays(); // TODO this is only used in aggs land and should be contained - public abstract BitsetFilterCache bitsetFilterCache(); public abstract TimeValue timeout(); diff --git a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java index f587f867dc512..1378161bf2632 100644 --- a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.logging.MockAppender; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -82,10 +81,9 @@ protected SearchContext createSearchContext(IndexService indexService) { } protected SearchContext createSearchContext(IndexService indexService, String... groupStats) { - BigArrays bigArrays = indexService.getBigArrays(); final ShardSearchRequest request = new ShardSearchRequest(new ShardId(indexService.index(), 0), 0L, null); - return new TestSearchContext(bigArrays, indexService) { + return new TestSearchContext(indexService) { @Override public List groupStats() { return Arrays.asList(groupStats); diff --git a/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java b/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java index 38292f61ccaa8..2d3c1c7b36c99 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.index.shard; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.internal.ReaderContext; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.ESTestCase; @@ -139,7 +140,7 @@ public void validateReaderContext(ReaderContext readerContext, TransportRequest Collections.shuffle(indexingOperationListeners, random()); SearchOperationListener.CompositeListener compositeListener = new SearchOperationListener.CompositeListener(indexingOperationListeners, logger); - SearchContext ctx = new TestSearchContext(null); + SearchContext ctx = new TestSearchContext((QueryShardContext) null); compositeListener.onQueryPhase(ctx, timeInNanos.get()); assertEquals(0, preFetch.get()); assertEquals(0, preQuery.get()); diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index 2f406a50bab85..e4384bd344a36 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -34,9 +34,6 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.common.util.MockBigArrays; -import org.elasticsearch.common.util.MockPageCacheRecycler; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.IndexCache; @@ -49,7 +46,6 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.LegacyReaderContext; import org.elasticsearch.search.internal.ReaderContext; @@ -122,8 +118,6 @@ public void testPreProcess() throws Exception { when(indexService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.getIndexSettings()).thenReturn(indexSettings); - BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()); - try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { @@ -150,7 +144,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) { ReaderContext readerWithoutScroll = new ReaderContext( newContextId(), indexService, indexShard, searcherSupplier.get(), randomNonNegativeLong(), false); DefaultSearchContext contextWithoutScroll = new DefaultSearchContext(readerWithoutScroll, shardSearchRequest, target, null, - bigArrays, null, timeout, null, false); + timeout, null, false); contextWithoutScroll.from(300); contextWithoutScroll.close(); @@ -166,7 +160,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) { ReaderContext readerContext = new LegacyReaderContext( newContextId(), indexService, indexShard, searcherSupplier.get(), shardSearchRequest, randomNonNegativeLong()); DefaultSearchContext context1 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null, - bigArrays, null, timeout, null, false); + timeout, null, false); context1.from(300); exception = expectThrows(IllegalArgumentException.class, () -> context1.preProcess(false)); assertThat(exception.getMessage(), equalTo("Batch size is too large, size must be less than or equal to: [" @@ -200,7 +194,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) { newContextId(), indexService, indexShard, searcherSupplier.get(), randomNonNegativeLong(), false); // rescore is null but sliceBuilder is not null DefaultSearchContext context2 = new DefaultSearchContext(readerContext, shardSearchRequest, target, - null, bigArrays, null, timeout, null, false); + null, timeout, null, false); SliceBuilder sliceBuilder = mock(SliceBuilder.class); int numSlices = maxSlicesPerScroll + randomIntBetween(1, 100); @@ -217,7 +211,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) { when(shardSearchRequest.indexBoost()).thenReturn(AbstractQueryBuilder.DEFAULT_BOOST); DefaultSearchContext context3 = new DefaultSearchContext(readerContext, shardSearchRequest, target, null, - bigArrays, null, timeout, null, false); + timeout, null, false); ParsedQuery parsedQuery = ParsedQuery.parsedMatchAllQuery(); context3.sliceBuilder(null).parsedQuery(parsedQuery).preProcess(false); assertEquals(context3.query(), context3.buildFilteredQuery(parsedQuery.query())); @@ -229,7 +223,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) { readerContext = new ReaderContext(newContextId(), indexService, indexShard, searcherSupplier.get(), randomNonNegativeLong(), false); DefaultSearchContext context4 = - new DefaultSearchContext(readerContext, shardSearchRequest, target, null, bigArrays, null, timeout, null, false); + new DefaultSearchContext(readerContext, shardSearchRequest, target, null, timeout, null, false); context4.sliceBuilder(new SliceBuilder(1,2)).parsedQuery(parsedQuery).preProcess(false); Query query1 = context4.query(); context4.sliceBuilder(new SliceBuilder(0,2)).parsedQuery(parsedQuery).preProcess(false); @@ -256,8 +250,6 @@ public void testClearQueryCancellationsOnClose() throws IOException { IndexService indexService = mock(IndexService.class); - BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()); - try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { @@ -282,7 +274,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) { ReaderContext readerContext = new ReaderContext( newContextId(), indexService, indexShard, searcherSupplier, randomNonNegativeLong(), false); DefaultSearchContext context = new DefaultSearchContext(readerContext, shardSearchRequest, target, null, - bigArrays, null, timeout, null, false); + timeout, null, false); assertThat(context.searcher().hasCancellations(), is(false)); context.searcher().addQueryCancellation(() -> {}); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregatorTests.java index 0cea28e5f4d0f..f6e0183070038 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregatorTests.java @@ -32,14 +32,10 @@ import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.LeafBucketCollector; -import org.elasticsearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer; -import org.elasticsearch.search.aggregations.support.AggregationContext.ProductionAggregationContext; -import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.aggregations.support.AggregationContext; import java.io.IOException; -import static org.mockito.Mockito.mock; - public class BucketsAggregatorTests extends AggregatorTestCase{ public BucketsAggregator buildMergeAggregator() throws IOException{ @@ -53,12 +49,11 @@ public BucketsAggregator buildMergeAggregator() throws IOException{ try (IndexReader indexReader = DirectoryReader.open(directory)) { IndexSearcher indexSearcher = new IndexSearcher(indexReader); - SearchContext searchContext = createSearchContext( + AggregationContext context = createAggregationContext( indexSearcher, null, new NumberFieldMapper.NumberFieldType("test", NumberFieldMapper.NumberType.INTEGER) ); - ProductionAggregationContext context = new ProductionAggregationContext(searchContext, mock(MultiBucketConsumer.class)); return new BucketsAggregator("test", AggregatorFactories.EMPTY, context, null, null, null) { @Override diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilderTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilderTests.java index 9af7f6cc0c714..304e19636f10e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilderTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.QueryBuilder; @@ -62,6 +63,7 @@ public void testFilterSizeLimitation() throws Exception { IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); when(indexShard.indexSettings()).thenReturn(indexSettings); when(queryShardContext.getIndexSettings()).thenReturn(indexSettings); + when(queryShardContext.bigArrays()).thenReturn(BigArrays.NON_RECYCLING_INSTANCE); when(indexShard.shardId()).thenReturn(new ShardId(new Index("test", "test"), 1)); SearchContext context = new TestSearchContext(queryShardContext, indexShard); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java index 5bc00a2e7dc28..09d74f6f9c106 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java @@ -38,8 +38,8 @@ import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; -import org.elasticsearch.search.internal.SearchContext; import org.hamcrest.Matcher; import java.io.IOException; @@ -1210,7 +1210,7 @@ private void aggregationImplementationChoiceTestCase( ); } try (IndexReader reader = indexWriter.getReader()) { - SearchContext context = createSearchContext(new IndexSearcher(reader), new MatchAllDocsQuery(), ft); + AggregationContext context = createAggregationContext(new IndexSearcher(reader), new MatchAllDocsQuery(), ft); Aggregator agg = createAggregator(builder, context); Matcher matcher = instanceOf(DateHistogramAggregator.FromDateRange.class); if (usesFromRange == false) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index f5697e0153f50..c119cc1e48458 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -88,11 +88,11 @@ import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.BucketScriptPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree; +import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; import org.elasticsearch.test.geo.RandomGeoGenerator; @@ -142,7 +142,8 @@ protected ScriptService getMockScriptService() { return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); } - protected A createAggregator(AggregationBuilder aggregationBuilder, SearchContext context) throws IOException { + protected A createAggregator(AggregationBuilder aggregationBuilder, AggregationContext context) + throws IOException { try { if (randomizeAggregatorImpl) { TermsAggregatorFactory.COLLECT_SEGMENT_ORDS = randomBoolean(); @@ -253,7 +254,7 @@ public void testSimple() throws Exception { .order(BucketOrder.key(true)); MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("string"); - SearchContext context = createSearchContext(indexSearcher, null, fieldType); + AggregationContext context = createAggregationContext(indexSearcher, null, fieldType); TermsAggregator aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -322,7 +323,7 @@ public void testStringIncludeExclude() throws Exception { .size(12) .order(BucketOrder.key(true)); - SearchContext context = createSearchContext(indexSearcher, null, fieldType); + AggregationContext context = createAggregationContext(indexSearcher, null, fieldType); TermsAggregator aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -357,7 +358,7 @@ public void testStringIncludeExclude() throws Exception { .field("sv_field") .order(BucketOrder.key(true)); - context = createSearchContext(indexSearcher, null, fieldType2); + context = createAggregationContext(indexSearcher, null, fieldType2); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -381,7 +382,7 @@ public void testStringIncludeExclude() throws Exception { .field("mv_field") .order(BucketOrder.key(true)); - context = createSearchContext(indexSearcher, null, fieldType); + context = createAggregationContext(indexSearcher, null, fieldType); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -410,7 +411,7 @@ public void testStringIncludeExclude() throws Exception { .includeExclude(new IncludeExclude(null, "val00.+")) .field("mv_field") .order(BucketOrder.key(true)); - context = createSearchContext(indexSearcher, null, fieldType); + context = createAggregationContext(indexSearcher, null, fieldType); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -427,7 +428,7 @@ public void testStringIncludeExclude() throws Exception { .includeExclude(new IncludeExclude(new String[]{"val000", "val010"}, null)) .field("mv_field") .order(BucketOrder.key(true)); - context = createSearchContext(indexSearcher, null, fieldType); + context = createAggregationContext(indexSearcher, null, fieldType); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -445,7 +446,7 @@ public void testStringIncludeExclude() throws Exception { "val005", "val006", "val007", "val008", "val009", "val011"})) .field("mv_field") .order(BucketOrder.key(true)); - context = createSearchContext(indexSearcher, null, fieldType); + context = createAggregationContext(indexSearcher, null, fieldType); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -463,7 +464,7 @@ public void testStringIncludeExclude() throws Exception { new String[]{"val001", "val002", "val003", "val004", "val005", "val006", "val007", "val008"})) .field("mv_field") .order(BucketOrder.key(true)); - context = createSearchContext(indexSearcher, null, fieldType); + context = createAggregationContext(indexSearcher, null, fieldType); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -481,7 +482,7 @@ public void testStringIncludeExclude() throws Exception { "val002", "val010"}, null)) .field("mv_field") .order(BucketOrder.key(true)); - context = createSearchContext(indexSearcher, null, fieldType); + context = createAggregationContext(indexSearcher, null, fieldType); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -536,7 +537,7 @@ public void testNumericIncludeExclude() throws Exception { .includeExclude(new IncludeExclude(new long[]{0, 5}, null)) .field("long_field") .order(BucketOrder.key(true)); - SearchContext context = createSearchContext(indexSearcher, null, fieldType); + AggregationContext context = createAggregationContext(indexSearcher, null, fieldType); TermsAggregator aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -553,7 +554,7 @@ public void testNumericIncludeExclude() throws Exception { .includeExclude(new IncludeExclude(null, new long[]{0, 5})) .field("long_field") .order(BucketOrder.key(true)); - context = createSearchContext(indexSearcher, null, fieldType); + context = createAggregationContext(indexSearcher, null, fieldType); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -576,7 +577,7 @@ public void testNumericIncludeExclude() throws Exception { .includeExclude(new IncludeExclude(new double[]{0.0, 5.0}, null)) .field("double_field") .order(BucketOrder.key(true)); - context = createSearchContext(indexSearcher, null, fieldType); + context = createAggregationContext(indexSearcher, null, fieldType); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -593,7 +594,7 @@ public void testNumericIncludeExclude() throws Exception { .includeExclude(new IncludeExclude(null, new double[]{0.0, 5.0})) .field("double_field") .order(BucketOrder.key(true)); - context = createSearchContext(indexSearcher, null, fieldType); + context = createAggregationContext(indexSearcher, null, fieldType); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -759,7 +760,7 @@ private void termsAggregator(ValueType valueType, MappedFieldType fieldType, .field("field") .order(bucketOrder); - SearchContext context = createSearchContext(indexSearcher, null, fieldType); + AggregationContext context = createAggregationContext(indexSearcher, null, fieldType); Aggregator aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -785,7 +786,7 @@ private void termsAggregator(ValueType valueType, MappedFieldType fieldType, .size(numTerms) .collectMode(randomFrom(Aggregator.SubAggCollectionMode.values())) .field("field")); - context = createSearchContext(indexSearcher, null, fieldType, filterFieldType); + context = createAggregationContext(indexSearcher, null, fieldType, filterFieldType); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -858,7 +859,7 @@ private void termsAggregatorWithNestedMaxAgg(ValueType valueType, MappedFiel MappedFieldType fieldType2 = new NumberFieldMapper.NumberFieldType("value", NumberFieldMapper.NumberType.LONG); - SearchContext context = createSearchContext(indexSearcher, null, fieldType, fieldType2); + AggregationContext context = createAggregationContext(indexSearcher, null, fieldType, fieldType2); Aggregator aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -887,7 +888,7 @@ public void testEmpty() throws Exception { TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") .userValueTypeHint(ValueType.STRING) .field("string"); - SearchContext context = createSearchContext(indexSearcher, null, fieldType1); + AggregationContext context = createAggregationContext(indexSearcher, null, fieldType1); Aggregator aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -896,7 +897,7 @@ public void testEmpty() throws Exception { assertEquals(0, result.getBuckets().size()); aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.LONG).field("long"); - context = createSearchContext(indexSearcher, null, fieldType2); + context = createAggregationContext(indexSearcher, null, fieldType2); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -905,7 +906,7 @@ public void testEmpty() throws Exception { assertEquals(0, result.getBuckets().size()); aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.DOUBLE).field("double"); - context = createSearchContext(indexSearcher, null, fieldType3); + context = createAggregationContext(indexSearcher, null, fieldType3); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -928,7 +929,7 @@ public void testUnmapped() throws Exception { TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") .userValueTypeHint(valueTypes[i]) .field(fieldNames[i]); - SearchContext context = createSearchContext(indexSearcher, null); + AggregationContext context = createAggregationContext(indexSearcher, null); Aggregator aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -964,7 +965,7 @@ public void testUnmappedWithMissing() throws Exception { TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") .userValueTypeHint(valueTypes[i]) .field(fieldNames[i]).missing(missingValues[i]); - SearchContext context = createSearchContext(indexSearcher, null, fieldType1); + AggregationContext context = createAggregationContext(indexSearcher, null, fieldType1); Aggregator aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -1036,7 +1037,7 @@ public void testIpField() throws Exception { IndexSearcher indexSearcher = newIndexSearcher(indexReader); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") .field(field); // Note - other places we throw IllegalArgumentException - SearchContext context = createSearchContext(indexSearcher, null, fieldType); + AggregationContext context = createAggregationContext(indexSearcher, null, fieldType); Aggregator aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -1084,7 +1085,7 @@ public void testNestedTermsAgg() throws Exception { MappedFieldType fieldType1 = new KeywordFieldMapper.KeywordFieldType("field1"); MappedFieldType fieldType2 = new KeywordFieldMapper.KeywordFieldType("field2"); - SearchContext context = createSearchContext(indexSearcher, null, fieldType1, fieldType2); + AggregationContext context = createAggregationContext(indexSearcher, null, fieldType1, fieldType2); Aggregator aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java index b41d726da54af..b74bb25cc600e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgAggregatorTests.java @@ -51,11 +51,11 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator; +import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.LeafDocLookup; import java.io.IOException; @@ -599,7 +599,7 @@ public void testCacheAggregation() throws IOException { AvgAggregationBuilder aggregationBuilder = new AvgAggregationBuilder("avg") .field("value"); - SearchContext context = createSearchContext(indexSearcher, null, fieldType); + AggregationContext context = createAggregationContext(indexSearcher, null, fieldType); AvgAggregator aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -611,7 +611,7 @@ public void testCacheAggregation() throws IOException { assertTrue(AggregationInspectionHelper.hasValue(avg)); // Test that an aggregation not using a script does get cached - assertTrue(context.getQueryShardContext().isCacheable()); + assertTrue(context.isCacheable()); multiReader.close(); directory.close(); @@ -645,7 +645,7 @@ public void testScriptCaching() throws IOException { .field("value") .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap())); - SearchContext context = createSearchContext(indexSearcher, null, fieldType); + AggregationContext context = createAggregationContext(indexSearcher, null, fieldType); AvgAggregator aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -657,13 +657,13 @@ public void testScriptCaching() throws IOException { assertTrue(AggregationInspectionHelper.hasValue(avg)); // Test that an aggregation using a deterministic script gets cached - assertTrue(context.getQueryShardContext().isCacheable()); + assertTrue(context.isCacheable()); aggregationBuilder = new AvgAggregationBuilder("avg") .field("value") .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, RANDOM_SCRIPT, Collections.emptyMap())); - context = createSearchContext(indexSearcher, null, fieldType); + context = createAggregationContext(indexSearcher, null, fieldType); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -676,7 +676,7 @@ public void testScriptCaching() throws IOException { assertTrue(AggregationInspectionHelper.hasValue(avg)); // Test that an aggregation using a nondeterministic script does not get cached - assertFalse(context.getQueryShardContext().isCacheable()); + assertFalse(context.isCacheable()); multiReader.close(); directory.close(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java index 3b980cbd8bb87..49952e61c2a47 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java @@ -70,11 +70,11 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator; +import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.LeafDocLookup; import java.io.IOException; @@ -907,7 +907,7 @@ public void testCacheAggregation() throws IOException { MaxAggregationBuilder aggregationBuilder = new MaxAggregationBuilder("max") .field("value"); - SearchContext context = createSearchContext(indexSearcher, null, fieldType); + AggregationContext context = createAggregationContext(indexSearcher, null, fieldType); MaxAggregator aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -919,7 +919,7 @@ public void testCacheAggregation() throws IOException { assertTrue(AggregationInspectionHelper.hasValue(max)); // Test that an aggregation not using a script does get cached - assertTrue(context.getQueryShardContext().isCacheable()); + assertTrue(context.isCacheable()); multiReader.close(); directory.close(); @@ -953,7 +953,7 @@ public void testScriptCaching() throws Exception { .field("value") .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap())); - SearchContext context = createSearchContext(indexSearcher, null, fieldType); + AggregationContext context = createAggregationContext(indexSearcher, null, fieldType); MaxAggregator aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -965,12 +965,12 @@ public void testScriptCaching() throws Exception { assertTrue(AggregationInspectionHelper.hasValue(max)); // Test that an aggregation using a script does not get cached - assertTrue(context.getQueryShardContext().isCacheable()); + assertTrue(context.isCacheable()); aggregationBuilder = new MaxAggregationBuilder("max") .field("value") .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, RANDOM_SCRIPT, Collections.emptyMap())); - context = createSearchContext(indexSearcher, null, fieldType); + context = createAggregationContext(indexSearcher, null, fieldType); aggregator = createAggregator(aggregationBuilder, context); aggregator.preCollection(); indexSearcher.search(new MatchAllDocsQuery(), aggregator); @@ -983,7 +983,7 @@ public void testScriptCaching() throws Exception { assertTrue(AggregationInspectionHelper.hasValue(max)); // Test that an aggregation using a nondeterministic script does not get cached - assertFalse(context.getQueryShardContext().isCacheable()); + assertFalse(context.isCacheable()); multiReader.close(); directory.close(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java index 0886ad91015f1..f833f9d8bc3bc 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java @@ -48,15 +48,10 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptEngine; @@ -76,6 +71,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.search.lookup.LeafDocLookup; @@ -99,7 +95,6 @@ public class MinAggregatorTests extends AggregatorTestCase { private final String SCRIPT_NAME = "script_name"; - private QueryShardContext queryShardContext; private final long SCRIPT_VALUE = 19L; /** Script to take a field name in params and sum the values of the field. */ @@ -164,14 +159,6 @@ protected ScriptService getMockScriptService() { return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); } - @Override - protected QueryShardContext queryShardContextMock(IndexSearcher searcher, MapperService mapperService, - IndexSettings indexSettings, CircuitBreakerService circuitBreakerService, - BigArrays bigArrays) { - this.queryShardContext = super.queryShardContextMock(searcher, mapperService, indexSettings, circuitBreakerService, bigArrays); - return queryShardContext; - } - public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7))); @@ -629,11 +616,9 @@ public void testCaching() throws IOException { try (IndexReader indexReader = DirectoryReader.open(directory)) { IndexSearcher indexSearcher = newSearcher(indexReader, true, true); - InternalMin min = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); - assertEquals(2.0, min.getValue(), 0); - assertTrue(AggregationInspectionHelper.hasValue(min)); - - assertTrue(queryShardContext.isCacheable()); + AggregationContext context = createAggregationContext(indexSearcher, new MatchAllDocsQuery(), fieldType); + createAggregator(aggregationBuilder, context); + assertTrue(context.isCacheable()); } } } @@ -660,19 +645,13 @@ public void testScriptCaching() throws IOException { try (IndexReader indexReader = DirectoryReader.open(directory)) { IndexSearcher indexSearcher = newSearcher(indexReader, true, true); - InternalMin min = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), nonDeterministicAggregationBuilder, fieldType); - assertTrue(min.getValue() >= 0.0 && min.getValue() <= 1.0); - assertTrue(AggregationInspectionHelper.hasValue(min)); - - assertFalse(queryShardContext.isCacheable()); - - indexSearcher = newSearcher(indexReader, true, true); - - min = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); - assertEquals(-7.0, min.getValue(), 0); - assertTrue(AggregationInspectionHelper.hasValue(min)); + AggregationContext context = createAggregationContext(indexSearcher, new MatchAllDocsQuery(), fieldType); + createAggregator(nonDeterministicAggregationBuilder, context); + assertFalse(context.isCacheable()); - assertTrue(queryShardContext.isCacheable()); + context = createAggregationContext(indexSearcher, new MatchAllDocsQuery(), fieldType); + createAggregator(aggregationBuilder, context); + assertTrue(context.isCacheable()); } } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index de9bed46fab14..befd469a5bf21 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -457,6 +457,11 @@ public CircuitBreaker breaker() { public Analyzer getIndexAnalyzer(Function unindexedFieldAnalyzer) { throw new UnsupportedOperationException(); } + + @Override + public boolean isCacheable() { + throw new UnsupportedOperationException(); + } }; } diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 4cbe515f5b4b1..8a961ce4b2bdd 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -52,8 +52,8 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; @@ -93,7 +93,9 @@ import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.mapper.TextFieldMapper; +import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesModule; @@ -120,6 +122,7 @@ import org.elasticsearch.search.fetch.subphase.FetchSourcePhase; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.internal.SubSearchContext; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.InternalAggregationTestCase; @@ -146,9 +149,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -159,11 +160,11 @@ */ public abstract class AggregatorTestCase extends ESTestCase { private static final String NESTEDFIELD_PREFIX = "nested_"; - private List releasables = new ArrayList<>(); + private List releasables = new ArrayList<>(); protected ValuesSourceRegistry valuesSourceRegistry; // A list of field types that should not be tested, or are not currently supported - private static List TYPE_TEST_BLACKLIST = List.of( + private static final List TYPE_TEST_BLACKLIST = List.of( ObjectMapper.CONTENT_TYPE, // Cannot aggregate objects GeoShapeFieldMapper.CONTENT_TYPE, // Cannot aggregate geoshapes (yet) @@ -180,8 +181,7 @@ protected Map getFieldAliases(MappedFieldType... fieldT return Collections.emptyMap(); } - private static void registerFieldTypes(SearchContext searchContext, MapperService mapperService, - Map fieldNameToType) { + private static void registerFieldTypes(MapperService mapperService, Map fieldNameToType) { for (Map.Entry entry : fieldNameToType.entrySet()) { String fieldName = entry.getKey(); MappedFieldType fieldType = entry.getValue(); @@ -207,89 +207,57 @@ protected List getSearchPlugins() { protected A createAggregator(AggregationBuilder aggregationBuilder, IndexSearcher searcher, MappedFieldType... fieldTypes) throws IOException { - return createAggregator(aggregationBuilder, createSearchContext(searcher, null, fieldTypes)); + return createAggregator(aggregationBuilder, createAggregationContext(searcher, null, fieldTypes)); } - protected A createAggregator(AggregationBuilder aggregationBuilder, SearchContext searchContext) - throws IOException { - MultiBucketConsumer bucketConsumer = new MultiBucketConsumer( - DEFAULT_MAX_BUCKETS, - searchContext.bigArrays().breakerService().getBreaker(CircuitBreaker.REQUEST) + protected A createAggregator(AggregationBuilder builder, AggregationContext context) throws IOException { + QueryRewriteContext rewriteContext = new QueryRewriteContext( + xContentRegistry(), + new NamedWriteableRegistry(List.of()), + null, + context::nowInMillis ); - return createAggregator(aggregationBuilder, searchContext, bucketConsumer); - } - - protected A createAggregator( - AggregationBuilder aggregationBuilder, - SearchContext searchContext, - MultiBucketConsumer multiBucketConsumer - ) throws IOException { - AggregationContext context = new ProductionAggregationContext(searchContext, multiBucketConsumer); @SuppressWarnings("unchecked") - A aggregator = (A) aggregationBuilder.rewrite(searchContext.getQueryShardContext()) - .build(context, null) - .create(null, CardinalityUpperBound.ONE); + A aggregator = (A) Rewriteable.rewrite(builder, rewriteContext, true).build(context, null).create(null, CardinalityUpperBound.ONE); return aggregator; } /** * Create a {@linkplain SearchContext} for testing an {@link Aggregator}. */ - protected SearchContext createSearchContext( // TODO replace me with AggregationContext + protected AggregationContext createAggregationContext( IndexSearcher indexSearcher, Query query, MappedFieldType... fieldTypes ) throws IOException { CircuitBreakerService breakerService = new NoneCircuitBreakerService(); - return createSearchContext(indexSearcher, createIndexSettings(), query, breakerService, fieldTypes); + return createAggregationContext(indexSearcher, createIndexSettings(), query, breakerService, DEFAULT_MAX_BUCKETS, fieldTypes); } - protected SearchContext createSearchContext(IndexSearcher indexSearcher, // TODO replace me with AggregationContext + protected AggregationContext createAggregationContext(IndexSearcher indexSearcher, IndexSettings indexSettings, Query query, - CircuitBreakerService circuitBreakerService, + CircuitBreakerService breakerService, + int maxBucket, MappedFieldType... fieldTypes) throws IOException { - QueryCache queryCache = new DisabledQueryCache(indexSettings); - QueryCachingPolicy queryCachingPolicy = new QueryCachingPolicy() { - @Override - public void onUse(Query query) { - } - - @Override - public boolean shouldCache(Query query) { - // never cache a query - return false; - } - }; - ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(indexSearcher.getIndexReader(), - indexSearcher.getSimilarity(), queryCache, queryCachingPolicy, false); - - SearchContext searchContext = mock(SearchContext.class); - when(searchContext.numberOfShards()).thenReturn(1); - when(searchContext.searcher()).thenReturn(contextIndexSearcher); - when(searchContext.fetchPhase()) - .thenReturn(new FetchPhase(Arrays.asList(new FetchSourcePhase(), new FetchDocValuesPhase()))); - when(searchContext.bitsetFilterCache()).thenReturn(new BitsetFilterCache(indexSettings, mock(Listener.class))); - IndexShard indexShard = mock(IndexShard.class); - when(indexShard.shardId()).thenReturn(new ShardId("test", "test", 0)); - when(searchContext.indexShard()).thenReturn(indexShard); - when(searchContext.query()).thenReturn(query); /* * Always use the circuit breaking big arrays instance so that the CircuitBreakerService * we're passed gets a chance to break. */ - BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), circuitBreakerService).withCircuitBreaking(); - when(searchContext.bigArrays()).thenReturn(bigArrays); + BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), breakerService).withCircuitBreaking(); // TODO: now just needed for top_hits, this will need to be revised for other agg unit tests: MapperService mapperService = mapperServiceMock(); when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.hasNested()).thenReturn(false); when(mapperService.indexAnalyzer(anyString(), any())).thenReturn(Lucene.STANDARD_ANALYZER); // for significant text - QueryShardContext queryShardContext = - queryShardContextMock(contextIndexSearcher, mapperService, indexSettings, circuitBreakerService, bigArrays); - when(searchContext.getQueryShardContext()).thenReturn(queryShardContext); - when(queryShardContext.getObjectMapper(anyString())).thenAnswer(invocation -> { + Map fieldNameToType = new HashMap<>(); + fieldNameToType.putAll(Arrays.stream(fieldTypes) + .filter(Objects::nonNull) + .collect(Collectors.toMap(MappedFieldType::name, Function.identity()))); + fieldNameToType.putAll(getFieldAliases(fieldTypes)); + registerFieldTypes(mapperService, fieldNameToType); + when(mapperService.getObjectMapper(anyString())).thenAnswer(invocation -> { String fieldName = (String) invocation.getArguments()[0]; if (fieldName.startsWith(NESTEDFIELD_PREFIX)) { return new ObjectMapper.Builder(fieldName, Version.CURRENT).nested(Nested.newNested()).build(new ContentPath()); @@ -297,25 +265,90 @@ public boolean shouldCache(Query query) { return null; }); - NestedDocuments nestedDocuments = new NestedDocuments(mapperService, searchContext.bitsetFilterCache()::getBitSetProducer); - when(searchContext.getNestedDocuments()) - .thenReturn(nestedDocuments); + TriFunction, IndexFieldData> fieldDataBuilder = ( + fieldType, + s, + searchLookup) -> fieldType.fielddataBuilder(mapperService.getIndexSettings().getIndex().getName(), searchLookup) + .build(new IndexFieldDataCache.None(), breakerService); + QueryShardContext queryShardContext = new QueryShardContext( + 0, + indexSettings, + bigArrays, + null, + fieldDataBuilder, + mapperService, + null, + getMockScriptService(), + xContentRegistry(), + writableRegistry(), + null, + indexSearcher, + System::currentTimeMillis, + null, + null, + () -> true, + valuesSourceRegistry, + emptyMap() + ); - Map fieldNameToType = new HashMap<>(); - fieldNameToType.putAll(Arrays.stream(fieldTypes) - .filter(Objects::nonNull) - .collect(Collectors.toMap(MappedFieldType::name, Function.identity()))); - fieldNameToType.putAll(getFieldAliases(fieldTypes)); + MultiBucketConsumer consumer = new MultiBucketConsumer(maxBucket, breakerService.getBreaker(CircuitBreaker.REQUEST)); + BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, mock(Listener.class)); + return new ProductionAggregationContext( + queryShardContext, + query, + null, + consumer, + () -> buildSubSearchContext(mapperService, queryShardContext, bitsetFilterCache), + releasables::add, + bitsetFilterCache, + randomInt(), + () -> 0L, + () -> false + ); + } - registerFieldTypes(searchContext, mapperService, - fieldNameToType); - doAnswer(invocation -> { - /* Store the release-ables so we can release them at the end of the test case. This is important because aggregations don't - * close their sub-aggregations. This is fairly similar to what the production code does. */ - releasables.add((Releasable) invocation.getArguments()[0]); - return null; - }).when(searchContext).addReleasable(anyObject()); - return searchContext; + /** + * Build a {@link SubSearchContext}s to power {@code top_hits}. + */ + private SubSearchContext buildSubSearchContext( + MapperService mapperService, + QueryShardContext queryShardContext, + BitsetFilterCache bitsetFilterCache + ) { + SearchContext ctx = mock(SearchContext.class); + QueryCache queryCache = new DisabledQueryCache(mapperService.getIndexSettings()); + QueryCachingPolicy queryCachingPolicy = new QueryCachingPolicy() { + @Override + public void onUse(Query query) { + } + + @Override + public boolean shouldCache(Query query) { + // never cache a query + return false; + } + }; + try { + when(ctx.searcher()).thenReturn( + new ContextIndexSearcher( + queryShardContext.searcher().getIndexReader(), + queryShardContext.searcher().getSimilarity(), + queryCache, + queryCachingPolicy, + false + ) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + when(ctx.fetchPhase()).thenReturn(new FetchPhase(Arrays.asList(new FetchSourcePhase(), new FetchDocValuesPhase()))); + when(ctx.getQueryShardContext()).thenReturn(queryShardContext); + NestedDocuments nestedDocuments = new NestedDocuments(mapperService, bitsetFilterCache::getBitSetProducer); + when(ctx.getNestedDocuments()).thenReturn(nestedDocuments); + IndexShard indexShard = mock(IndexShard.class); + when(indexShard.shardId()).thenReturn(new ShardId("test", "test", 0)); + when(ctx.indexShard()).thenReturn(indexShard); + return new SubSearchContext(ctx); } protected IndexSettings createIndexSettings() { @@ -336,32 +369,6 @@ protected MapperService mapperServiceMock() { return mock(MapperService.class); } - /** - * sub-tests that need a more complex mock can overwrite this - */ - protected QueryShardContext queryShardContextMock(IndexSearcher searcher, - MapperService mapperService, - IndexSettings indexSettings, - CircuitBreakerService circuitBreakerService, - BigArrays bigArrays) { - - return new QueryShardContext(0, indexSettings, bigArrays, null, - getIndexFieldDataLookup(mapperService, circuitBreakerService), - mapperService, null, getMockScriptService(), xContentRegistry(), - writableRegistry(), null, searcher, System::currentTimeMillis, null, null, () -> true, - valuesSourceRegistry, emptyMap()); - } - - /** - * Sub-tests that need a more complex index field data provider can override this - */ - protected TriFunction, IndexFieldData> getIndexFieldDataLookup( - MapperService mapperService, CircuitBreakerService circuitBreakerService) { - return (fieldType, s, searchLookup) -> fieldType.fielddataBuilder( - mapperService.getIndexSettings().getIndex().getName(), searchLookup) - .build(new IndexFieldDataCache.None(), circuitBreakerService); - } - /** * Sub-tests that need scripting can override this method to provide a script service and pre-baked scripts */ @@ -411,9 +418,8 @@ protected A searchAndReduc List aggs = new ArrayList<>(); Query rewritten = searcher.rewrite(query); CircuitBreakerService breakerService = new NoneCircuitBreakerService(); - MultiBucketConsumer bucketConsumer = new MultiBucketConsumer(maxBucket, breakerService.getBreaker(CircuitBreaker.REQUEST)); - SearchContext searchContext = createSearchContext(searcher, indexSettings, query, breakerService, fieldTypes); - C root = createAggregator(builder, searchContext, bucketConsumer); + AggregationContext context = createAggregationContext(searcher, indexSettings, query, breakerService, maxBucket, fieldTypes); + C root = createAggregator(builder, context); if (randomBoolean() && searcher.getIndexReader().leaves().size() > 0) { assertThat(ctx, instanceOf(CompositeReaderContext.class)); @@ -425,7 +431,7 @@ protected A searchAndReduc subSearchers[searcherIDX] = new ShardSearcher(leave, compCTX); } for (ShardSearcher subSearcher : subSearchers) { - C a = createAggregator(builder, searchContext, bucketConsumer); + C a = createAggregator(builder, context); a.preCollection(); Weight weight = subSearcher.createWeight(rewritten, ScoreMode.COMPLETE, 1f); subSearcher.search(weight, a); @@ -443,9 +449,9 @@ protected A searchAndReduc Collections.shuffle(aggs, random()); int r = randomIntBetween(1, toReduceSize); List toReduce = aggs.subList(0, r); - InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forPartialReduction( - searchContext.bigArrays(), getMockScriptService(), () -> PipelineAggregator.PipelineTree.EMPTY); - A reduced = (A) aggs.get(0).reduce(toReduce, context); + InternalAggregation.ReduceContext reduceContext = InternalAggregation.ReduceContext.forPartialReduction( + context.bigArrays(), getMockScriptService(), () -> PipelineAggregator.PipelineTree.EMPTY); + A reduced = (A) aggs.get(0).reduce(toReduce, reduceContext); aggs = new ArrayList<>(aggs.subList(r, toReduceSize)); aggs.add(reduced); } @@ -453,18 +459,18 @@ protected A searchAndReduc // now do the final reduce MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumer(maxBucket, new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)); - InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction( - searchContext.bigArrays(), getMockScriptService(), reduceBucketConsumer, pipelines); + InternalAggregation.ReduceContext reduceContext = InternalAggregation.ReduceContext.forFinalReduction( + context.bigArrays(), getMockScriptService(), reduceBucketConsumer, pipelines); @SuppressWarnings("unchecked") - A internalAgg = (A) aggs.get(0).reduce(aggs, context); + A internalAgg = (A) aggs.get(0).reduce(aggs, reduceContext); // materialize any parent pipelines - internalAgg = (A) internalAgg.reducePipelines(internalAgg, context, pipelines); + internalAgg = (A) internalAgg.reducePipelines(internalAgg, reduceContext, pipelines); // materialize any sibling pipelines at top level for (PipelineAggregator pipelineAggregator : pipelines.aggregators()) { - internalAgg = (A) pipelineAggregator.reduce(internalAgg, context); + internalAgg = (A) pipelineAggregator.reduce(internalAgg, reduceContext); } doAssertReducedMultiBucketConsumer(internalAgg, reduceBucketConsumer); return internalAgg; @@ -509,7 +515,7 @@ protected void withAggregator( try (DirectoryReader unwrapped = DirectoryReader.open(directory); IndexReader indexReader = wrapDirectoryReader(unwrapped)) { IndexSearcher searcher = newIndexSearcher(indexReader); - SearchContext context = createSearchContext(searcher, query, fieldTypes); + AggregationContext context = createAggregationContext(searcher, query, fieldTypes); verify.accept(searcher, createAggregator(aggregationBuilder, context)); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 4d00f33000990..f87bce6784688 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -19,6 +19,7 @@ package org.elasticsearch.test; import com.carrotsearch.randomizedtesting.RandomizedContext; + import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; @@ -34,7 +35,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -323,8 +323,7 @@ public Index resolveIndex(String index) { * Create a new search context. */ protected SearchContext createSearchContext(IndexService indexService) { - BigArrays bigArrays = indexService.getBigArrays(); - return new TestSearchContext(bigArrays, indexService); + return new TestSearchContext(indexService); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index d05a375410266..1a15be52bb1ba 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -25,7 +25,6 @@ import org.elasticsearch.action.search.SearchShardTask; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.query.ParsedQuery; @@ -69,7 +68,6 @@ public class TestSearchContext extends SearchContext { public static final SearchShardTarget SHARD_TARGET = new SearchShardTarget("test", new ShardId("test", "test", 0), null, OriginalIndices.NONE); - final BigArrays bigArrays; final IndexService indexService; final BitsetFilterCache fixedBitSetFilterCache; final Map, Collector> queryCollectors = new HashMap<>(); @@ -94,8 +92,7 @@ public class TestSearchContext extends SearchContext { private final Map searchExtBuilders = new HashMap<>(); - public TestSearchContext(BigArrays bigArrays, IndexService indexService) { - this.bigArrays = bigArrays.withCircuitBreaking(); + public TestSearchContext(IndexService indexService) { this.indexService = indexService; this.fixedBitSetFilterCache = indexService.cache().bitsetFilterCache(); this.indexShard = indexService.getShardOrNull(0); @@ -116,7 +113,6 @@ public TestSearchContext(QueryShardContext queryShardContext, IndexShard indexSh public TestSearchContext(QueryShardContext queryShardContext, IndexShard indexShard, ContextIndexSearcher searcher, ScrollContext scrollContext) { - this.bigArrays = null; this.indexService = null; this.fixedBitSetFilterCache = null; this.indexShard = indexShard; @@ -277,11 +273,6 @@ public IndexShard indexShard() { return indexShard; } - @Override - public BigArrays bigArrays() { - return bigArrays; - } - @Override public BitsetFilterCache bitsetFilterCache() { return fixedBitSetFilterCache; diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java index 5f104cdb665db..d8aaff5f41e59 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregatorTests.java @@ -54,13 +54,11 @@ import org.elasticsearch.search.aggregations.CardinalityUpperBound; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.LeafBucketCollector; -import org.elasticsearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer; +import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.support.AggregationContext; -import org.elasticsearch.search.aggregations.support.AggregationContext.ProductionAggregationContext; import org.elasticsearch.search.aggregations.support.MultiValuesSourceFieldConfig; -import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.GeoDistanceSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; @@ -363,15 +361,15 @@ public long addWithoutBreaking(long bytes) { try (IndexReader indexReader = DirectoryReader.open(directory)) { IndexSearcher indexSearcher = newSearcher(indexReader, false, false); - SearchContext searchContext = createSearchContext( + AggregationContext context = createAggregationContext( indexSearcher, createIndexSettings(), new MatchAllDocsQuery(), breaker, + MultiBucketConsumerService.DEFAULT_MAX_BUCKETS, doubleFields() ); TopMetricsAggregationBuilder builder = simpleBuilder(new FieldSortBuilder("s").order(SortOrder.ASC)); - AggregationContext context = new ProductionAggregationContext(searchContext, mock(MultiBucketConsumer.class)); Aggregator aggregator = builder.build(context, null).create(null, CardinalityUpperBound.ONE); aggregator.preCollection(); assertThat(indexReader.leaves(), hasSize(1));