Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport] GITHUB-11838 Add api to allow concurrent query rewrite #12197

Merged
merged 13 commits into from
May 12, 2023
Merged

[Backport] GITHUB-11838 Add api to allow concurrent query rewrite #12197

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
17cac27
GITHUB-11838 Change API to allow concurrent query rewrite (#11840)
zhaih Oct 19, 2022
d49a6c9
fixing up backport
benwtrent Mar 9, 2023
e6a0f55
Merge branch 'branch_9x' into backport/pr#11840
benwtrent Mar 9, 2023
9a79ce0
Update lucene/core/src/java/org/apache/lucene/search/Query.java
benwtrent Mar 9, 2023
d62b990
committing suggestion
benwtrent Mar 9, 2023
d828aa8
adding docs, updating changes
benwtrent Mar 11, 2023
5bee5bd
formatting
benwtrent Mar 11, 2023
ea9ed4b
Merge remote-tracking branch 'upstream/branch_9x' into backport/pr#11840
benwtrent Mar 23, 2023
37d03e5
Merge remote-tracking branch 'upstream/branch_9x' into backport/pr#11840
benwtrent Apr 5, 2023
2e03b50
Merge remote-tracking branch 'upstream/branch_9x' into backport/pr#11840
benwtrent May 9, 2023
7b3bac5
adding test that verifies override defaults
benwtrent May 9, 2023
9b0b8cf
adding tests for bwc and making virtual method things private
benwtrent May 9, 2023
31a5f41
Merge remote-tracking branch 'upstream/branch_9x' into backport/pr#11840
benwtrent May 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions lucene/CHANGES.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ API Changes

* GITHUB#11746: Deprecate LongValueFacetCounts#getTopChildrenSortByCount. (Greg Miller)

* GITHUB#11840: Query rewrite now takes an IndexSearcher instead of IndexReader to enable concurrent
rewriting. (Patrick Zhai)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please feel free to add yourself as co-author, the backport work is quite hard!


New Features
---------------------

Expand Down
4 changes: 3 additions & 1 deletion lucene/classification/src/java/org/apache/lucene/classification/utils/NearestFuzzyQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.FuzzyTermsEnum;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.TermQuery;
Expand Down Expand Up @@ -214,7 +215,8 @@ private Query newTermQuery(IndexReader reader, Term term) throws IOException {
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
IndexReader reader = indexSearcher.getIndexReader();
ScoreTermQueue q = new ScoreTermQueue(MAX_NUM_TERMS);
// load up the list of possible terms
for (FieldVals f : fieldVals) {
Expand Down
5 changes: 2 additions & 3 deletions lucene/core/src/java/org/apache/lucene/document/BinaryRangeFieldRangeQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import java.util.Arrays;
import java.util.Objects;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.ConstantScoreScorer;
Expand Down Expand Up @@ -90,8 +89,8 @@ public void visit(QueryVisitor visitor) {
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
return super.rewrite(reader);
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
return super.rewrite(indexSearcher);
}

private BinaryRangeDocValues getValues(LeafReader reader, String field) throws IOException {
Expand Down
6 changes: 3 additions & 3 deletions lucene/core/src/java/org/apache/lucene/document/DoubleRangeSlowRangeQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import java.io.IOException;
import java.util.Arrays;
import java.util.Objects;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;

Expand Down Expand Up @@ -79,8 +79,8 @@ public String toString(String field) {
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
return super.rewrite(reader);
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
return super.rewrite(indexSearcher);
}

private static byte[] encodeRanges(double[] min, double[] max) {
Expand Down
9 changes: 5 additions & 4 deletions lucene/core/src/java/org/apache/lucene/document/FeatureField.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.similarities.BM25Similarity;
Expand Down Expand Up @@ -223,7 +224,7 @@ abstract static class FeatureFunction {

abstract Explanation explain(String field, String feature, float w, int freq);

FeatureFunction rewrite(IndexReader reader) throws IOException {
FeatureFunction rewrite(IndexSearcher indexSearcher) throws IOException {
return this;
}
}
Expand Down Expand Up @@ -340,11 +341,11 @@ static final class SaturationFunction extends FeatureFunction {
}

@Override
public FeatureFunction rewrite(IndexReader reader) throws IOException {
public FeatureFunction rewrite(IndexSearcher indexSearcher) throws IOException {
if (pivot != null) {
return super.rewrite(reader);
return super.rewrite(indexSearcher);
}
float newPivot = computePivotFeatureValue(reader, field, feature);
float newPivot = computePivotFeatureValue(indexSearcher.getIndexReader(), field, feature);
return new SaturationFunction(field, feature, newPivot);
}

Expand Down
7 changes: 3 additions & 4 deletions lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import java.util.Objects;
import org.apache.lucene.document.FeatureField.FeatureFunction;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
Expand Down Expand Up @@ -52,12 +51,12 @@ final class FeatureQuery extends Query {
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
FeatureFunction rewritten = function.rewrite(reader);
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
FeatureFunction rewritten = function.rewrite(indexSearcher);
if (function != rewritten) {
return new FeatureQuery(fieldName, featureName, rewritten);
}
return super.rewrite(reader);
return super.rewrite(indexSearcher);
}

@Override
Expand Down
6 changes: 3 additions & 3 deletions lucene/core/src/java/org/apache/lucene/document/FloatRangeSlowRangeQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import java.io.IOException;
import java.util.Arrays;
import java.util.Objects;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;

Expand Down Expand Up @@ -79,8 +79,8 @@ public String toString(String field) {
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
return super.rewrite(reader);
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
return super.rewrite(indexSearcher);
}

private static byte[] encodeRanges(float[] min, float[] max) {
Expand Down
6 changes: 3 additions & 3 deletions lucene/core/src/java/org/apache/lucene/document/IntRangeSlowRangeQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import java.io.IOException;
import java.util.Arrays;
import java.util.Objects;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;

Expand Down Expand Up @@ -77,8 +77,8 @@ public String toString(String field) {
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
return super.rewrite(reader);
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
return super.rewrite(indexSearcher);
}

private static byte[] encodeRanges(int[] min, int[] max) {
Expand Down
6 changes: 3 additions & 3 deletions lucene/core/src/java/org/apache/lucene/document/LongRangeSlowRangeQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import java.io.IOException;
import java.util.Arrays;
import java.util.Objects;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;

Expand Down Expand Up @@ -79,8 +79,8 @@ public String toString(String field) {
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
return super.rewrite(reader);
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
return super.rewrite(indexSearcher);
}

private static byte[] encodeRanges(long[] min, long[] max) {
Expand Down
5 changes: 2 additions & 3 deletions lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import java.io.IOException;
import java.util.Objects;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
Expand Down Expand Up @@ -84,11 +83,11 @@ public String toString(String field) {
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
if (lowerValue == Long.MIN_VALUE && upperValue == Long.MAX_VALUE) {
return new FieldExistsQuery(field);
}
return super.rewrite(reader);
return super.rewrite(indexSearcher);
}

@Override
Expand Down
5 changes: 2 additions & 3 deletions lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesSetQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import java.util.Arrays;
import java.util.Objects;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
Expand Down Expand Up @@ -85,11 +84,11 @@ public long ramBytesUsed() {
}

@Override
public Query rewrite(IndexReader indexReader) throws IOException {
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
if (numbers.size() == 0) {
return new MatchNoDocsQuery();
}
return super.rewrite(indexReader);
return super.rewrite(indexSearcher);
}

@Override
Expand Down
5 changes: 2 additions & 3 deletions lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import java.io.IOException;
import java.util.Objects;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
Expand Down Expand Up @@ -98,11 +97,11 @@ public String toString(String field) {
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
if (lowerValue == null && upperValue == null) {
return new FieldExistsQuery(field);
}
return super.rewrite(reader);
return super.rewrite(indexSearcher);
}

@Override
Expand Down
4 changes: 2 additions & 2 deletions lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,12 @@ public AbstractKnnVectorQuery(String field, int k, Query filter) {
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
IndexReader reader = indexSearcher.getIndexReader();
TopDocs[] perLeafResults = new TopDocs[reader.leaves().size()];

Weight filterWeight = null;
if (filter != null) {
IndexSearcher indexSearcher = new IndexSearcher(reader);
BooleanQuery booleanQuery =
new BooleanQuery.Builder()
.add(filter, BooleanClause.Occur.FILTER)
Expand Down
10 changes: 5 additions & 5 deletions lucene/core/src/java/org/apache/lucene/search/BlendedTermQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
Expand Down Expand Up @@ -268,11 +267,12 @@ public String toString(String field) {
}

@Override
public final Query rewrite(IndexReader reader) throws IOException {
public final Query rewrite(IndexSearcher indexSearcher) throws IOException {
final TermStates[] contexts = ArrayUtil.copyOfSubArray(this.contexts, 0, this.contexts.length);
for (int i = 0; i < contexts.length; ++i) {
if (contexts[i] == null || contexts[i].wasBuiltFor(reader.getContext()) == false) {
contexts[i] = TermStates.build(reader.getContext(), terms[i], true);
if (contexts[i] == null
|| contexts[i].wasBuiltFor(indexSearcher.getTopReaderContext()) == false) {
contexts[i] = TermStates.build(indexSearcher.getTopReaderContext(), terms[i], true);
}
}

Expand All @@ -287,7 +287,7 @@ public final Query rewrite(IndexReader reader) throws IOException {
}

for (int i = 0; i < contexts.length; ++i) {
contexts[i] = adjustFrequencies(reader.getContext(), contexts[i], df, ttf);
contexts[i] = adjustFrequencies(indexSearcher.getTopReaderContext(), contexts[i], df, ttf);
}

Query[] termQueries = new Query[terms.length];
Expand Down
9 changes: 4 additions & 5 deletions lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import java.util.Objects;
import java.util.Set;
import java.util.function.Predicate;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.BooleanClause.Occur;

/**
Expand Down Expand Up @@ -247,7 +246,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
if (clauses.size() == 0) {
return new MatchNoDocsQuery("empty BooleanQuery");
}
Expand Down Expand Up @@ -286,12 +285,12 @@ public Query rewrite(IndexReader reader) throws IOException {
Query rewritten;
if (occur == Occur.FILTER || occur == Occur.MUST_NOT) {
// Clauses that are not involved in scoring can get some extra simplifications
rewritten = new ConstantScoreQuery(query).rewrite(reader);
rewritten = new ConstantScoreQuery(query).rewrite(indexSearcher);
if (rewritten instanceof ConstantScoreQuery) {
rewritten = ((ConstantScoreQuery) rewritten).getQuery();
}
} else {
rewritten = query.rewrite(reader);
rewritten = query.rewrite(indexSearcher);
}
if (rewritten != query || query.getClass() == MatchNoDocsQuery.class) {
// rewrite clause
Expand Down Expand Up @@ -566,7 +565,7 @@ public Query rewrite(IndexReader reader) throws IOException {
}
}

return super.rewrite(reader);
return super.rewrite(indexSearcher);
}

@Override
Expand Down
7 changes: 3 additions & 4 deletions lucene/core/src/java/org/apache/lucene/search/BoostQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import java.io.IOException;
import java.util.Objects;
import org.apache.lucene.index.IndexReader;

/**
* A {@link Query} wrapper that allows to give a boost to the wrapped query. Boost values that are
Expand Down Expand Up @@ -73,8 +72,8 @@ public int hashCode() {
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
final Query rewritten = query.rewrite(reader);
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
final Query rewritten = query.rewrite(indexSearcher);

if (boost == 1f) {
return rewritten;
Expand All @@ -99,7 +98,7 @@ public Query rewrite(IndexReader reader) throws IOException {
return new BoostQuery(rewritten, boost);
}

return super.rewrite(reader);
return super.rewrite(indexSearcher);
}

@Override
Expand Down
8 changes: 4 additions & 4 deletions lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import java.io.IOException;
import java.util.Objects;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.Bits;

Expand All @@ -40,8 +39,9 @@ public Query getQuery() {
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
Query rewritten = query.rewrite(reader);
public Query rewrite(IndexSearcher indexSearcher) throws IOException {

Query rewritten = query.rewrite(indexSearcher);

// Do some extra simplifications that are legal since scores are not needed on the wrapped
// query.
Expand Down Expand Up @@ -70,7 +70,7 @@ public Query rewrite(IndexReader reader) throws IOException {
return new ConstantScoreQuery(((BoostQuery) rewritten).getQuery());
}

return super.rewrite(reader);
return super.rewrite(indexSearcher);
}

@Override
Expand Down
Loading