diff --git a/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java b/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java index 5222899957b16..d78037dee7460 100644 --- a/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java +++ b/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java @@ -23,11 +23,13 @@ import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.support.single.shard.SingleShardOperationRequest; import org.elasticsearch.client.Requests; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.fetch.source.FetchSourceContext; import java.io.IOException; @@ -43,8 +45,9 @@ public class ExplainRequest extends SingleShardOperationRequest private String routing; private String preference; private BytesReference source; - private String[] fields; private boolean sourceUnsafe; + private String[] fields; + private FetchSourceContext fetchSourceContext; private String[] filteringAlias = Strings.EMPTY_ARRAY; @@ -121,6 +124,19 @@ public ExplainRequest source(BytesReference source, boolean unsafe) { return this; } + /** + * Allows setting the {@link FetchSourceContext} for this request, controlling if and how _source should be returned. + */ + public ExplainRequest fetchSourceContext(FetchSourceContext context) { + this.fetchSourceContext = context; + return this; + } + + public FetchSourceContext fetchSourceContext() { + return fetchSourceContext; + } + + public String[] fields() { return fields; } @@ -178,6 +194,8 @@ public void readFrom(StreamInput in) throws IOException { if (in.readBoolean()) { fields = in.readStringArray(); } + + fetchSourceContext = FetchSourceContext.optionalReadFromStream(in); } @Override @@ -195,5 +213,7 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeBoolean(false); } + + FetchSourceContext.optionalWriteToStream(fetchSourceContext, out); } } diff --git a/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java b/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java index 69d9a97957479..358d88c8127c1 100644 --- a/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java @@ -23,8 +23,11 @@ import org.elasticsearch.action.support.single.shard.SingleShardOperationRequestBuilder; import org.elasticsearch.client.Client; import org.elasticsearch.client.internal.InternalClient; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.fetch.source.FetchSourceContext; /** * A builder for {@link ExplainRequest}. @@ -105,6 +108,57 @@ public ExplainRequestBuilder setFields(String... fields) { return this; } + /** + * Indicates whether the response should contain the stored _source + * + * + * @param fetch + * @return + */ + public ExplainRequestBuilder setFetchSource(boolean fetch) { + FetchSourceContext context = request.fetchSourceContext(); + if (context == null) { + request.fetchSourceContext(new FetchSourceContext(fetch)); + } + else { + context.fetchSource(fetch); + } + return this; + } + + /** + * Indicate that _source should be returned, with an "include" and/or "exclude" set which can include simple wildcard + * elements. + * + * @param include An optional include (optionally wildcarded) pattern to filter the returned _source + * @param exclude An optional exclude (optionally wildcarded) pattern to filter the returned _source + */ + public ExplainRequestBuilder setFetchSource(@Nullable String include, @Nullable String exclude) { + return setFetchSource( + include == null? Strings.EMPTY_ARRAY : new String[] {include}, + exclude == null? Strings.EMPTY_ARRAY : new String[] {exclude}); + } + + /** + * Indicate that _source should be returned, with an "include" and/or "exclude" set which can include simple wildcard + * elements. + * + * @param includes An optional list of include (optionally wildcarded) pattern to filter the returned _source + * @param excludes An optional list of exclude (optionally wildcarded) pattern to filter the returned _source + */ + public ExplainRequestBuilder setFetchSource(@Nullable String[] includes, @Nullable String[] excludes) { + FetchSourceContext context = request.fetchSourceContext(); + if (context == null) { + request.fetchSourceContext(new FetchSourceContext(includes, excludes)); + } + else { + context.fetchSource(true); + context.includes(includes); + context.excludes(excludes); + } + return this; + } + /** * Sets the full source of the explain request (for example, wrapping an actual query). */ diff --git a/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java index b528eaed1523b..269be146ca267 100644 --- a/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java +++ b/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java @@ -119,14 +119,11 @@ protected ExplainResponse shardOperation(ExplainRequest request, int shardId) th } else { explanation = context.searcher().explain(context.query(), topLevelDocId); } - if (request.fields() != null) { - if (request.fields().length == 1 && "_source".equals(request.fields()[0])) { - request.fields(null); // Load the _source field - } + if (request.fields() != null || (request.fetchSourceContext() != null && request.fetchSourceContext().fetchSource())) { // Advantage is that we're not opening a second searcher to retrieve the _source. Also // because we are working in the same searcher in engineGetResult we can be sure that a // doc isn't deleted between the initial get and this call. - GetResult getResult = indexShard.getService().get(result, request.id(), request.type(), request.fields()); + GetResult getResult = indexShard.getService().get(result, request.id(), request.type(), request.fields(), request.fetchSourceContext()); return new ExplainResponse(true, explanation, getResult); } else { return new ExplainResponse(true, explanation); diff --git a/src/main/java/org/elasticsearch/action/get/GetRequest.java b/src/main/java/org/elasticsearch/action/get/GetRequest.java index 13ccf8d6e9cd8..41dfa75e603e4 100644 --- a/src/main/java/org/elasticsearch/action/get/GetRequest.java +++ b/src/main/java/org/elasticsearch/action/get/GetRequest.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.index.VersionType; +import org.elasticsearch.search.fetch.source.FetchSourceContext; import java.io.IOException; @@ -51,6 +52,8 @@ public class GetRequest extends SingleShardOperationRequest { private String[] fields; + private FetchSourceContext fetchSourceContext; + private boolean refresh = false; Boolean realtime; @@ -162,6 +165,18 @@ public String preference() { return this.preference; } + /** + * Allows setting the {@link FetchSourceContext} for this request, controlling if and how _source should be returned. + */ + public GetRequest fetchSourceContext(FetchSourceContext context) { + this.fetchSourceContext = context; + return this; + } + + public FetchSourceContext fetchSourceContext() { + return fetchSourceContext; + } + /** * Explicitly specify the fields that will be returned. By default, the _source * field will be returned. @@ -248,8 +263,11 @@ public void readFrom(StreamInput in) throws IOException { } else if (realtime == 1) { this.realtime = true; } + this.versionType = VersionType.fromValue(in.readByte()); this.version = in.readVLong(); + + fetchSourceContext = FetchSourceContext.optionalReadFromStream(in); } @Override @@ -279,6 +297,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeByte(versionType.getValue()); out.writeVLong(version); + + FetchSourceContext.optionalWriteToStream(fetchSourceContext, out); } @Override diff --git a/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java b/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java index 06bcb1b96adce..0db03f3e78291 100644 --- a/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java +++ b/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java @@ -25,6 +25,8 @@ import org.elasticsearch.client.internal.InternalClient; import org.elasticsearch.common.Nullable; import org.elasticsearch.index.VersionType; +import org.elasticsearch.common.Strings; +import org.elasticsearch.search.fetch.source.FetchSourceContext; /** * A get document action request builder. @@ -93,6 +95,56 @@ public GetRequestBuilder setFields(String... fields) { return this; } + /** + * Indicates whether the response should contain the stored _source + * + * @param fetch + * @return + */ + public GetRequestBuilder setFetchSource(boolean fetch) { + FetchSourceContext context = request.fetchSourceContext(); + if (context == null) { + request.fetchSourceContext(new FetchSourceContext(fetch)); + } + else { + context.fetchSource(fetch); + } + return this; + } + + /** + * Indicate that _source should be returned, with an "include" and/or "exclude" set which can include simple wildcard + * elements. + * + * @param include An optional include (optionally wildcarded) pattern to filter the returned _source + * @param exclude An optional exclude (optionally wildcarded) pattern to filter the returned _source + */ + public GetRequestBuilder setFetchSource(@Nullable String include, @Nullable String exclude) { + return setFetchSource( + include == null? Strings.EMPTY_ARRAY : new String[] {include}, + exclude == null? Strings.EMPTY_ARRAY : new String[] {exclude}); + } + + /** + * Indicate that _source should be returned, with an "include" and/or "exclude" set which can include simple wildcard + * elements. + * + * @param includes An optional list of include (optionally wildcarded) pattern to filter the returned _source + * @param excludes An optional list of exclude (optionally wildcarded) pattern to filter the returned _source + */ + public GetRequestBuilder setFetchSource(@Nullable String[] includes, @Nullable String[] excludes) { + FetchSourceContext context = request.fetchSourceContext(); + if (context == null) { + request.fetchSourceContext(new FetchSourceContext(includes, excludes)); + } + else { + context.fetchSource(true); + context.includes(includes); + context.excludes(excludes); + } + return this; + } + /** * Should a refresh be executed before this get operation causing the operation to * return the latest value. Note, heavy get should not set this to true. Defaults @@ -129,4 +181,6 @@ public GetRequestBuilder setVersionType(VersionType versionType) { protected void doExecute(ActionListener listener) { ((Client) client).get(request, listener); } + + } diff --git a/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java b/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java index 07707d733f3ab..a41c3335ad1ee 100644 --- a/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java +++ b/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java @@ -20,10 +20,12 @@ package org.elasticsearch.action.get; import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.ElasticSearchParseException; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ValidateActions; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; @@ -33,6 +35,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.VersionType; +import org.elasticsearch.search.fetch.source.FetchSourceContext; import java.io.IOException; import java.util.ArrayList; @@ -51,6 +54,7 @@ public static class Item implements Streamable { private String[] fields; private long version = Versions.MATCH_ANY; private VersionType versionType = VersionType.INTERNAL; + private FetchSourceContext fetchSourceContext; Item() { @@ -132,6 +136,18 @@ public Item versionType(VersionType versionType) { return this; } + public FetchSourceContext fetchSourceContext() { + return this.fetchSourceContext; + } + + /** + * Allows setting the {@link FetchSourceContext} for this request, controlling if and how _source should be returned. + */ + public Item fetchSourceContext(FetchSourceContext fetchSourceContext) { + this.fetchSourceContext = fetchSourceContext; + return this; + } + public static Item readItem(StreamInput in) throws IOException { Item item = new Item(); item.readFrom(in); @@ -153,6 +169,8 @@ public void readFrom(StreamInput in) throws IOException { } version = in.readVLong(); versionType = VersionType.fromValue(in.readByte()); + + fetchSourceContext = FetchSourceContext.optionalReadFromStream(in); } @Override @@ -169,8 +187,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(field); } } + out.writeVLong(version); out.writeByte(versionType.getValue()); + + FetchSourceContext.optionalWriteToStream(fetchSourceContext, out); } } @@ -243,11 +264,11 @@ public MultiGetRequest refresh(boolean refresh) { return this; } - public void add(@Nullable String defaultIndex, @Nullable String defaultType, @Nullable String[] defaultFields, byte[] data, int from, int length) throws Exception { - add(defaultIndex, defaultType, defaultFields, new BytesArray(data, from, length)); + public void add(@Nullable String defaultIndex, @Nullable String defaultType, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSource, byte[] data, int from, int length) throws Exception { + add(defaultIndex, defaultType, defaultFields, defaultFetchSource, new BytesArray(data, from, length)); } - public void add(@Nullable String defaultIndex, @Nullable String defaultType, @Nullable String[] defaultFields, BytesReference data) throws Exception { + public void add(@Nullable String defaultIndex, @Nullable String defaultType, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSource, BytesReference data) throws Exception { XContentParser parser = XContentFactory.xContent(data).createParser(data); try { XContentParser.Token token; @@ -270,6 +291,8 @@ public void add(@Nullable String defaultIndex, @Nullable String defaultType, @Nu long version = Versions.MATCH_ANY; VersionType versionType = VersionType.INTERNAL; + FetchSourceContext fetchSourceContext = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); @@ -291,6 +314,12 @@ public void add(@Nullable String defaultIndex, @Nullable String defaultType, @Nu version = parser.longValue(); } else if ("_version_type".equals(currentFieldName) || "_versionType".equals(currentFieldName) || "version_type".equals(currentFieldName) || "versionType".equals(currentFieldName)) { versionType = VersionType.fromString(parser.text()); + } else if ("_source".equals(currentFieldName)) { + if (token == XContentParser.Token.VALUE_BOOLEAN) { + fetchSourceContext = new FetchSourceContext(parser.booleanValue()); + } else if (token == XContentParser.Token.VALUE_STRING) { + fetchSourceContext = new FetchSourceContext(new String[]{parser.text()}); + } } } else if (token == XContentParser.Token.START_ARRAY) { if ("fields".equals(currentFieldName)) { @@ -298,6 +327,42 @@ public void add(@Nullable String defaultIndex, @Nullable String defaultType, @Nu while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { fields.add(parser.text()); } + } else if ("_source".equals(currentFieldName)) { + ArrayList includes = new ArrayList(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + includes.add(parser.text()); + } + fetchSourceContext = new FetchSourceContext(includes.toArray(Strings.EMPTY_ARRAY)); + } + + } else if (token == XContentParser.Token.START_OBJECT) { + if ("_source".equals(currentFieldName)) { + List currentList = null, includes = null, excludes = null; + + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + if ("includes".equals(currentFieldName) || "include".equals(currentFieldName)) { + currentList = includes != null ? includes : (includes = new ArrayList(2)); + } else if ("excludes".equals(currentFieldName) || "exclude".equals(currentFieldName)) { + currentList = excludes != null ? excludes : (excludes = new ArrayList(2)); + } else { + throw new ElasticSearchParseException("Source definition may not contain " + parser.text()); + } + } else if (token == XContentParser.Token.START_ARRAY) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + currentList.add(parser.text()); + } + } else if (token.isValue()) { + currentList.add(parser.text()); + } else { + throw new ElasticSearchParseException("unexpected token while parsing source settings"); + } + } + + fetchSourceContext = new FetchSourceContext( + includes == null ? Strings.EMPTY_ARRAY : includes.toArray(new String[includes.size()]), + excludes == null ? Strings.EMPTY_ARRAY : excludes.toArray(new String[excludes.size()])); } } } @@ -307,14 +372,15 @@ public void add(@Nullable String defaultIndex, @Nullable String defaultType, @Nu } else { aFields = defaultFields; } - add(new Item(index, type, id).routing(routing).fields(aFields).parent(parent).version(version).versionType(versionType)); + add(new Item(index, type, id).routing(routing).fields(aFields).parent(parent).version(version).versionType(versionType) + .fetchSourceContext(fetchSourceContext == null ? defaultFetchSource : fetchSourceContext)); } } else if ("ids".equals(currentFieldName)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (!token.isValue()) { throw new ElasticSearchIllegalArgumentException("ids array element should only contain ids"); } - add(new Item(defaultIndex, defaultType, parser.text()).fields(defaultFields)); + add(new Item(defaultIndex, defaultType, parser.text()).fields(defaultFields).fetchSourceContext(defaultFetchSource)); } } } diff --git a/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java b/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java index fc570afe97f83..4b9e0e879ce34 100644 --- a/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java +++ b/src/main/java/org/elasticsearch/action/get/MultiGetShardRequest.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.VersionType; +import org.elasticsearch.search.fetch.source.FetchSourceContext; import java.io.IOException; import java.util.ArrayList; @@ -44,6 +45,7 @@ public class MultiGetShardRequest extends SingleShardOperationRequest fields; TLongArrayList versions; List versionTypes; + List fetchSourceContexts; MultiGetShardRequest() { @@ -58,6 +60,7 @@ public class MultiGetShardRequest extends SingleShardOperationRequest(); versions = new TLongArrayList(); versionTypes = new ArrayList(); + fetchSourceContexts = new ArrayList(); } public int shardId() { @@ -96,13 +99,14 @@ public MultiGetShardRequest refresh(boolean refresh) { return this; } - public void add(int location, @Nullable String type, String id, String[] fields, long version, VersionType versionType) { + public void add(int location, @Nullable String type, String id, String[] fields, long version, VersionType versionType, FetchSourceContext fetchSourceContext) { this.locations.add(location); this.types.add(type); this.ids.add(id); this.fields.add(fields); this.versions.add(version); this.versionTypes.add(versionType); + this.fetchSourceContexts.add(fetchSourceContext); } @Override @@ -115,6 +119,7 @@ public void readFrom(StreamInput in) throws IOException { fields = new ArrayList(size); versions = new TLongArrayList(size); versionTypes = new ArrayList(size); + fetchSourceContexts = new ArrayList(size); for (int i = 0; i < size; i++) { locations.add(in.readVInt()); if (in.readBoolean()) { @@ -135,6 +140,8 @@ public void readFrom(StreamInput in) throws IOException { } versions.add(in.readVLong()); versionTypes.add(VersionType.fromValue(in.readByte())); + + fetchSourceContexts.add(FetchSourceContext.optionalReadFromStream(in)); } preference = in.readOptionalString(); @@ -170,6 +177,8 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeVLong(versions.get(i)); out.writeByte(versionTypes.get(i).getValue()); + FetchSourceContext fetchSourceContext = fetchSourceContexts.get(i); + FetchSourceContext.optionalWriteToStream(fetchSourceContext, out); } out.writeOptionalString(preference); @@ -181,5 +190,7 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeByte((byte) 1); } + + } } diff --git a/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index b2dcc814aa3a2..50a51297cd0a0 100644 --- a/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -100,7 +100,7 @@ protected GetResponse shardOperation(GetRequest request, int shardId) throws Ela } GetResult result = indexShard.getService().get(request.type(), request.id(), request.fields(), - request.realtime(), request.version(), request.versionType()); + request.realtime(), request.version(), request.versionType(), request.fetchSourceContext()); return new GetResponse(result); } diff --git a/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java b/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java index dde95fb6980e3..8819208bfb454 100644 --- a/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java +++ b/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java @@ -82,7 +82,7 @@ protected void doExecute(final MultiGetRequest request, final ActionListener fields; public CustomFieldsVisitor(Set fields, boolean loadSource) { - this.loadAllFields = false; this.loadSource = loadSource; this.fields = fields; } - public CustomFieldsVisitor(boolean loadAllFields, boolean loadSource) { - this.loadAllFields = loadAllFields; - this.loadSource = loadSource; - this.fields = null; - } - @Override public Status needsField(FieldInfo fieldInfo) throws IOException { - if (loadAllFields) { - return Status.YES; - } + if (loadSource && SourceFieldMapper.NAME.equals(fieldInfo.name)) { return Status.YES; } diff --git a/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 684c9e16143ff..6030eeab3874c 100644 --- a/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -23,6 +23,7 @@ import com.google.common.collect.Sets; import org.apache.lucene.index.Term; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; @@ -49,6 +50,7 @@ import org.elasticsearch.index.translog.Translog; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.SearchScript; +import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; @@ -95,11 +97,13 @@ public ShardGetService setIndexShard(IndexShard indexShard) { return this; } - public GetResult get(String type, String id, String[] gFields, boolean realtime, long version, VersionType versionType) throws ElasticSearchException { + public GetResult get(String type, String id, String[] gFields, boolean realtime, long version, VersionType versionType, FetchSourceContext fetchSourceContext) + throws ElasticSearchException { currentMetric.inc(); try { long now = System.nanoTime(); - GetResult getResult = innerGet(type, id, gFields, realtime, version, versionType); + GetResult getResult = innerGet(type, id, gFields, realtime, version, versionType, fetchSourceContext); + if (getResult.isExists()) { existsMetric.inc(System.nanoTime() - now); } else { @@ -118,7 +122,7 @@ public GetResult get(String type, String id, String[] gFields, boolean realtime, *

* Note: Call must release engine searcher associated with engineGetResult! */ - public GetResult get(Engine.GetResult engineGetResult, String id, String type, String[] fields) { + public GetResult get(Engine.GetResult engineGetResult, String id, String type, String[] fields, FetchSourceContext fetchSourceContext) { if (!engineGetResult.exists()) { return new GetResult(shardId.index().name(), type, id, -1, false, null, null); } @@ -131,8 +135,8 @@ public GetResult get(Engine.GetResult engineGetResult, String id, String type, S missingMetric.inc(System.nanoTime() - now); return new GetResult(shardId.index().name(), type, id, -1, false, null, null); } - - GetResult getResult = innerGetLoadFromStoredFields(type, id, fields, engineGetResult, docMapper); + fetchSourceContext = normalizeFetchSourceContent(fetchSourceContext, fields); + GetResult getResult = innerGetLoadFromStoredFields(type, id, fields, fetchSourceContext, engineGetResult, docMapper); if (getResult.isExists()) { existsMetric.inc(System.nanoTime() - now); } else { @@ -144,8 +148,29 @@ public GetResult get(Engine.GetResult engineGetResult, String id, String type, S } } - public GetResult innerGet(String type, String id, String[] gFields, boolean realtime, long version, VersionType versionType) throws ElasticSearchException { - boolean loadSource = gFields == null || gFields.length > 0; + /** + * decides what needs to be done based on the request input and always returns a valid non-null FetchSourceContext + */ + protected FetchSourceContext normalizeFetchSourceContent(@Nullable FetchSourceContext context, @Nullable String[] gFields) { + if (context != null) { + return context; + } + if (gFields == null) { + return FetchSourceContext.FETCH_SOURCE; + } + for (String field : gFields) { + if (SourceFieldMapper.NAME.equals(field)) { + return FetchSourceContext.FETCH_SOURCE; + } + } + return FetchSourceContext.DO_NOT_FETCH_SOURCE; + } + + public GetResult innerGet(String type, String id, String[] gFields, boolean realtime, long version, VersionType versionType, FetchSourceContext fetchSourceContext) throws ElasticSearchException { + fetchSourceContext = normalizeFetchSourceContent(fetchSourceContext, gFields); + + boolean loadSource = (gFields != null && gFields.length > 0) || fetchSourceContext.fetchSource(); + Engine.GetResult get = null; if (type == null || type.equals("_all")) { for (String typeX : mapperService.types()) { @@ -183,25 +208,19 @@ public GetResult innerGet(String type, String id, String[] gFields, boolean real try { // break between having loaded it from translog (so we only have _source), and having a document to load if (get.docIdAndVersion() != null) { - return innerGetLoadFromStoredFields(type, id, gFields, get, docMapper); + return innerGetLoadFromStoredFields(type, id, gFields, fetchSourceContext, get, docMapper); } else { Translog.Source source = get.source(); Map fields = null; - boolean sourceRequested = false; + SearchLookup searchLookup = null; // we can only load scripts that can run against the source - if (gFields == null) { - sourceRequested = true; - } else if (gFields.length == 0) { - // no fields, and no source - sourceRequested = false; - } else { + if (gFields != null && gFields.length > 0) { Map sourceAsMap = null; - SearchLookup searchLookup = null; for (String field : gFields) { - if (field.equals("_source")) { - sourceRequested = true; + if (SourceFieldMapper.NAME.equals(field)) { + // dealt with when normalizing fetchSourceContext. continue; } Object value = null; @@ -279,27 +298,38 @@ public GetResult innerGet(String type, String id, String[] gFields, boolean real } } - // if source is not enabled, don't return it even though we have it from the translog - if (sourceRequested && !docMapper.sourceMapper().enabled()) { - sourceRequested = false; - } - - // Cater for source excludes/includes at the cost of performance + // deal with source, but only if it's enabled (we always have it from the translog) BytesReference sourceToBeReturned = null; - if (sourceRequested) { + SourceFieldMapper sourceFieldMapper = docMapper.sourceMapper(); + if (fetchSourceContext.fetchSource() && sourceFieldMapper.enabled()) { + sourceToBeReturned = source.source; - SourceFieldMapper sourceFieldMapper = docMapper.sourceMapper(); - if (sourceFieldMapper.enabled()) { - boolean filtered = sourceFieldMapper.includes().length > 0 || sourceFieldMapper.excludes().length > 0; - if (filtered) { - Tuple> mapTuple = XContentHelper.convertToMap(source.source, true); - Map filteredSource = XContentMapValues.filter(mapTuple.v2(), sourceFieldMapper.includes(), sourceFieldMapper.excludes()); - try { - sourceToBeReturned = XContentFactory.contentBuilder(mapTuple.v1()).map(filteredSource).bytes(); - } catch (IOException e) { - throw new ElasticSearchException("Failed to get type [" + type + "] and id [" + id + "] with includes/excludes set", e); - } + // Cater for source excludes/includes at the cost of performance + // We must first apply the field mapper filtering to make sure we get correct results + // in the case that the fetchSourceContext white lists something that's not included by the field mapper + + Map filteredSource = null; + XContentType sourceContentType = null; + if (sourceFieldMapper.includes().length > 0 || sourceFieldMapper.excludes().length > 0) { + // TODO: The source might parsed and available in the sourceLookup but that one uses unordered maps so different. Do we care? + Tuple> typeMapTuple = XContentHelper.convertToMap(source.source, true); + sourceContentType = typeMapTuple.v1(); + filteredSource = XContentMapValues.filter(typeMapTuple.v2(), sourceFieldMapper.includes(), sourceFieldMapper.excludes()); + } + if (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0) { + if (filteredSource == null) { + Tuple> typeMapTuple = XContentHelper.convertToMap(source.source, true); + sourceContentType = typeMapTuple.v1(); + filteredSource = typeMapTuple.v2(); + } + filteredSource = XContentMapValues.filter(filteredSource, fetchSourceContext.includes(), fetchSourceContext.excludes()); + } + if (filteredSource != null) { + try { + sourceToBeReturned = XContentFactory.contentBuilder(sourceContentType).map(filteredSource).bytes(); + } catch (IOException e) { + throw new ElasticSearchException("Failed to get type [" + type + "] and id [" + id + "] with includes/excludes set", e); } } } @@ -311,11 +341,11 @@ public GetResult innerGet(String type, String id, String[] gFields, boolean real } } - private GetResult innerGetLoadFromStoredFields(String type, String id, String[] gFields, Engine.GetResult get, DocumentMapper docMapper) { + private GetResult innerGetLoadFromStoredFields(String type, String id, String[] gFields, FetchSourceContext fetchSourceContext, Engine.GetResult get, DocumentMapper docMapper) { Map fields = null; BytesReference source = null; Versions.DocIdAndVersion docIdAndVersion = get.docIdAndVersion(); - FieldsVisitor fieldVisitor = buildFieldsVisitors(gFields); + FieldsVisitor fieldVisitor = buildFieldsVisitors(gFields, fetchSourceContext); if (fieldVisitor != null) { try { docIdAndVersion.context.reader().document(docIdAndVersion.docId, fieldVisitor); @@ -341,11 +371,13 @@ private GetResult innerGetLoadFromStoredFields(String type, String id, String[] if (field.contains("_source.") || field.contains("doc[")) { if (searchLookup == null) { searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type}); + searchLookup.source().setNextSource(source); + searchLookup.setNextReader(docIdAndVersion.context); + searchLookup.setNextDocId(docIdAndVersion.docId); } SearchScript searchScript = scriptService.search(searchLookup, "mvel", field, null); searchScript.setNextReader(docIdAndVersion.context); searchScript.setNextDocId(docIdAndVersion.docId); - try { value = searchScript.run(); } catch (RuntimeException e) { @@ -360,6 +392,7 @@ private GetResult innerGetLoadFromStoredFields(String type, String id, String[] if (searchLookup == null) { searchLookup = new SearchLookup(mapperService, fieldDataService, new String[]{type}); searchLookup.setNextReader(docIdAndVersion.context); + searchLookup.source().setNextSource(source); searchLookup.setNextDocId(docIdAndVersion.docId); } value = searchLookup.source().extractValue(field); @@ -390,19 +423,30 @@ private GetResult innerGetLoadFromStoredFields(String type, String id, String[] } } + if (!fetchSourceContext.fetchSource()) { + source = null; + } else if (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0) { + Map filteredSource; + XContentType sourceContentType = null; + // TODO: The source might parsed and available in the sourceLookup but that one uses unordered maps so different. Do we care? + Tuple> typeMapTuple = XContentHelper.convertToMap(source, true); + sourceContentType = typeMapTuple.v1(); + filteredSource = XContentMapValues.filter(typeMapTuple.v2(), fetchSourceContext.includes(), fetchSourceContext.excludes()); + try { + source = XContentFactory.contentBuilder(sourceContentType).map(filteredSource).bytes(); + } catch (IOException e) { + throw new ElasticSearchException("Failed to get type [" + type + "] and id [" + id + "] with includes/excludes set", e); + } + } + return new GetResult(shardId.index().name(), type, id, get.version(), get.exists(), source, fields); } - private static FieldsVisitor buildFieldsVisitors(String... fields) { - if (fields == null) { - return new JustSourceFieldsVisitor(); - } - - // don't load anything - if (fields.length == 0) { - return null; + private static FieldsVisitor buildFieldsVisitors(String[] fields, FetchSourceContext fetchSourceContext) { + if (fields == null || fields.length == 0) { + return fetchSourceContext.fetchSource() ? new JustSourceFieldsVisitor() : null; } - return new CustomFieldsVisitor(Sets.newHashSet(fields), false); + return new CustomFieldsVisitor(Sets.newHashSet(fields), fetchSourceContext.fetchSource()); } } diff --git a/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java b/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java index a65cee3f483fa..68211ffb45d78 100644 --- a/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java +++ b/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.explain.ExplainResponse; import org.elasticsearch.action.explain.ExplainSourceBuilder; import org.elasticsearch.client.Client; +import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.inject.Inject; @@ -36,6 +37,7 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryStringQueryBuilder; import org.elasticsearch.rest.*; +import org.elasticsearch.search.fetch.source.FetchSourceContext; import java.io.IOException; @@ -100,6 +102,8 @@ public void handleRequest(final RestRequest request, final RestChannel channel) } } + explainRequest.fetchSourceContext(FetchSourceContext.parseFromRestRequest(request)); + client.explain(explainRequest, new ActionListener() { @Override diff --git a/src/main/java/org/elasticsearch/rest/action/get/RestGetAction.java b/src/main/java/org/elasticsearch/rest/action/get/RestGetAction.java index 5d744d6391e35..1743833e8e31a 100644 --- a/src/main/java/org/elasticsearch/rest/action/get/RestGetAction.java +++ b/src/main/java/org/elasticsearch/rest/action/get/RestGetAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -30,6 +31,7 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.support.RestActions; +import org.elasticsearch.search.fetch.source.FetchSourceContext; import java.io.IOException; @@ -71,6 +73,7 @@ public void handleRequest(final RestRequest request, final RestChannel channel) getRequest.version(RestActions.parseVersion(request)); getRequest.versionType(VersionType.fromString(request.param("version_type"), getRequest.versionType())); + getRequest.fetchSourceContext(FetchSourceContext.parseFromRestRequest(request)); client.get(getRequest, new ActionListener() { @Override diff --git a/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java b/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java index 65588d31a16f5..32827b45c1275 100644 --- a/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java +++ b/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java @@ -23,11 +23,13 @@ import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.*; import org.elasticsearch.rest.action.support.RestXContentBuilder; +import org.elasticsearch.search.fetch.source.FetchSourceContext; import java.io.IOException; @@ -58,6 +60,23 @@ public void handleRequest(final RestRequest request, final RestChannel channel) getRequest.preference(request.param("preference")); getRequest.realtime(request.paramAsBooleanOptional("realtime", null)); + String[] includes = null, excludes = null; + String sIncludes = request.param("include"); + sIncludes = request.param("includes", sIncludes); + if (sIncludes != null) { + includes = Strings.splitStringByCommaToArray(sIncludes); + } + + String sExcludes = request.param("exclude"); + sExcludes = request.param("excludes", sExcludes); + if (sExcludes != null) { + excludes = Strings.splitStringByCommaToArray(sExcludes); + } + + if (includes != null || excludes != null) { + getRequest.fetchSourceContext(new FetchSourceContext(includes, excludes)); + } + client.get(getRequest, new ActionListener() { @Override public void onResponse(GetResponse response) { diff --git a/src/main/java/org/elasticsearch/rest/action/get/RestMultiGetAction.java b/src/main/java/org/elasticsearch/rest/action/get/RestMultiGetAction.java index 2694ef3bc0768..7f923ca70b356 100644 --- a/src/main/java/org/elasticsearch/rest/action/get/RestMultiGetAction.java +++ b/src/main/java/org/elasticsearch/rest/action/get/RestMultiGetAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.*; +import org.elasticsearch.search.fetch.source.FetchSourceContext; import java.io.IOException; @@ -64,8 +65,10 @@ public void handleRequest(final RestRequest request, final RestChannel channel) sFields = Strings.splitStringByCommaToArray(sField); } + FetchSourceContext defaultFetchSource = FetchSourceContext.parseFromRestRequest(request); + try { - multiGetRequest.add(request.param("index"), request.param("type"), sFields, request.content()); + multiGetRequest.add(request.param("index"), request.param("type"), sFields, defaultFetchSource, request.content()); } catch (Exception e) { try { XContentBuilder builder = restContentBuilder(request); diff --git a/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index db348a1001890..0334871f33c2d 100644 --- a/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -36,6 +36,7 @@ import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.sort.SortOrder; import java.io.IOException; @@ -225,8 +226,15 @@ private SearchSourceBuilder parseSearchSource(RestRequest request) { } } } + FetchSourceContext fetchSourceContext = FetchSourceContext.parseFromRestRequest(request); + if (fetchSourceContext != null) { + if (searchSourceBuilder == null) { + searchSourceBuilder = new SearchSourceBuilder(); + } + searchSourceBuilder.fetchSource(fetchSourceContext); + } - if(request.hasParam("track_scores")) { + if (request.hasParam("track_scores")) { if (searchSourceBuilder == null) { searchSourceBuilder = new SearchSourceBuilder(); } diff --git a/src/main/java/org/elasticsearch/search/SearchModule.java b/src/main/java/org/elasticsearch/search/SearchModule.java index 6a76537a7ae80..c5dd3b4197b69 100644 --- a/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/src/main/java/org/elasticsearch/search/SearchModule.java @@ -32,6 +32,7 @@ import org.elasticsearch.search.fetch.matchedfilters.MatchedFiltersFetchSubPhase; import org.elasticsearch.search.fetch.partial.PartialFieldsFetchSubPhase; import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase; +import org.elasticsearch.search.fetch.source.FetchSourceSubPhase; import org.elasticsearch.search.fetch.version.VersionFetchSubPhase; import org.elasticsearch.search.highlight.HighlightModule; import org.elasticsearch.search.highlight.HighlightPhase; @@ -59,6 +60,7 @@ protected void configure() { bind(ExplainFetchSubPhase.class).asEagerSingleton(); bind(ScriptFieldsFetchSubPhase.class).asEagerSingleton(); bind(PartialFieldsFetchSubPhase.class).asEagerSingleton(); + bind(FetchSourceSubPhase.class).asEagerSingleton(); bind(VersionFetchSubPhase.class).asEagerSingleton(); bind(MatchedFiltersFetchSubPhase.class).asEagerSingleton(); bind(HighlightPhase.class).asEagerSingleton(); diff --git a/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 52758a93865ef..7d0828241d470 100644 --- a/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -27,6 +27,7 @@ import org.elasticsearch.ElasticSearchGenerationException; import org.elasticsearch.client.Requests; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.TimeValue; @@ -37,6 +38,7 @@ import org.elasticsearch.index.query.FilterBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.facet.FacetBuilder; +import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.HighlightBuilder; import org.elasticsearch.search.rescore.RescoreBuilder; import org.elasticsearch.search.sort.SortBuilder; @@ -98,6 +100,7 @@ public static HighlightBuilder highlight() { private List fieldNames; private List scriptFields; private List partialFields; + private FetchSourceContext fetchSourceContext; private List facets; @@ -420,6 +423,52 @@ public RescoreBuilder rescore() { return rescoreBuilder; } + /** + * Indicates whether the response should contain the stored _source for every hit + * + * @param fetch + * @return + */ + public SearchSourceBuilder fetchSource(boolean fetch) { + if (this.fetchSourceContext == null) { + this.fetchSourceContext = new FetchSourceContext(fetch); + } else { + this.fetchSourceContext.fetchSource(fetch); + } + return this; + } + + /** + * Indicate that _source should be returned with every hit, with an "include" and/or "exclude" set which can include simple wildcard + * elements. + * + * @param include An optional include (optionally wildcarded) pattern to filter the returned _source + * @param exclude An optional exclude (optionally wildcarded) pattern to filter the returned _source + */ + public SearchSourceBuilder fetchSource(@Nullable String include, @Nullable String exclude) { + return fetchSource(include == null ? Strings.EMPTY_ARRAY : new String[]{include}, include == null ? Strings.EMPTY_ARRAY : new String[]{exclude}); + } + + /** + * Indicate that _source should be returned with every hit, with an "include" and/or "exclude" set which can include simple wildcard + * elements. + * + * @param includes An optional list of include (optionally wildcarded) pattern to filter the returned _source + * @param excludes An optional list of exclude (optionally wildcarded) pattern to filter the returned _source + */ + public SearchSourceBuilder fetchSource(@Nullable String[] includes, @Nullable String[] excludes) { + fetchSourceContext = new FetchSourceContext(includes, excludes); + return this; + } + + /** + * Indicate how the _source should be fetched. + */ + public SearchSourceBuilder fetchSource(@Nullable FetchSourceContext fetchSourceContext) { + this.fetchSourceContext = fetchSourceContext; + return this; + } + /** * Sets no fields to be loaded, resulting in only id and type to be returned per field. */ @@ -634,6 +683,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("explain", explain); } + if (fetchSourceContext != null) { + if (!fetchSourceContext.fetchSource()) { + builder.field("_source", false); + } else { + builder.startObject("_source"); + builder.array("includes", fetchSourceContext.includes()); + builder.array("excludes", fetchSourceContext.excludes()); + builder.endObject(); + } + } + if (fieldNames != null) { if (fieldNames.size() == 1) { builder.field("fields", fieldNames.get(0)); diff --git a/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 39b69bb3a8f76..be6bf30be8013 100644 --- a/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -25,10 +25,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.text.StringAndBytesText; import org.elasticsearch.common.text.Text; -import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor; -import org.elasticsearch.index.fieldvisitor.FieldsVisitor; -import org.elasticsearch.index.fieldvisitor.JustUidFieldsVisitor; -import org.elasticsearch.index.fieldvisitor.UidAndSourceFieldsVisitor; +import org.elasticsearch.index.fieldvisitor.*; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMappers; import org.elasticsearch.index.mapper.internal.SourceFieldMapper; @@ -39,6 +36,8 @@ import org.elasticsearch.search.fetch.matchedfilters.MatchedFiltersFetchSubPhase; import org.elasticsearch.search.fetch.partial.PartialFieldsFetchSubPhase; import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase; +import org.elasticsearch.search.fetch.source.FetchSourceContext; +import org.elasticsearch.search.fetch.source.FetchSourceSubPhase; import org.elasticsearch.search.fetch.version.VersionFetchSubPhase; import org.elasticsearch.search.highlight.HighlightPhase; import org.elasticsearch.search.internal.InternalSearchHit; @@ -60,8 +59,10 @@ public class FetchPhase implements SearchPhase { @Inject public FetchPhase(HighlightPhase highlightPhase, ScriptFieldsFetchSubPhase scriptFieldsPhase, PartialFieldsFetchSubPhase partialFieldsPhase, - MatchedFiltersFetchSubPhase matchFiltersPhase, ExplainFetchSubPhase explainPhase, VersionFetchSubPhase versionPhase) { - this.fetchSubPhases = new FetchSubPhase[]{scriptFieldsPhase, partialFieldsPhase, matchFiltersPhase, explainPhase, highlightPhase, versionPhase}; + MatchedFiltersFetchSubPhase matchFiltersPhase, ExplainFetchSubPhase explainPhase, VersionFetchSubPhase versionPhase, + FetchSourceSubPhase fetchSourceSubPhase) { + this.fetchSubPhases = new FetchSubPhase[]{scriptFieldsPhase, partialFieldsPhase, matchFiltersPhase, explainPhase, highlightPhase, + fetchSourceSubPhase, versionPhase}; } @Override @@ -81,20 +82,24 @@ public void preProcess(SearchContext context) { public void execute(SearchContext context) { FieldsVisitor fieldsVisitor; List extractFieldNames = null; - boolean sourceRequested = false; + if (!context.hasFieldNames()) { if (context.hasPartialFields()) { - // partial fields need the source, so fetch it, but don't return it + // partial fields need the source, so fetch it fieldsVisitor = new UidAndSourceFieldsVisitor(); - } else if (context.hasScriptFields()) { - // we ask for script fields, and no field names, don't load the source - fieldsVisitor = new JustUidFieldsVisitor(); } else { - sourceRequested = true; - fieldsVisitor = new UidAndSourceFieldsVisitor(); + // no fields specified, default to return source if no explicit indication + if (!context.hasScriptFields() && !context.hasFetchSourceContext()) { + context.fetchSourceContext(new FetchSourceContext(true)); + } + fieldsVisitor = context.sourceRequested() ? new UidAndSourceFieldsVisitor() : new JustUidFieldsVisitor(); } } else if (context.fieldNames().isEmpty()) { - fieldsVisitor = new JustUidFieldsVisitor(); + if (context.sourceRequested()) { + fieldsVisitor = new UidAndSourceFieldsVisitor(); + } else { + fieldsVisitor = new JustUidFieldsVisitor(); + } } else { boolean loadAllStored = false; Set fieldNames = null; @@ -104,7 +109,11 @@ public void execute(SearchContext context) { continue; } if (fieldName.equals(SourceFieldMapper.NAME)) { - sourceRequested = true; + if (context.hasFetchSourceContext()) { + context.fetchSourceContext().fetchSource(true); + } else { + context.fetchSourceContext(new FetchSourceContext(true)); + } continue; } FieldMappers x = context.smartNameFieldMappers(fieldName); @@ -121,15 +130,11 @@ public void execute(SearchContext context) { } } if (loadAllStored) { - if (sourceRequested || extractFieldNames != null) { - fieldsVisitor = new CustomFieldsVisitor(true, true); // load everything, including _source - } else { - fieldsVisitor = new CustomFieldsVisitor(true, false); - } + fieldsVisitor = new AllFieldsVisitor(); // load everything, including _source } else if (fieldNames != null) { - boolean loadSource = extractFieldNames != null || sourceRequested; + boolean loadSource = extractFieldNames != null || context.sourceRequested(); fieldsVisitor = new CustomFieldsVisitor(fieldNames, loadSource); - } else if (extractFieldNames != null || sourceRequested) { + } else if (extractFieldNames != null || context.sourceRequested()) { fieldsVisitor = new UidAndSourceFieldsVisitor(); } else { fieldsVisitor = new JustUidFieldsVisitor(); @@ -158,7 +163,7 @@ public void execute(SearchContext context) { } else { typeText = documentMapper.typeText(); } - InternalSearchHit searchHit = new InternalSearchHit(docId, fieldsVisitor.uid().id(), typeText, sourceRequested ? fieldsVisitor.source() : null, searchFields); + InternalSearchHit searchHit = new InternalSearchHit(docId, fieldsVisitor.uid().id(), typeText, searchFields); hits[index] = searchHit; diff --git a/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceContext.java b/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceContext.java new file mode 100644 index 0000000000000..053c85bf1dff2 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceContext.java @@ -0,0 +1,165 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.fetch.source; + +import org.elasticsearch.common.Booleans; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.rest.RestRequest; + +import java.io.IOException; + +/** + */ +public class FetchSourceContext implements Streamable { + + public static final FetchSourceContext FETCH_SOURCE = new FetchSourceContext(true); + public static final FetchSourceContext DO_NOT_FETCH_SOURCE = new FetchSourceContext(false); + private boolean fetchSource; + private String[] includes; + private String[] excludes; + + + FetchSourceContext() { + + } + + public FetchSourceContext(boolean fetchSource) { + this(fetchSource, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY); + } + + public FetchSourceContext(String include) { + this(include, null); + } + + public FetchSourceContext(String include, String exclude) { + this(true, + include == null ? Strings.EMPTY_ARRAY : new String[]{include}, + exclude == null ? Strings.EMPTY_ARRAY : new String[]{exclude}); + } + + public FetchSourceContext(String[] includes) { + this(true, includes, Strings.EMPTY_ARRAY); + } + + public FetchSourceContext(String[] includes, String[] excludes) { + this(true, includes, excludes); + } + + public FetchSourceContext(boolean fetchSource, String[] includes, String[] excludes) { + this.fetchSource = fetchSource; + this.includes = includes == null ? Strings.EMPTY_ARRAY : includes; + this.excludes = excludes == null ? Strings.EMPTY_ARRAY : excludes; + } + + public boolean fetchSource() { + return this.fetchSource; + } + + public FetchSourceContext fetchSource(boolean fetchSource) { + this.fetchSource = fetchSource; + return this; + } + + public String[] includes() { + return this.includes; + } + + public FetchSourceContext includes(String[] includes) { + this.includes = includes; + return this; + } + + public String[] excludes() { + return this.excludes; + } + + public FetchSourceContext excludes(String[] excludes) { + this.excludes = excludes; + return this; + } + + public static FetchSourceContext optionalReadFromStream(StreamInput in) throws IOException { + if (!in.readBoolean()) { + return null; + } + FetchSourceContext context = new FetchSourceContext(); + context.readFrom(in); + return context; + } + + public static void optionalWriteToStream(FetchSourceContext context, StreamOutput out) throws IOException { + if (context == null) { + out.writeBoolean(false); + return; + } + out.writeBoolean(true); + context.writeTo(out); + } + + public static FetchSourceContext parseFromRestRequest(RestRequest request) { + Boolean fetchSource = null; + String[] source_excludes = null; + String[] source_includes = null; + + String source = request.param("_source"); + if (source != null) { + if (Booleans.isExplicitTrue(source)) { + fetchSource = true; + } else if (Booleans.isExplicitFalse(source)) { + fetchSource = false; + } else { + source_includes = Strings.splitStringByCommaToArray(source); + } + } + String sIncludes = request.param("_source_includes"); + sIncludes = request.param("_source_include", sIncludes); + if (sIncludes != null) { + source_includes = Strings.splitStringByCommaToArray(sIncludes); + } + + String sExcludes = request.param("_source_excludes"); + sExcludes = request.param("_source_exclude", sExcludes); + if (sExcludes != null) { + source_excludes = Strings.splitStringByCommaToArray(sExcludes); + } + + if (fetchSource != null || source_includes != null || source_excludes != null) { + return new FetchSourceContext(fetchSource == null ? true : fetchSource, source_includes, source_excludes); + } + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + fetchSource = in.readBoolean(); + includes = in.readStringArray(); + excludes = in.readStringArray(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(fetchSource); + out.writeStringArray(includes); + out.writeStringArray(excludes); + } +} diff --git a/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceParseElement.java b/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceParseElement.java new file mode 100644 index 0000000000000..23e97ba58dec0 --- /dev/null +++ b/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceParseElement.java @@ -0,0 +1,96 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.fetch.source; + +import org.elasticsearch.ElasticSearchParseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchParseElement; +import org.elasticsearch.search.internal.SearchContext; + +import java.util.ArrayList; +import java.util.List; + +/** + * 

+ * "source" : true/false
+ * "source" : "field"
+ * "source" : [ "include", "include" ]
+ * "source" : {
+ *     "include" : ["obj"]
+ *     "exclude" : ["obj"]
+ * }
+ *
+ */ +public class FetchSourceParseElement implements SearchParseElement { + + @Override + public void parse(XContentParser parser, SearchContext context) throws Exception { + XContentParser.Token token; + + List includes = null, excludes = null; + String currentFieldName = null; + token = parser.currentToken(); // we get it on the value + if (token == XContentParser.Token.VALUE_BOOLEAN) { + context.fetchSourceContext(new FetchSourceContext(parser.booleanValue())); + return; + } else if (token == XContentParser.Token.VALUE_STRING) { + context.fetchSourceContext(new FetchSourceContext(new String[]{parser.text()})); + return; + } else if (token == XContentParser.Token.START_ARRAY) { + includes = new ArrayList(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + includes.add(parser.text()); + } + } else if (token == XContentParser.Token.START_OBJECT) { + + List currentList = null; + + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + if ("includes".equals(currentFieldName) || "include".equals(currentFieldName)) { + currentList = includes != null ? includes : (includes = new ArrayList(2)); + } else if ("excludes".equals(currentFieldName) || "exclude".equals(currentFieldName)) { + currentList = excludes != null ? excludes : (excludes = new ArrayList(2)); + } else { + throw new ElasticSearchParseException("Source definition may not contain " + parser.text()); + } + } else if (token == XContentParser.Token.START_ARRAY) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + currentList.add(parser.text()); + } + } else if (token.isValue()) { + currentList.add(parser.text()); + } else { + throw new ElasticSearchParseException("unexpected token while parsing source settings"); + } + } + } else { + throw new ElasticSearchParseException("source element value can be of type " + token.name()); + } + + + context.fetchSourceContext(new FetchSourceContext( + includes == null ? Strings.EMPTY_ARRAY : includes.toArray(new String[includes.size()]), + excludes == null ? Strings.EMPTY_ARRAY : excludes.toArray(new String[excludes.size()]))); + + } +} diff --git a/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java b/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java new file mode 100644 index 0000000000000..cc33b2a6d438e --- /dev/null +++ b/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java @@ -0,0 +1,84 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.fetch.source; + +import com.google.common.collect.ImmutableMap; +import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.search.SearchParseElement; +import org.elasticsearch.search.fetch.FetchSubPhase; +import org.elasticsearch.search.internal.InternalSearchHit; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Map; + +/** + */ +public class FetchSourceSubPhase implements FetchSubPhase { + + @Inject + public FetchSourceSubPhase() { + + } + + @Override + public Map parseElements() { + ImmutableMap.Builder parseElements = ImmutableMap.builder(); + parseElements.put("_source", new FetchSourceParseElement()); + return parseElements.build(); + } + + @Override + public boolean hitsExecutionNeeded(SearchContext context) { + return false; + } + + @Override + public void hitsExecute(SearchContext context, InternalSearchHit[] hits) throws ElasticSearchException { + } + + @Override + public boolean hitExecutionNeeded(SearchContext context) { + return context.sourceRequested(); + } + + @Override + public void hitExecute(SearchContext context, HitContext hitContext) throws ElasticSearchException { + FetchSourceContext fetchSourceContext = context.fetchSourceContext(); + assert fetchSourceContext.fetchSource(); + if (fetchSourceContext.includes().length == 0 && fetchSourceContext.excludes().length == 0) { + hitContext.hit().sourceRef(context.lookup().source().internalSourceRef()); + return; + } + + Object value = context.lookup().source().filter(fetchSourceContext.includes(), fetchSourceContext.excludes()); + try { + XContentBuilder builder = XContentFactory.contentBuilder(context.lookup().source().sourceContentType()); + builder.value(value); + hitContext.hit().sourceRef(builder.bytes()); + } catch (IOException e) { + throw new ElasticSearchException("Error filtering source", e); + } + + } +} diff --git a/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java b/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java index 97b7850ae153e..e1afa4ff5d309 100644 --- a/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java +++ b/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java @@ -90,11 +90,10 @@ private InternalSearchHit() { } - public InternalSearchHit(int docId, String id, Text type, BytesReference source, Map fields) { + public InternalSearchHit(int docId, String id, Text type, Map fields) { this.docId = docId; this.id = new StringAndBytesText(id); this.type = type; - this.source = source; this.fields = fields; } @@ -177,6 +176,16 @@ public BytesReference sourceRef() { } } + /** + * Sets representation, might be compressed.... + */ + public InternalSearchHit sourceRef(BytesReference source) { + this.source = source; + this.sourceAsBytes = null; + this.sourceAsMap = null; + return this; + } + @Override public BytesReference getSourceRef() { return sourceRef(); @@ -189,6 +198,7 @@ public BytesReference internalSourceRef() { return source; } + @Override public byte[] source() { if (source == null) { diff --git a/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/src/main/java/org/elasticsearch/search/internal/SearchContext.java index ed3ed08c9031e..6680906d01825 100644 --- a/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -58,6 +58,7 @@ import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.fetch.partial.PartialFieldsContext; import org.elasticsearch.search.fetch.script.ScriptFieldsContext; +import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.SearchContextHighlight; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.query.QuerySearchResult; @@ -142,6 +143,7 @@ public static interface Rewrite { private List fieldNames; private ScriptFieldsContext scriptFields; private PartialFieldsContext partialFields; + private FetchSourceContext fetchSourceContext; private int from = -1; @@ -259,7 +261,6 @@ public Filter searchFilter(String[] types) { } } - public long id() { return this.id; } @@ -370,6 +371,28 @@ public PartialFieldsContext partialFields() { return this.partialFields; } + /** + * A shortcut function to see whether there is a fetchSourceContext and it says the source is requested. + * + * @return + */ + public boolean sourceRequested() { + return fetchSourceContext != null && fetchSourceContext.fetchSource(); + } + + public boolean hasFetchSourceContext() { + return fetchSourceContext != null; + } + + public FetchSourceContext fetchSourceContext() { + return this.fetchSourceContext; + } + + public SearchContext fetchSourceContext(FetchSourceContext fetchSourceContext) { + this.fetchSourceContext = fetchSourceContext; + return this; + } + public ContextIndexSearcher searcher() { return this.searcher; } diff --git a/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index 3771ed44a7a50..c37a609250c8a 100644 --- a/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -24,7 +24,9 @@ import org.apache.lucene.index.AtomicReaderContext; import org.elasticsearch.ElasticSearchParseException; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fieldvisitor.JustSourceFieldsVisitor; @@ -36,7 +38,6 @@ /** * */ -// TODO: If we are processing it in the per hit fetch phase, we cna initialize it with a source if it was loaded.. public class SourceLookup implements Map { private AtomicReader reader; @@ -45,17 +46,24 @@ public class SourceLookup implements Map { private BytesReference sourceAsBytes; private Map source; + private XContentType sourceContentType; public Map source() { return source; } + public XContentType sourceContentType() { + return sourceContentType; + } + private Map loadSourceIfNeeded() { if (source != null) { return source; } if (sourceAsBytes != null) { - source = sourceAsMap(sourceAsBytes); + Tuple> tuple = sourceAsMapAndType(sourceAsBytes); + sourceContentType = tuple.v1(); + source = tuple.v2(); return source; } try { @@ -64,8 +72,11 @@ private Map loadSourceIfNeeded() { BytesReference source = sourceFieldVisitor.source(); if (source == null) { this.source = ImmutableMap.of(); + this.sourceContentType = null; } else { - this.source = sourceAsMap(source); + Tuple> tuple = sourceAsMapAndType(source); + this.sourceContentType = tuple.v1(); + this.source = tuple.v2(); } } catch (Exception e) { throw new ElasticSearchParseException("failed to parse / load source", e); @@ -73,12 +84,20 @@ private Map loadSourceIfNeeded() { return this.source; } + public static Tuple> sourceAsMapAndType(BytesReference source) throws ElasticSearchParseException { + return XContentHelper.convertToMap(source, false); + } + public static Map sourceAsMap(BytesReference source) throws ElasticSearchParseException { - return XContentHelper.convertToMap(source, false).v2(); + return sourceAsMapAndType(source).v2(); + } + + public static Tuple> sourceAsMapAndType(byte[] bytes, int offset, int length) throws ElasticSearchParseException { + return XContentHelper.convertToMap(bytes, offset, length, false); } public static Map sourceAsMap(byte[] bytes, int offset, int length) throws ElasticSearchParseException { - return XContentHelper.convertToMap(bytes, offset, length, false).v2(); + return sourceAsMapAndType(bytes, offset, length).v2(); } public void setNextReader(AtomicReaderContext context) { @@ -108,6 +127,13 @@ public void setNextSource(Map source) { this.source = source; } + /** + * Internal source representation, might be compressed.... + */ + public BytesReference internalSourceRef() { + return sourceAsBytes; + } + /** * Returns the values associated with the path. Those are "low" level values, and it can * handle path expression where an array/list is navigated within. diff --git a/src/test/java/org/elasticsearch/test/integration/explain/ExplainActionTests.java b/src/test/java/org/elasticsearch/test/integration/explain/ExplainActionTests.java index 8149d4bb68b6a..de47da90a4c4d 100644 --- a/src/test/java/org/elasticsearch/test/integration/explain/ExplainActionTests.java +++ b/src/test/java/org/elasticsearch/test/integration/explain/ExplainActionTests.java @@ -44,7 +44,8 @@ public void testSimple() throws Exception { cluster().ensureAtLeastNumNodes(2); try { client().admin().indices().prepareDelete("test").execute().actionGet(); - } catch (IndexMissingException e) {} + } catch (IndexMissingException e) { + } client().admin().indices().prepareCreate("test").setSettings( ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1) ).execute().actionGet(); @@ -103,12 +104,14 @@ public void testSimple() throws Exception { assertFalse(response.isMatch()); } + @SuppressWarnings("unchecked") @Test public void testExplainWithFields() throws Exception { cluster().ensureAtLeastNumNodes(2); try { client().admin().indices().prepareDelete("test").execute().actionGet(); - } catch (IndexMissingException e) {} + } catch (IndexMissingException e) { + } client().admin().indices().prepareCreate("test").execute().actionGet(); client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet(); @@ -116,10 +119,10 @@ public void testExplainWithFields() throws Exception { .setSource( jsonBuilder().startObject() .startObject("obj1") - .field("field1", "value1") - .field("field2", "value2") + .field("field1", "value1") + .field("field2", "value2") + .endObject() .endObject() - .endObject() ).execute().actionGet(); client().admin().indices().prepareRefresh("test").execute().actionGet(); @@ -136,6 +139,24 @@ public void testExplainWithFields() throws Exception { assertThat(response.getGetResult().getId(), equalTo("1")); assertThat(response.getGetResult().getFields().size(), equalTo(1)); assertThat(response.getGetResult().getFields().get("obj1.field1").getValue().toString(), equalTo("value1")); + assertThat(response.getGetResult().isSourceEmpty(), equalTo(true)); + + client().admin().indices().prepareRefresh("test").execute().actionGet(); + response = client().prepareExplain("test", "test", "1") + .setQuery(QueryBuilders.matchAllQuery()) + .setFields("obj1.field1") + .setFetchSource(true) + .get(); + assertNotNull(response); + assertTrue(response.isMatch()); + assertNotNull(response.getExplanation()); + assertTrue(response.getExplanation().isMatch()); + assertThat(response.getExplanation().getValue(), equalTo(1.0f)); + assertThat(response.getGetResult().isExists(), equalTo(true)); + assertThat(response.getGetResult().getId(), equalTo("1")); + assertThat(response.getGetResult().getFields().size(), equalTo(1)); + assertThat(response.getGetResult().getFields().get("obj1.field1").getValue().toString(), equalTo("value1")); + assertThat(response.getGetResult().isSourceEmpty(), equalTo(false)); response = client().prepareExplain("test", "test", "1") .setQuery(QueryBuilders.matchAllQuery()) @@ -150,12 +171,59 @@ public void testExplainWithFields() throws Exception { assertThat(fields.get("field2"), equalTo("value2")); } + @SuppressWarnings("unchecked") + @Test + public void testExplainWitSource() throws Exception { + cluster().ensureAtLeastNumNodes(2); + try { + client().admin().indices().prepareDelete("test").execute().actionGet(); + } catch (IndexMissingException e) { + } + client().admin().indices().prepareCreate("test").execute().actionGet(); + client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet(); + + client().prepareIndex("test", "test", "1") + .setSource( + jsonBuilder().startObject() + .startObject("obj1") + .field("field1", "value1") + .field("field2", "value2") + .endObject() + .endObject() + ).execute().actionGet(); + + client().admin().indices().prepareRefresh("test").execute().actionGet(); + ExplainResponse response = client().prepareExplain("test", "test", "1") + .setQuery(QueryBuilders.matchAllQuery()) + .setFetchSource("obj1.field1", null) + .get(); + assertNotNull(response); + assertTrue(response.isMatch()); + assertNotNull(response.getExplanation()); + assertTrue(response.getExplanation().isMatch()); + assertThat(response.getExplanation().getValue(), equalTo(1.0f)); + assertThat(response.getGetResult().isExists(), equalTo(true)); + assertThat(response.getGetResult().getId(), equalTo("1")); + assertThat(response.getGetResult().getSource().size(), equalTo(1)); + assertThat(((Map) response.getGetResult().getSource().get("obj1")).get("field1").toString(), equalTo("value1")); + + response = client().prepareExplain("test", "test", "1") + .setQuery(QueryBuilders.matchAllQuery()) + .setFetchSource(null, "obj1.field2") + .execute().actionGet(); + assertNotNull(response); + assertTrue(response.isMatch()); + assertThat(((Map) response.getGetResult().getSource().get("obj1")).get("field1").toString(), equalTo("value1")); + } + + @Test public void testExplainWithAlias() throws Exception { cluster().ensureAtLeastNumNodes(2); try { client().admin().indices().prepareDelete("test").execute().actionGet(); - } catch (IndexMissingException e) {} + } catch (IndexMissingException e) { + } client().admin().indices().prepareCreate("test") .execute().actionGet(); client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet(); diff --git a/src/test/java/org/elasticsearch/test/integration/get/GetActionTests.java b/src/test/java/org/elasticsearch/test/integration/get/GetActionTests.java index 841a214e41f2e..b8f43cffef018 100644 --- a/src/test/java/org/elasticsearch/test/integration/get/GetActionTests.java +++ b/src/test/java/org/elasticsearch/test/integration/get/GetActionTests.java @@ -35,6 +35,8 @@ import org.elasticsearch.test.integration.AbstractSharedClusterTest; import org.junit.Test; +import java.util.Map; + import static org.elasticsearch.client.Requests.clusterHealthRequest; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.*; @@ -46,15 +48,13 @@ public void simpleGetTests() { client().admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1)).execute().actionGet(); - ClusterHealthResponse clusterHealth = client().admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet(); - assertThat(clusterHealth.isTimedOut(), equalTo(false)); - assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN)); + ensureGreen(); GetResponse response = client().prepareGet("test", "type1", "1").execute().actionGet(); assertThat(response.isExists(), equalTo(false)); logger.info("--> index doc 1"); - client().prepareIndex("test", "type1", "1").setSource("field1", "value1", "field2", "value2").execute().actionGet(); + client().prepareIndex("test", "type1", "1").setSource("field1", "value1", "field2", "value2").get(); logger.info("--> realtime get 1"); response = client().prepareGet("test", "type1", "1").execute().actionGet(); @@ -62,11 +62,18 @@ public void simpleGetTests() { assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1")); assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2")); - logger.info("--> realtime get 1 (no source)"); - response = client().prepareGet("test", "type1", "1").setFields(Strings.EMPTY_ARRAY).execute().actionGet(); + logger.info("--> realtime get 1 (no source, implicit)"); + response = client().prepareGet("test", "type1", "1").setFields(Strings.EMPTY_ARRAY).get(); + assertThat(response.isExists(), equalTo(true)); + assertThat(response.getFields().size(), equalTo(0)); + assertThat(response.getSourceAsBytes(), nullValue()); + + logger.info("--> realtime get 1 (no source, explicit)"); + response = client().prepareGet("test", "type1", "1").setFetchSource(false).get(); assertThat(response.isExists(), equalTo(true)); + assertThat(response.getFields().size(), equalTo(0)); assertThat(response.getSourceAsBytes(), nullValue()); - + logger.info("--> realtime get 1 (no type)"); response = client().prepareGet("test", null, "1").execute().actionGet(); assertThat(response.isExists(), equalTo(true)); @@ -84,6 +91,14 @@ public void simpleGetTests() { assertThat(response.getField("field1").getValues().get(0).toString(), equalTo("value1")); assertThat(response.getField("field2"), nullValue()); + logger.info("--> realtime fetch of field & source (requires fetching parsing source)"); + response = client().prepareGet("test", "type1", "1").setFields("field1").setFetchSource("field1", null).execute().actionGet(); + assertThat(response.isExists(), equalTo(true)); + assertThat(response.getSourceAsMap(), hasKey("field1")); + assertThat(response.getSourceAsMap(), not(hasKey("field2"))); + assertThat(response.getField("field1").getValues().get(0).toString(), equalTo("value1")); + assertThat(response.getField("field2"), nullValue()); + logger.info("--> flush the index, so we load it from it"); client().admin().indices().prepareFlush().execute().actionGet(); @@ -106,6 +121,13 @@ public void simpleGetTests() { assertThat(response.getField("field1").getValues().get(0).toString(), equalTo("value1")); assertThat(response.getField("field2"), nullValue()); + logger.info("--> realtime fetch of field & source (loaded from index)"); + response = client().prepareGet("test", "type1", "1").setFields("field1").setFetchSource(true).execute().actionGet(); + assertThat(response.isExists(), equalTo(true)); + assertThat(response.getSourceAsBytes(), not(nullValue())); + assertThat(response.getField("field1").getValues().get(0).toString(), equalTo("value1")); + assertThat(response.getField("field2"), nullValue()); + logger.info("--> update doc 1"); client().prepareIndex("test", "type1", "1").setSource("field1", "value1_1", "field2", "value2_1").execute().actionGet(); @@ -455,6 +477,7 @@ public void testThatGetFromTranslogShouldWorkWithInclude() throws Exception { assertThat(responseBeforeFlush.getSourceAsString(), is(responseAfterFlush.getSourceAsString())); } + @SuppressWarnings("unchecked") @Test public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFields() throws Exception { client().admin().indices().prepareDelete().execute().actionGet(); @@ -480,21 +503,37 @@ public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFields() throw client().prepareIndex(index, type, "1") .setSource(jsonBuilder().startObject() .field("field", "1", "2") - .field("included", "should be seen") - .field("excluded", "should not be seen") + .startObject("included").field("field", "should be seen").field("field2", "extra field to remove").endObject() + .startObject("excluded").field("field", "should not be seen").field("field2", "should not be seen").endObject() .endObject()) .execute().actionGet(); GetResponse responseBeforeFlush = client().prepareGet(index, type, "1").setFields("_source", "included", "excluded").execute().actionGet(); - client().admin().indices().prepareFlush(index).execute().actionGet(); - GetResponse responseAfterFlush = client().prepareGet(index, type, "1").setFields("_source", "included", "excluded").execute().actionGet(); - assertThat(responseBeforeFlush.isExists(), is(true)); - assertThat(responseAfterFlush.isExists(), is(true)); assertThat(responseBeforeFlush.getSourceAsMap(), not(hasKey("excluded"))); assertThat(responseBeforeFlush.getSourceAsMap(), not(hasKey("field"))); assertThat(responseBeforeFlush.getSourceAsMap(), hasKey("included")); + + // now tests that extra source filtering works as expected + GetResponse responseBeforeFlushWithExtraFilters = client().prepareGet(index, type, "1").setFields("included", "excluded") + .setFetchSource(new String[]{"field", "*.field"}, new String[]{"*.field2"}).get(); + assertThat(responseBeforeFlushWithExtraFilters.isExists(), is(true)); + assertThat(responseBeforeFlushWithExtraFilters.getSourceAsMap(), not(hasKey("excluded"))); + assertThat(responseBeforeFlushWithExtraFilters.getSourceAsMap(), not(hasKey("field"))); + assertThat(responseBeforeFlushWithExtraFilters.getSourceAsMap(), hasKey("included")); + assertThat((Map) responseBeforeFlushWithExtraFilters.getSourceAsMap().get("included"), hasKey("field")); + assertThat((Map) responseBeforeFlushWithExtraFilters.getSourceAsMap().get("included"), not(hasKey("field2"))); + + client().admin().indices().prepareFlush(index).execute().actionGet(); + GetResponse responseAfterFlush = client().prepareGet(index, type, "1").setFields("_source", "included", "excluded").execute().actionGet(); + GetResponse responseAfterFlushWithExtraFilters = client().prepareGet(index, type, "1").setFields("included", "excluded") + .setFetchSource("*.field", "*.field2").get(); + + assertThat(responseAfterFlush.isExists(), is(true)); assertThat(responseBeforeFlush.getSourceAsString(), is(responseAfterFlush.getSourceAsString())); + + assertThat(responseAfterFlushWithExtraFilters.isExists(), is(true)); + assertThat(responseBeforeFlushWithExtraFilters.getSourceAsString(), is(responseAfterFlushWithExtraFilters.getSourceAsString())); } @Test diff --git a/src/test/java/org/elasticsearch/test/integration/mget/SimpleMgetTests.java b/src/test/java/org/elasticsearch/test/integration/mget/SimpleMgetTests.java index be58eccb7081a..9de5a2a4d40c5 100644 --- a/src/test/java/org/elasticsearch/test/integration/mget/SimpleMgetTests.java +++ b/src/test/java/org/elasticsearch/test/integration/mget/SimpleMgetTests.java @@ -18,16 +18,20 @@ */ package org.elasticsearch.test.integration.mget; +import org.elasticsearch.action.get.MultiGetItemResponse; import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetRequestBuilder; import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.test.integration.AbstractSharedClusterTest; import org.junit.Test; import java.io.IOException; +import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.*; public class SimpleMgetTests extends AbstractSharedClusterTest { @@ -72,9 +76,9 @@ public void testThatParentPerDocumentIsSupported() throws Exception { .startObject("_parent") .field("type", "foo") .endObject() - .endObject(). - endObject() - ).execute().actionGet(); + .endObject() + .endObject() + ).get(); client().prepareIndex("test", "test", "1").setParent("4").setRefresh(true) .setSource(jsonBuilder().startObject().field("foo", "bar").endObject()) @@ -93,6 +97,48 @@ public void testThatParentPerDocumentIsSupported() throws Exception { assertThat(mgetResponse.getResponses()[1].getResponse().isExists(), is(false)); } + @SuppressWarnings("unchecked") + @Test + public void testThatSourceFilteringIsSupported() throws Exception { + createIndex("test"); + ensureYellow(); + BytesReference sourceBytesRef = jsonBuilder().startObject() + .field("field", "1", "2") + .startObject("included").field("field", "should be seen").field("hidden_field", "should not be seen").endObject() + .field("excluded", "should not be seen") + .endObject().bytes(); + for (int i = 0; i < 100; i++) { + client().prepareIndex("test", "type", Integer.toString(i)).setSource(sourceBytesRef).get(); + } + + MultiGetRequestBuilder request = client().prepareMultiGet(); + for (int i = 0; i < 100; i++) { + if (i % 2 == 0) { + request.add(new MultiGetRequest.Item("test", "type", Integer.toString(i)).fetchSourceContext(new FetchSourceContext("included", "*.hidden_field"))); + } else { + request.add(new MultiGetRequest.Item("test", "type", Integer.toString(i)).fetchSourceContext(new FetchSourceContext(false))); + } + } + + MultiGetResponse response = request.get(); + + assertThat(response.getResponses().length, equalTo(100)); + for (int i = 0; i < 100; i++) { + MultiGetItemResponse responseItem = response.getResponses()[i]; + if (i % 2 == 0) { + Map source = responseItem.getResponse().getSourceAsMap(); + assertThat(source.size(), equalTo(1)); + assertThat(source, hasKey("included")); + assertThat(((Map) source.get("included")).size(), equalTo(1)); + assertThat(((Map) source.get("included")), hasKey("field")); + } else { + assertThat(responseItem.getResponse().getSourceAsBytes(), nullValue()); + } + } + + + } + @Test public void testThatRoutingPerDocumentIsSupported() throws Exception { createIndex("test"); diff --git a/src/test/java/org/elasticsearch/test/integration/search/source/SourceFetchingTests.java b/src/test/java/org/elasticsearch/test/integration/search/source/SourceFetchingTests.java new file mode 100644 index 0000000000000..ec9d9df508650 --- /dev/null +++ b/src/test/java/org/elasticsearch/test/integration/search/source/SourceFetchingTests.java @@ -0,0 +1,85 @@ +package org.elasticsearch.test.integration.search.source; +/* + * Licensed to ElasticSearch under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.test.integration.AbstractSharedClusterTest; +import org.junit.Test; + +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.core.IsEqual.equalTo; + +public class SourceFetchingTests extends AbstractSharedClusterTest { + + @Test + public void testSourceDefaultBehavior() { + createIndex("test"); + ensureGreen(); + + index("test", "type1", "1", "field", "value"); + refresh(); + + SearchResponse response = client().prepareSearch("test").get(); + assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue()); + + response = client().prepareSearch("test").addField("bla").get(); + assertThat(response.getHits().getAt(0).getSourceAsString(), nullValue()); + + response = client().prepareSearch("test").addField("_source").get(); + assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue()); + + response = client().prepareSearch("test").addPartialField("test", "field", null).get(); + assertThat(response.getHits().getAt(0).getSourceAsString(), nullValue()); + + } + + @Test + public void testSourceFiltering() { + createIndex("test"); + ensureGreen(); + + client().prepareIndex("test", "type1", "1").setSource("field1", "value", "field2", "value2").get(); + refresh(); + + SearchResponse response = client().prepareSearch("test").setFetchSource(false).get(); + assertThat(response.getHits().getAt(0).getSourceAsString(), nullValue()); + + response = client().prepareSearch("test").setFetchSource(true).get(); + assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue()); + + response = client().prepareSearch("test").setFetchSource("field1", null).get(); + assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue()); + assertThat(response.getHits().getAt(0).getSource().size(), equalTo(1)); + assertThat((String) response.getHits().getAt(0).getSource().get("field1"), equalTo("value")); + + response = client().prepareSearch("test").setFetchSource("hello", null).get(); + assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue()); + assertThat(response.getHits().getAt(0).getSource().size(), equalTo(0)); + + response = client().prepareSearch("test").setFetchSource(new String[]{"*"}, new String[]{"field2"}).get(); + assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue()); + assertThat(response.getHits().getAt(0).getSource().size(), equalTo(1)); + assertThat((String) response.getHits().getAt(0).getSource().get("field1"), equalTo("value")); + + } + + +}