Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Mappings: Remove ability to disable _source field #10915

Merged
merged 1 commit into from
May 6, 2015
Merged

Mappings: Remove ability to disable _source field #10915

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 3 additions & 12 deletions docs/reference/mapping/fields/source-field.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,6 @@ actual JSON that was used as the indexed document. It is not indexed
<<search-search,search>>, the `_source` field is
returned by default.

Though very handy to have around, the source field does incur storage
overhead within the index. For this reason, it can be disabled. For
example:

[source,js]
--------------------------------------------------
{
"tweet" : {
"_source" : {"enabled" : false}
}
}
--------------------------------------------------
Many APIs may use the `_source` field. For example, the
<<docs-update,Update API>>. To minimize the storage cost of
`_source`, set `index.codec: best_compression` in index settings.
9 changes: 9 additions & 0 deletions docs/reference/migration/migrate_2_0.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,15 @@ to provide special features. They now have limited configuration options.
* `_field_names` configuration is limited to disabling the field.
* `_size` configuration is limited to enabling the field.

==== Source field limitations
The `_source` field could previously be disabled dynamically. Since this field
is a critical piece of many features like the Update API, it is no longer
possible to disable.

The options for `compress` and `compress_threshold` have also been removed.
The source field is already compressed. To minimize the storage cost,
set `index.codec: best_compression` in index settings.

==== Boolean fields

Boolean fields used to have a string fielddata with `F` meaning `false` and `T`
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,15 +145,15 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("enabled")) {
if (fieldName.equals("enabled") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
builder.enabled(nodeBooleanValue(fieldNode));
iterator.remove();
} else if (fieldName.equals("compress")) {
} else if (fieldName.equals("compress") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
if (fieldNode != null) {
builder.compress(nodeBooleanValue(fieldNode));
}
iterator.remove();
} else if (fieldName.equals("compress_threshold")) {
} else if (fieldName.equals("compress_threshold") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
if (fieldNode != null) {
if (fieldNode instanceof Number) {
builder.compressThreshold(((Number) fieldNode).longValue());
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ public void doAfterNodes(int numNodes, Client client) throws Exception {

logger.info("--> add some metadata, additional type and template");
client.admin().indices().preparePutMapping("test").setType("type2")
.setSource(jsonBuilder().startObject().startObject("type2").startObject("_source").field("enabled", false).endObject().endObject().endObject())
.setSource(jsonBuilder().startObject().startObject("type2").endObject().endObject())
.execute().actionGet();
client.admin().indices().preparePutTemplate("template_1")
.setTemplate("te*")
Expand Down
34 changes: 15 additions & 19 deletions src/test/java/org/elasticsearch/get/GetActionTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,9 @@ public void simpleMultiGetTests() throws Exception {
}

@Test
public void realtimeGetWithCompress() throws Exception {
assertAcked(prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
public void realtimeGetWithCompressBackcompat() throws Exception {
assertAcked(prepareCreate("test")
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1).put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id))
.addMapping("type", jsonBuilder().startObject().startObject("type").startObject("_source").field("compress", true).endObject().endObject().endObject()));
ensureGreen();

Expand All @@ -249,9 +250,8 @@ public void realtimeGetWithCompress() throws Exception {
@Test
public void getFieldsWithDifferentTypes() throws Exception {
assertAcked(prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
.addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("_source").field("enabled", true).endObject().endObject().endObject())
.addMapping("type1", jsonBuilder().startObject().startObject("type1").endObject().endObject())
.addMapping("type2", jsonBuilder().startObject().startObject("type2")
.startObject("_source").field("enabled", false).endObject()
.startObject("properties")
.startObject("str").field("type", "string").field("store", "yes").endObject()
.startObject("strs").field("type", "string").field("store", "yes").endObject()
Expand Down Expand Up @@ -339,7 +339,6 @@ public void testGetDocWithMultivaluedFields() throws Exception {
.startObject("properties")
.startObject("field").field("type", "string").field("store", "yes").endObject()
.endObject()
.startObject("_source").field("enabled", false).endObject()
.endObject().endObject().string();
assertAcked(prepareCreate("test")
.addMapping("type1", mapping1)
Expand Down Expand Up @@ -397,7 +396,7 @@ public void testGetDocWithMultivaluedFields() throws Exception {
}

@Test
public void testThatGetFromTranslogShouldWorkWithExclude() throws Exception {
public void testThatGetFromTranslogShouldWorkWithExcludeBackcompat() throws Exception {
String index = "test";
String type = "type1";

Expand Down Expand Up @@ -431,7 +430,7 @@ public void testThatGetFromTranslogShouldWorkWithExclude() throws Exception {
}

@Test
public void testThatGetFromTranslogShouldWorkWithInclude() throws Exception {
public void testThatGetFromTranslogShouldWorkWithIncludeBackcompat() throws Exception {
String index = "test";
String type = "type1";

Expand Down Expand Up @@ -466,7 +465,7 @@ public void testThatGetFromTranslogShouldWorkWithInclude() throws Exception {

@SuppressWarnings("unchecked")
@Test
public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFields() throws Exception {
public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFieldsBackcompat() throws Exception {
String index = "test";
String type = "type1";

Expand Down Expand Up @@ -925,9 +924,6 @@ public void testUngeneratedFieldsThatAreNeverStored() throws IOException {
" },\n" +
" \"mappings\": {\n" +
" \"doc\": {\n" +
" \"_source\": {\n" +
" \"enabled\": \"" + randomBoolean() + "\"\n" +
" },\n" +
" \"properties\": {\n" +
" \"suggest\": {\n" +
" \"type\": \"completion\"\n" +
Expand Down Expand Up @@ -970,9 +966,6 @@ public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException {
" \"mappings\": {\n" +
" \"parentdoc\": {},\n" +
" \"doc\": {\n" +
" \"_source\": {\n" +
" \"enabled\": " + randomBoolean() + "\n" +
" },\n" +
" \"_parent\": {\n" +
" \"type\": \"parentdoc\"\n" +
" },\n" +
Expand Down Expand Up @@ -1002,7 +995,7 @@ public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException {
}

@Test
public void testUngeneratedFieldsPartOfSourceUnstoredSourceDisabled() throws IOException {
public void testUngeneratedFieldsPartOfSourceUnstoredSourceDisabledBackcompat() throws IOException {
indexSingleDocumentWithUngeneratedFieldsThatArePartOf_source(false, false);
String[] fieldsList = {};
// before refresh - document is only in translog
Expand All @@ -1016,7 +1009,7 @@ public void testUngeneratedFieldsPartOfSourceUnstoredSourceDisabled() throws IOE
}

@Test
public void testUngeneratedFieldsPartOfSourceEitherStoredOrSourceEnabled() throws IOException {
public void testUngeneratedFieldsPartOfSourceEitherStoredOrSourceEnabledBackcompat() throws IOException {
boolean stored = randomBoolean();
boolean sourceEnabled = true;
if (stored) {
Expand All @@ -1039,7 +1032,8 @@ void indexSingleDocumentWithUngeneratedFieldsThatArePartOf_source(boolean stored
String createIndexSource = "{\n" +
" \"settings\": {\n" +
" \"index.translog.disable_flush\": true,\n" +
" \"refresh_interval\": \"-1\"\n" +
" \"refresh_interval\": \"-1\",\n" +
" \"" + IndexMetaData.SETTING_VERSION_CREATED + "\": " + Version.V_1_4_2.id + "\n" +
" },\n" +
" \"mappings\": {\n" +
" \"doc\": {\n" +
Expand Down Expand Up @@ -1161,7 +1155,8 @@ void indexSingleDocumentWithStringFieldsGeneratedFromText(boolean stored, boolea
String createIndexSource = "{\n" +
" \"settings\": {\n" +
" \"index.translog.disable_flush\": true,\n" +
" \"refresh_interval\": \"-1\"\n" +
" \"refresh_interval\": \"-1\",\n" +
" \"" + IndexMetaData.SETTING_VERSION_CREATED + "\": " + Version.V_1_4_2.id + "\n" +
" },\n" +
" \"mappings\": {\n" +
" \"doc\": {\n" +
Expand Down Expand Up @@ -1215,7 +1210,8 @@ void indexSingleDocumentWithNumericFieldsGeneratedFromText(boolean stored, boole
String createIndexSource = "{\n" +
" \"settings\": {\n" +
" \"index.translog.disable_flush\": true,\n" +
" \"refresh_interval\": \"-1\"\n" +
" \"refresh_interval\": \"-1\",\n" +
" \"" + IndexMetaData.SETTING_VERSION_CREATED + "\": " + Version.V_1_4_2.id + "\n" +
" },\n" +
" \"mappings\": {\n" +
" \"doc\": {\n" +
Expand Down
1 change: 0 additions & 1 deletion src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,6 @@ public void testRootMappersStillWorking() {
//just pick some example from DocumentMapperParser.rootTypeParsers
rootTypes.put(SizeFieldMapper.NAME, "{\"enabled\" : true}");
rootTypes.put(IndexFieldMapper.NAME, "{\"enabled\" : true}");
rootTypes.put(SourceFieldMapper.NAME, "{\"enabled\" : true}");
rootTypes.put("include_in_all", "true");
rootTypes.put("dynamic_date_formats", "[\"yyyy-MM-dd\", \"dd-MM-yyyy\"]");
rootTypes.put("numeric_detection", "true");
Expand Down
11 changes: 8 additions & 3 deletions src/test/java/org/elasticsearch/index/mapper/source/CompressSourceMappingTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,11 @@
package org.elasticsearch.index.mapper.source;

import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.compress.CompressorFactory;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.ParsedDocument;
Expand All @@ -33,14 +37,15 @@
*
*/
public class CompressSourceMappingTests extends ElasticsearchSingleNodeTest {
Settings settings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();

@Test
public void testCompressDisabled() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_source").field("compress", false).endObject()
.endObject().endObject().string();

DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);

ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
.field("field1", "value1")
Expand All @@ -56,7 +61,7 @@ public void testCompressEnabled() throws Exception {
.startObject("_source").field("compress", true).endObject()
.endObject().endObject().string();

DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);

ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
.field("field1", "value1")
Expand All @@ -73,7 +78,7 @@ public void testCompressThreshold() throws Exception {
.startObject("_source").field("compress_threshold", "200b").endObject()
.endObject().endObject().string();

DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);

ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
.field("field1", "value1")
Expand Down
Loading