Skip to content

Commit

Permalink
Mappings: Remove includes and excludes from _source
Browse files Browse the repository at this point in the history 
Regardless of the outcome of #8142, we should at least enforce that
when _source is enabled, it is sufficient to reindex. This change
removes the excludes and includes settings, since these modify
the source, causing us to lose the ability to reindex some fields.
  • Loading branch information
@rjernst
rjernst committed Apr 27, 2015
1 parent 91e2bb1 commit f35f01a
Show file tree
Hide file tree
Showing 6 changed files with 66 additions and 142 deletions.
5 changes: 3 additions & 2 deletions src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
Expand Down Expand Up @@ -166,15 +167,15 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
} else if ("format".equals(fieldName)) {
builder.format(nodeStringValue(fieldNode, null));
iterator.remove();
} else if (fieldName.equals("includes")) {
} else if (fieldName.equals("includes") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
List<Object> values = (List<Object>) fieldNode;
String[] includes = new String[values.size()];
for (int i = 0; i < includes.length; i++) {
includes[i] = values.get(i).toString();
}
builder.includes(includes);
iterator.remove();
} else if (fieldName.equals("excludes")) {
} else if (fieldName.equals("excludes") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
List<Object> values = (List<Object>) fieldNode;
String[] excludes = new String[values.size()];
for (int i = 0; i < excludes.length; i++) {
Expand Down
8 changes: 5 additions & 3 deletions src/test/java/org/elasticsearch/get/GetActionTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,13 @@

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.flush.FlushResponse;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.*;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Base64;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
Expand Down Expand Up @@ -412,7 +414,7 @@ public void testThatGetFromTranslogShouldWorkWithExclude() throws Exception {

assertAcked(prepareCreate(index)
.addMapping(type, mapping)
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1)));
.setSettings("index.refresh_interval", -1, IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id));

client().prepareIndex(index, type, "1")
.setSource(jsonBuilder().startObject().field("field", "1", "2").field("excluded", "should not be seen").endObject())
Expand Down Expand Up @@ -446,7 +448,7 @@ public void testThatGetFromTranslogShouldWorkWithInclude() throws Exception {

assertAcked(prepareCreate(index)
.addMapping(type, mapping)
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1)));
.setSettings("index.refresh_interval", -1, IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id));

client().prepareIndex(index, type, "1")
.setSource(jsonBuilder().startObject().field("field", "1", "2").field("included", "should be seen").endObject())
Expand Down Expand Up @@ -482,7 +484,7 @@ public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFields() throw

assertAcked(prepareCreate(index)
.addMapping(type, mapping)
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1)));
.setSettings("index.refresh_interval", -1, IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id));

client().prepareIndex(index, type, "1")
.setSource(jsonBuilder().startObject()
Expand Down
124 changes: 44 additions & 80 deletions src/test/java/org/elasticsearch/index/mapper/source/DefaultSourceMappingTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,13 @@
package org.elasticsearch.index.mapper.source;

import org.apache.lucene.index.IndexableField;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.compress.CompressorFactory;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
Expand All @@ -38,12 +41,8 @@

import static org.hamcrest.Matchers.*;

/**
*
*/
public class DefaultSourceMappingTests extends ElasticsearchSingleNodeTest {

@Test
public void testNoFormat() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_source").endObject()
Expand All @@ -65,7 +64,6 @@ public void testNoFormat() throws Exception {
assertThat(XContentFactory.xContentType(doc.source()), equalTo(XContentType.SMILE));
}

@Test
public void testJsonFormat() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_source").field("format", "json").endObject()
Expand All @@ -87,7 +85,6 @@ public void testJsonFormat() throws Exception {
assertThat(XContentFactory.xContentType(doc.source()), equalTo(XContentType.JSON));
}

@Test
public void testJsonFormatCompressed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_source").field("format", "json").field("compress", true).endObject()
Expand All @@ -113,26 +110,58 @@ public void testJsonFormatCompressed() throws Exception {
assertThat(XContentFactory.xContentType(uncompressed), equalTo(XContentType.JSON));
}

@Test
public void testIncludeExclude() throws Exception {
public void testIncludesBackcompat() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_source").field("includes", new String[]{"path1*"}).endObject()
.endObject().endObject().string();
.startObject("_source").field("includes", new String[]{"path1*"}).endObject()
.endObject().endObject().string();

try {
createIndex("testbad").mapperService().documentMapperParser().parse(mapping);
fail("includes should not be allowed");
} catch (MapperParsingException e) {
assertTrue(e.getMessage().contains("unsupported parameters"));
}

DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
Settings settings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);

ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
.startObject("path1").field("field1", "value1").endObject()
.startObject("path2").field("field2", "value2").endObject()
.endObject().bytes());
.startObject("path1").field("field1", "value1").endObject()
.startObject("path2").field("field2", "value2").endObject()
.endObject().bytes());

IndexableField sourceField = doc.rootDoc().getField("_source");
Map<String, Object> sourceAsMap = XContentFactory.xContent(XContentType.JSON).createParser(new BytesArray(sourceField.binaryValue())).mapAndClose();
assertThat(sourceAsMap.containsKey("path1"), equalTo(true));
assertThat(sourceAsMap.containsKey("path2"), equalTo(false));
}

@Test
public void testExcludesBackcompat() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_source").field("excludes", new String[]{"path1*"}).endObject()
.endObject().endObject().string();

try {
createIndex("testbad").mapperService().documentMapperParser().parse(mapping);
fail("excludes should not be allowed");
} catch (MapperParsingException e) {
assertTrue(e.getMessage().contains("unsupported parameters"));
}

Settings settings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);

ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
.startObject("path1").field("field1", "value1").endObject()
.startObject("path2").field("field2", "value2").endObject()
.endObject().bytes());

IndexableField sourceField = doc.rootDoc().getField("_source");
Map<String, Object> sourceAsMap = XContentFactory.xContent(XContentType.JSON).createParser(new BytesArray(sourceField.binaryValue())).mapAndClose();
assertThat(sourceAsMap.containsKey("path1"), equalTo(false));
assertThat(sourceAsMap.containsKey("path2"), equalTo(true));
}

public void testDefaultMappingAndNoMapping() throws Exception {
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
.startObject("_source").field("enabled", false).endObject()
Expand Down Expand Up @@ -161,7 +190,6 @@ public void testDefaultMappingAndNoMapping() throws Exception {
}
}

@Test
public void testDefaultMappingAndWithMappingOverride() throws Exception {
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
.startObject("_source").field("enabled", false).endObject()
Expand All @@ -176,7 +204,6 @@ public void testDefaultMappingAndWithMappingOverride() throws Exception {
assertThat(mapper.sourceMapper().enabled(), equalTo(true));
}

@Test
public void testDefaultMappingAndNoMappingWithMapperService() throws Exception {
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
.startObject("_source").field("enabled", false).endObject()
Expand All @@ -190,7 +217,6 @@ public void testDefaultMappingAndNoMappingWithMapperService() throws Exception {
assertThat(mapper.sourceMapper().enabled(), equalTo(false));
}

@Test
public void testDefaultMappingAndWithMappingOverrideWithMapperService() throws Exception {
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
.startObject("_source").field("enabled", false).endObject()
Expand All @@ -208,66 +234,4 @@ public void testDefaultMappingAndWithMappingOverrideWithMapperService() throws E
assertThat(mapper.type(), equalTo("my_type"));
assertThat(mapper.sourceMapper().enabled(), equalTo(true));
}

@Test
public void testParsingWithDefaultAppliedAndNotApplied() throws Exception {
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
.startObject("_source").array("includes", "default_field_path.").endObject()
.endObject().endObject().string();

MapperService mapperService = createIndex("test").mapperService();
mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedString(defaultMapping), true);

String mapping = XContentFactory.jsonBuilder().startObject().startObject("my_type")
.startObject("_source").array("includes", "custom_field_path.").endObject()
.endObject().endObject().string();
mapperService.merge("my_type", new CompressedString(mapping), true);
DocumentMapper mapper = mapperService.documentMapper("my_type");
assertThat(mapper.type(), equalTo("my_type"));
assertThat(mapper.sourceMapper().includes().length, equalTo(2));
assertThat(mapper.sourceMapper().includes(), hasItemInArray("default_field_path."));
assertThat(mapper.sourceMapper().includes(), hasItemInArray("custom_field_path."));

mapping = XContentFactory.jsonBuilder().startObject().startObject("my_type")
.startObject("properties").startObject("text").field("type", "string").endObject().endObject()
.endObject().endObject().string();
mapperService.merge("my_type", new CompressedString(mapping), false);
mapper = mapperService.documentMapper("my_type");
assertThat(mapper.type(), equalTo("my_type"));
assertThat(mapper.sourceMapper().includes(), hasItemInArray("default_field_path."));
assertThat(mapper.sourceMapper().includes(), hasItemInArray("custom_field_path."));
assertThat(mapper.sourceMapper().includes().length, equalTo(2));
}

public void testDefaultNotAppliedOnUpdate() throws Exception {
XContentBuilder defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
.startObject("_source").array("includes", "default_field_path.").endObject()
.endObject().endObject();

IndexService indexService = createIndex("test", ImmutableSettings.EMPTY, MapperService.DEFAULT_MAPPING, defaultMapping);

String mapping = XContentFactory.jsonBuilder().startObject().startObject("my_type")
.startObject("_source").array("includes", "custom_field_path.").endObject()
.endObject().endObject().string();
client().admin().indices().preparePutMapping("test").setType("my_type").setSource(mapping).get();

DocumentMapper mapper = indexService.mapperService().documentMapper("my_type");
assertThat(mapper.type(), equalTo("my_type"));
assertThat(mapper.sourceMapper().includes().length, equalTo(2));
List<String> includes = Arrays.asList(mapper.sourceMapper().includes());
assertThat("default_field_path.", isIn(includes));
assertThat("custom_field_path.", isIn(includes));

mapping = XContentFactory.jsonBuilder().startObject().startObject("my_type")
.startObject("properties").startObject("text").field("type", "string").endObject().endObject()
.endObject().endObject().string();
client().admin().indices().preparePutMapping("test").setType("my_type").setSource(mapping).get();

mapper = indexService.mapperService().documentMapper("my_type");
assertThat(mapper.type(), equalTo("my_type"));
includes = Arrays.asList(mapper.sourceMapper().includes());
assertThat("default_field_path.", isIn(includes));
assertThat("custom_field_path.", isIn(includes));
assertThat(mapper.sourceMapper().includes().length, equalTo(2));
}
}
24 changes: 10 additions & 14 deletions ...h/indices/mapping/UpdateMappingTests.java → ...apping/UpdateMappingIntegrationTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,15 @@

import com.google.common.collect.Lists;

import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.collect.ImmutableOpenMap;
Expand All @@ -54,7 +56,7 @@
import static org.hamcrest.Matchers.*;

@ClusterScope(randomDynamicTemplates = false)
public class UpdateMappingTests extends ElasticsearchIntegrationTest {
public class UpdateMappingIntegrationTests extends ElasticsearchIntegrationTest {

@Test
public void dynamicUpdates() throws Exception {
Expand Down Expand Up @@ -213,13 +215,13 @@ public void updateMappingNoChanges() throws Exception {

@SuppressWarnings("unchecked")
@Test
public void updateIncludeExclude() throws Exception {
assertAcked(prepareCreate("test").addMapping("type",
jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("normal").field("type", "long").endObject()
.startObject("exclude").field("type", "long").endObject()
.startObject("include").field("type", "long").endObject()
.endObject().endObject().endObject()));
public void updateIncludeExcludeBackcompat() throws Exception {
assertAcked(prepareCreate("test").setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id)
.addMapping("type", jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("normal").field("type", "long").endObject()
.startObject("exclude").field("type", "long").endObject()
.startObject("include").field("type", "long").endObject()
.endObject().endObject().endObject()));
ensureGreen(); // make sure that replicas are initialized so the refresh command will work them too

logger.info("Index doc");
Expand All @@ -229,7 +231,6 @@ public void updateIncludeExclude() throws Exception {
);
refresh(); // commit it for later testing.


logger.info("Adding exclude settings");
PutMappingResponse putResponse = client().admin().indices().preparePutMapping("test").setType("type").setSource(
JsonXContent.contentBuilder().startObject().startObject("type")
Expand Down Expand Up @@ -259,7 +260,6 @@ public void updateIncludeExclude() throws Exception {
assertThat(getResponse.getSource(), not(hasKey("exclude")));
assertThat(getResponse.getSource(), hasKey("include"));


logger.info("Changing mapping to includes");
putResponse = client().admin().indices().preparePutMapping("test").setType("type").setSource(
JsonXContent.contentBuilder().startObject().startObject("type")
Expand All @@ -278,7 +278,6 @@ public void updateIncludeExclude() throws Exception {
assertThat((Map<String, Object>) typeMapping.getSourceAsMap().get("_source"), hasKey("excludes"));
assertThat((ArrayList<String>) ((Map<String, Object>) typeMapping.getSourceAsMap().get("_source")).get("excludes"), emptyIterable());


logger.info("Indexing doc yet again");
index("test", "type", "1", JsonXContent.contentBuilder().startObject()
.field("normal", 3).field("exclude", 3).field("include", 3)
Expand All @@ -290,7 +289,6 @@ public void updateIncludeExclude() throws Exception {
assertThat(getResponse.getSource(), not(hasKey("exclude")));
assertThat(getResponse.getSource(), hasKey("include"));


logger.info("Adding excludes, but keep includes");
putResponse = client().admin().indices().preparePutMapping("test").setType("type").setSource(
JsonXContent.contentBuilder().startObject().startObject("type")
Expand All @@ -308,8 +306,6 @@ public void updateIncludeExclude() throws Exception {
assertThat((Map<String, Object>) typeMapping.getSourceAsMap().get("_source"), hasKey("excludes"));
ArrayList<String> excludes = (ArrayList<String>) ((Map<String, Object>) typeMapping.getSourceAsMap().get("_source")).get("excludes");
assertThat(excludes, contains("*.excludes"));


}

@SuppressWarnings("unchecked")
Expand Down
Loading

0 comments on commit f35f01a

Please sign in to comment.