Skip to content

Commit

Permalink
Implements Template Support :
Browse files Browse the repository at this point in the history
  - Upgrade elasticsearch version to 0.19-3-SNAPSHOT to use this feature elastic/elasticsearch#1860
  - Rename forceReinit to forceMapping in ElasticsearchAbstractClientFactoryBean
  - Add Test ElasticsearchTemplateTest.java
  • Loading branch information
nhuray authored and nhuray committed Apr 16, 2012
1 parent 7158860 commit 977478e
Show file tree
Hide file tree
Showing 12 changed files with 243 additions and 20 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,5 @@
/.classpath
/.settings
/target
*.iml
*.idea
4 changes: 2 additions & 2 deletions README.textile
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ If merging fails, the factory will not start.
h3. Force rebuild mappings (use with caution)

For test purpose or for continuous integration, you could force the factory to clean the previous @type@ when starting the client.
It will *remove all your datas* for that @type@. Just set @forceReinit@ property to @true@.
It will *remove all your datas* for that @type@. Just set @forceMapping@ property to @true@.

bc. <bean id="esClient"
class="fr.pilato.spring.elasticsearch.ElasticsearchClientFactoryBean" >
Expand All @@ -169,5 +169,5 @@ bc. <bean id="esClient"
<value>twitter/tweet</value>
</list>
</property>
<property name="forceReinit" value="true" />
<property name="forceMapping" value="true" />
</bean>
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

<properties>
<spring.version>3.1.1.RELEASE</spring.version>
<elasticsearch.version>0.19.1</elasticsearch.version>
<elasticsearch.version>0.19.3-SNAPSHOT</elasticsearch.version>
</properties>

<developers>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,23 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
import org.elasticsearch.action.admin.indices.close.CloseIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.open.OpenIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
import org.elasticsearch.action.admin.indices.settings.UpdateSettingsRequestBuilder;
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.FactoryBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.util.Assert;

/**
* An abstract {@link FactoryBean} used to create an ElasticSearch
Expand Down Expand Up @@ -79,8 +81,14 @@
* <value>alltheworld:rss</value>
* </list>
* </property>
* <property name="forceReinit" value="false" />
* <property name="templates">
* <list>
* <value>rss_template</value>
* </list>
* </property>
* <property name="forceMapping" value="false" />
* <property name="mergeMapping" value="true" />
* <property name="forceTemplate" value="false" />
* <property name="mergeSettings" value="true" />
* <property name="settingsFile" value="es.properties" />
* </bean>
Expand Down Expand Up @@ -131,7 +139,9 @@ public abstract class ElasticsearchAbstractClientFactoryBean extends Elasticsear

protected Client client;

protected boolean forceReinit;
protected boolean forceMapping;

protected boolean forceTemplate;

protected boolean mergeMapping;

Expand All @@ -141,6 +151,8 @@ public abstract class ElasticsearchAbstractClientFactoryBean extends Elasticsear

protected String[] aliases;

protected String[] templates;

protected String classpathRoot = "/es";

// TODO Let the user decide
Expand All @@ -158,12 +170,21 @@ public abstract class ElasticsearchAbstractClientFactoryBean extends Elasticsear

/**
* Set to true if you want to force reinit indexes/mapping
* @param forceReinit
* @param forceMapping
*/
public void setForceReinit(boolean forceReinit) {
this.forceReinit = forceReinit;
public void setForceMapping(boolean forceMapping) {
this.forceMapping = forceMapping;
}

/**
* Set to true if you want to force recreate templates
*
* @param forceTemplate
*/
public void setForceTemplate(boolean forceTemplate) {
this.forceTemplate = forceTemplate;
}

/**
* Set to true if you want to try to merge mappings
* @param mergeMapping
Expand Down Expand Up @@ -220,6 +241,28 @@ public void setMappings(String[] mappings) {
public void setAliases(String[] aliases) {
this.aliases = aliases;
}

/**
* Define templates you want to manage with this factory <br/>
* <p>
* Example :<br/>
*
* <pre>
* {@code
* <property name="templates">
* <list>
* <value>template_1</value>
* <value>template_2</value>
* </list>
* </property>
* }
* </pre>
*
* @param templates list of template
*/
public void setTemplates(String[] templates) {
this.templates = templates;
}

/**
* Classpath root for index and mapping files (default : /es)
Expand All @@ -244,6 +287,7 @@ public void afterPropertiesSet() throws Exception {
logger.info("Starting ElasticSearch client");

client = buildClient();
initTemplates();
initMappings();
initAliases();
}
Expand All @@ -256,7 +300,7 @@ public void destroy() throws Exception {
client.close();
}
} catch (final Exception e) {
logger.error("Error closing Elasticsearch client: ", e);
logger.error("Error closing ElasticSearch client: ", e);
}
}

Expand All @@ -275,9 +319,29 @@ public boolean isSingleton() {
return true;
}

/**
* Init templates if needed.
* <p>
* Note that you can force to recreate template using
* {@link #setForceTemplate(boolean)}
*
* @throws Exception
*/
private void initTemplates() throws Exception {
if (templates != null && templates.length > 0) {
for (int i = 0; i < templates.length; i++) {
String template = templates[i];
Assert.hasText(template, "Can not read template in ["
+ templates[i]
+ "]. Check that templates is not empty.");
createTemplate(template, forceTemplate);
}
}
}

/**
* Init mapping if needed.
* <p>Note that you can force to reinit mapping using {@link #setForceReinit(boolean)}
* <p>Note that you can force to reinit mapping using {@link #setForceMapping(boolean)}
* @throws Exception
*/
private void initMappings() throws Exception {
Expand Down Expand Up @@ -317,7 +381,7 @@ private void initMappings() throws Exception {
for (Iterator<String> iterator = mappings.iterator(); iterator
.hasNext();) {
String type = iterator.next();
pushMapping(index, type, forceReinit, mergeMapping);
pushMapping(index, type, forceMapping, mergeMapping);
}
}
}
Expand Down Expand Up @@ -369,6 +433,59 @@ private void createAlias(String alias, String index) throws Exception {
if (!response.acknowledged()) throw new Exception("Could not define alias [" + alias + "] for index [" + index + "].");
if (logger.isTraceEnabled()) logger.trace("/createAlias("+alias+","+index+")");
}

/**
* Create a template if needed
*
* @param template template name
* @param force force recreate template
* @throws Exception
*/
private void createTemplate(String template, boolean force)
throws Exception {
if (logger.isTraceEnabled())
logger.trace("createTemplate(" + template + ")");
checkClient();

// If template already exists and if we are in force mode, we delete the
// type and its mapping
if (force && isTemplateExist(template)) {
if (logger.isDebugEnabled())
logger.debug("Force remove template [" + template + "]");
// Remove template in ElasticSearch !
final DeleteIndexTemplateResponse response = client.admin()
.indices().prepareDeleteTemplate(template).execute()
.actionGet();
}

// Read the template json file if exists and use it
String source = readTemplate(template);
if (source != null) {
if (logger.isTraceEnabled())
logger.trace("Template [" + template + "]=" + source);
// Create template
final PutIndexTemplateResponse response = client.admin().indices()
.preparePutTemplate(template).setSource(source).execute()
.actionGet();
if (!response.acknowledged()) {
throw new Exception("Could not define template [" + template
+ "].");
} else {
if (logger.isDebugEnabled()) {
logger.debug("Template [" + template
+ "] successfully created.");
}
}
} else {
if (logger.isWarnEnabled()) {
logger.warn("No template definition for [" + template
+ "]. Ignoring.");
}
}

if (logger.isTraceEnabled())
logger.trace("/createTemplate(" + template + ")");
}

/**
* Check if an index already exists
Expand All @@ -395,6 +512,23 @@ private boolean isMappingExist(String index, String type) {
if (mdd != null) return true;
return false;
}

/**
* Check if a template already exists
*
* @param template template name
* @return true if template exists
*/
private boolean isTemplateExist(String template) {
ClusterState cs = client.admin().cluster().prepareState()
.setFilterIndexTemplates(template).execute().actionGet()
.getState();
final IndexTemplateMetaData mdd = cs.getMetaData().templates()
.get(template);

if (mdd != null) return true;
return false;
}

/**
* Define a type for a given index and if exists with its mapping definition
Expand Down Expand Up @@ -486,9 +620,8 @@ private void createIndex(String index) throws Exception {
}

/**
* Create a new index in Elasticsearch
* Create a new index in ElasticSearch
* @param index Index name
* @param merge Try to merge settings ?
* @throws Exception
*/
private void mergeIndexSettings(String index) throws Exception {
Expand Down Expand Up @@ -532,6 +665,18 @@ private String readMapping(String index, String type) throws Exception {
return readFileInClasspath(classpathRoot + "/" + index + "/" + type + jsonFileExtension);
}

/**
* Read the template.<br>
* Shortcut to readFileInClasspath(classpathRoot + "/_template/" + template + jsonFileExtension);
*
* @param template Template name
* @return Template if exists. Null otherwise.
* @throws Exception
*/
private String readTemplate(String template) throws Exception {
return readFileInClasspath(classpathRoot + "/_template/" + template + jsonFileExtension);
}

/**
* Read settings for an index.<br>
* Shortcut to readFileInClasspath(classpathRoot + "/" + index + "/" + indexSettingsFileName);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package fr.pilato.spring.elasticsearch.xml;

import org.elasticsearch.client.Client;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

public class ElasticsearchTemplateTest {

static protected ConfigurableApplicationContext ctx;

@BeforeClass
static public void setup() {
ctx = new ClassPathXmlApplicationContext(
"fr/pilato/spring/elasticsearch/xml/es-template-test-context.xml");
}

@AfterClass
static public void tearDown() {
if (ctx != null) {
ctx.close();
}
}

@Test
public void test_transport_client() {
Client client = ctx.getBean("esClient", Client.class);
Assert.assertNotNull("Client must not be null...", client);
}

}
2 changes: 1 addition & 1 deletion src/test/resources/es-context.xml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ http://www.springframework.org/schema/context http://www.springframework.org/sch
<value>alltheworld:rss</value>
</list>
</property>
<property name="forceReinit" value="true" />
<property name="forceMapping" value="true" />
</bean>

</beans>
16 changes: 16 additions & 0 deletions src/test/resources/es6/_template/twitter_template.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"template" : "twee*",
"settings" : {
"number_of_shards" : 1
},
"mappings" : {
"tweet" : {
"properties" : {
"message" : {
"type" : "string",
"store" : "yes"
}
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<value>twitter/tweet</value>
</list>
</property>
<property name="forceReinit" value="false" />
<property name="forceMapping" value="false" />
</bean>

<bean id="esClient2"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<value>twitter/tweet</value>
</list>
</property>
<property name="forceReinit" value="false" />
<property name="forceMapping" value="false" />
</bean>

<bean id="esClient2"
Expand Down
Loading

0 comments on commit 977478e

Please sign in to comment.